Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): replace Sentry with optional OpenTelemetry OTLP export (#4555)

Gergő Móriczbca56457b0d8
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 35ef78bb1..5905d11f7 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -153,3 +153,9 @@ CDP_API_KEY_SECRET=""
## ENDPOINT PRICING TEST $0.01
X402_ENDPOINT_PRICE_USD=0.01
+
+# === OpenTelemetry (optional) ===
+# Set an OTLP endpoint to export traces (http/protobuf, 100% sampled). Spans from
+# zero-data-retention requests are never exported.
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+# OTEL_SERVICE_NAME=firecrawl-api
diff --git a/apps/api/.gitignore b/apps/api/.gitignore
index 1cc479f83..d953aacec 100644
--- a/apps/api/.gitignore
+++ b/apps/api/.gitignore
@@ -8,7 +8,6 @@ dump.rdb
/.next/
.rdb
-.sentryclirc
.env.*
diff --git a/apps/api/package.json b/apps/api/package.json
index a0240f4da..8a1a7e16a 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -14,7 +14,6 @@
"start:dev": "tsc-watch --compiler ./node_modules/typescript-7/bin/tsc --onSuccess \"node dist/src/index.js\"",
"build": "tsc",
"branding:print": "node src/scraper/scrapeURL/engines/fire-engine/branding-script/print-script.js",
- "build:nosentry": "tsc",
"test": "vitest run --exclude='**/__tests__/e2e_noAuth/**'",
"test:local-no-auth": "vitest run --exclude='**/__tests__/e2e_withAuth/**'",
"test:full": "vitest run --exclude='**/__tests__/e2e_noAuth/**' --exclude='**/__tests__/e2e_withAuth/**'",
@@ -41,14 +40,12 @@
"zdr-worker:production": "node dist/src/services/zdr-worker.js",
"mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
"mongo-docker-console": "docker exec -it mongodb mongosh",
- "sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist",
"prepare": "cd ../.. && husky ./apps/api/.husky",
"knip": "knip"
},
"author": "",
"license": "ISC",
"devDependencies": {
- "@sentry/core": "^10.67.0",
"@types/amqplib": "^0.10.7",
"@types/body-parser": "^1.19.2",
"@types/cors": "^2.8.13",
@@ -91,8 +88,11 @@
"@google-cloud/storage": "^7.21.0",
"@mendable/firecrawl-rs": "workspace:*",
"@openrouter/ai-sdk-provider": "^2.9.1",
- "@sentry/cli": "^2.58.2",
- "@sentry/node": "^10.67.0",
+ "@opentelemetry/api": "^1.9.1",
+ "@opentelemetry/exporter-trace-otlp-proto": "^0.222.0",
+ "@opentelemetry/resources": "^2.11.0",
+ "@opentelemetry/sdk-trace-node": "^2.11.0",
+ "@opentelemetry/semantic-conventions": "^1.43.0",
"@types/ws": "^8.5.12",
"ai": "6.0.86",
"ajv": "^8.18.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 4afe58716..4b37e5fbc 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -105,12 +105,21 @@ importers:
'@openrouter/ai-sdk-provider':
specifier: ^2.9.1
- '@sentry/cli':
- specifier: ^2.58.2
- version: 2.58.2
- '@sentry/node':
- specifier: ^10.67.0
- version: 10.67.0(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@opentelemetry/api':
+ specifier: ^1.9.1
+ version: 1.9.1
+ '@opentelemetry/exporter-trace-otlp-proto':
+ specifier: ^0.222.0
+ version: 0.222.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources':
+ specifier: ^2.11.0
+ version: 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace-node':
+ specifier: ^2.11.0
+ version: 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions':
+ specifier: ^1.43.0
+ version: 1.43.0
'@types/ws':
specifier: ^8.5.12
version: 8.5.12
@@ -206,7 +215,7 @@ importers:
version: 2.9.0
langsmith:
specifier: ^0.6.0
- version: 0.6.3(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected])
+ version: 0.6.3(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected])
lodash:
specifier: ^4.18.0
version: 4.18.1
@@ -289,9 +298,6 @@ importers:
specifier: 4.1.12
version: 4.1.12
devDependencies:
- '@sentry/core':
- specifier: ^10.67.0
- version: 10.67.0
'@types/amqplib':
specifier: ^0.10.7
version: 0.10.7
@@ -514,17 +520,6 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
- '@apm-js-collab/[email protected]':
- resolution: {integrity: sha512-Yidf5GOl60db80UxUtNdKK3pnY7obU/gs0xOfA0SCdnvVLMCvfYIer/egC3TqpPiT0Jg22eg3RlzcO+zKfPMcA==}
- engines: {node: '>=18.0.0'}
-
- '@apm-js-collab/[email protected]':
- resolution: {integrity: sha512-aN3Oq8r1J3gPJtCwErP664gM0+HhM1I1lujPr9TMTCcEl/joQQbpGpeMdts9B1+W2wHMsvioDMv5F4PvMWE6gw==}
- hasBin: true
-
- '@apm-js-collab/[email protected]':
- resolution: {integrity: sha512-mTvWz9rnQwx1U3h0XPTHaX7bgfkpipLLTQyjlC2cdhQpQEuoLT0AGzoydeoq2NxfEVv6fWOOETcSbb2nptleyw==}
-
'@asamuzakjp/[email protected]':
resolution: {integrity: sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==}
engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
@@ -1510,8 +1505,8 @@ packages:
ai: ^6.0.0
zod: ^3.25.0 || ^4.0.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-9mb1If+IF6u0ZVXkHQ6ogEae5HwA6ajIVUgpSDQyRASxft6BSXHvBvPooRle3yFN/fKnCdSOnuu0OC3PLcF6+g==}
engines: {node: '>=8.0.0'}
'@opentelemetry/[email protected]':
@@ -1522,38 +1517,86 @@ packages:
resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
engines: {node: '>=8.0.0'}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-Tr79DyWI8itsBdg+jH+opjfrwLzX+erk1/ExkIwhWoAVjVrJIn2y5+cGjTC0Vy8fyNIA/y8wuJPZwr1T3xCZeQ==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.0.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.0.0 <1.10.0'
+
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-m2nckMT80NnmjTYSPjJQObBJ+8dgkoajEOUbznL8AHZ3T3yHRk2P7gI1PhEBc1+lOnrYE9UWrWHqJDsmqjmNbw==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': '>=1.0.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-imLrLzNhnABJt3tUyT63h6MqfZrG3Gdv3TE7+OUQTvwD088RfCUx3rF01lcgfu2KXBFk13vwi0yKZLvFEalkIA==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-YbywG3veEm2Fb6TbdxRkuquWob6eVWXuA8/Ba1tXz9jHfUqpdE3keilOHEtPboC4CvS1bjeeVfNkWGOOrLj+lw==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-/F3BZ89+CJQnZkMh2tCrtcdB+XT2Dxhj4FFE+WPQ//413hmFL0/RfEX6vgOIWGhiSzrkHWTK3+6SiT7K5/g/jQ==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': ^1.3.0
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.3.0 <1.10.0'
+
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': '>=1.3.0 <1.10.0'
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-+19YHODIjaUCArxleaJtuufFZVpz/xvvK+VllQqE+W8hHolxdoRwHfK/s667zezwh1hkx6FFF+oYzetYgqK+Bg==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.4.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.9.0 <1.10.0'
+
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': '>=1.9.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-cp9zmTl62R8PJrpvFcmc8N2JQU/xfa0S+61q511Nji+QxCfZ8Ifvg7H27G8cANe4crg4RTrWsVvanHiXjSp6ag==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': '>=1.3.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-sGA19HvtrrSKYsseHphluH6j3p6Xa3fqc7c7y8f/7mYWejc1lyDFcpSdD1kYa50HCLUeEo4zA5bW0pniaPszuw==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-CuvCMJmZxswhNLlM2LfuLOW3h3fZujA4hsG4B+Sz4dX2zvaXO8Ng74cnDHWD64gLszTlhiG3c0iNUjj4g+0/sA==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.0.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': '>=1.3.0 <1.10.0'
@@ -1858,103 +1901,6 @@ packages:
'@selderee/[email protected]':
resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==}
- '@sentry/[email protected]':
- resolution: {integrity: sha512-MArsb3zLhA2/cbd4rTm09SmTpnEuZCoZOpuZYkrpDw1qzBVJmRFA1W1hGAQ9puzBIk/ubY3EUhhzuU3zN2uD6w==}
- engines: {node: '>=10'}
- os: [darwin]
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-ay3OeObnbbPrt45cjeUyQjsx5ain1laj1tRszWj37NkKu55NZSp4QCg1gGBZ0gBGhckI9nInEsmKtix00alw2g==}
- engines: {node: '>=10'}
- cpu: [arm64]
- os: [linux, freebsd, android]
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-HU9lTCzcHqCz/7Mt5n+cv+nFuJdc1hGD2h35Uo92GgxX3/IujNvOUfF+nMX9j6BXH6hUt73R5c0Ycq9+a3Parg==}
- engines: {node: '>=10'}
- cpu: [arm]
- os: [linux, freebsd, android]
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-CN9p0nfDFsAT1tTGBbzOUGkIllwS3hygOUyTK7LIm9z+UHw5uNgNVqdM/3Vg+02ymjkjISNB3/+mqEM5osGXdA==}
- engines: {node: '>=10'}
- cpu: [x86, ia32]
- os: [linux, freebsd, android]
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-oX/LLfvWaJO50oBVOn4ZvG2SDWPq0MN8SV9eg5tt2nviq+Ryltfr7Rtoo+HfV+eyOlx1/ZXhq9Wm7OT3cQuz+A==}
- engines: {node: '>=10'}
- cpu: [x64]
- os: [linux, freebsd, android]
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-+cl3x2HPVMpoSVGVM1IDWlAEREZrrVQj4xBb0TRKII7g3hUxRsAIcsrr7+tSkie++0FuH4go/b5fGAv51OEF3w==}
- engines: {node: '>=10'}
- cpu: [arm64]
- os: [win32]
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-omFVr0FhzJ8oTJSg1Kf+gjLgzpYklY0XPfLxZ5iiMiYUKwF5uo1RJRdkUOiEAv0IqpUKnmKcmVCLaDxsWclB7Q==}
- engines: {node: '>=10'}
- cpu: [x86, ia32]
- os: [win32]
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-2NAFs9UxVbRztQbgJSP5i8TB9eJQ7xraciwj/93djrSMHSEbJ0vC47TME0iifgvhlHMs5vqETOKJtfbbpQAQFA==}
- engines: {node: '>=10'}
- cpu: [x64]
- os: [win32]
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-U4u62V4vaTWF+o40Mih8aOpQKqKUbZQt9A3LorIJwaE3tO3XFLRI70eWtW2se1Qmy0RZ74zB14nYcFNFl2t4Rw==}
- engines: {node: '>= 10'}
- hasBin: true
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ==}
- engines: {node: '>=14'}
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-b6U3pJ8AUvN9aouq0vl+VZI8KT8RslBsfGMFuNwRr313zOmdmFJBZqTiUw9VGgJ2jGKxLO9alm9rlxBfX4hf+w==}
- engines: {node: '>=18'}
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-dBHHRwZyan1pOnFJ+sNBvR8TkXbZAfZU/jpxmALS3JZ2/8AGR7cQKL+b7SleKuJ7iUDZyklN3Nqi0i5JkcA+HA==}
- engines: {node: '>=18'}
- peerDependencies:
- '@opentelemetry/api': ^1.9.0
- '@opentelemetry/core': 2.9.0
- '@opentelemetry/exporter-trace-otlp-http': '>=0.57.0 <1'
- '@opentelemetry/instrumentation': '>=0.57.1 <1'
- '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0
- peerDependenciesMeta:
- '@opentelemetry/api':
- optional: true
- '@opentelemetry/core':
- optional: true
- '@opentelemetry/exporter-trace-otlp-http':
- optional: true
- '@opentelemetry/instrumentation':
- optional: true
- '@opentelemetry/sdk-trace-base':
- optional: true
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-SFKpZGqOCEFSmP93NdDP6ikZp4NS7A/JR8+2ofK3jF6Y9Vyox7pX0pxdOnPLpFcPtMybMahfWqSAWJvsFs4RmA==}
- engines: {node: '>=18'}
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-oLTOrAK1rOqmYRktOJZwz37B1seXPx1W2FTMVtzTVNjMFA/LZwGzePeZzhUOgzZgfLHixMd/ceWtGqoxAndcjQ==}
- engines: {node: '>=18'}
- peerDependencies:
- '@opentelemetry/api': ^1.9.0
- '@opentelemetry/core': 2.9.0
- '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0
-
- '@sentry/[email protected]':
- resolution: {integrity: sha512-GQ9t+RSTx5s3b/aZrLFuL4nrwPLMah5NiZk5cjxJmgmOSgm3nMdO8gdqCedDch5B13F3YsxtaQYjSJnzLz8M5A==}
- engines: {node: '>=18'}
-
'@standard-schema/[email protected]':
resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}
@@ -2315,10 +2261,6 @@ packages:
resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==}
engines: {node: '>=12'}
- resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==}
- hasBin: true
-
resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==}
@@ -2455,9 +2397,6 @@ packages:
resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}
engines: {node: '>= 6'}
- resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==}
-
resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==}
engines: {node: '>=18'}
@@ -2874,9 +2813,6 @@ packages:
resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==}
- resolution: {integrity: sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==}
-
resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==}
engines: {node: '>= 0.4'}
@@ -2904,14 +2840,6 @@ packages:
resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
engines: {node: '>=10'}
- resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==}
- engines: {node: '>=0.10'}
-
- resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==}
- engines: {node: '>=4.0'}
-
resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==}
@@ -3287,10 +3215,6 @@ packages:
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
- resolution: {integrity: sha512-jTd2FfOgOWOdgjkHuk/1Ms8VKFXkPs15ymYBETw1sAOrO/dY3XeGVRWir9qBbw7pXr0T2eTFwfCZ+N02HmiNGA==}
- engines: {node: '>=18'}
-
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
@@ -3660,10 +3584,6 @@ packages:
resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==}
engines: {node: '>= 8'}
- resolution: {integrity: sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ==}
- engines: {node: '>=18.0.0'}
-
resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
engines: {node: '>= 0.6'}
@@ -3713,9 +3633,6 @@ packages:
resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}
engines: {node: '>=16 || 14 >=14.17'}
- resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==}
-
resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}
@@ -4027,10 +3944,6 @@ packages:
resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}
engines: {node: '>= 0.6.0'}
- resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
- engines: {node: '>=0.4.0'}
-
resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
engines: {node: ^16 || ^18 || >=20}
@@ -4051,9 +3964,6 @@ packages:
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
engines: {node: '>= 0.10'}
- resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
-
resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==}
engines: {node: '>=10'}
@@ -4152,10 +4062,6 @@ packages:
resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==}
engines: {node: '>=0.10.0'}
- resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==}
- engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'}
-
resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==}
@@ -4239,9 +4145,6 @@ packages:
resolution: {integrity: sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==}
- resolution: {integrity: sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw==}
-
resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==}
engines: {node: '>=10'}
@@ -4317,10 +4220,6 @@ packages:
resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}
engines: {node: '>=0.10.0'}
- resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
- engines: {node: '>=0.10.0'}
-
resolution: {integrity: sha512-qxQJTx2ryR0Dw0ITYyekNQWpz6f8dGd7vffGNflQQ3Iqj9NJ6qiZ7ELpZsJ/QBhIVAiDfXdag3+Gp8RvWa62AA==}
engines: {node: '>=12'}
@@ -4976,30 +4875,6 @@ snapshots:
'@ai-sdk/provider-utils': 4.0.23([email protected])
zod: 4.1.12
- '@apm-js-collab/[email protected]':
- dependencies:
- '@apm-js-collab/code-transformer': 0.18.0
- es-module-lexer: 2.3.1
- magic-string: 0.30.21
- module-details-from-path: 1.0.4
-
- '@apm-js-collab/[email protected]':
- dependencies:
- '@types/estree': 1.0.9
- astring: 1.9.0
- esquery: 1.7.0
- meriyah: 6.1.4
- semifies: 1.0.0
- source-map: 0.6.1
-
- '@apm-js-collab/[email protected]':
- dependencies:
- '@apm-js-collab/code-transformer': 0.18.0
- debug: 4.4.3
- module-details-from-path: 1.0.4
- transitivePeerDependencies:
- - supports-color
-
'@asamuzakjp/[email protected]':
dependencies:
'@asamuzakjp/generational-cache': 1.0.1
@@ -5881,7 +5756,7 @@ snapshots:
ai: 6.0.86([email protected])
zod: 4.1.12
- '@opentelemetry/[email protected]':
+ '@opentelemetry/[email protected]':
dependencies:
'@opentelemetry/api': 1.9.1
@@ -5889,19 +5764,48 @@ snapshots:
'@opentelemetry/[email protected]': {}
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/semantic-conventions': 1.43.0
+
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/semantic-conventions': 1.43.0
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.1
- '@opentelemetry/api-logs': 0.220.0
- import-in-the-middle: 3.3.2
- require-in-the-middle: 8.0.1
- transitivePeerDependencies:
- - supports-color
+ '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/[email protected])
+ '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/api-logs': 0.222.0
+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-logs': 0.222.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-metrics': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
@@ -5909,25 +5813,46 @@ snapshots:
'@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
'@opentelemetry/semantic-conventions': 1.43.0
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/api-logs': 0.222.0
+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])
+
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
'@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.1
- '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
- '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/[email protected])
'@opentelemetry/semantic-conventions': 1.43.0
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.1
- '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
- '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/context-async-hooks': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace-base': 2.11.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])
'@opentelemetry/semantic-conventions': 1.43.0
'@opentelemetry/[email protected]': {}
@@ -6112,101 +6037,6 @@ snapshots:
domhandler: 5.0.3
selderee: 0.11.0
- '@sentry/[email protected]':
- optional: true
-
- '@sentry/[email protected]':
- optional: true
-
- '@sentry/[email protected]':
- optional: true
-
- '@sentry/[email protected]':
- optional: true
-
- '@sentry/[email protected]':
- optional: true
-
- '@sentry/[email protected]':
- optional: true
-
- '@sentry/[email protected]':
- optional: true
-
- '@sentry/[email protected]':
- optional: true
-
- '@sentry/[email protected]':
- dependencies:
- https-proxy-agent: 5.0.1
- node-fetch: 2.7.0(patch_hash=cf2058381802eaf328e1cb26b185762ba41e2157a5545e7ed9bba56b7de0c49b)
- progress: 2.0.3
- proxy-from-env: 1.1.0
- which: 2.0.2
- optionalDependencies:
- '@sentry/cli-darwin': 2.58.2
- '@sentry/cli-linux-arm': 2.58.2
- '@sentry/cli-linux-arm64': 2.58.2
- '@sentry/cli-linux-i686': 2.58.2
- '@sentry/cli-linux-x64': 2.58.2
- '@sentry/cli-win32-arm64': 2.58.2
- '@sentry/cli-win32-i686': 2.58.2
- '@sentry/cli-win32-x64': 2.58.2
- transitivePeerDependencies:
- - encoding
- - supports-color
-
- '@sentry/[email protected]': {}
-
- '@sentry/[email protected]':
- dependencies:
- '@sentry/conventions': 0.16.0
-
- '@sentry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':
- dependencies:
- '@sentry/conventions': 0.16.0
- '@sentry/core': 10.67.0
- '@sentry/opentelemetry': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
- import-in-the-middle: 3.3.2
- optionalDependencies:
- '@opentelemetry/api': 1.9.1
- '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.220.0(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])
-
- '@sentry/[email protected](@opentelemetry/[email protected](@opentelemetry/[email protected]))':
- dependencies:
- '@opentelemetry/api': 1.9.1
- '@opentelemetry/instrumentation': 0.220.0(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])
- '@sentry/conventions': 0.16.0
- '@sentry/core': 10.67.0
- '@sentry/node-core': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
- '@sentry/opentelemetry': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
- '@sentry/server-utils': 10.67.0
- import-in-the-middle: 3.3.2
- transitivePeerDependencies:
- - '@opentelemetry/core'
- - '@opentelemetry/exporter-trace-otlp-http'
- - supports-color
-
- '@sentry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':
- dependencies:
- '@opentelemetry/api': 1.9.1
- '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])
- '@sentry/conventions': 0.16.0
- '@sentry/core': 10.67.0
-
- '@sentry/[email protected]':
- dependencies:
- '@apm-js-collab/code-transformer-bundler-plugins': 0.7.1
- '@apm-js-collab/tracing-hooks': 0.13.0
- '@sentry/conventions': 0.16.0
- '@sentry/core': 10.67.0
- transitivePeerDependencies:
- - supports-color
-
'@standard-schema/[email protected]': {}
'@taplo/[email protected]': {}
@@ -6527,8 +6357,6 @@ snapshots:
-
dependencies:
tslib: 2.8.1
@@ -6690,8 +6518,6 @@ snapshots:
parse5: 7.1.2
parse5-htmlparser2-tree-adapter: 7.0.0
-
dependencies:
restore-cursor: 5.1.0
@@ -6949,8 +6775,6 @@ snapshots:
-
dependencies:
es-errors: 1.3.0
@@ -6999,12 +6823,6 @@ snapshots:
- dependencies:
- estraverse: 5.3.0
-
-
dependencies:
'@types/estree': 1.0.9
@@ -7515,12 +7333,6 @@ snapshots:
- dependencies:
- cjs-module-lexer: 2.2.0
- es-module-lexer: 2.3.1
- module-details-from-path: 1.0.4
-
@@ -7690,12 +7502,13 @@ snapshots:
+ [email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected]):
dependencies:
p-queue: 6.6.2
optionalDependencies:
'@opentelemetry/api': 1.9.1
- '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/exporter-trace-otlp-proto': 0.222.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace-base': 2.11.0(@opentelemetry/[email protected])
ws: 8.21.0
@@ -7857,8 +7670,6 @@ snapshots:
-
@@ -7892,8 +7703,6 @@ snapshots:
-
@@ -8187,11 +7996,9 @@ snapshots:
-
dependencies:
- '@opentelemetry/api': 1.9.0
+ '@opentelemetry/api': 1.9.1
tdigest: 0.1.2
@@ -8219,8 +8026,6 @@ snapshots:
forwarded: 0.2.0
ipaddr.js: 1.9.1
-
@@ -8328,13 +8133,6 @@ snapshots:
- dependencies:
- debug: 4.4.3
- module-details-from-path: 1.0.4
- transitivePeerDependencies:
- - supports-color
-
@@ -8441,8 +8239,6 @@ snapshots:
dependencies:
parseley: 0.12.1
-
@@ -8532,8 +8328,6 @@ snapshots:
-
diff --git a/apps/api/pnpm-workspace.yaml b/apps/api/pnpm-workspace.yaml
index c5137401d..f6d8c7174 100644
--- a/apps/api/pnpm-workspace.yaml
+++ b/apps/api/pnpm-workspace.yaml
@@ -60,8 +60,6 @@ patchedDependencies:
allowBuilds:
"@mendable/firecrawl-rs": true
- "@sentry-internal/node-cpu-profiler": true
- "@sentry/cli": true
bigint-buffer: true
# Optional native accelerator for fast-crc32c (pulled in by
# @google-cloud/bigtable); the pure-JS fallback is fine for our payload
diff --git a/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts b/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts
index da2bec978..67118ccca 100644
--- a/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts
+++ b/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts
@@ -6,9 +6,6 @@ vi.mock("ai", () => ({
vi.mock("../../../lib/generic-ai", () => ({
getModel: vi.fn().mockReturnValue({}),
}));
-vi.mock("../../../services/sentry", () => ({
- captureExceptionWithZdrCheck: vi.fn(),
-}));
import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";
import { mergeBrandingResults } from "../../../lib/branding/merge";
diff --git a/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts b/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts
index 1217c8ace..2626b9c98 100644
--- a/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts
+++ b/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts
@@ -34,8 +34,12 @@ function appUnderTest() {
v2.post("/scrape", (_req, res) => res.status(200).json({ success: true }));
// Mirrors the real /v2/crawl/:jobId table (src/routes/v2.ts): only GET and
// DELETE are registered, never PATCH.
- v2.get("/crawl/:jobId", (_req, res) => res.status(200).json({ success: true }));
- v2.delete("/crawl/:jobId", (_req, res) => res.status(200).json({ success: true }));
+ v2.get("/crawl/:jobId", (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
+ v2.delete("/crawl/:jobId", (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
app.use("/v2", v2);
app.use(notFoundHandler);
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 5ecce4076..fb1e655ec 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -523,11 +523,15 @@ const configSchema = z.object({
SYS_INFO_MAX_CACHE_DURATION: z.coerce.number().default(150),
USE_GO_MARKDOWN_PARSER: z.stringbool().optional(),
- // Sentry
- SENTRY_DSN: z.string().optional(),
- SENTRY_TRACE_SAMPLE_RATE: z.coerce.number().default(0.01),
- SENTRY_ERROR_SAMPLE_RATE: z.coerce.number().default(0.05),
SENTRY_ENVIRONMENT: z.string().default("production"),
+
+ // OpenTelemetry. Tracing is off unless an OTLP endpoint is set; spans are then
+ // exported over http/protobuf at 100% sampling, and the SDK honors the
+ // standard OTEL_EXPORTER_OTLP_* / OTEL_BSP_* / OTEL_RESOURCE_ATTRIBUTES
+ // variables. Zero-data-retention spans are never exported (see otel-tracer).
+ OTEL_EXPORTER_OTLP_ENDPOINT: emptyStringAsUndefined(z.string().url()),
+ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: emptyStringAsUndefined(z.string().url()),
+ OTEL_SERVICE_NAME: emptyStringAsUndefined(z.string()),
NUQ_POD_NAME: z.string().default("main"),
// Billing
diff --git a/apps/api/src/controllers/v0/admin/check-fire-engine.ts b/apps/api/src/controllers/v0/admin/check-fire-engine.ts
index ae78abd9e..bc038633c 100644
--- a/apps/api/src/controllers/v0/admin/check-fire-engine.ts
+++ b/apps/api/src/controllers/v0/admin/check-fire-engine.ts
@@ -1,6 +1,5 @@
import { logger } from "../../../lib/logger";
import { config } from "../../../config";
-import * as Sentry from "@sentry/node";
import { Request, Response } from "express";
export async function checkFireEngine(req: Request, res: Response) {
@@ -63,14 +62,12 @@ export async function checkFireEngine(req: Request, res: Response) {
method: "checkFireEngine",
error: lastError,
});
- Sentry.captureException(lastError);
return res.status(500).json({
success: false,
error: "Internal server error - all retry attempts failed",
});
} catch (error) {
logger.error(error);
- Sentry.captureException(error);
return res.status(500).json({
success: false,
error: "Internal server error",
diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts
index fa552014f..c86836a38 100644
--- a/apps/api/src/controllers/v0/crawl-cancel.ts
+++ b/apps/api/src/controllers/v0/crawl-cancel.ts
@@ -3,7 +3,6 @@ import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
import { logger } from "../../../src/lib/logger";
import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis";
-import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { redisEvictConnection } from "../../../src/services/redis";
import { crawlGroup } from "../../services/worker/nuq-router";
@@ -83,7 +82,6 @@ export async function crawlCancelController(req: Request, res: Response) {
status: "cancelled",
});
} catch (error) {
- Sentry.captureException(error);
logger.error(error);
return res.status(500).json({ error: error.message });
}
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index ee23113c8..8726214e3 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -10,7 +10,6 @@ import {
type StoredCrawl,
} from "../../../src/lib/crawl-redis";
import { supabaseGetScrapesByRequestId } from "../../../src/lib/supabase-jobs";
-import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { toLegacyDocument } from "../v1/types";
import type { DBScrape, PseudoJob } from "../v1/crawl-status";
@@ -297,7 +296,6 @@ export async function crawlStatusController(req: Request, res: Response) {
.map(x => toLegacyDocument(x, sc.internalOptions)),
});
} catch (error) {
- Sentry.captureException(error);
logger.error(error);
return res.status(500).json({ error: error.message });
}
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index fe1bccb57..2e497a9b5 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -26,7 +26,6 @@ import {
} from "../../../src/lib/crawl-redis";
import { redisEvictConnection } from "../../../src/services/redis";
import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
-import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
import { url as urlSchema } from "../v1/types";
import { ZodError } from "zod";
@@ -312,7 +311,6 @@ export async function crawlController(req: Request, res: Response) {
logger,
);
for (const job of jobs) {
- // add with sentry instrumentation
await addScrapeJob(job.data, job.jobId, job.priority);
}
});
@@ -347,7 +345,6 @@ export async function crawlController(req: Request, res: Response) {
res.json({ jobId: id });
} catch (error) {
- Sentry.captureException(error);
logger.error(error);
return res.status(500).json({
error: error instanceof ZodError ? "Invalid URL" : error.message,
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 9d71790c1..30f3787e6 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -15,7 +15,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { redisEvictConnection } from "../../../src/services/redis";
import { v7 as uuidv7 } from "uuid";
import { logger } from "../../lib/logger";
-import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
import { ZodError } from "zod";
import { Document as V0Document } from "./../../lib/entities";
@@ -323,7 +322,6 @@ export async function scrapeController(req: Request, res: Response) {
return res.status(result.returnCode).json(result);
} catch (error) {
- Sentry.captureException(error);
logger.error("Scrape error occcurred", { error });
return res.status(500).json({
error:
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 432799157..de30f90c8 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -12,7 +12,6 @@ import { v7 as uuidv7 } from "uuid";
import { logger } from "../../lib/logger";
import { redisEvictConnection } from "../../../src/services/redis";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
-import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
import {
fromLegacyScrapeOptions,
@@ -261,7 +260,6 @@ export async function searchController(req: Request, res: Response) {
return res.status(402).json({ error: "Insufficient credits" });
}
} catch (error) {
- Sentry.captureException(error);
logger.error(error);
return res.status(500).json({ error: "Internal server error" });
}
@@ -301,7 +299,6 @@ export async function searchController(req: Request, res: Response) {
return res.status(408).json({ error: error.message });
}
- Sentry.captureException(error);
logger.error("Unhandled error occurred in search", { error });
return res.status(500).json({ error: error.message });
}
diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts
index 2e1d80c82..7fc58c1cb 100644
--- a/apps/api/src/controllers/v1/crawl-cancel.ts
+++ b/apps/api/src/controllers/v1/crawl-cancel.ts
@@ -1,7 +1,6 @@
import { Response } from "express";
import { logger } from "../../lib/logger";
import { getCrawl, getCrawlJobs, saveCrawl } from "../../lib/crawl-redis";
-import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { RequestWithAuth } from "./types";
import { crawlGroup } from "../../services/worker/nuq-router";
@@ -49,7 +48,6 @@ export async function crawlCancelController(
status: "cancelled",
});
} catch (error) {
- Sentry.captureException(error);
logger.error(error);
return res.status(500).json({ error: error.message });
}
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index 762c25213..ea01e508b 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -17,7 +17,6 @@ import {
getDoneJobsOrdered,
} from "../../lib/crawl-redis";
import { getJobs, PseudoJob } from "./crawl-status";
-import * as Sentry from "@sentry/node";
import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq-router";
import { getErrorContactMessage } from "../../lib/deployment";
@@ -188,8 +187,6 @@ export async function crawlStatusWSController(
await crawlStatusWS(ws, req);
} catch (err) {
- Sentry.captureException(err);
-
const id = uuidv7();
let verbose = JSON.stringify(err);
if (verbose === "{}") {
diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.ts
index 919126963..b6faff9a6 100644
--- a/apps/api/src/controllers/v1/deep-research.ts
+++ b/apps/api/src/controllers/v1/deep-research.ts
@@ -2,7 +2,6 @@ import { v7 as uuidv7 } from "uuid";
import { Request, Response } from "express";
import { ErrorResponse, extractOptions, RequestWithAuth } from "./types";
import { getDeepResearchQueue } from "../../services/queue-service";
-import * as Sentry from "@sentry/node";
import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";
import { z } from "zod";
import { logRequest } from "../../services/logging/log_job";
diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.ts
index ced86715d..3ad13dda1 100644
--- a/apps/api/src/controllers/v1/generate-llmstxt.ts
+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts
@@ -7,7 +7,6 @@ import {
RequestWithAuth,
} from "./types";
import { getGenerateLlmsTxtQueue } from "../../services/queue-service";
-import * as Sentry from "@sentry/node";
import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 6269822f0..279ac1dd3 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -27,8 +27,14 @@ import { AbortManagerThrownError } from "../../scraper/scrapeURL/lib/abortManage
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { getErrorContactMessage } from "../../lib/deployment";
-import { captureExceptionWithZdrCheck } from "../../services/sentry";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+ withSpan,
+ setSpanAttributes,
+ recordSpanException,
+ SpanKind,
+ type Span,
+} from "../../lib/otel-tracer";
import {
adjustKeylessCredits,
keylessLimitBody,
@@ -43,6 +49,31 @@ import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
res: Response<ScrapeResponse>,
+) {
+ // Resolved before the root span starts so the whole request trace stays
+ // unrecorded for zero-data-retention requests (see otel-tracer).
+ const zeroDataRetentionTrace =
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body?.zeroDataRetention === true;
+
+ return withSpan(
+ "api.scrape.request",
+ span => scrapeControllerInner(req, res, span),
+ {
+ kind: SpanKind.SERVER,
+ attributes: {
+ "api.version": "v1",
+ "scrape.team_id": req.auth.team_id,
+ },
+ zeroDataRetention: zeroDataRetentionTrace,
+ },
+ );
+}
+
+async function scrapeControllerInner(
+ req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
+ res: Response<ScrapeResponse>,
+ span: Span,
) {
// Get timing data from middleware (includes all middleware processing time)
const middlewareStartTime =
@@ -342,17 +373,10 @@ export async function scrapeController(
path: req.path,
teamId: req.auth.team_id,
});
- captureExceptionWithZdrCheck(e, {
- tags: {
- errorId: id,
- version: "v1",
- teamId: req.auth.team_id,
- },
- extra: {
- path: req.path,
- url: req.body.url,
- },
- zeroDataRetention,
+ recordSpanException(span, e);
+ setSpanAttributes(span, {
+ "scrape.status_code": 500,
+ "scrape.error_id": id,
});
return res.status(500).json({
success: false,
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index e5e641473..9429d3338 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -20,7 +20,6 @@ import {
} from "../../lib/key-restriction";
import type { Logger } from "winston";
import { ScrapeJobTimeoutError } from "../../lib/error";
-import { captureExceptionWithZdrCheck } from "../../services/sentry";
import { z } from "zod";
import { executeSearch } from "../../search/execute";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
@@ -374,9 +373,6 @@ export async function searchController(
});
}
- captureExceptionWithZdrCheck(error, {
- extra: { zeroDataRetention },
- });
logger.error("Unhandled error occurred in search", {
version: "v1",
error,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 291a02729..2381a32ec 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1,4 +1,4 @@
-import { Request, Response } from "express";
+import { Request } from "express";
import { config } from "../../config";
import { z } from "zod";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
@@ -1410,11 +1410,6 @@ export interface RequestWithAuth<
account?: Account;
}
-export interface ResponseWithSentry<ResBody = undefined>
- extends Response<ResBody> {
- sentry?: string;
-}
-
export function toLegacyCrawlerOptions(x: CrawlerOptions) {
return {
includes: x.includePaths,
diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
index 6b2f93c4d..fb4abcb1a 100644
--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
@@ -51,11 +51,6 @@ vi.mock("../../../lib/key-restriction", () => ({
checkKeyEndpointRestriction: vi.fn().mockResolvedValue({ allowed: true }),
}));
-vi.mock("../../../services/sentry", () => ({
- applyZdrScope: vi.fn(),
- captureExceptionWithZdrCheck: vi.fn(),
-}));
-
vi.mock("../../../lib/logger", () => ({
logger: {
info: vi.fn(),
diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts
index a304860ac..17484b891 100644
--- a/apps/api/src/controllers/v2/crawl-cancel.ts
+++ b/apps/api/src/controllers/v2/crawl-cancel.ts
@@ -6,7 +6,6 @@ import {
saveCrawl,
StoredCrawl,
} from "../../lib/crawl-redis";
-import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { RequestWithAuth, scrapeOptions } from "./types";
import { crawlGroup } from "../../services/worker/nuq-router";
@@ -63,7 +62,6 @@ export async function crawlCancelController(
status: "cancelled",
});
} catch (error) {
- Sentry.captureException(error);
logger.error(error);
return res.status(500).json({ error: error.message });
}
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index b44137223..d1ec7fa2f 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -18,7 +18,6 @@ import {
getDoneJobsOrdered,
} from "../../lib/crawl-redis";
import { getJobs, PseudoJob } from "./crawl-status";
-import * as Sentry from "@sentry/node";
import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq-router";
import { getErrorContactMessage } from "../../lib/deployment";
@@ -217,8 +216,6 @@ export async function crawlStatusWSController(
await crawlStatusWS(ws, req);
} catch (err) {
- Sentry.captureException(err);
-
const id = uuidv7();
let verbose = JSON.stringify(err);
if (verbose === "{}") {
diff --git a/apps/api/src/controllers/v2/feedback/record.ts b/apps/api/src/controllers/v2/feedback/record.ts
index 0eb116d83..a43b86e22 100644
--- a/apps/api/src/controllers/v2/feedback/record.ts
+++ b/apps/api/src/controllers/v2/feedback/record.ts
@@ -1,11 +1,17 @@
import { v7 as uuidv7 } from "uuid";
import { config } from "../../../config";
import { logger as _logger } from "../../../lib/logger";
+import {
+ withSpan,
+ setSpanAttributes,
+ recordSpanException,
+ type Span,
+} from "../../../lib/otel-tracer";
+import { getScrapeZDR, getSearchZDR } from "../../../lib/zdr-helpers";
import {
autumnService,
featureIdForBillingEndpoint,
} from "../../../services/autumn/autumn.service";
-import { captureExceptionWithZdrCheck } from "../../../services/sentry";
import {
EndpointFeedbackErrorCode,
RequestWithAuth,
@@ -238,9 +244,49 @@ async function refundCredits(params: {
}
}
+/**
+ * Team-level zero data retention for the endpoint the feedback refers to. Unlike
+ * `shouldSkipPersistenceForForcedZdr` this ignores the persistence override:
+ * the trace must stay unrecorded whenever the team is forced into ZDR.
+ */
+function isForcedZdrTeam(
+ req: RequestWithAuth<any, any, any>,
+ options: FeedbackRecordOptions,
+): boolean {
+ if (options.endpoint === "search") {
+ const searchZDR = getSearchZDR(req.acuc?.flags);
+ return searchZDR === "forced-zdr" || searchZDR === "forced-anon";
+ }
+
+ if (options.endpoint === "scrape" || options.endpoint === "parse") {
+ return getScrapeZDR(req.acuc?.flags) === "forced";
+ }
+
+ return false;
+}
+
export async function recordEndpointFeedback(
req: RequestWithAuth<any, any, any>,
options: FeedbackRecordOptions,
+): Promise<FeedbackRecordResult> {
+ return withSpan(
+ "api.feedback.record",
+ span => recordEndpointFeedbackInner(req, options, span),
+ {
+ attributes: {
+ "feedback.endpoint": options.endpoint,
+ "feedback.job_id": options.jobId,
+ "feedback.team_id": req.auth.team_id,
+ },
+ zeroDataRetention: isForcedZdrTeam(req, options),
+ },
+ );
+}
+
+async function recordEndpointFeedbackInner(
+ req: RequestWithAuth<any, any, any>,
+ options: FeedbackRecordOptions,
+ span: Span,
): Promise<FeedbackRecordResult> {
const logger = _logger.child({
module: "api/v2",
@@ -252,6 +298,7 @@ export async function recordEndpointFeedback(
if (shouldSkipPersistenceForForcedZdr(req, options)) {
logger.info("Skipping feedback persistence for forced ZDR team");
+ setSpanAttributes(span, { "feedback.zero_data_retention": true });
return zdrFeedbackSuccess(options);
}
@@ -265,7 +312,9 @@ export async function recordEndpointFeedback(
if ("status" in jobOrFailure) return jobOrFailure;
if (shouldSkipPersistenceForJobZdr(jobOrFailure, options)) {
+ // Learned late: the attribute drops this span and flags the trace.
logger.info("Skipping feedback persistence for ZDR job");
+ setSpanAttributes(span, { "feedback.zero_data_retention": true });
return zdrFeedbackSuccess(options);
}
@@ -365,10 +414,10 @@ export async function recordEndpointFeedback(
},
};
} catch (error) {
- captureExceptionWithZdrCheck(error);
logger.error("Unhandled error while recording endpoint feedback", {
error,
});
+ recordSpanException(span, error);
return feedbackFailure(
500,
"INTERNAL",
diff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.ts
index d3f204ada..f971ee99e 100644
--- a/apps/api/src/controllers/v2/parse-upload.ts
+++ b/apps/api/src/controllers/v2/parse-upload.ts
@@ -11,6 +11,7 @@ import { getRedisConnection } from "../../services/queue-service";
import { RequestWithAuth, UploadedParseFile } from "./types";
import { detectUploadedFileKind, getSupportedParseFileTypes } from "./parse";
import { isImageOcrEnabled } from "../../lib/image-ocr-gate";
+import { getScrapeZDR } from "../../lib/zdr-helpers";
import { reportPipelineError } from "../../lib/redis-pipeline";
const PARSE_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;
@@ -278,140 +279,148 @@ export async function parseUploadUrlController(
req: RequestWithAuth<{}, any, any>,
res: Response,
) {
- return withSpan("api.parse.upload_url", async span => {
- const parsed = uploadInitSchema.safeParse(req.body ?? {});
- if (!parsed.success) {
- return res.status(400).json({
- success: false,
- code: "BAD_REQUEST",
- error:
- parsed.error.issues[0]?.message ?? "Invalid upload init request.",
- });
- }
-
- const { filename, contentType, declaredSizeBytes } = parsed.data;
- const imageOcrEnabled = isImageOcrEnabled(req.acuc?.flags);
- if (!isSupportedParseUpload(filename, contentType, imageOcrEnabled)) {
- return res.status(400).json({
- success: false,
- code: "UNSUPPORTED_FILE_TYPE",
- error: `Unsupported upload type. Supported file extensions include ${getSupportedParseFileTypes(imageOcrEnabled)}, or matching supported MIME types.`,
- });
- }
-
- cleanupExpiredLocalUploads();
- const driver = getParseUploadDriver();
- if (!getEffectiveRefSecret()) {
- return res.status(503).json({
- success: false,
- code: "PARSE_UPLOAD_REF_SECRET_MISSING",
- error: "Parse upload references are not configured.",
- });
- }
-
- if (driver === "local" && !isLocalUploadAdapterAllowed()) {
- return res.status(503).json({
- success: false,
- code: "PARSE_UPLOAD_STORAGE_DISABLED",
- error:
- "Local parse upload storage is disabled outside development/test.",
- });
- }
+ return withSpan(
+ "api.parse.upload_url",
+ async span => {
+ const parsed = uploadInitSchema.safeParse(req.body ?? {});
+ if (!parsed.success) {
+ return res.status(400).json({
+ success: false,
+ code: "BAD_REQUEST",
+ error:
+ parsed.error.issues[0]?.message ?? "Invalid upload init request.",
+ });
+ }
- const uploadId = uuidv7();
- const expiresAt = Date.now() + PARSE_UPLOAD_TTL_MS;
- const objectPath = makeObjectPath(req.auth.team_id, uploadId, filename);
- const refPayload: ParseUploadRefPayload = {
- v: 1,
- driver,
- uploadId,
- teamId: req.auth.team_id,
- objectPath,
- filename: sanitizeFilename(filename),
- contentType,
- expiresAt,
- maxBytes: PARSE_UPLOAD_MAX_BYTES,
- };
- const uploadRef = signUploadRef(refPayload);
+ const { filename, contentType, declaredSizeBytes } = parsed.data;
+ const imageOcrEnabled = isImageOcrEnabled(req.acuc?.flags);
+ if (!isSupportedParseUpload(filename, contentType, imageOcrEnabled)) {
+ return res.status(400).json({
+ success: false,
+ code: "UNSUPPORTED_FILE_TYPE",
+ error: `Unsupported upload type. Supported file extensions include ${getSupportedParseFileTypes(imageOcrEnabled)}, or matching supported MIME types.`,
+ });
+ }
- setSpanAttributes(span, {
- "parse_upload.driver": driver,
- "parse_upload.team_id": req.auth.team_id,
- "parse_upload.declared_size": declaredSizeBytes,
- });
+ cleanupExpiredLocalUploads();
+ const driver = getParseUploadDriver();
+ if (!getEffectiveRefSecret()) {
+ return res.status(503).json({
+ success: false,
+ code: "PARSE_UPLOAD_REF_SECRET_MISSING",
+ error: "Parse upload references are not configured.",
+ });
+ }
- if (driver === "gcs") {
- if (!config.GCS_PARSE_UPLOAD_BUCKET_NAME) {
+ if (driver === "local" && !isLocalUploadAdapterAllowed()) {
return res.status(503).json({
success: false,
code: "PARSE_UPLOAD_STORAGE_DISABLED",
- error: "Parse upload storage is not configured.",
+ error:
+ "Local parse upload storage is disabled outside development/test.",
});
}
- const bucket = getStorageClient().bucket(
- config.GCS_PARSE_UPLOAD_BUCKET_NAME,
- );
- const file = bucket.file(objectPath);
- const [policy] = await file.generateSignedPostPolicyV4({
- expires: expiresAt,
- fields: {
- "Content-Type": contentType || "application/octet-stream",
- },
- conditions: [
- ["content-length-range", 1, PARSE_UPLOAD_MAX_BYTES],
- ["eq", "$Content-Type", contentType || "application/octet-stream"],
- ],
+ const uploadId = uuidv7();
+ const expiresAt = Date.now() + PARSE_UPLOAD_TTL_MS;
+ const objectPath = makeObjectPath(req.auth.team_id, uploadId, filename);
+ const refPayload: ParseUploadRefPayload = {
+ v: 1,
+ driver,
+ uploadId,
+ teamId: req.auth.team_id,
+ objectPath,
+ filename: sanitizeFilename(filename),
+ contentType,
+ expiresAt,
+ maxBytes: PARSE_UPLOAD_MAX_BYTES,
+ };
+ const uploadRef = signUploadRef(refPayload);
+
+ setSpanAttributes(span, {
+ "parse_upload.driver": driver,
+ "parse_upload.team_id": req.auth.team_id,
+ "parse_upload.declared_size": declaredSizeBytes,
});
+ if (driver === "gcs") {
+ if (!config.GCS_PARSE_UPLOAD_BUCKET_NAME) {
+ return res.status(503).json({
+ success: false,
+ code: "PARSE_UPLOAD_STORAGE_DISABLED",
+ error: "Parse upload storage is not configured.",
+ });
+ }
+
+ const bucket = getStorageClient().bucket(
+ config.GCS_PARSE_UPLOAD_BUCKET_NAME,
+ );
+ const file = bucket.file(objectPath);
+ const [policy] = await file.generateSignedPostPolicyV4({
+ expires: expiresAt,
+ fields: {
+ "Content-Type": contentType || "application/octet-stream",
+ },
+ conditions: [
+ ["content-length-range", 1, PARSE_UPLOAD_MAX_BYTES],
+ ["eq", "$Content-Type", contentType || "application/octet-stream"],
+ ],
+ });
+
+ if (
+ await reserveUnparsedUploadRefOrRespond(
+ res,
+ req.auth.team_id,
+ uploadId,
+ )
+ ) {
+ return;
+ }
+
+ return res.status(200).json({
+ success: true,
+ data: {
+ uploadUrl: policy.url,
+ uploadRef,
+ method: "POST",
+ headers: {},
+ fields: policy.fields,
+ expiresAt: new Date(expiresAt).toISOString(),
+ maxSizeBytes: PARSE_UPLOAD_MAX_BYTES,
+ },
+ });
+ }
+
if (
await reserveUnparsedUploadRefOrRespond(res, req.auth.team_id, uploadId)
) {
return;
}
+ localUploads.set(uploadId, {
+ filename: sanitizeFilename(filename),
+ contentType,
+ teamId: req.auth.team_id,
+ expiresAt,
+ maxBytes: PARSE_UPLOAD_MAX_BYTES,
+ });
+
return res.status(200).json({
success: true,
data: {
- uploadUrl: policy.url,
+ uploadUrl: `${buildPublicBaseUrl(req)}/v2/parse/upload/${uploadId}?uploadRef=${encodeURIComponent(uploadRef)}`,
uploadRef,
- method: "POST",
- headers: {},
- fields: policy.fields,
+ method: "PUT",
+ headers: {
+ "Content-Type": contentType || "application/octet-stream",
+ },
expiresAt: new Date(expiresAt).toISOString(),
maxSizeBytes: PARSE_UPLOAD_MAX_BYTES,
},
});
- }
-
- if (
- await reserveUnparsedUploadRefOrRespond(res, req.auth.team_id, uploadId)
- ) {
- return;
- }
-
- localUploads.set(uploadId, {
- filename: sanitizeFilename(filename),
- contentType,
- teamId: req.auth.team_id,
- expiresAt,
- maxBytes: PARSE_UPLOAD_MAX_BYTES,
- });
-
- return res.status(200).json({
- success: true,
- data: {
- uploadUrl: `${buildPublicBaseUrl(req)}/v2/parse/upload/${uploadId}?uploadRef=${encodeURIComponent(uploadRef)}`,
- uploadRef,
- method: "PUT",
- headers: {
- "Content-Type": contentType || "application/octet-stream",
- },
- expiresAt: new Date(expiresAt).toISOString(),
- maxSizeBytes: PARSE_UPLOAD_MAX_BYTES,
- },
- });
- });
+ },
+ { zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced" },
+ );
}
export async function parseLocalUploadController(req: Request, res: Response) {
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index fef769b70..4b689d4e7 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -24,7 +24,6 @@ import { getJobPriority } from "../../lib/job-priority";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { getErrorContactMessage } from "../../lib/deployment";
-import { captureExceptionWithZdrCheck } from "../../services/sentry";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
@@ -295,6 +294,12 @@ export async function parseController(
req: RequestWithAuth<{}, ScrapeResponse, ParseRequest>,
res: Response<ScrapeResponse>,
) {
+ // Resolved before the root span starts so the whole request trace stays
+ // unrecorded for zero-data-retention requests (see otel-tracer).
+ const zeroDataRetentionTrace =
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body?.zeroDataRetention === true;
+
return withSpan(
"api.parse.request",
async span => {
@@ -658,18 +663,6 @@ export async function parseController(
path: req.path,
teamId: req.auth.team_id,
});
- captureExceptionWithZdrCheck(e, {
- tags: {
- errorId: id,
- version: "v2",
- teamId: req.auth.team_id,
- },
- extra: {
- path: req.path,
- fileName: req.body.file.filename,
- },
- zeroDataRetention,
- });
setSpanAttributes(span, {
"parse.status_code": 500,
"parse.error_id": id,
@@ -771,6 +764,7 @@ export async function parseController(
"http.route": "/v2/parse",
},
kind: SpanKind.SERVER,
+ zeroDataRetention: zeroDataRetentionTrace,
},
);
}
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 61197cda9..9615dba9f 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -30,7 +30,6 @@ import { getJobPriority } from "../../lib/job-priority";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { getErrorContactMessage } from "../../lib/deployment";
-import { captureExceptionWithZdrCheck } from "../../services/sentry";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
@@ -51,6 +50,13 @@ export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
res: Response<ScrapeResponse>,
) {
+ // Resolved before the root span starts so the whole request trace stays
+ // unrecorded for zero-data-retention requests (see otel-tracer).
+ const zeroDataRetentionTrace =
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body?.zeroDataRetention === true ||
+ req.body?.lockdown === true;
+
return withSpan(
"api.scrape.request",
async span => {
@@ -525,18 +531,6 @@ export async function scrapeController(
path: req.path,
teamId: req.auth.team_id,
});
- captureExceptionWithZdrCheck(e, {
- tags: {
- errorId: id,
- version: "v2",
- teamId: req.auth.team_id,
- },
- extra: {
- path: req.path,
- url: req.body.url,
- },
- zeroDataRetention,
- });
setSpanAttributes(span, {
"scrape.status_code": 500,
"scrape.error_id": id,
@@ -642,6 +636,7 @@ export async function scrapeController(
"http.route": "/v2/scrape",
},
kind: SpanKind.SERVER,
+ zeroDataRetention: zeroDataRetentionTrace,
},
);
}
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 4d773290a..b115cae2a 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -24,13 +24,16 @@ import { logger as _logger } from "../../lib/logger";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { z } from "zod";
import { CategoryOption } from "../../lib/search-query-builder";
-import {
- applyZdrScope,
- captureExceptionWithZdrCheck,
-} from "../../services/sentry";
import { executeSearch } from "../../search/execute";
import type { BillingMetadata } from "../../services/billing/types";
import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";
+import {
+ withSpan,
+ setSpanAttributes,
+ recordSpanException,
+ SpanKind,
+ type Span,
+} from "../../lib/otel-tracer";
import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
@@ -47,6 +50,35 @@ import { isAgentInteropSecretValid } from "../../lib/agent-interop";
export async function searchController(
req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
res: Response<SearchResponse>,
+) {
+ // Resolved before any span starts so the whole request stays unrecorded for
+ // zero-data-retention and anonymous searches (see otel-tracer).
+ const enterprise: unknown[] = Array.isArray(req.body?.enterprise)
+ ? req.body.enterprise
+ : [];
+ const zeroDataRetentionTrace =
+ Boolean(getSearchForcedKind(req.acuc?.flags)) ||
+ enterprise.includes("zdr") ||
+ enterprise.includes("anon");
+
+ return withSpan(
+ "api.search.request",
+ span => searchControllerInner(req, res, span),
+ {
+ kind: SpanKind.SERVER,
+ attributes: {
+ "api.version": "v2",
+ "search.team_id": req.auth.team_id,
+ },
+ zeroDataRetention: zeroDataRetentionTrace,
+ },
+ );
+}
+
+async function searchControllerInner(
+ req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
+ res: Response<SearchResponse>,
+ span: Span,
) {
const middlewareStartTime =
(req as any).requestTiming?.startTime || new Date().getTime();
@@ -169,7 +201,6 @@ export async function searchController(
const isZDROrAnon = isZDR || isAnon;
zeroDataRetention = isZDROrAnon ?? false;
logger = logger.child({ zeroDataRetention });
- applyZdrScope(zeroDataRetention);
// Verify the team has searchZDR enabled before allowing enterprise ZDR/anon
if (isZDROrAnon && !teamForcedKind) {
@@ -406,13 +437,12 @@ export async function searchController(
});
}
- captureExceptionWithZdrCheck(error, {
- extra: { zeroDataRetention },
- });
logger.error("Unhandled error occurred in search", {
version: "v2",
error,
});
+ recordSpanException(span, error);
+ setSpanAttributes(span, { "search.status_code": 500 });
return res.status(500).json({
success: false,
error: error.message,
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 8beb321a2..8e1b29bf1 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -1,8 +1,6 @@
import "dotenv/config";
import { config } from "./config";
-import "./services/sentry";
-import { setSentryServiceTag } from "./services/sentry";
-import * as Sentry from "@sentry/node";
+import { shutdownTracing } from "./otel";
import express, { NextFunction, Request, Response } from "express";
import bodyParser from "body-parser";
import cors from "cors";
@@ -20,11 +18,7 @@ import http from "node:http";
import https from "node:https";
import { v1Router } from "./routes/v1";
import expressWs from "express-ws";
-import {
- ErrorResponse,
- RequestWithMaybeACUC,
- ResponseWithSentry,
-} from "./controllers/v1/types";
+import { ErrorResponse, RequestWithMaybeACUC } from "./controllers/v1/types";
import { ZodError } from "zod";
import { QueueFullError } from "./lib/queue-full-error";
import { v7 as uuidv7 } from "uuid";
@@ -75,8 +69,6 @@ const app = ws.app;
global.isProduction = config.IS_PRODUCTION;
-setSentryServiceTag("api");
-
// Capture the exact request bytes so integrations that sign the raw payload
// (e.g. Slack's X-Slack-Signature) can verify it after body parsing. Typed with
// the node http types body-parser's `verify` hook expects.
@@ -187,6 +179,7 @@ async function startServer(port = DEFAULT_PORT) {
shutdownWebhookQueue().finally(() => {
shutdownIndexerQueue().finally(async () => {
await shutdownPubSubLogging();
+ await shutdownTracing();
logger.info("NUQ shutdown complete");
process.exit(0);
});
@@ -265,13 +258,11 @@ app.use(
},
);
-Sentry.setupExpressErrorHandler(app);
-
app.use(
(
err: unknown,
req: RequestWithMaybeACUC<{}, ErrorResponse, undefined>,
- res: ResponseWithSentry<ErrorResponse>,
+ res: Response<ErrorResponse>,
next: NextFunction,
) => {
if (
@@ -298,7 +289,7 @@ app.use(
});
}
- const id = res.sentry ?? uuidv7();
+ const id = uuidv7();
logger.error(
"Error occurred in request! (" + req.path + ") -- ID " + id + " -- ",
diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.ts
index b877615fd..bdd96c58b 100644
--- a/apps/api/src/lib/branding/llm.ts
+++ b/apps/api/src/lib/branding/llm.ts
@@ -5,7 +5,6 @@ import { BrandingEnhancement, getBrandingEnhancementSchema } from "./schema";
import { buildBrandingPrompt } from "./prompt";
import { BrandingLLMInput } from "./types";
import { getModel } from "../generic-ai";
-import { captureExceptionWithZdrCheck } from "../../services/sentry";
function isDebugBrandingEnabled(input: BrandingLLMInput): boolean {
return (
@@ -121,6 +120,9 @@ export async function enhanceBrandingWithLLM(
temperature: 0.1,
experimental_telemetry: {
isEnabled: true,
+ // The input carries the page screenshot / raw page content; too large
+ // for span attributes. Outputs stay recorded.
+ recordInputs: false,
functionId: "enhanceBrandingWithLLM",
metadata: {
teamId: input.teamId || "unknown",
@@ -193,22 +195,6 @@ export async function enhanceBrandingWithLLM(
},
);
} else {
- captureExceptionWithZdrCheck(error, {
- tags: {
- feature: "branding-llm",
- model: modelName,
- ...(input.scrapeId ? { scrape_id: input.scrapeId } : {}),
- ...(input.teamId ? { team_id: input.teamId } : {}),
- },
- extra: {
- url: input.url,
- buttonsCount: input.buttons?.length || 0,
- logoCandidatesCount: input.logoCandidates?.length || 0,
- promptLength: prompt.length,
- hasScreenshot: !!input.screenshot,
- },
- zeroDataRetention: input.zeroDataRetention,
- });
logger.error("LLM branding enhancement failed", {
error,
buttonsCount: input.buttons?.length || 0,
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index f9a2b1ee1..52a2633bb 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -39,35 +39,39 @@ export type StoredCrawl = {
};
export async function saveCrawl(id: string, crawl: StoredCrawl) {
- return await withSpan("firecrawl-redis-save-crawl", async span => {
- setSpanAttributes(span, {
- "crawl.id": id,
- "crawl.team_id": crawl.team_id,
- "crawl.zero_data_retention": crawl.zeroDataRetention || false,
- operation: "save_crawl",
- });
+ return await withSpan(
+ "firecrawl-redis-save-crawl",
+ async span => {
+ setSpanAttributes(span, {
+ "crawl.id": id,
+ "crawl.team_id": crawl.team_id,
+ "crawl.zero_data_retention": crawl.zeroDataRetention || false,
+ operation: "save_crawl",
+ });
- _logger.debug("Saving crawl " + id + " to Redis...", {
- crawl,
- module: "crawl-redis",
- method: "saveCrawl",
- crawlId: id,
- teamId: crawl.team_id,
- zeroDataRetention: crawl.zeroDataRetention,
- });
+ _logger.debug("Saving crawl " + id + " to Redis...", {
+ crawl,
+ module: "crawl-redis",
+ method: "saveCrawl",
+ crawlId: id,
+ teamId: crawl.team_id,
+ zeroDataRetention: crawl.zeroDataRetention,
+ });
- await redisEvictConnection.set(
- "crawl:" + id,
- JSON.stringify(crawl),
- "EX",
- 24 * 60 * 60,
- );
- await redisEvictConnection.sadd("crawls_by_team_id:" + crawl.team_id, id);
- await redisEvictConnection.expire(
- "crawls_by_team_id:" + crawl.team_id,
- 24 * 60 * 60,
- );
- });
+ await redisEvictConnection.set(
+ "crawl:" + id,
+ JSON.stringify(crawl),
+ "EX",
+ 24 * 60 * 60,
+ );
+ await redisEvictConnection.sadd("crawls_by_team_id:" + crawl.team_id, id);
+ await redisEvictConnection.expire(
+ "crawls_by_team_id:" + crawl.team_id,
+ 24 * 60 * 60,
+ );
+ },
+ { zeroDataRetention: crawl.zeroDataRetention },
+ );
}
export async function recordRobotsBlocked(crawlId: string, url: string) {
@@ -134,6 +138,7 @@ export async function getCrawl(id: string): Promise<StoredCrawl | null> {
setSpanAttributes(span, {
"crawl.found": true,
"crawl.team_id": crawl.team_id,
+ "crawl.zero_data_retention": crawl.zeroDataRetention === true,
});
return crawl;
@@ -669,103 +674,58 @@ export async function lockURL(
url: string,
__logger: Logger = _logger,
): Promise<boolean> {
- return await withSpan("firecrawl-redis-lock-url", async span => {
- const normalizedUrl = normalizeURL(url, sc);
- setSpanAttributes(span, {
- "crawl.id": id,
- "crawl.url": normalizedUrl,
- "crawl.team_id": sc.team_id,
- operation: "lock_url",
- });
-
- if (typeof sc.crawlerOptions?.limit === "number") {
- if (
- (await redisEvictConnection.scard("crawl:" + id + ":visited_unique")) >=
- sc.crawlerOptions.limit
- ) {
- setSpanAttributes(span, { "crawl.limit_reached": true });
- return false;
- }
- }
-
- const lockTarget = !sc.crawlerOptions?.deduplicateSimilarURLs
- ? normalizedUrl
- : generateURLPermutations(normalizedUrl)[0].href;
-
- // The commands are idempotent, so retry the pipeline on failure: the
- // SADD may land before a failing command (e.g. a failed EXPIRE), and
- // giving up then would leave the URL marked visited without being
- // queued — a caller retry would read SADD=0 and skip it. Track whether
- // any attempt reported the SADD as newly added, since a retry after a
- // partial success reads 0.
- let sawNewLock = false;
- let locked = false;
- let lastError: unknown = null;
- for (let attempt = 1; attempt <= 3; attempt++) {
- let attemptError: unknown = null;
- try {
- const pipeline = redisEvictConnection.pipeline();
- pipeline.sadd("crawl:" + id + ":visited", lockTarget);
- pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);
- const attemptResults = await pipeline.exec();
- if (attemptResults?.[0]?.[1] === 1) {
- sawNewLock = true;
- }
- attemptError = firstPipelineError(attemptResults);
- } catch (error) {
- attemptError = error;
- }
-
- if (attemptError === null) {
- locked = true;
- lastError = null;
- break;
- }
- lastError = attemptError;
-
- __logger.error("Redis pipeline command failed", {
- module: "crawl-redis",
- method: "lockURL",
- crawlId: id,
- url: normalizedUrl,
- attempt,
- error: attemptError,
+ return await withSpan(
+ "firecrawl-redis-lock-url",
+ async span => {
+ const normalizedUrl = normalizeURL(url, sc);
+ setSpanAttributes(span, {
+ "crawl.id": id,
+ "crawl.url": normalizedUrl,
+ "crawl.team_id": sc.team_id,
+ operation: "lock_url",
});
- if (attempt < 3) {
- await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));
+ if (typeof sc.crawlerOptions?.limit === "number") {
+ if (
+ (await redisEvictConnection.scard(
+ "crawl:" + id + ":visited_unique",
+ )) >= sc.crawlerOptions.limit
+ ) {
+ setSpanAttributes(span, { "crawl.limit_reached": true });
+ return false;
+ }
}
- }
-
- if (!locked) {
- throw new Error("URL lock pipeline failed after retries", {
- cause: lastError,
- });
- }
-
- const res = sawNewLock;
- if (res) {
- // The visited SADD has already landed, so a caller retry would skip
- // this URL as already-visited and it would be lost from the crawl.
- // The visited_unique bookkeeping is idempotent — retry it before
- // exposing the failure.
+ const lockTarget = !sc.crawlerOptions?.deduplicateSimilarURLs
+ ? normalizedUrl
+ : generateURLPermutations(normalizedUrl)[0].href;
+
+ // The commands are idempotent, so retry the pipeline on failure: the
+ // SADD may land before a failing command (e.g. a failed EXPIRE), and
+ // giving up then would leave the URL marked visited without being
+ // queued — a caller retry would read SADD=0 and skip it. Track whether
+ // any attempt reported the SADD as newly added, since a retry after a
+ // partial success reads 0.
+ let sawNewLock = false;
+ let locked = false;
let lastError: unknown = null;
for (let attempt = 1; attempt <= 3; attempt++) {
let attemptError: unknown = null;
try {
- const uniquePipeline = redisEvictConnection.pipeline();
- uniquePipeline.sadd("crawl:" + id + ":visited_unique", normalizedUrl);
- uniquePipeline.expire(
- "crawl:" + id + ":visited_unique",
- 24 * 60 * 60,
- );
- attemptError = firstPipelineError(await uniquePipeline.exec());
+ const pipeline = redisEvictConnection.pipeline();
+ pipeline.sadd("crawl:" + id + ":visited", lockTarget);
+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+ const attemptResults = await pipeline.exec();
+ if (attemptResults?.[0]?.[1] === 1) {
+ sawNewLock = true;
+ }
+ attemptError = firstPipelineError(attemptResults);
} catch (error) {
attemptError = error;
}
if (attemptError === null) {
+ locked = true;
lastError = null;
break;
}
@@ -785,29 +745,42 @@ export async function lockURL(
}
}
- if (lastError !== null) {
- // Roll back the locks we just acquired, so a caller retry can
- // re-lock and queue this URL instead of skipping it as
- // already-visited. visited_unique may have landed during a retried
- // attempt whose EXPIRE failed, so remove it too — otherwise the
- // stale member consumes crawl-limit budget for a URL that was
- // never queued. The rollback itself is retried; if it still fails,
- // Redis is having a sustained outage and the canonical logs are
- // the signal.
+ if (!locked) {
+ throw new Error("URL lock pipeline failed after retries", {
+ cause: lastError,
+ });
+ }
+
+ const res = sawNewLock;
+
+ if (res) {
+ // The visited SADD has already landed, so a caller retry would skip
+ // this URL as already-visited and it would be lost from the crawl.
+ // The visited_unique bookkeeping is idempotent — retry it before
+ // exposing the failure.
+ let lastError: unknown = null;
for (let attempt = 1; attempt <= 3; attempt++) {
- let rollbackError: unknown = null;
+ let attemptError: unknown = null;
try {
- const rollback = redisEvictConnection.pipeline();
- rollback.srem("crawl:" + id + ":visited", lockTarget);
- rollback.srem("crawl:" + id + ":visited_unique", normalizedUrl);
- rollbackError = firstPipelineError(await rollback.exec());
+ const uniquePipeline = redisEvictConnection.pipeline();
+ uniquePipeline.sadd(
+ "crawl:" + id + ":visited_unique",
+ normalizedUrl,
+ );
+ uniquePipeline.expire(
+ "crawl:" + id + ":visited_unique",
+ 24 * 60 * 60,
+ );
+ attemptError = firstPipelineError(await uniquePipeline.exec());
} catch (error) {
- rollbackError = error;
+ attemptError = error;
}
- if (rollbackError === null) {
+ if (attemptError === null) {
+ lastError = null;
break;
}
+ lastError = attemptError;
__logger.error("Redis pipeline command failed", {
module: "crawl-redis",
@@ -815,7 +788,7 @@ export async function lockURL(
crawlId: id,
url: normalizedUrl,
attempt,
- error: rollbackError,
+ error: attemptError,
});
if (attempt < 3) {
@@ -824,15 +797,57 @@ export async function lockURL(
);
}
}
- throw new Error("Unique URL lock pipeline failed after retries", {
- cause: lastError,
- });
+
+ if (lastError !== null) {
+ // Roll back the locks we just acquired, so a caller retry can
+ // re-lock and queue this URL instead of skipping it as
+ // already-visited. visited_unique may have landed during a retried
+ // attempt whose EXPIRE failed, so remove it too — otherwise the
+ // stale member consumes crawl-limit budget for a URL that was
+ // never queued. The rollback itself is retried; if it still fails,
+ // Redis is having a sustained outage and the canonical logs are
+ // the signal.
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ let rollbackError: unknown = null;
+ try {
+ const rollback = redisEvictConnection.pipeline();
+ rollback.srem("crawl:" + id + ":visited", lockTarget);
+ rollback.srem("crawl:" + id + ":visited_unique", normalizedUrl);
+ rollbackError = firstPipelineError(await rollback.exec());
+ } catch (error) {
+ rollbackError = error;
+ }
+
+ if (rollbackError === null) {
+ break;
+ }
+
+ __logger.error("Redis pipeline command failed", {
+ module: "crawl-redis",
+ method: "lockURL",
+ crawlId: id,
+ url: normalizedUrl,
+ attempt,
+ error: rollbackError,
+ });
+
+ if (attempt < 3) {
+ await new Promise(resolve =>
+ setTimeout(resolve, 50 * 2 ** attempt),
+ );
+ }
+ }
+ throw new Error("Unique URL lock pipeline failed after retries", {
+ cause: lastError,
+ });
+ }
}
- }
- setSpanAttributes(span, { "crawl.url_locked": res });
- return res;
- });
+ setSpanAttributes(span, { "crawl.url_locked": res });
+ return res;
+ },
+ { zeroDataRetention: sc.zeroDataRetention },
+ );
}
/// NOTE: does not check limit. only use if limit is checked beforehand e.g. with sitemap
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 67ed0f5a9..39cd45096 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -389,7 +389,7 @@ export class ActionsNotSupportedError extends TransportableError {
/**
* Error thrown when a job is cancelled (expected flow control, not a real error)
- * This should not be sent to Sentry as it's expected behavior when a crawl/batch is cancelled
+ * This is expected behavior when a crawl/batch is cancelled
*/
export class JobCancelledError extends Error {
constructor() {
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 3203da93a..92d86f179 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -90,74 +90,78 @@ async function saveJobToGCS(params: {
const attempts: GCSOperationAttempt[] = [];
- return await withSpan("firecrawl-gcs-save-job", async span => {
- setSpanAttributes(span, {
- "gcs.operation": "save_job",
- "job.id": params.id,
- "job.request_id": params.request_id,
- "job.team_id": params.team_id,
- "job.mode": params.mode,
- "job.success": params.is_successful,
- "job.num_docs": params.num_docs,
- });
-
- if (!config.GCS_BUCKET_NAME) {
- setSpanAttributes(span, { "gcs.bucket_configured": false });
- return;
- }
+ return await withSpan(
+ "firecrawl-gcs-save-job",
+ async span => {
+ setSpanAttributes(span, {
+ "gcs.operation": "save_job",
+ "job.id": params.id,
+ "job.request_id": params.request_id,
+ "job.team_id": params.team_id,
+ "job.mode": params.mode,
+ "job.success": params.is_successful,
+ "job.num_docs": params.num_docs,
+ });
- const bucket = storageManualRetries.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(filename);
+ if (!config.GCS_BUCKET_NAME) {
+ setSpanAttributes(span, { "gcs.bucket_configured": false });
+ return;
+ }
- let backoffUsed = BACKOFF_PARAMS;
+ const bucket = storageManualRetries.bucket(config.GCS_BUCKET_NAME);
+ const blob = bucket.file(filename);
- const data = JSON.stringify(params.data);
+ let backoffUsed = BACKOFF_PARAMS;
- // Save job docs with retry
- // Due to retries and resumable uploads, this is:
- // if data is smaller than or exactly 3MB: best case 1 request, worst case 3 requests
- // if data is larger than 3MB: best case 2 requests, worst case 6 requests
- for (let i = 0; i < backoffUsed.length; i++) {
- const backoffMs = backoffUsed[i];
- if (backoffMs > 0) {
- await new Promise(resolve => setTimeout(resolve, backoffMs));
- }
+ const data = JSON.stringify(params.data);
- const saveStart = Date.now();
- try {
- await blob.save(data, {
- metadata: {
- contentType: "application/json",
- metadata: params.metadata,
- },
- resumable: data.length > 3 * 1024 * 1024, // 3MB, 5MB official limit
- });
- attempts.push({
- error: null,
- timeMs: Date.now() - saveStart,
- backoffMs,
- });
- break;
- } catch (error) {
- if (
- error instanceof ApiError &&
- (error.code === 429 || error.code === 503)
- ) {
- // switch to slower backoff parameters for rate limiting or server overloaded errors
- backoffUsed = BACKOFF_SLOWDOWN_PARAMS;
+ // Save job docs with retry
+ // Due to retries and resumable uploads, this is:
+ // if data is smaller than or exactly 3MB: best case 1 request, worst case 3 requests
+ // if data is larger than 3MB: best case 2 requests, worst case 6 requests
+ for (let i = 0; i < backoffUsed.length; i++) {
+ const backoffMs = backoffUsed[i];
+ if (backoffMs > 0) {
+ await new Promise(resolve => setTimeout(resolve, backoffMs));
}
- attempts.push({ error, timeMs: Date.now() - saveStart, backoffMs });
-
- if (i === BACKOFF_PARAMS.length - 1) {
- setSpanAttributes(span, { "gcs.save_successful": false });
- throw error;
+ const saveStart = Date.now();
+ try {
+ await blob.save(data, {
+ metadata: {
+ contentType: "application/json",
+ metadata: params.metadata,
+ },
+ resumable: data.length > 3 * 1024 * 1024, // 3MB, 5MB official limit
+ });
+ attempts.push({
+ error: null,
+ timeMs: Date.now() - saveStart,
+ backoffMs,
+ });
+ break;
+ } catch (error) {
+ if (
+ error instanceof ApiError &&
+ (error.code === 429 || error.code === 503)
+ ) {
+ // switch to slower backoff parameters for rate limiting or server overloaded errors
+ backoffUsed = BACKOFF_SLOWDOWN_PARAMS;
+ }
+
+ attempts.push({ error, timeMs: Date.now() - saveStart, backoffMs });
+
+ if (i === BACKOFF_PARAMS.length - 1) {
+ setSpanAttributes(span, { "gcs.save_successful": false });
+ throw error;
+ }
}
}
- }
- setSpanAttributes(span, { "gcs.save_successful": true });
- })
+ setSpanAttributes(span, { "gcs.save_successful": true });
+ },
+ { zeroDataRetention: params.zeroDataRetention },
+ )
.then(x => {
if (attempts.length === 0) {
return x;
diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.ts
index bdaf12cd0..20d7ced40 100644
--- a/apps/api/src/lib/html-to-markdown-client.ts
+++ b/apps/api/src/lib/html-to-markdown-client.ts
@@ -9,7 +9,6 @@ import axios, { AxiosInstance, AxiosError } from "axios";
import { config } from "../config";
import { logger } from "./logger";
import type { Logger } from "winston";
-import * as Sentry from "@sentry/node";
interface ConvertRequest {
html: string;
@@ -112,22 +111,6 @@ export async function convertHTMLToMarkdownWithHttpService(
serviceUrl: url,
});
- // Capture in Sentry with additional context (omit identifying fields
- // and content sizes when ZDR so nothing is retained in Sentry either).
- Sentry.captureException(error, {
- tags: {
- service: "html-to-markdown",
- status_code: statusCode,
- ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),
- },
- extra: {
- serviceUrl: url,
- errorMessage,
- errorDetails,
- ...(zeroDataRetention ? {} : { inputSize: html.length }),
- },
- });
-
// Include details in error message if available
const fullErrorMessage = errorDetails
? `HTML to Markdown conversion failed: ${errorMessage} - ${errorDetails}`
@@ -141,11 +124,6 @@ export async function convertHTMLToMarkdownWithHttpService(
error,
},
);
- Sentry.captureException(error, {
- tags: {
- ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),
- },
- });
throw error;
}
}
diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts
index 1b217f3b5..eeed6d66a 100644
--- a/apps/api/src/lib/html-to-markdown.ts
+++ b/apps/api/src/lib/html-to-markdown.ts
@@ -1,7 +1,5 @@
import koffi from "koffi";
import { config } from "../config";
-import "../services/sentry";
-import * as Sentry from "@sentry/node";
import { logger } from "./logger";
import type { Logger } from "winston";
import { stat } from "fs/promises";
@@ -82,12 +80,6 @@ export async function parseMarkdown(
"Error converting HTML to Markdown with HTTP service, falling back to original parser",
{ error },
);
- Sentry.captureException(error, {
- tags: {
- fallback: "original_parser",
- ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),
- },
- });
}
}
@@ -103,11 +95,6 @@ export async function parseMarkdown(
!(error instanceof Error) ||
error.message !== "Go shared library not found"
) {
- Sentry.captureException(error, {
- tags: {
- ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),
- },
- });
contextLogger.error(
`Error converting HTML to Markdown with Go parser: ${error}`,
);
diff --git a/apps/api/src/lib/otel-tracer.test.ts b/apps/api/src/lib/otel-tracer.test.ts
new file mode 100644
index 000000000..134b6e0a1
--- /dev/null
+++ b/apps/api/src/lib/otel-tracer.test.ts
@@ -0,0 +1,244 @@
+import {
+ context,
+ propagation,
+ SpanStatusCode,
+ trace,
+} from "@opentelemetry/api";
+import {
+ InMemorySpanExporter,
+ type NodeTracerProvider,
+} from "@opentelemetry/sdk-trace-node";
+import {
+ createTracerProvider,
+ serializeTraceContext,
+ setSpanAttributes,
+ SpanKind,
+ withSpan,
+ withTraceContextAsync,
+ withZeroDataRetention,
+ type SerializedTraceContext,
+} from "./otel-tracer";
+
+describe("otel-tracer", () => {
+ const exporter = new InMemorySpanExporter();
+ let provider: NodeTracerProvider;
+ // The provider's env detector reads these; keep the suite hermetic.
+ const savedEnv = {
+ OTEL_SERVICE_NAME: process.env.OTEL_SERVICE_NAME,
+ OTEL_RESOURCE_ATTRIBUTES: process.env.OTEL_RESOURCE_ATTRIBUTES,
+ };
+
+ beforeAll(() => {
+ delete process.env.OTEL_SERVICE_NAME;
+ delete process.env.OTEL_RESOURCE_ATTRIBUTES;
+ provider = createTracerProvider({
+ exporter,
+ serviceName: "test-service",
+ serviceInstanceId: "pod-1",
+ });
+ provider.register();
+ });
+
+ afterAll(async () => {
+ await provider.shutdown();
+ trace.disable();
+ context.disable();
+ propagation.disable();
+ for (const [key, value] of Object.entries(savedEnv)) {
+ if (value === undefined) delete process.env[key];
+ else process.env[key] = value;
+ }
+ });
+
+ beforeEach(() => {
+ exporter.reset();
+ });
+
+ async function exportedSpans() {
+ await provider.forceFlush();
+ return exporter.getFinishedSpans();
+ }
+
+ it("exports spans with kind, attributes, status, and resource", async () => {
+ const result = await withSpan(
+ "test.root",
+ async span => {
+ setSpanAttributes(span, {
+ "a.string": "x",
+ "a.number": 1,
+ "a.undefined": undefined,
+ "a.null": null,
+ });
+ return 42;
+ },
+ { kind: SpanKind.SERVER, attributes: { initial: true } },
+ );
+
+ expect(result).toBe(42);
+ const spans = await exportedSpans();
+ expect(spans).toHaveLength(1);
+ expect(spans[0].name).toBe("test.root");
+ expect(spans[0].kind).toBe(SpanKind.SERVER);
+ expect(spans[0].attributes).toEqual({
+ initial: true,
+ "a.string": "x",
+ "a.number": 1,
+ });
+ expect(spans[0].status.code).toBe(SpanStatusCode.OK);
+ expect(spans[0].resource.attributes["service.name"]).toBe("test-service");
+ expect(spans[0].resource.attributes["service.instance.id"]).toBe("pod-1");
+ });
+
+ it("records the exception, marks the span as errored, and rethrows", async () => {
+ await expect(
+ withSpan("test.fail", async () => {
+ throw new Error("boom");
+ }),
+ ).rejects.toThrow("boom");
+
+ const [span] = await exportedSpans();
+ expect(span.status).toEqual({
+ code: SpanStatusCode.ERROR,
+ message: "boom",
+ });
+ expect(span.events.map(event => event.name)).toContain("exception");
+ });
+
+ it("parents nested and library spans under the active span", async () => {
+ await withSpan("test.parent", async () => {
+ await withSpan("test.child", async () => {});
+ // The AI SDK reports through the global tracer, like this.
+ trace.getTracer("ai").startSpan("ai.generateText").end();
+ });
+
+ const spans = await exportedSpans();
+ const parent = spans.find(span => span.name === "test.parent")!;
+ expect(spans.map(span => span.name).sort()).toEqual([
+ "ai.generateText",
+ "test.child",
+ "test.parent",
+ ]);
+ for (const span of spans.filter(span => span !== parent)) {
+ expect(span.parentSpanContext?.spanId).toBe(parent.spanContext().spanId);
+ }
+ });
+
+ it("records nothing started under zero data retention", async () => {
+ await withSpan(
+ "zdr.root",
+ async span => {
+ expect(span.isRecording()).toBe(false);
+ await withSpan("zdr.child", async () => {});
+ trace.getTracer("ai").startSpan("ai.generateText").end();
+ },
+ { zeroDataRetention: true },
+ );
+ await withZeroDataRetention(true, () =>
+ withSpan("zdr.nested", async () => {}),
+ );
+ await withSpan("kept.root", async () => {}, { zeroDataRetention: false });
+
+ expect((await exportedSpans()).map(span => span.name)).toEqual([
+ "kept.root",
+ ]);
+ });
+
+ it("drops spans that end with a zero_data_retention attribute", async () => {
+ await withSpan("attr.dropped", async span => {
+ setSpanAttributes(span, { "nuq.zero_data_retention": true });
+ });
+ await withSpan("attr.kept", async span => {
+ setSpanAttributes(span, { "nuq.zero_data_retention": false });
+ });
+
+ expect((await exportedSpans()).map(span => span.name)).toEqual([
+ "attr.kept",
+ ]);
+ });
+
+ it("suppresses descendants once a live span is flagged zero data retention", async () => {
+ await withSpan("late.root", async span => {
+ setSpanAttributes(span, { "nuq.zero_data_retention": true });
+ await withSpan("late.child", async () => {});
+ trace.getTracer("ai").startSpan("ai.generateText").end();
+ });
+
+ expect(await exportedSpans()).toHaveLength(0);
+ });
+
+ it("drops in-flight spans of a trace flagged after they started", async () => {
+ await withSpan("late.root", async root => {
+ await withSpan("late.child", async () => {
+ setSpanAttributes(root, { "crawl.zero_data_retention": true });
+ });
+ });
+ await withSpan("other.root", async () => {});
+
+ expect((await exportedSpans()).map(span => span.name)).toEqual([
+ "other.root",
+ ]);
+ });
+
+ it("continues a serialized trace and inherits the sampled decision", async () => {
+ let serialized: SerializedTraceContext = {};
+ let apiSpanId = "";
+ let traceId = "";
+ await withSpan("api.request", async span => {
+ serialized = serializeTraceContext();
+ apiSpanId = span.spanContext().spanId;
+ traceId = span.spanContext().traceId;
+ });
+ expect(serialized.traceparent).toBe(`00-${traceId}-${apiSpanId}-01`);
+
+ await withTraceContextAsync(serialized, () =>
+ withSpan("worker.process", async () => {}),
+ );
+
+ const worker = (await exportedSpans()).find(
+ span => span.name === "worker.process",
+ )!;
+ expect(worker.spanContext().traceId).toBe(traceId);
+ expect(worker.parentSpanContext?.spanId).toBe(apiSpanId);
+ });
+
+ it("keeps a continued zero-data-retention trace unrecorded", async () => {
+ let serialized: SerializedTraceContext = {};
+ await withSpan(
+ "api.zdr",
+ async () => {
+ serialized = serializeTraceContext();
+ },
+ { zeroDataRetention: true },
+ );
+ expect(serialized.traceparent).toMatch(/-00$/);
+
+ await withTraceContextAsync(serialized, () =>
+ withSpan("worker.process", async () => {}),
+ );
+
+ expect(await exportedSpans()).toHaveLength(0);
+ });
+
+ it("starts a fresh trace for missing or legacy serialized contexts", async () => {
+ await withTraceContextAsync(undefined, () =>
+ withSpan("worker.none", async () => {}),
+ );
+ await withTraceContextAsync(
+ { sentryTrace: "legacy" } as unknown as SerializedTraceContext,
+ () => withSpan("worker.legacy", async () => {}),
+ );
+
+ const spans = await exportedSpans();
+ expect(spans.map(span => span.name).sort()).toEqual([
+ "worker.legacy",
+ "worker.none",
+ ]);
+ for (const span of spans) {
+ expect(span.parentSpanContext).toBeUndefined();
+ }
+ });
+
+ it("serializes an empty context outside of a span", () => {
+ expect(serializeTraceContext()).toEqual({});
+ });
+});
diff --git a/apps/api/src/lib/otel-tracer.ts b/apps/api/src/lib/otel-tracer.ts
index bca592fe1..aaeb53c2f 100644
--- a/apps/api/src/lib/otel-tracer.ts
+++ b/apps/api/src/lib/otel-tracer.ts
@@ -1,71 +1,286 @@
-import * as Sentry from "@sentry/node";
-import type { Span } from "@sentry/node";
-import type { SpanAttributeValue, SpanAttributes } from "@sentry/core";
+import {
+ context,
+ createContextKey,
+ isSpanContextValid,
+ propagation,
+ SpanKind,
+ SpanStatusCode,
+ trace,
+ type Attributes,
+ type AttributeValue,
+ type Context,
+ type Link,
+ type Span,
+} from "@opentelemetry/api";
+import {
+ AlwaysOnSampler,
+ BatchSpanProcessor,
+ NodeTracerProvider,
+ ParentBasedSampler,
+ SamplingDecision,
+ type ReadableSpan,
+ type Sampler,
+ type SamplingResult,
+ type SpanExporter,
+ type SpanProcessor,
+} from "@opentelemetry/sdk-trace-node";
+import {
+ defaultResource,
+ detectResources,
+ envDetector,
+ resourceFromAttributes,
+} from "@opentelemetry/resources";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
-export enum SpanKind {
- INTERNAL = 0,
- SERVER = 1,
- CLIENT = 2,
- PRODUCER = 3,
- CONSUMER = 4,
+export { SpanKind } from "@opentelemetry/api";
+export type { Span } from "@opentelemetry/api";
+
+const TRACER_NAME = "firecrawl-api";
+
+// Zero data retention (ZDR) spans must never leave the process. Two layers
+// enforce that:
+// 1. `withZeroDataRetention` flags the active context; the sampler turns every
+// span started under it (our own and library spans such as the AI SDK's)
+// into a non-recording span, so nothing is even buffered.
+// 2. Any span that ends carrying a `zero_data_retention` /
+// `*.zero_data_retention` attribute set to `true` is dropped before export,
+// covering spans that only learn about ZDR after they started.
+const ZERO_DATA_RETENTION_CONTEXT_KEY = createContextKey(
+ "firecrawl.zero_data_retention",
+);
+
+export function withZeroDataRetention<T>(
+ zeroDataRetention: boolean | undefined,
+ fn: () => T,
+): T {
+ if (!zeroDataRetention) {
+ return fn();
+ }
+
+ return context.with(
+ context.active().setValue(ZERO_DATA_RETENTION_CONTEXT_KEY, true),
+ fn,
+ );
+}
+
+function isZeroDataRetentionContext(ctx: Context): boolean {
+ return ctx.getValue(ZERO_DATA_RETENTION_CONTEXT_KEY) === true;
+}
+
+// Traces flagged ZDR after their root span started (a `*.zero_data_retention`
+// attribute set from inside a *recording* span). Descendants started afterwards
+// are not recorded and spans of these traces still in flight are dropped when
+// they end. Only recording spans register here: a non-recording span already
+// suppresses its descendants through parent-based sampling, so the registry
+// only sees the rare late-discovery case and stays far below its cap. The cap
+// is a memory guard, not an expected steady state.
+const ZERO_DATA_RETENTION_TRACE_LIMIT = 100_000;
+const zeroDataRetentionTraces = new Set<string>();
+
+function markTraceZeroDataRetention(traceId: string): void {
+ if (zeroDataRetentionTraces.has(traceId)) {
+ return;
+ }
+
+ zeroDataRetentionTraces.add(traceId);
+ if (zeroDataRetentionTraces.size > ZERO_DATA_RETENTION_TRACE_LIMIT) {
+ const oldest = zeroDataRetentionTraces.values().next().value;
+ if (oldest !== undefined) {
+ zeroDataRetentionTraces.delete(oldest);
+ }
+ }
+}
+
+function hasZeroDataRetentionAttribute(attributes: Attributes): boolean {
+ for (const [key, value] of Object.entries(attributes)) {
+ if (
+ value === true &&
+ (key === "zero_data_retention" || key.endsWith(".zero_data_retention"))
+ ) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+class ZeroDataRetentionSampler implements Sampler {
+ constructor(private readonly delegate: Sampler) {}
+
+ shouldSample(
+ ctx: Context,
+ traceId: string,
+ spanName: string,
+ spanKind: SpanKind,
+ attributes: Attributes,
+ links: Link[],
+ ): SamplingResult {
+ if (hasZeroDataRetentionAttribute(attributes)) {
+ markTraceZeroDataRetention(traceId);
+ return { decision: SamplingDecision.NOT_RECORD };
+ }
+
+ if (
+ isZeroDataRetentionContext(ctx) ||
+ zeroDataRetentionTraces.has(traceId)
+ ) {
+ return { decision: SamplingDecision.NOT_RECORD };
+ }
+
+ return this.delegate.shouldSample(
+ ctx,
+ traceId,
+ spanName,
+ spanKind,
+ attributes,
+ links,
+ );
+ }
+
+ toString(): string {
+ return `ZeroDataRetentionSampler(${this.delegate.toString()})`;
+ }
+}
+
+class ZeroDataRetentionSpanProcessor implements SpanProcessor {
+ constructor(private readonly delegate: SpanProcessor) {}
+
+ onStart(
+ span: Parameters<SpanProcessor["onStart"]>[0],
+ parentContext: Context,
+ ): void {
+ this.delegate.onStart(span, parentContext);
+ }
+
+ onEnd(span: ReadableSpan): void {
+ const traceId = span.spanContext().traceId;
+ if (hasZeroDataRetentionAttribute(span.attributes)) {
+ markTraceZeroDataRetention(traceId);
+ return;
+ }
+
+ if (zeroDataRetentionTraces.has(traceId)) {
+ return;
+ }
+
+ this.delegate.onEnd(span);
+ }
+
+ forceFlush(): Promise<void> {
+ return this.delegate.forceFlush();
+ }
+
+ shutdown(): Promise<void> {
+ return this.delegate.shutdown();
+ }
+}
+
+interface TracerProviderOptions {
+ exporter: SpanExporter;
+ serviceName: string;
+ serviceInstanceId?: string;
}
-const SPAN_STATUS_OK = 1;
-const SPAN_STATUS_ERROR = 2;
+/**
+ * Builds the tracer provider used by every Firecrawl process: 100% sampling
+ * (respecting the sampled flag of a remote parent), ZDR filtering, and batched
+ * export. Callers decide whether to `register()` it globally.
+ */
+export function createTracerProvider(
+ options: TracerProviderOptions,
+): NodeTracerProvider {
+ const resource = defaultResource()
+ .merge(detectResources({ detectors: [envDetector] }))
+ .merge(
+ resourceFromAttributes({
+ [ATTR_SERVICE_NAME]: options.serviceName,
+ ...(options.serviceInstanceId
+ ? { "service.instance.id": options.serviceInstanceId }
+ : {}),
+ }),
+ );
+ return new NodeTracerProvider({
+ resource,
+ sampler: new ZeroDataRetentionSampler(
+ new ParentBasedSampler({ root: new AlwaysOnSampler() }),
+ ),
+ spanProcessors: [
+ new ZeroDataRetentionSpanProcessor(
+ new BatchSpanProcessor(options.exporter),
+ ),
+ ],
+ });
+}
+
+/**
+ * W3C trace context carrier, stored on queue jobs so a worker can continue the
+ * trace the API request started.
+ */
export interface SerializedTraceContext {
- sentryTrace?: string;
+ traceparent?: string;
+ tracestate?: string;
baggage?: string;
}
export function serializeTraceContext(): SerializedTraceContext {
- const traceData = Sentry.getTraceData();
+ const carrier: Record<string, string> = {};
+ propagation.inject(context.active(), carrier);
- return {
- sentryTrace: traceData["sentry-trace"],
- baggage: traceData["baggage"],
- };
+ const serialized: SerializedTraceContext = {};
+ if (carrier.traceparent) serialized.traceparent = carrier.traceparent;
+ if (carrier.tracestate) serialized.tracestate = carrier.tracestate;
+ if (carrier.baggage) serialized.baggage = carrier.baggage;
+ return serialized;
}
export async function withTraceContextAsync<T>(
- serializedContext: SerializedTraceContext,
+ serializedContext: SerializedTraceContext | undefined,
fn: () => Promise<T>,
): Promise<T> {
- if (!serializedContext.sentryTrace) {
+ if (!serializedContext?.traceparent) {
return fn();
}
- return Sentry.continueTrace(
- {
- sentryTrace: serializedContext.sentryTrace,
- baggage: serializedContext.baggage,
- },
- fn,
- );
+ const carrier: Record<string, string> = {
+ traceparent: serializedContext.traceparent,
+ };
+ if (serializedContext.tracestate) {
+ carrier.tracestate = serializedContext.tracestate;
+ }
+ if (serializedContext.baggage) {
+ carrier.baggage = serializedContext.baggage;
+ }
+
+ return context.with(propagation.extract(context.active(), carrier), fn);
}
-type Attributes = SpanAttributes;
+type SpanAttributeInput = Record<string, AttributeValue | null | undefined>;
-interface SpanOptions {
- attributes?: Attributes;
- kind?: SpanKind;
- op?: string;
-}
+function cleanAttributes(
+ attributes: SpanAttributeInput | undefined,
+): Attributes | undefined {
+ if (!attributes) {
+ return undefined;
+ }
-function spanKindToOp(kind?: SpanKind): string | undefined {
- switch (kind) {
- case SpanKind.SERVER:
- return "http.server";
- case SpanKind.CLIENT:
- return "http.client";
- case SpanKind.PRODUCER:
- return "queue.publish";
- case SpanKind.CONSUMER:
- return "queue.process";
- case SpanKind.INTERNAL:
- default:
- return undefined;
+ const cleaned: Attributes = {};
+ for (const [key, value] of Object.entries(attributes)) {
+ if (value !== undefined && value !== null) {
+ cleaned[key] = value;
+ }
}
+ return cleaned;
+}
+
+interface SpanOptions {
+ attributes?: SpanAttributeInput;
+ kind?: SpanKind;
+ /**
+ * Marks the span and everything started inside it as zero data retention:
+ * nothing is recorded or exported.
+ */
+ zeroDataRetention?: boolean;
}
export async function withSpan<T>(
@@ -73,35 +288,61 @@ export async function withSpan<T>(
fn: (span: Span) => Promise<T>,
options?: SpanOptions,
): Promise<T> {
- const op = options?.op || spanKindToOp(options?.kind);
+ const tracer = trace.getTracer(TRACER_NAME);
- return Sentry.startSpan(
- {
+ return withZeroDataRetention(options?.zeroDataRetention, () =>
+ tracer.startActiveSpan(
name,
- op,
- attributes: options?.attributes,
- },
- async span => {
- try {
- const result = await fn(span);
- span.setStatus({ code: SPAN_STATUS_OK });
- return result;
- } catch (error) {
- span.setStatus({
- code: SPAN_STATUS_ERROR,
- message: error instanceof Error ? error.message : String(error),
- });
-
- throw error;
- }
- },
+ { kind: options?.kind, attributes: cleanAttributes(options?.attributes) },
+ async span => {
+ try {
+ const result = await fn(span);
+ span.setStatus({ code: SpanStatusCode.OK });
+ return result;
+ } catch (error) {
+ recordSpanException(span, error);
+ throw error;
+ } finally {
+ span.end();
+ }
+ },
+ ),
);
}
-export function setSpanAttributes(span: Span, attributes: Attributes): void {
- Object.entries(attributes).forEach(([key, value]) => {
- if (value !== undefined && value !== null) {
- span.setAttribute(key, value as SpanAttributeValue);
- }
+/**
+ * Marks a span as failed and attaches the error as an exception event, for
+ * failures that are handled (returned) rather than thrown out of `withSpan`.
+ */
+export function recordSpanException(span: Span, error: unknown): void {
+ span.recordException(error instanceof Error ? error : String(error));
+ span.setStatus({
+ code: SpanStatusCode.ERROR,
+ message: error instanceof Error ? error.message : String(error),
});
}
+
+export function setSpanAttributes(
+ span: Span,
+ attributes: SpanAttributeInput,
+): void {
+ const cleaned = cleanAttributes(attributes);
+ if (!cleaned) {
+ return;
+ }
+
+ span.setAttributes(cleaned);
+
+ // Flagging a live, recording span as ZDR also covers everything started under
+ // it from now on, plus spans of the trace that are still in flight. A
+ // non-recording span needs no registry entry: its descendants are already
+ // dropped by parent-based sampling.
+ const spanContext = span.spanContext();
+ if (
+ span.isRecording() &&
+ hasZeroDataRetentionAttribute(cleaned) &&
+ isSpanContextValid(spanContext)
+ ) {
+ markTraceZeroDataRetention(spanContext.traceId);
+ }
+}
diff --git a/apps/api/src/lib/queue-full-error.ts b/apps/api/src/lib/queue-full-error.ts
index 5e9e44245..3cea8e5ca 100644
--- a/apps/api/src/lib/queue-full-error.ts
+++ b/apps/api/src/lib/queue-full-error.ts
@@ -1,5 +1,4 @@
const QUEUE_FULL_ERROR_NAME = "QueueFullError";
-const QUEUE_FULL_ERROR_MESSAGE_PREFIX = "Queue limit reached:";
export class QueueFullError extends Error {
statusCode = 429;
@@ -11,41 +10,3 @@ export class QueueFullError extends Error {
this.name = QUEUE_FULL_ERROR_NAME;
}
}
-
-function getStringField(
- error: Record<string, unknown>,
- fields: string[],
-): string {
- for (const field of fields) {
- if (field in error && error[field] !== undefined && error[field] !== null) {
- return String(error[field]);
- }
- }
-
- return "";
-}
-
-export function isQueueFullError(error: unknown): boolean {
- const objectError =
- error && typeof error === "object"
- ? (error as Record<string, unknown>)
- : null;
-
- const errorName = objectError
- ? getStringField(objectError, ["name", "type"])
- : "";
- if (errorName === QUEUE_FULL_ERROR_NAME) {
- return true;
- }
-
- const errorMessage =
- error instanceof Error
- ? error.message
- : typeof error === "string"
- ? error
- : objectError
- ? getStringField(objectError, ["message", "value"])
- : "";
-
- return errorMessage.startsWith(QUEUE_FULL_ERROR_MESSAGE_PREFIX);
-}
diff --git a/apps/api/src/lib/retry-utils.ts b/apps/api/src/lib/retry-utils.ts
index e32269949..c794bbbdb 100644
--- a/apps/api/src/lib/retry-utils.ts
+++ b/apps/api/src/lib/retry-utils.ts
@@ -1,4 +1,3 @@
-import * as Sentry from "@sentry/node";
import { logger } from "./logger";
const RETRY_DELAYS = [500, 1500, 3000] as const;
@@ -47,7 +46,6 @@ export async function attemptRequest<T>(
}
} catch (error) {
logger.error("Fire Engine API request failed:", error);
- Sentry.captureException(error);
}
return null;
}
@@ -107,7 +105,6 @@ export async function executeWithRetry<T>(
}
logger.error(`Attempt ${attempt + 1} failed:`, error);
- Sentry.captureException(error);
}
// Wait before retry (except on last attempt)
diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts
index 0c9d8db59..46e6672eb 100644
--- a/apps/api/src/lib/supabase-jobs.ts
+++ b/apps/api/src/lib/supabase-jobs.ts
@@ -3,7 +3,6 @@ import { eq, inArray, and } from "drizzle-orm";
import { db, dbRr } from "../db/connection";
import * as schema from "../db/schema";
import { logger } from "./logger";
-import * as Sentry from "@sentry/node";
/**
* Get a single scrape by ID from the scrapes table
@@ -38,7 +37,6 @@ export const supabaseGetScrapesById = async (
.where(inArray(schema.scrapes.id, scrapeIds));
} catch (error) {
logger.error(`Error in supabaseGetScrapesById: ${error}`);
- Sentry.captureException(error);
return [];
}
};
@@ -58,7 +56,6 @@ export const supabaseGetScrapesByRequestId = async (
.where(eq(schema.scrapes.request_id, requestId));
} catch (error) {
logger.error(`Error in supabaseGetScrapesByRequestId: ${error}`);
- Sentry.captureException(error);
return [];
}
};
diff --git a/apps/api/src/lib/withAuth.ts b/apps/api/src/lib/withAuth.ts
index a9d602cbd..305322aaa 100644
--- a/apps/api/src/lib/withAuth.ts
+++ b/apps/api/src/lib/withAuth.ts
@@ -1,6 +1,5 @@
import { AuthResponse } from "../../src/types";
import { logger } from "./logger";
-import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { config } from "../config";
configDotenv();
diff --git a/apps/api/src/otel.ts b/apps/api/src/otel.ts
new file mode 100644
index 000000000..feecb4709
--- /dev/null
+++ b/apps/api/src/otel.ts
@@ -0,0 +1,57 @@
+import { diag, DiagLogLevel, type DiagLogger } from "@opentelemetry/api";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
+import { config } from "./config";
+import { logger } from "./lib/logger";
+import { createTracerProvider } from "./lib/otel-tracer";
+import type { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+
+// Imported for its side effects by every entry point, before anything that
+// starts spans. Tracing stays off unless an OTLP endpoint is configured, in
+// which case spans go to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` or
+// `OTEL_EXPORTER_OTLP_ENDPOINT` + `/v1/traces` over http/protobuf. The exporter
+// reads the standard `OTEL_EXPORTER_OTLP_*` variables (headers, timeout,
+// compression) itself.
+const endpoint =
+ config.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ||
+ config.OTEL_EXPORTER_OTLP_ENDPOINT;
+
+let provider: NodeTracerProvider | undefined;
+
+if (endpoint) {
+ const otelLogger = logger.child({ module: "otel" });
+ const diagLogger: DiagLogger = {
+ error: (message, ...args) => otelLogger.error(message, { args }),
+ warn: (message, ...args) => otelLogger.warn(message, { args }),
+ info: (message, ...args) => otelLogger.info(message, { args }),
+ debug: (message, ...args) => otelLogger.debug(message, { args }),
+ verbose: (message, ...args) => otelLogger.debug(message, { args }),
+ };
+ diag.setLogger(diagLogger, DiagLogLevel.WARN);
+
+ const serviceName = config.OTEL_SERVICE_NAME ?? "firecrawl-api";
+ provider = createTracerProvider({
+ exporter: new OTLPTraceExporter(),
+ serviceName,
+ serviceInstanceId: config.NUQ_POD_NAME,
+ });
+ provider.register();
+
+ otelLogger.info("OpenTelemetry tracing enabled", { endpoint, serviceName });
+}
+
+/**
+ * Flushes buffered spans and stops the exporter. Bounded so a stuck collector
+ * cannot hold up process shutdown.
+ */
+export async function shutdownTracing(timeoutMs = 5000): Promise<void> {
+ if (!provider) {
+ return;
+ }
+
+ const active = provider;
+ provider = undefined;
+ await Promise.race([
+ active.shutdown().catch(() => {}),
+ new Promise<void>(resolve => setTimeout(resolve, timeoutMs).unref()),
+ ]);
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index e847dae1f..d200a13c3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -1,5 +1,4 @@
import { Logger } from "winston";
-import * as Sentry from "@sentry/node";
import { z } from "zod";
import { robustFetch } from "../../lib/fetch";
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 3c4f35aeb..67b1082ed 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -25,7 +25,6 @@ import {
FEPageLoadFailed,
ProxySelectionError,
} from "../../error";
-import * as Sentry from "@sentry/node";
import { gunzipSync } from "node:zlib";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import {
@@ -202,7 +201,6 @@ async function performFireEngineScrape<
`An unexpeceted error occurred while calling checkStatus. Error counter is now at ${errors.length}.`,
{ error, jobId: (scrape as any).jobId },
);
- Sentry.captureException(error);
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 955489d5e..f3875d2c5 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -58,7 +58,6 @@ import { runFirePdfByReferenceAttempt } from "./fire-pdf/by-reference-flow";
import { decideFirePdfAsyncRoute } from "./fire-pdf/routing";
import { scrapePDFWithParsePDF } from "./pdfParse";
import { toPublicBlocks } from "./blocks";
-import { captureExceptionWithZdrCheck } from "../../../../services/sentry";
import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";
import { comparePdfOutputs } from "./shadowComparison";
import { withPdfExtractionPermit } from "./semaphore";
@@ -370,14 +369,6 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
error,
url: meta.rewrittenUrl ?? meta.url,
});
- captureExceptionWithZdrCheck(error, {
- extra: {
- zeroDataRetention: meta.internalOptions.zeroDataRetention ?? false,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- url: meta.rewrittenUrl ?? meta.url,
- },
- });
}
} else {
// Rust extraction enabled (fast / auto modes).
@@ -483,14 +474,6 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
error,
url: meta.rewrittenUrl ?? meta.url,
});
- captureExceptionWithZdrCheck(error, {
- extra: {
- zeroDataRetention: meta.internalOptions.zeroDataRetention ?? false,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- url: meta.rewrittenUrl ?? meta.url,
- },
- });
// effectivePageCount stays 0 — skip time budget check
}
}
@@ -882,15 +865,6 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
"RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",
{ error },
);
- captureExceptionWithZdrCheck(error, {
- extra: {
- zeroDataRetention:
- meta.internalOptions.zeroDataRetention ?? false,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- url: meta.rewrittenUrl ?? meta.url,
- },
- });
const muV1DurationMs = Date.now() - muV1StartedAt;
meta.logger
.child({ method: "scrapePDF/MUv1Experiment" })
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index dfe3bae08..3f797fcd2 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1,8 +1,10 @@
import { Logger } from "winston";
import { config } from "../../config";
-import * as Sentry from "@sentry/node";
-import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
-import { captureExceptionWithZdrCheck } from "../../services/sentry";
+import {
+ withSpan,
+ setSpanAttributes,
+ recordSpanException,
+} from "../../lib/otel-tracer";
import {
applyScrapeOptionsDefaults,
@@ -1227,597 +1229,621 @@ export async function scrapeURL(
internalOptions: InternalOptions,
costTracking: CostTracking,
): Promise<ScrapeUrlResponse> {
- return withSpan("scrape.pipeline", async span => {
- const meta = await buildMetaObject(
- id,
- url,
- options,
- internalOptions,
- costTracking,
- );
-
- const startTime = Date.now();
+ return withSpan(
+ "scrape.pipeline",
+ async span => {
+ const meta = await buildMetaObject(
+ id,
+ url,
+ options,
+ internalOptions,
+ costTracking,
+ );
- // Set initial span attributes
- setSpanAttributes(span, {
- "scrape.id": id,
- "scrape.url": url,
- "scrape.team_id": internalOptions.teamId,
- "scrape.crawl_id": internalOptions.crawlId,
- "scrape.zero_data_retention": internalOptions.zeroDataRetention,
- "scrape.force_engine": Array.isArray(internalOptions.forceEngine)
- ? internalOptions.forceEngine.join(",")
- : internalOptions.forceEngine,
- "scrape.features": Array.from(meta.featureFlags).join(","),
- });
+ const startTime = Date.now();
- meta.logger.info("scrapeURL entered");
-
- // Threat protection: check the target URL BEFORE any engine selection
- // or outbound fetch. The policy is resolved at the controller layer and
- // threaded through internalOptions; absent policy = zero overhead.
- // The dedup map is scoped to this one scrape: the initial check and any
- // redirect re-check on the same URL share a single scan (one fee); a
- // redirect to a different URL is a second scan and bills a second fee
- // (see calculateThreatScanCredits).
- const threatPolicy = internalOptions.threatProtection;
- const threatDedup: ThreatCheckDedup = new Map();
- if (threatPolicy && threatPolicy.mode !== "off") {
- const initialUrl = meta.rewrittenUrl ?? meta.url;
- const decision = await checkUrl(initialUrl, threatPolicy, {
- teamId: internalOptions.teamId,
- dedup: threatDedup,
+ // Set initial span attributes
+ setSpanAttributes(span, {
+ "scrape.id": id,
+ "scrape.url": url,
+ "scrape.team_id": internalOptions.teamId,
+ "scrape.crawl_id": internalOptions.crawlId,
+ "scrape.zero_data_retention": internalOptions.zeroDataRetention,
+ "scrape.force_engine": Array.isArray(internalOptions.forceEngine)
+ ? internalOptions.forceEngine.join(",")
+ : internalOptions.forceEngine,
+ "scrape.features": Array.from(meta.featureFlags).join(","),
});
- meta.threatDecisions.push(decision);
- if (!decision.allowed) {
- meta.logger.info("URL blocked by threat protection policy", {
- url: initialUrl,
- domain: decision.domain,
- rule: decision.rule,
+
+ meta.logger.info("scrapeURL entered");
+
+ // Threat protection: check the target URL BEFORE any engine selection
+ // or outbound fetch. The policy is resolved at the controller layer and
+ // threaded through internalOptions; absent policy = zero overhead.
+ // The dedup map is scoped to this one scrape: the initial check and any
+ // redirect re-check on the same URL share a single scan (one fee); a
+ // redirect to a different URL is a second scan and bills a second fee
+ // (see calculateThreatScanCredits).
+ const threatPolicy = internalOptions.threatProtection;
+ const threatDedup: ThreatCheckDedup = new Map();
+ if (threatPolicy && threatPolicy.mode !== "off") {
+ const initialUrl = meta.rewrittenUrl ?? meta.url;
+ const decision = await checkUrl(initialUrl, threatPolicy, {
+ teamId: internalOptions.teamId,
+ dedup: threatDedup,
});
+ meta.threatDecisions.push(decision);
+ if (!decision.allowed) {
+ meta.logger.info("URL blocked by threat protection policy", {
+ url: initialUrl,
+ domain: decision.domain,
+ rule: decision.rule,
+ });
+ setSpanAttributes(span, {
+ "scrape.blocked_by_threat_protection": true,
+ });
+ return {
+ success: false,
+ error: new UnsafeDomainBlockedError(initialUrl, decision),
+ threatDecisions: meta.threatDecisions,
+ };
+ }
+ }
+
+ if (meta.rewrittenUrl) {
+ meta.logger.info("Rewriting URL");
setSpanAttributes(span, {
- "scrape.blocked_by_threat_protection": true,
+ "scrape.rewritten_url": meta.rewrittenUrl,
});
- return {
- success: false,
- error: new UnsafeDomainBlockedError(initialUrl, decision),
- threatDecisions: meta.threatDecisions,
- };
}
- }
- if (meta.rewrittenUrl) {
- meta.logger.info("Rewriting URL");
- setSpanAttributes(span, {
- "scrape.rewritten_url": meta.rewrittenUrl,
- });
- }
-
- if (internalOptions.isPreCrawl === true) {
- setSpanAttributes(span, {
- "scrape.is_precrawl": true,
- });
- }
-
- if (shouldCheckRobots(options, internalOptions)) {
- const result = await withSpan("scrape.robots_check", async robotsSpan => {
- const urlToCheck = meta.rewrittenUrl || meta.url;
- meta.logger.info("Checking robots.txt", { url: urlToCheck });
-
- const urlObj = new URL(urlToCheck);
- const isRobotsTxtPath = urlObj.pathname === "/robots.txt";
-
- setSpanAttributes(robotsSpan, {
- "robots.url": urlToCheck,
- "robots.is_robots_txt_path": isRobotsTxtPath,
+ if (internalOptions.isPreCrawl === true) {
+ setSpanAttributes(span, {
+ "scrape.is_precrawl": true,
});
+ }
- if (!isRobotsTxtPath) {
- try {
- let robotsTxt: string | undefined;
- if (internalOptions.crawlId) {
- const crawl = await getCrawl(internalOptions.crawlId);
- robotsTxt = crawl?.robots;
- }
-
- if (!robotsTxt) {
- const { content } = await fetchRobotsTxt(
- {
- url: urlToCheck,
- zeroDataRetention: internalOptions.zeroDataRetention || false,
- location: options.location,
- },
- id,
- meta.logger,
- meta.abort.asSignal(),
- );
- robotsTxt = content;
- }
+ if (shouldCheckRobots(options, internalOptions)) {
+ const result = await withSpan(
+ "scrape.robots_check",
+ async robotsSpan => {
+ const urlToCheck = meta.rewrittenUrl || meta.url;
+ meta.logger.info("Checking robots.txt", { url: urlToCheck });
- const checker = createRobotsChecker(urlToCheck, robotsTxt);
- const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);
+ const urlObj = new URL(urlToCheck);
+ const isRobotsTxtPath = urlObj.pathname === "/robots.txt";
setSpanAttributes(robotsSpan, {
- "robots.allowed": isAllowed,
+ "robots.url": urlToCheck,
+ "robots.is_robots_txt_path": isRobotsTxtPath,
});
- if (!isAllowed) {
- meta.logger.info("URL blocked by robots.txt", {
- url: urlToCheck,
- });
- setSpanAttributes(span, {
- "scrape.blocked_by_robots": true,
- });
- throw new CrawlDenialError("URL blocked by robots.txt");
- }
- } catch (error) {
- if (error instanceof CrawlDenialError) {
- throw error;
+ if (!isRobotsTxtPath) {
+ try {
+ let robotsTxt: string | undefined;
+ if (internalOptions.crawlId) {
+ const crawl = await getCrawl(internalOptions.crawlId);
+ robotsTxt = crawl?.robots;
+ }
+
+ if (!robotsTxt) {
+ const { content } = await fetchRobotsTxt(
+ {
+ url: urlToCheck,
+ zeroDataRetention:
+ internalOptions.zeroDataRetention || false,
+ location: options.location,
+ },
+ id,
+ meta.logger,
+ meta.abort.asSignal(),
+ );
+ robotsTxt = content;
+ }
+
+ const checker = createRobotsChecker(urlToCheck, robotsTxt);
+ const isAllowed = isUrlAllowedByRobots(
+ urlToCheck,
+ checker.robots,
+ );
+
+ setSpanAttributes(robotsSpan, {
+ "robots.allowed": isAllowed,
+ });
+
+ if (!isAllowed) {
+ meta.logger.info("URL blocked by robots.txt", {
+ url: urlToCheck,
+ });
+ setSpanAttributes(span, {
+ "scrape.blocked_by_robots": true,
+ });
+ throw new CrawlDenialError("URL blocked by robots.txt");
+ }
+ } catch (error) {
+ if (error instanceof CrawlDenialError) {
+ throw error;
+ }
+ meta.logger.debug(
+ "Failed to fetch robots.txt, allowing scrape",
+ {
+ error,
+ url: urlToCheck,
+ },
+ );
+ setSpanAttributes(robotsSpan, {
+ "robots.fetch_failed": true,
+ });
+ }
}
- meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {
+ },
+ ).catch(error => {
+ if (error instanceof CrawlDenialError) {
+ return {
+ success: false as const,
error,
- url: urlToCheck,
- });
- setSpanAttributes(robotsSpan, {
- "robots.fetch_failed": true,
- });
+ };
}
- }
- }).catch(error => {
- if (error instanceof CrawlDenialError) {
+ throw error;
+ });
+ if (result) {
return {
- success: false as const,
- error,
+ ...result,
+ threatDecisions: meta.threatDecisions,
};
}
- throw error;
- });
- if (result) {
- return {
- ...result,
- threatDecisions: meta.threatDecisions,
- };
}
- }
- // Initialize retry tracker with configured limits
- const retryTracker = new ScrapeRetryTracker(
- {
- maxAttempts: config.SCRAPE_MAX_ATTEMPTS,
- maxFeatureToggles: config.SCRAPE_MAX_FEATURE_TOGGLES,
- maxFeatureRemovals: config.SCRAPE_MAX_FEATURE_REMOVALS,
- maxPdfPrefetches: config.SCRAPE_MAX_PDF_PREFETCHES,
- maxDocumentPrefetches: config.SCRAPE_MAX_DOCUMENT_PREFETCHES,
- },
- meta.logger,
- );
+ // Initialize retry tracker with configured limits
+ const retryTracker = new ScrapeRetryTracker(
+ {
+ maxAttempts: config.SCRAPE_MAX_ATTEMPTS,
+ maxFeatureToggles: config.SCRAPE_MAX_FEATURE_TOGGLES,
+ maxFeatureRemovals: config.SCRAPE_MAX_FEATURE_REMOVALS,
+ maxPdfPrefetches: config.SCRAPE_MAX_PDF_PREFETCHES,
+ maxDocumentPrefetches: config.SCRAPE_MAX_DOCUMENT_PREFETCHES,
+ },
+ meta.logger,
+ );
- try {
- let result: ScrapeUrlResponse;
- while (true) {
- try {
- result = await scrapeURLLoop(meta);
- break;
- } catch (error) {
- if (
- error instanceof AddFeatureError &&
- (meta.internalOptions.forceEngine === undefined ||
- Array.isArray(meta.internalOptions.forceEngine))
- ) {
- retryTracker.record("feature_toggle", error);
- // A file handoff names the one parser that can open the file;
- // the file flag the URL extension implied earlier gives way to it.
- const nextFeatureFlags = applyHandoffFeatureFlags(
- meta.featureFlags,
- error.featureFlags,
- );
- meta.logger.debug(
- "More feature flags requested by scraper: adding " +
- error.featureFlags.join(", "),
- {
- error,
- existingFlags: meta.featureFlags,
- droppedFlags: [...meta.featureFlags].filter(
- flag => !nextFeatureFlags.has(flag),
- ),
- },
- );
- meta.featureFlags = nextFeatureFlags;
- if (error.pdfPrefetch) {
- meta.pdfPrefetch = error.pdfPrefetch;
- } else if (error.pdfPrefetch === null) {
- // Browser round trip ran but delivered no file. Preserve the
- // null sentinel: the antibot branches below still retry (the
- // empty handoff may be transient), but the proxy-failure
- // branches fail fast instead of re-running the browser.
- meta.pdfPrefetch = null;
- }
- if (error.documentPrefetch) {
- meta.documentPrefetch = error.documentPrefetch;
- } else if (error.documentPrefetch === null) {
- meta.documentPrefetch = null;
- }
- if (error.imagePrefetch) {
- meta.imagePrefetch = error.imagePrefetch;
- } else if (error.imagePrefetch === null) {
- meta.imagePrefetch = null;
- }
- } else if (
- error instanceof RemoveFeatureError &&
- (meta.internalOptions.forceEngine === undefined ||
- Array.isArray(meta.internalOptions.forceEngine))
- ) {
- retryTracker.record("feature_removal", error);
- meta.logger.debug(
- "Incorrect feature flags reported by scraper: removing " +
- error.featureFlags.join(","),
- { error, existingFlags: meta.featureFlags },
- );
- meta.featureFlags = new Set(
- [...meta.featureFlags].filter(
- x => !error.featureFlags.includes(x),
- ),
- );
- } else if (
- error instanceof PDFAntibotError &&
- meta.internalOptions.forceEngine === undefined
- ) {
- // null = browser ran but delivered no file (possibly transient) —
- // still worth one more browser round trip, so only a real
- // prefetch object fails here.
- if (meta.pdfPrefetch != null) {
- meta.logger.error(
- "PDF was prefetched and still blocked by antibot, failing",
+ try {
+ let result: ScrapeUrlResponse;
+ while (true) {
+ try {
+ result = await scrapeURLLoop(meta);
+ break;
+ } catch (error) {
+ if (
+ error instanceof AddFeatureError &&
+ (meta.internalOptions.forceEngine === undefined ||
+ Array.isArray(meta.internalOptions.forceEngine))
+ ) {
+ retryTracker.record("feature_toggle", error);
+ // A file handoff names the one parser that can open the file;
+ // the file flag the URL extension implied earlier gives way to it.
+ const nextFeatureFlags = applyHandoffFeatureFlags(
+ meta.featureFlags,
+ error.featureFlags,
);
- throw error;
- } else {
- retryTracker.record("pdf_antibot", error);
meta.logger.debug(
- "PDF was blocked by anti-bot, prefetching with chrome-cdp",
- );
- meta.featureFlags = new Set(
- [...meta.featureFlags].filter(x => x !== "pdf"),
- );
- }
- } else if (
- error instanceof PDFFetchProxyError &&
- meta.internalOptions.forceEngine === undefined
- ) {
- // meta.pdfPrefetch distinguishes "browser never attempted"
- // (undefined — clear the pdf flag so the browser engine fetches
- // the file through fire-engine's proxies) from "browser attempted,
- // came back empty" (null — fail fast: another round trip would
- // only burn the shared antibot+proxy prefetch budget).
- if (meta.pdfPrefetch !== undefined) {
- meta.logger.error(
- "PDF was prefetched and the direct fetch still failed at the proxy, failing",
+ "More feature flags requested by scraper: adding " +
+ error.featureFlags.join(", "),
+ {
+ error,
+ existingFlags: meta.featureFlags,
+ droppedFlags: [...meta.featureFlags].filter(
+ flag => !nextFeatureFlags.has(flag),
+ ),
+ },
);
- throw error;
- } else {
- retryTracker.record("pdf_fetch_proxy", error);
+ meta.featureFlags = nextFeatureFlags;
+ if (error.pdfPrefetch) {
+ meta.pdfPrefetch = error.pdfPrefetch;
+ } else if (error.pdfPrefetch === null) {
+ // Browser round trip ran but delivered no file. Preserve the
+ // null sentinel: the antibot branches below still retry (the
+ // empty handoff may be transient), but the proxy-failure
+ // branches fail fast instead of re-running the browser.
+ meta.pdfPrefetch = null;
+ }
+ if (error.documentPrefetch) {
+ meta.documentPrefetch = error.documentPrefetch;
+ } else if (error.documentPrefetch === null) {
+ meta.documentPrefetch = null;
+ }
+ if (error.imagePrefetch) {
+ meta.imagePrefetch = error.imagePrefetch;
+ } else if (error.imagePrefetch === null) {
+ meta.imagePrefetch = null;
+ }
+ } else if (
+ error instanceof RemoveFeatureError &&
+ (meta.internalOptions.forceEngine === undefined ||
+ Array.isArray(meta.internalOptions.forceEngine))
+ ) {
+ retryTracker.record("feature_removal", error);
meta.logger.debug(
- "PDF direct download failed at the proxy, prefetching with chrome-cdp",
+ "Incorrect feature flags reported by scraper: removing " +
+ error.featureFlags.join(","),
+ { error, existingFlags: meta.featureFlags },
);
meta.featureFlags = new Set(
- [...meta.featureFlags].filter(x => x !== "pdf"),
- );
- }
- } else if (
- error instanceof DocumentAntibotError &&
- meta.internalOptions.forceEngine === undefined
- ) {
- // null = browser ran but delivered no file (possibly transient) —
- // still worth one more browser round trip, so only a real
- // prefetch object fails here.
- if (meta.documentPrefetch != null) {
- meta.logger.error(
- "Document was prefetched and still blocked by antibot, failing",
+ [...meta.featureFlags].filter(
+ x => !error.featureFlags.includes(x),
+ ),
);
- throw error;
+ } else if (
+ error instanceof PDFAntibotError &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ // null = browser ran but delivered no file (possibly transient) —
+ // still worth one more browser round trip, so only a real
+ // prefetch object fails here.
+ if (meta.pdfPrefetch != null) {
+ meta.logger.error(
+ "PDF was prefetched and still blocked by antibot, failing",
+ );
+ throw error;
+ } else {
+ retryTracker.record("pdf_antibot", error);
+ meta.logger.debug(
+ "PDF was blocked by anti-bot, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "pdf"),
+ );
+ }
+ } else if (
+ error instanceof PDFFetchProxyError &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ // meta.pdfPrefetch distinguishes "browser never attempted"
+ // (undefined — clear the pdf flag so the browser engine fetches
+ // the file through fire-engine's proxies) from "browser attempted,
+ // came back empty" (null — fail fast: another round trip would
+ // only burn the shared antibot+proxy prefetch budget).
+ if (meta.pdfPrefetch !== undefined) {
+ meta.logger.error(
+ "PDF was prefetched and the direct fetch still failed at the proxy, failing",
+ );
+ throw error;
+ } else {
+ retryTracker.record("pdf_fetch_proxy", error);
+ meta.logger.debug(
+ "PDF direct download failed at the proxy, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "pdf"),
+ );
+ }
+ } else if (
+ error instanceof DocumentAntibotError &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ // null = browser ran but delivered no file (possibly transient) —
+ // still worth one more browser round trip, so only a real
+ // prefetch object fails here.
+ if (meta.documentPrefetch != null) {
+ meta.logger.error(
+ "Document was prefetched and still blocked by antibot, failing",
+ );
+ throw error;
+ } else {
+ retryTracker.record("document_antibot", error);
+ meta.logger.debug(
+ "Document was blocked by anti-bot, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "document"),
+ );
+ }
+ } else if (
+ error instanceof DocumentFetchProxyError &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ // Same undefined-vs-null distinction as the PDF branch above.
+ if (meta.documentPrefetch !== undefined) {
+ meta.logger.error(
+ "Document was prefetched and the direct fetch still failed at the proxy, failing",
+ );
+ throw error;
+ } else {
+ retryTracker.record("document_fetch_proxy", error);
+ meta.logger.debug(
+ "Document direct download failed at the proxy, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "document"),
+ );
+ }
} else {
- retryTracker.record("document_antibot", error);
- meta.logger.debug(
- "Document was blocked by anti-bot, prefetching with chrome-cdp",
- );
- meta.featureFlags = new Set(
- [...meta.featureFlags].filter(x => x !== "document"),
- );
- }
- } else if (
- error instanceof DocumentFetchProxyError &&
- meta.internalOptions.forceEngine === undefined
- ) {
- // Same undefined-vs-null distinction as the PDF branch above.
- if (meta.documentPrefetch !== undefined) {
- meta.logger.error(
- "Document was prefetched and the direct fetch still failed at the proxy, failing",
- );
throw error;
- } else {
- retryTracker.record("document_fetch_proxy", error);
- meta.logger.debug(
- "Document direct download failed at the proxy, prefetching with chrome-cdp",
- );
- meta.featureFlags = new Set(
- [...meta.featureFlags].filter(x => x !== "document"),
- );
}
- } else {
- throw error;
}
}
- }
- // Threat protection: if the scrape ended up on a different URL than
- // requested (redirect), re-check the destination URL. This closes the
- // "clean URL redirects to a blocked URL" bypass vector — including
- // same-domain redirects onto a flagged path.
- if (threatPolicy && threatPolicy.mode !== "off" && result.success) {
- const initialUrl = meta.rewrittenUrl ?? meta.url;
- const finalUrl = result.document.metadata.url;
- if (
- finalUrl &&
- canonicalizeUrl(finalUrl) !== canonicalizeUrl(initialUrl)
- ) {
- const decision = await checkUrl(finalUrl, threatPolicy, {
- teamId: internalOptions.teamId,
- dedup: threatDedup,
- });
- meta.threatDecisions.push(decision);
- if (!decision.allowed) {
- meta.logger.info(
- "Redirect destination blocked by threat protection policy",
- {
- url: finalUrl,
- domain: decision.domain,
- initialUrl,
- rule: decision.rule,
- },
- );
- setSpanAttributes(span, {
- "scrape.blocked_by_threat_protection": true,
+ // Threat protection: if the scrape ended up on a different URL than
+ // requested (redirect), re-check the destination URL. This closes the
+ // "clean URL redirects to a blocked URL" bypass vector — including
+ // same-domain redirects onto a flagged path.
+ if (threatPolicy && threatPolicy.mode !== "off" && result.success) {
+ const initialUrl = meta.rewrittenUrl ?? meta.url;
+ const finalUrl = result.document.metadata.url;
+ if (
+ finalUrl &&
+ canonicalizeUrl(finalUrl) !== canonicalizeUrl(initialUrl)
+ ) {
+ const decision = await checkUrl(finalUrl, threatPolicy, {
+ teamId: internalOptions.teamId,
+ dedup: threatDedup,
});
- throw new UnsafeDomainBlockedError(finalUrl, decision);
+ meta.threatDecisions.push(decision);
+ if (!decision.allowed) {
+ meta.logger.info(
+ "Redirect destination blocked by threat protection policy",
+ {
+ url: finalUrl,
+ domain: decision.domain,
+ initialUrl,
+ rule: decision.rule,
+ },
+ );
+ setSpanAttributes(span, {
+ "scrape.blocked_by_threat_protection": true,
+ });
+ throw new UnsafeDomainBlockedError(finalUrl, decision);
+ }
}
}
- }
-
- meta.logger.debug("scrapeURL metrics", {
- module: "scrapeURL/metrics",
- timeTaken: Date.now() - startTime,
- maxAgeValid: (meta.options.maxAge ?? 0) > 0,
- shouldUseIndex: shouldUseIndex(meta),
- success: result.success,
- indexHit:
- result.success && result.document.metadata.cacheState === "hit",
- });
- if (useIndex) {
- meta.logger.debug("scrapeURL index metrics", {
- module: "scrapeURL/index-metrics",
+ meta.logger.debug("scrapeURL metrics", {
+ module: "scrapeURL/metrics",
timeTaken: Date.now() - startTime,
- changeTrackingEnabled: !!hasFormatOfType(
- meta.options.formats,
- "changeTracking",
- ),
- summaryEnabled: !!hasFormatOfType(meta.options.formats, "summary"),
- jsonEnabled: !!hasFormatOfType(meta.options.formats, "json"),
- screenshotEnabled: !!hasFormatOfType(
- meta.options.formats,
- "screenshot",
- ),
- imagesEnabled: !!hasFormatOfType(meta.options.formats, "images"),
- brandingEnabled: !!hasFormatOfType(meta.options.formats, "branding"),
- pdfMaxPages: getPDFMaxPages(meta.options.parsers),
- maxAge: meta.options.maxAge,
- headers: meta.options.headers
- ? Object.keys(meta.options.headers).length
- : 0,
- actions: meta.options.actions?.length ?? 0,
- proxy: meta.options.proxy,
+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,
+ shouldUseIndex: shouldUseIndex(meta),
success: result.success,
indexHit:
result.success && result.document.metadata.cacheState === "hit",
});
- }
- setSpanAttributes(span, {
- "scrape.success": true,
- "scrape.duration_ms": Date.now() - startTime,
- "scrape.index_hit":
- result.success && result.document.metadata.cacheState === "hit",
- });
+ if (useIndex) {
+ meta.logger.debug("scrapeURL index metrics", {
+ module: "scrapeURL/index-metrics",
+ timeTaken: Date.now() - startTime,
+ changeTrackingEnabled: !!hasFormatOfType(
+ meta.options.formats,
+ "changeTracking",
+ ),
+ summaryEnabled: !!hasFormatOfType(meta.options.formats, "summary"),
+ jsonEnabled: !!hasFormatOfType(meta.options.formats, "json"),
+ screenshotEnabled: !!hasFormatOfType(
+ meta.options.formats,
+ "screenshot",
+ ),
+ imagesEnabled: !!hasFormatOfType(meta.options.formats, "images"),
+ brandingEnabled: !!hasFormatOfType(
+ meta.options.formats,
+ "branding",
+ ),
+ pdfMaxPages: getPDFMaxPages(meta.options.parsers),
+ maxAge: meta.options.maxAge,
+ headers: meta.options.headers
+ ? Object.keys(meta.options.headers).length
+ : 0,
+ actions: meta.options.actions?.length ?? 0,
+ proxy: meta.options.proxy,
+ success: result.success,
+ indexHit:
+ result.success && result.document.metadata.cacheState === "hit",
+ });
+ }
- return meta.threatDecisions.length > 0
- ? { ...result, threatDecisions: meta.threatDecisions }
- : result;
- } catch (error) {
- // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {
- // throw new FEPageLoadFailed();
- // } else
- // A timed-out large-PDF scrape leaves its fire-pdf job running by
- // design (fire-pdf/async.ts cancel policy); upgrade the timeout
- // error IN PLACE so the caller learns processing continues and
- // when a retry of the same URL picks the result up. Covers both
- // the engine race's own timer and the abort manager's inner
- // timeout, which exits through the early rethrow below.
- const timeoutCandidate =
- error instanceof AbortManagerThrownError ? error.inner : error;
- if (
- meta.largePdfProcessing?.current &&
- timeoutCandidate instanceof ScrapeJobTimeoutError &&
- timeoutCandidate.processing === undefined
- ) {
- const composed = composeTimeoutProcessing({
- ...meta.largePdfProcessing.current,
- nowMs: Date.now(),
+ setSpanAttributes(span, {
+ "scrape.success": true,
+ "scrape.duration_ms": Date.now() - startTime,
+ "scrape.index_hit":
+ result.success && result.document.metadata.cacheState === "hit",
});
- timeoutCandidate.processing = composed.details;
- timeoutCandidate.message = composed.message;
- }
- meta.logger.debug("scrapeURL metrics", {
- module: "scrapeURL/metrics",
- timeTaken: Date.now() - startTime,
- maxAgeValid: (meta.options.maxAge ?? 0) > 0,
- shouldUseIndex: shouldUseIndex(meta),
- success: false,
- indexHit: false,
- });
+ return meta.threatDecisions.length > 0
+ ? { ...result, threatDecisions: meta.threatDecisions }
+ : result;
+ } catch (error) {
+ // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {
+ // throw new FEPageLoadFailed();
+ // } else
+ // A timed-out large-PDF scrape leaves its fire-pdf job running by
+ // design (fire-pdf/async.ts cancel policy); upgrade the timeout
+ // error IN PLACE so the caller learns processing continues and
+ // when a retry of the same URL picks the result up. Covers both
+ // the engine race's own timer and the abort manager's inner
+ // timeout, which exits through the early rethrow below.
+ const timeoutCandidate =
+ error instanceof AbortManagerThrownError ? error.inner : error;
+ if (
+ meta.largePdfProcessing?.current &&
+ timeoutCandidate instanceof ScrapeJobTimeoutError &&
+ timeoutCandidate.processing === undefined
+ ) {
+ const composed = composeTimeoutProcessing({
+ ...meta.largePdfProcessing.current,
+ nowMs: Date.now(),
+ });
+ timeoutCandidate.processing = composed.details;
+ timeoutCandidate.message = composed.message;
+ }
- if (useIndex) {
- meta.logger.debug("scrapeURL index metrics", {
- module: "scrapeURL/index-metrics",
+ meta.logger.debug("scrapeURL metrics", {
+ module: "scrapeURL/metrics",
timeTaken: Date.now() - startTime,
- changeTrackingEnabled: !!hasFormatOfType(
- meta.options.formats,
- "changeTracking",
- ),
- summaryEnabled: !!hasFormatOfType(meta.options.formats, "summary"),
- jsonEnabled: !!hasFormatOfType(meta.options.formats, "json"),
- screenshotEnabled: !!hasFormatOfType(
- meta.options.formats,
- "screenshot",
- ),
- imagesEnabled: !!hasFormatOfType(meta.options.formats, "images"),
- brandingEnabled: !!hasFormatOfType(meta.options.formats, "branding"),
- pdfMaxPages: getPDFMaxPages(meta.options.parsers),
- maxAge: meta.options.maxAge,
- headers: meta.options.headers
- ? Object.keys(meta.options.headers).length
- : 0,
- actions: meta.options.actions?.length ?? 0,
- proxy: meta.options.proxy,
+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,
+ shouldUseIndex: shouldUseIndex(meta),
success: false,
indexHit: false,
});
- }
- // Set error attributes on span
- let errorType = "unknown";
- if (error instanceof NoEnginesLeftError) {
- errorType = "NoEnginesLeftError";
- meta.logger.warn("scrapeURL: All scraping engines failed!", { error });
- } else if (error instanceof LLMRefusalError) {
- errorType = "LLMRefusalError";
- meta.logger.warn("scrapeURL: LLM refused to extract content", {
- error,
- });
- } else if (
- error instanceof Error &&
- error.message.includes("Invalid schema for response_format")
- ) {
- errorType = "LLMSchemaError";
- // TODO: separate into custom error
- meta.logger.warn("scrapeURL: LLM schema error", { error });
- // TODO: results?
- } else if (error instanceof SiteError) {
- errorType = "SiteError";
- meta.logger.warn("scrapeURL: Site failed to load in browser", {
- error,
- });
- } else if (error instanceof SSLError) {
- errorType = "SSLError";
- meta.logger.warn("scrapeURL: SSL error", { error });
- } else if (error instanceof ActionError) {
- errorType = "ActionError";
- meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });
- } else if (error instanceof UnsupportedFileError) {
- errorType = "UnsupportedFileError";
- meta.logger.warn("scrapeURL: Tried to scrape unsupported file", {
- error,
- });
- } else if (error instanceof PDFInsufficientTimeError) {
- errorType = "PDFInsufficientTimeError";
- meta.logger.warn("scrapeURL: Insufficient time to process PDF", {
- error,
- });
- } else if (error instanceof PDFOCRRequiredError) {
- errorType = "PDFOCRRequiredError";
- meta.logger.warn(
- "scrapeURL: PDF requires OCR but fast mode was requested",
- {
+ if (useIndex) {
+ meta.logger.debug("scrapeURL index metrics", {
+ module: "scrapeURL/index-metrics",
+ timeTaken: Date.now() - startTime,
+ changeTrackingEnabled: !!hasFormatOfType(
+ meta.options.formats,
+ "changeTracking",
+ ),
+ summaryEnabled: !!hasFormatOfType(meta.options.formats, "summary"),
+ jsonEnabled: !!hasFormatOfType(meta.options.formats, "json"),
+ screenshotEnabled: !!hasFormatOfType(
+ meta.options.formats,
+ "screenshot",
+ ),
+ imagesEnabled: !!hasFormatOfType(meta.options.formats, "images"),
+ brandingEnabled: !!hasFormatOfType(
+ meta.options.formats,
+ "branding",
+ ),
+ pdfMaxPages: getPDFMaxPages(meta.options.parsers),
+ maxAge: meta.options.maxAge,
+ headers: meta.options.headers
+ ? Object.keys(meta.options.headers).length
+ : 0,
+ actions: meta.options.actions?.length ?? 0,
+ proxy: meta.options.proxy,
+ success: false,
+ indexHit: false,
+ });
+ }
+
+ // Set error attributes on span
+ let errorType = "unknown";
+ if (error instanceof NoEnginesLeftError) {
+ errorType = "NoEnginesLeftError";
+ meta.logger.warn("scrapeURL: All scraping engines failed!", {
error,
- },
- );
- } else if (error instanceof PDFPrefetchFailed) {
- errorType = "PDFPrefetchFailed";
- meta.logger.warn(
- "scrapeURL: Failed to prefetch PDF that is protected by anti-bot",
- { error },
- );
- } else if (error instanceof DocumentPrefetchFailed) {
- errorType = "DocumentPrefetchFailed";
- meta.logger.warn(
- "scrapeURL: Failed to prefetch document that is protected by anti-bot",
- { error },
- );
- } else if (error instanceof PDFFetchProxyError) {
- errorType = "PDFFetchProxyError";
- meta.logger.warn(
- "scrapeURL: PDF download failed at the proxy and could not be recovered via browser prefetch",
- { error },
- );
- } else if (error instanceof DocumentFetchProxyError) {
- errorType = "DocumentFetchProxyError";
- meta.logger.warn(
- "scrapeURL: Document download failed at the proxy and could not be recovered via browser prefetch",
- { error },
- );
- } else if (error instanceof BrandingNotSupportedError) {
- errorType = "BrandingNotSupportedError";
- meta.logger.warn("scrapeURL: Branding not supported for this content", {
- error,
- });
- } else if (error instanceof ProxySelectionError) {
- errorType = "ProxySelectionError";
- meta.logger.warn("scrapeURL: Proxy selection error", { error });
- } else if (error instanceof DNSResolutionError) {
- errorType = "DNSResolutionError";
- meta.logger.warn("scrapeURL: DNS resolution error", { error });
- } else if (error instanceof ScrapeRetryLimitError) {
- errorType = "ScrapeRetryLimitError";
- meta.logger.warn("scrapeURL: Retry limit reached", {
- error,
- retryStats: error.stats,
- });
- } else if (error instanceof UnsafeDomainBlockedError) {
- errorType = "UnsafeDomainBlockedError";
- meta.logger.warn(
- "scrapeURL: Domain blocked by threat protection policy",
- {
+ });
+ } else if (error instanceof LLMRefusalError) {
+ errorType = "LLMRefusalError";
+ meta.logger.warn("scrapeURL: LLM refused to extract content", {
error,
- domain: error.domain,
- rule: error.decision.rule,
- },
- );
- } else if (error instanceof AbortManagerThrownError) {
- errorType = "AbortManagerThrownError";
- throw error.inner;
- } else {
- captureExceptionWithZdrCheck(error, {
- extra: {
- zeroDataRetention: internalOptions.zeroDataRetention ?? false,
- },
- });
- meta.logger.error("scrapeURL: Unexpected error happened", { error });
- // TODO: results?
- }
+ });
+ } else if (
+ error instanceof Error &&
+ error.message.includes("Invalid schema for response_format")
+ ) {
+ errorType = "LLMSchemaError";
+ // TODO: separate into custom error
+ meta.logger.warn("scrapeURL: LLM schema error", { error });
+ // TODO: results?
+ } else if (error instanceof SiteError) {
+ errorType = "SiteError";
+ meta.logger.warn("scrapeURL: Site failed to load in browser", {
+ error,
+ });
+ } else if (error instanceof SSLError) {
+ errorType = "SSLError";
+ meta.logger.warn("scrapeURL: SSL error", { error });
+ } else if (error instanceof ActionError) {
+ errorType = "ActionError";
+ meta.logger.warn("scrapeURL: Action(s) failed to complete", {
+ error,
+ });
+ } else if (error instanceof UnsupportedFileError) {
+ errorType = "UnsupportedFileError";
+ meta.logger.warn("scrapeURL: Tried to scrape unsupported file", {
+ error,
+ });
+ } else if (error instanceof PDFInsufficientTimeError) {
+ errorType = "PDFInsufficientTimeError";
+ meta.logger.warn("scrapeURL: Insufficient time to process PDF", {
+ error,
+ });
+ } else if (error instanceof PDFOCRRequiredError) {
+ errorType = "PDFOCRRequiredError";
+ meta.logger.warn(
+ "scrapeURL: PDF requires OCR but fast mode was requested",
+ {
+ error,
+ },
+ );
+ } else if (error instanceof PDFPrefetchFailed) {
+ errorType = "PDFPrefetchFailed";
+ meta.logger.warn(
+ "scrapeURL: Failed to prefetch PDF that is protected by anti-bot",
+ { error },
+ );
+ } else if (error instanceof DocumentPrefetchFailed) {
+ errorType = "DocumentPrefetchFailed";
+ meta.logger.warn(
+ "scrapeURL: Failed to prefetch document that is protected by anti-bot",
+ { error },
+ );
+ } else if (error instanceof PDFFetchProxyError) {
+ errorType = "PDFFetchProxyError";
+ meta.logger.warn(
+ "scrapeURL: PDF download failed at the proxy and could not be recovered via browser prefetch",
+ { error },
+ );
+ } else if (error instanceof DocumentFetchProxyError) {
+ errorType = "DocumentFetchProxyError";
+ meta.logger.warn(
+ "scrapeURL: Document download failed at the proxy and could not be recovered via browser prefetch",
+ { error },
+ );
+ } else if (error instanceof BrandingNotSupportedError) {
+ errorType = "BrandingNotSupportedError";
+ meta.logger.warn(
+ "scrapeURL: Branding not supported for this content",
+ {
+ error,
+ },
+ );
+ } else if (error instanceof ProxySelectionError) {
+ errorType = "ProxySelectionError";
+ meta.logger.warn("scrapeURL: Proxy selection error", { error });
+ } else if (error instanceof DNSResolutionError) {
+ errorType = "DNSResolutionError";
+ meta.logger.warn("scrapeURL: DNS resolution error", { error });
+ } else if (error instanceof ScrapeRetryLimitError) {
+ errorType = "ScrapeRetryLimitError";
+ meta.logger.warn("scrapeURL: Retry limit reached", {
+ error,
+ retryStats: error.stats,
+ });
+ } else if (error instanceof UnsafeDomainBlockedError) {
+ errorType = "UnsafeDomainBlockedError";
+ meta.logger.warn(
+ "scrapeURL: Domain blocked by threat protection policy",
+ {
+ error,
+ domain: error.domain,
+ rule: error.decision.rule,
+ },
+ );
+ } else if (error instanceof AbortManagerThrownError) {
+ errorType = "AbortManagerThrownError";
+ throw error.inner;
+ } else {
+ meta.logger.error("scrapeURL: Unexpected error happened", { error });
+ recordSpanException(span, error);
+ // TODO: results?
+ }
- setSpanAttributes(span, {
- "scrape.success": false,
- "scrape.error": error instanceof Error ? error.message : String(error),
- "scrape.error_type": errorType,
- "scrape.duration_ms": Date.now() - startTime,
- });
+ setSpanAttributes(span, {
+ "scrape.success": false,
+ "scrape.error":
+ error instanceof Error ? error.message : String(error),
+ "scrape.error_type": errorType,
+ "scrape.duration_ms": Date.now() - startTime,
+ });
- return {
- success: false,
- error,
- ...(meta.threatDecisions.length > 0
- ? { threatDecisions: meta.threatDecisions }
- : {}),
- };
- }
- });
+ return {
+ success: false,
+ error,
+ ...(meta.threatDecisions.length > 0
+ ? { threatDecisions: meta.threatDecisions }
+ : {}),
+ };
+ }
+ },
+ { zeroDataRetention: internalOptions.zeroDataRetention },
+ );
}
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index 59c26f45f..498a9ed1a 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -1,6 +1,5 @@
import { Logger } from "winston";
import { z, ZodError } from "zod";
-import * as Sentry from "@sentry/node";
import { MockState, saveMock } from "./mock";
import { fireEngineURL } from "../engines/fire-engine/scrape";
import { fetch, Response, FormData, Agent } from "undici";
@@ -137,7 +136,6 @@ export async function robustFetch<
if (error instanceof AbortManagerThrownError) {
throw error;
} else if (!ignoreFailure) {
- Sentry.captureException(error);
if (tryCount > 1) {
logger.debug(
"Request failed, trying " + (tryCount - 1) + " more times",
diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
index cdc80fa85..ed7dabc4a 100644
--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
@@ -17,7 +17,6 @@ import {
} from "../../../lib/cost-tracking";
import { calculateCost } from "../transformers/llmExtract";
import { PromptInjectionDetectedError } from "../error";
-import { captureExceptionWithZdrCheck } from "../../../services/sentry";
const GUARD_MODEL = "gpt-4o-mini";
const GUARD_MAX_CHUNK_CHARS = 32000;
@@ -109,6 +108,9 @@ async function classifyChunk(
},
experimental_telemetry: {
isEnabled: !zeroDataRetention,
+ // The input carries the page screenshot / raw page content; too large
+ // for span attributes. Outputs stay recorded.
+ recordInputs: false,
functionId: metadata.functionId
? metadata.functionId + "/promptInjectionGuard"
: "promptInjectionGuard",
@@ -160,13 +162,6 @@ async function classifyChunk(
);
}
- if (!NoObjectGeneratedError.isInstance(error)) {
- captureExceptionWithZdrCheck(error, {
- tags: { feature: "prompt-injection-guard" },
- extra: { teamId: metadata.teamId },
- });
- }
-
// Deliberately fail-open so a guard outage doesn't take down the whole scrape.
logger.warn(
"Prompt injection guard call failed; proceeding without a guard verdict (fail-open)",
diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.ts
index 0529e499e..6070285f5 100644
--- a/apps/api/src/search/fireEngine.ts
+++ b/apps/api/src/search/fireEngine.ts
@@ -1,6 +1,5 @@
import { config } from "../config";
import { SearchResult } from "../../src/lib/entities";
-import * as Sentry from "@sentry/node";
import { logger } from "../lib/logger";
import { executeWithRetry, attemptRequest } from "../lib/retry-utils";
import { useFireEngine } from "../scraper/scrapeURL/engines/fire-engine/available";
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index 88bea6a14..fd27262a9 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -4,7 +4,6 @@ import {
SearchV2Response,
SearchResultType,
} from "../../lib/entities";
-import * as Sentry from "@sentry/node";
import { logger } from "../../lib/logger";
import { executeWithRetry, attemptRequest } from "../../lib/retry-utils";
import { useFireEngine } from "../../scraper/scrapeURL/engines/fire-engine/available";
diff --git a/apps/api/src/services/billing/__tests__/batch_billing.test.ts b/apps/api/src/services/billing/__tests__/batch_billing.test.ts
index d207476f3..2bef47022 100644
--- a/apps/api/src/services/billing/__tests__/batch_billing.test.ts
+++ b/apps/api/src/services/billing/__tests__/batch_billing.test.ts
@@ -4,33 +4,23 @@ import { vi } from "vitest";
// reads at build time must be created in vi.hoisted(). (Jest left jest.mock
// un-hoisted here because `jest` was imported from @jest/globals.) The `redis`
// stub below stays module-level: its factory only captures it lazily.
-const {
- captureException,
- logger,
- withAuth,
- trackCredits,
- refundCredits,
- billTeam7,
-} = vi.hoisted(() => {
- const logger: any = {
- info: vi.fn(),
- warn: vi.fn(),
- error: vi.fn(),
- child: vi.fn(() => logger),
- };
- return {
- captureException: vi.fn(),
- logger,
- withAuth: vi.fn((fn: any) => fn),
- trackCredits: vi.fn<(args: any) => Promise<boolean>>(),
- refundCredits: vi.fn<(args: any) => Promise<void>>(),
- billTeam7: vi.fn<(params: any) => Promise<{ api_key: string }[]>>(),
- };
-});
-
-vi.mock("@sentry/node", () => ({
- captureException,
-}));
+const { logger, withAuth, trackCredits, refundCredits, billTeam7 } = vi.hoisted(
+ () => {
+ const logger: any = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn(() => logger),
+ };
+ return {
+ logger,
+ withAuth: vi.fn((fn: any) => fn),
+ trackCredits: vi.fn<(args: any) => Promise<boolean>>(),
+ refundCredits: vi.fn<(args: any) => Promise<void>>(),
+ billTeam7: vi.fn<(params: any) => Promise<{ api_key: string }[]>>(),
+ };
+ },
+);
vi.mock("../../../lib/logger", () => ({
logger,
@@ -151,7 +141,6 @@ describe("processBillingBatch", () => {
expect(billTeam7).toHaveBeenCalled();
expect(trackCredits).not.toHaveBeenCalled();
- expect(captureException).not.toHaveBeenCalled();
});
it("does not re-track even when the op was already tracked at request time", async () => {
@@ -179,10 +168,9 @@ describe("processBillingBatch", () => {
},
featureId: "CREDITS",
});
- expect(captureException).toHaveBeenCalled();
});
- it("captures exceptions and refunds when billing throws", async () => {
+ it("refunds when billing throws", async () => {
queue = [makeOp({ autumnTrackInRequest: true })];
billTeam7.mockRejectedValueOnce(new Error("rpc exploded"));
@@ -198,7 +186,6 @@ describe("processBillingBatch", () => {
},
featureId: "CREDITS",
});
- expect(captureException).toHaveBeenCalled();
});
it("continues processing later groups when an Autumn refund fails", async () => {
@@ -232,6 +219,5 @@ describe("processBillingBatch", () => {
expect(billTeam7).toHaveBeenCalledTimes(2);
// The batch never tracks usage to Autumn, regardless of the request-time flag.
expect(trackCredits).not.toHaveBeenCalled();
- expect(captureException).toHaveBeenCalled();
});
});
diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.ts
index f09c80175..56bf07bac 100644
--- a/apps/api/src/services/billing/batch_billing.ts
+++ b/apps/api/src/services/billing/batch_billing.ts
@@ -1,7 +1,6 @@
import { logger } from "../../lib/logger";
import { getRedisConnection } from "../queue-service";
import { billTeam7 } from "../../db/rpc";
-import * as Sentry from "@sentry/node";
import { withAuth } from "../../lib/withAuth";
import {
autumnService,
@@ -53,10 +52,7 @@ async function confirmExchangeOutcomes(
// Exchange queues at most EXCHANGE_CONFIRM_CONCURRENCY waiters per
// invocation on the shared budget instead of one per operation.
let nextIndex = 0;
- const workerCount = Math.min(
- EXCHANGE_CONFIRM_CONCURRENCY,
- operations.length,
- );
+ const workerCount = Math.min(EXCHANGE_CONFIRM_CONCURRENCY, operations.length);
await Promise.all(
Array.from({ length: workerCount }, async () => {
while (true) {
@@ -157,13 +153,6 @@ async function refundRequestTrackedCredits(group: GroupedBillingOperation) {
credits: requestTrackedCredits,
billing: group.billing,
});
- Sentry.captureException(error, {
- data: {
- operation: "batch_billing_refund",
- team_id: group.team_id,
- credits: requestTrackedCredits,
- },
- });
}
}
@@ -299,24 +288,12 @@ export async function processBillingBatch() {
error,
group,
});
- Sentry.captureException(error, {
- data: {
- operation: "batch_billing",
- team_id: group.team_id,
- credits: group.total_credits,
- },
- });
}
}
logger.info("✅ Billing batch processing completed successfully");
} catch (error) {
logger.error("Error processing billing batch", { error });
- Sentry.captureException(error, {
- data: {
- operation: "batch_billing_process",
- },
- });
} finally {
await releaseLock();
}
@@ -412,13 +389,6 @@ export async function queueBillingOperation(
return { success: true };
} catch (error) {
logger.error("Error queueing billing operation", { error, team_id });
- Sentry.captureException(error, {
- data: {
- operation: "queue_billing",
- team_id,
- credits,
- },
- });
return { success: false, error };
}
}
@@ -455,7 +425,6 @@ async function supaBillTeam(
is_extract,
});
} catch (error) {
- Sentry.captureException(error);
_logger.error("Failed to bill team.", { error });
return { success: false, error };
}
diff --git a/apps/api/src/services/cclog-worker.ts b/apps/api/src/services/cclog-worker.ts
index 1637dfaf7..84cf10e9c 100644
--- a/apps/api/src/services/cclog-worker.ts
+++ b/apps/api/src/services/cclog-worker.ts
@@ -1,7 +1,6 @@
import "dotenv/config";
import { config } from "../config";
-import "./sentry";
-import { setSentryServiceTag } from "./sentry";
+import { shutdownTracing } from "../otel";
import Express from "express";
import { logger as _logger } from "../lib/logger";
import {
@@ -34,8 +33,6 @@ async function sendHeartbeat() {
}
(async () => {
- setSentryServiceTag("cclog-worker");
-
let isShuttingDown = false;
let tickInFlight = false;
@@ -68,7 +65,8 @@ async function sendHeartbeat() {
await sleep(1000);
}
- server.close(() => {
+ server.close(async () => {
+ await shutdownTracing();
_logger.info("cclog worker shut down");
process.exit(0);
});
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index 15e07efb4..043652bae 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -1,8 +1,6 @@
import "dotenv/config";
import { config } from "../config";
-import "./sentry";
-import { setSentryServiceTag } from "./sentry";
-import * as Sentry from "@sentry/node";
+import { shutdownTracing } from "../otel";
import { logger as _logger } from "../lib/logger";
import { configDotenv } from "dotenv";
import { ExtractResult } from "../lib/extract/types";
@@ -11,7 +9,6 @@ import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f
import { createWebhookSender, WebhookEvent } from "./webhook";
import Express from "express";
import { getErrorContactMessage } from "../lib/deployment";
-import { TransportableError } from "../lib/error";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";
import {
@@ -104,15 +101,6 @@ const processExtractJob = async (
error,
});
- // Filter out TransportableErrors (flow control)
- if (!(error instanceof TransportableError)) {
- Sentry.captureException(error, {
- data: {
- extractId: data.extractId,
- },
- });
- }
-
await updateExtract(data.extractId, {
status: "failed",
error:
@@ -216,6 +204,7 @@ async function shutdown() {
_logger.info("Shutting down extract worker...");
await shutdownExtractQueue();
await shutdownPubSubLogging();
+ await shutdownTracing();
_logger.info("Extract worker shut down");
process.exit(0);
}
@@ -233,8 +222,6 @@ if (require.main === module) {
}
(async () => {
- setSentryServiceTag("extract-worker");
-
await initializeBlocklist().catch(e => {
_logger.error("Failed to initialize blocklist", { error: e });
process.exit(1);
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 2de767688..22258ecae 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -1,8 +1,6 @@
import "dotenv/config";
import { config } from "../../config";
-import "../sentry";
-import { setSentryServiceTag } from "../sentry";
-import * as Sentry from "@sentry/node";
+import { shutdownTracing } from "../../otel";
import { Job, Queue, Worker } from "bullmq";
import { logger as _logger, logger } from "../../lib/logger";
import {
@@ -125,7 +123,6 @@ const processBillingJobInternal = async (token: string, job: Job) => {
await job.moveToCompleted({ success: true }, token, false);
} catch (error) {
logger.error("Error processing billing job", { error });
- Sentry.captureException(error);
err = error;
await job.moveToFailed(error, token, false);
} finally {
@@ -555,7 +552,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {
submittedCrawls++;
} catch (e) {
logger.error("Error adding precrawl job to queue", { error: e });
- Sentry.captureException(e);
}
}
@@ -580,7 +576,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {
});
} catch (e) {
logger.error("Error processing precrawl job", { error: e });
- Sentry.captureException(e);
await job.moveToFailed(e, token, false);
} finally {
clearInterval(extendLockInterval);
@@ -687,6 +682,7 @@ const workerFun = async (
}
logger.info("All jobs finished. Worker exiting!");
await shutdownPubSubLogging();
+ await shutdownTracing();
process.exit(0);
};
@@ -699,8 +695,6 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;
// Start the workers
(async () => {
- setSentryServiceTag("index-worker");
-
// Start billing worker and batch processing
startBillingBatchProcessing();
const billingWorkerPromise = workerFun(
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index d4f39b59b..ce6eda1f8 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -4,7 +4,6 @@ import { vi } from "vitest";
// vi.hoisted() (also hoisted). Under Jest these worked because importing `jest`
// from @jest/globals disables jest.mock hoisting.
const {
- captureException,
logger,
values,
insert,
@@ -38,7 +37,6 @@ const {
};
});
return {
- captureException: vi.fn(),
logger,
values,
insert,
@@ -58,10 +56,6 @@ vi.mock("@google-cloud/pubsub", () => ({
},
}));
-vi.mock("@sentry/node", () => ({
- captureException,
-}));
-
vi.mock("../../config", () => ({
config: {
GCS_BUCKET_NAME: undefined,
@@ -177,28 +171,6 @@ describe("logSearch", () => {
expect(search.options.query).toBe("nested\u0000query");
});
- it("uses sanitized data in Sentry insert failure context", async () => {
- values.mockRejectedValueOnce(
- Object.assign(new Error("unsupported Unicode escape sequence"), {
- code: "22P05",
- }),
- );
-
- await logSearch(
- makeSearch({
- query: "bad\u0000query",
- options: { query: "bad\u0000query" },
- }),
- );
-
- expect(captureException).toHaveBeenCalled();
- const context = captureException.mock.calls[0][1] as {
- extra: { data: string };
- };
- expect(context.extra.data).not.toContain("\\u0000");
- expect(context.extra.data).toContain("badquery");
- });
-
it("reports serialization failures without losing the PostgreSQL attempt", async () => {
const search = makeSearch({ options: { unsupported: 1n } });
@@ -286,7 +258,6 @@ describe("logRequest", () => {
"Failed to publish log to Pub/Sub",
expect.objectContaining({ error: expect.any(Error) }),
);
- expect(captureException).toHaveBeenCalled();
});
it("does not hold the caller on a slow publish", async () => {
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 7e3ada468..9ed445b8a 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -6,7 +6,6 @@ import "dotenv/config";
import { logger as _logger } from "../../lib/logger";
import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";
import { configDotenv } from "dotenv";
-import * as Sentry from "@sentry/node";
import type { PgTable } from "drizzle-orm/pg-core";
import {
saveDeepResearchToGCS,
@@ -122,9 +121,6 @@ function getPubSubClient(logger: Logger): PubSub | null {
} catch (error) {
pubSubClient = null;
logger.error("Failed to initialize Pub/Sub log publisher", { error });
- Sentry.captureException(error, {
- tags: { operation: "initializePubSubLogPublisher" },
- });
return null;
}
}
@@ -197,10 +193,6 @@ async function publishLog(table: string, data: any, logger: Logger) {
logId: data.id,
durationMs: Date.now() - startedAt,
});
- Sentry.captureException(error, {
- tags: { table, operation: "publishPubSubLog" },
- extra: { logId: data.id },
- });
}
}
@@ -259,10 +251,6 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {
if (errors.length > 0) {
logger.error("Failed to drain Pub/Sub log publisher", { errors });
- Sentry.captureException(errors[0], {
- tags: { operation: "flushPubSubLogPublisher" },
- extra: { failures: errors.length },
- });
} else {
logger.info("Pub/Sub log publisher drained", {
durationMs: Date.now() - startedAt,
@@ -283,9 +271,6 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {
]);
} catch (error) {
logger.error("Failed to close Pub/Sub log publisher", { error });
- Sentry.captureException(error, {
- tags: { operation: "closePubSubLogPublisher" },
- });
} finally {
clearTimeout(closeDeadline);
}
@@ -355,25 +340,6 @@ async function robustInsert(
});
} else {
logger.error("Failed to insert into database", { attempts });
- // Report to Sentry with context
- Sentry.captureException(
- attempts[attempts.length - 1]?.error ||
- new Error("Database insert failed after 10 attempts"),
- {
- tags: {
- table,
- operation: "robustInsert",
- },
- extra: {
- table,
- data: JSON.stringify(data).substring(0, 500), // Limit size
- attempts: 10,
- lastError: attempts[attempts.length - 1]?.error
- ? JSON.stringify(attempts[attempts.length - 1].error)
- : null,
- },
- },
- );
}
} else {
const start = Date.now();
@@ -384,18 +350,6 @@ async function robustInsert(
} catch (error) {
attempts.push({ error, timeMs: Date.now() - start, backoffMs: 0 });
logger.error("Failed to insert into database", { attempts });
- // Report to Sentry
- Sentry.captureException(error, {
- tags: {
- table,
- operation: "robustInsert",
- force: "false",
- },
- extra: {
- table,
- data: JSON.stringify(data).substring(0, 500), // Limit size
- },
- });
}
}
}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index ffe3085c0..3f98da518 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -1,8 +1,6 @@
import "dotenv/config";
import { config } from "../config";
-import "./sentry";
-import { setSentryServiceTag } from "./sentry";
-import * as Sentry from "@sentry/node";
+import { shutdownTracing } from "../otel";
import {
getDeepResearchQueue,
getGenerateLlmsTxtQueue,
@@ -24,7 +22,6 @@ import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forc
import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
import { finishCrawlSuper } from "./worker/crawl-logic";
import { getCrawl } from "../lib/crawl-redis";
-import { TransportableError } from "../lib/error";
import {
processMonitorCheckJob,
reconcileRunningMonitorChecks,
@@ -102,15 +99,6 @@ const processDeepResearchJobInternal = async (
} catch (error) {
logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });
- // Skip TransportableErrors: they're flow control, not failures.
- if (!(error instanceof TransportableError)) {
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
- }
-
try {
await job.moveToFailed(error, token, false);
} catch (e) {
@@ -178,15 +166,6 @@ const processGenerateLlmsTxtJobInternal = async (
} catch (error) {
logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });
- // Skip TransportableErrors: they're flow control, not failures.
- if (!(error instanceof TransportableError)) {
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
- }
-
try {
await job.moveToFailed(error, token, false);
} catch (e) {
@@ -456,8 +435,6 @@ app.listen(workerPort, (error?: Error) => {
});
(async () => {
- setSentryServiceTag("queue-worker");
-
await initializeBlocklist().catch(e => {
_logger.error("Failed to initialize blocklist", { error: e });
process.exit(1);
@@ -514,5 +491,6 @@ app.listen(workerPort, (error?: Error) => {
_logger.info("All jobs finished. Shutting down...");
await shutdownPubSubLogging();
+ await shutdownTracing();
process.exit(0);
})();
diff --git a/apps/api/src/services/sentry.test.ts b/apps/api/src/services/sentry.test.ts
deleted file mode 100644
index f23917b5b..000000000
--- a/apps/api/src/services/sentry.test.ts
+++ /dev/null
@@ -1,133 +0,0 @@
-import { vi } from "vitest";
-
-// vi.mock is hoisted to the top of the file, so anything its factories reference
-// must be created in vi.hoisted() (which also hoists). Under Jest these worked
-// because importing `jest` from @jest/globals disables jest.mock hoisting.
-const { captureException, init, setTag, scopeSetExtra, scopeSetTag } = vi.hoisted(
- () => ({
- captureException: vi.fn(),
- init: vi.fn(),
- setTag: vi.fn(),
- scopeSetExtra: vi.fn(),
- scopeSetTag: vi.fn(),
- }),
-);
-
-vi.mock("@sentry/node", () => ({
- captureException,
- getCurrentScope: () => ({
- setExtra: scopeSetExtra,
- setTag: scopeSetTag,
- }),
- init,
- setTag,
- vercelAIIntegration: vi.fn(() => ({})),
-}));
-
-vi.mock("../config", () => ({
- config: {
- NUQ_POD_NAME: "test-pod",
- SENTRY_DSN: undefined,
- SENTRY_ENVIRONMENT: "test",
- SENTRY_ERROR_SAMPLE_RATE: 1,
- SENTRY_TRACE_SAMPLE_RATE: 0,
- },
-}));
-
-vi.mock("../lib/logger", () => ({
- logger: {
- info: vi.fn(),
- },
-}));
-
-vi.mock("../scraper/scrapeURL/error", () => ({
- AddFeatureError: class AddFeatureError extends Error {},
- RemoveFeatureError: class RemoveFeatureError extends Error {},
- EngineError: class EngineError extends Error {},
-}));
-
-vi.mock("../scraper/scrapeURL/lib/abortManager", () => ({
- AbortManagerThrownError: class AbortManagerThrownError extends Error {},
-}));
-
-vi.mock("../lib/error", () => ({
- JobCancelledError: class JobCancelledError extends Error {},
-}));
-
-import { QueueFullError } from "../lib/queue-full-error";
-import {
- captureExceptionWithZdrCheck,
- shouldIgnoreSentryException,
-} from "./sentry";
-
-describe("sentry filtering", () => {
- beforeEach(() => {
- vi.clearAllMocks();
- });
-
- it("does not capture queue full errors", () => {
- captureExceptionWithZdrCheck(new QueueFullError(50000, 50000));
-
- expect(captureException).not.toHaveBeenCalled();
- });
-
- it("recognizes Sentry exception payloads for queue full errors", () => {
- expect(
- shouldIgnoreSentryException({
- type: "QueueFullError",
- value:
- "Queue limit reached: your team has 50000 jobs queued (limit: 50000).",
- }),
- ).toBe(true);
- });
-
- it("recognizes serialized transportable errors in Sentry exception values", () => {
- expect(
- shouldIgnoreSentryException({
- type: "Error",
- value: 'SCRAPE_TIMEOUT|{"message":"timeout"}',
- }),
- ).toBe(true);
- });
-
- it("recognizes actions-not-supported Sentry exception values", () => {
- expect(
- shouldIgnoreSentryException({
- type: "Error",
- value:
- 'SCRAPE_ACTIONS_NOT_SUPPORTED|{"message":"Actions are not supported by any available engines. Actions require Fire Engine (fire-engine) to be enabled."}',
- }),
- ).toBe(true);
- });
-
- it("recognizes cancellation errors in Sentry exception values", () => {
- expect(
- shouldIgnoreSentryException({
- type: "Error",
- value: "Parent crawl/batch scrape was cancelled",
- }),
- ).toBe(true);
- });
-
- it("does not ignore unexpected Sentry exception values", () => {
- expect(
- shouldIgnoreSentryException({
- type: "Error",
- value: "boom",
- }),
- ).toBe(false);
- });
-
- it("still captures unexpected errors", () => {
- const error = new Error("boom");
-
- captureExceptionWithZdrCheck(error, {
- tags: { module: "test" },
- zeroDataRetention: false,
- });
-
- expect(captureException).toHaveBeenCalledWith(error, {
- tags: { module: "test" },
- });
- });
-});
diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts
deleted file mode 100644
index f02ae61b9..000000000
--- a/apps/api/src/services/sentry.ts
+++ /dev/null
@@ -1,214 +0,0 @@
-import * as Sentry from "@sentry/node";
-import { logger } from "../lib/logger";
-import { config } from "../config";
-import {
- AddFeatureError,
- RemoveFeatureError,
- EngineError,
-} from "../scraper/scrapeURL/error";
-import { AbortManagerThrownError } from "../scraper/scrapeURL/lib/abortManager";
-import { JobCancelledError } from "../lib/error";
-import { isQueueFullError } from "../lib/queue-full-error";
-
-type CaptureContext = {
- tags?: Record<string, string>;
- extra?: Record<string, any>;
- level?: Sentry.SeverityLevel;
- fingerprint?: string[];
- contexts?: Record<string, any>;
- user?: {
- id?: string;
- username?: string;
- email?: string;
- ip_address?: string;
- [key: string]: any;
- };
- [key: string]: any;
-};
-
-const transportableErrorCodes = [
- "SCRAPE_ALL_ENGINES_FAILED",
- "SCRAPE_DNS_RESOLUTION_ERROR",
- "SCRAPE_SITE_ERROR",
- "SCRAPE_SSL_ERROR",
- "SCRAPE_PROXY_SELECTION_ERROR",
- "SCRAPE_ZDR_VIOLATION_ERROR",
- "SCRAPE_UNSUPPORTED_FILE_ERROR",
- "SCRAPE_PDF_ANTIBOT_ERROR",
- "SCRAPE_PDF_FETCH_PROXY_ERROR",
- "SCRAPE_ACTION_ERROR",
- "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR",
- "SCRAPE_PDF_PREFETCH_FAILED",
- "SCRAPE_DOCUMENT_ANTIBOT_ERROR",
- "SCRAPE_DOCUMENT_FETCH_PROXY_ERROR",
- "SCRAPE_DOCUMENT_PREFETCH_FAILED",
- "SCRAPE_ACTIONS_NOT_SUPPORTED",
- "SCRAPE_TIMEOUT",
- "MAP_TIMEOUT",
- "SCRAPE_UNKNOWN_ERROR",
- "SCRAPE_RACED_REDIRECT_ERROR",
- "SCRAPE_SITEMAP_ERROR",
- "CRAWL_DENIAL",
-];
-
-function getStringField(
- error: Record<string, unknown>,
- fields: string[],
-): string {
- for (const field of fields) {
- if (error[field] !== undefined && error[field] !== null) {
- return String(error[field]);
- }
- }
-
- return "";
-}
-
-function getErrorMessage(error: unknown): string {
- if (error instanceof Error) {
- return String(error.message || "");
- }
-
- if (typeof error === "string") {
- return error;
- }
-
- if (error && typeof error === "object") {
- return getStringField(error as Record<string, unknown>, [
- "message",
- "value",
- ]);
- }
-
- return "";
-}
-
-export function shouldIgnoreSentryException(error: unknown): boolean {
- if (isQueueFullError(error)) {
- return true;
- }
-
- if (!error || typeof error !== "object") {
- return false;
- }
-
- if (
- error instanceof AddFeatureError ||
- error instanceof RemoveFeatureError ||
- error instanceof AbortManagerThrownError ||
- error instanceof EngineError ||
- error instanceof JobCancelledError
- ) {
- return true;
- }
-
- // We sometimes rethrow TransportableErrors across process boundaries as a
- // plain Error with message like `${code}|${json}` (see error-serde.ts).
- // In that case, `error.code` is missing, so extract it from the message.
- const errorCodeFromField = "code" in error ? String(error.code) : "";
- const errorMessage = getErrorMessage(error);
-
- // Ignore cancellation errors (fallback for serialized errors)
- if (
- errorMessage === "Parent crawl/batch scrape was cancelled" ||
- errorMessage.includes("Parent crawl/batch scrape was cancelled")
- ) {
- return true;
- }
-
- const errorCodeFromMessage =
- errorCodeFromField ||
- (errorMessage.includes("|") ? errorMessage.split("|", 1)[0] : "");
- const errorCode = errorCodeFromMessage;
-
- return transportableErrorCodes.includes(errorCode);
-}
-
-if (config.SENTRY_DSN) {
- logger.info("Setting up Sentry...");
-
- Sentry.init({
- dsn: config.SENTRY_DSN,
- integrations: integrations => [
- ...integrations,
- Sentry.vercelAIIntegration({
- recordInputs: false,
- recordOutputs: false,
- }),
- ],
- tracesSampler: samplingContext => {
- // trace all AI spans, sample 1% of all others
- return samplingContext.name?.startsWith("ai.")
- ? 1.0
- : config.SENTRY_TRACE_SAMPLE_RATE;
- },
- sampleRate: config.SENTRY_ERROR_SAMPLE_RATE,
- serverName: config.NUQ_POD_NAME,
- environment: config.SENTRY_ENVIRONMENT,
- beforeSend(event, hint) {
- const zeroDataRetention =
- event.tags?.zeroDataRetention === "true" ||
- event.tags?.zero_data_retention === "true" ||
- event.extra?.zeroDataRetention === true ||
- event.extra?.zero_data_retention === true;
-
- if (zeroDataRetention) {
- return null;
- }
-
- const error = hint?.originalException;
- const sentryException = event.exception?.values?.[0];
-
- if (
- shouldIgnoreSentryException(error) ||
- shouldIgnoreSentryException(sentryException)
- ) {
- return null;
- }
-
- return event;
- },
- });
-}
-
-export function captureExceptionWithZdrCheck(
- error: any,
- context?: (CaptureContext & { zeroDataRetention?: boolean }) | null,
-) {
- const zeroDataRetention =
- context?.zeroDataRetention ??
- (context?.extra as any)?.zeroDataRetention ??
- (context?.data as any)?.zeroDataRetention;
-
- if (zeroDataRetention) {
- return;
- }
-
- if (shouldIgnoreSentryException(error)) {
- return;
- }
-
- const { zeroDataRetention: _zdr, ...sentryContext } = context || {};
-
- return Sentry.captureException(error, sentryContext);
-}
-
-export function applyZdrScope(zeroDataRetention?: boolean) {
- if (!zeroDataRetention) {
- return;
- }
-
- const scope = Sentry.getCurrentScope();
- scope.setTag("zeroDataRetention", "true");
- scope.setExtra("zeroDataRetention", true);
-}
-
-/**
- * Set the service type tag for this Sentry instance
- * This helps distinguish between API server and worker errors in Sentry
- */
-export function setSentryServiceTag(serviceType: string) {
- if (config.SENTRY_DSN) {
- Sentry.setTag("service_type", serviceType);
- }
-}
diff --git a/apps/api/src/services/worker/nuq-fdb-worker.ts b/apps/api/src/services/worker/nuq-fdb-worker.ts
index f51104bc2..b971018c9 100644
--- a/apps/api/src/services/worker/nuq-fdb-worker.ts
+++ b/apps/api/src/services/worker/nuq-fdb-worker.ts
@@ -1,6 +1,5 @@
import "dotenv/config";
-import "../sentry";
-import { setSentryServiceTag } from "../sentry";
+import "../../otel";
import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
import { getCrawl } from "../../lib/crawl-redis";
@@ -133,8 +132,6 @@ function startCrawlFinishedLoop() {
}
(async () => {
- setSentryServiceTag("nuq-fdb-worker");
-
let crawlFinishedLoop: ReturnType<typeof startCrawlFinishedLoop> | null =
null;
diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.ts
index 3456e88f3..5b4d8b69a 100644
--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts
+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts
@@ -1,7 +1,6 @@
import "dotenv/config";
import { config } from "../../config";
-import "../sentry";
-import { setSentryServiceTag } from "../sentry";
+import { shutdownTracing } from "../../otel";
import {
scrapeQueue,
nuqGetLocalMetrics,
@@ -13,8 +12,6 @@ import Express from "express";
import { logger } from "../../lib/logger";
(async () => {
- setSentryServiceTag("nuq-prefetch-worker");
-
const app = Express();
app.get("/metrics", (_, res) =>
@@ -46,6 +43,7 @@ import { logger } from "../../lib/logger";
async function shutdown() {
server.close();
await nuqShutdown();
+ await shutdownTracing();
process.exit(0);
}
diff --git a/apps/api/src/services/worker/nuq-reconciler-worker.ts b/apps/api/src/services/worker/nuq-reconciler-worker.ts
index 865da6549..16e6f9322 100644
--- a/apps/api/src/services/worker/nuq-reconciler-worker.ts
+++ b/apps/api/src/services/worker/nuq-reconciler-worker.ts
@@ -1,7 +1,6 @@
import "dotenv/config";
import { config } from "../../config";
-import "../sentry";
-import { setSentryServiceTag } from "../sentry";
+import { shutdownTracing } from "../../otel";
import { logger as _logger } from "../../lib/logger";
import { reconcileConcurrencyQueue } from "../../lib/concurrency-queue-reconciler";
import { repairCrawlJobDoneMarkers } from "../../lib/crawl-redis";
@@ -26,8 +25,6 @@ const reconcilerJobsRecoveredTotal = new Counter({
});
(async () => {
- setSentryServiceTag("nuq-reconciler-worker");
-
let isShuttingDown = false;
let reconcilerInFlight = false;
@@ -72,7 +69,8 @@ const reconcilerJobsRecoveredTotal = new Counter({
await new Promise(resolve => setTimeout(resolve, 1000));
}
- server.close(() => {
+ server.close(async () => {
+ await shutdownTracing();
_logger.info("NuQ reconciler worker shut down");
process.exit(0);
});
diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.ts
index 3af1a4145..259a332e1 100644
--- a/apps/api/src/services/worker/nuq-worker-runner.ts
+++ b/apps/api/src/services/worker/nuq-worker-runner.ts
@@ -1,4 +1,6 @@
import { config } from "../../config";
+import { withZeroDataRetention } from "../../lib/otel-tracer";
+import { shutdownTracing } from "../../otel";
import { logger as _logger } from "../../lib/logger";
import { jobDurationSeconds } from "../../lib/job-metrics";
import { processJobInternal } from "./scrape-worker";
@@ -175,35 +177,43 @@ export async function runNuqWorker(options: {
clearInterval(lockRenewInterval);
- if (processResult.ok) {
- endJobTimer({ status: "success" });
- if (
- !(await options.queue.jobFinish(
- job.id,
- job.lock!,
- processResult.data,
- logger,
- ))
- ) {
- logger.warn("Could not update job status");
- }
- } else {
- endJobTimer({ status: "failed" });
- if (
- !(await options.queue.jobFail(
- job.id,
- job.lock!,
- processResult.error instanceof Error
- ? processResult.error.message
- : typeof processResult.error === "string"
- ? processResult.error
- : JSON.stringify(processResult.error),
- logger,
- ))
- ) {
- logger.warn("Could not update job status");
+ // The job span has ended by now; keep the queue bookkeeping (which records
+ // the failure reason) unrecorded for zero-data-retention jobs too.
+ const zeroDataRetention =
+ (job.data as { zeroDataRetention?: boolean } | undefined)
+ ?.zeroDataRetention === true;
+
+ await withZeroDataRetention(zeroDataRetention, async () => {
+ if (processResult.ok) {
+ endJobTimer({ status: "success" });
+ if (
+ !(await options.queue.jobFinish(
+ job.id,
+ job.lock!,
+ processResult.data,
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ } else {
+ endJobTimer({ status: "failed" });
+ if (
+ !(await options.queue.jobFail(
+ job.id,
+ job.lock!,
+ processResult.error instanceof Error
+ ? processResult.error.message
+ : typeof processResult.error === "string"
+ ? processResult.error
+ : JSON.stringify(processResult.error),
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
}
- }
+ });
}
_logger.info("NuQ worker shutting down", {
@@ -215,6 +225,7 @@ export async function runNuqWorker(options: {
await options.beforeShutdown?.();
await options.shutdown?.();
await shutdownPubSubLogging();
+ await shutdownTracing();
_logger.info("NuQ worker shut down", { module: options.serviceName });
process.exit(0);
});
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
index dab1132f8..c4e23b7de 100644
--- a/apps/api/src/services/worker/nuq-worker.ts
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -1,12 +1,9 @@
import "dotenv/config";
-import "../sentry";
-import { setSentryServiceTag } from "../sentry";
+import "../../otel";
import { nuqGetLocalMetrics, nuqHealthCheck, scrapeQueue } from "./nuq";
import { runNuqWorker } from "./nuq-worker-runner";
(async () => {
- setSentryServiceTag("nuq-worker");
-
await runNuqWorker({
serviceName: "nuq-worker",
queue: scrapeQueue,
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index fb5b1ca64..c65d1785c 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -810,60 +810,64 @@ class NuQ<JobData = any, JobReturnValue = any> {
data: JobData,
options: NuQJobOptions,
): Promise<NuQJob<JobData, JobReturnValue>> {
- return withSpan("nuq.addJob", async span => {
- setSpanAttributes(span, {
- "nuq.queue_name": this.queueName,
- "nuq.job_id": id,
- "nuq.priority": options.priority ?? 0,
- "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
- "nuq.listenable": options.listenable ?? false,
- });
-
- const start = Date.now();
- try {
- const result = this.rowToJob(
- (
- await nuqPool.query(
- `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,
- [
- id,
- data,
- options.priority ?? 0,
- options.listenable ? this.listenChannelId : null,
- normalizeOwnerId(options.ownerId),
- options.groupId ?? null,
- ...(options.backlogged
- ? [
- options.backloggedTimesOutAt
- ? options.backloggedTimesOutAt.toISOString()
- : null,
- ]
- : []),
- ],
- )
- ).rows[0],
- options.backlogged,
- )!;
-
+ return withSpan(
+ "nuq.addJob",
+ async span => {
setSpanAttributes(span, {
- "nuq.job_created": true,
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
+ "nuq.priority": options.priority ?? 0,
+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
+ "nuq.listenable": options.listenable ?? false,
});
- return result;
- } finally {
- const duration = Date.now() - start;
- setSpanAttributes(span, {
- "nuq.duration_ms": duration,
- });
- logger.info("nuqAddJob metrics", {
- module: "nuq/metrics",
- method: "nuqAddJob",
- duration,
- scrapeId: id,
- zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
- });
- }
- });
+ const start = Date.now();
+ try {
+ const result = this.rowToJob(
+ (
+ await nuqPool.query(
+ `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,
+ [
+ id,
+ data,
+ options.priority ?? 0,
+ options.listenable ? this.listenChannelId : null,
+ normalizeOwnerId(options.ownerId),
+ options.groupId ?? null,
+ ...(options.backlogged
+ ? [
+ options.backloggedTimesOutAt
+ ? options.backloggedTimesOutAt.toISOString()
+ : null,
+ ]
+ : []),
+ ],
+ )
+ ).rows[0],
+ options.backlogged,
+ )!;
+
+ setSpanAttributes(span, {
+ "nuq.job_created": true,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ logger.info("nuqAddJob metrics", {
+ module: "nuq/metrics",
+ method: "nuqAddJob",
+ duration,
+ scrapeId: id,
+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
+ });
+ }
+ },
+ { zeroDataRetention: (data as any)?.zeroDataRetention === true },
+ );
}
public async addJobIfNotExists(
@@ -871,60 +875,64 @@ class NuQ<JobData = any, JobReturnValue = any> {
data: JobData,
options: NuQJobOptions,
): Promise<NuQJob<JobData, JobReturnValue> | null> {
- return withSpan("nuq.addJob", async span => {
- setSpanAttributes(span, {
- "nuq.queue_name": this.queueName,
- "nuq.job_id": id,
- "nuq.priority": options.priority ?? 0,
- "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
- "nuq.listenable": options.listenable ?? false,
- });
-
- const start = Date.now();
- try {
- const result = this.rowToJob(
- (
- await nuqPool.query(
- `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) ON CONFLICT (id) DO NOTHING RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,
- [
- id,
- data,
- options.priority ?? 0,
- options.listenable ? this.listenChannelId : null,
- normalizeOwnerId(options.ownerId),
- options.groupId ?? null,
- ...(options.backlogged
- ? [
- options.backloggedTimesOutAt
- ? options.backloggedTimesOutAt.toISOString()
- : null,
- ]
- : []),
- ],
- )
- ).rows[0],
- options.backlogged,
- );
-
+ return withSpan(
+ "nuq.addJob",
+ async span => {
setSpanAttributes(span, {
- "nuq.job_created": result !== null,
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
+ "nuq.priority": options.priority ?? 0,
+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
+ "nuq.listenable": options.listenable ?? false,
});
- return result;
- } finally {
- const duration = Date.now() - start;
- setSpanAttributes(span, {
- "nuq.duration_ms": duration,
- });
- logger.info("nuqAddJob metrics", {
- module: "nuq/metrics",
- method: "nuqAddJob",
- duration,
- scrapeId: id,
- zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
- });
- }
- });
+ const start = Date.now();
+ try {
+ const result = this.rowToJob(
+ (
+ await nuqPool.query(
+ `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) ON CONFLICT (id) DO NOTHING RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,
+ [
+ id,
+ data,
+ options.priority ?? 0,
+ options.listenable ? this.listenChannelId : null,
+ normalizeOwnerId(options.ownerId),
+ options.groupId ?? null,
+ ...(options.backlogged
+ ? [
+ options.backloggedTimesOutAt
+ ? options.backloggedTimesOutAt.toISOString()
+ : null,
+ ]
+ : []),
+ ],
+ )
+ ).rows[0],
+ options.backlogged,
+ );
+
+ setSpanAttributes(span, {
+ "nuq.job_created": result !== null,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ logger.info("nuqAddJob metrics", {
+ module: "nuq/metrics",
+ method: "nuqAddJob",
+ duration,
+ scrapeId: id,
+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
+ });
+ }
+ },
+ { zeroDataRetention: (data as any)?.zeroDataRetention === true },
+ );
}
public async addJobs(
@@ -934,142 +942,150 @@ class NuQ<JobData = any, JobReturnValue = any> {
options: NuQJobOptions;
}>,
): Promise<NuQJob<JobData, JobReturnValue>[]> {
- return withSpan("nuq.addJobs", async span => {
- setSpanAttributes(span, {
- "nuq.queue_name": this.queueName,
- "nuq.jobs_count": jobs.length,
- });
+ return withSpan(
+ "nuq.addJobs",
+ async span => {
+ setSpanAttributes(span, {
+ "nuq.queue_name": this.queueName,
+ "nuq.jobs_count": jobs.length,
+ });
- if (jobs.length === 0) {
- return [];
- }
+ if (jobs.length === 0) {
+ return [];
+ }
- const start = Date.now();
- try {
- // Separate jobs into backlogged and non-backlogged groups
- const regularJobs: typeof jobs = [];
- const backloggedJobs: typeof jobs = [];
+ const start = Date.now();
+ try {
+ // Separate jobs into backlogged and non-backlogged groups
+ const regularJobs: typeof jobs = [];
+ const backloggedJobs: typeof jobs = [];
- for (const job of jobs) {
- if (job.options.backlogged) {
- backloggedJobs.push(job);
- } else {
- regularJobs.push(job);
+ for (const job of jobs) {
+ if (job.options.backlogged) {
+ backloggedJobs.push(job);
+ } else {
+ regularJobs.push(job);
+ }
}
- }
- const results: NuQJob<JobData, JobReturnValue>[] = [];
-
- // Batch size: 6 params per job, stay well under PG's 65535 param limit
- // 1000 jobs = 6000 params, leaving plenty of headroom
- const BATCH_SIZE = 1000;
-
- // Helper function to build and execute bulk insert with batching
- const bulkInsert = async (
- jobsToInsert: typeof jobs,
- tableSuffix: string,
- ) => {
- if (jobsToInsert.length === 0) return;
-
- // Process in batches
- for (
- let offset = 0;
- offset < jobsToInsert.length;
- offset += BATCH_SIZE
- ) {
- const batch = jobsToInsert.slice(offset, offset + BATCH_SIZE);
-
- // Build the VALUES clause and parameters array
- const valuesPlaceholders: string[] = [];
- const params: any[] = [];
-
- const columns = [
- "id",
- "data",
- "priority",
- "listen_channel_id",
- "owner_id",
- "group_id",
- ...(tableSuffix === "_backlog" ? ["times_out_at"] : []),
- ];
-
- for (let i = 0; i < batch.length; i++) {
- const job = batch[i];
- const baseIdx = i * columns.length + 1;
-
- valuesPlaceholders.push(
- `(${new Array(columns.length)
- .fill(0)
- .map((_, i) => "$" + (baseIdx + i))
- .join(", ")})`,
- );
+ const results: NuQJob<JobData, JobReturnValue>[] = [];
+
+ // Batch size: 6 params per job, stay well under PG's 65535 param limit
+ // 1000 jobs = 6000 params, leaving plenty of headroom
+ const BATCH_SIZE = 1000;
+
+ // Helper function to build and execute bulk insert with batching
+ const bulkInsert = async (
+ jobsToInsert: typeof jobs,
+ tableSuffix: string,
+ ) => {
+ if (jobsToInsert.length === 0) return;
+
+ // Process in batches
+ for (
+ let offset = 0;
+ offset < jobsToInsert.length;
+ offset += BATCH_SIZE
+ ) {
+ const batch = jobsToInsert.slice(offset, offset + BATCH_SIZE);
+
+ // Build the VALUES clause and parameters array
+ const valuesPlaceholders: string[] = [];
+ const params: any[] = [];
+
+ const columns = [
+ "id",
+ "data",
+ "priority",
+ "listen_channel_id",
+ "owner_id",
+ "group_id",
+ ...(tableSuffix === "_backlog" ? ["times_out_at"] : []),
+ ];
+
+ for (let i = 0; i < batch.length; i++) {
+ const job = batch[i];
+ const baseIdx = i * columns.length + 1;
+
+ valuesPlaceholders.push(
+ `(${new Array(columns.length)
+ .fill(0)
+ .map((_, i) => "$" + (baseIdx + i))
+ .join(", ")})`,
+ );
- params.push(
- ...[
- job.id,
- job.data,
- job.options.priority ?? 0,
- job.options.listenable ? this.listenChannelId : null,
- normalizeOwnerId(job.options.ownerId),
- job.options.groupId ?? null,
- ...(tableSuffix === "_backlog"
- ? [
- job.options.backloggedTimesOutAt
- ? job.options.backloggedTimesOutAt.toISOString()
- : null,
- ]
- : []),
- ],
- );
- }
+ params.push(
+ ...[
+ job.id,
+ job.data,
+ job.options.priority ?? 0,
+ job.options.listenable ? this.listenChannelId : null,
+ normalizeOwnerId(job.options.ownerId),
+ job.options.groupId ?? null,
+ ...(tableSuffix === "_backlog"
+ ? [
+ job.options.backloggedTimesOutAt
+ ? job.options.backloggedTimesOutAt.toISOString()
+ : null,
+ ]
+ : []),
+ ],
+ );
+ }
- const query = `INSERT INTO ${this.queueName}${tableSuffix} (${columns.join(", ")}) VALUES ${valuesPlaceholders.join(", ")} RETURNING ${(tableSuffix === "_backlog" ? this.jobBacklogReturning : this.jobReturning).join(", ")};`;
+ const query = `INSERT INTO ${this.queueName}${tableSuffix} (${columns.join(", ")}) VALUES ${valuesPlaceholders.join(", ")} RETURNING ${(tableSuffix === "_backlog" ? this.jobBacklogReturning : this.jobReturning).join(", ")};`;
- const result = await nuqPool.query(query, params);
+ const result = await nuqPool.query(query, params);
- // Convert rows to jobs and maintain order
- const jobMap = new Map(
- result.rows.map(row => [
- row.id,
- this.rowToJob(row, tableSuffix === "_backlog")!,
- ]),
- );
+ // Convert rows to jobs and maintain order
+ const jobMap = new Map(
+ result.rows.map(row => [
+ row.id,
+ this.rowToJob(row, tableSuffix === "_backlog")!,
+ ]),
+ );
- for (const job of batch) {
- const insertedJob = jobMap.get(job.id);
- if (insertedJob) {
- results.push(insertedJob);
+ for (const job of batch) {
+ const insertedJob = jobMap.get(job.id);
+ if (insertedJob) {
+ results.push(insertedJob);
+ }
}
}
- }
- };
+ };
- // Insert regular jobs
- await bulkInsert(regularJobs, "");
+ // Insert regular jobs
+ await bulkInsert(regularJobs, "");
- // Insert backlogged jobs
- await bulkInsert(backloggedJobs, "_backlog");
+ // Insert backlogged jobs
+ await bulkInsert(backloggedJobs, "_backlog");
- setSpanAttributes(span, {
- "nuq.jobs_created": results.length,
- "nuq.regular_jobs_count": regularJobs.length,
- "nuq.backlogged_jobs_count": backloggedJobs.length,
- });
+ setSpanAttributes(span, {
+ "nuq.jobs_created": results.length,
+ "nuq.regular_jobs_count": regularJobs.length,
+ "nuq.backlogged_jobs_count": backloggedJobs.length,
+ });
- return results;
- } finally {
- const duration = Date.now() - start;
- setSpanAttributes(span, {
- "nuq.duration_ms": duration,
- });
- logger.info("nuqAddJobs metrics", {
- module: "nuq/metrics",
- method: "nuqAddJobs",
- duration,
- jobsCount: jobs.length,
- });
- }
- });
+ return results;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ logger.info("nuqAddJobs metrics", {
+ module: "nuq/metrics",
+ method: "nuqAddJobs",
+ duration,
+ jobsCount: jobs.length,
+ });
+ }
+ },
+ {
+ zeroDataRetention: jobs.some(
+ job => (job.data as any)?.zeroDataRetention === true,
+ ),
+ },
+ );
}
public async promoteJobFromBacklogOrAdd(
@@ -1077,21 +1093,23 @@ class NuQ<JobData = any, JobReturnValue = any> {
data: JobData,
options: NuQJobOptions,
): Promise<NuQJob<JobData, JobReturnValue> | null> {
- return withSpan("nuq.promoteJobFromBacklogOrAdd", async span => {
- setSpanAttributes(span, {
- "nuq.queue_name": this.queueName,
- "nuq.job_id": id,
- "nuq.priority": options.priority ?? 0,
- "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
- "nuq.listenable": options.listenable ?? false,
- });
+ return withSpan(
+ "nuq.promoteJobFromBacklogOrAdd",
+ async span => {
+ setSpanAttributes(span, {
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
+ "nuq.priority": options.priority ?? 0,
+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
+ "nuq.listenable": options.listenable ?? false,
+ });
- const start = Date.now();
- try {
- const result = this.rowToJob(
- (
- await nuqPool.query(
- `
+ const start = Date.now();
+ try {
+ const result = this.rowToJob(
+ (
+ await nuqPool.query(
+ `
WITH ins AS (
INSERT INTO ${this.queueName} (id, data, created_at, priority, listen_channel_id, owner_id, group_id)
SELECT b.id, b.data, b.created_at, b.priority, b.listen_channel_id, b.owner_id, b.group_id
@@ -1106,31 +1124,33 @@ class NuQ<JobData = any, JobReturnValue = any> {
)
SELECT * FROM ins
`,
- [id],
- )
- ).rows[0],
- );
+ [id],
+ )
+ ).rows[0],
+ );
+
+ if (!result) {
+ return await this.addJobIfNotExists(id, data, {
+ ...options,
+ backlogged: false,
+ });
+ }
- if (!result) {
- return await this.addJobIfNotExists(id, data, {
- ...options,
- backlogged: false,
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ logger.info("nuqPromoteJobFromBacklogOrAdd metrics", {
+ module: "nuq/metrics",
+ method: "nuqPromoteJobFromBacklogOrAdd",
+ duration,
});
}
-
- return result;
- } finally {
- const duration = Date.now() - start;
- setSpanAttributes(span, {
- "nuq.duration_ms": duration,
- });
- logger.info("nuqPromoteJobFromBacklogOrAdd metrics", {
- module: "nuq/metrics",
- method: "nuqPromoteJobFromBacklogOrAdd",
- duration,
- });
- }
- });
+ },
+ { zeroDataRetention: (data as any)?.zeroDataRetention === true },
+ );
}
private readonly nuqWaitMode =
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 87b838f69..e8b3737c1 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -1,7 +1,5 @@
import { configDotenv } from "dotenv";
import { config } from "../../config";
-import * as Sentry from "@sentry/node";
-import { applyZdrScope, captureExceptionWithZdrCheck } from "../sentry";
import http from "http";
import https from "https";
@@ -93,6 +91,7 @@ import { scrapeSitemap } from "../../scraper/crawler/sitemap";
import {
withTraceContextAsync,
withSpan,
+ withZeroDataRetention,
setSpanAttributes,
} from "../../lib/otel-tracer";
import { ScrapeUrlResponse } from "../../scraper/scrapeURL";
@@ -303,9 +302,6 @@ async function billScrapeJob(
status: "void",
});
}
- captureExceptionWithZdrCheck(error, {
- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
- });
return creditsToBeBilled;
}
}
@@ -363,7 +359,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
teamId: job.data?.team_id ?? undefined,
zeroDataRetention: job.data?.zeroDataRetention ?? false,
});
- applyZdrScope(job.data?.zeroDataRetention);
logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });
const start = job.data.startTime ?? Date.now();
const remainingTime = job.data.scrapeOptions.timeout
@@ -995,16 +990,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
} else {
logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });
- // Filter out TransportableErrors (flow control)
- if (!(error instanceof TransportableError)) {
- captureExceptionWithZdrCheck(error, {
- data: {
- job: job.id,
- },
- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
- });
- }
-
if (error instanceof CustomError) {
// Here we handle the error, then save the failed job
logger.error(error.message); // or any other error handling
@@ -1662,8 +1647,13 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {
zeroDataRetention: job.data?.zeroDataRetention ?? false,
});
- // Restore trace context if available and execute within span
- if (job.data.traceContext) {
+ // Restore trace context if available and execute within span. The ZDR
+ // context is applied either way so nothing below records for ZDR jobs.
+ return withZeroDataRetention(job.data.zeroDataRetention === true, () => {
+ if (!job.data.traceContext) {
+ return processJobWithTracing(job, logger);
+ }
+
return withTraceContextAsync(job.data.traceContext, () =>
withSpan("worker.scrape.process", async span => {
setSpanAttributes(span, {
@@ -1677,9 +1667,7 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {
return processJobWithTracing(job, logger);
}),
);
- } else {
- return processJobWithTracing(job, logger);
- }
+ });
};
async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
@@ -1767,20 +1755,6 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
} catch (error) {
logger.warn("Job failed", { error });
- // Filter out expected errors (flow control, not real errors)
- if (
- error instanceof TransportableError ||
- error instanceof JobCancelledError ||
- error instanceof RacedRedirectError ||
- error instanceof ScrapeJobTimeoutError
- ) {
- // These are expected flow control errors, don't send to Sentry
- } else {
- captureExceptionWithZdrCheck(error, {
- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
- });
- }
-
if (job.data.skipNuq) {
throw error;
} else {
diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.ts
index 5dde4ddec..1ed439d7d 100644
--- a/apps/api/src/services/zdr-worker.ts
+++ b/apps/api/src/services/zdr-worker.ts
@@ -1,6 +1,5 @@
import "dotenv/config";
-import "./sentry";
-import { setSentryServiceTag } from "./sentry";
+import { shutdownTracing } from "../otel";
import { logger } from "../lib/logger";
import { zdrcleaner } from "../lib/zdrcleaner";
@@ -17,12 +16,11 @@ process.on("SIGTERM", () => {
});
(async () => {
- setSentryServiceTag("zdr-worker");
-
while (!isShuttingDown) {
await zdrcleaner();
}
+ await shutdownTracing();
logger.info("zdr-worker exiting");
process.exit(0);
})();
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index ef87771b0..31fe6dbc2 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -11,7 +11,7 @@ import { InternalOptions } from "./scraper/scrapeURL";
import type { CostTracking } from "./lib/cost-tracking";
import type { BillingMetadata } from "./services/billing/types";
import { webhookSchema } from "./services/webhook/schema";
-import { SerializedTraceContext } from "./lib/otel-tracer";
+import type { SerializedTraceContext } from "./lib/otel-tracer";
type ScrapeJobCommon = {
concurrencyLimited?: boolean;
@@ -60,7 +60,6 @@ type ScrapeJobSingleUrlsUnique = {
from_extract?: boolean;
startTime?: number;
- sentry?: any;
is_extract?: boolean;
apiKeyId: number | null;