Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

`WebScraper` refactor into `scrapeURL` (#714)

GergΕ‘ MΓ³ricz8d467c8ca764
diff --git a/.github/archive/js-sdk.yml b/.github/archive/js-sdk.yml
index c84bb8b19..7ef096d4a 100644
--- a/.github/archive/js-sdk.yml
+++ b/.github/archive/js-sdk.yml
@@ -8,7 +8,6 @@ env:
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
HOST: ${{ secrets.HOST }}
LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
- LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
@@ -21,7 +20,6 @@ env:
SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
HDX_NODE_BETA_MODE: 1
jobs:
diff --git a/.github/archive/python-sdk.yml b/.github/archive/python-sdk.yml
index 274498885..bdefeab62 100644
--- a/.github/archive/python-sdk.yml
+++ b/.github/archive/python-sdk.yml
@@ -8,7 +8,6 @@ env:
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
HOST: ${{ secrets.HOST }}
LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
- LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
@@ -21,7 +20,6 @@ env:
SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
HDX_NODE_BETA_MODE: 1
jobs:
diff --git a/.github/archive/rust-sdk.yml b/.github/archive/rust-sdk.yml
index 62deeaab5..792e06c21 100644
--- a/.github/archive/rust-sdk.yml
+++ b/.github/archive/rust-sdk.yml
@@ -8,7 +8,6 @@ env:
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
HOST: ${{ secrets.HOST }}
LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
- LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
@@ -21,7 +20,6 @@ env:
SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
HDX_NODE_BETA_MODE: 1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8a9a74ccf..ef7d1cba5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,7 +12,6 @@ env:
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
HOST: ${{ secrets.HOST }}
LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
- LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
@@ -25,7 +24,6 @@ env:
SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
HDX_NODE_BETA_MODE: 1
FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }}
diff --git a/.github/workflows/deploy-image-staging.yml b/.github/workflows/deploy-image-staging.yml
new file mode 100644
index 000000000..e74aba9a7
--- /dev/null
+++ b/.github/workflows/deploy-image-staging.yml
@@ -0,0 +1,32 @@
+name: STAGING Deploy Images to GHCR
+
+env:
+ DOTNET_VERSION: '6.0.x'
+
+on:
+ push:
+ branches:
+ - mog/webscraper-refactor
+ workflow_dispatch:
+
+jobs:
+ push-app-image:
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: './apps/api'
+ steps:
+ - name: 'Checkout GitHub Action'
+ uses: actions/checkout@main
+
+ - name: 'Login to GitHub Container Registry'
+ uses: docker/login-action@v1
+ with:
+ registry: ghcr.io
+ username: ${{github.actor}}
+ password: ${{secrets.GITHUB_TOKEN}}
+
+ - name: 'Build Inventory Image'
+ run: |
+ docker build . --tag ghcr.io/mendableai/firecrawl-staging:latest
+ docker push ghcr.io/mendableai/firecrawl-staging:latest
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 73ccf0e60..b8c1f0a59 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -41,7 +41,6 @@ TEST_API_KEY= # use if you've set up authentication and want to test with a real
SCRAPING_BEE_API_KEY= #Set if you'd like to use scraping Be to handle JS blocking
OPENAI_API_KEY= # add for LLM dependednt features (image alt generation, etc.)
BULL_AUTH_KEY= @
-LOGTAIL_KEY= # Use if you're configuring basic logging with logtail
PLAYWRIGHT_MICROSERVICE_URL= # set if you'd like to run a playwright fallback
LLAMAPARSE_API_KEY= #Set if you have a llamaparse key you'd like to use to parse pdfs
SLACK_WEBHOOK_URL= # set if you'd like to send slack server health status messages
diff --git a/SELF_HOST.md b/SELF_HOST.md
index 782284857..46e08db95 100644
--- a/SELF_HOST.md
+++ b/SELF_HOST.md
@@ -62,7 +62,6 @@ TEST_API_KEY= # use if you've set up authentication and want to test with a real
SCRAPING_BEE_API_KEY= # use if you'd like to use as a fallback scraper
OPENAI_API_KEY= # add for LLM-dependent features (e.g., image alt generation)
BULL_AUTH_KEY= @
-LOGTAIL_KEY= # Use if you're configuring basic logging with logtail
PLAYWRIGHT_MICROSERVICE_URL= # set if you'd like to run a playwright fallback
LLAMAPARSE_API_KEY= #Set if you have a llamaparse key you'd like to use to parse pdfs
SLACK_WEBHOOK_URL= # set if you'd like to send slack server health status messages
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 6ba49daa6..d54c696d2 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -33,8 +33,6 @@ SCRAPING_BEE_API_KEY=
# add for LLM dependednt features (image alt generation, etc.)
OPENAI_API_KEY=
BULL_AUTH_KEY=@
-# use if you're configuring basic logging with logtail
-LOGTAIL_KEY=
# set if you have a llamaparse key you'd like to use to parse pdfs
LLAMAPARSE_API_KEY=
# set if you'd like to send slack server health status messages
@@ -54,9 +52,6 @@ STRIPE_PRICE_ID_STANDARD_NEW_YEARLY=
STRIPE_PRICE_ID_GROWTH=
STRIPE_PRICE_ID_GROWTH_YEARLY=
-HYPERDX_API_KEY=
-HDX_NODE_BETA_MODE=1
-
# set if you'd like to use the fire engine closed beta
FIRE_ENGINE_BETA_URL=
diff --git a/apps/api/jest.setup.js b/apps/api/jest.setup.js
index c158ca426..0b3b09b7c 100644
--- a/apps/api/jest.setup.js
+++ b/apps/api/jest.setup.js
@@ -1 +1 @@
-global.fetch = require('jest-fetch-mock');
+// global.fetch = require('jest-fetch-mock');
diff --git a/apps/api/package.json b/apps/api/package.json
index a0f9cf8e5..bb4ea268b 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -32,9 +32,11 @@
"@tsconfig/recommended": "^1.0.3",
"@types/body-parser": "^1.19.2",
"@types/cors": "^2.8.13",
+ "@types/escape-html": "^1.0.4",
"@types/express": "^4.17.17",
"@types/jest": "^29.5.12",
"@types/node": "^20.14.1",
+ "@types/pdf-parse": "^1.1.4",
"body-parser": "^1.20.1",
"express": "^4.18.2",
"jest": "^29.6.3",
@@ -53,9 +55,7 @@
"@bull-board/api": "^5.20.5",
"@bull-board/express": "^5.20.5",
"@devil7softwares/pos": "^1.0.2",
- "@dqbd/tiktoken": "^1.0.13",
- "@hyperdx/node-opentelemetry": "^0.8.1",
- "@logtail/node": "^0.4.12",
+ "@dqbd/tiktoken": "^1.0.16",
"@nangohq/node": "^0.40.8",
"@sentry/cli": "^2.33.1",
"@sentry/node": "^8.26.0",
@@ -78,6 +78,7 @@
"date-fns": "^3.6.0",
"dotenv": "^16.3.1",
"dotenv-cli": "^7.4.2",
+ "escape-html": "^1.0.3",
"express-rate-limit": "^7.3.1",
"express-ws": "^5.0.2",
"form-data": "^4.0.0",
@@ -92,6 +93,7 @@
"languagedetect": "^2.0.0",
"logsnag": "^1.0.0",
"luxon": "^3.4.3",
+ "marked": "^14.1.2",
"md5": "^2.3.0",
"moment": "^2.29.4",
"mongoose": "^8.4.4",
@@ -114,6 +116,8 @@
"typesense": "^1.5.4",
"unstructured-client": "^0.11.3",
"uuid": "^10.0.0",
+ "winston": "^3.14.2",
+ "winston-transport": "^4.8.0",
"wordpos": "^2.1.0",
"ws": "^8.18.0",
"xml2js": "^0.6.2",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 095b507c3..3350c74ea 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -24,14 +24,8 @@ importers:
specifier: ^1.0.2
version: 1.0.2
'@dqbd/tiktoken':
- specifier: ^1.0.13
- version: 1.0.15
- '@hyperdx/node-opentelemetry':
- specifier: ^0.8.1
- version: 0.8.1
- '@logtail/node':
- specifier: ^0.4.12
- version: 0.4.21
+ specifier: ^1.0.16
+ version: 1.0.16
'@nangohq/node':
specifier: ^0.40.8
version: 0.40.8
@@ -98,6 +92,9 @@ importers:
dotenv-cli:
specifier: ^7.4.2
version: 7.4.2
+ escape-html:
+ specifier: ^1.0.3
+ version: 1.0.3
express-rate-limit:
specifier: ^7.3.1
version: 7.3.1([email protected])
@@ -140,6 +137,9 @@ importers:
luxon:
specifier: ^3.4.3
version: 3.4.4
+ marked:
+ specifier: ^14.1.2
+ version: 14.1.2
md5:
specifier: ^2.3.0
version: 2.3.0
@@ -206,6 +206,12 @@ importers:
uuid:
specifier: ^10.0.0
version: 10.0.0
+ winston:
+ specifier: ^3.14.2
+ version: 3.14.2
+ winston-transport:
+ specifier: ^4.8.0
+ version: 4.8.0
wordpos:
specifier: ^2.1.0
version: 2.1.0
@@ -237,6 +243,9 @@ importers:
'@types/cors':
specifier: ^2.8.13
version: 2.8.17
+ '@types/escape-html':
+ specifier: ^1.0.4
+ version: 1.0.4
'@types/express':
specifier: ^4.17.17
version: 4.17.21
@@ -246,6 +255,9 @@ importers:
'@types/node':
specifier: ^20.14.1
version: 20.14.1
+ '@types/pdf-parse':
+ specifier: ^1.1.4
+ version: 1.1.4
body-parser:
specifier: ^1.20.1
version: 1.20.2
@@ -494,42 +506,22 @@ packages:
resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==}
+
'@devil7softwares/[email protected]':
resolution: {integrity: sha512-49Ke26+++Ix8C5LChi4uga7aWgMuc5zV1NqjGxXxE7DralZwe+hvUuSJmBDWS+HHZaK9rFzLNdufV4HAvvOxPA==}
engines: {node: '>=0'}
deprecated: This package has been renamed to `fast-tag-pos`
- resolution: {integrity: sha512-a6I67K1xUkuqcuwulobIJiLikkoE7egMaviI1Jg5bxSn2V7QGqXsGE3jTKr8UIOU/o74mAAd5TkeXFNBtaKF4A==}
+ resolution: {integrity: sha512-4uIrs5qxAwFVFFEP507HZIZhGOsgfaEMEWDXWalr+v+XP+wJwP60EVmkZtQyQe70IsKGVkx5umBxw4NfmU0pPg==}
'@flydotio/[email protected]':
resolution: {integrity: sha512-L52UAfrOhmAn3T4TxpeRofQOSO+Kctg+uraB4nLzo4mvvh+4Z7HYxSi7Dnq0Kirz+xx6fDIc4OMNT1EdaORecA==}
engines: {node: '>=16.0.0'}
hasBin: true
- resolution: {integrity: sha512-HPa/K5NX6ahMoeBv15njAc/sfF4/jmiXLar9UlC2UfHFKZzsCVLc3wbe7+7qua7w9VPh2/L6EBxyAV7/E8Wftg==}
- engines: {node: '>=12.10.0'}
-
- resolution: {integrity: sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==}
- engines: {node: '>=6'}
- hasBin: true
-
- '@hyperdx/[email protected]':
- resolution: {integrity: sha512-Jgk7JY5J07Mq9fgXApGVhSkS4+WdzzRcWLReAZhxgo46KShxE6w614mFqUSnuo+z6ghlehsy4ForViUfxrFyew==}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@hyperdx/[email protected]':
- resolution: {integrity: sha512-n8d/K/8M2owL2w4FNfV+lSVW6yoznEj5SdRCysV/ZIfyrZwpijiiSn7gkRcrOfKHmrxrupyp7DVg5L19cGuH6A==}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@hyperdx/[email protected]':
- resolution: {integrity: sha512-wNw0yQf54j/9KXVWeEOu8G6C5FT5EFlrz4dcmscTkwCvo6fQOLRZa/NbGcqugt0LSFMc0/6/Q5RDWVqDpEn0LQ==}
- hasBin: true
-
'@ioredis/[email protected]':
resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
@@ -636,9 +628,6 @@ packages:
'@jridgewell/[email protected]':
resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==}
- '@js-sdsl/[email protected]':
- resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==}
-
'@langchain/[email protected]':
resolution: {integrity: sha512-zaKvUcWU1Cxcpd/fxklygY6iUrxls10KTRzyHZGBAIKJq1JD/B10vX59YlFgBs7nqqVTEvaChfIE0O0e2qBttA==}
engines: {node: '>=18'}
@@ -651,28 +640,12 @@ packages:
resolution: {integrity: sha512-cXWgKE3sdWLSqAa8ykbCcUsUF1Kyr5J3HOWYGuobhPEycXW4WI++d5DhzdpL238mzoEXTi90VqfSCra37l5YqA==}
engines: {node: '>=18'}
- '@logtail/[email protected]':
- resolution: {integrity: sha512-QDq194+24bwi4e+a/pxyf4X67NewhTvBmh9iwM2NhbSVSQz4Fo8xQn1Ul8zuUrXETycu/Od2D8wT2tZFNFx/7A==}
-
- '@logtail/[email protected]':
- resolution: {integrity: sha512-zpwkhJgcYaM+vsjotHRJthc0ot1vP0CAVy+fwrkL8XjfdC3NHiWb6f0agQpHlqdRX8RTsAbcYpWNXKPpFB5U9Q==}
-
- '@logtail/[email protected]':
- resolution: {integrity: sha512-xIaolScUwJEikllopGphxBX0lVlN/rA8pLAZiNCMNJXpPbwitoFKLW3w4qRuYdKoFCCJZKwOdwEqU2Fv0i9Cuw==}
-
- '@logtail/[email protected]':
- resolution: {integrity: sha512-nYsum10eJMTo+ySBlYXvSrvgD1NDCVUeOlxLBbelq3XUmHu9L48VNR3P0BOmhLamYCTEgjatTj0PyPLfjL1W9g==}
-
'@mixmark-io/[email protected]':
resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
'@mongodb-js/[email protected]':
resolution: {integrity: sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q==}
- '@msgpack/[email protected]':
- resolution: {integrity: sha512-h9u4u/jiIRKbq25PM+zymTyW6bhTzELvOoUd+AvYriWOAKpLGnIamaET3pnHYoI5iYphAHBI4ayx0MehR+VVPQ==}
- engines: {node: '>= 10'}
-
'@msgpackr-extract/[email protected]':
resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==}
cpu: [arm64]
@@ -710,10 +683,6 @@ packages:
'@one-ini/[email protected]':
resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==}
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-E3skn949Pk1z2XtXu/lxf6QAZpawuTM/IUEXcAzpiUkTd73Hmvw26FiN3cJuTmkpM5hZzHwkomVdtrh/n/zzwA==}
- engines: {node: '>=14'}
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-qnSqB2DQ9TPP96dl8cDubDvrUyWc0/sK81xHTK8eSUspzDM3bsewX903qclQFvVhgStjRWdC5bLb3kQqMkfV5A==}
engines: {node: '>=14'}
@@ -722,647 +691,198 @@ packages:
resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
engines: {node: '>=8.0.0'}
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-s0CwmY9KYtPawOhV5YO2Gf62uVOQRNvT6Or8IZ0S4gr/kPVNhoMehTsQvqBwSWQfoFrkmW3KKOHiKJEp4dVGXg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.4.1
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-R5r6DO4kgEOVBxFXhXjwospLQkv+sYxwCfjvoZBe7Zm6KKXAV9kDSJhi/D1BweowdZmO+sdbENLs374gER8hpQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.9.0'
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-UW/ge9zjvAEmRWVapOP0qyCvPulWU6cQxGxDbWEFfGOj1VBBZAuOqTo3X6yWmDTD3Xe15ysCZChHncr2xFMIfQ==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': '>=1.0.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-wMSGfsdmibI88K9wB498zXY04yThPexo8jvwNNlm542HZB7XrrMRBbAyKJqG8qDRJwIBdBrPMi4V9ZPW/sqrcg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.9.0'
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-GeT/l6rBYWVQ4XArluLVB6WWQ8flHbdb6r2FCHC3smtdOAbrJBIv35tpV/yp9bmYUJf+xmZpu9DRTIeJVhFbEQ==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': '>=1.0.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-cd6GZ9IqCrmvOJwi1HjRR7o9ihF7xhZTekgxUsoyTsPF+SjKMsLF9ur6HeBYkYhk+YjZ1ken3XUMH47oUTvu8Q==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-oFXvif9iksHUxrzG3P8ohMLt7xSrl+oDMqxD/3XXndU761RFAKSbRDpfrQs25U5D+A2aMV3qk+4kfUWdJhZ77g==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-jhj8xD6S4cydXGCuf2tp56+4QI0DbDH6g+0MiPPJVdXjxLj+iycQuqB2cwljWpByblFaOjyUsL/VKtm8C7sQ9A==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-P9+Hkszih95ITvldGZ+kXvj9HpD1QfS+PwooyHK72GYA+Bgm+yUSAsDkUkDms8+s9HW6poxURv3LcjaMuBBpVQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-n+LhLPsX07URh+HhV2SHVSvz1t4G/l/CE5BjpmhAPqeTceFac1VpyQkavWEJbvnK5bUEXijWt4LxAxFpt2fXyw==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-SE9f0/6V6EeXC9i+WA4WFjS1EYgaBCpAnI5+lxWvZ7iO7EU1IvHvZhP6Kojr0nLldo83gqg6G7OWFqsID3uF+w==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-+Rl/VFmu2n6eaRMnVbyfZx1DqR/1KNyWebYuHyQBZaEAVIn/ZLgmofRpXN1X2nhJ4BNaptQUNxAstCYYz6dKoQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-XjOHeAOreh0XX4jlzTTUWWqu1dIGvMWM8yvd43JJdRMAmTZisezjKsxLjMEMIvF0PzQdoXwh9DiS9nYE4/QmpA==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-/BLG+0DQr2tCILFGJKJH2Fg6eyjhqOlVflYpNddUEXnzyQ/PAhTdgirkqbICFgeSW2XYcEY9zXpuRldrVNw9cA==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-7+8WMY0LQeqv6KIObXK+Py44qNFLeCU0ZLLxSZtXEbZ2wJlQISP1St65jRto0NV7isnZoyuOxb2+ZpypPPNv7Q==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-ThNcgTE22W7PKzTzz5qfGxb5Gf7rA3EORousYo2nJWHHcF6gqiMNv2+GXY3MdpjLBr8IgCfhtvbQdD6rlIPUpA==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-ML4Vw0it2uIpETfX6skuSIGLHF9D3TUKOfdfrk9lnrzzWSzg2aS6pl3UeepkQX4wXHdzlxVRB0USrUqsmxMd5Q==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-xI5Q/CMmzBmHshPnzzjD19ptFaYO/rQWzokpNio4QixZYWhJsa35QgRvN9FhPkwgtuJIbt/CWWAufJ3egJNHEA==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-2/nRnx3pjYEmdPIaBwtgtSviTKHWnDZN3R+TkRUnhIVrvBKVcq+I5B2rtd6mr6Fe9cHlZ9Ojcuh7pkNh/xdWWg==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-90eAF2JPSbPAsOuGfYyctYaoYXqy4Clbxt0j/uUgg6dto4oqwUw3AvTyHQEztLGxeXwEzC1EQigDtVPg5ZexYA==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-fiyCOAw+tlbneok1x7P5UseoGW5nS60CWWx7NXzYW+WOexpSmDQQW7olttGa8fqE6/sVCoi1l+QdfVoETZi/NQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-NWRbQ7q0E3co/CNTWLZZvUzZoKhB1iTitY282IM8HDTXkA6VRssCfOcvaHw5ezOh23TJbAeYxmmpVj4hFvDPYQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-AG8U7z7D0JcBu/7dDcwb47UMEzj9/FMiJV2iQZqrsZnxR3FjB9J9oIH2iszJYci2eUdp2WbdvtpD9RV/zmME5A==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-uRx0V3LPGzjn2bxAnV8eUsDT82vT7NTwI0ezEuPMBOTOsnPpGhWdhcdNdhH80sM4TrWrOfXm9HGEdfWE3TRIww==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-3Nfm43PI0I+3EX+1YbSy6xbDu276R1Dh1tqAk68yd4yirnIh52Kd5B+nJ8CgHA7o3UKakpBjj6vSzi5vNCzJIA==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-HBVLpTSYpkQZ87/Df3N0gAw7VzYZV3n28THIBrJWfuqw3Or7UqdhnjeuMIPQ04BKk3aZc0cWn2naSQObbh5vXw==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-Waf+2hekJRxIwq1PmivxOWLdMOtYbY22hKr34gEtfbv2CArSv8FBJH4BmQxB9o5ZcwkdKu589qs009dbuSfNmQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-pVc8P5AgliC1DphyyBUgsxXlm2XaPH4BpYvt7rAZDMIqUpRk8gs19SioABtKqqxvFzg5jPtgJfJsdxq0Y+maLw==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-CExAEqJvK8jYxrhN8cl6EaGg57EGJi+qsSKouLC5lndXi68gZLOKbZIMZg4pF0kNfp/D4BFaGmA6Ap7d5WoPTw==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-LVRdEHWACWOczv2imD+mhUrLMxsEjPPi32vIZJT57zygR5aUiA4em8X3aiGOCycgbMWkIu8xOSGSxdx3JmzN+w==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-N8SOwoKL9KQSX7z3gOaw5UaTeVQcfDO1c21csVHnmnmGUoqsXbArK2B8VuwPWcv6/BC/i3io+xTo7QGRZ/z28Q==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-coRTugFL7De/VNH/1NqPlxnfik87jS+jBXsny+Y/lMhXIA3x8t71IyL9ihuewkD+lNtIxIz6Y7Sq6kPuOqz5dQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-ZcOqEuwuutTDYIjhDIStix22ECblG/i9pHje23QGs4Q4YS4RMaZ5hKCoQJxW88Z4K7T53rQkdISmoXFKDV8xMg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-8U/w7Ifumtd2bSN1OLaSwAAFhb9FyqWUki3lMMB0ds+1+HdSxYBe9aspEJEgvxAqOkrQnVniAPTEGf1pGM7SOw==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-6b3nZnFFEz/3xZ6w8bVxctPUWIPWiXuPQ725530JgxnN1cvYFd8CJ75PrHZNjynmzSSnqBkN3ef4R9N+RpMh8Q==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-dG/aevWhaP+7OLv4BQQSEKMJv8GyeOp3Wxl31NHqE8xo9/fYMfEljiZphUHIfyg4gnZ9swMyWjfOQs5GUQe54Q==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-Jv/fH7KhpWe4KBirsiqeUJIYrsdR2iu2l4nWhfOlRvaZ+zYIiLEzTQR6QhBbyRoAbU4OuYJzjWusOmmpGBnwng==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-P11H168EKvBB9TUSasNDOGJCSkpT44XgoM6d3gRIWAa9ghLpYhl0uRkS8//MqPzcJVHr3h3RmfXIpiYLjyIZTw==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-6bEuiI+yMf3D0+ZWZE2AKmXhIhBvZ0brdO/0A8lUqeqeS+sS4fTcjA1F2CclsCNxYWEgcs8o3QyQqPceBeVRlg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-dJc3H/bKMcgUYcQpLF+1IbmUKus0e5Fnn/+ru/3voIRHwMADT3rFSUcGLWSczkg68BCgz0vFWGDTvPtcWIFr7A==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-H1BEmnMhho8o8HuNRq5zEI4+SIHDIglNB7BPKohZyWG4fWNuR7yM4GTlR01Syq21vODAS7z5omblScJD/eZdKw==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-dHLrn55qVWsHJQYdForPWPUWDk2HZ2jjzkT+WoQSqpYT1j4HxfoiLfBTF+I3EbEYFAJnDRmRAUfA6nU5GPdCLQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-5efkT8ZfN8il5z+yfKYFGm2YR3mhlhaJoGfNOAylKE/6tUH3WDTTWaP7nrURtWGc+fuvDktcEch18Se8qsGS7w==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-bMKej7Y76QVUD3l55Q9YqizXybHUzF3pujsBFjqbZrRn2WYqtsDtTUlbCK7fvXNPwFInqZ2KhnTqd0gwo8MzaQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-VF/MicZ5UOBiXrqBslzwxhN7TVqzu1/LN/QDpkskqM0Zm0aZ4CVRbUygL8d7lrjLn15x5kGIe8VsSphMfPJzlA==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-zaeiasdnRjXe6VhYCBMdkmAVh1S5MmXC/0spet+yqoaViGnYst/DOxPvhwg3yT4Yag5crZNWsVXnA538UjP6Ow==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-niRi5ZUnkgzRhIGMOozTyoZIvJKNJyhijQI4nF4iFSb+FUx2v5fngfR+8XLmdQAO7xmsD8E5vEGdDVYVtKbZew==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-qkpHMgWSDTYVB1vlZ9sspf7l2wdS5DDq/rbIepDwX5BA0N0068JTQqh0CgAh34tdFqSCnWXIhcyOXC2TtRb0sg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-0xfS1xcqUmY7WE1uWjlmI67Xg3QsSUlNT+AcXHeA4BDUPwZtWqF4ezIwLgpVZfHOnkAEheqGfNSWd1PIu3Wnfg==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-+iBAawUaTfX/HAlvySwozx0C2B6LBfNPXX1W8Z2On1Uva33AGkw2UjL9XgIg1Pj4eLZ9R4EoJ/aFz+Xj4E/7Fw==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-d7ja8yizsOCNMYIJt5PH/fKZXjb/mS48zLROO4BzZTtDfhNCl2UM/9VIomP2qkGIFVouSJrGr/T00EzY7bPtKA==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-ebYQjHZEmGHWEALwwDGhSQVLBaurFnuLIkZD5igPXrt7ohfF4lc5/4al1LO+vKc0NHk8SJWStuRueT86ISA8Vg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-mewVhEXdikyvIZoMIUry8eb8l3HUjuQjSjVbmLVTt4NQi35tkpnHQrG9bTRBrl3403LoWZ2njMPJyg4l6HfKvA==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-rZlbSgwAJys8lpug+xIeAdO98ypYMAPVqrHqc4AHuUl5S4MULHEcjGLMZLoE/guEGO4xAQ5XUezpRFGM1SAnsg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-BSlhpivzBD77meQNZY9fS4aKgydA8AJBzv2dqvxXFy/Hq64b7HURgw/ztbmwFeYwdF5raZZUifiiNSMLpOJoSA==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-og23KLyoxdnAeFs1UWqzSonuCkePUzCX30keSYigIzJe/6WSYA8rnEI5lobcxPEzg+GcU06J7jzokuEHbjVJNw==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-uA17F2iP77o3NculB63QD2zv3jkJ093Gfb0GxHLEqTIqpYs1ToJ53ybWwjJwqFByxk7GrliaxaxVtWC23PKzBg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-Zpfqfi83KeKgVQ0C2083GZPon3ZPYQ5E59v9FAbhubtOoUb9Rh7n111YD8FPW3sgx6JKp1odXmBmfQhWCaTOpQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-H7IfGTqW2reLXqput4yzAe8YpDC0fmVNal95GHMLOrS89W+qWUKIqxolSh63hJyfmwPSFwXASzj7wpSk8Az+Dg==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-HUjTerD84jRJnSyDrRPqn6xQ7K91o9qLflRPZqzRvq0GRj5PMfc6TJ/z3q/ayWy/2Kzffhrp7HCIVp0u0TkgUg==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-a9TijXZZbk0vI5TGLZl+0kxyFfrXHhX6Svtz7Pp2/VBlCSKrazuULEyoJQrOknJyFWNMEmbbJgOciHCCpQcisw==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-VYK47Z9GBaZX5MQLL7kZDdzQDdyUtHRD4J/GSr6kdwmIpdpUQXLsV3EnboeB8P+BlpucF57FyJKE8yWTOEMfnA==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-uXJbYU/5/MBHjMp1FqrILLRuiJCs3Ofk0MeRDk8g1S1gD47U8X3JnSwcMO1rtRo1x1a7zKaQHaoYu49p/4eSKw==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-+OPcm7C9I5oPqnpStE+1WkdPWjRx0k5XKratxQmIDFZrmhRcqvMte3vrrzE/OBPg9iqh2tKrSe0y7+0sRfTJyQ==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-faYX1N0gpLhej/6nyp6bgRjzAKXn5GOEMYY7YhciSfCoITAktLUtQ36d24QEWNA1/WA1y6qQunCe0OhHRkVl9g==}
engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-/NOgUF5gf3T5c3GMyy6fnQxaVzbOf9j2xcetgymIIX2HSN3Gk7o64G7KDvwHwhaa20ZiF0QDLb3m4AT+tn9eRg==}
- engines: {node: '>=14.10.0'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-4J2ehk5mJyDT6j2yJCOuPxAjit5QB1Fwzhx0LID5jjvhI9LxzZIGDNAPTTHyghSiaRDeNMzceXKkkEQJkg2MNw==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-pkZT+iFYIZsVn6+GzM0kSX+u3MSLCY9md+lIJOoKl/P+gJFfxJte/60Usdp8Ce4rOs8GduUpSPNe1ddGyDT1sQ==}
engines: {node: '>=14'}
peerDependencies:
- '@opentelemetry/api': ^1.3.0
+ '@opentelemetry/api': '>=1.0.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-maSXMxgS0szU52khQzAROV4nWr+3M8mZajMQOc3/7tYjo+Q3HlWAowOuagPvp4pwROK4x6oDaFYlY+ZSj1qjYA==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-9Mb7q5ioFL4E4dDrc4wC/A3NTHDat44v4I3p2pLPSxRvqUbDIQyMVr9uK+EU69+HWhlET1VaSrRzwdckWqY15Q==}
engines: {node: '>=14'}
peerDependencies:
- '@opentelemetry/api': ^1.3.0
+ '@opentelemetry/api': '>=1.3.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-RH9WdVRtpnyp8kvya2RYqKsJouPxvHl7jKPsIfrbL8u2QCKloAGi0uEqDHoOS15ZRYPQTDXZ7d8jSpUgSQmvpA==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-C8k4hnEbc5FamuZQ92nTOp8X/diCY56XUTnMiv9UTuJitCzaNNHAVsdm5+HLCdI8SLQsLWIrG38tddMxLVoftw==}
engines: {node: '>=14'}
peerDependencies:
- '@opentelemetry/api': ^1.7.0
+ '@opentelemetry/api': '>=1.0.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-vOx55fxdNjo2XojJf8JN4jP7VVvQCh7UQzzQ2Q2FpGJpt8Z3EErKaY8xOBkOuJH0TtL/Q72rmIn9c+mRG46BxA==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-ZDjMJJQRlyk8A1KZFCc+bCbsyrn1wTwdNt56F7twdfUfnHUZUq77/WfONCj8p72NZOyP7pNTdUWSTYC3GTbuuQ==}
engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-a9TijXZZbk0vI5TGLZl+0kxyFfrXHhX6Svtz7Pp2/VBlCSKrazuULEyoJQrOknJyFWNMEmbbJgOciHCCpQcisw==}
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-nSDlnHSqzC3pXn/wZEZVLuAuJ1MYMXPBwtv2qAbCa3847SaHItdE7SzUq/Jtb0KZmh1zfAbNi3AAMjztTT4Ugg==}
engines: {node: '>=14'}
peerDependencies:
- '@opentelemetry/api': ^1.3.0
+ '@opentelemetry/api': ^1.1.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-JIrvhpgqY6437QIqToyozrUG1h5UhwHkaGK/WAX+fkrpyPtc+RO5FkRtUd9BH0MibabHHvqsnBGKfKVijbmp8w==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
+ '@pdf-lib/[email protected]':
+ resolution: {integrity: sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==}
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-uXJbYU/5/MBHjMp1FqrILLRuiJCs3Ofk0MeRDk8g1S1gD47U8X3JnSwcMO1rtRo1x1a7zKaQHaoYu49p/4eSKw==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.3.0
+ '@pdf-lib/[email protected]':
+ resolution: {integrity: sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==}
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-UYlnOYyDdzo1Gw559EHCzru0RwhvuXCwoH8jGo9J4gO1TE58GjnEmIjomMsKBCym3qWNJfIQXw+9SZCV0DdQNg==}
+ '@pkgjs/[email protected]':
+ resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-ZAS+4pq8o7dsugGTwV9s6JMKSxi+guIHdn0acOv0bqj26e9pWDFx5Ky+bI0aY46uR9Y0JyXqY+KAEYM/SO3DFA==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
+ '@prisma/[email protected]':
+ resolution: {integrity: sha512-c1Sle4ji8aasMcYfBBHFM56We4ljfenVtRmS8aY06BllS7SoU6SmJBwG7vil+GHiR0Yrh+t9iBwt4AY0Jr4KNQ==}
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-gxxxwfk0inDMb5DLeuxQ3L8TtptxSiTNHE4nnAJH34IQXAVRhXSXW1rK8PmDKDngRPIZ6J7ncUCjjIn8b+AgqQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
+ '@puppeteer/[email protected]':
+ resolution: {integrity: sha512-bJ0UBsk0ESOs6RFcLXOt99a3yTDcOKlzfjad+rhFwdaG1Lu/Wzq58GHYCDTlZ9z6mldf4g+NTb+TXEfe0PpnsQ==}
+ engines: {node: '>=18'}
+ hasBin: true
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-OppYOXwV9LQqqtYUCywqoOqX/JT9LQ5/FMuPZ//eTkvuHdUC4ZMwz2c6uSoT2R90GWvvGnF1iEqTGyTT3xAt2Q==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.3.0 <1.9.0'
+ '@react-email/[email protected]':
+ resolution: {integrity: sha512-/pT5dBu0y1mogrfEpc002rgRcXpbShK6PFtxTVU6LZZ+bccvZPgk67HKc01lxpa1eYGQgZ6I+VQ02GRnMDclTg==}
+ engines: {node: '>=18.0.0'}
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-hhTW8pFp9PSyosYzzuUL9rdm7HF97w3OCyElufFHyUnYnKkCBbu8ne2LyF/KSdI/xZ81ubxWZs78hX4S7pLq5g==}
- engines: {node: '>=14'}
+ '@redis/[email protected]':
+ resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
peerDependencies:
- '@opentelemetry/api': ^1.0.0
+ '@redis/client': ^1.0.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-soZQdO9EAROMwa9bL2C0VLadbrfRjSA9t7g6X8sL0X1B8V59pzOayYMyTW9qTECn9uuJV98A7qOnJm6KH6yk8w==}
+ '@redis/[email protected]':
+ resolution: {integrity: sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==}
engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.10.0'
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-nda97ZwhpZKyUJTXqQuKzNhPMUgMLunbbGWn8kroBwegn+nh6OhtyGkrVQsQLNdVKJl0KeB5z0ZgeWszrYhwFw==}
- engines: {node: '>=14'}
+ '@redis/[email protected]':
+ resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==}
peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.9.0'
+ '@redis/client': ^1.0.0
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-7bRBJn3FG1l195A1m+xXRHvgzAOBsfmRi9uZ5Da18oTh7BLmNDiA8+kpk51FpTsU1PCikPVpRDNPhKVB6lyzZg==}
- engines: {node: '>=14'}
+ '@redis/[email protected]':
+ resolution: {integrity: sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==}
peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.9.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-faYX1N0gpLhej/6nyp6bgRjzAKXn5GOEMYY7YhciSfCoITAktLUtQ36d24QEWNA1/WA1y6qQunCe0OhHRkVl9g==}
- engines: {node: '>=14'}
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-TZv/1Y2QCL6sJ+X9SsPPBXe4786bc/Qsw0hQXFsNTbJzDTGGUmOAlSZ2qPiuqAd4ZheUYfD+QA20IvAjUz9Hhg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-+IUh4gAwJf49vOJM6PIjmgOapRH5zr21ZpFnNU0QZmxRi52AXVhZN7A89pKW6GAQheWnVQLD7iUN87ieYt70tw==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-16Z6kyrmszoa7J1uj1kbSAgZuk11K07yEDj6fa3I9XBf8Debi8y4K8ex94kpxbCfEraWagXji3bCWvaq3k4dRg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-22ndMDakxX+nuhAYwqsciexV8/w26JozRUV0FN9kJiqSWtA1b5dCVtlp3J6JivG5t8kDN9UF5efatNnVbqRT9Q==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-rm2HKJ9lsdoVvrbmkr9dkOzg3Uk0FksXNxvNBgrCprM1XhMoJwThI5i0h/5sJypISUAJlEeJS6gn6nROj/NpkQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.0.0
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-cyv0MwAaPF7O86x5hk3NNgenMObeejZFLJJDVuSeSMIsknlsj3oOZzRv3qSzlwYomXsICfBeFFlxwHQte5mGXQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.9.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-pkZT+iFYIZsVn6+GzM0kSX+u3MSLCY9md+lIJOoKl/P+gJFfxJte/60Usdp8Ce4rOs8GduUpSPNe1ddGyDT1sQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.10.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-ULQQtl82b673PpZc5/0EtH4V+BrwVOgKJZEB7tYZnGTG3I98tQVk89S9/JSixomDr++F4ih+LSJTCqIKBz+MQQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.4.0 <1.9.0'
- '@opentelemetry/api-logs': '>=0.39.1'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-FrAqCbbGao9iKI+Mgh+OsC9+U2YMoXnlDHe06yH7dvavCKzE3S892dGtX54+WhSFVxHR/TMRVJiK/CV93GR0TQ==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.3.0 <1.9.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-9Mb7q5ioFL4E4dDrc4wC/A3NTHDat44v4I3p2pLPSxRvqUbDIQyMVr9uK+EU69+HWhlET1VaSrRzwdckWqY15Q==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.3.0 <1.10.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-GgmNF9C+6esr8PIJxCqHw84rEOkYm6XdFWZ2+Wyc3qaUt92ACoN7uSw5iKNvaUq62W0xii1wsGxwHzyENtPP8w==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.3.0 <1.9.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-zz+N423IcySgjihl2NfjBf0qw1RWe11XIAWVrTNOSSI6dtSPJiVom2zipFB2AEEtJWpv0Iz6DY6+TjnyTV5pWg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.9.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-C8k4hnEbc5FamuZQ92nTOp8X/diCY56XUTnMiv9UTuJitCzaNNHAVsdm5+HLCdI8SLQsLWIrG38tddMxLVoftw==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.10.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-/FZX8uWaGIAwsDhqI8VvQ+qWtfMNlXjaFYGc+vmxgdRFppCSSIRwrPyIhJO1qx61okyYhoyxVEZAfoiNxrfJCg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.9.0'
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-VkliWlS4/+GHLLW7J/rVBA00uXus1SWvwFvcUDxDwmFxYfg/2VI6ekwdXS28cjI8Qz2ky2BzG8OUHo+WeYIWqw==}
- engines: {node: '>=14'}
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-ZDjMJJQRlyk8A1KZFCc+bCbsyrn1wTwdNt56F7twdfUfnHUZUq77/WfONCj8p72NZOyP7pNTdUWSTYC3GTbuuQ==}
- engines: {node: '>=14'}
-
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-nSDlnHSqzC3pXn/wZEZVLuAuJ1MYMXPBwtv2qAbCa3847SaHItdE7SzUq/Jtb0KZmh1zfAbNi3AAMjztTT4Ugg==}
- engines: {node: '>=14'}
- peerDependencies:
- '@opentelemetry/api': ^1.1.0
-
- '@pdf-lib/[email protected]':
- resolution: {integrity: sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==}
-
- '@pdf-lib/[email protected]':
- resolution: {integrity: sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==}
-
- '@pkgjs/[email protected]':
- resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
- engines: {node: '>=14'}
-
- '@prisma/[email protected]':
- resolution: {integrity: sha512-c1Sle4ji8aasMcYfBBHFM56We4ljfenVtRmS8aY06BllS7SoU6SmJBwG7vil+GHiR0Yrh+t9iBwt4AY0Jr4KNQ==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
-
- '@protobufjs/[email protected]':
- resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==}
-
- '@puppeteer/[email protected]':
- resolution: {integrity: sha512-bJ0UBsk0ESOs6RFcLXOt99a3yTDcOKlzfjad+rhFwdaG1Lu/Wzq58GHYCDTlZ9z6mldf4g+NTb+TXEfe0PpnsQ==}
- engines: {node: '>=18'}
- hasBin: true
-
- '@react-email/[email protected]':
- resolution: {integrity: sha512-/pT5dBu0y1mogrfEpc002rgRcXpbShK6PFtxTVU6LZZ+bccvZPgk67HKc01lxpa1eYGQgZ6I+VQ02GRnMDclTg==}
- engines: {node: '>=18.0.0'}
-
- '@redis/[email protected]':
- resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
- peerDependencies:
- '@redis/client': ^1.0.0
-
- '@redis/[email protected]':
- resolution: {integrity: sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==}
- engines: {node: '>=14'}
-
- '@redis/[email protected]':
- resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==}
- peerDependencies:
- '@redis/client': ^1.0.0
-
- '@redis/[email protected]':
- resolution: {integrity: sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==}
- peerDependencies:
- '@redis/client': ^1.0.0
+ '@redis/client': ^1.0.0
resolution: {integrity: sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==}
@@ -1423,10 +943,6 @@ packages:
engines: {node: '>= 10'}
hasBin: true
- '@sentry/[email protected]':
- resolution: {integrity: sha512-N9Qg4ZGxZWp8eb2eUUHVVKgjBLtFIjS805nG92s6yJmkvOpKm6mLtcUaT/iDf3Hta6nG+xRkhbE3r+Z4cbXG8w==}
- engines: {node: '>=14.18'}
-
'@sentry/[email protected]':
resolution: {integrity: sha512-g/tVmTZD4GNbLFf++hKJfBpcCAtduFEMLnbfa9iT/QEZjlmP+EzY+GsH9bafM5VsNe8DiOUp+kJKWtShzlVdBA==}
engines: {node: '>=14.18'}
@@ -1450,18 +966,10 @@ packages:
engines: {node: '>=14.18'}
hasBin: true
- '@sentry/[email protected]':
- resolution: {integrity: sha512-r63s/H5gvQnQM9tTGBXz2xErUbxZALh4e2Lg/1aHj4zIvGLBjA2z5qWsh6TEZYbpmgAyGShLDr6+rWeUVf9yBQ==}
- engines: {node: '>=14.18'}
-
'@sentry/[email protected]':
resolution: {integrity: sha512-zKmh6SWsJh630rpt7a9vP4Cm4m1C2gDTUqUiH565CajCL/4cePpNWYrNwalSqsOSL7B9OrczA1+n6a6XvND+ng==}
engines: {node: '>=14.18'}
- '@sentry/[email protected]':
- resolution: {integrity: sha512-PxV0v9VbGWH9zP37P5w2msLUFDr287nYjoY2XVF+RSolyiTs1CQNI5ZMUO3o4MsSac/dpXxjyrZXQd72t/jRYA==}
- engines: {node: '>=14.18'}
-
'@sentry/[email protected]':
resolution: {integrity: sha512-xvlPU9Hd2BlyT+FhWHGNwnxWqdVRk2AHnDtVcW4Ma0Ri5EwS+uy4Jeik5UkSv8C5RVb9VlxFmS8LN3I1MPJsLw==}
engines: {node: '>=14.18'}
@@ -1613,12 +1121,6 @@ packages:
'@tsconfig/[email protected]':
resolution: {integrity: sha512-0IKu9GHYF1NGTJiYgfWwqnOQSlnE9V9R7YohHNNf0/fj/SyOZWzdd06JFr0fLpg1Mqw0kGbYg8w5xdkSqLKM9g==}
- '@types/[email protected]':
- resolution: {integrity: sha512-Pay9fq2lM2wXPWbteBsRAGiWH2hig4ZE2asK+mm7kUzlxRTfL961rj89I6zV/E3PcIkDqyuBEcMxFT7rccugeQ==}
-
- '@types/[email protected]':
- resolution: {integrity: sha512-vBkIh9AY22kVOCEKo5CJlyCgmSWvasC+SWUxL/x/vOwRobMpI/HG1xp/Ae3AqmSiZeLUbOhW0FCD3ZjqqUxmXw==}
-
resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==}
@@ -1634,24 +1136,18 @@ packages:
resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==}
- '@types/[email protected]':
- resolution: {integrity: sha512-ZqS9JGpBxVOvsawzmVt30sP++gSQMTejCkIAQ3VdadOcRE8izTyW66hufvwLeH+YEGP6Js2AW7Gz+RMyvrEbmw==}
-
resolution: {integrity: sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w==}
resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==}
- '@types/[email protected]':
- resolution: {integrity: sha512-QVSSvno3dE0MgO76pJhmv4Qyi/j0Yk9pBp0Y7TJ2Tlj+KCgJWY6qX7nnxCOLkZ3VYRSIk1WTxCvwUSdx6CCLdg==}
-
- '@types/[email protected]':
- resolution: {integrity: sha512-40Zk8qR147RABiQ7NQnBzWzDcjKzNrntB5BAmeGCb2p/MIyOE+4BVvc17wumsUqUw00bJYqoXFHYygQnEFh4/Q==}
-
resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-qZ72SFTgUAZ5a7Tj6kf2SHLetiH5S6f8G5frB2SPQ3EyF02kxdyBFf4Tz4banE3xCgGnKgWLt//a6VuYHKYJTg==}
+
resolution: {integrity: sha512-KOzM7MhcBFlmnlr/fzISFF5vGWVSvN6fTd4T+ExOt08bA/dA5kpSzY52nMsI1KDFmUREpJelPYyuslLRSjjgCg==}
@@ -1664,9 +1160,6 @@ packages:
resolution: {integrity: sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==}
- '@types/[email protected]':
- resolution: {integrity: sha512-4+tE/lwdAahgZT1g30Jkdm9PzFRde0xwxBNUyRsCitRvCQB90iuA2uJYdUnhnANRcqGXaWOGY4FEoxeElNAK2g==}
-
resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==}
@@ -1682,21 +1175,6 @@ packages:
resolution: {integrity: sha512-eDC8bTvT/QhYdxJAulQikueigY5AsdBRH2yDKW3yveW7svY3+DzN84/2NUgkw10RTiJbWqZrTtoGVdYlvFJdLw==}
- '@types/[email protected]':
- resolution: {integrity: sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==}
-
- '@types/[email protected]':
- resolution: {integrity: sha512-4Olc63RY+MKvxMwVknCUDhRQX1pFQoBZ/lXcRLP69PQkEpze/0cr8LNqJQe5NFb/b19DWi2a5bTi2VAlQzhJuA==}
-
- '@types/[email protected]':
- resolution: {integrity: sha512-DTDUyznHGNHAl+wd1n0z1jxNajduyTh8R53xoewuerdBzGo6Ogj6F2299BFtrexJw4NtgjsI5SMPCmV9gZwGXA==}
-
- '@types/[email protected]':
- resolution: {integrity: sha512-5YUJVv6NwM1z7m6FuYpKfNLTZ932Z6EF6xy2BbtpJSyn13DKNQEkXVffFVSnJHxvwwWh2SAeumpjAYUELqgjyw==}
-
- '@types/[email protected]':
- resolution: {integrity: sha512-AM9smvZN55Gzs2wRrqeMHVP7KE8KWgCJO/XL5yCly2xF6EKa4YlbpK+cLSAH4NG/Ah64HrlegmGqW8kYws7Vxg==}
-
resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==}
@@ -1712,6 +1190,9 @@ packages:
resolution: {integrity: sha512-T2MzSGEu+ysB/FkWfqmhV3PLyQlowdptmmgD20C6QxsS8Fmv5SjpZ1ayXaEC0S21/h5UJ9iA6W/5vSNU5l00OA==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-+gbBHbNCVGGYw1S9lAIIvrHW47UYOhMIFUsJcMkMrzy1Jf0vulBN3XQIjPgnoOXveMuHnF3b57fXROnY/Or7eg==}
+
resolution: {integrity: sha512-qZAvkv1K3QbmHHFYSNRYPkRjOWRLBYrL4B9c+wG0GSVGBw0NtJwPcgx/DSddeDJvRGMHCEQ4VMEVfuJ/0gZ3XQ==}
@@ -1739,15 +1220,9 @@ packages:
resolution: {integrity: sha512-9Hp0ObzwwO57DpLFF0InUjUm/II8GmKAvzbefxQTihCb7KI6yc9yzf0nLc4mVdby5N4DRCgQM2wCup9KTieeww==}
- '@types/[email protected]':
- resolution: {integrity: sha512-TgfOX+mGY/NyNxJLIbDWrO9DjGoVSW9+aB8H2yy1fy32jsvxijhmyJI9fDFgvz3YP4lvJaq9DzdR/M1bOgVc9g==}
-
resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==}
- '@types/[email protected]':
- resolution: {integrity: sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==}
-
resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==}
@@ -1974,9 +1449,6 @@ packages:
resolution: {integrity: sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==}
engines: {node: '>=10.0.0'}
- resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==}
-
resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -1988,9 +1460,6 @@ packages:
resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==}
- resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
-
resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==}
@@ -2118,22 +1587,10 @@ packages:
resolution: {integrity: sha512-a3KdPAANPbNE4ZUv9h6LckSl9zLsYOP4MBmhIPkRaeyybt+r4UghLvq+xw/YwUcC1gqylCkL4rdVs3Lwupjm4Q==}
- resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==}
- engines: {node: '>=8'}
-
- resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==}
- engines: {node: '>=6'}
-
resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==}
engines: {node: '>=12'}
- resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==}
- engines: {node: '>=0.8'}
-
resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
engines: {node: '>=0.10.0'}
@@ -2165,6 +1622,15 @@ packages:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
+ resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==}
+
+ resolution: {integrity: sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==}
+
+ resolution: {integrity: sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==}
+
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
engines: {node: '>= 0.8'}
@@ -2323,17 +1789,10 @@ packages:
resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==}
engines: {node: '>=0.10.0'}
- resolution: {integrity: sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==}
-
resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==}
engines: {node: '>= 0.4'}
- resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==}
engines: {node: '>= 14'}
@@ -2440,6 +1899,9 @@ packages:
resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
+ resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==}
+
resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==}
engines: {node: '>= 0.8'}
@@ -2542,9 +2004,6 @@ packages:
resolution: {integrity: sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==}
engines: {node: '>= 0.10.0'}
- resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
-
resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==}
engines: {node: '>= 10.17.0'}
@@ -2597,6 +2056,9 @@ packages:
resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==}
hasBin: true
+ resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==}
+
resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==}
engines: {node: '>=4.0'}
@@ -2651,14 +2113,6 @@ packages:
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
- resolution: {integrity: sha512-DSrkyMTfAnAm4ks9Go20QGOcXEyW/NmZhvTYBU2rb4afBB393WIMQPWPEDMl/k8xqiNN9HYq2zao3oWXsdl2Tg==}
- engines: {node: '>=14'}
-
- resolution: {integrity: sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==}
- engines: {node: '>=14'}
-
resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}
engines: {node: '>= 4'}
@@ -2831,9 +2285,6 @@ packages:
resolution: {integrity: sha512-1LrZPDtW+atAxH42S6288qyDFNQ2YCty+2mxEPRtfazH6Z5QwkaBSTS2ods7hnVJioF6rkRfNoA6A/MstpFXLg==}
- resolution: {integrity: sha512-Lk+qzWmiQuRPPulGQeK5qq0v32k2bHnWrRPFgqyvhw7Kkov5L6MOLOIU3pcWeujc9W4q54Cp3Q2WV16eQkc7Bg==}
-
resolution: {integrity: sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==}
engines: {node: '>=8'}
@@ -2871,6 +2322,9 @@ packages:
resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==}
+ resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
+
resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
engines: {node: '>=8'}
@@ -2881,11 +2335,6 @@ packages:
resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}
- resolution: {integrity: sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==}
- engines: {node: '>=8'}
- hasBin: true
-
resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
engines: {node: '>=0.10.0'}
@@ -2902,10 +2351,6 @@ packages:
resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
engines: {node: '>=0.10.0'}
- resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
engines: {node: '>=0.12.0'}
@@ -2922,14 +2367,6 @@ packages:
resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
engines: {node: '>=8'}
- resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==}
- engines: {node: '>=10'}
-
- resolution: {integrity: sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==}
@@ -3138,9 +2575,6 @@ packages:
engines: {node: '>=4'}
hasBin: true
- resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==}
-
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
@@ -3151,9 +2585,6 @@ packages:
resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
- resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
-
resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
engines: {node: '>=6'}
@@ -3184,6 +2615,9 @@ packages:
resolution: {integrity: sha512-KCsuJ2gM58n6bNdR2Z7gqsh/3TchxxQFbVgax2/UvAjRTgwNSYAJDx9E3jrkBP4jEDHWRCfE47Y2OG+/fiSvEw==}
+ resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==}
+
resolution: {integrity: sha512-kb2IOMA71xH8e6EXFg0l4S+QSMC/c796pj1+7mPBkR91HHwoyHZhFRrBaZv4tV+Td+Ba91J2uEDBmySklZLpNQ==}
engines: {node: '>=18'}
@@ -3378,24 +2812,12 @@ packages:
resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==}
engines: {node: '>=8'}
- resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==}
-
resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
- resolution: {integrity: sha512-3/Qptq2vr7WeJbB4KHUSKlq8Pl7ASXi3UG6CMbBm8WRtXi8+GHm7mKaU3urfpSEzWe2wCIChs6/sdocUsTKJiA==}
-
- resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==}
-
- resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==}
-
resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==}
@@ -3408,14 +2830,14 @@ packages:
resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
- resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==}
- engines: {node: '>=10'}
-
resolution: {integrity: sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==}
engines: {node: '>= 12.0.0'}
+ resolution: {integrity: sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==}
+ engines: {node: '>= 12.0.0'}
+
resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==}
engines: {node: '>= 0.6.0'}
@@ -3423,9 +2845,6 @@ packages:
resolution: {integrity: sha512-HMzjh75OR5EVY7Be4Rw8TcDTIY5UPsrXF1HvQ6EzDi21x5cQcDzi4Ts0Y/ruPCbxKY2KG17YjeeTzErXFewFBg==}
- resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==}
-
resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}
hasBin: true
@@ -3467,6 +2886,11 @@ packages:
engines: {node: '>=12.0.0'}
hasBin: true
+ resolution: {integrity: sha512-f3r0yqpz31VXiDB/wj9GaOB0a2PRLQl6vJmXiFrniNwjkKdvakqJRULhjFKJpxOchlCRiG5fcacoUZY5Xa6PEQ==}
+ engines: {node: '>= 18'}
+ hasBin: true
+
resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==}
@@ -3736,14 +3160,13 @@ packages:
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
+ resolution: {integrity: sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==}
+
resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==}
engines: {node: '>=6'}
- resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==}
- engines: {node: '>=12'}
-
resolution: {integrity: sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==}
@@ -3772,10 +3195,6 @@ packages:
resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==}
- resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==}
- engines: {node: '>=10'}
-
resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==}
engines: {node: '>=4'}
@@ -3919,9 +3338,6 @@ packages:
resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}
engines: {node: '>=8.6'}
- resolution: {integrity: sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q==}
-
resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==}
engines: {node: '>= 6'}
@@ -3986,10 +3402,6 @@ packages:
resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==}
- resolution: {integrity: sha512-RXyHaACeqXeqAKGLDl68rQKbmObRsTIn4TYVUUug1KfS47YWCo5MacGITEryugIgZqORCvJWEk4l449POg5Txg==}
- engines: {node: '>=12.0.0'}
-
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
engines: {node: '>= 0.10'}
@@ -4139,10 +3551,6 @@ packages:
resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}
hasBin: true
- resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
engines: {node: '>= 4'}
@@ -4213,10 +3621,6 @@ packages:
resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==}
engines: {node: '>= 0.8.0'}
- resolution: {integrity: sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==}
- engines: {node: '>=10'}
-
resolution: {integrity: sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==}
engines: {node: '>= 0.8.0'}
@@ -4259,6 +3663,9 @@ packages:
resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
engines: {node: '>=14'}
+ resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
+
resolution: {integrity: sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==}
engines: {node: '>=8.10.0'}
@@ -4422,6 +3829,9 @@ packages:
resolution: {integrity: sha512-TmLJNj6UgX8xcUZo4UDStGQtDiTzF7BzWlzn9g7UWrjkpHr5uJTK1ld16wZ3LXb2vb6jH8qU89dW5whuMdXYdw==}
+ resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==}
+
resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}
@@ -4509,10 +3919,6 @@ packages:
resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==}
engines: {node: '>=4'}
- resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==}
- engines: {node: '>=10'}
-
resolution: {integrity: sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==}
engines: {node: '>=10'}
@@ -4615,9 +4021,6 @@ packages:
resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==}
- resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==}
-
resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
engines: {node: '>= 8'}
@@ -4652,8 +4055,12 @@ packages:
engines: {node: '>= 8'}
hasBin: true
- resolution: {integrity: sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==}
+ resolution: {integrity: sha512-qxSTKswC6llEMZKgCQdaWgDuMJQnhuvF5f2Nk3SNXc4byfQ+voo2mX1Px9dkNOuR8p0KAjfPG29PuYUSIb+vSA==}
+ engines: {node: '>= 12.0.0'}
+
+ resolution: {integrity: sha512-CO8cdpBB2yqzEf8v895L+GNKYJiEq8eKlHU38af3snQBQ+sdAIUepjMSguOIJC7ICbzm0ZI+Af2If4vIJrtmOg==}
engines: {node: '>= 12.0.0'}
@@ -5038,9 +4445,15 @@ snapshots:
dependencies:
'@jridgewell/trace-mapping': 0.3.9
+ dependencies:
+ colorspace: 1.1.4
+ enabled: 2.0.0
+ kuler: 2.0.0
+
'@devil7softwares/[email protected]': {}
- '@dqbd/[email protected]': {}
+ '@dqbd/[email protected]': {}
'@flydotio/[email protected]':
dependencies:
@@ -5050,83 +4463,6 @@ snapshots:
shell-quote: 1.8.1
yargs: 17.7.2
- dependencies:
- '@grpc/proto-loader': 0.7.13
- '@js-sdsl/ordered-map': 4.4.2
-
- dependencies:
- lodash.camelcase: 4.3.0
- long: 5.2.3
- protobufjs: 7.3.2
- yargs: 17.7.2
-
- '@hyperdx/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@hyperdx/instrumentation-sentry-node': 0.1.0(@opentelemetry/[email protected])
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@sentry/core': 8.13.0
- '@sentry/types': 8.13.0
- '@sentry/utils': 8.13.0
- json-stringify-safe: 5.0.1
- shimmer: 1.2.1
- tslib: 2.6.3
- transitivePeerDependencies:
- - supports-color
-
- '@hyperdx/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- json-stringify-safe: 5.0.1
- shimmer: 1.2.1
- tslib: 2.6.3
- transitivePeerDependencies:
- - supports-color
-
- '@hyperdx/[email protected]':
- dependencies:
- '@hyperdx/instrumentation-exception': 0.1.0(@opentelemetry/[email protected])
- '@hyperdx/instrumentation-sentry-node': 0.1.0(@opentelemetry/[email protected])
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/api-logs': 0.51.1
- '@opentelemetry/auto-instrumentations-node': 0.46.1(@opentelemetry/[email protected])
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-logs-otlp-http': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-metrics-otlp-proto': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-trace-otlp-proto': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-http': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-runtime-node': 0.4.0(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-logs': 0.51.1(@opentelemetry/[email protected])(@opentelemetry/[email protected])
- '@opentelemetry/sdk-metrics': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-node': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- cli-spinners: 2.9.2
- json-stringify-safe: 5.0.1
- lodash.isobject: 3.0.2
- lodash.isplainobject: 4.0.6
- lodash.isstring: 4.0.1
- node-fetch: 2.7.0
- open: 8.4.2
- ora: 5.4.1
- pino-abstract-transport: 1.2.0
- semver: 7.6.2
- shimmer: 1.2.1
- tslib: 2.6.3
- winston-transport: 4.7.0
- transitivePeerDependencies:
- - encoding
- - supports-color
-
'@ioredis/[email protected]': {}
'@isaacs/[email protected]':
@@ -5336,8 +4672,6 @@ snapshots:
'@jridgewell/resolve-uri': 3.1.2
'@jridgewell/sourcemap-codec': 1.4.15
- '@js-sdsl/[email protected]': {}
-
dependencies:
ansi-styles: 5.2.0
@@ -5375,38 +4709,11 @@ snapshots:
- langchain
- openai
- '@logtail/[email protected]':
- dependencies:
- '@logtail/tools': 0.4.21
- '@logtail/types': 0.4.20
- serialize-error: 8.1.0
-
- '@logtail/[email protected]':
- dependencies:
- '@logtail/core': 0.4.21
- '@logtail/types': 0.4.20
- '@msgpack/msgpack': 2.8.0
- '@types/stack-trace': 0.0.29
- cross-fetch: 3.1.8
- minimatch: 3.1.2
- serialize-error: 8.1.0
- stack-trace: 0.0.10
- transitivePeerDependencies:
- - encoding
+ '@mixmark-io/[email protected]': {}
- '@logtail/[email protected]':
+ '@mongodb-js/[email protected]':
dependencies:
- '@logtail/types': 0.4.20
-
- '@logtail/[email protected]': {}
-
- '@mixmark-io/[email protected]': {}
-
- '@mongodb-js/[email protected]':
- dependencies:
- sparse-bitfield: 3.0.3
-
- '@msgpack/[email protected]': {}
+ sparse-bitfield: 3.0.3
'@msgpackr-extract/[email protected]':
optional: true
@@ -5434,210 +4741,21 @@ snapshots:
'@one-ini/[email protected]': {}
- '@opentelemetry/[email protected]':
- dependencies:
- '@opentelemetry/api': 1.9.0
-
'@opentelemetry/[email protected]':
dependencies:
'@opentelemetry/api': 1.9.0
'@opentelemetry/[email protected]': {}
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-amqplib': 0.37.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-aws-lambda': 0.41.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-aws-sdk': 0.41.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-bunyan': 0.38.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-cassandra-driver': 0.38.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-connect': 0.36.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-cucumber': 0.6.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-dataloader': 0.9.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-dns': 0.36.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-express': 0.39.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-fastify': 0.36.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-fs': 0.12.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-generic-pool': 0.36.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-graphql': 0.40.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-grpc': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-hapi': 0.38.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-http': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-ioredis': 0.40.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-knex': 0.36.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-koa': 0.40.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-lru-memoizer': 0.37.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-memcached': 0.36.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-mongodb': 0.43.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-mongoose': 0.38.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-mysql': 0.38.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-mysql2': 0.38.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-nestjs-core': 0.37.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-net': 0.36.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-pg': 0.41.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-pino': 0.39.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-redis': 0.39.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-redis-4': 0.39.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-restify': 0.38.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-router': 0.37.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-socket.io': 0.39.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-tedious': 0.10.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-undici': 0.2.0(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation-winston': 0.37.0(@opentelemetry/[email protected])
- '@opentelemetry/resource-detector-alibaba-cloud': 0.28.10(@opentelemetry/[email protected])
- '@opentelemetry/resource-detector-aws': 1.5.1(@opentelemetry/[email protected])
- '@opentelemetry/resource-detector-azure': 0.2.9(@opentelemetry/[email protected])
- '@opentelemetry/resource-detector-container': 0.3.11(@opentelemetry/[email protected])
- '@opentelemetry/resource-detector-gcp': 0.29.10(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-node': 0.51.1(@opentelemetry/[email protected])
- transitivePeerDependencies:
- - encoding
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/semantic-conventions': 1.24.1
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
'@opentelemetry/semantic-conventions': 1.25.1
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/api-logs': 0.51.1
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-transformer': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-logs': 0.51.1(@opentelemetry/[email protected])(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-transformer': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-metrics-otlp-http': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-proto-exporter-base': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-transformer': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@grpc/grpc-js': 1.10.10
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-grpc-exporter-base': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-transformer': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-transformer': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-proto-exporter-base': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-transformer': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.24.1
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/propagator-aws-xray': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@types/aws-lambda': 8.10.122
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/propagation-utils': 0.30.10(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/api-logs': 0.51.1
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@types/bunyan': 1.8.9
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@types/connect': 3.4.36
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5648,39 +4766,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- semver: 7.6.2
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5690,15 +4775,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5708,14 +4784,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5724,21 +4792,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5746,23 +4799,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.24.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5772,16 +4808,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.24.1
- semver: 7.6.2
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5792,15 +4818,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/redis-common': 0.36.2
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5810,25 +4827,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@types/koa': 2.14.0
- '@types/koa__router': 12.0.3
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5838,31 +4836,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@types/memcached': 2.2.10
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-metrics': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5872,15 +4845,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -5890,176 +4854,49 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@opentelemetry/sql-common': 0.40.1(@opentelemetry/[email protected])
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
'@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@opentelemetry/sql-common': 0.40.1(@opentelemetry/[email protected])
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@types/mysql': 2.15.22
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@types/mysql': 2.15.22
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@opentelemetry/sql-common': 0.40.1(@opentelemetry/[email protected])
- '@types/pg': 8.6.1
- '@types/pg-pool': 2.0.4
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- '@opentelemetry/sql-common': 0.40.1(@opentelemetry/[email protected])
- '@types/pg': 8.6.1
- '@types/pg-pool': 2.0.4
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/redis-common': 0.36.2
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
- '@opentelemetry/redis-common': 0.36.2
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/redis-common': 0.36.2
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.25.1
+ '@opentelemetry/sql-common': 0.40.1(@opentelemetry/[email protected])
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
'@opentelemetry/semantic-conventions': 1.25.1
+ '@types/mysql': 2.15.22
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
'@opentelemetry/semantic-conventions': 1.25.1
- '@types/tedious': 4.0.14
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.25.1
+ '@opentelemetry/sql-common': 0.40.1(@opentelemetry/[email protected])
+ '@types/pg': 8.6.1
+ '@types/pg-pool': 2.0.4
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
- '@opentelemetry/api-logs': 0.51.1
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/[email protected])
+ '@opentelemetry/redis-common': 0.36.2
+ '@opentelemetry/semantic-conventions': 1.25.1
transitivePeerDependencies:
- supports-color
@@ -6075,18 +4912,6 @@ snapshots:
- supports-color
optional: true
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/api-logs': 0.51.1
- '@types/shimmer': 1.0.5
- import-in-the-middle: 1.7.4
- require-in-the-middle: 7.3.0
- semver: 7.6.2
- shimmer: 1.2.1
- transitivePeerDependencies:
- - supports-color
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -6099,119 +4924,14 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@grpc/grpc-js': 1.10.10
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/[email protected])
- protobufjs: 7.3.2
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/[email protected])
- protobufjs: 7.3.2
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/api-logs': 0.51.1
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-logs': 0.51.1(@opentelemetry/[email protected])(@opentelemetry/[email protected])
- '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
-
'@opentelemetry/[email protected]': {}
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.25.1
- gcp-metadata: 6.1.0
- transitivePeerDependencies:
- - encoding
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.24.1
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
'@opentelemetry/core': 1.25.1(@opentelemetry/[email protected])
'@opentelemetry/semantic-conventions': 1.25.1
- '@opentelemetry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/api-logs': 0.51.1
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- lodash.merge: 4.6.2
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -6219,32 +4939,6 @@ snapshots:
'@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
lodash.merge: 4.6.2
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/api-logs': 0.51.1
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-trace-otlp-grpc': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-trace-otlp-http': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-trace-otlp-proto': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-zipkin': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/instrumentation': 0.51.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-logs': 0.51.1(@opentelemetry/[email protected])(@opentelemetry/[email protected])
- '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-node': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.24.1
- transitivePeerDependencies:
- - supports-color
-
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/resources': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions': 1.24.1
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.0
@@ -6252,18 +4946,6 @@ snapshots:
'@opentelemetry/resources': 1.25.1(@opentelemetry/[email protected])
'@opentelemetry/semantic-conventions': 1.25.1
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.0
- '@opentelemetry/context-async-hooks': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/core': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/propagator-b3': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/propagator-jaeger': 1.24.1(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/[email protected])
- semver: 7.6.2
-
- '@opentelemetry/[email protected]': {}
-
'@opentelemetry/[email protected]': {}
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
@@ -6290,29 +4972,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@protobufjs/[email protected]': {}
-
- '@protobufjs/[email protected]': {}
-
- '@protobufjs/[email protected]': {}
-
- '@protobufjs/[email protected]': {}
-
- '@protobufjs/[email protected]':
- dependencies:
- '@protobufjs/aspromise': 1.1.2
- '@protobufjs/inquire': 1.1.0
-
- '@protobufjs/[email protected]': {}
-
- '@protobufjs/[email protected]': {}
-
- '@protobufjs/[email protected]': {}
-
- '@protobufjs/[email protected]': {}
-
- '@protobufjs/[email protected]': {}
-
'@puppeteer/[email protected]':
dependencies:
debug: 4.3.4
@@ -6405,11 +5064,6 @@ snapshots:
- encoding
- supports-color
- '@sentry/[email protected]':
- dependencies:
- '@sentry/types': 8.13.0
- '@sentry/utils': 8.13.0
-
'@sentry/[email protected]':
dependencies:
'@sentry/types': 8.26.0
@@ -6473,14 +5127,8 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@sentry/[email protected]': {}
-
'@sentry/[email protected]': {}
- '@sentry/[email protected]':
- dependencies:
- '@sentry/types': 8.13.0
-
'@sentry/[email protected]':
dependencies:
'@sentry/types': 8.26.0
@@ -6631,12 +5279,6 @@ snapshots:
'@tsconfig/[email protected]': {}
- '@types/[email protected]':
- dependencies:
- '@types/node': 20.14.1
-
- '@types/[email protected]': {}
-
dependencies:
'@babel/parser': 7.24.6
@@ -6663,10 +5305,6 @@ snapshots:
'@types/connect': 3.4.38
'@types/node': 20.14.1
- '@types/[email protected]':
- dependencies:
- '@types/node': 20.14.1
-
dependencies:
'@types/node': 20.14.1
@@ -6675,19 +5313,12 @@ snapshots:
dependencies:
'@types/node': 20.14.1
- '@types/[email protected]': {}
-
- '@types/[email protected]':
- dependencies:
- '@types/connect': 3.4.38
- '@types/express': 4.17.21
- '@types/keygrip': 1.0.6
- '@types/node': 20.14.1
-
dependencies:
'@types/node': 20.14.1
+ '@types/[email protected]': {}
+
dependencies:
'@types/node': 20.14.1
@@ -6712,8 +5343,6 @@ snapshots:
dependencies:
'@types/node': 20.14.1
- '@types/[email protected]': {}
-
'@types/[email protected]': {}
'@types/[email protected]': {}
@@ -6731,31 +5360,6 @@ snapshots:
expect: 29.7.0
pretty-format: 29.7.0
- '@types/[email protected]': {}
-
- '@types/[email protected]':
- dependencies:
- '@types/koa': 2.14.0
-
- '@types/[email protected]':
- dependencies:
- '@types/accepts': 1.3.7
- '@types/content-disposition': 0.5.8
- '@types/cookies': 0.9.0
- '@types/http-assert': 1.5.5
- '@types/http-errors': 2.0.4
- '@types/keygrip': 1.0.6
- '@types/koa-compose': 3.2.8
- '@types/node': 20.14.1
-
- '@types/[email protected]':
- dependencies:
- '@types/koa': 2.14.0
-
- '@types/[email protected]':
- dependencies:
- '@types/node': 20.14.1
-
'@types/[email protected]': {}
@@ -6775,6 +5379,8 @@ snapshots:
dependencies:
undici-types: 5.26.5
+ '@types/[email protected]': {}
+
dependencies:
'@types/pg': 8.6.1
@@ -6806,14 +5412,8 @@ snapshots:
'@types/[email protected]': {}
- '@types/[email protected]': {}
-
'@types/[email protected]': {}
- '@types/[email protected]':
- dependencies:
- '@types/node': 20.14.1
-
'@types/[email protected]': {}
'@types/[email protected]': {}
@@ -7064,8 +5664,6 @@ snapshots:
-
dependencies:
cmd-shim: 6.0.3
@@ -7077,12 +5675,6 @@ snapshots:
- dependencies:
- buffer: 5.7.1
- inherits: 2.0.4
- readable-stream: 3.6.2
-
@@ -7243,20 +5835,12 @@ snapshots:
- dependencies:
- restore-cursor: 3.1.0
-
-
dependencies:
string-width: 4.2.3
strip-ansi: 6.0.1
wrap-ansi: 7.0.0
-
@@ -7279,6 +5863,21 @@ snapshots:
+ dependencies:
+ color-name: 1.1.4
+ simple-swizzle: 0.2.2
+
+ dependencies:
+ color-convert: 1.9.3
+ color-string: 1.9.1
+
+ dependencies:
+ color: 3.2.1
+ text-hex: 1.0.0
+
dependencies:
delayed-stream: 1.0.0
@@ -7407,18 +6006,12 @@ snapshots:
- dependencies:
- clone: 1.0.4
-
dependencies:
es-define-property: 1.0.0
es-errors: 1.3.0
gopd: 1.0.1
-
dependencies:
ast-types: 0.13.4
@@ -7508,6 +6101,8 @@ snapshots:
+
@@ -7628,8 +6223,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
-
dependencies:
debug: 4.3.4
@@ -7692,6 +6285,8 @@ snapshots:
+
@@ -7740,25 +6335,6 @@ snapshots:
- dependencies:
- extend: 3.0.2
- https-proxy-agent: 7.0.5
- is-stream: 2.0.1
- node-fetch: 2.7.0
- uuid: 10.0.0
- transitivePeerDependencies:
- - encoding
- - supports-color
-
- dependencies:
- gaxios: 6.7.0
- json-bigint: 1.0.0
- transitivePeerDependencies:
- - encoding
- - supports-color
-
@@ -7978,13 +6554,6 @@ snapshots:
module-details-from-path: 1.0.3
optional: true
- dependencies:
- acorn: 8.12.0
- acorn-import-attributes: 1.9.5([email protected])
- cjs-module-lexer: 1.3.1
- module-details-from-path: 1.0.3
-
dependencies:
pkg-dir: 4.2.0
@@ -8026,6 +6595,8 @@ snapshots:
+
dependencies:
binary-extensions: 2.3.0
@@ -8036,8 +6607,6 @@ snapshots:
dependencies:
hasown: 2.0.2
-
@@ -8048,8 +6617,6 @@ snapshots:
dependencies:
is-extglob: 2.1.1
-
@@ -8058,12 +6625,6 @@ snapshots:
-
- dependencies:
- is-docker: 2.2.1
-
@@ -8476,18 +7037,12 @@ snapshots:
- dependencies:
- bignumber.js: 9.1.2
-
-
@@ -8513,6 +7068,8 @@ snapshots:
+
dependencies:
@@ -8579,18 +7136,10 @@ snapshots:
dependencies:
p-locate: 4.1.0
-
-
-
-
@@ -8599,12 +7148,16 @@ snapshots:
dependencies:
- chalk: 4.1.2
- is-unicode-supported: 0.1.0
+ '@colors/colors': 1.6.0
+ '@types/triple-beam': 1.3.5
+ fecha: 4.2.3
+ ms: 2.1.3
+ safe-stable-stringify: 2.4.3
+ triple-beam: 1.4.1
dependencies:
'@colors/colors': 1.6.0
'@types/triple-beam': 1.3.5
@@ -8621,8 +7174,6 @@ snapshots:
transitivePeerDependencies:
- encoding
-
dependencies:
js-tokens: 4.0.0
@@ -8670,6 +7221,8 @@ snapshots:
underscore: 1.13.6
xmlbuilder: 10.1.1
+
dependencies:
charenc: 0.0.2
@@ -8937,15 +7490,13 @@ snapshots:
dependencies:
wrappy: 1.0.2
dependencies:
- mimic-fn: 2.1.0
+ fn.name: 1.1.0
dependencies:
- define-lazy-prop: 2.0.0
- is-docker: 2.2.1
- is-wsl: 2.2.0
+ mimic-fn: 2.1.0
dependencies:
@@ -8985,18 +7536,6 @@ snapshots:
- dependencies:
- bl: 4.1.0
- chalk: 4.1.2
- cli-cursor: 3.1.0
- cli-spinners: 2.9.2
- is-interactive: 1.0.0
- is-unicode-supported: 0.1.0
- log-symbols: 4.1.0
- strip-ansi: 6.0.1
- wcwidth: 1.0.1
-
@@ -9148,11 +7687,6 @@ snapshots:
- dependencies:
- readable-stream: 4.5.2
- split2: 4.2.0
-
@@ -9221,21 +7755,6 @@ snapshots:
- dependencies:
- '@protobufjs/aspromise': 1.1.2
- '@protobufjs/base64': 1.1.2
- '@protobufjs/codegen': 2.0.4
- '@protobufjs/eventemitter': 1.1.0
- '@protobufjs/fetch': 1.1.0
- '@protobufjs/float': 1.0.2
- '@protobufjs/inquire': 1.1.0
- '@protobufjs/path': 1.1.2
- '@protobufjs/pool': 1.1.0
- '@protobufjs/utf8': 1.1.0
- '@types/node': 20.14.1
- long: 5.2.3
-
dependencies:
forwarded: 0.2.0
@@ -9417,11 +7936,6 @@ snapshots:
path-parse: 1.0.7
supports-preserve-symlinks-flag: 1.0.0
- dependencies:
- onetime: 5.1.2
- signal-exit: 3.0.7
-
@@ -9490,10 +8004,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- dependencies:
- type-fest: 0.20.2
-
dependencies:
encodeurl: 1.0.2
@@ -9539,6 +8049,10 @@ snapshots:
+ dependencies:
+ is-arrayish: 0.3.2
+
dependencies:
semver: 7.0.0
@@ -9725,6 +8239,8 @@ snapshots:
dependencies:
b4a: 1.6.6
+
@@ -9795,8 +8311,6 @@ snapshots:
-
@@ -9880,10 +8394,6 @@ snapshots:
dependencies:
makeerror: 1.0.12
- dependencies:
- defaults: 1.0.4
-
@@ -9912,11 +8422,25 @@ snapshots:
dependencies:
isexe: 2.0.0
+ dependencies:
+ logform: 2.6.1
+ readable-stream: 4.5.2
+ triple-beam: 1.4.1
+
dependencies:
+ '@colors/colors': 1.6.0
+ '@dabh/diagnostics': 2.0.3
+ async: 3.2.5
+ is-stream: 2.0.1
logform: 2.6.0
+ one-time: 1.0.0
readable-stream: 3.6.2
+ safe-stable-stringify: 2.4.3
+ stack-trace: 0.0.10
triple-beam: 1.4.1
+ winston-transport: 4.8.0
diff --git a/apps/api/requests.http b/apps/api/requests.http
index 3e7bd2b78..809bae7bb 100644
--- a/apps/api/requests.http
+++ b/apps/api/requests.http
@@ -1,15 +1,15 @@
### Crawl Website
POST http://localhost:3002/v0/scrape HTTP/1.1
-Authorization: Bearer fc-
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
content-type: application/json
{
- "url":"corterix.com"
+ "url":"firecrawl.dev"
}
### Check Job Status
GET http://localhost:3002/v1/crawl/1dd0f924-a36f-4b96-94ea-32ed954dac67 HTTP/1.1
-Authorization: Bearer fc-
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
### Check Job Status
@@ -18,7 +18,7 @@ GET http://localhost:3002/v0/jobs/active HTTP/1.1
### Scrape Website
POST http://localhost:3002/v0/crawl HTTP/1.1
-Authorization: Bearer fc-
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
content-type: application/json
{
@@ -45,7 +45,7 @@ content-type: application/json
### Scrape Website
POST http://localhost:3002/v0/scrape HTTP/1.1
-Authorization: Bearer
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
content-type: application/json
{
@@ -56,12 +56,12 @@ content-type: application/json
### Check Job Status
GET http://localhost:3002/v0/crawl/status/a6053912-d602-4709-841f-3d2cb46fea0a HTTP/1.1
-Authorization: Bearer
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
### Get Job Result
POST https://api.firecrawl.dev/v0/crawl HTTP/1.1
-Authorization: Bearer
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
content-type: application/json
{
@@ -70,7 +70,7 @@ content-type: application/json
### Check Job Status
GET https://api.firecrawl.dev/v0/crawl/status/cfcb71ac-23a3-4da5-bd85-d4e58b871d66
-Authorization: Bearer
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
### Get Active Jobs Count
GET http://localhost:3002/serverHealthCheck
diff --git a/apps/api/sharedLibs/go-html-to-md/.gitignore b/apps/api/sharedLibs/go-html-to-md/.gitignore
new file mode 100644
index 000000000..bdab47c67
--- /dev/null
+++ b/apps/api/sharedLibs/go-html-to-md/.gitignore
@@ -0,0 +1,2 @@
+html-to-markdown.so
+html-to-markdown.h
\ No newline at end of file
diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
index b1708abc2..dec77131b 100644
--- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
@@ -844,7 +844,7 @@ describe("E2E Tests for API Routes", () => {
expect(crawlInitResponse.statusCode).toBe(200);
expect(crawlInitResponse.body).toHaveProperty("jobId");
- let crawlStatus: string;
+ let crawlStatus: string = "scraping";
let crawlData = [];
while (crawlStatus !== "completed") {
const statusResponse = await request(TEST_URL)
diff --git a/apps/api/src/__tests__/e2e_noAuth/index.test.ts b/apps/api/src/__tests__/e2e_noAuth/index.test.ts
index acb22780b..83f676b8b 100644
--- a/apps/api/src/__tests__/e2e_noAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_noAuth/index.test.ts
@@ -20,7 +20,6 @@ describe("E2E Tests for API Routes with No Authentication", () => {
process.env.SCRAPING_BEE_API_KEY = "";
process.env.OPENAI_API_KEY = "";
process.env.BULL_AUTH_KEY = "";
- process.env.LOGTAIL_KEY = "";
process.env.PLAYWRIGHT_MICROSERVICE_URL = "";
process.env.LLAMAPARSE_API_KEY = "";
process.env.TEST_API_KEY = "";
diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
index a41634726..8c3d1731c 100644
--- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
@@ -1,7 +1,7 @@
import request from "supertest";
import { configDotenv } from "dotenv";
import {
- ScrapeRequest,
+ ScrapeRequestInput,
ScrapeResponseRequestTest,
} from "../../controllers/v1/types";
@@ -44,7 +44,7 @@ describe("E2E Tests for v1 API Routes", () => {
});
it.concurrent("should throw error for blocklisted URL", async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://facebook.com/fake-test",
};
@@ -73,7 +73,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent(
"should return a successful response with a valid API key",
async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://roastmywebsite.ai",
};
@@ -125,7 +125,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent(
"should return a successful response with a valid API key",
async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://arxiv.org/abs/2410.04840",
};
@@ -167,7 +167,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent(
"should return a successful response with a valid API key and includeHtml set to true",
async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://roastmywebsite.ai",
formats: ["markdown", "html"],
};
@@ -194,7 +194,7 @@ describe("E2E Tests for v1 API Routes", () => {
30000
);
it.concurrent('should return a successful response for a valid scrape with PDF file', async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://arxiv.org/pdf/astro-ph/9301001.pdf"
// formats: ["markdown", "html"],
};
@@ -217,7 +217,7 @@ describe("E2E Tests for v1 API Routes", () => {
}, 60000);
it.concurrent('should return a successful response for a valid scrape with PDF file without explicit .pdf extension', async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://arxiv.org/pdf/astro-ph/9301001"
};
const response: ScrapeResponseRequestTest = await request(TEST_URL)
@@ -240,7 +240,7 @@ describe("E2E Tests for v1 API Routes", () => {
}, 60000);
it.concurrent("should return a successful response with a valid API key with removeTags option", async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://www.scrapethissite.com/",
onlyMainContent: false // default is true
};
@@ -261,7 +261,7 @@ describe("E2E Tests for v1 API Routes", () => {
expect(responseWithoutRemoveTags.body.data.markdown).toContain("[FAQ](/faq/)"); // .nav
expect(responseWithoutRemoveTags.body.data.markdown).toContain("Hartley Brody 2023"); // #footer
- const scrapeRequestWithRemoveTags: ScrapeRequest = {
+ const scrapeRequestWithRemoveTags: ScrapeRequestInput = {
url: "https://www.scrapethissite.com/",
excludeTags: ['.nav', '#footer', 'strong'],
onlyMainContent: false // default is true
@@ -407,7 +407,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent(
"should return a successful response with a valid API key and includeHtml set to true",
async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://roastmywebsite.ai",
formats: ["html","rawHtml"],
};
@@ -438,7 +438,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent(
"should return a successful response with waitFor",
async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://ycombinator.com/companies",
formats: ["markdown"],
waitFor: 8000
@@ -471,7 +471,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent(
"should return a successful response with a valid links on page",
async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://roastmywebsite.ai",
formats: ["links"],
};
@@ -672,7 +672,7 @@ describe("POST /v1/crawl", () => {
});
it.concurrent("should throw error for blocklisted URL", async () => {
- const scrapeRequest: ScrapeRequest = {
+ const scrapeRequest: ScrapeRequestInput = {
url: "https://facebook.com/fake-test",
};
diff --git a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
new file mode 100644
index 000000000..5c7feb1f1
--- /dev/null
+++ b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
@@ -0,0 +1,603 @@
+import request from "supertest";
+import { configDotenv } from "dotenv";
+import {
+ ScrapeRequest,
+ ScrapeResponseRequestTest,
+} from "../../controllers/v1/types";
+
+configDotenv();
+const FIRECRAWL_API_URL = "http://127.0.0.1:3002";
+const E2E_TEST_SERVER_URL = "http://firecrawl-e2e-test.vercel.app"; // @rafaelsideguide/firecrawl-e2e-test
+
+describe("E2E Tests for v1 API Routes", () => {
+
+ it.concurrent('should return a successful response for a scrape with 403 page', async () => {
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post('/v1/scrape')
+ .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
+ .set('Content-Type', 'application/json')
+ .send({ url: 'https://httpstat.us/403' });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty('data');
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ expect(response.body.data).toHaveProperty('markdown');
+ expect(response.body.data).toHaveProperty('metadata');
+ expect(response.body.data.metadata.statusCode).toBe(403);
+ }, 30000);
+
+ it.concurrent("should handle 'formats:markdown (default)' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty("data");
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+ expect(response.body.data).toHaveProperty("markdown");
+
+ expect(response.body.data.markdown).toContain("This page is used for end-to-end (e2e) testing with Firecrawl.");
+ expect(response.body.data.markdown).toContain("Content with id #content-1");
+ // expect(response.body.data.markdown).toContain("Loading...");
+ expect(response.body.data.markdown).toContain("Click me!");
+ expect(response.body.data.markdown).toContain("Power your AI apps with clean data crawled from any website. It's also open-source."); // firecrawl.dev inside an iframe
+ expect(response.body.data.markdown).toContain("This content loads only when you see it. Don't blink! πŸ‘Ό"); // the browser always scroll to the bottom
+ expect(response.body.data.markdown).not.toContain("Header"); // Only main content is returned by default
+ expect(response.body.data.markdown).not.toContain("footer"); // Only main content is returned by default
+ expect(response.body.data.markdown).not.toContain("This content is only visible on mobile");
+ },
+ 30000);
+
+ it.concurrent("should handle 'formats:html' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ formats: ["html"]
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty("data");
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+
+ expect(response.body.data).not.toHaveProperty("markdown");
+ expect(response.body.data).toHaveProperty("html");
+
+ expect(response.body.data.html).not.toContain("<header class=\"row-start-1\" style=\"\">Header</header>");
+ expect(response.body.data.html).toContain("<p style=\"\">This page is used for end-to-end (e2e) testing with Firecrawl.</p>");
+ },
+ 30000);
+
+ it.concurrent("should handle 'rawHtml' in 'formats' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ formats: ["rawHtml"]
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty("data");
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+ expect(response.body.data).not.toHaveProperty("markdown");
+ expect(response.body.data).toHaveProperty("rawHtml");
+
+ expect(response.body.data.rawHtml).toContain(">This page is used for end-to-end (e2e) testing with Firecrawl.</p>");
+ expect(response.body.data.rawHtml).toContain(">Header</header>");
+ },
+ 30000);
+
+ // - TODO: tests for links
+ // - TODO: tests for screenshot
+ // - TODO: tests for screenshot@fullPage
+
+ it.concurrent("should handle 'headers' parameter correctly", async () => {
+ // @ts-ignore
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ headers: { "e2e-header-test": "firecrawl" }
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty("data");
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+ expect(response.body.data.markdown).toContain("e2e-header-test: firecrawl");
+ }, 30000);
+
+ it.concurrent("should handle 'includeTags' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ includeTags: ['#content-1']
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty("data");
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+ expect(response.body.data.markdown).not.toContain("<p>This page is used for end-to-end (e2e) testing with Firecrawl.</p>");
+ expect(response.body.data.markdown).toContain("Content with id #content-1");
+ },
+ 30000);
+
+ it.concurrent("should handle 'excludeTags' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ excludeTags: ['#content-1']
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty("data");
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+ expect(response.body.data.markdown).toContain("This page is used for end-to-end (e2e) testing with Firecrawl.");
+ expect(response.body.data.markdown).not.toContain("Content with id #content-1");
+ },
+ 30000);
+
+ it.concurrent("should handle 'onlyMainContent' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ formats: ["html", "markdown"],
+ onlyMainContent: false
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty("data");
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+ expect(response.body.data.markdown).toContain("This page is used for end-to-end (e2e) testing with Firecrawl.");
+ expect(response.body.data.html).toContain("<header class=\"row-start-1\" style=\"\">Header</header>");
+ },
+ 30000);
+
+ it.concurrent("should handle 'timeout' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ timeout: 500
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(408);
+
+ if (!("error" in response.body)) {
+ throw new Error("Expected response body to have 'error' property");
+ }
+ expect(response.body.error).toBe("Request timed out");
+ expect(response.body.success).toBe(false);
+ }, 30000);
+
+
+ it.concurrent("should handle 'mobile' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ mobile: true
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ expect(response.body.data.markdown).toContain("This content is only visible on mobile");
+ },
+ 30000);
+
+ it.concurrent("should handle 'parsePDF' parameter correctly",
+ async () => {
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send({ url: 'https://arxiv.org/pdf/astro-ph/9301001.pdf'});
+ await new Promise((r) => setTimeout(r, 6000));
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toHaveProperty('data');
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+ expect(response.body.data.markdown).toContain('arXiv:astro-ph/9301001v1 7 Jan 1993');
+ expect(response.body.data.markdown).not.toContain('h7uKu14adDL6yGfnGf2qycY5uq8kC3OKCWkPxm');
+
+ const responseNoParsePDF: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send({ url: 'https://arxiv.org/pdf/astro-ph/9301001.pdf', parsePDF: false });
+ await new Promise((r) => setTimeout(r, 6000));
+
+ expect(responseNoParsePDF.statusCode).toBe(200);
+ expect(responseNoParsePDF.body).toHaveProperty('data');
+ if (!("data" in responseNoParsePDF.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ expect(responseNoParsePDF.body.data.markdown).toContain('h7uKu14adDL6yGfnGf2qycY5uq8kC3OKCWkPxm');
+ },
+ 30000);
+
+ // it.concurrent("should handle 'location' parameter correctly",
+ // async () => {
+ // const scrapeRequest: ScrapeRequest = {
+ // url: "https://roastmywebsite.ai",
+ // location: {
+ // country: "US",
+ // languages: ["en"]
+ // }
+ // };
+
+ // const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ // .post("/v1/scrape")
+ // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Content-Type", "application/json")
+ // .send(scrapeRequest);
+
+ // expect(response.statusCode).toBe(200);
+ // // Add assertions to verify location is handled correctly
+ // },
+ // 30000);
+
+ it.concurrent("should handle 'skipTlsVerification' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: "https://expired.badssl.com/",
+ timeout: 120000
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+ console.log("Error1a")
+ // console.log(response.body)
+ expect(response.statusCode).toBe(200);
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ expect(response.body.data.metadata.pageStatusCode).toBe(500);
+ console.log("Error?")
+
+ const scrapeRequestWithSkipTlsVerification = {
+ url: "https://expired.badssl.com/",
+ skipTlsVerification: true,
+ timeout: 120000
+
+ } as ScrapeRequest;
+
+ const responseWithSkipTlsVerification: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequestWithSkipTlsVerification);
+
+ console.log("Error1b")
+ // console.log(responseWithSkipTlsVerification.body)
+ expect(responseWithSkipTlsVerification.statusCode).toBe(200);
+ if (!("data" in responseWithSkipTlsVerification.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ // console.log(responseWithSkipTlsVerification.body.data)
+ expect(responseWithSkipTlsVerification.body.data.markdown).toContain("badssl.com");
+ },
+ 60000);
+
+ it.concurrent("should handle 'removeBase64Images' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ removeBase64Images: true
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ // console.log(response.body.data.markdown)
+ // - TODO: not working for every image
+ // expect(response.body.data.markdown).toContain("Image-Removed");
+ },
+ 30000);
+
+ it.concurrent("should handle 'action wait' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ actions: [{
+ type: "wait",
+ milliseconds: 10000
+ }]
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ expect(response.body.data.markdown).not.toContain("Loading...");
+ expect(response.body.data.markdown).toContain("Content loaded after 5 seconds!");
+ },
+ 30000);
+
+ // screenshot
+ it.concurrent("should handle 'action screenshot' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ actions: [{
+ type: "screenshot"
+ }]
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ if (!response.body.data.actions?.screenshots) {
+ throw new Error("Expected response body to have screenshots array");
+ }
+ expect(response.body.data.actions.screenshots[0].length).toBeGreaterThan(0);
+ expect(response.body.data.actions.screenshots[0]).toContain("https://service.firecrawl.dev/storage/v1/object/public/media/screenshot-");
+
+ // TODO compare screenshot with expected screenshot
+ },
+ 30000);
+
+ it.concurrent("should handle 'action screenshot@fullPage' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ actions: [{
+ type: "screenshot",
+ fullPage: true
+ },
+ {
+ type:"scrape"
+ }]
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ // console.log(response.body.data.actions?.screenshots[0])
+ if (!response.body.data.actions?.screenshots) {
+ throw new Error("Expected response body to have screenshots array");
+ }
+ expect(response.body.data.actions.screenshots[0].length).toBeGreaterThan(0);
+ expect(response.body.data.actions.screenshots[0]).toContain("https://service.firecrawl.dev/storage/v1/object/public/media/screenshot-");
+
+ if (!response.body.data.actions?.scrapes) {
+ throw new Error("Expected response body to have scrapes array");
+ }
+ expect(response.body.data.actions.scrapes[0].url).toBe("https://firecrawl-e2e-test.vercel.app/");
+ expect(response.body.data.actions.scrapes[0].html).toContain("This page is used for end-to-end (e2e) testing with Firecrawl.</p>");
+ // TODO compare screenshot with expected full page screenshot
+ },
+ 30000);
+
+ it.concurrent("should handle 'action click' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ actions: [{
+ type: "click",
+ selector: "#click-me"
+ }]
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+ expect(response.body.data.markdown).not.toContain("Click me!");
+ expect(response.body.data.markdown).toContain("Text changed after click!");
+ },
+ 30000);
+
+ it.concurrent("should handle 'action write' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ formats: ["html"],
+ actions: [{
+ type: "click",
+ selector: "#input-1"
+ },
+ {
+ type: "write",
+ text: "Hello, world!"
+ }
+ ]} as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ expect(response.statusCode).toBe(200);
+ if (!("data" in response.body)) {
+ throw new Error("Expected response body to have 'data' property");
+ }
+
+ // TODO: fix this test (need to fix fire-engine first)
+ // uncomment the following line:
+ // expect(response.body.data.html).toContain("<input id=\"input-1\" type=\"text\" placeholder=\"Enter text here...\" style=\"padding:8px;margin:10px;border:1px solid #ccc;border-radius:4px;background-color:#000\" value=\"Hello, world!\">");
+ },
+ 30000);
+
+ // TODO: fix this test (need to fix fire-engine first)
+ it.concurrent("should handle 'action pressKey' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ formats: ["markdown"],
+ actions: [
+ {
+ type: "press",
+ key: "ArrowDown"
+ }
+ ]
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ // // TODO: fix this test (need to fix fire-engine first)
+ // // right now response.body is: { success: false, error: '(Internal server error) - null' }
+ // expect(response.statusCode).toBe(200);
+ // if (!("data" in response.body)) {
+ // throw new Error("Expected response body to have 'data' property");
+ // }
+ // expect(response.body.data.markdown).toContain("Last Key Clicked: ArrowDown")
+ },
+ 30000);
+
+ // TODO: fix this test (need to fix fire-engine first)
+ it.concurrent("should handle 'action scroll' parameter correctly",
+ async () => {
+ const scrapeRequest = {
+ url: E2E_TEST_SERVER_URL,
+ formats: ["markdown"],
+ actions: [
+ {
+ type: "click",
+ selector: "#scroll-bottom-loader"
+ },
+ {
+ type: "scroll",
+ direction: "down",
+ amount: 2000
+ }
+ ]
+ } as ScrapeRequest;
+
+ const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(scrapeRequest);
+
+ // TODO: uncomment this tests
+ // expect(response.statusCode).toBe(200);
+ // if (!("data" in response.body)) {
+ // throw new Error("Expected response body to have 'data' property");
+ // }
+ //
+ // expect(response.body.data.markdown).toContain("You have reached the bottom!")
+ },
+ 30000);
+
+ // TODO: test scrape action
+
+});
\ No newline at end of file
diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index 26caf63ea..1646843fb 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -776,7 +776,8 @@ describe("E2E Tests for v0 API Routes", () => {
await new Promise((r) => setTimeout(r, 10000));
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .maxResponseSize(4000000000);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index c9f693c50..de74fed0b 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -9,9 +9,8 @@ import {
import { supabase_service } from "../services/supabase";
import { withAuth } from "../lib/withAuth";
import { RateLimiterRedis } from "rate-limiter-flexible";
-import { setTraceAttributes } from "@hyperdx/node-opentelemetry";
import { sendNotification } from "../services/notification/email_notification";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
import { redlock } from "../services/redlock";
import { deleteKey, getValue } from "../services/redis";
import { setValue } from "../services/redis";
@@ -40,8 +39,8 @@ function normalizedApiIsUuid(potentialUuid: string): boolean {
export async function setCachedACUC(
api_key: string,
acuc:
- | AuthCreditUsageChunk
- | ((acuc: AuthCreditUsageChunk) => AuthCreditUsageChunk)
+ | AuthCreditUsageChunk | null
+ | ((acuc: AuthCreditUsageChunk) => AuthCreditUsageChunk | null)
) {
const cacheKeyACUC = `acuc_${api_key}`;
const redLockKey = `lock_${cacheKeyACUC}`;
@@ -49,7 +48,7 @@ export async function setCachedACUC(
try {
await redlock.using([redLockKey], 10000, {}, async (signal) => {
if (typeof acuc === "function") {
- acuc = acuc(JSON.parse(await getValue(cacheKeyACUC)));
+ acuc = acuc(JSON.parse(await getValue(cacheKeyACUC) ?? "null"));
if (acuc === null) {
if (signal.aborted) {
@@ -69,7 +68,7 @@ export async function setCachedACUC(
await setValue(cacheKeyACUC, JSON.stringify(acuc), 600, true);
});
} catch (error) {
- Logger.error(`Error updating cached ACUC ${cacheKeyACUC}: ${error}`);
+ logger.error(`Error updating cached ACUC ${cacheKeyACUC}: ${error}`);
}
}
@@ -103,7 +102,7 @@ export async function getACUC(
break;
}
- Logger.warn(
+ logger.warn(
`Failed to retrieve authentication and credit usage data after ${retries}, trying again...`
);
retries++;
@@ -146,33 +145,14 @@ export async function authenticateUser(
res,
mode?: RateLimiterMode
): Promise<AuthResponse> {
- return withAuth(supaAuthenticateUser)(req, res, mode);
-}
-
-function setTrace(team_id: string, api_key: string) {
- try {
- setTraceAttributes({
- team_id,
- api_key,
- });
- } catch (error) {
- Sentry.captureException(error);
- Logger.error(`Error setting trace attributes: ${error.message}`);
- }
+ return withAuth(supaAuthenticateUser, { success: true, chunk: null, team_id: "bypass" })(req, res, mode);
}
export async function supaAuthenticateUser(
req,
res,
mode?: RateLimiterMode
-): Promise<{
- success: boolean;
- team_id?: string;
- error?: string;
- status?: number;
- plan?: PlanType;
- chunk?: AuthCreditUsageChunk;
-}> {
+): Promise<AuthResponse> {
const authHeader =
req.headers.authorization ??
(req.headers["sec-websocket-protocol"]
@@ -200,7 +180,7 @@ export async function supaAuthenticateUser(
let teamId: string | null = null;
let priceId: string | null = null;
- let chunk: AuthCreditUsageChunk;
+ let chunk: AuthCreditUsageChunk | null = null;
if (token == "this_is_just_a_preview_token") {
if (mode == RateLimiterMode.CrawlStatus) {
@@ -233,8 +213,6 @@ export async function supaAuthenticateUser(
priceId = chunk.price_id;
const plan = getPlanByPriceId(priceId);
- // HyperDX Logging
- setTrace(teamId, normalizedApi);
subscriptionData = {
team_id: teamId,
plan,
@@ -291,7 +269,7 @@ export async function supaAuthenticateUser(
try {
await rateLimiter.consume(team_endpoint_token);
} catch (rateLimiterRes) {
- Logger.error(`Rate limit exceeded: ${rateLimiterRes}`);
+ logger.error(`Rate limit exceeded: ${rateLimiterRes}`);
const secs = Math.round(rateLimiterRes.msBeforeNext / 1000) || 1;
const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext);
@@ -318,7 +296,7 @@ export async function supaAuthenticateUser(
mode === RateLimiterMode.CrawlStatus ||
mode === RateLimiterMode.Search)
) {
- return { success: true, team_id: "preview" };
+ return { success: true, team_id: "preview", chunk: null };
// check the origin of the request and make sure its from firecrawl.dev
// const origin = req.headers.origin;
// if (origin && origin.includes("firecrawl.dev")){
@@ -333,12 +311,12 @@ export async function supaAuthenticateUser(
return {
success: true,
- team_id: subscriptionData.team_id,
- plan: (subscriptionData.plan ?? "") as PlanType,
+ team_id: teamId ?? undefined,
+ plan: (subscriptionData?.plan ?? "") as PlanType,
chunk,
};
}
-function getPlanByPriceId(price_id: string): PlanType {
+function getPlanByPriceId(price_id: string | null): PlanType {
switch (price_id) {
case process.env.STRIPE_PRICE_ID_STARTER:
return "starter";
diff --git a/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts b/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts
index 876ca98aa..75acd60ac 100644
--- a/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts
+++ b/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts
@@ -1,7 +1,7 @@
import { Request, Response } from "express";
import { supabase_service } from "../../../services/supabase";
import { clearACUC } from "../../auth";
-import { Logger } from "../../../lib/logger";
+import { logger } from "../../../lib/logger";
export async function acucCacheClearController(req: Request, res: Response) {
try {
@@ -12,11 +12,11 @@ export async function acucCacheClearController(req: Request, res: Response) {
.select("*")
.eq("team_id", team_id);
- await Promise.all(keys.data.map((x) => clearACUC(x.key)));
+ await Promise.all((keys.data ?? []).map((x) => clearACUC(x.key)));
res.json({ ok: true });
} catch (error) {
- Logger.error(`Error clearing ACUC cache via API route: ${error}`);
+ logger.error(`Error clearing ACUC cache via API route: ${error}`);
res.status(500).json({ error: "Internal server error" });
}
}
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts
index 717480026..6ef8a992a 100644
--- a/apps/api/src/controllers/v0/admin/queue.ts
+++ b/apps/api/src/controllers/v0/admin/queue.ts
@@ -1,7 +1,7 @@
import { Request, Response } from "express";
import { Job } from "bullmq";
-import { Logger } from "../../../lib/logger";
+import { logger } from "../../../lib/logger";
import { getScrapeQueue } from "../../../services/queue-service";
import { checkAlerts } from "../../../services/alerts";
import { sendSlackWebhook } from "../../../services/alerts/slack";
@@ -10,7 +10,7 @@ export async function cleanBefore24hCompleteJobsController(
req: Request,
res: Response
) {
- Logger.info("πŸ‚ Cleaning jobs older than 24h");
+ logger.info("πŸ‚ Cleaning jobs older than 24h");
try {
const scrapeQueue = getScrapeQueue();
const batchSize = 10;
@@ -31,7 +31,7 @@ export async function cleanBefore24hCompleteJobsController(
).flat();
const before24hJobs =
completedJobs.filter(
- (job) => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000
+ (job) => job.finishedOn !== undefined && job.finishedOn < Date.now() - 24 * 60 * 60 * 1000
) || [];
let count = 0;
@@ -45,12 +45,12 @@ export async function cleanBefore24hCompleteJobsController(
await job.remove();
count++;
} catch (jobError) {
- Logger.error(`πŸ‚ Failed to remove job with ID ${job.id}: ${jobError}`);
+ logger.error(`πŸ‚ Failed to remove job with ID ${job.id}: ${jobError}`);
}
}
return res.status(200).send(`Removed ${count} completed jobs.`);
} catch (error) {
- Logger.error(`πŸ‚ Failed to clean last 24h complete jobs: ${error}`);
+ logger.error(`πŸ‚ Failed to clean last 24h complete jobs: ${error}`);
return res.status(500).send("Failed to clean jobs");
}
}
@@ -60,7 +60,7 @@ export async function checkQueuesController(req: Request, res: Response) {
await checkAlerts();
return res.status(200).send("Alerts initialized");
} catch (error) {
- Logger.debug(`Failed to initialize alerts: ${error}`);
+ logger.debug(`Failed to initialize alerts: ${error}`);
return res.status(500).send("Failed to initialize alerts");
}
}
@@ -81,7 +81,7 @@ export async function queuesController(req: Request, res: Response) {
noActiveJobs,
});
} catch (error) {
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: error.message });
}
}
@@ -165,7 +165,7 @@ export async function autoscalerController(req: Request, res: Response) {
}
if (targetMachineCount !== activeMachines) {
- Logger.info(
+ logger.info(
`πŸ‚ Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting`
);
@@ -193,7 +193,7 @@ export async function autoscalerController(req: Request, res: Response) {
count: activeMachines,
});
} catch (error) {
- Logger.error(error);
+ logger.error(error);
return res.status(500).send("Failed to initialize autoscaler");
}
}
diff --git a/apps/api/src/controllers/v0/admin/redis-health.ts b/apps/api/src/controllers/v0/admin/redis-health.ts
index dc58d7454..dc5876067 100644
--- a/apps/api/src/controllers/v0/admin/redis-health.ts
+++ b/apps/api/src/controllers/v0/admin/redis-health.ts
@@ -1,6 +1,6 @@
import { Request, Response } from "express";
import Redis from "ioredis";
-import { Logger } from "../../../lib/logger";
+import { logger } from "../../../lib/logger";
import { redisRateLimitClient } from "../../../services/rate-limiter";
export async function redisHealthController(req: Request, res: Response) {
@@ -10,14 +10,14 @@ export async function redisHealthController(req: Request, res: Response) {
return await operation();
} catch (error) {
if (attempt === retries) throw error;
- Logger.warn(`Attempt ${attempt} failed: ${error.message}. Retrying...`);
+ logger.warn(`Attempt ${attempt} failed: ${error.message}. Retrying...`);
await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait 2 seconds before retrying
}
}
};
try {
- const queueRedis = new Redis(process.env.REDIS_URL);
+ const queueRedis = new Redis(process.env.REDIS_URL!);
const testKey = "test";
const testValue = "test";
@@ -29,7 +29,7 @@ export async function redisHealthController(req: Request, res: Response) {
queueRedisHealth = await retryOperation(() => queueRedis.get(testKey));
await retryOperation(() => queueRedis.del(testKey));
} catch (error) {
- Logger.error(`queueRedis health check failed: ${error}`);
+ logger.error(`queueRedis health check failed: ${error}`);
queueRedisHealth = null;
}
@@ -42,7 +42,7 @@ export async function redisHealthController(req: Request, res: Response) {
);
await retryOperation(() => redisRateLimitClient.del(testKey));
} catch (error) {
- Logger.error(`redisRateLimitClient health check failed: ${error}`);
+ logger.error(`redisRateLimitClient health check failed: ${error}`);
redisRateLimitHealth = null;
}
@@ -56,10 +56,10 @@ export async function redisHealthController(req: Request, res: Response) {
healthStatus.queueRedis === "healthy" &&
healthStatus.redisRateLimitClient === "healthy"
) {
- Logger.info("Both Redis instances are healthy");
+ logger.info("Both Redis instances are healthy");
return res.status(200).json({ status: "healthy", details: healthStatus });
} else {
- Logger.info(
+ logger.info(
`Redis instances health check: ${JSON.stringify(healthStatus)}`
);
// await sendSlackWebhook(
@@ -73,7 +73,7 @@ export async function redisHealthController(req: Request, res: Response) {
.json({ status: "unhealthy", details: healthStatus });
}
} catch (error) {
- Logger.error(`Redis health check failed: ${error}`);
+ logger.error(`Redis health check failed: ${error}`);
// await sendSlackWebhook(
// `[REDIS DOWN] Redis instances health check: ${error.message}`,
// true
diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts
index efcd454a8..e81064f21 100644
--- a/apps/api/src/controllers/v0/crawl-cancel.ts
+++ b/apps/api/src/controllers/v0/crawl-cancel.ts
@@ -2,7 +2,7 @@ import { Request, Response } from "express";
import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
import { supabase_service } from "../../../src/services/supabase";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
@@ -12,15 +12,17 @@ export async function crawlCancelController(req: Request, res: Response) {
try {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
- const { success, team_id, error, status } = await authenticateUser(
+ const auth = await authenticateUser(
req,
res,
RateLimiterMode.CrawlStatus
);
- if (!success) {
- return res.status(status).json({ error });
+ if (!auth.success) {
+ return res.status(auth.status).json({ error: auth.error });
}
+ const { team_id } = auth;
+
const sc = await getCrawl(req.params.jobId);
if (!sc) {
return res.status(404).json({ error: "Job not found" });
@@ -46,7 +48,7 @@ export async function crawlCancelController(req: Request, res: Response) {
sc.cancelled = true;
await saveCrawl(req.params.jobId, sc);
} catch (error) {
- Logger.error(error);
+ logger.error(error);
}
res.json({
@@ -54,7 +56,7 @@ export async function crawlCancelController(req: Request, res: Response) {
});
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: error.message });
}
}
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index 66522bcf4..7b6e610ad 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -2,15 +2,17 @@ import { Request, Response } from "express";
import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
import { getScrapeQueue } from "../../../src/services/queue-service";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";
import { supabaseGetJobsByCrawlId } from "../../../src/lib/supabase-jobs";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
+import { Job } from "bullmq";
+import { toLegacyDocument } from "../v1/types";
configDotenv();
export async function getJobs(crawlId: string, ids: string[]) {
- const jobs = (await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))).filter(x => x);
+ const jobs = (await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))).filter(x => x) as Job[];
if (process.env.USE_DB_AUTHENTICATION === "true") {
const supabaseData = await supabaseGetJobsByCrawlId(crawlId);
@@ -32,15 +34,17 @@ export async function getJobs(crawlId: string, ids: string[]) {
export async function crawlStatusController(req: Request, res: Response) {
try {
- const { success, team_id, error, status } = await authenticateUser(
+ const auth = await authenticateUser(
req,
res,
RateLimiterMode.CrawlStatus
);
- if (!success) {
- return res.status(status).json({ error });
+ if (!auth.success) {
+ return res.status(auth.status).json({ error: auth.error });
}
+ const { team_id } = auth;
+
const sc = await getCrawl(req.params.jobId);
if (!sc) {
return res.status(404).json({ error: "Job not found" });
@@ -90,12 +94,12 @@ export async function crawlStatusController(req: Request, res: Response) {
status: jobStatus,
current: jobStatuses.filter(x => x === "completed" || x === "failed").length,
total: jobs.length,
- data: jobStatus === "completed" ? data : null,
- partial_data: jobStatus === "completed" ? [] : data.filter(x => x !== null),
+ data: jobStatus === "completed" ? data.map(x => toLegacyDocument(x, sc.internalOptions)) : null,
+ partial_data: jobStatus === "completed" ? [] : data.filter(x => x !== null).map(x => toLegacyDocument(x, sc.internalOptions)),
});
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: error.message });
}
}
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index fb4121478..d502d1425 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -9,24 +9,28 @@ import { validateIdempotencyKey } from "../../../src/services/idempotency/valida
import { createIdempotencyKey } from "../../../src/services/idempotency/create";
import { defaultCrawlPageOptions, defaultCrawlerOptions, defaultOrigin } from "../../../src/lib/default-values";
import { v4 as uuidv4 } from "uuid";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import { addCrawlJob, addCrawlJobs, crawlToCrawler, lockURL, lockURLs, saveCrawl, StoredCrawl } from "../../../src/lib/crawl-redis";
import { getScrapeQueue } from "../../../src/services/queue-service";
import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
+import { fromLegacyScrapeOptions, url as urlSchema } from "../v1/types";
+import { ZodError } from "zod";
export async function crawlController(req: Request, res: Response) {
try {
- const { success, team_id, error, status, plan, chunk } = await authenticateUser(
+ const auth = await authenticateUser(
req,
res,
RateLimiterMode.Crawl
);
- if (!success) {
- return res.status(status).json({ error });
+ if (!auth.success) {
+ return res.status(auth.status).json({ error: auth.error });
}
+ const { team_id, plan, chunk } = auth;
+
if (req.headers["x-idempotency-key"]) {
const isIdempotencyValid = await validateIdempotencyKey(req);
if (!isIdempotencyValid) {
@@ -35,7 +39,7 @@ export async function crawlController(req: Request, res: Response) {
try {
createIdempotencyKey(req);
} catch (error) {
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: error.message });
}
}
@@ -77,7 +81,7 @@ export async function crawlController(req: Request, res: Response) {
// TODO: need to do this to v1
crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit);
- let url = req.body.url;
+ let url = urlSchema.parse(req.body.url);
if (!url) {
return res.status(400).json({ error: "Url is required" });
}
@@ -123,7 +127,7 @@ export async function crawlController(req: Request, res: Response) {
// documents: docs,
// });
// } catch (error) {
- // Logger.error(error);
+ // logger.error(error);
// return res.status(500).json({ error: error.message });
// }
// }
@@ -132,10 +136,13 @@ export async function crawlController(req: Request, res: Response) {
await logCrawl(id, team_id);
+ const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(pageOptions, undefined, undefined);
+
const sc: StoredCrawl = {
originUrl: url,
crawlerOptions,
- pageOptions,
+ scrapeOptions,
+ internalOptions,
team_id,
plan,
createdAt: Date.now(),
@@ -170,10 +177,11 @@ export async function crawlController(req: Request, res: Response) {
data: {
url,
mode: "single_urls",
- crawlerOptions: crawlerOptions,
+ crawlerOptions,
+ scrapeOptions,
+ internalOptions,
team_id,
plan,
- pageOptions: pageOptions,
origin: req.body.origin ?? defaultOrigin,
crawl_id: id,
sitemapped: true,
@@ -208,10 +216,11 @@ export async function crawlController(req: Request, res: Response) {
{
url,
mode: "single_urls",
- crawlerOptions: crawlerOptions,
+ crawlerOptions,
+ scrapeOptions,
+ internalOptions,
team_id,
- plan,
- pageOptions: pageOptions,
+ plan: plan!,
origin: req.body.origin ?? defaultOrigin,
crawl_id: id,
},
@@ -226,7 +235,9 @@ export async function crawlController(req: Request, res: Response) {
res.json({ jobId: id });
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
- return res.status(500).json({ error: error.message });
+ logger.error(error);
+ return res.status(500).json({ error: error instanceof ZodError
+ ? "Invalid URL"
+ : error.message });
}
}
diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts
index e9f6e8062..8b82bef88 100644
--- a/apps/api/src/controllers/v0/crawlPreview.ts
+++ b/apps/api/src/controllers/v0/crawlPreview.ts
@@ -3,15 +3,16 @@ import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";
import { v4 as uuidv4 } from "uuid";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import { addCrawlJob, crawlToCrawler, lockURL, saveCrawl, StoredCrawl } from "../../../src/lib/crawl-redis";
import { addScrapeJob } from "../../../src/services/queue-jobs";
import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
import * as Sentry from "@sentry/node";
+import { fromLegacyScrapeOptions } from "../v1/types";
export async function crawlPreviewController(req: Request, res: Response) {
try {
- const { success, error, status, team_id:a, plan } = await authenticateUser(
+ const auth = await authenticateUser(
req,
res,
RateLimiterMode.Preview
@@ -19,10 +20,12 @@ export async function crawlPreviewController(req: Request, res: Response) {
const team_id = "preview";
- if (!success) {
- return res.status(status).json({ error });
+ if (!auth.success) {
+ return res.status(auth.status).json({ error: auth.error });
}
+ const { plan } = auth;
+
let url = req.body.url;
if (!url) {
return res.status(400).json({ error: "Url is required" });
@@ -71,7 +74,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
// documents: docs,
// });
// } catch (error) {
- // Logger.error(error);
+ // logger.error(error);
// return res.status(500).json({ error: error.message });
// }
// }
@@ -84,10 +87,13 @@ export async function crawlPreviewController(req: Request, res: Response) {
robots = await this.getRobotsTxt();
} catch (_) {}
+ const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(pageOptions, undefined, undefined);
+
const sc: StoredCrawl = {
originUrl: url,
crawlerOptions,
- pageOptions,
+ scrapeOptions,
+ internalOptions,
team_id,
plan,
robots,
@@ -107,10 +113,11 @@ export async function crawlPreviewController(req: Request, res: Response) {
await addScrapeJob({
url,
mode: "single_urls",
- crawlerOptions: crawlerOptions,
team_id,
- plan,
- pageOptions: pageOptions,
+ plan: plan!,
+ crawlerOptions,
+ scrapeOptions,
+ internalOptions,
origin: "website-preview",
crawl_id: id,
sitemapped: true,
@@ -123,10 +130,11 @@ export async function crawlPreviewController(req: Request, res: Response) {
await addScrapeJob({
url,
mode: "single_urls",
- crawlerOptions: crawlerOptions,
team_id,
- plan,
- pageOptions: pageOptions,
+ plan: plan!,
+ crawlerOptions,
+ scrapeOptions,
+ internalOptions,
origin: "website-preview",
crawl_id: id,
}, {}, jobId);
@@ -136,7 +144,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
res.json({ jobId: id });
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: error.message });
}
}
diff --git a/apps/api/src/controllers/v0/keyAuth.ts b/apps/api/src/controllers/v0/keyAuth.ts
index b70d672a1..639153024 100644
--- a/apps/api/src/controllers/v0/keyAuth.ts
+++ b/apps/api/src/controllers/v0/keyAuth.ts
@@ -8,13 +8,14 @@ import { authenticateUser } from "../auth";
export const keyAuthController = async (req: Request, res: Response) => {
try {
// make sure to authenticate user first, Bearer <token>
- const { success, team_id, error, status } = await authenticateUser(
+ const auth = await authenticateUser(
req,
res
);
- if (!success) {
- return res.status(status).json({ error });
+ if (!auth.success) {
+ return res.status(auth.status).json({ error: auth.error });
}
+
// if success, return success: true
return res.status(200).json({ success: true });
} catch (error) {
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 5e6b7c6f7..05fb3c412 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -7,7 +7,7 @@ import {
import { authenticateUser } from "../auth";
import { PlanType, RateLimiterMode } from "../../types";
import { logJob } from "../../services/logging/log_job";
-import { Document } from "../../lib/entities";
+import { Document, fromLegacyCombo, toLegacyDocument, url as urlSchema } from "../v1/types";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; // Import the isUrlBlocked function
import { numTokensFromString } from "../../lib/LLM-extraction/helpers";
import {
@@ -19,9 +19,11 @@ import {
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getScrapeQueue } from "../../services/queue-service";
import { v4 as uuidv4 } from "uuid";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
+import { fromLegacyScrapeOptions } from "../v1/types";
+import { ZodError } from "zod";
export async function scrapeHelper(
jobId: string,
@@ -35,10 +37,10 @@ export async function scrapeHelper(
): Promise<{
success: boolean;
error?: string;
- data?: Document;
+ data?: Document | { url: string };
returnCode: number;
}> {
- const url = req.body.url;
+ const url = urlSchema.parse(req.body.url);
if (typeof url !== "string") {
return { success: false, error: "Url is required", returnCode: 400 };
}
@@ -54,15 +56,16 @@ export async function scrapeHelper(
const jobPriority = await getJobPriority({ plan, team_id, basePriority: 10 });
+ const { scrapeOptions, internalOptions } = fromLegacyCombo(pageOptions, extractorOptions, timeout, crawlerOptions);
+
await addScrapeJob(
{
url,
mode: "single_urls",
- crawlerOptions,
team_id,
- pageOptions,
- plan,
- extractorOptions,
+ scrapeOptions,
+ internalOptions,
+ plan: plan!,
origin: req.body.origin ?? defaultOrigin,
is_scrape: true,
},
@@ -81,7 +84,7 @@ export async function scrapeHelper(
},
async (span) => {
try {
- doc = (await waitForJob(jobId, timeout))[0];
+ doc = (await waitForJob<Document>(jobId, timeout));
} catch (e) {
if (e instanceof Error && e.message.startsWith("Job wait")) {
span.setAttribute("timedOut", true);
@@ -149,7 +152,7 @@ export async function scrapeHelper(
return {
success: true,
- data: doc,
+ data: toLegacyDocument(doc, internalOptions),
returnCode: 200,
};
}
@@ -158,15 +161,17 @@ export async function scrapeController(req: Request, res: Response) {
try {
let earlyReturn = false;
// make sure to authenticate user first, Bearer <token>
- const { success, team_id, error, status, plan, chunk } = await authenticateUser(
+ const auth = await authenticateUser(
req,
res,
RateLimiterMode.Scrape
);
- if (!success) {
- return res.status(status).json({ error });
+ if (!auth.success) {
+ return res.status(auth.status).json({ error: auth.error });
}
+ const { team_id, plan, chunk } = auth;
+
const crawlerOptions = req.body.crawlerOptions ?? {};
const pageOptions = { ...defaultPageOptions, ...req.body.pageOptions };
const extractorOptions = {
@@ -200,7 +205,7 @@ export async function scrapeController(req: Request, res: Response) {
return res.status(402).json({ error: "Insufficient credits. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing" });
}
} catch (error) {
- Logger.error(error);
+ logger.error(error);
earlyReturn = true;
return res.status(500).json({
error:
@@ -224,8 +229,8 @@ export async function scrapeController(req: Request, res: Response) {
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - startTime) / 1000;
const numTokens =
- result.data && result.data.markdown
- ? numTokensFromString(result.data.markdown, "gpt-3.5-turbo")
+ result.data && (result.data as Document).markdown
+ ? numTokensFromString((result.data as Document).markdown!, "gpt-3.5-turbo")
: 0;
if (result.success) {
@@ -246,7 +251,7 @@ export async function scrapeController(req: Request, res: Response) {
if (creditsToBeBilled > 0) {
// billing for doc done on queue end, bill only for llm extraction
billTeam(team_id, chunk?.sub_id, creditsToBeBilled).catch(error => {
- Logger.error(`Failed to bill team ${team_id} for ${creditsToBeBilled} credits: ${error}`);
+ logger.error(`Failed to bill team ${team_id} for ${creditsToBeBilled} credits: ${error}`);
// Optionally, you could notify an admin or add to a retry queue here
});
}
@@ -254,17 +259,19 @@ export async function scrapeController(req: Request, res: Response) {
let doc = result.data;
if (!pageOptions || !pageOptions.includeRawHtml) {
- if (doc && doc.rawHtml) {
- delete doc.rawHtml;
+ if (doc && (doc as Document).rawHtml) {
+ delete (doc as Document).rawHtml;
}
}
if(pageOptions && pageOptions.includeExtract) {
- if(!pageOptions.includeMarkdown && doc && doc.markdown) {
- delete doc.markdown;
+ if(!pageOptions.includeMarkdown && doc && (doc as Document).markdown) {
+ delete (doc as Document).markdown;
}
}
+ const { scrapeOptions } = fromLegacyScrapeOptions(pageOptions, extractorOptions, timeout);
+
logJob({
job_id: jobId,
success: result.success,
@@ -276,21 +283,22 @@ export async function scrapeController(req: Request, res: Response) {
mode: "scrape",
url: req.body.url,
crawlerOptions: crawlerOptions,
- pageOptions: pageOptions,
+ scrapeOptions,
origin: origin,
- extractor_options: extractorOptions,
num_tokens: numTokens,
});
return res.status(result.returnCode).json(result);
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({
error:
- typeof error === "string"
- ? error
- : error?.message ?? "Internal Server Error",
+ error instanceof ZodError
+ ? "Invalid URL"
+ : typeof error === "string"
+ ? error
+ : error?.message ?? "Internal Server Error",
});
}
}
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 67cff8eb8..e0102406f 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -1,5 +1,4 @@
import { Request, Response } from "express";
-import { WebScraperDataProvider } from "../../scraper/WebScraper";
import { billTeam, checkTeamCredits } from "../../services/billing/credit_billing";
import { authenticateUser } from "../auth";
import { PlanType, RateLimiterMode } from "../../types";
@@ -8,21 +7,23 @@ import { PageOptions, SearchOptions } from "../../lib/entities";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { v4 as uuidv4 } from "uuid";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import { getScrapeQueue } from "../../services/queue-service";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
+import { Job } from "bullmq";
+import { Document, fromLegacyCombo, fromLegacyScrapeOptions, toLegacyDocument } from "../v1/types";
export async function searchHelper(
jobId: string,
req: Request,
team_id: string,
- subscription_id: string,
+ subscription_id: string | null | undefined,
crawlerOptions: any,
pageOptions: PageOptions,
searchOptions: SearchOptions,
- plan: PlanType
+ plan: PlanType | undefined
): Promise<{
success: boolean;
error?: string;
@@ -35,8 +36,8 @@ export async function searchHelper(
return { success: false, error: "Query is required", returnCode: 400 };
}
- const tbs = searchOptions.tbs ?? null;
- const filter = searchOptions.filter ?? null;
+ const tbs = searchOptions.tbs ?? undefined;
+ const filter = searchOptions.filter ?? undefined;
let num_results = Math.min(searchOptions.limit ?? 7, 10);
if (team_id === "d97c4ceb-290b-4957-8432-2b2a02727d95") {
@@ -57,11 +58,12 @@ export async function searchHelper(
});
let justSearch = pageOptions.fetchPageContent === false;
-
+
+ const { scrapeOptions, internalOptions } = fromLegacyCombo(pageOptions, undefined, 60000, crawlerOptions);
if (justSearch) {
billTeam(team_id, subscription_id, res.length).catch(error => {
- Logger.error(`Failed to bill team ${team_id} for ${res.length} credits: ${error}`);
+ logger.error(`Failed to bill team ${team_id} for ${res.length} credits: ${error}`);
// Optionally, you could notify an admin or add to a retry queue here
});
return { success: true, data: res, returnCode: 200 };
@@ -88,9 +90,9 @@ export async function searchHelper(
data: {
url,
mode: "single_urls",
- crawlerOptions: crawlerOptions,
team_id: team_id,
- pageOptions: pageOptions,
+ scrapeOptions,
+ internalOptions,
},
opts: {
jobId: uuid,
@@ -104,7 +106,7 @@ export async function searchHelper(
await addScrapeJob(job.data as any, {}, job.opts.jobId, job.opts.priority)
}
- const docs = (await Promise.all(jobDatas.map(x => waitForJob(x.opts.jobId, 60000)))).map(x => x[0]);
+ const docs = (await Promise.all(jobDatas.map(x => waitForJob<Document>(x.opts.jobId, 60000)))).map(x => toLegacyDocument(x, internalOptions));
if (docs.length === 0) {
return { success: true, error: "No search results found", returnCode: 200 };
@@ -115,7 +117,7 @@ export async function searchHelper(
// make sure doc.content is not empty
const filteredDocs = docs.filter(
- (doc: { content?: string }) => doc && doc.content && doc.content.trim().length > 0
+ (doc: any) => doc && doc.content && doc.content.trim().length > 0
);
if (filteredDocs.length === 0) {
@@ -132,14 +134,15 @@ export async function searchHelper(
export async function searchController(req: Request, res: Response) {
try {
// make sure to authenticate user first, Bearer <token>
- const { success, team_id, error, status, plan, chunk } = await authenticateUser(
+ const auth = await authenticateUser(
req,
res,
RateLimiterMode.Search
);
- if (!success) {
- return res.status(status).json({ error });
+ if (!auth.success) {
+ return res.status(auth.status).json({ error: auth.error });
}
+ const { team_id, plan, chunk } = auth;
const crawlerOptions = req.body.crawlerOptions ?? {};
const pageOptions = req.body.pageOptions ?? {
includeHtml: req.body.pageOptions?.includeHtml ?? false,
@@ -162,7 +165,7 @@ export async function searchController(req: Request, res: Response) {
}
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: "Internal server error" });
}
const startTime = new Date().getTime();
@@ -189,7 +192,6 @@ export async function searchController(req: Request, res: Response) {
mode: "search",
url: req.body.query,
crawlerOptions: crawlerOptions,
- pageOptions: pageOptions,
origin: origin,
});
return res.status(result.returnCode).json(result);
@@ -199,7 +201,7 @@ export async function searchController(req: Request, res: Response) {
}
Sentry.captureException(error);
- Logger.error(error);
+ logger.error("Unhandled error occurred in search", { error });
return res.status(500).json({ error: error.message });
}
}
diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.ts
index bf8d28344..c5eafc2dc 100644
--- a/apps/api/src/controllers/v0/status.ts
+++ b/apps/api/src/controllers/v0/status.ts
@@ -1,5 +1,5 @@
import { Request, Response } from "express";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";
import { getJobs } from "./crawl-status";
import * as Sentry from "@sentry/node";
@@ -37,7 +37,7 @@ export async function crawlJobStatusPreviewController(req: Request, res: Respons
});
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: error.message });
}
}
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 286163dfe..9c6a288c3 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -4,8 +4,6 @@ import {
BatchScrapeRequest,
batchScrapeRequestSchema,
CrawlResponse,
- legacyExtractorOptions,
- legacyScrapeOptions,
RequestWithAuth,
} from "./types";
import {
@@ -29,19 +27,16 @@ export async function batchScrapeController(
await logCrawl(id, req.auth.team_id);
- let { remainingCredits } = req.account;
+ let { remainingCredits } = req.account!;
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
if(!useDbAuthentication){
remainingCredits = Infinity;
}
- const pageOptions = legacyScrapeOptions(req.body);
- const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined;
-
-
const sc: StoredCrawl = {
crawlerOptions: null,
- pageOptions,
+ scrapeOptions: req.body,
+ internalOptions: {},
team_id: req.auth.team_id,
createdAt: Date.now(),
plan: req.auth.plan,
@@ -64,10 +59,9 @@ export async function batchScrapeController(
url: x,
mode: "single_urls" as const,
team_id: req.auth.team_id,
- plan: req.auth.plan,
+ plan: req.auth.plan!,
crawlerOptions: null,
- pageOptions,
- extractorOptions,
+ scrapeOptions: req.body,
origin: "api",
crawl_id: id,
sitemapped: true,
diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts
index f8fba824a..958318b51 100644
--- a/apps/api/src/controllers/v1/crawl-cancel.ts
+++ b/apps/api/src/controllers/v1/crawl-cancel.ts
@@ -1,6 +1,6 @@
import { Response } from "express";
import { supabase_service } from "../../services/supabase";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import { getCrawl, saveCrawl } from "../../lib/crawl-redis";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
@@ -36,7 +36,7 @@ export async function crawlCancelController(req: RequestWithAuth<{ jobId: string
sc.cancelled = true;
await saveCrawl(req.params.jobId, sc);
} catch (error) {
- Logger.error(error);
+ logger.error(error);
}
res.json({
@@ -44,7 +44,7 @@ export async function crawlCancelController(req: RequestWithAuth<{ jobId: string
});
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: error.message });
}
}
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index 3738e3a21..18222edca 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -1,14 +1,15 @@
import { authMiddleware } from "../../routes/v1";
import { RateLimiterMode } from "../../types";
import { authenticateUser } from "../auth";
-import { CrawlStatusParams, CrawlStatusResponse, Document, ErrorResponse, legacyDocumentConverter, RequestWithAuth } from "./types";
+import { CrawlStatusParams, CrawlStatusResponse, Document, ErrorResponse, RequestWithAuth } from "./types";
import { WebSocket } from "ws";
import { v4 as uuidv4 } from "uuid";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import { getCrawl, getCrawlExpiry, getCrawlJobs, getDoneJobsOrdered, getDoneJobsOrderedLength, getThrottledJobs, isCrawlFinished, isCrawlFinishedLocked } from "../../lib/crawl-redis";
import { getScrapeQueue } from "../../services/queue-service";
import { getJob, getJobs } from "./crawl-status";
import * as Sentry from "@sentry/node";
+import { Job, JobState } from "bullmq";
type ErrorMessage = {
type: "error",
@@ -56,7 +57,7 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusPara
return close(ws, 3003, { type: "error", error: "Forbidden" });
}
- let doneJobIDs = [];
+ let doneJobIDs: string[] = [];
let finished = false;
const loop = async () => {
@@ -70,15 +71,14 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusPara
const notDoneJobIDs = jobIDs.filter(x => !doneJobIDs.includes(x));
const jobStatuses = await Promise.all(notDoneJobIDs.map(async x => [x, await getScrapeQueue().getJobState(x)]));
- const newlyDoneJobIDs = jobStatuses.filter(x => x[1] === "completed" || x[1] === "failed").map(x => x[0]);
-
- for (const jobID of newlyDoneJobIDs) {
- const job = await getJob(jobID);
+ const newlyDoneJobIDs: string[] = jobStatuses.filter(x => x[1] === "completed" || x[1] === "failed").map(x => x[0]);
+ const newlyDoneJobs: Job[] = (await Promise.all(newlyDoneJobIDs.map(x => getJob(x)))).filter(x => x !== undefined) as Job[]
+ for (const job of newlyDoneJobs) {
if (job.returnvalue) {
send(ws, {
type: "document",
- data: legacyDocumentConverter(job.returnvalue),
+ data: job.returnvalue,
})
} else {
return close(ws, 3000, { type: "error", error: job.failedReason });
@@ -100,8 +100,8 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusPara
const throttledJobsSet = new Set(throttledJobs);
- const validJobStatuses = [];
- const validJobIDs = [];
+ const validJobStatuses: [string, JobState | "unknown"][] = [];
+ const validJobIDs: string[] = [];
for (const [id, status] of jobStatuses) {
if (!throttledJobsSet.has(id) && status !== "failed" && status !== "unknown") {
@@ -126,7 +126,7 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusPara
completed: doneJobIDs.length,
creditsUsed: jobIDs.length,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
- data: data.map(x => legacyDocumentConverter(x)),
+ data: data,
}
});
@@ -139,19 +139,21 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusPara
// Basically just middleware and error wrapping
export async function crawlStatusWSController(ws: WebSocket, req: RequestWithAuth<CrawlStatusParams, undefined, undefined>) {
try {
- const { success, team_id, error, status, plan } = await authenticateUser(
+ const auth = await authenticateUser(
req,
null,
RateLimiterMode.CrawlStatus,
);
- if (!success) {
+ if (!auth.success) {
return close(ws, 3000, {
type: "error",
- error,
+ error: auth.error,
});
}
+ const { team_id, plan } = auth;
+
req.auth = { team_id, plan };
await crawlStatusWS(ws, req);
@@ -170,7 +172,7 @@ export async function crawlStatusWSController(ws: WebSocket, req: RequestWithAut
}
}
- Logger.error("Error occurred in WebSocket! (" + req.path + ") -- ID " + id + " -- " + verbose);
+ logger.error("Error occurred in WebSocket! (" + req.path + ") -- ID " + id + " -- " + verbose);
return close(ws, 1011, {
type: "error",
error: "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " + id
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index a8d782939..f150ddc4a 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -1,9 +1,10 @@
import { Response } from "express";
-import { CrawlStatusParams, CrawlStatusResponse, ErrorResponse, legacyDocumentConverter, RequestWithAuth } from "./types";
+import { CrawlStatusParams, CrawlStatusResponse, ErrorResponse, RequestWithAuth } from "./types";
import { getCrawl, getCrawlExpiry, getCrawlJobs, getDoneJobsOrdered, getDoneJobsOrderedLength, getThrottledJobs } from "../../lib/crawl-redis";
import { getScrapeQueue } from "../../services/queue-service";
import { supabaseGetJobById, supabaseGetJobsById } from "../../lib/supabase-jobs";
import { configDotenv } from "dotenv";
+import { Job, JobState } from "bullmq";
configDotenv();
export async function getJob(id: string) {
@@ -24,7 +25,7 @@ export async function getJob(id: string) {
}
export async function getJobs(ids: string[]) {
- const jobs = (await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))).filter(x => x);
+ const jobs: (Job & { id: string })[] = (await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))).filter(x => x) as (Job & {id: string})[];
if (process.env.USE_DB_AUTHENTICATION === "true") {
const supabaseData = await supabaseGetJobsById(ids);
@@ -63,8 +64,8 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
const throttledJobsSet = new Set(throttledJobs);
- const validJobStatuses = [];
- const validJobIDs = [];
+ const validJobStatuses: [string, JobState | "unknown"][] = [];
+ const validJobIDs: string[] = [];
for (const [id, status] of jobStatuses) {
if (!throttledJobsSet.has(id) && status !== "failed" && status !== "unknown") {
@@ -81,7 +82,7 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
const doneJobsLength = await getDoneJobsOrderedLength(req.params.jobId);
const doneJobsOrder = await getDoneJobsOrdered(req.params.jobId, start, end ?? -1);
- let doneJobs = [];
+ let doneJobs: Job[] = [];
if (end === undefined) { // determine 10 megabyte limit
let bytes = 0;
@@ -98,7 +99,7 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
for (let ii = 0; ii < jobs.length && bytes < bytesLimit; ii++) {
const job = jobs[ii];
doneJobs.push(job);
- bytes += JSON.stringify(legacyDocumentConverter(job.returnvalue)).length;
+ bytes += JSON.stringify(job.returnvalue).length;
}
}
@@ -122,7 +123,7 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
}
if (data.length > 0) {
- if (!doneJobs[0].data.pageOptions.includeRawHtml) {
+ if (!doneJobs[0].data.scrapeOptions.formats.includes("rawHtml")) {
for (let ii = 0; ii < doneJobs.length; ii++) {
if (data[ii]) {
delete data[ii].rawHtml;
@@ -142,7 +143,7 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
status !== "scraping" && (start + data.length) === doneJobsLength // if there's not gonna be any documents after this
? undefined
: nextURL.href,
- data: data.map(x => legacyDocumentConverter(x)),
+ data: data,
});
}
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 53cc04e8a..f9f60e717 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -4,9 +4,8 @@ import {
CrawlRequest,
crawlRequestSchema,
CrawlResponse,
- legacyCrawlerOptions,
- legacyScrapeOptions,
RequestWithAuth,
+ toLegacyCrawlerOptions,
} from "./types";
import {
addCrawlJob,
@@ -20,9 +19,10 @@ import {
import { logCrawl } from "../../services/logging/crawl_log";
import { getScrapeQueue } from "../../services/queue-service";
import { addScrapeJob } from "../../services/queue-jobs";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import { getJobPriority } from "../../lib/job-priority";
import { callWebhook } from "../../services/webhook";
+import { scrapeOptions as scrapeOptionsSchema } from "./types";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -34,18 +34,22 @@ export async function crawlController(
await logCrawl(id, req.auth.team_id);
- let { remainingCredits } = req.account;
+ let { remainingCredits } = req.account!;
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
if(!useDbAuthentication){
remainingCredits = Infinity;
}
- const crawlerOptions = legacyCrawlerOptions(req.body);
- const pageOptions = legacyScrapeOptions(req.body.scrapeOptions);
+ const crawlerOptions = {
+ ...req.body,
+ url: undefined,
+ scrapeOptions: undefined,
+ };
+ const scrapeOptions = req.body.scrapeOptions;
// TODO: @rafa, is this right? copied from v0
- if (Array.isArray(crawlerOptions.includes)) {
- for (const x of crawlerOptions.includes) {
+ if (Array.isArray(crawlerOptions.includePaths)) {
+ for (const x of crawlerOptions.includePaths) {
try {
new RegExp(x);
} catch (e) {
@@ -54,8 +58,8 @@ export async function crawlController(
}
}
- if (Array.isArray(crawlerOptions.excludes)) {
- for (const x of crawlerOptions.excludes) {
+ if (Array.isArray(crawlerOptions.excludePaths)) {
+ for (const x of crawlerOptions.excludePaths) {
try {
new RegExp(x);
} catch (e) {
@@ -68,8 +72,9 @@ export async function crawlController(
const sc: StoredCrawl = {
originUrl: req.body.url,
- crawlerOptions,
- pageOptions,
+ crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),
+ scrapeOptions,
+ internalOptions: {},
team_id: req.auth.team_id,
createdAt: Date.now(),
plan: req.auth.plan,
@@ -78,9 +83,9 @@ export async function crawlController(
const crawler = crawlToCrawler(id, sc);
try {
- sc.robots = await crawler.getRobotsTxt(pageOptions.skipTlsVerification);
+ sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification);
} catch (e) {
- Logger.debug(
+ logger.debug(
`[Crawl] Failed to get robots.txt (this is probably fine!): ${JSON.stringify(
e
)}`
@@ -112,7 +117,7 @@ export async function crawlController(
team_id: req.auth.team_id,
plan: req.auth.plan,
crawlerOptions,
- pageOptions,
+ scrapeOptions,
origin: "api",
crawl_id: id,
sitemapped: true,
@@ -142,10 +147,10 @@ export async function crawlController(
{
url: req.body.url,
mode: "single_urls",
- crawlerOptions: crawlerOptions,
team_id: req.auth.team_id,
- plan: req.auth.plan,
- pageOptions: pageOptions,
+ crawlerOptions,
+ scrapeOptions: scrapeOptionsSchema.parse(scrapeOptions),
+ plan: req.auth.plan!,
origin: "api",
crawl_id: id,
webhook: req.body.webhook,
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 5ed3dd51c..458565438 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -1,9 +1,9 @@
import { Response } from "express";
import { v4 as uuidv4 } from "uuid";
import {
- legacyCrawlerOptions,
mapRequestSchema,
RequestWithAuth,
+ scrapeOptions,
} from "./types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
import { MapResponse, MapRequest } from "./types";
@@ -18,11 +18,11 @@ import { fireEngineMap } from "../../search/fireEngine";
import { billTeam } from "../../services/billing/credit_billing";
import { logJob } from "../../services/logging/log_job";
import { performCosineSimilarity } from "../../lib/map-cosine";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import Redis from "ioredis";
configDotenv();
-const redis = new Redis(process.env.REDIS_URL);
+const redis = new Redis(process.env.REDIS_URL!);
// Max Links that /map can return
const MAX_MAP_LIMIT = 5000;
@@ -44,8 +44,12 @@ export async function mapController(
const sc: StoredCrawl = {
originUrl: req.body.url,
- crawlerOptions: legacyCrawlerOptions(req.body),
- pageOptions: {},
+ crawlerOptions: {
+ ...req.body,
+ scrapeOptions: undefined,
+ },
+ scrapeOptions: scrapeOptions.parse({}),
+ internalOptions: {},
team_id: req.auth.team_id,
createdAt: Date.now(),
plan: req.auth.plan,
@@ -65,8 +69,8 @@ export async function mapController(
const cacheKey = `fireEngineMap:${mapUrl}`;
const cachedResult = null;
- let allResults: any[];
- let pagePromises: Promise<any>[];
+ let allResults: any[] = [];
+ let pagePromises: Promise<any>[] = [];
if (cachedResult) {
allResults = JSON.parse(cachedResult);
@@ -139,7 +143,7 @@ export async function mapController(
return null;
}
})
- .filter((x) => x !== null);
+ .filter((x) => x !== null) as string[];
// allows for subdomains to be included
links = links.filter((x) => isSameDomain(x, req.body.url));
@@ -153,7 +157,7 @@ export async function mapController(
links = removeDuplicateUrls(links);
billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {
- Logger.error(
+ logger.error(
`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`
);
// Optionally, you could notify an admin or add to a retry queue here
@@ -175,9 +179,8 @@ export async function mapController(
mode: "map",
url: req.body.url,
crawlerOptions: {},
- pageOptions: {},
+ scrapeOptions: {},
origin: req.body.origin,
- extractor_options: { mode: "markdown" },
num_tokens: 0,
});
diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.ts
index 5e0aecb6a..db50f7d3d 100644
--- a/apps/api/src/controllers/v1/scrape-status.ts
+++ b/apps/api/src/controllers/v1/scrape-status.ts
@@ -12,7 +12,7 @@ export async function scrapeStatusController(req: any, res: any) {
const job = await supabaseGetJobByIdOnlyData(req.params.jobId);
- if(job.team_id !== "41bdbfe1-0579-4d9b-b6d5-809f16be12f5"){
+ if(job?.team_id !== "41bdbfe1-0579-4d9b-b6d5-809f16be12f5"){
return res.status(403).json({
success: false,
error: "You are not allowed to access this resource.",
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 6fa855f7a..3f6609afb 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -1,10 +1,7 @@
-import { Request, Response } from "express";
-import { Logger } from "../../lib/logger";
+import { Response } from "express";
+import { logger } from "../../lib/logger";
import {
Document,
- legacyDocumentConverter,
- legacyExtractorOptions,
- legacyScrapeOptions,
RequestWithAuth,
ScrapeRequest,
scrapeRequestSchema,
@@ -12,7 +9,6 @@ import {
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
import { v4 as uuidv4 } from "uuid";
-import { numTokensFromString } from "../../lib/LLM-extraction/helpers";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
import { getJobPriority } from "../../lib/job-priority";
@@ -28,8 +24,6 @@ export async function scrapeController(
const origin = req.body.origin;
const timeout = req.body.timeout;
- const pageOptions = legacyScrapeOptions(req.body);
- const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined;
const jobId = uuidv4();
const startTime = new Date().getTime();
@@ -43,11 +37,10 @@ export async function scrapeController(
{
url: req.body.url,
mode: "single_urls",
- crawlerOptions: {},
team_id: req.auth.team_id,
- plan: req.auth.plan,
- pageOptions,
- extractorOptions,
+ scrapeOptions: req.body,
+ internalOptions: {},
+ plan: req.auth.plan!,
origin: req.body.origin,
is_scrape: true,
},
@@ -56,13 +49,13 @@ export async function scrapeController(
jobPriority
);
- const totalWait = (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds : 0) + a, 0);
+ const totalWait = (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds ?? 0 : 0) + a, 0);
- let doc: any | undefined;
+ let doc: Document;
try {
- doc = (await waitForJob(jobId, timeout + totalWait))[0];
+ doc = await waitForJob<Document>(jobId, timeout + totalWait); // TODO: better types for this
} catch (e) {
- Logger.error(`Error in scrapeController: ${e}`);
+ logger.error(`Error in scrapeController: ${e}`);
if (e instanceof Error && e.message.startsWith("Job wait")) {
return res.status(408).json({
success: false,
@@ -71,34 +64,19 @@ export async function scrapeController(
} else {
return res.status(500).json({
success: false,
- error: `(Internal server error) - ${e && e?.message ? e.message : e} ${
- extractorOptions && extractorOptions.mode !== "markdown"
- ? " - Could be due to LLM parsing issues"
- : ""
- }`,
+ error: `(Internal server error) - ${e && e?.message ? e.message : e}`,
});
}
}
await getScrapeQueue().remove(jobId);
- if (!doc) {
- console.error("!!! PANIC DOC IS", doc);
- return res.status(200).json({
- success: true,
- warning: "No page found",
- data: doc,
- });
- }
-
- delete doc.index;
- delete doc.provider;
-
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - startTime) / 1000;
const numTokens =
- doc && doc.markdown
- ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
+ doc && doc.extract
+ // ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
+ ? 0 // TODO: fix
: 0;
let creditsToBeBilled = 1; // Assuming 1 credit per document
@@ -111,22 +89,16 @@ export async function scrapeController(
}
billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(error => {
- Logger.error(`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`);
+ logger.error(`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`);
// Optionally, you could notify an admin or add to a retry queue here
});
- if (!pageOptions || !pageOptions.includeRawHtml) {
+ if (!req.body.formats.includes("rawHtml")) {
if (doc && doc.rawHtml) {
delete doc.rawHtml;
}
}
- if(pageOptions && pageOptions.includeExtract) {
- if(!pageOptions.includeMarkdown && doc && doc.markdown) {
- delete doc.markdown;
- }
- }
-
logJob({
job_id: jobId,
success: true,
@@ -137,16 +109,14 @@ export async function scrapeController(
team_id: req.auth.team_id,
mode: "scrape",
url: req.body.url,
- crawlerOptions: {},
- pageOptions: pageOptions,
+ scrapeOptions: req.body,
origin: origin,
- extractor_options: extractorOptions,
num_tokens: numTokens,
});
return res.status(200).json({
success: true,
- data: legacyDocumentConverter(doc),
+ data: doc,
scrape_id: origin?.includes("website") ? jobId : undefined,
});
}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 9585175e8..82b24c222 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1,10 +1,11 @@
import { Request, Response } from "express";
import { z } from "zod";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import { Action, ExtractorOptions, PageOptions } from "../../lib/entities";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { PlanType } from "../../types";
import { countries } from "../../lib/validate-country";
+import { ExtractorOptions, PageOptions, ScrapeActionContent, Document as V0Document } from "../../lib/entities";
+import { InternalOptions } from "../../scraper/scrapeURL";
export type Format =
| "markdown"
@@ -167,6 +168,7 @@ export const scrapeRequestSchema = scrapeOptions.extend({
});
export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
+export type ScrapeRequestInput = z.input<typeof scrapeRequestSchema>;
export const batchScrapeRequestSchema = scrapeOptions.extend({
urls: url.array(),
@@ -240,7 +242,7 @@ export const mapRequestSchema = crawlerOptions.extend({
includeSubdomains: z.boolean().default(true),
search: z.string().optional(),
ignoreSitemap: z.boolean().default(false),
- limit: z.number().min(1).max(5000).default(5000).optional(),
+ limit: z.number().min(1).max(5000).default(5000),
}).strict(strictMessage);
// export type MapRequest = {
@@ -252,13 +254,14 @@ export type MapRequest = z.infer<typeof mapRequestSchema>;
export type Document = {
markdown?: string;
- extract?: string;
+ extract?: any;
html?: string;
rawHtml?: string;
links?: string[];
screenshot?: string;
actions?: {
- screenshots: string[];
+ screenshots?: string[];
+ scrapes?: ScrapeActionContent[];
};
warning?: string;
metadata: {
@@ -291,11 +294,11 @@ export type Document = {
publishedTime?: string;
articleTag?: string;
articleSection?: string;
+ url?: string;
sourceURL?: string;
statusCode?: number;
error?: string;
[key: string]: string | string[] | number | undefined;
-
};
};
@@ -366,7 +369,7 @@ export type CrawlStatusResponse =
type AuthObject = {
team_id: string;
- plan: PlanType;
+ plan: PlanType | undefined;
};
type Account = {
@@ -439,7 +442,7 @@ export interface ResponseWithSentry<
sentry?: string,
}
-export function legacyCrawlerOptions(x: CrawlerOptions) {
+export function toLegacyCrawlerOptions(x: CrawlerOptions) {
return {
includes: x.includePaths,
excludes: x.excludePaths,
@@ -453,68 +456,90 @@ export function legacyCrawlerOptions(x: CrawlerOptions) {
};
}
-export function legacyScrapeOptions(x: ScrapeOptions): PageOptions {
+export function fromLegacyCrawlerOptions(x: any): { crawlOptions: CrawlerOptions; internalOptions: InternalOptions } {
return {
- includeMarkdown: x.formats.includes("markdown"),
- includeHtml: x.formats.includes("html"),
- includeRawHtml: x.formats.includes("rawHtml"),
- includeExtract: x.formats.includes("extract"),
- onlyIncludeTags: x.includeTags,
- removeTags: x.excludeTags,
- onlyMainContent: x.onlyMainContent,
- waitFor: x.waitFor,
- headers: x.headers,
- includeLinks: x.formats.includes("links"),
- screenshot: x.formats.includes("screenshot"),
- fullPageScreenshot: x.formats.includes("screenshot@fullPage"),
- parsePDF: x.parsePDF,
- actions: x.actions as Action[], // no strict null checking grrrr - mogery
- geolocation: x.location ?? x.geolocation,
- skipTlsVerification: x.skipTlsVerification,
- removeBase64Images: x.removeBase64Images,
- mobile: x.mobile,
+ crawlOptions: crawlerOptions.parse({
+ includePaths: x.includes,
+ excludePaths: x.excludes,
+ limit: x.maxCrawledLinks ?? x.limit,
+ maxDepth: x.maxDepth,
+ allowBackwardLinks: x.allowBackwardCrawling,
+ allowExternalLinks: x.allowExternalContentLinks,
+ ignoreSitemap: x.ignoreSitemap,
+ // TODO: returnOnlyUrls support
+ }),
+ internalOptions: {
+ v0CrawlOnlyUrls: x.returnOnlyUrls,
+ },
};
}
-export function legacyExtractorOptions(x: ExtractOptions): ExtractorOptions {
+export function fromLegacyScrapeOptions(pageOptions: PageOptions, extractorOptions: ExtractorOptions | undefined, timeout: number | undefined): { scrapeOptions: ScrapeOptions, internalOptions: InternalOptions } {
return {
- mode: x.mode ? "llm-extraction" : "markdown",
- extractionPrompt: x.prompt ?? "Based on the information on the page, extract the information from the schema.",
- extractionSchema: x.schema,
- userPrompt: x.prompt ?? "",
- };
+ scrapeOptions: scrapeOptions.parse({
+ formats: [
+ (pageOptions.includeMarkdown ?? true) ? "markdown" as const : null,
+ (pageOptions.includeHtml ?? false) ? "html" as const : null,
+ (pageOptions.includeRawHtml ?? false) ? "rawHtml" as const : null,
+ (pageOptions.screenshot ?? false) ? "screenshot" as const : null,
+ (pageOptions.fullPageScreenshot ?? false) ? "screenshot@fullPage" as const : null,
+ (extractorOptions !== undefined && extractorOptions.mode.includes("llm-extraction")) ? "extract" as const : null,
+ "links"
+ ].filter(x => x !== null),
+ waitFor: pageOptions.waitFor,
+ headers: pageOptions.headers,
+ includeTags: (typeof pageOptions.onlyIncludeTags === "string" ? [pageOptions.onlyIncludeTags] : pageOptions.onlyIncludeTags),
+ excludeTags: (typeof pageOptions.removeTags === "string" ? [pageOptions.removeTags] : pageOptions.removeTags),
+ onlyMainContent: pageOptions.onlyMainContent ?? false,
+ timeout: timeout,
+ parsePDF: pageOptions.parsePDF,
+ actions: pageOptions.actions,
+ location: pageOptions.geolocation,
+ skipTlsVerification: pageOptions.skipTlsVerification,
+ removeBase64Images: pageOptions.removeBase64Images,
+ extract: extractorOptions !== undefined && extractorOptions.mode.includes("llm-extraction") ? {
+ systemPrompt: extractorOptions.extractionPrompt,
+ prompt: extractorOptions.userPrompt,
+ schema: extractorOptions.extractionSchema,
+ } : undefined,
+ mobile: pageOptions.mobile,
+ }),
+ internalOptions: {
+ atsv: pageOptions.atsv,
+ v0DisableJsDom: pageOptions.disableJsDom,
+ v0UseFastMode: pageOptions.useFastMode,
+ },
+ // TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
+ }
}
-export function legacyDocumentConverter(doc: any): Document {
- if (doc === null || doc === undefined) return null;
-
- if (doc.metadata) {
- if (doc.metadata.screenshot) {
- doc.screenshot = doc.metadata.screenshot;
- delete doc.metadata.screenshot;
- }
+export function fromLegacyCombo(pageOptions: PageOptions, extractorOptions: ExtractorOptions | undefined, timeout: number | undefined, crawlerOptions: any): { scrapeOptions: ScrapeOptions, internalOptions: InternalOptions} {
+ const { scrapeOptions, internalOptions: i1 } = fromLegacyScrapeOptions(pageOptions, extractorOptions, timeout);
+ const { internalOptions: i2 } = fromLegacyCrawlerOptions(crawlerOptions);
+ return { scrapeOptions, internalOptions: Object.assign(i1, i2) };
+}
- if (doc.metadata.fullPageScreenshot) {
- doc.fullPageScreenshot = doc.metadata.fullPageScreenshot;
- delete doc.metadata.fullPageScreenshot;
- }
+export function toLegacyDocument(document: Document, internalOptions: InternalOptions): V0Document | { url: string; } {
+ if (internalOptions.v0CrawlOnlyUrls) {
+ return { url: document.metadata.sourceURL! };
}
return {
- markdown: doc.markdown,
- links: doc.linksOnPage,
- rawHtml: doc.rawHtml,
- html: doc.html,
- extract: doc.llm_extraction,
- screenshot: doc.screenshot ?? doc.fullPageScreenshot,
- actions: doc.actions ?? undefined,
- warning: doc.warning ?? undefined,
+ content: document.markdown!,
+ markdown: document.markdown!,
+ html: document.html,
+ rawHtml: document.rawHtml,
+ linksOnPage: document.links,
+ llm_extraction: document.extract,
metadata: {
- ...doc.metadata,
- pageError: undefined,
- pageStatusCode: undefined,
- error: doc.metadata?.pageError,
- statusCode: doc.metadata?.pageStatusCode,
+ ...document.metadata,
+ error: undefined,
+ statusCode: undefined,
+ pageError: document.metadata.error,
+ pageStatusCode: document.metadata.statusCode,
+ screenshot: document.screenshot,
},
- };
+ actions: document.actions ,
+ warning: document.warning,
+ }
}
diff --git a/apps/api/src/example.ts b/apps/api/src/example.ts
deleted file mode 100644
index edf0faefd..000000000
--- a/apps/api/src/example.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import { WebScraperDataProvider } from "./scraper/WebScraper";
-
-async function example() {
- const example = new WebScraperDataProvider();
-
- await example.setOptions({
- jobId: "TEST",
- mode: "crawl",
- urls: ["https://mendable.ai"],
- crawlerOptions: {},
- });
- const docs = await example.getDocuments(false);
- docs.map((doc) => {
- console.log(doc.metadata.sourceURL);
- });
- console.log(docs.length);
-}
-
-// example();
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 5ccbb9cc5..7f7ec036c 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -6,28 +6,24 @@ import bodyParser from "body-parser";
import cors from "cors";
import { getScrapeQueue } from "./services/queue-service";
import { v0Router } from "./routes/v0";
-import { initSDK } from "@hyperdx/node-opentelemetry";
import os from "os";
-import { Logger } from "./lib/logger";
+import { logger } from "./lib/logger";
import { adminRouter } from "./routes/admin";
-import { ScrapeEvents } from "./lib/scrape-events";
import http from 'node:http';
import https from 'node:https';
import CacheableLookup from 'cacheable-lookup';
import { v1Router } from "./routes/v1";
import expressWs from "express-ws";
-import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws";
import { ErrorResponse, ResponseWithSentry } from "./controllers/v1/types";
import { ZodError } from "zod";
import { v4 as uuidv4 } from "uuid";
-import dns from 'node:dns';
const { createBullBoard } = require("@bull-board/api");
const { BullAdapter } = require("@bull-board/api/bullAdapter");
const { ExpressAdapter } = require("@bull-board/express");
const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;
-Logger.info(`Number of CPUs: ${numCPUs} available`);
+logger.info(`Number of CPUs: ${numCPUs} available`);
const cacheable = new CacheableLookup()
@@ -55,7 +51,6 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
serverAdapter: serverAdapter,
});
-
app.use(
`/admin/${process.env.BULL_AUTH_KEY}/queues`,
serverAdapter.getRouter()
@@ -78,15 +73,10 @@ app.use(adminRouter);
const DEFAULT_PORT = process.env.PORT ?? 3002;
const HOST = process.env.HOST ?? "localhost";
-// HyperDX OpenTelemetry
-if (process.env.ENV === "production") {
- initSDK({ consoleCapture: true, additionalInstrumentations: [] });
-}
-
function startServer(port = DEFAULT_PORT) {
const server = app.listen(Number(port), HOST, () => {
- Logger.info(`Worker ${process.pid} listening on port ${port}`);
- Logger.info(
+ logger.info(`Worker ${process.pid} listening on port ${port}`);
+ logger.info(
`For the Queue UI, open: http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`
);
});
@@ -103,7 +93,6 @@ app.get(`/serverHealthCheck`, async (req, res) => {
const [waitingJobs] = await Promise.all([
scrapeQueue.getWaitingCount(),
]);
-
const noWaitingJobs = waitingJobs === 0;
// 200 if no active jobs, 503 if there are active jobs
return res.status(noWaitingJobs ? 200 : 500).json({
@@ -111,7 +100,7 @@ app.get(`/serverHealthCheck`, async (req, res) => {
});
} catch (error) {
Sentry.captureException(error);
- Logger.error(error);
+ logger.error(error);
return res.status(500).json({ error: error.message });
}
});
@@ -140,7 +129,7 @@ app.get("/serverHealthCheck/notify", async (req, res) => {
// Re-check the waiting jobs count after the timeout
waitingJobsCount = await getWaitingJobsCount();
if (waitingJobsCount >= treshold) {
- const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL;
+ const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL!;
const message = {
text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${
timeout / 60000
@@ -156,14 +145,14 @@ app.get("/serverHealthCheck/notify", async (req, res) => {
});
if (!response.ok) {
- Logger.error("Failed to send Slack notification");
+ logger.error("Failed to send Slack notification");
}
}
}, timeout);
}
} catch (error) {
Sentry.captureException(error);
- Logger.debug(error);
+ logger.debug(error);
}
};
@@ -178,7 +167,7 @@ app.get("/is-production", (req, res) => {
app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response<ErrorResponse>, next: NextFunction) => {
if (err instanceof ZodError) {
if (Array.isArray(err.errors) && err.errors.find(x => x.message === "URL uses unsupported protocol")) {
- Logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));
+ logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));
}
res.status(400).json({ success: false, error: "Bad Request", details: err.errors });
@@ -206,11 +195,11 @@ app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response
}
}
- Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose);
+ logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose);
res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " + id });
});
-Logger.info(`Worker ${process.pid} started`);
+logger.info(`Worker ${process.pid} started`);
// const sq = getScrapeQueue();
diff --git a/apps/api/src/lib/LLM-extraction/index.ts b/apps/api/src/lib/LLM-extraction/index.ts
index d05f9bd7a..430dc1d42 100644
--- a/apps/api/src/lib/LLM-extraction/index.ts
+++ b/apps/api/src/lib/LLM-extraction/index.ts
@@ -4,19 +4,19 @@ const ajv = new Ajv(); // Initialize AJV for JSON schema validation
import { generateOpenAICompletions } from "./models";
import { Document, ExtractorOptions } from "../entities";
-import { Logger } from "../logger";
+import { logger } from "../logger";
// Generate completion using OpenAI
export async function generateCompletions(
documents: Document[],
- extractionOptions: ExtractorOptions,
+ extractionOptions: ExtractorOptions | undefined,
mode: "markdown" | "raw-html"
): Promise<Document[]> {
// const schema = zodToJsonSchema(options.schema)
- const schema = extractionOptions.extractionSchema;
- const systemPrompt = extractionOptions.extractionPrompt;
- const prompt = extractionOptions.userPrompt;
+ const schema = extractionOptions?.extractionSchema;
+ const systemPrompt = extractionOptions?.extractionPrompt;
+ const prompt = extractionOptions?.userPrompt;
const switchVariable = "openAI"; // Placholder, want to think more about how we abstract the model provider
@@ -51,7 +51,7 @@ export async function generateCompletions(
return completionResult;
} catch (error) {
- Logger.error(`Error generating completions: ${error}`);
+ logger.error(`Error generating completions: ${error}`);
throw error;
}
default:
diff --git a/apps/api/src/lib/LLM-extraction/models.ts b/apps/api/src/lib/LLM-extraction/models.ts
index 23147b12c..f777dce96 100644
--- a/apps/api/src/lib/LLM-extraction/models.ts
+++ b/apps/api/src/lib/LLM-extraction/models.ts
@@ -95,7 +95,7 @@ export async function generateOpenAICompletions({
try {
llmExtraction = JSON.parse(
- jsonCompletion.choices[0].message.content.trim()
+ (jsonCompletion.choices[0].message.content ?? "").trim()
);
} catch (e) {
throw new Error("Invalid JSON");
diff --git a/apps/api/src/lib/batch-process.ts b/apps/api/src/lib/batch-process.ts
index 30289fd0c..802d1eb1a 100644
--- a/apps/api/src/lib/batch-process.ts
+++ b/apps/api/src/lib/batch-process.ts
@@ -3,7 +3,7 @@ export async function batchProcess<T>(
batchSize: number,
asyncFunction: (item: T, index: number) => Promise<void>
): Promise<void> {
- const batches = [];
+ const batches: T[][] = [];
for (let i = 0; i < array.length; i += batchSize) {
const batch = array.slice(i, i + batchSize);
batches.push(batch);
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 379bc1797..41cbb07c8 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -1,13 +1,16 @@
+import { InternalOptions } from "../scraper/scrapeURL";
+import { ScrapeOptions } from "../controllers/v1/types";
import { WebCrawler } from "../scraper/WebScraper/crawler";
import { redisConnection } from "../services/queue-service";
-import { Logger } from "./logger";
+import { logger } from "./logger";
export type StoredCrawl = {
originUrl?: string;
crawlerOptions: any;
- pageOptions: any;
+ scrapeOptions: Omit<ScrapeOptions, "timeout">;
+ internalOptions: InternalOptions;
team_id: string;
- plan: string;
+ plan?: string;
robots?: string;
cancelled?: boolean;
createdAt: number;
@@ -100,7 +103,7 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
urlO.hash = "";
url = urlO.href;
} catch (error) {
- Logger.warn("Failed to normalize URL " + JSON.stringify(url) + ": " + error);
+ logger.warn("Failed to normalize URL " + JSON.stringify(url) + ": " + error);
}
const res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0
@@ -117,7 +120,7 @@ export async function lockURLs(id: string, urls: string[]): Promise<boolean> {
urlO.hash = "";
return urlO.href;
} catch (error) {
- Logger.warn("Failed to normalize URL " + JSON.stringify(url) + ": " + error);
+ logger.warn("Failed to normalize URL " + JSON.stringify(url) + ": " + error);
}
return url;
@@ -131,7 +134,7 @@ export async function lockURLs(id: string, urls: string[]): Promise<boolean> {
export function crawlToCrawler(id: string, sc: StoredCrawl): WebCrawler {
const crawler = new WebCrawler({
jobId: id,
- initialUrl: sc.originUrl,
+ initialUrl: sc.originUrl!,
includes: sc.crawlerOptions?.includes ?? [],
excludes: sc.crawlerOptions?.excludes ?? [],
maxCrawledLinks: sc.crawlerOptions?.maxCrawledLinks ?? 1000,
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 2c8f376de..9b68f4256 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -1,3 +1,5 @@
+import type { Document as V1Document } from "../controllers/v1/types";
+
export interface Progress {
current: number;
total: number;
@@ -129,7 +131,8 @@ export class Document {
provider?: string;
warning?: string;
actions?: {
- screenshots: string[];
+ screenshots?: string[];
+ scrapes?: ScrapeActionContent[];
}
index?: number;
diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts
index 396b7fe72..8800d9167 100644
--- a/apps/api/src/lib/html-to-markdown.ts
+++ b/apps/api/src/lib/html-to-markdown.ts
@@ -5,7 +5,7 @@ import "../services/sentry"
import * as Sentry from "@sentry/node";
import dotenv from 'dotenv';
-import { Logger } from './logger';
+import { logger } from './logger';
dotenv.config();
// TODO: add a timeout to the Go parser
@@ -40,7 +40,7 @@ class GoMarkdownConverter {
}
}
-export async function parseMarkdown(html: string): Promise<string> {
+export async function parseMarkdown(html: string | null | undefined): Promise<string> {
if (!html) {
return '';
}
@@ -52,12 +52,12 @@ export async function parseMarkdown(html: string): Promise<string> {
markdownContent = processMultiLineLinks(markdownContent);
markdownContent = removeSkipToContentLinks(markdownContent);
- Logger.info(`HTML to Markdown conversion using Go parser successful`);
+ logger.info(`HTML to Markdown conversion using Go parser successful`);
return markdownContent;
}
} catch (error) {
Sentry.captureException(error);
- Logger.error(`Error converting HTML to Markdown with Go parser: ${error}`);
+ logger.error(`Error converting HTML to Markdown with Go parser: ${error}`);
}
// Fallback to TurndownService if Go parser fails or is not enabled
diff --git a/apps/api/src/lib/job-priority.ts b/apps/api/src/lib/job-priority.ts
index 9d046052a..27e452301 100644
--- a/apps/api/src/lib/job-priority.ts
+++ b/apps/api/src/lib/job-priority.ts
@@ -1,6 +1,6 @@
import { redisConnection } from "../../src/services/queue-service";
import { PlanType } from "../../src/types";
-import { Logger } from "./logger";
+import { logger } from "./logger";
const SET_KEY_PREFIX = "limit_team_id:";
export async function addJobPriority(team_id, job_id) {
@@ -13,7 +13,7 @@ export async function addJobPriority(team_id, job_id) {
// This approach will reset the expiration time to 60 seconds every time a new job is added to the set.
await redisConnection.expire(setKey, 60);
} catch (e) {
- Logger.error(`Add job priority (sadd) failed: ${team_id}, ${job_id}`);
+ logger.error(`Add job priority (sadd) failed: ${team_id}, ${job_id}`);
}
}
@@ -24,7 +24,7 @@ export async function deleteJobPriority(team_id, job_id) {
// remove job_id from the set
await redisConnection.srem(setKey, job_id);
} catch (e) {
- Logger.error(`Delete job priority (srem) failed: ${team_id}, ${job_id}`);
+ logger.error(`Delete job priority (srem) failed: ${team_id}, ${job_id}`);
}
}
@@ -33,7 +33,7 @@ export async function getJobPriority({
team_id,
basePriority = 10,
}: {
- plan: PlanType;
+ plan: PlanType | undefined;
team_id: string;
basePriority?: number;
}): Promise<number> {
@@ -95,7 +95,7 @@ export async function getJobPriority({
);
}
} catch (e) {
- Logger.error(
+ logger.error(
`Get job priority failed: ${team_id}, ${plan}, ${basePriority}`
);
return basePriority;
diff --git a/apps/api/src/lib/load-testing-example.ts b/apps/api/src/lib/load-testing-example.ts
deleted file mode 100644
index 01b61db92..000000000
--- a/apps/api/src/lib/load-testing-example.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-// import { scrapWithFireEngine } from "../../src/scraper/WebScraper/single_url";
-
-// const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-// const scrapInBatches = async (
-// urls: string[],
-// batchSize: number,
-// delayMs: number
-// ) => {
-// let successCount = 0;
-// let errorCount = 0;
-
-// for (let i = 0; i < urls.length; i += batchSize) {
-// const batch = urls
-// .slice(i, i + batchSize)
-// .map((url) => scrapWithFireEngine(url));
-// try {
-// const results = await Promise.all(batch);
-// results.forEach((data, index) => {
-// if (data.trim() === "") {
-// errorCount++;
-// } else {
-// successCount++;
-// console.log(
-// `Scraping result ${i + index + 1}:`,
-// data.trim().substring(0, 20) + "..."
-// );
-// }
-// });
-// } catch (error) {
-// console.error("Error during scraping:", error);
-// }
-// await delay(delayMs);
-// }
-
-// console.log(`Total successful scrapes: ${successCount}`);
-// console.log(`Total errored scrapes: ${errorCount}`);
-// };
-// function run() {
-// const urls = Array.from({ length: 200 }, () => "https://scrapethissite.com");
-// scrapInBatches(urls, 10, 1000);
-// }
diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts
index 7eca1ef0e..33aa949bf 100644
--- a/apps/api/src/lib/logger.ts
+++ b/apps/api/src/lib/logger.ts
@@ -1,57 +1,82 @@
+import * as winston from "winston";
+
import { configDotenv } from "dotenv";
+import Transport from "winston-transport";
configDotenv();
-enum LogLevel {
- NONE = 'NONE', // No logs will be output.
- ERROR = 'ERROR', // For logging error messages that indicate a failure in a specific operation.
- WARN = 'WARN', // For logging potentially harmful situations that are not necessarily errors.
- INFO = 'INFO', // For logging informational messages that highlight the progress of the application.
- DEBUG = 'DEBUG', // For logging detailed information on the flow through the system, primarily used for debugging.
- TRACE = 'TRACE' // For logging more detailed information than the DEBUG level.
-}
-export class Logger {
- static colors = {
- ERROR: '\x1b[31m%s\x1b[0m', // Red
- WARN: '\x1b[33m%s\x1b[0m', // Yellow
- INFO: '\x1b[34m%s\x1b[0m', // Blue
- DEBUG: '\x1b[36m%s\x1b[0m', // Cyan
- TRACE: '\x1b[35m%s\x1b[0m' // Magenta
- };
-
- static log (message: string, level: LogLevel) {
- const logLevel: LogLevel = LogLevel[process.env.LOGGING_LEVEL as keyof typeof LogLevel] || LogLevel.TRACE;
- const levels = [LogLevel.NONE, LogLevel.ERROR, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG, LogLevel.TRACE];
- const currentLevelIndex = levels.indexOf(logLevel);
- const messageLevelIndex = levels.indexOf(level);
-
- if (currentLevelIndex >= messageLevelIndex) {
- const color = Logger.colors[level];
- console[level.toLowerCase()](color, `[${new Date().toISOString()}]${level} - ${message}`);
-
- // const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
- // if (useDbAuthentication) {
- // save to supabase? another place?
- // supabase.from('logs').insert({ level: level, message: message, timestamp: new Date().toISOString(), success: boolean });
- // }
+const logFormat = winston.format.printf(info =>
+ `${info.timestamp} ${info.level} [${info.metadata.module ?? ""}:${info.metadata.method ?? ""}]: ${info.message} ${info.level.includes("error") || info.level.includes("warn") ? JSON.stringify(
+ info.metadata,
+ (_, value) => {
+ if (value instanceof Error) {
+ return {
+ ...value,
+ name: value.name,
+ message: value.message,
+ stack: value.stack,
+ cause: value.cause,
+ }
+ } else {
+ return value;
+ }
}
- }
- static error(message: string | any) {
- Logger.log(message, LogLevel.ERROR);
- }
+ ) : ""}`
+)
- static warn(message: string) {
- Logger.log(message, LogLevel.WARN);
- }
+export const logger = winston.createLogger({
+ level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug",
+ format: winston.format.json({
+ replacer(key, value) {
+ if (value instanceof Error) {
+ return {
+ ...value,
+ name: value.name,
+ message: value.message,
+ stack: value.stack,
+ cause: value.cause,
+ }
+ } else {
+ return value;
+ }
+ }
+ }),
+ transports: [
+ new winston.transports.Console({
+ format: winston.format.combine(
+ winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
+ winston.format.metadata({ fillExcept: ["message", "level", "timestamp"] }),
+ ...(((process.env.ENV === "production" && process.env.SENTRY_ENVIRONMENT === "dev") || (process.env.ENV !== "production")) ? [winston.format.colorize(), logFormat] : []),
+ ),
+ }),
+ ],
+});
- static info(message: string) {
- Logger.log(message, LogLevel.INFO);
- }
+export type ArrayTransportOptions = Transport.TransportStreamOptions & {
+ array: any[];
+ scrapeId?: string;
+};
- static debug(message: string) {
- Logger.log(message, LogLevel.DEBUG);
+export class ArrayTransport extends Transport {
+ private array: any[];
+ private scrapeId?: string;
+
+ constructor(opts: ArrayTransportOptions) {
+ super(opts);
+ this.array = opts.array;
+ this.scrapeId = opts.scrapeId;
}
- static trace(message: string) {
- Logger.log(message, LogLevel.TRACE);
+ log(info, next) {
+ setImmediate(() => {
+ this.emit("logged", info);
+ });
+
+ if (this.scrapeId !== undefined && info.scrapeId !== this.scrapeId) {
+ return next();
+ }
+
+ this.array.push(info);
+
+ next();
}
-}
+}
\ No newline at end of file
diff --git a/apps/api/src/lib/map-cosine.ts b/apps/api/src/lib/map-cosine.ts
index db2491a93..2a089548f 100644
--- a/apps/api/src/lib/map-cosine.ts
+++ b/apps/api/src/lib/map-cosine.ts
@@ -1,4 +1,4 @@
-import { Logger } from "./logger";
+import { logger } from "./logger";
export function performCosineSimilarity(links: string[], searchQuery: string) {
try {
@@ -40,7 +40,7 @@ export function performCosineSimilarity(links: string[], searchQuery: string) {
links = a.map((item) => item.link);
return links;
} catch (error) {
- Logger.error(`Error performing cosine similarity: ${error}`);
+ logger.error(`Error performing cosine similarity: ${error}`);
return links;
}
}
diff --git a/apps/api/src/lib/scrape-events.ts b/apps/api/src/lib/scrape-events.ts
index ad70dfefe..83873a58a 100644
--- a/apps/api/src/lib/scrape-events.ts
+++ b/apps/api/src/lib/scrape-events.ts
@@ -1,8 +1,8 @@
import { Job } from "bullmq";
-import type { baseScrapers } from "../scraper/WebScraper/single_url";
import { supabase_service as supabase } from "../services/supabase";
-import { Logger } from "./logger";
+import { logger } from "./logger";
import { configDotenv } from "dotenv";
+import { Engine } from "../scraper/scrapeURL/engines";
configDotenv();
export type ScrapeErrorEvent = {
@@ -15,7 +15,7 @@ export type ScrapeScrapeEvent = {
type: "scrape",
url: string,
worker?: string,
- method: (typeof baseScrapers)[number],
+ method: Engine,
result: null | {
success: boolean,
response_code?: number,
@@ -49,7 +49,7 @@ export class ScrapeEvents {
}).select().single();
return (result.data as any).id;
} catch (error) {
- // Logger.error(`Error inserting scrape event: ${error}`);
+ // logger.error(`Error inserting scrape event: ${error}`);
return null;
}
}
@@ -69,7 +69,7 @@ export class ScrapeEvents {
}
}).eq("id", logId);
} catch (error) {
- Logger.error(`Error updating scrape result: ${error}`);
+ logger.error(`Error updating scrape result: ${error}`);
}
}
@@ -81,7 +81,7 @@ export class ScrapeEvents {
worker: process.env.FLY_MACHINE_ID,
});
} catch (error) {
- Logger.error(`Error logging job event: ${error}`);
+ logger.error(`Error logging job event: ${error}`);
}
}
}
diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts
index c418a6e09..c9be72a3c 100644
--- a/apps/api/src/lib/supabase-jobs.ts
+++ b/apps/api/src/lib/supabase-jobs.ts
@@ -1,5 +1,5 @@
import { supabase_service } from "../services/supabase";
-import { Logger } from "./logger";
+import { logger } from "./logger";
import * as Sentry from "@sentry/node";
/**
@@ -37,7 +37,7 @@ export const supabaseGetJobsById = async (jobIds: string[]) => {
.in("job_id", jobIds);
if (error) {
- Logger.error(`Error in supabaseGetJobsById: ${error}`);
+ logger.error(`Error in supabaseGetJobsById: ${error}`);
Sentry.captureException(error);
return [];
}
@@ -61,7 +61,7 @@ export const supabaseGetJobsByCrawlId = async (crawlId: string) => {
.eq("crawl_id", crawlId)
if (error) {
- Logger.error(`Error in supabaseGetJobsByCrawlId: ${error}`);
+ logger.error(`Error in supabaseGetJobsByCrawlId: ${error}`);
Sentry.captureException(error);
return [];
}
diff --git a/apps/api/src/lib/withAuth.ts b/apps/api/src/lib/withAuth.ts
index b45b8973a..a6cd539d5 100644
--- a/apps/api/src/lib/withAuth.ts
+++ b/apps/api/src/lib/withAuth.ts
@@ -1,30 +1,25 @@
import { AuthResponse } from "../../src/types";
-import { Logger } from "./logger";
+import { logger } from "./logger";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
configDotenv();
let warningCount = 0;
-export function withAuth<T extends AuthResponse, U extends any[]>(
- originalFunction: (...args: U) => Promise<T>
+export function withAuth<T, U extends any[]>(
+ originalFunction: (...args: U) => Promise<T>,
+ mockSuccess: T,
) {
return async function (...args: U): Promise<T> {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
if (!useDbAuthentication) {
if (warningCount < 5) {
- Logger.warn("You're bypassing authentication");
+ logger.warn("You're bypassing authentication");
warningCount++;
}
return { success: true } as T;
} else {
- try {
- return await originalFunction(...args);
- } catch (error) {
- Sentry.captureException(error);
- Logger.error(`Error in withAuth function: ${error}`);
- return { success: false, error: error.message } as T;
- }
+ return await originalFunction(...args);
}
};
}
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 8bd0c12c1..90d4a47f2 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -1,151 +1,127 @@
import { Job } from "bullmq";
import {
- CrawlResult,
WebScraperOptions,
RunWebScraperParams,
RunWebScraperResult,
} from "../types";
-import { WebScraperDataProvider } from "../scraper/WebScraper";
-import { DocumentUrl, Progress } from "../lib/entities";
import { billTeam } from "../services/billing/credit_billing";
-import { Document } from "../lib/entities";
+import { Document } from "../controllers/v1/types";
import { supabase_service } from "../services/supabase";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
import { ScrapeEvents } from "../lib/scrape-events";
import { configDotenv } from "dotenv";
+import { EngineResultsTracker, scrapeURL, ScrapeUrlResponse } from "../scraper/scrapeURL";
+import { Engine } from "../scraper/scrapeURL/engines";
configDotenv();
export async function startWebScraperPipeline({
job,
token,
}: {
- job: Job<WebScraperOptions>;
+ job: Job<WebScraperOptions> & { id: string };
token: string;
}) {
- let partialDocs: Document[] = [];
return (await runWebScraper({
url: job.data.url,
mode: job.data.mode,
- crawlerOptions: job.data.crawlerOptions,
- extractorOptions: job.data.extractorOptions,
- pageOptions: {
- ...job.data.pageOptions,
+ scrapeOptions: {
+ ...job.data.scrapeOptions,
...(job.data.crawl_id ? ({
- includeRawHtml: true,
+ formats: job.data.scrapeOptions.formats.concat(["rawHtml"]),
}): {}),
},
- inProgress: (progress) => {
- Logger.debug(`πŸ‚ Job in progress ${job.id}`);
- if (progress.currentDocument) {
- partialDocs.push(progress.currentDocument);
- if (partialDocs.length > 50) {
- partialDocs = partialDocs.slice(-50);
- }
- // job.updateProgress({ ...progress, partialDocs: partialDocs });
- }
- },
- onSuccess: (result, mode) => {
- Logger.debug(`πŸ‚ Job completed ${job.id}`);
- saveJob(job, result, token, mode);
- },
- onError: (error) => {
- Logger.error(`πŸ‚ Job failed ${job.id}`);
- ScrapeEvents.logJobEvent(job, "failed");
- job.moveToFailed(error, token, false);
- },
+ internalOptions: job.data.internalOptions,
+ // onSuccess: (result, mode) => {
+ // logger.debug(`πŸ‚ Job completed ${job.id}`);
+ // saveJob(job, result, token, mode);
+ // },
+ // onError: (error) => {
+ // logger.error(`πŸ‚ Job failed ${job.id}`);
+ // ScrapeEvents.logJobEvent(job, "failed");
+ // },
team_id: job.data.team_id,
bull_job_id: job.id.toString(),
priority: job.opts.priority,
is_scrape: job.data.is_scrape ?? false,
- })) as { success: boolean; message: string; docs: Document[] };
+ }));
}
export async function runWebScraper({
url,
mode,
- crawlerOptions,
- pageOptions,
- extractorOptions,
- inProgress,
- onSuccess,
- onError,
+ scrapeOptions,
+ internalOptions,
+ // onSuccess,
+ // onError,
team_id,
bull_job_id,
priority,
is_scrape=false,
-}: RunWebScraperParams): Promise<RunWebScraperResult> {
+}: RunWebScraperParams): Promise<ScrapeUrlResponse> {
+ let response: ScrapeUrlResponse | undefined = undefined;
+ let engines: EngineResultsTracker = {};
try {
- const provider = new WebScraperDataProvider();
- if (mode === "crawl") {
- await provider.setOptions({
- jobId: bull_job_id,
- mode: mode,
- urls: [url],
- extractorOptions,
- crawlerOptions: crawlerOptions,
- pageOptions: pageOptions,
- bullJobId: bull_job_id,
- priority,
- });
- } else {
- await provider.setOptions({
- jobId: bull_job_id,
- mode: mode,
- urls: url.split(","),
- extractorOptions,
- crawlerOptions: crawlerOptions,
- pageOptions: pageOptions,
- priority,
- teamId: team_id
- });
- }
- const docs = (await provider.getDocuments(false, (progress: Progress) => {
- inProgress(progress);
- })) as Document[];
-
- if (docs.length === 0) {
- return {
- success: true,
- message: "No pages found",
- docs: [],
- };
+ response = await scrapeURL(bull_job_id, url, scrapeOptions, { priority, ...internalOptions });
+ if (!response.success) {
+ if (response.error instanceof Error) {
+ throw response.error;
+ } else {
+ throw new Error("scrapeURL error: " + (Array.isArray(response.error) ? JSON.stringify(response.error) : typeof response.error === "object" ? JSON.stringify({ ...response.error }) : response.error));
+ }
}
- // remove docs with empty content
- const filteredDocs = crawlerOptions?.returnOnlyUrls
- ? docs.map((doc) => {
- if (doc.metadata.sourceURL) {
- return { url: doc.metadata.sourceURL };
- }
- })
- : docs;
-
if(is_scrape === false) {
let creditsToBeBilled = 1; // Assuming 1 credit per document
- if (extractorOptions && (extractorOptions.mode === "llm-extraction" || extractorOptions.mode === "extract")) {
+ if (scrapeOptions.extract) {
creditsToBeBilled = 5;
}
- billTeam(team_id, undefined, creditsToBeBilled * filteredDocs.length).catch(error => {
- Logger.error(`Failed to bill team ${team_id} for ${creditsToBeBilled * filteredDocs.length} credits: ${error}`);
+ billTeam(team_id, undefined, creditsToBeBilled).catch(error => {
+ logger.error(`Failed to bill team ${team_id} for ${creditsToBeBilled} credits: ${error}`);
// Optionally, you could notify an admin or add to a retry queue here
});
}
-
-
// This is where the returnvalue from the job is set
- onSuccess(filteredDocs, mode);
+ // onSuccess(response.document, mode);
- // this return doesn't matter too much for the job completion result
- return { success: true, message: "", docs: filteredDocs };
+ engines = response.engines;
+ return response;
} catch (error) {
- onError(error);
- return { success: false, message: error.message, docs: [] };
+ engines = response !== undefined ? response.engines : ((typeof error === "object" && error !== null ? (error as any).results ?? {} : {}));
+
+ if (response !== undefined) {
+ return {
+ ...response,
+ success: false,
+ error,
+ }
+ } else {
+ return { success: false, error, logs: ["no logs -- error coming from runWebScraper"], engines };
+ }
+ // onError(error);
+ } finally {
+ const engineOrder = Object.entries(engines).sort((a, b) => a[1].startedAt - b[1].startedAt).map(x => x[0]) as Engine[];
+
+ for (const engine of engineOrder) {
+ const result = engines[engine] as Exclude<EngineResultsTracker[Engine], undefined>;
+ ScrapeEvents.insert(bull_job_id, {
+ type: "scrape",
+ url,
+ method: engine,
+ result: {
+ success: result.state === "success",
+ response_code: (result.state === "success" ? result.result.statusCode : undefined),
+ response_size: (result.state === "success" ? result.result.html.length : undefined),
+ error: (result.state === "error" ? result.error : result.state === "timeout" ? "Timed out" : undefined),
+ time_taken: result.finishedAt - result.startedAt,
+ },
+ });
+ }
}
}
-const saveJob = async (job: Job, result: any, token: string, mode: string) => {
+const saveJob = async (job: Job, result: any, token: string, mode: string, engines?: EngineResultsTracker) => {
try {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
if (useDbAuthentication) {
@@ -173,6 +149,6 @@ const saveJob = async (job: Job, result: any, token: string, mode: string) => {
}
ScrapeEvents.logJobEvent(job, "completed");
} catch (error) {
- Logger.error(`πŸ‚ Failed to update job status: ${error}`);
+ logger.error(`πŸ‚ Failed to update job status: ${error}`);
}
};
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 881590602..ac61519a8 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -6,8 +6,8 @@ import {
cleanBefore24hCompleteJobsController,
queuesController,
} from "../controllers/v0/admin/queue";
-import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
import { wrap } from "./v1";
+import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
export const adminRouter = express.Router();
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index 4e4b6052d..3eaace3b2 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -14,7 +14,7 @@ import expressWs from "express-ws";
import { crawlStatusWSController } from "../controllers/v1/crawl-status-ws";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
import { crawlCancelController } from "../controllers/v1/crawl-cancel";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
import { scrapeStatusController } from "../controllers/v1/scrape-status";
import { concurrencyCheckController } from "../controllers/v1/concurrency-check";
import { batchScrapeController } from "../controllers/v1/batch-scrape";
@@ -32,10 +32,12 @@ function checkCreditsMiddleware(minimum?: number): (req: RequestWithAuth, res: R
if (!minimum && req.body) {
minimum = (req.body as any)?.limit ?? (req.body as any)?.urls?.length ?? 1;
}
- const { success, remainingCredits, chunk } = await checkTeamCredits(req.acuc, req.auth.team_id, minimum);
- req.acuc = chunk;
+ const { success, remainingCredits, chunk } = await checkTeamCredits(req.acuc, req.auth.team_id, minimum ?? 1);
+ if (chunk) {
+ req.acuc = chunk;
+ }
if (!success) {
- Logger.error(`Insufficient credits: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`);
+ logger.error(`Insufficient credits: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`);
if (!res.headersSent) {
return res.status(402).json({ success: false, error: "Insufficient credits to perform this request. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing or try changing the request limit to a lower value." });
}
@@ -50,20 +52,27 @@ function checkCreditsMiddleware(minimum?: number): (req: RequestWithAuth, res: R
export function authMiddleware(rateLimiterMode: RateLimiterMode): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {
return (req, res, next) => {
(async () => {
- const { success, team_id, error, status, plan, chunk } = await authenticateUser(
+ const auth = await authenticateUser(
req,
res,
rateLimiterMode,
);
- if (!success) {
+ if (!auth.success) {
if (!res.headersSent) {
- return res.status(status).json({ success: false, error });
+ return res.status(auth.status).json({ success: false, error: auth.error });
+ } else {
+ return;
}
}
+ const { team_id, plan, chunk } = auth;
+
req.auth = { team_id, plan };
- req.acuc = chunk;
+ req.acuc = chunk ?? undefined;
+ if (chunk) {
+ req.account = { remainingCredits: chunk.remaining_credits };
+ }
next();
})()
.catch(err => next(err));
diff --git a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
index 20419ffa0..eba0ddb4e 100644
--- a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
+++ b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
@@ -2,7 +2,6 @@
import { WebCrawler } from '../crawler';
import axios from 'axios';
import robotsParser from 'robots-parser';
-import { getAdjustedMaxDepth } from '../utils/maxDepthUtils';
jest.mock('axios');
jest.mock('robots-parser');
@@ -35,165 +34,6 @@ describe('WebCrawler', () => {
});
});
- it('should filter out links that exceed maxDepth param of 2 based on enterURL depth of 0 ', async () => {
- const initialUrl = 'http://example.com'; // Set initial URL for this test
- const enteredMaxCrawledDepth = 2;
- maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
-
-
- crawler = new WebCrawler({
- jobId: "TEST",
- initialUrl: initialUrl,
- includes: [],
- excludes: [],
- limit: 100,
- maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
- });
-
- // Mock sitemap fetching function to return controlled links
- crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
- initialUrl, // depth 0
- initialUrl + '/page1', // depth 1
- initialUrl + '/page1/page2', // depth 2
- initialUrl + '/page1/page2/page3' // depth 3, should be filtered out
- ]);
-
- const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
- expect(results).toEqual([
- { url: initialUrl, html: '' },
- { url: initialUrl + '/page1', html: '' },
- { url: initialUrl + '/page1/page2', html: '' }
- ]);
-
-
- // Ensure that the link with depth 3 is not included
- expect(results.some(r => r.url === initialUrl + '/page1/page2/page3')).toBe(false);
- });
-
- it('should filter out links that exceed maxDepth param of 0 based on enterURL depth of 0 ', async () => {
- const initialUrl = 'http://example.com'; // Set initial URL for this test
- const enteredMaxCrawledDepth = 0;
- maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
-
-
- crawler = new WebCrawler({
- jobId: "TEST",
- initialUrl: initialUrl,
- includes: [],
- excludes: [],
- limit: 100,
- maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
- });
-
- // Mock sitemap fetching function to return controlled links
- crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
- initialUrl, // depth 0
- initialUrl + '/page1', // depth 1
- initialUrl + '/page1/page2', // depth 2
- initialUrl + '/page1/page2/page3' // depth 3, should be filtered out
- ]);
-
- const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
- expect(results).toEqual([
- { url: initialUrl, html: '' },
- ]);
- });
-
- it('should filter out links that exceed maxDepth param of 1 based on enterURL depth of 1 ', async () => {
- const initialUrl = 'http://example.com/page1'; // Set initial URL for this test
- const enteredMaxCrawledDepth = 1;
- maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
-
-
- crawler = new WebCrawler({
- jobId: "TEST",
- initialUrl: initialUrl,
- includes: [],
- excludes: [],
- limit: 100,
- maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
- });
-
- // Mock sitemap fetching function to return controlled links
- crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
- initialUrl, // depth 0
- initialUrl + '/page2', // depth 1
- initialUrl + '/page2/page3', // depth 2
- initialUrl + '/page2/page3/page4' // depth 3, should be filtered out
- ]);
-
- const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
- expect(results).toEqual([
- { url: initialUrl, html: '' },
- { url: initialUrl + '/page2', html: '' }
- ]);
- });
-
- it('should filter out links that exceed maxDepth param of 1 based on enterURL depth of 2 ', async () => {
- const initialUrl = 'http://example.com/page1'; // Set initial URL for this test
- const enteredMaxCrawledDepth = 2;
- maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
-
-
- crawler = new WebCrawler({
- jobId: "TEST",
- initialUrl: initialUrl,
- includes: [],
- excludes: [],
- limit: 100,
- maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
- });
-
- // Mock sitemap fetching function to return controlled links
- crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
- initialUrl, // depth 0
- initialUrl + '/page2', // depth 1
- initialUrl + '/page2/page3', // depth 2
- initialUrl + '/page2/page3/page4' // depth 3, should be filtered out
- ]);
-
- const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
- expect(results).toEqual([
- { url: initialUrl, html: '' },
- { url: initialUrl + '/page2', html: '' },
- { url: initialUrl + '/page2/page3', html: '' }
- ]);
- });
-
- it('should handle allowBackwardCrawling option correctly', async () => {
- const initialUrl = 'https://mendable.ai/blog';
-
- // Setup the crawler with the specific test case options
- const crawler = new WebCrawler({
- jobId: "TEST",
- initialUrl: initialUrl,
- includes: [],
- excludes: [],
- limit: 100,
- maxCrawledDepth: 3, // Example depth
- allowBackwardCrawling: true
- });
-
- // Mock the sitemap fetching function to simulate backward crawling
- crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
- initialUrl,
- 'https://mendable.ai', // backward link
- initialUrl + '/page1',
- initialUrl + '/page1/page2'
- ]);
-
- const results = await crawler.start();
- expect(results).toEqual([
- { url: initialUrl, html: '' },
- { url: 'https://mendable.ai', html: '' }, // Expect the backward link to be included
- { url: initialUrl + '/page1', html: '' },
- { url: initialUrl + '/page1/page2', html: '' }
- ]);
-
- // Check that the backward link is included if allowBackwardCrawling is true
- expect(results.some(r => r.url === 'https://mendable.ai')).toBe(true);
- });
-
it('should respect the limit parameter by not returning more links than specified', async () => {
const initialUrl = 'http://example.com';
const limit = 2; // Set a limit for the number of links
diff --git a/apps/api/src/scraper/WebScraper/__tests__/single_url.test.ts b/apps/api/src/scraper/WebScraper/__tests__/single_url.test.ts
deleted file mode 100644
index 02c8a7e09..000000000
--- a/apps/api/src/scraper/WebScraper/__tests__/single_url.test.ts
+++ /dev/null
@@ -1,37 +0,0 @@
-import { scrapSingleUrl } from '../single_url';
-import { PageOptions } from '../../../lib/entities';
-
-
-jest.mock('../single_url', () => {
- const originalModule = jest.requireActual('../single_url');
- originalModule.fetchHtmlContent = jest.fn().mockResolvedValue('<html><head><title>Test</title></head><body><h1>Roast</h1></body></html>');
-
- return originalModule;
-});
-
-describe('scrapSingleUrl', () => {
- it('should handle includeHtml option correctly', async () => {
- const url = 'https://roastmywebsite.ai';
- const pageOptionsWithHtml: PageOptions = { includeHtml: true };
- const pageOptionsWithoutHtml: PageOptions = { includeHtml: false };
-
- const resultWithHtml = await scrapSingleUrl("TEST", url, pageOptionsWithHtml);
- const resultWithoutHtml = await scrapSingleUrl("TEST", url, pageOptionsWithoutHtml);
-
- expect(resultWithHtml.html).toBeDefined();
- expect(resultWithoutHtml.html).toBeUndefined();
- }, 10000);
-});
-
-it('should return a list of links on the firecrawl.ai page', async () => {
- const url = 'https://flutterbricks.com';
- const pageOptions: PageOptions = { includeHtml: true };
-
- const result = await scrapSingleUrl("TEST", url, pageOptions);
-
- // Check if the result contains a list of links
- expect(result.linksOnPage).toBeDefined();
- expect(Array.isArray(result.linksOnPage)).toBe(true);
- expect(result.linksOnPage.length).toBeGreaterThan(0);
- expect(result.linksOnPage).toContain('https://flutterbricks.com/features')
-}, 15000);
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 1ae166b49..9e3f7cd21 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -2,13 +2,10 @@ import axios, { AxiosError } from "axios";
import cheerio, { load } from "cheerio";
import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
-import async from "async";
-import { CrawlerOptions, PageOptions, Progress } from "../../lib/entities";
-import { scrapSingleUrl } from "./single_url";
import robotsParser from "robots-parser";
import { getURLDepth } from "./utils/maxDepthUtils";
import { axiosTimeout } from "../../../src/lib/timeout";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import https from "https";
export class WebCrawler {
private jobId: string;
@@ -73,7 +70,7 @@ export class WebCrawler {
try {
url = new URL(link.trim(), this.baseUrl);
} catch (error) {
- Logger.debug(`Error processing link: ${link} | Error: ${error.message}`);
+ logger.debug(`Error processing link: ${link} | Error: ${error.message}`);
return false;
}
const path = url.pathname;
@@ -132,7 +129,7 @@ export class WebCrawler {
const isAllowed = this.robots.isAllowed(link, "FireCrawlAgent") ?? true;
// Check if the link is disallowed by robots.txt
if (!isAllowed) {
- Logger.debug(`Link disallowed by robots.txt: ${link}`);
+ logger.debug(`Link disallowed by robots.txt: ${link}`);
return false;
}
@@ -161,7 +158,7 @@ export class WebCrawler {
}
public async tryGetSitemap(): Promise<{ url: string; html: string; }[] | null> {
- Logger.debug(`Fetching sitemap links from ${this.initialUrl}`);
+ logger.debug(`Fetching sitemap links from ${this.initialUrl}`);
const sitemapLinks = await this.tryFetchSitemapLinks(this.initialUrl);
if (sitemapLinks.length > 0) {
let filteredLinks = this.filterLinks(sitemapLinks, this.limit, this.maxCrawledDepth);
@@ -170,115 +167,6 @@ export class WebCrawler {
return null;
}
- public async start(
- inProgress?: (progress: Progress) => void,
- pageOptions?: PageOptions,
- crawlerOptions?: CrawlerOptions,
- concurrencyLimit: number = 5,
- limit: number = 10000,
- maxDepth: number = 10
- ): Promise<{ url: string, html: string }[]> {
-
- Logger.debug(`Crawler starting with ${this.initialUrl}`);
- // Fetch and parse robots.txt
- try {
- const txt = await this.getRobotsTxt();
- this.importRobotsTxt(txt);
- Logger.debug(`Crawler robots.txt fetched with ${this.robotsTxtUrl}`);
- } catch (error) {
- Logger.debug(`Failed to fetch robots.txt from ${this.robotsTxtUrl}`);
- }
-
- if (!crawlerOptions?.ignoreSitemap){
- const sm = await this.tryGetSitemap();
- if (sm !== null) {
- return sm;
- }
- }
-
- const urls = await this.crawlUrls(
- [this.initialUrl],
- pageOptions,
- concurrencyLimit,
- inProgress
- );
-
- if (
- urls.length === 0 &&
- this.filterLinks([this.initialUrl], limit, this.maxCrawledDepth).length > 0
- ) {
- return [{ url: this.initialUrl, html: "" }];
- }
-
- // make sure to run include exclude here again
- const filteredUrls = this.filterLinks(urls.map(urlObj => urlObj.url), limit, this.maxCrawledDepth);
- return filteredUrls.map(url => ({ url, html: urls.find(urlObj => urlObj.url === url)?.html || "" }));
- }
-
- private async crawlUrls(
- urls: string[],
- pageOptions: PageOptions,
- concurrencyLimit: number,
- inProgress?: (progress: Progress) => void,
- ): Promise<{ url: string, html: string }[]> {
- const queue = async.queue(async (task: string, callback) => {
- Logger.debug(`Crawling ${task}`);
- if (this.crawledUrls.size >= Math.min(this.maxCrawledLinks, this.limit)) {
- if (callback && typeof callback === "function") {
- callback();
- }
- return;
- }
- const newUrls = await this.crawl(task, pageOptions);
- // add the initial url if not already added
- // if (this.visited.size === 1) {
- // let normalizedInitial = this.initialUrl;
- // if (!normalizedInitial.endsWith("/")) {
- // normalizedInitial = normalizedInitial + "/";
- // }
- // if (!newUrls.some(page => page.url === this.initialUrl)) {
- // newUrls.push({ url: this.initialUrl, html: "" });
- // }
- // }
-
- newUrls.forEach((page) => this.crawledUrls.set(page.url, page.html));
-
- if (inProgress && newUrls.length > 0) {
- inProgress({
- current: this.crawledUrls.size,
- total: Math.min(this.maxCrawledLinks, this.limit),
- status: "SCRAPING",
- currentDocumentUrl: newUrls[newUrls.length - 1].url,
- });
- } else if (inProgress) {
- inProgress({
- current: this.crawledUrls.size,
- total: Math.min(this.maxCrawledLinks, this.limit),
- status: "SCRAPING",
- currentDocumentUrl: task,
- });
- }
- await this.crawlUrls(newUrls.map((p) => p.url), pageOptions, concurrencyLimit, inProgress);
- if (callback && typeof callback === "function") {
- callback();
- }
- }, concurrencyLimit);
-
- Logger.debug(`πŸ‚ Pushing ${urls.length} URLs to the queue`);
- queue.push(
- urls.filter(
- (url) =>
- !this.visited.has(url) && this.robots.isAllowed(url, "FireCrawlAgent")
- ),
- (err) => {
- if (err) Logger.error(`πŸ‚ Error pushing URLs to the queue: ${err}`);
- }
- );
- await queue.drain();
- Logger.debug(`πŸ‚ Crawled ${this.crawledUrls.size} URLs, Queue drained.`);
- return Array.from(this.crawledUrls.entries()).map(([url, html]) => ({ url, html }));
- }
-
public filterURL(href: string, url: string): string | null {
let fullUrl = href;
if (!href.startsWith("http")) {
@@ -346,79 +234,9 @@ export class WebCrawler {
return links;
}
- async crawl(url: string, pageOptions: PageOptions): Promise<{url: string, html: string, pageStatusCode?: number, pageError?: string}[]> {
- if (this.visited.has(url) || !this.robots.isAllowed(url, "FireCrawlAgent")) {
- return [];
- }
- this.visited.add(url);
-
- if (!url.startsWith("http")) {
- url = "https://" + url;
- }
- if (url.endsWith("/")) {
- url = url.slice(0, -1);
- }
-
- if (this.isFile(url) || this.isSocialMediaOrEmail(url)) {
- return [];
- }
-
- try {
- let content: string = "";
- let pageStatusCode: number;
- let pageError: string | undefined = undefined;
-
- // If it is the first link, fetch with single url
- if (this.visited.size === 1) {
- const page = await scrapSingleUrl(this.jobId, url, { ...pageOptions, includeHtml: true });
- content = page.html ?? "";
- pageStatusCode = page.metadata?.pageStatusCode;
- pageError = page.metadata?.pageError || undefined;
- } else {
- const response = await axios.get(url, { timeout: axiosTimeout });
- content = response.data ?? "";
- pageStatusCode = response.status;
- pageError = response.statusText != "OK" ? response.statusText : undefined;
- }
-
- const $ = load(content);
- let links: { url: string, html: string, pageStatusCode?: number, pageError?: string }[] = [];
-
- // Add the initial URL to the list of links
- if (this.visited.size === 1) {
- links.push({ url, html: content, pageStatusCode, pageError });
- }
-
- links.push(...this.extractLinksFromHTML(content, url).map(url => ({ url, html: content, pageStatusCode, pageError })));
-
- if (this.visited.size === 1) {
- return links;
- }
-
- // Create a new list to return to avoid modifying the visited list
- return links.filter((link) => !this.visited.has(link.url));
- } catch (error) {
- return [];
- }
- }
-
private isRobotsAllowed(url: string): boolean {
return (this.robots ? (this.robots.isAllowed(url, "FireCrawlAgent") ?? true) : true)
}
- private normalizeCrawlUrl(url: string): string {
- try{
- const urlObj = new URL(url);
- urlObj.searchParams.sort(); // Sort query parameters to normalize
- return urlObj.toString();
- } catch (error) {
- return url;
- }
- }
-
- private matchesIncludes(url: string): boolean {
- if (this.includes.length === 0 || this.includes[0] == "") return true;
- return this.includes.some((pattern) => new RegExp(pattern).test(url));
- }
private matchesExcludes(url: string, onlyDomains: boolean = false): boolean {
return this.excludes.some((pattern) => {
@@ -503,7 +321,7 @@ export class WebCrawler {
const urlWithoutQuery = url.split('?')[0].toLowerCase();
return fileExtensions.some((ext) => urlWithoutQuery.endsWith(ext));
} catch (error) {
- Logger.error(`Error processing URL in isFile: ${error}`);
+ logger.error(`Error processing URL in isFile: ${error}`);
return false;
}
}
@@ -524,7 +342,6 @@ export class WebCrawler {
return socialMediaOrEmail.some((ext) => url.includes(ext));
}
- //
private async tryFetchSitemapLinks(url: string): Promise<string[]> {
const normalizeUrl = (url: string) => {
url = url.replace(/^https?:\/\//, "").replace(/^www\./, "");
@@ -546,7 +363,7 @@ export class WebCrawler {
sitemapLinks = await getLinksFromSitemap({ sitemapUrl });
}
} catch (error) {
- Logger.debug(`Failed to fetch sitemap with axios from ${sitemapUrl}: ${error}`);
+ logger.debug(`Failed to fetch sitemap with axios from ${sitemapUrl}: ${error}`);
if (error instanceof AxiosError && error.response?.status === 404) {
// ignore 404
} else {
@@ -565,7 +382,7 @@ export class WebCrawler {
sitemapLinks = await getLinksFromSitemap({ sitemapUrl: baseUrlSitemap, mode: 'fire-engine' });
}
} catch (error) {
- Logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}: ${error}`);
+ logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}: ${error}`);
if (error instanceof AxiosError && error.response?.status === 404) {
// ignore 404
} else {
diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
index e5841978d..48aa2ffd8 100644
--- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
+++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
@@ -1,4 +1,4 @@
-import { Logger } from "../../../lib/logger";
+import { logger } from "../../../lib/logger";
export async function handleCustomScraping(
text: string,
@@ -6,7 +6,7 @@ export async function handleCustomScraping(
): Promise<{ scraper: string; url: string; waitAfterLoad?: number, pageOptions?: { scrollXPaths?: string[] } } | null> {
// Check for Readme Docs special case
if (text.includes('<meta name="readme-deploy"') && !url.includes('developers.notion.com')) {
- Logger.debug(
+ logger.debug(
`Special use case detected for ${url}, using Fire Engine with wait time 1000ms`
);
return {
@@ -21,7 +21,7 @@ export async function handleCustomScraping(
// Check for Vanta security portals
if (text.includes('<link href="https://static.vanta.com')) {
- Logger.debug(
+ logger.debug(
`Vanta link detected for ${url}, using Fire Engine with wait time 3000ms`
);
return {
@@ -36,7 +36,7 @@ export async function handleCustomScraping(
const googleDriveMetaMatch = text.match(googleDriveMetaPattern);
if (googleDriveMetaMatch) {
const url = googleDriveMetaMatch[1];
- Logger.debug(`Google Drive PDF link detected: ${url}`);
+ logger.debug(`Google Drive PDF link detected: ${url}`);
const fileIdMatch = url.match(/https:\/\/drive\.google\.com\/file\/d\/([^\/]+)\/view/);
if (fileIdMatch) {
diff --git a/apps/api/src/scraper/WebScraper/global.ts b/apps/api/src/scraper/WebScraper/global.ts
deleted file mode 100644
index 7233fe789..000000000
--- a/apps/api/src/scraper/WebScraper/global.ts
+++ /dev/null
@@ -1 +0,0 @@
-export const universalTimeout = 15000;
\ No newline at end of file
diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts
deleted file mode 100644
index 79b312d59..000000000
--- a/apps/api/src/scraper/WebScraper/index.ts
+++ /dev/null
@@ -1,743 +0,0 @@
-import {
- Document,
- ExtractorOptions,
- PageOptions,
- WebScraperOptions,
-} from "../../lib/entities";
-import { Progress } from "../../lib/entities";
-import { scrapSingleUrl } from "./single_url";
-import { SitemapEntry, fetchSitemapData, getLinksFromSitemap } from "./sitemap";
-import { WebCrawler } from "./crawler";
-import { getValue, setValue } from "../../services/redis";
-import { getImageDescription } from "./utils/imageDescription";
-import { fetchAndProcessPdf } from "./utils/pdfProcessor";
-import {
- replaceImgPathsWithAbsolutePaths,
- replacePathsWithAbsolutePaths,
-} from "./utils/replacePaths";
-import { generateCompletions } from "../../lib/LLM-extraction";
-import { fetchAndProcessDocx } from "./utils/docxProcessor";
-import { getAdjustedMaxDepth, getURLDepth } from "./utils/maxDepthUtils";
-import { Logger } from "../../lib/logger";
-import { ScrapeEvents } from "../../lib/scrape-events";
-
-export class WebScraperDataProvider {
- private jobId: string;
- private bullJobId: string;
- private urls: string[] = [""];
- private mode: "single_urls" | "sitemap" | "crawl" = "single_urls";
- private includes: string | string[];
- private excludes: string | string[];
- private maxCrawledLinks: number;
- private maxCrawledDepth: number = 10;
- private returnOnlyUrls: boolean;
- private limit: number = 10000;
- private concurrentRequests: number = 20;
- private generateImgAltText: boolean = false;
- private ignoreSitemap: boolean = false;
- private pageOptions?: PageOptions;
- private extractorOptions?: ExtractorOptions;
- private replaceAllPathsWithAbsolutePaths?: boolean = false;
- private generateImgAltTextModel: "gpt-4-turbo" | "claude-3-opus" =
- "gpt-4-turbo";
- private crawlerMode: string = "default";
- private allowBackwardCrawling: boolean = false;
- private allowExternalContentLinks: boolean = false;
- private priority?: number;
- private teamId?: string;
-
- authorize(): void {
- throw new Error("Method not implemented.");
- }
-
- authorizeNango(): Promise<void> {
- throw new Error("Method not implemented.");
- }
-
- private async convertUrlsToDocuments(
- urls: string[],
- inProgress?: (progress: Progress) => void,
- allHtmls?: string[]
- ): Promise<Document[]> {
- const totalUrls = urls.length;
- let processedUrls = 0;
-
- const results: (Document | null)[] = new Array(urls.length).fill(null);
- for (let i = 0; i < urls.length; i += this.concurrentRequests) {
- const batchUrls = urls.slice(i, i + this.concurrentRequests);
- await Promise.all(
- batchUrls.map(async (url, index) => {
- const existingHTML = allHtmls ? allHtmls[i + index] : "";
- const result = await scrapSingleUrl(
- this.jobId,
- url,
- this.pageOptions,
- this.extractorOptions,
- existingHTML,
- this.priority,
- this.teamId,
- );
- processedUrls++;
- if (inProgress) {
- inProgress({
- current: processedUrls,
- total: totalUrls,
- status: "SCRAPING",
- currentDocumentUrl: url,
- currentDocument: { ...result, index: processedUrls },
- });
- }
-
- results[i + index] = result;
- })
- );
- }
- return results.filter((result) => result !== null) as Document[];
- }
-
- async getDocuments(
- useCaching: boolean = false,
- inProgress?: (progress: Progress) => void
- ): Promise<Document[]> {
- this.validateInitialUrl();
- if (!useCaching) {
- return this.processDocumentsWithoutCache(inProgress);
- }
-
- return this.processDocumentsWithCache(inProgress);
- }
-
- private validateInitialUrl(): void {
- if (this.urls[0].trim() === "") {
- throw new Error("Url is required");
- }
- }
-
- /**
- * Process documents without cache handling each mode
- * @param inProgress inProgress
- * @returns documents
- */
- private async processDocumentsWithoutCache(
- inProgress?: (progress: Progress) => void
- ): Promise<Document[]> {
- switch (this.mode) {
- case "crawl":
- return this.handleCrawlMode(inProgress);
- case "single_urls":
- return this.handleSingleUrlsMode(inProgress);
- case "sitemap":
- return this.handleSitemapMode(inProgress);
- default:
- return [];
- }
- }
-
- private async cleanIrrelevantPath(links: string[]) {
- return links.filter((link) => {
- const normalizedInitialUrl = new URL(this.urls[0]);
- const normalizedLink = new URL(link);
-
- // Normalize the hostname to account for www and non-www versions
- const initialHostname = normalizedInitialUrl.hostname.replace(
- /^www\./,
- ""
- );
- const linkHostname = normalizedLink.hostname.replace(/^www\./, "");
-
- // Ensure the protocol and hostname match, and the path starts with the initial URL's path
- return (
- linkHostname === initialHostname &&
- normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)
- );
- });
- }
-
- private async handleCrawlMode(
- inProgress?: (progress: Progress) => void
- ): Promise<Document[]> {
- let includes: string[];
- if (Array.isArray(this.includes)) {
- if (this.includes[0] != "") {
- includes = this.includes;
- }
- } else {
- includes = this.includes.split(',');
- }
-
- let excludes: string[];
- if (Array.isArray(this.excludes)) {
- if (this.excludes[0] != "") {
- excludes = this.excludes;
- }
- } else {
- excludes = this.excludes.split(',');
- }
-
- const crawler = new WebCrawler({
- jobId: this.jobId,
- initialUrl: this.urls[0],
- includes,
- excludes,
- maxCrawledLinks: this.maxCrawledLinks,
- maxCrawledDepth: getAdjustedMaxDepth(this.urls[0], this.maxCrawledDepth),
- limit: this.limit,
- generateImgAltText: this.generateImgAltText,
- allowBackwardCrawling: this.allowBackwardCrawling,
- allowExternalContentLinks: this.allowExternalContentLinks,
- });
-
- let links = await crawler.start(
- inProgress,
- this.pageOptions,
- {
- ignoreSitemap: this.ignoreSitemap,
- },
- 5,
- this.limit,
- this.maxCrawledDepth
- );
-
- let allLinks = links.map((e) => e.url);
- const allHtmls = links.map((e) => e.html);
-
- if (this.returnOnlyUrls) {
- return this.returnOnlyUrlsResponse(allLinks, inProgress);
- }
-
- let documents = [];
- // check if fast mode is enabled and there is html inside the links
- if (this.crawlerMode === "fast" && links.some((link) => link.html)) {
- documents = await this.processLinks(allLinks, inProgress, allHtmls);
- } else {
- documents = await this.processLinks(allLinks, inProgress);
- }
-
- return this.cacheAndFinalizeDocuments(documents, allLinks);
- }
-
- private async handleSingleUrlsMode(
- inProgress?: (progress: Progress) => void
- ): Promise<Document[]> {
- const links = this.urls;
-
- let documents = await this.processLinks(links, inProgress);
- return documents;
- }
-
- private async handleSitemapMode(
- inProgress?: (progress: Progress) => void
- ): Promise<Document[]> {
- let links = await getLinksFromSitemap({ sitemapUrl: this.urls[0] });
- links = await this.cleanIrrelevantPath(links);
-
- if (this.returnOnlyUrls) {
- return this.returnOnlyUrlsResponse(links, inProgress);
- }
-
- let documents = await this.processLinks(links, inProgress);
- return this.cacheAndFinalizeDocuments(documents, links);
- }
-
- private async returnOnlyUrlsResponse(
- links: string[],
- inProgress?: (progress: Progress) => void
- ): Promise<Document[]> {
- inProgress?.({
- current: links.length,
- total: links.length,
- status: "COMPLETED",
- currentDocumentUrl: this.urls[0],
- });
- return links.map((url) => ({
- content: "",
- html: this.pageOptions?.includeHtml ? "" : undefined,
- markdown: "",
- metadata: { sourceURL: url, pageStatusCode: 200 },
- }));
- }
-
- private async processLinks(
- links: string[],
- inProgress?: (progress: Progress) => void,
- allHtmls?: string[]
- ): Promise<Document[]> {
- const pdfLinks = links.filter((link) => link.endsWith(".pdf"));
- const docLinks = links.filter(
- (link) => link.endsWith(".doc") || link.endsWith(".docx")
- );
-
- const [pdfDocuments, docxDocuments] = await Promise.all([
- this.fetchPdfDocuments(pdfLinks),
- this.fetchDocxDocuments(docLinks),
- ]);
-
- links = links.filter(
- (link) => !pdfLinks.includes(link) && !docLinks.includes(link)
- );
-
- let [documents, sitemapData] = await Promise.all([
- this.convertUrlsToDocuments(links, inProgress, allHtmls),
- this.mode === "single_urls" && links.length > 0
- ? this.getSitemapDataForSingleUrl(this.urls[0], links[0], 1500).catch(
- (error) => {
- Logger.debug(`Failed to fetch sitemap data: ${error}`);
- return null;
- }
- )
- : Promise.resolve(null),
- ]);
-
- if (this.mode === "single_urls" && documents.length > 0) {
- documents[0].metadata.sitemap = sitemapData ?? undefined;
- } else {
- documents = await this.getSitemapData(this.urls[0], documents);
- }
-
- if (this.pageOptions.includeMarkdown) {
- documents = this.applyPathReplacements(documents);
- }
-
- if (!this.pageOptions.includeHtml) {
- for (let document of documents) {
- delete document.html;
- }
- }
-
- // documents = await this.applyImgAltText(documents);
- if (this.mode === "single_urls" && this.pageOptions.includeExtract) {
- const extractionMode = this.extractorOptions?.mode ?? "markdown";
- const completionMode = extractionMode === "llm-extraction-from-raw-html" ? "raw-html" : "markdown";
-
- if (
- extractionMode === "llm-extraction" ||
- extractionMode === "llm-extraction-from-markdown" ||
- extractionMode === "llm-extraction-from-raw-html"
- ) {
- documents = await generateCompletions(
- documents,
- this.extractorOptions,
- completionMode
- );
- }
- }
- return documents.concat(pdfDocuments).concat(docxDocuments);
- }
-
- private async fetchPdfDocuments(pdfLinks: string[]): Promise<Document[]> {
- return Promise.all(
- pdfLinks.map(async (pdfLink) => {
- const timer = Date.now();
- const logInsertPromise = ScrapeEvents.insert(this.jobId, {
- type: "scrape",
- url: pdfLink,
- worker: process.env.FLY_MACHINE_ID,
- method: "pdf-scrape",
- result: null,
- });
-
- const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(
- pdfLink,
- this.pageOptions.parsePDF
- );
-
- const insertedLogId = await logInsertPromise;
- ScrapeEvents.updateScrapeResult(insertedLogId, {
- response_size: content.length,
- success: !(pageStatusCode && pageStatusCode >= 400) && !!content && (content.trim().length >= 100),
- error: pageError,
- response_code: pageStatusCode,
- time_taken: Date.now() - timer,
- });
- return {
- content: content,
- markdown: content,
- metadata: { sourceURL: pdfLink, pageStatusCode, pageError },
- provider: "web-scraper",
- };
- })
- );
- }
- private async fetchDocxDocuments(docxLinks: string[]): Promise<Document[]> {
- return Promise.all(
- docxLinks.map(async (docxLink) => {
- const timer = Date.now();
- const logInsertPromise = ScrapeEvents.insert(this.jobId, {
- type: "scrape",
- url: docxLink,
- worker: process.env.FLY_MACHINE_ID,
- method: "docx-scrape",
- result: null,
- });
-
- const { content, pageStatusCode, pageError } = await fetchAndProcessDocx(
- docxLink
- );
-
- const insertedLogId = await logInsertPromise;
- ScrapeEvents.updateScrapeResult(insertedLogId, {
- response_size: content.length,
- success: !(pageStatusCode && pageStatusCode >= 400) && !!content && (content.trim().length >= 100),
- error: pageError,
- response_code: pageStatusCode,
- time_taken: Date.now() - timer,
- });
-
- return {
- content,
- metadata: { sourceURL: docxLink, pageStatusCode, pageError },
- provider: "web-scraper",
- };
- })
- );
- }
-
- private applyPathReplacements(documents: Document[]): Document[] {
- if (this.replaceAllPathsWithAbsolutePaths) {
- documents = replacePathsWithAbsolutePaths(documents);
- }
- return replaceImgPathsWithAbsolutePaths(documents);
- }
-
- private async applyImgAltText(documents: Document[]): Promise<Document[]> {
- return this.generateImgAltText
- ? this.generatesImgAltText(documents)
- : documents;
- }
-
- private async cacheAndFinalizeDocuments(
- documents: Document[],
- links: string[]
- ): Promise<Document[]> {
- // await this.setCachedDocuments(documents, links);
- documents = this.removeChildLinks(documents);
- return documents.splice(0, this.limit);
- }
-
- private async processDocumentsWithCache(
- inProgress?: (progress: Progress) => void
- ): Promise<Document[]> {
- let documents = await this.getCachedDocuments(
- this.urls.slice(0, this.limit)
- );
- if (documents.length < this.limit) {
- const newDocuments: Document[] = await this.getDocuments(
- false,
- inProgress
- );
- documents = this.mergeNewDocuments(documents, newDocuments);
- }
- documents = this.filterDocsExcludeInclude(documents);
- documents = this.filterDepth(documents);
- documents = this.removeChildLinks(documents);
- return documents.splice(0, this.limit);
- }
-
- private mergeNewDocuments(
- existingDocuments: Document[],
- newDocuments: Document[]
- ): Document[] {
- newDocuments.forEach((doc) => {
- if (
- !existingDocuments.some(
- (d) =>
- this.normalizeUrl(d.metadata.sourceURL) ===
- this.normalizeUrl(doc.metadata?.sourceURL)
- )
- ) {
- existingDocuments.push(doc);
- }
- });
- return existingDocuments;
- }
-
- private filterDocsExcludeInclude(documents: Document[]): Document[] {
- return documents.filter((document) => {
- const url = new URL(document.metadata.sourceURL);
- const path = url.pathname;
-
- if (!Array.isArray(this.excludes)) {
- this.excludes = this.excludes.split(',');
- }
-
- if (this.excludes.length > 0 && this.excludes[0] !== "") {
- // Check if the link should be excluded
- if (
- this.excludes.some((excludePattern) =>
- new RegExp(excludePattern).test(path)
- )
- ) {
- return false;
- }
- }
-
- if (!Array.isArray(this.includes)) {
- this.includes = this.includes.split(',');
- }
-
- if (this.includes.length > 0 && this.includes[0] !== "") {
- // Check if the link matches the include patterns, if any are specified
- if (this.includes.length > 0) {
- return this.includes.some((includePattern) =>
- new RegExp(includePattern).test(path)
- );
- }
- }
- return true;
- });
- }
-
- private normalizeUrl(url: string): string {
- if (url.includes("//www.")) {
- return url.replace("//www.", "//");
- }
- return url;
- }
-
- private removeChildLinks(documents: Document[]): Document[] {
- for (let document of documents) {
- if (document?.childrenLinks) delete document.childrenLinks;
- }
- return documents;
- }
-
- async setCachedDocuments(documents: Document[], childrenLinks?: string[]) {
- for (const document of documents) {
- if (document.content.trim().length === 0) {
- continue;
- }
- const normalizedUrl = this.normalizeUrl(document.metadata.sourceURL);
- await setValue(
- "web-scraper-cache:" + normalizedUrl,
- JSON.stringify({
- ...document,
- childrenLinks: childrenLinks || [],
- }),
- 60 * 60
- ); // 10 days
- }
- }
-
- async getCachedDocuments(urls: string[]): Promise<Document[]> {
- let documents: Document[] = [];
- for (const url of urls) {
- const normalizedUrl = this.normalizeUrl(url);
- Logger.debug(
- "Getting cached document for web-scraper-cache:" + normalizedUrl
- );
- const cachedDocumentString = await getValue(
- "web-scraper-cache:" + normalizedUrl
- );
- if (cachedDocumentString) {
- const cachedDocument = JSON.parse(cachedDocumentString);
- documents.push(cachedDocument);
-
- // get children documents
- for (const childUrl of cachedDocument.childrenLinks || []) {
- const normalizedChildUrl = this.normalizeUrl(childUrl);
- const childCachedDocumentString = await getValue(
- "web-scraper-cache:" + normalizedChildUrl
- );
- if (childCachedDocumentString) {
- const childCachedDocument = JSON.parse(childCachedDocumentString);
- if (
- !documents.find(
- (doc) =>
- doc.metadata.sourceURL ===
- childCachedDocument.metadata.sourceURL
- )
- ) {
- documents.push(childCachedDocument);
- }
- }
- }
- }
- }
- return documents;
- }
-
- setOptions(options: WebScraperOptions): void {
- if (!options.urls) {
- throw new Error("Urls are required");
- }
-
- this.jobId = options.jobId;
- this.bullJobId = options.bullJobId;
- this.urls = options.urls;
- this.mode = options.mode;
- this.concurrentRequests = options.concurrentRequests ?? 20;
- this.includes = options.crawlerOptions?.includes ?? [];
- this.excludes = options.crawlerOptions?.excludes ?? [];
- this.maxCrawledLinks = options.crawlerOptions?.maxCrawledLinks ?? 1000;
- this.maxCrawledDepth = options.crawlerOptions?.maxDepth ?? 10;
- this.returnOnlyUrls = options.crawlerOptions?.returnOnlyUrls ?? false;
- this.limit = options.crawlerOptions?.limit ?? 10000;
- this.generateImgAltText =
- options.crawlerOptions?.generateImgAltText ?? false;
- this.pageOptions = {
- onlyMainContent: options.pageOptions?.onlyMainContent ?? false,
- includeHtml: options.pageOptions?.includeHtml ?? false,
- replaceAllPathsWithAbsolutePaths: options.pageOptions?.replaceAllPathsWithAbsolutePaths ?? true,
- parsePDF: options.pageOptions?.parsePDF ?? true,
- onlyIncludeTags: options.pageOptions?.onlyIncludeTags ?? [],
- removeTags: options.pageOptions?.removeTags ?? [],
- includeMarkdown: options.pageOptions?.includeMarkdown ?? true,
- includeRawHtml: options.pageOptions?.includeRawHtml ?? false,
- includeExtract: options.pageOptions?.includeExtract ?? (options.extractorOptions?.mode && options.extractorOptions?.mode !== "markdown") ?? false,
- waitFor: options.pageOptions?.waitFor ?? undefined,
- headers: options.pageOptions?.headers ?? undefined,
- includeLinks: options.pageOptions?.includeLinks ?? true,
- fullPageScreenshot: options.pageOptions?.fullPageScreenshot ?? false,
- screenshot: options.pageOptions?.screenshot ?? false,
- useFastMode: options.pageOptions?.useFastMode ?? false,
- disableJsDom: options.pageOptions?.disableJsDom ?? false,
- atsv: options.pageOptions?.atsv ?? false,
- actions: options.pageOptions?.actions ?? undefined,
- geolocation: options.pageOptions?.geolocation ?? undefined,
- skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false,
- removeBase64Images: options.pageOptions?.removeBase64Images ?? true,
- mobile: options.pageOptions?.mobile ?? false,
- };
- this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };
- this.replaceAllPathsWithAbsolutePaths =
- options.crawlerOptions?.replaceAllPathsWithAbsolutePaths ??
- options.pageOptions?.replaceAllPathsWithAbsolutePaths ??
- false;
-
- if (typeof options.crawlerOptions?.excludes === 'string') {
- this.excludes = options.crawlerOptions?.excludes.split(',').filter((item) => item.trim() !== "");
- }
-
- if (typeof options.crawlerOptions?.includes === 'string') {
- this.includes = options.crawlerOptions?.includes.split(',').filter((item) => item.trim() !== "");
- }
-
- this.crawlerMode = options.crawlerOptions?.mode ?? "default";
- this.ignoreSitemap = options.crawlerOptions?.ignoreSitemap ?? false;
- this.allowBackwardCrawling =
- options.crawlerOptions?.allowBackwardCrawling ?? false;
- this.allowExternalContentLinks =
- options.crawlerOptions?.allowExternalContentLinks ?? false;
- this.priority = options.priority;
- this.teamId = options.teamId ?? null;
-
-
-
- // make sure all urls start with https://
- this.urls = this.urls.map((url) => {
- if (!url.trim().startsWith("http")) {
- return `https://${url}`;
- }
- return url;
- });
- }
-
- private async getSitemapData(baseUrl: string, documents: Document[]) {
- const sitemapData = await fetchSitemapData(baseUrl);
- if (sitemapData) {
- for (let i = 0; i < documents.length; i++) {
- const docInSitemapData = sitemapData.find(
- (data) =>
- this.normalizeUrl(data.loc) ===
- this.normalizeUrl(documents[i].metadata.sourceURL)
- );
- if (docInSitemapData) {
- let sitemapDocData: Partial<SitemapEntry> = {};
- if (docInSitemapData.changefreq) {
- sitemapDocData.changefreq = docInSitemapData.changefreq;
- }
- if (docInSitemapData.priority) {
- sitemapDocData.priority = Number(docInSitemapData.priority);
- }
- if (docInSitemapData.lastmod) {
- sitemapDocData.lastmod = docInSitemapData.lastmod;
- }
- if (Object.keys(sitemapDocData).length !== 0) {
- documents[i].metadata.sitemap = sitemapDocData;
- }
- }
- }
- }
- return documents;
- }
- private async getSitemapDataForSingleUrl(
- baseUrl: string,
- url: string,
- timeout?: number
- ) {
- const sitemapData = await fetchSitemapData(baseUrl, timeout);
- if (sitemapData) {
- const docInSitemapData = sitemapData.find(
- (data) => this.normalizeUrl(data.loc) === this.normalizeUrl(url)
- );
- if (docInSitemapData) {
- let sitemapDocData: Partial<SitemapEntry> = {};
- if (docInSitemapData.changefreq) {
- sitemapDocData.changefreq = docInSitemapData.changefreq;
- }
- if (docInSitemapData.priority) {
- sitemapDocData.priority = Number(docInSitemapData.priority);
- }
- if (docInSitemapData.lastmod) {
- sitemapDocData.lastmod = docInSitemapData.lastmod;
- }
- if (Object.keys(sitemapDocData).length !== 0) {
- return sitemapDocData;
- }
- }
- }
- return null;
- }
- generatesImgAltText = async (documents: Document[]): Promise<Document[]> => {
- await Promise.all(
- documents.map(async (document) => {
- const images = document.content.match(/!\[.*?\]\((.*?)\)/g) || [];
-
- await Promise.all(
- images.map(async (image: string) => {
- let imageUrl = image.match(/\(([^)]+)\)/)[1];
- let altText = image.match(/\[(.*?)\]/)[1];
-
- if (
- !altText &&
- !imageUrl.startsWith("data:image") &&
- /\.(png|jpeg|gif|webp)$/.test(imageUrl)
- ) {
- const imageIndex = document.content.indexOf(image);
- const contentLength = document.content.length;
- let backText = document.content.substring(
- imageIndex + image.length,
- Math.min(imageIndex + image.length + 1000, contentLength)
- );
- let frontTextStartIndex = Math.max(imageIndex - 1000, 0);
- let frontText = document.content.substring(
- frontTextStartIndex,
- imageIndex
- );
- altText = await getImageDescription(
- imageUrl,
- backText,
- frontText,
- this.generateImgAltTextModel
- );
- }
-
- document.content = document.content.replace(
- image,
- `![${altText}](${imageUrl})`
- );
- })
- );
- })
- );
-
- return documents;
- };
-
- filterDepth(documents: Document[]): Document[] {
- return documents.filter((document) => {
- const url = new URL(document.metadata.sourceURL);
- return getURLDepth(url.toString()) <= this.maxCrawledDepth;
- });
- }
-}
diff --git a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts b/apps/api/src/scraper/WebScraper/scrapers/fetch.ts
deleted file mode 100644
index 0df3be724..000000000
--- a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts
+++ /dev/null
@@ -1,89 +0,0 @@
-import axios from "axios";
-import { logScrape } from "../../../services/logging/scrape_log";
-import { fetchAndProcessPdf } from "../utils/pdfProcessor";
-import { universalTimeout } from "../global";
-import { Logger } from "../../../lib/logger";
-
-/**
- * Scrapes a URL with Axios
- * @param url The URL to scrape
- * @param pageOptions The options for the page
- * @returns The scraped content
- */
-export async function scrapWithFetch(
- url: string,
- pageOptions: { parsePDF?: boolean } = { parsePDF: true }
-): Promise<{ content: string; pageStatusCode?: number; pageError?: string }> {
- const logParams = {
- url,
- scraper: "fetch",
- success: false,
- response_code: null,
- time_taken_seconds: null,
- error_message: null,
- html: "",
- startTime: Date.now(),
- };
-
- try {
- const response = await axios.get(url, {
- headers: {
- "Content-Type": "application/json",
- },
- timeout: universalTimeout,
- transformResponse: [(data) => data], // Prevent axios from parsing JSON automatically
- });
-
- if (response.status !== 200) {
- Logger.debug(
- `⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`
- );
- logParams.error_message = response.statusText;
- logParams.response_code = response.status;
- return {
- content: "",
- pageStatusCode: response.status,
- pageError: response.statusText,
- };
- }
-
- const contentType = response.headers["content-type"];
- if (contentType && contentType.includes("application/pdf")) {
- logParams.success = true;
- const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(
- url,
- pageOptions?.parsePDF
- );
- logParams.response_code = pageStatusCode;
- logParams.error_message = pageError;
- return { content, pageStatusCode: response.status, pageError };
- } else {
- const text = response.data;
- logParams.success = true;
- logParams.html = text;
- logParams.response_code = response.status;
- return {
- content: text,
- pageStatusCode: response.status,
- pageError: null,
- };
- }
- } catch (error) {
- if (error.code === "ECONNABORTED") {
- logParams.error_message = "Request timed out";
- Logger.debug(`⛏️ Axios: Request timed out for ${url}`);
- } else {
- logParams.error_message = error.message || error;
- Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`);
- }
- return {
- content: "",
- pageStatusCode: error.response?.status ?? null,
- pageError: logParams.error_message,
- };
- } finally {
- const endTime = Date.now();
- logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;
- await logScrape(logParams);
- }
-}
diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
deleted file mode 100644
index 7616774a7..000000000
--- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
+++ /dev/null
@@ -1,230 +0,0 @@
-import axios from "axios";
-import { Action, FireEngineOptions, FireEngineResponse } from "../../../lib/entities";
-import { logScrape } from "../../../services/logging/scrape_log";
-import { generateRequestParams } from "../single_url";
-import { fetchAndProcessPdf } from "../utils/pdfProcessor";
-import { universalTimeout } from "../global";
-import { Logger } from "../../../lib/logger";
-import * as Sentry from "@sentry/node";
-import axiosRetry from 'axios-retry';
-
-axiosRetry(axios, { retries: 3 , onRetry:()=>{
- console.log("Retrying (fire-engine)...");
-}, retryDelay: axiosRetry.exponentialDelay});
-/**
- * Scrapes a URL with Fire-Engine
- * @param url The URL to scrape
- * @param waitFor The time to wait for the page to load
- * @param screenshot Whether to take a screenshot
- * @param fullPageScreenshot Whether to take a full page screenshot
- * @param pageOptions The options for the page
- * @param headers The headers to send with the request
- * @param options The options for the request
- * @returns The scraped content
- */
-export async function scrapWithFireEngine({
- url,
- actions,
- waitFor = 0,
- screenshot = false,
- fullPageScreenshot = false,
- pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true, mobile: false },
- fireEngineOptions = {},
- headers,
- options,
- priority,
- teamId,
-}: {
- url: string;
- actions?: Action[];
- waitFor?: number;
- screenshot?: boolean;
- fullPageScreenshot?: boolean;
- pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean, mobile?: boolean };
- fireEngineOptions?: FireEngineOptions;
- headers?: Record<string, string>;
- options?: any;
- priority?: number;
- teamId?: string;
-}): Promise<FireEngineResponse> {
- const logParams = {
- url,
- scraper: "fire-engine",
- success: false,
- response_code: null,
- time_taken_seconds: null,
- error_message: null,
- html: "",
- startTime: Date.now(),
- };
-
- try {
- const reqParams = await generateRequestParams(url);
- let waitParam = reqParams["params"]?.wait ?? waitFor;
- let engineParam = reqParams["params"]?.engine ?? reqParams["params"]?.fireEngineOptions?.engine ?? fireEngineOptions?.engine ?? "chrome-cdp";
- let screenshotParam = reqParams["params"]?.screenshot ?? screenshot;
- let fullPageScreenshotParam = reqParams["params"]?.fullPageScreenshot ?? fullPageScreenshot;
- let fireEngineOptionsParam : FireEngineOptions = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions;
-
-
- let endpoint = "/scrape";
-
- if(options?.endpoint === "request") {
- endpoint = "/request";
- }
-
- let engine = engineParam; // do we want fireEngineOptions as first choice?
-
- if (pageOptions?.useFastMode) {
- fireEngineOptionsParam.engine = "tlsclient";
- engine = "tlsclient";
- }
-
- Logger.info(
- `⛏️ Fire-Engine (${engine}): Scraping ${url} | params: { actions: ${JSON.stringify((actions ?? []).map(x => x.type))}, method: ${fireEngineOptionsParam?.method ?? "null"} }`
- );
-
- // atsv is only available for beta customers
- const betaCustomersString = process.env.BETA_CUSTOMERS;
- const betaCustomers = betaCustomersString ? betaCustomersString.split(",") : [];
-
- if (pageOptions?.atsv && betaCustomers.includes(teamId)) {
- fireEngineOptionsParam.atsv = true;
- } else {
- pageOptions.atsv = false;
- }
-
- const axiosInstance = axios.create({
- headers: { "Content-Type": "application/json" }
- });
-
- const startTime = Date.now();
- const _response = await Sentry.startSpan({
- name: "Call to fire-engine"
- }, async span => {
-
- return await axiosInstance.post(
- process.env.FIRE_ENGINE_BETA_URL + endpoint,
- {
- url: url,
- headers: headers,
- wait: waitParam,
- screenshot: screenshotParam,
- fullPageScreenshot: fullPageScreenshotParam,
- disableJsDom: pageOptions?.disableJsDom ?? false,
- priority,
- engine,
- instantReturn: true,
- mobile: pageOptions?.mobile ?? false,
- ...fireEngineOptionsParam,
- atsv: pageOptions?.atsv ?? false,
- scrollXPaths: pageOptions?.scrollXPaths ?? [],
- geolocation: pageOptions?.geolocation,
- skipTlsVerification: pageOptions?.skipTlsVerification ?? false,
- removeBase64Images: pageOptions?.removeBase64Images ?? true,
- actions: actions,
- },
- {
- headers: {
- "Content-Type": "application/json",
- ...(Sentry.isInitialized() ? ({
- "sentry-trace": Sentry.spanToTraceHeader(span),
- "baggage": Sentry.spanToBaggageHeader(span),
- }) : {}),
- }
- }
- );
- });
-
- const waitTotal = (actions ?? []).filter(x => x.type === "wait").reduce((a, x) => (x as { type: "wait"; milliseconds: number; }).milliseconds + a, 0);
-
- let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`);
-
- // added 5 seconds to the timeout to account for 'smart wait'
- while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitTotal + 5000) {
- await new Promise(resolve => setTimeout(resolve, 250)); // wait 0.25 seconds
- checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`);
- }
-
- if (checkStatusResponse.data.processing) {
- Logger.debug(`⛏️ Fire-Engine (${engine}): deleting request - jobId: ${_response.data.jobId}`);
- axiosInstance.delete(
- process.env.FIRE_ENGINE_BETA_URL + `/scrape/${_response.data.jobId}`, {
- validateStatus: (status) => true
- }
- ).catch((error) => {
- Logger.debug(`⛏️ Fire-Engine (${engine}): Failed to delete request - jobId: ${_response.data.jobId} | error: ${error}`);
- });
-
- Logger.debug(`⛏️ Fire-Engine (${engine}): Request timed out for ${url}`);
- logParams.error_message = "Request timed out";
- return { html: "", pageStatusCode: null, pageError: "" };
- }
-
- if (checkStatusResponse.status !== 200 || checkStatusResponse.data.error) {
- Logger.debug(
- `⛏️ Fire-Engine (${engine}): Failed to fetch url: ${url} \t status: ${checkStatusResponse.status}\t ${checkStatusResponse.data.error}`
- );
-
- logParams.error_message = checkStatusResponse.data?.pageError ?? checkStatusResponse.data?.error;
- logParams.response_code = checkStatusResponse.data?.pageStatusCode;
-
- if(checkStatusResponse.data && checkStatusResponse.data?.pageStatusCode !== 200) {
- Logger.debug(`⛏️ Fire-Engine (${engine}): Failed to fetch url: ${url} \t status: ${checkStatusResponse.data?.pageStatusCode}`);
- }
-
- const pageStatusCode = checkStatusResponse.data?.pageStatusCode ? checkStatusResponse.data?.pageStatusCode : checkStatusResponse.data?.error && checkStatusResponse.data?.error.includes("Dns resolution error for hostname") ? 404 : undefined;
-
- return {
- html: "",
- pageStatusCode,
- pageError: checkStatusResponse.data?.pageError ?? checkStatusResponse.data?.error,
- };
- }
-
- const contentType = checkStatusResponse.data.responseHeaders?.["content-type"];
-
- if (contentType && contentType.includes("application/pdf")) {
- const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(
- url,
- pageOptions?.parsePDF
- );
- logParams.success = true;
- logParams.response_code = pageStatusCode;
- logParams.error_message = pageError;
- return { html: content, pageStatusCode, pageError };
- } else {
- const data = checkStatusResponse.data;
-
- logParams.success =
- (data.pageStatusCode >= 200 && data.pageStatusCode < 300) ||
- data.pageStatusCode === 404;
- logParams.html = data.content ?? "";
- logParams.response_code = data.pageStatusCode;
- logParams.error_message = data.pageError ?? data.error;
-
- return {
- html: data.content ?? "",
- screenshots: data.screenshots ?? [data.screenshot] ?? [],
- pageStatusCode: data.pageStatusCode,
- pageError: data.pageError ?? data.error,
- scrapeActionContent: data?.actionContent ?? [],
- };
- }
- } catch (error) {
- if (error.code === "ECONNABORTED") {
- Logger.debug(`⛏️ Fire-Engine (catch block): Request timed out for ${url}`);
- logParams.error_message = "Request timed out";
- } else {
- Logger.debug(`⛏️ Fire-Engine(catch block): Failed to fetch url: ${url} | Error: ${error}`);
- logParams.error_message = error.message || error;
- }
- return { html: "", pageStatusCode: null, pageError: logParams.error_message };
- } finally {
- const endTime = Date.now();
- logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;
- await logScrape(logParams, pageOptions);
- }
-}
-
-
diff --git a/apps/api/src/scraper/WebScraper/scrapers/playwright.ts b/apps/api/src/scraper/WebScraper/scrapers/playwright.ts
deleted file mode 100644
index 09c7353bb..000000000
--- a/apps/api/src/scraper/WebScraper/scrapers/playwright.ts
+++ /dev/null
@@ -1,111 +0,0 @@
-import axios from "axios";
-import { logScrape } from "../../../services/logging/scrape_log";
-import { generateRequestParams } from "../single_url";
-import { fetchAndProcessPdf } from "../utils/pdfProcessor";
-import { universalTimeout } from "../global";
-import { Logger } from "../../../lib/logger";
-
-/**
- * Scrapes a URL with Playwright
- * @param url The URL to scrape
- * @param waitFor The time to wait for the page to load
- * @param headers The headers to send with the request
- * @param pageOptions The options for the page
- * @returns The scraped content
- */
-export async function scrapWithPlaywright(
- url: string,
- waitFor: number = 0,
- headers?: Record<string, string>,
- pageOptions: { parsePDF?: boolean } = { parsePDF: true }
-): Promise<{ content: string; pageStatusCode?: number; pageError?: string }> {
- const logParams = {
- url,
- scraper: "playwright",
- success: false,
- response_code: null,
- time_taken_seconds: null,
- error_message: null,
- html: "",
- startTime: Date.now(),
- };
-
- try {
- const reqParams = await generateRequestParams(url);
- // If the user has passed a wait parameter in the request, use that
- const waitParam = reqParams["params"]?.wait ?? waitFor;
-
- const response = await axios.post(
- process.env.PLAYWRIGHT_MICROSERVICE_URL,
- {
- url: url,
- wait_after_load: waitParam,
- timeout: universalTimeout + waitParam,
- headers: headers,
- },
- {
- headers: {
- "Content-Type": "application/json",
- },
- timeout: universalTimeout + waitParam, // Add waitParam to timeout to account for the wait time
- transformResponse: [(data) => data], // Prevent axios from parsing JSON automatically
- }
- );
-
- if (response.status !== 200) {
- Logger.debug(
- `⛏️ Playwright: Failed to fetch url: ${url} | status: ${response.status}, error: ${response.data?.pageError}`
- );
- logParams.error_message = response.data?.pageError;
- logParams.response_code = response.data?.pageStatusCode;
- return {
- content: "",
- pageStatusCode: response.data?.pageStatusCode,
- pageError: response.data?.pageError,
- };
- }
-
- const contentType = response.headers["content-type"];
- if (contentType && contentType.includes("application/pdf")) {
- logParams.success = true;
- const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
- logParams.response_code = pageStatusCode;
- logParams.error_message = pageError;
- return { content, pageStatusCode, pageError };
- } else {
- const textData = response.data;
- try {
- const data = JSON.parse(textData);
- const html = data.content;
- logParams.success = true;
- logParams.html = html;
- logParams.response_code = data.pageStatusCode;
- logParams.error_message = data.pageError;
- return {
- content: html ?? "",
- pageStatusCode: data.pageStatusCode,
- pageError: data.pageError,
- };
- } catch (jsonError) {
- logParams.error_message = jsonError.message || jsonError;
- Logger.debug(
- `⛏️ Playwright: Error parsing JSON response for url: ${url} | Error: ${jsonError}`
- );
- return { content: "", pageStatusCode: null, pageError: logParams.error_message };
- }
- }
- } catch (error) {
- if (error.code === "ECONNABORTED") {
- logParams.error_message = "Request timed out";
- Logger.debug(`⛏️ Playwright: Request timed out for ${url}`);
- } else {
- logParams.error_message = error.message || error;
- Logger.debug(`⛏️ Playwright: Failed to fetch url: ${url} | Error: ${error}`);
- }
- return { content: "", pageStatusCode: null, pageError: logParams.error_message };
- } finally {
- const endTime = Date.now();
- logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;
- await logScrape(logParams);
- }
-}
diff --git a/apps/api/src/scraper/WebScraper/scrapers/scrapingBee.ts b/apps/api/src/scraper/WebScraper/scrapers/scrapingBee.ts
deleted file mode 100644
index b72fa8b2c..000000000
--- a/apps/api/src/scraper/WebScraper/scrapers/scrapingBee.ts
+++ /dev/null
@@ -1,92 +0,0 @@
-import { logScrape } from "../../../services/logging/scrape_log";
-import { generateRequestParams } from "../single_url";
-import { fetchAndProcessPdf } from "../utils/pdfProcessor";
-import { universalTimeout } from "../global";
-import { ScrapingBeeClient } from "scrapingbee";
-import { Logger } from "../../../lib/logger";
-
-/**
- * Scrapes a URL with ScrapingBee
- * @param url The URL to scrape
- * @param wait_browser The browser event to wait for
- * @param timeout The timeout for the scrape
- * @param pageOptions The options for the page
- * @returns The scraped content
- */
-export async function scrapWithScrapingBee(
- url: string,
- wait_browser: string = "domcontentloaded",
- timeout: number = universalTimeout,
- pageOptions: { parsePDF?: boolean } = { parsePDF: true }
- ): Promise<{ content: string; pageStatusCode?: number; pageError?: string }> {
- const logParams = {
- url,
- scraper: wait_browser === "networkidle2" ? "scrapingBeeLoad" : "scrapingBee",
- success: false,
- response_code: null,
- time_taken_seconds: null,
- error_message: null,
- html: "",
- startTime: Date.now(),
- };
- try {
- const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY);
- const clientParams = await generateRequestParams(
- url,
- wait_browser,
- timeout
- );
- const response = await client.get({
- ...clientParams,
- params: {
- ...clientParams.params,
- transparent_status_code: "True",
- },
- });
- Logger.info(
- `⛏️ ScrapingBee: Scraping ${url}`
- );
- const contentType = response.headers["content-type"];
- if (contentType && contentType.includes("application/pdf")) {
- logParams.success = true;
- const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
- logParams.response_code = pageStatusCode;
- logParams.error_message = pageError;
- return { content, pageStatusCode, pageError };
- } else {
- let text = "";
- try {
- const decoder = new TextDecoder();
- text = decoder.decode(response.data);
- logParams.success = true;
- } catch (decodeError) {
- Logger.debug(
- `⛏️ ScrapingBee: Error decoding response data for url: ${url} | Error: ${decodeError}`
- );
- logParams.error_message = decodeError.message || decodeError;
- }
- logParams.response_code = response.status;
- logParams.html = text;
- logParams.success = response.status >= 200 && response.status < 300 || response.status === 404;
- logParams.error_message = response.statusText !== "OK" ? response.statusText : undefined;
- return {
- content: text,
- pageStatusCode: response.status,
- pageError: response.statusText !== "OK" ? response.statusText : undefined,
- };
- }
- } catch (error) {
- Logger.debug(`⛏️ ScrapingBee: Error fetching url: ${url} | Error: ${error}`);
- logParams.error_message = error.message || error;
- logParams.response_code = error.response?.status;
- return {
- content: "",
- pageStatusCode: error.response?.status,
- pageError: error.response?.statusText,
- };
- } finally {
- const endTime = Date.now();
- logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;
- await logScrape(logParams);
- }
- }
\ No newline at end of file
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
deleted file mode 100644
index 38ec74f03..000000000
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ /dev/null
@@ -1,506 +0,0 @@
-import * as cheerio from "cheerio";
-import { extractMetadata } from "./utils/metadata";
-import dotenv from "dotenv";
-import {
- Document,
- PageOptions,
- FireEngineResponse,
- ExtractorOptions,
- Action,
-} from "../../lib/entities";
-import { parseMarkdown } from "../../lib/html-to-markdown";
-import { urlSpecificParams } from "./utils/custom/website_params";
-import { fetchAndProcessPdf } from "./utils/pdfProcessor";
-import { handleCustomScraping } from "./custom/handleCustomScraping";
-import { removeUnwantedElements } from "./utils/removeUnwantedElements";
-import { scrapWithFetch } from "./scrapers/fetch";
-import { scrapWithFireEngine } from "./scrapers/fireEngine";
-import { scrapWithPlaywright } from "./scrapers/playwright";
-import { scrapWithScrapingBee } from "./scrapers/scrapingBee";
-import { extractLinks } from "./utils/utils";
-import { Logger } from "../../lib/logger";
-import { ScrapeEvents } from "../../lib/scrape-events";
-import { clientSideError } from "../../strings";
-import { ScrapeActionContent } from "../../lib/entities";
-import { removeBase64Images } from "./utils/removeBase64Images";
-
-dotenv.config();
-
-const useScrapingBee = process.env.SCRAPING_BEE_API_KEY !== '' && process.env.SCRAPING_BEE_API_KEY !== undefined;
-const useFireEngine = process.env.FIRE_ENGINE_BETA_URL !== '' && process.env.FIRE_ENGINE_BETA_URL !== undefined;
-
-export const baseScrapers = [
- useFireEngine ? "fire-engine;chrome-cdp" : undefined,
- useFireEngine ? "fire-engine" : undefined,
- useScrapingBee ? "scrapingBee" : undefined,
- useFireEngine ? undefined : "playwright",
- useScrapingBee ? "scrapingBeeLoad" : undefined,
- "fetch",
-].filter(Boolean);
-
-export async function generateRequestParams(
- url: string,
- wait_browser: string = "domcontentloaded",
- timeout: number = 15000
-): Promise<any> {
- const defaultParams = {
- url: url,
- params: { timeout: timeout, wait_browser: wait_browser },
- headers: { "ScrapingService-Request": "TRUE" },
- };
-
- try {
- const urlKey = new URL(url).hostname.replace(/^www\./, "");
- if (urlSpecificParams.hasOwnProperty(urlKey)) {
- return { ...defaultParams, ...urlSpecificParams[urlKey] };
- } else {
- return defaultParams;
- }
- } catch (error) {
- Logger.error(`Error generating URL key: ${error}`);
- return defaultParams;
- }
-}
-
-/**
- * Get the order of scrapers to be used for scraping a URL
- * If the user doesn't have envs set for a specific scraper, it will be removed from the order.
- * @param defaultScraper The default scraper to use if the URL does not have a specific scraper order defined
- * @returns The order of scrapers to be used for scraping a URL
- */
-function getScrapingFallbackOrder(
- defaultScraper?: string,
- isWaitPresent: boolean = false,
- isScreenshotPresent: boolean = false,
- isHeadersPresent: boolean = false,
- isActionsPresent: boolean = false,
-) {
- if (isActionsPresent) {
- return useFireEngine ? ["fire-engine;chrome-cdp"] : [];
- }
-
- const availableScrapers = baseScrapers.filter((scraper) => {
- switch (scraper) {
- case "scrapingBee":
- case "scrapingBeeLoad":
- return !!process.env.SCRAPING_BEE_API_KEY;
- case "fire-engine":
- return !!process.env.FIRE_ENGINE_BETA_URL;
- case "fire-engine;chrome-cdp":
- return !!process.env.FIRE_ENGINE_BETA_URL;
- case "playwright":
- return !!process.env.PLAYWRIGHT_MICROSERVICE_URL;
- default:
- return true;
- }
- });
-
- let defaultOrder = [
- useFireEngine ? "fire-engine;chrome-cdp" : undefined,
- useFireEngine ? "fire-engine" : undefined,
- useScrapingBee ? "scrapingBee" : undefined,
- useScrapingBee ? "scrapingBeeLoad" : undefined,
- useFireEngine ? undefined : "playwright",
- "fetch",
- ].filter(Boolean);
-
- // if (isWaitPresent || isScreenshotPresent || isHeadersPresent) {
- // defaultOrder = [
- // "fire-engine",
- // useFireEngine ? undefined : "playwright",
- // ...defaultOrder.filter(
- // (scraper) => scraper !== "fire-engine" && scraper !== "playwright"
- // ),
- // ].filter(Boolean);
- // }
-
- const filteredDefaultOrder = defaultOrder.filter(
- (scraper: (typeof baseScrapers)[number]) =>
- availableScrapers.includes(scraper)
- );
- const uniqueScrapers = new Set(
- defaultScraper
- ? [defaultScraper, ...filteredDefaultOrder, ...availableScrapers]
- : [...filteredDefaultOrder, ...availableScrapers]
- );
-
- const scrapersInOrder = Array.from(uniqueScrapers);
- return scrapersInOrder as (typeof baseScrapers)[number][];
-}
-
-
-
-export async function scrapSingleUrl(
- jobId: string,
- urlToScrap: string,
- pageOptions: PageOptions,
- extractorOptions?: ExtractorOptions,
- existingHtml?: string,
- priority?: number,
- teamId?: string
-): Promise<Document> {
- pageOptions = {
- includeMarkdown: pageOptions.includeMarkdown ?? true,
- includeExtract: pageOptions.includeExtract ?? false,
- onlyMainContent: pageOptions.onlyMainContent ?? false,
- includeHtml: pageOptions.includeHtml ?? false,
- includeRawHtml: pageOptions.includeRawHtml ?? false,
- waitFor: pageOptions.waitFor ?? undefined,
- screenshot: pageOptions.screenshot ?? false,
- fullPageScreenshot: pageOptions.fullPageScreenshot ?? false,
- headers: pageOptions.headers ?? undefined,
- includeLinks: pageOptions.includeLinks ?? true,
- replaceAllPathsWithAbsolutePaths: pageOptions.replaceAllPathsWithAbsolutePaths ?? true,
- parsePDF: pageOptions.parsePDF ?? true,
- removeTags: pageOptions.removeTags ?? [],
- onlyIncludeTags: pageOptions.onlyIncludeTags ?? [],
- useFastMode: pageOptions.useFastMode ?? false,
- disableJsDom: pageOptions.disableJsDom ?? false,
- atsv: pageOptions.atsv ?? false,
- actions: pageOptions.actions ?? undefined,
- geolocation: pageOptions.geolocation ?? undefined,
- skipTlsVerification: pageOptions.skipTlsVerification ?? false,
- removeBase64Images: pageOptions.removeBase64Images ?? true,
- mobile: pageOptions.mobile ?? false,
- }
-
- if (extractorOptions) {
- extractorOptions = {
- mode: extractorOptions?.mode ?? "llm-extraction-from-markdown",
- }
- }
-
- if (!existingHtml) {
- existingHtml = "";
- }
-
- urlToScrap = urlToScrap.trim();
-
- const attemptScraping = async (
- url: string,
- method: (typeof baseScrapers)[number]
- ) => {
- let scraperResponse: {
- text: string;
- screenshot: string;
- actions?: {
- screenshots?: string[];
- scrapes?: ScrapeActionContent[];
- };
- metadata: { pageStatusCode?: number; pageError?: string | null };
- } = { text: "", screenshot: "", metadata: {} };
- let screenshot = "";
-
- const timer = Date.now();
- const logInsertPromise = ScrapeEvents.insert(jobId, {
- type: "scrape",
- url,
- worker: process.env.FLY_MACHINE_ID,
- method,
- result: null,
- });
-
- switch (method) {
- case "fire-engine":
- case "fire-engine;chrome-cdp":
-
- let engine: "playwright" | "chrome-cdp" | "tlsclient" = "playwright";
- if (method === "fire-engine;chrome-cdp") {
- engine = "chrome-cdp";
- }
-
- if (process.env.FIRE_ENGINE_BETA_URL) {
- const processedActions: Action[] = pageOptions.actions?.flatMap((action: Action, index: number, array: Action[]) => {
- if (action.type === "click" || action.type === "write" || action.type === "press") {
- const result: Action[] = [];
- // Don't add a wait if the previous action is a wait
- // if (index === 0 || array[index - 1].type !== "wait") {
- // result.push({ type: "wait", milliseconds: 1200 } as Action);
- // }
- // Fire-engine now handles wait times automatically, leaving the code here for now
- result.push(action);
- // Don't add a wait if the next action is a wait
- // if (index === array.length - 1 || array[index + 1].type !== "wait") {
- // result.push({ type: "wait", milliseconds: 1200 } as Action);
- // }
- return result;
- }
- return [action as Action];
- }) ?? [] as Action[];
-
- const response = await scrapWithFireEngine({
- url,
- ...(engine === "chrome-cdp" ? ({
- actions: [
- ...(pageOptions.waitFor ? [{
- type: "wait" as const,
- milliseconds: pageOptions.waitFor,
- }] : []),
- ...((pageOptions.screenshot || pageOptions.fullPageScreenshot) ? [{
- type: "screenshot" as const,
- fullPage: !!pageOptions.fullPageScreenshot,
- }] : []),
- ...processedActions,
- ],
- }) : ({
- waitFor: pageOptions.waitFor,
- screenshot: pageOptions.screenshot,
- fullPageScreenshot: pageOptions.fullPageScreenshot,
- })),
- pageOptions: pageOptions,
- headers: pageOptions.headers,
- fireEngineOptions: {
- engine: engine,
- atsv: pageOptions.atsv,
- disableJsDom: pageOptions.disableJsDom,
- },
- priority,
- teamId,
- });
- scraperResponse.text = response.html;
- if (pageOptions.screenshot || pageOptions.fullPageScreenshot) {
- scraperResponse.screenshot = (response.screenshots ?? []).splice(0, 1)[0] ?? "";
- }
- if (pageOptions.actions) {
- scraperResponse.actions = {
- screenshots: response.screenshots ?? [],
- scrapes: response.scrapeActionContent ?? [],
- };
- }
- scraperResponse.metadata.pageStatusCode = response.pageStatusCode;
- scraperResponse.metadata.pageError = response.pageError;
- }
- break;
- case "scrapingBee":
- if (process.env.SCRAPING_BEE_API_KEY) {
- const response = await scrapWithScrapingBee(
- url,
- "domcontentloaded",
- pageOptions.fallback === false ? 7000 : 15000
- );
- scraperResponse.text = response.content;
- scraperResponse.metadata.pageStatusCode = response.pageStatusCode;
- scraperResponse.metadata.pageError = response.pageError;
- }
- break;
- case "playwright":
- if (process.env.PLAYWRIGHT_MICROSERVICE_URL) {
- const response = await scrapWithPlaywright(
- url,
- pageOptions.waitFor,
- pageOptions.headers
- );
- scraperResponse.text = response.content;
- scraperResponse.metadata.pageStatusCode = response.pageStatusCode;
- scraperResponse.metadata.pageError = response.pageError;
- }
- break;
- case "scrapingBeeLoad":
- if (process.env.SCRAPING_BEE_API_KEY) {
- const response = await scrapWithScrapingBee(url, "networkidle2");
- scraperResponse.text = response.content;
- scraperResponse.metadata.pageStatusCode = response.pageStatusCode;
- scraperResponse.metadata.pageError = response.pageError;
- }
- break;
- case "fetch":
- const response = await scrapWithFetch(url);
- scraperResponse.text = response.content;
- scraperResponse.metadata.pageStatusCode = response.pageStatusCode;
- scraperResponse.metadata.pageError = response.pageError;
- break;
- }
-
- let customScrapedContent: FireEngineResponse | null = null;
-
- // Check for custom scraping conditions
- const customScraperResult = await handleCustomScraping(
- scraperResponse.text,
- url
- );
-
- if (customScraperResult) {
- switch (customScraperResult.scraper) {
- case "fire-engine":
- customScrapedContent = await scrapWithFireEngine({
- url: customScraperResult.url,
- actions: customScraperResult.waitAfterLoad ? ([
- {
- type: "wait",
- milliseconds: customScraperResult.waitAfterLoad,
- }
- ]) : ([]),
- pageOptions: customScraperResult.pageOptions,
- });
- break;
- case "pdf":
- const { content, pageStatusCode, pageError } =
- await fetchAndProcessPdf(
- customScraperResult.url,
- pageOptions?.parsePDF
- );
- customScrapedContent = {
- html: content,
- pageStatusCode,
- pageError,
- };
- break;
- }
- }
-
- if (customScrapedContent) {
- scraperResponse.text = customScrapedContent.html;
- }
- //* TODO: add an optional to return markdown or structured/extracted content
- let cleanedHtml = removeUnwantedElements(scraperResponse.text, pageOptions);
- let text = await parseMarkdown(cleanedHtml);
- if (pageOptions.removeBase64Images) {
- text = await removeBase64Images(text);
- }
-
- const insertedLogId = await logInsertPromise;
- ScrapeEvents.updateScrapeResult(insertedLogId, {
- response_size: scraperResponse.text.length,
- success: !(scraperResponse.metadata.pageStatusCode && scraperResponse.metadata.pageStatusCode >= 400) && !!text && (text.trim().length >= 100),
- error: scraperResponse.metadata.pageError,
- response_code: scraperResponse.metadata.pageStatusCode,
- time_taken: Date.now() - timer,
- });
-
- return {
- text,
- html: cleanedHtml,
- rawHtml: scraperResponse.text,
- screenshot: scraperResponse.screenshot,
- actions: scraperResponse.actions,
- pageStatusCode: scraperResponse.metadata.pageStatusCode,
- pageError: scraperResponse.metadata.pageError || undefined,
- };
- };
-
- let { text, html, rawHtml, screenshot, actions, pageStatusCode, pageError } = {
- text: "",
- html: "",
- rawHtml: "",
- screenshot: "",
- actions: undefined,
- pageStatusCode: 200,
- pageError: undefined,
- };
- try {
- let urlKey = urlToScrap;
- try {
- urlKey = new URL(urlToScrap).hostname.replace(/^www\./, "");
- } catch (error) {
- Logger.error(`Invalid URL key, trying: ${urlToScrap}`);
- }
- const defaultScraper = urlSpecificParams[urlKey]?.defaultScraper ?? "";
- const scrapersInOrder = getScrapingFallbackOrder(
- defaultScraper,
- pageOptions && pageOptions.waitFor && pageOptions.waitFor > 0,
- pageOptions && (pageOptions.screenshot || pageOptions.fullPageScreenshot) && (pageOptions.screenshot === true || pageOptions.fullPageScreenshot === true),
- pageOptions && pageOptions.headers && pageOptions.headers !== undefined,
- pageOptions && Array.isArray(pageOptions.actions) && pageOptions.actions.length > 0,
- );
-
- for (const scraper of scrapersInOrder) {
- // If exists text coming from crawler, use it
- if (existingHtml && existingHtml.trim().length >= 100 && !existingHtml.includes(clientSideError)) {
- let cleanedHtml = removeUnwantedElements(existingHtml, pageOptions);
- text = await parseMarkdown(cleanedHtml);
- html = cleanedHtml;
- break;
- }
-
- const attempt = await attemptScraping(urlToScrap, scraper);
- text = attempt.text ?? "";
- html = attempt.html ?? "";
- rawHtml = attempt.rawHtml ?? "";
- screenshot = attempt.screenshot ?? "";
- actions = attempt.actions ?? undefined;
-
- if (attempt.pageStatusCode) {
- pageStatusCode = attempt.pageStatusCode;
- }
-
- if (attempt.pageError && (attempt.pageStatusCode >= 400 || scrapersInOrder.indexOf(scraper) === scrapersInOrder.length - 1)) { // force pageError if it's the last scraper and it failed too
- pageError = attempt.pageError;
-
- if (attempt.pageStatusCode < 400 || !attempt.pageStatusCode) {
- pageStatusCode = 500;
- }
- } else if (attempt && attempt.pageStatusCode && attempt.pageStatusCode < 400) {
- pageError = undefined;
- }
-
- if ((text && text.trim().length >= 100) || (typeof screenshot === "string" && screenshot.length > 0)) {
- Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with text length >= 100 or screenshot, breaking`);
- break;
- }
- if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400)) {
- Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with status code ${pageStatusCode}, breaking`);
- break;
- }
- // const nextScraperIndex = scrapersInOrder.indexOf(scraper) + 1;
- // if (nextScraperIndex < scrapersInOrder.length) {
- // Logger.debug(`⛏️ ${scraper} Failed to fetch URL: ${urlToScrap} with status: ${pageStatusCode}, error: ${pageError} | Falling back to ${scrapersInOrder[nextScraperIndex]}`);
- // }
- }
-
- if (!text) {
- throw new Error(`All scraping methods failed for URL: ${urlToScrap}`);
- }
-
- const soup = cheerio.load(rawHtml);
- const metadata = extractMetadata(soup, urlToScrap);
-
- let linksOnPage: string[] | undefined;
-
- if (pageOptions.includeLinks) {
- linksOnPage = extractLinks(rawHtml, urlToScrap);
- }
-
- let document: Document = {
- content: text,
- markdown: pageOptions.includeMarkdown || pageOptions.includeExtract ? text : undefined,
- html: pageOptions.includeHtml ? html : undefined,
- rawHtml:
- pageOptions.includeRawHtml ||
- (extractorOptions?.mode === "llm-extraction-from-raw-html" && pageOptions.includeExtract)
- ? rawHtml
- : undefined,
- linksOnPage: pageOptions.includeLinks ? linksOnPage : undefined,
- actions,
- metadata: {
- ...metadata,
- ...(screenshot && screenshot.length > 0 ? ({
- screenshot,
- }) : {}),
- sourceURL: urlToScrap,
- pageStatusCode: pageStatusCode,
- pageError: pageError,
- },
- };
-
- return document;
- } catch (error) {
- Logger.debug(`⛏️ Error: ${error.message} - Failed to fetch URL: ${urlToScrap}`);
- ScrapeEvents.insert(jobId, {
- type: "error",
- message: typeof error === "string" ? error : typeof error.message === "string" ? error.message : JSON.stringify(error),
- stack: error.stack,
- });
-
- return {
- content: "",
- markdown: pageOptions.includeMarkdown || pageOptions.includeExtract ? "" : undefined,
- html: "",
- linksOnPage: pageOptions.includeLinks ? [] : undefined,
- metadata: {
- sourceURL: urlToScrap,
- pageStatusCode: pageStatusCode,
- pageError: pageError,
- },
- } as Document;
- }
-}
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 756cd7651..05b3d00d8 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -1,9 +1,10 @@
import axios from "axios";
import { axiosTimeout } from "../../lib/timeout";
import { parseStringPromise } from "xml2js";
-import { scrapWithFireEngine } from "./scrapers/fireEngine";
import { WebCrawler } from "./crawler";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
+import { scrapeURL } from "../scrapeURL";
+import { scrapeOptions } from "../../controllers/v1/types";
export async function getLinksFromSitemap(
{
@@ -17,17 +18,20 @@ export async function getLinksFromSitemap(
}
): Promise<string[]> {
try {
- let content: string;
+ let content: string = "";
try {
if (mode === 'axios' || process.env.FIRE_ENGINE_BETA_URL === '') {
const response = await axios.get(sitemapUrl, { timeout: axiosTimeout });
content = response.data;
} else if (mode === 'fire-engine') {
- const response = await scrapWithFireEngine({ url: sitemapUrl, fireEngineOptions: { engine:"playwright" } });
- content = response.html;
+ const response = await scrapeURL("sitemap", sitemapUrl, scrapeOptions.parse({ formats: ["rawHtml"] }), { forceEngine: "fire-engine;playwright" });;
+ if (!response.success) {
+ throw response.error;
+ }
+ content = response.document.rawHtml!;
}
} catch (error) {
- Logger.error(`Request failed for ${sitemapUrl}: ${error.message}`);
+ logger.error(`Request failed for ${sitemapUrl}: ${error.message}`);
return allUrls;
}
@@ -47,7 +51,7 @@ export async function getLinksFromSitemap(
allUrls.push(...validUrls);
}
} catch (error) {
- Logger.debug(`Error processing sitemapUrl: ${sitemapUrl} | Error: ${error.message}`);
+ logger.debug(`Error processing sitemapUrl: ${sitemapUrl} | Error: ${error.message}`);
}
return allUrls;
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/docxProcessor.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/docxProcessor.test.ts
deleted file mode 100644
index 53237ef89..000000000
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/docxProcessor.test.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-import * as docxProcessor from "../docxProcessor";
-
-describe("DOCX Processing Module - Integration Test", () => {
- it("should correctly process a simple DOCX file without the LLAMAPARSE_API_KEY", async () => {
- delete process.env.LLAMAPARSE_API_KEY;
- const { content, pageStatusCode, pageError } = await docxProcessor.fetchAndProcessDocx(
- "https://nvca.org/wp-content/uploads/2019/06/NVCA-Model-Document-Stock-Purchase-Agreement.docx"
- );
- expect(content.trim()).toContain(
- "SERIES A PREFERRED STOCK PURCHASE AGREEMENT"
- );
- expect(pageStatusCode).toBe(200);
- expect(pageError).toBeUndefined();
- });
-});
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/parseTable.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/parseTable.test.ts
deleted file mode 100644
index 8d644c7bc..000000000
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/parseTable.test.ts
+++ /dev/null
@@ -1,128 +0,0 @@
-import { parseTablesToMarkdown, convertTableElementToMarkdown, convertTableRowElementToMarkdown, createMarkdownDividerRow } from '../parseTable';
-import cheerio from 'cheerio';
-
-describe('parseTablesToMarkdown', () => {
- it('converts a simple HTML table to Markdown', async () => {
- const html = `
- <table>
- <tr><th>Header 1</th><th>Header 2</th></tr>
- <tr><td>Row 1 Col 1</td><td>Row 1 Col 2</td></tr>
- <tr><td>Row 2 Col 1</td><td>Row 2 Col 2</td></tr>
- </table>
- `;
- const expectedMarkdown = `<div>| Header 1 | Header 2 |\n| --- | --- |\n| Row 1 Col 1 | Row 1 Col 2 |\n| Row 2 Col 1 | Row 2 Col 2 |</div>`;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
- it('converts a table with a single row to Markdown', async () => {
- const html = `
- <table>
- <tr><th>Header 1</th><th>Header 2</th></tr>
- <tr><td>Row 1 Col 1</td><td>Row 1 Col 2</td></tr>
- </table>
- `;
- const expectedMarkdown = `<div>| Header 1 | Header 2 |\n| --- | --- |\n| Row 1 Col 1 | Row 1 Col 2 |</div>`;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
- it('converts a table with a single column to Markdown', async () => {
- const html = `
- <table>
- <tr><th>Header 1</th></tr>
- <tr><td>Row 1 Col 1</td></tr>
- <tr><td>Row 2 Col 1</td></tr>
- </table>
- `;
- const expectedMarkdown = `<div>| Header 1 |\n| --- |\n| Row 1 Col 1 |\n| Row 2 Col 1 |</div>`;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
- it('converts a table with a single cell to Markdown', async () => {
- const html = `
- <table>
- <tr><th>Header 1</th></tr>
- <tr><td>Row 1 Col 1</td></tr>
- </table>
- `;
- const expectedMarkdown = `<div>| Header 1 |\n| --- |\n| Row 1 Col 1 |</div>`;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
- it('converts a table with no header to Markdown', async () => {
- const html = `
- <table>
- <tr><td>Row 1 Col 1</td><td>Row 1 Col 2</td></tr>
- <tr><td>Row 2 Col 1</td><td>Row 2 Col 2</td></tr>
- </table>
- `;
- const expectedMarkdown = `<div>| Row 1 Col 1 | Row 1 Col 2 |\n| Row 2 Col 1 | Row 2 Col 2 |</div>`;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
- it('converts a table with no rows to Markdown', async () => {
- const html = `
- <table>
- </table>
- `;
- const expectedMarkdown = `<div></div>`;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
- it('converts a table with no cells to Markdown', async () => {
- const html = `
- <table>
- <tr></tr>
- </table>
- `;
- const expectedMarkdown = `<div></div>`;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
- it('converts a table with no columns to Markdown', async () => {
- const html = `
- <table>
- <tr><th></th></tr>
- </table>
- `;
- const expectedMarkdown = `<div></div>`;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
- it('converts a table with no table to Markdown', async () => {
- const html = ``;
- const expectedMarkdown = ``;
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
- });
-
-it('converts a table inside of a bunch of html noise', async () => {
- const html = `
- <div>
- <p>Some text before</p>
- <table>
- <tr><td>Row 1 Col 1</td><td>Row 1 Col 2</td></tr>
- <tr><td>Row 2 Col 1</td><td>Row 2 Col 2</td></tr>
- </table>
- <p>Some text after</p>
- </div>
- `;
- const expectedMarkdown = `<div>
- <p>Some text before</p>
- <div>| Row 1 Col 1 | Row 1 Col 2 |
-| Row 2 Col 1 | Row 2 Col 2 |</div>
- <p>Some text after</p>
- </div>`;
-
- const markdown = await parseTablesToMarkdown(html);
- expect(markdown).toBe(expectedMarkdown);
-});
-
-});
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/pdfProcessor.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/pdfProcessor.test.ts
deleted file mode 100644
index 18302654d..000000000
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/pdfProcessor.test.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import * as pdfProcessor from '../pdfProcessor';
-
-describe('PDF Processing Module - Integration Test', () => {
- it('should correctly process a simple PDF file without the LLAMAPARSE_API_KEY', async () => {
- delete process.env.LLAMAPARSE_API_KEY;
- const { content, pageStatusCode, pageError } = await pdfProcessor.fetchAndProcessPdf('https://s3.us-east-1.amazonaws.com/storage.mendable.ai/rafa-testing/test%20%281%29.pdf', true);
- expect(content.trim()).toEqual("Dummy PDF file");
- expect(pageStatusCode).toEqual(200);
- expect(pageError).toBeUndefined();
- });
-
- it('should return a successful response for a valid scrape with PDF file and parsePDF set to false', async () => {
- const { content, pageStatusCode, pageError } = await pdfProcessor.fetchAndProcessPdf('https://arxiv.org/pdf/astro-ph/9301001.pdf', false);
- expect(pageStatusCode).toBe(200);
- expect(pageError).toBeUndefined();
- expect(content).toContain('/Title(arXiv:astro-ph/9301001v1 7 Jan 1993)>>endobj');
- }, 60000); // 60 seconds
-
-});
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
deleted file mode 100644
index b3d4a244c..000000000
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
+++ /dev/null
@@ -1,192 +0,0 @@
-import { removeUnwantedElements } from "../removeUnwantedElements";
-import { PageOptions } from "../../../../lib/entities";
-
-describe('removeUnwantedElements', () => {
- it('should remove script, style, iframe, noscript, meta, and head tags', () => {
- const html = `<html><head><title>Test</title></head><body><script>alert('test');</script><div>Content</div></body></html>`;
- const options: PageOptions = {};
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('<script>');
- expect(result).not.toContain('<head>');
- expect(result).toContain('Content');
- });
-
- it('should remove specified tags passed as string', () => {
- const html = `<div><span>Remove</span><p>Keep</p></div>`;
- const options: PageOptions = { removeTags: 'span' };
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('<span>');
- expect(result).toContain('<p>Keep</p>');
- });
-
- it('should remove specified tags passed as array', () => {
- const html = `<div><span>Remove</span><p>Remove</p><a>Keep</a></div>`;
- const options: PageOptions = { removeTags: ['span', 'p'] };
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('<span>');
- expect(result).not.toContain('<p>');
- expect(result).toContain('<a>Keep</a>');
- });
-
- it('should handle class selectors', () => {
- const html = `<div class="test">Remove</div><div class="keep">Keep</div>`;
- const options: PageOptions = { removeTags: '.test' };
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('class="test"');
- expect(result).toContain('class="keep"');
- });
-
- it('should handle id selectors', () => {
- const html = `<div id="test">Remove</div><div id="keep">Keep</div>`;
- const options: PageOptions = { removeTags: '#test' };
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('id="test"');
- expect(result).toContain('id="keep"');
- });
-
- it('should handle regex patterns in class names', () => {
- const html = `<div class="test-123">Remove</div><div class="test-abc">Remove</div><div class="keep">Keep</div>`;
- const options: PageOptions = { removeTags: ['*test*'] };
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('class="test-123"');
- expect(result).not.toContain('class="test-abc"');
- expect(result).toContain('class="keep"');
- });
-
- it('should remove non-main content if onlyMainContent is true', () => {
- const html = `<div><main>Main Content</main><aside>Remove</aside></div>`;
- const options: PageOptions = { onlyMainContent: true };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('Main Content');
- expect(result).not.toContain('<aside>');
- });
-
- it('should handle complex regex patterns for class names', () => {
- const html = `<div class="test-123">Remove</div><div class="test-abc">Remove</div><div class="keep">Keep</div><div class="test-xyz">Remove</div>`;
- const options: PageOptions = { removeTags: ['*.test-[a-z]+*'] };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('class="test-123"');
- expect(result).not.toContain('class="test-abc"');
- expect(result).not.toContain('class="test-xyz"');
- expect(result).toContain('class="keep"');
- });
-
- it('should handle complex regex patterns for attributes', () => {
- const html = `<div data-info="12345">Remove</div><div data-info="abcde">Keep</div><div data-info="67890">Remove</div>`;
- const options: PageOptions = { removeTags: ['*data-info="\\d+"*'] }; // Matches data-info that starts with digits
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('data-info="12345"');
- expect(result).not.toContain('data-info="67890"');
- expect(result).toContain('data-info="abcde"');
- });
-
- it('should handle mixed selectors with regex', () => {
- const html = `<div class="remove-this">Remove</div><div id="remove-this">Remove</div><div class="keep-this">Keep</div>`;
- const options: PageOptions = { removeTags: ['.remove-this', '#remove-this'] };
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('class="remove-this"');
- expect(result).not.toContain('id="remove-this"');
- expect(result).toContain('class="keep-this"');
- });
-
- it('should handle multiple regex patterns', () => {
- const html = `<div attr="test-123">Remove</div><div class="class-remove">Remove</div><div class="keep">Keep</div><div class="remove-this">Remove</div><div id="remove-this">Remove</div>`;
- const options: PageOptions = { removeTags: ['*test*', '.class-remove', '*.remove-[a-z]+*', '#remove-this'] };
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain('class="test-123"');
- expect(result).not.toContain('class="test-abc"');
- expect(result).not.toContain('class="remove"');
- expect(result).not.toContain('class="remove-this"');
- expect(result).not.toContain('id="remove-this"');
- expect(result).toContain('class="keep"');
- });
-
- it('should only include specified tags', () => {
- const html = `<div><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div>`;
- const options: PageOptions = { onlyIncludeTags: ['main', 'footer'] };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('<main>Main Content</main>');
- expect(result).toContain('<footer>Footer Content</footer>');
- expect(result).not.toContain('<aside>');
- });
-
- it('should handle multiple specified tags', () => {
- const html = `<div><header>Header Content</header><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div>`;
- const options: PageOptions = { onlyIncludeTags: ['header', 'main', 'footer'] };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('<header>Header Content</header>');
- expect(result).toContain('<main>Main Content</main>');
- expect(result).toContain('<footer>Footer Content</footer>');
- expect(result).not.toContain('<aside>');
- });
-
- it('should handle nested specified tags', () => {
- const html = `<div><main><section>Main Section</section></main><aside>Remove</aside><footer>Footer Content</footer></div>`;
- const options: PageOptions = { onlyIncludeTags: ['main', 'footer'] };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('<main><section>Main Section</section></main>');
- expect(result).toContain('<footer>Footer Content</footer>');
- expect(result).not.toContain('<aside>');
- });
-
- it('should not handle no specified tags, return full content', () => {
- const html = `<html><body><div><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div></body></html>`;
- const options: PageOptions = { onlyIncludeTags: [] };
- const result = removeUnwantedElements(html, options);
- expect(result).toBe(html);
- });
-
- it('should handle specified tags as a string', () => {
- const html = `<div><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div>`;
- const options: PageOptions = { onlyIncludeTags: 'main' };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('<main>Main Content</main>');
- expect(result).not.toContain('<aside>');
- expect(result).not.toContain('<footer>');
- });
-
- it('should include specified tags with class', () => {
- const html = `<div><main class="main-content">Main Content</main><aside class="remove">Remove</aside><footer class="footer-content">Footer Content</footer></div>`;
- const options: PageOptions = { onlyIncludeTags: ['.main-content', '.footer-content'] };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('<main class="main-content">Main Content</main>');
- expect(result).toContain('<footer class="footer-content">Footer Content</footer>');
- expect(result).not.toContain('<aside class="remove">');
- });
-
- it('should include specified tags with id', () => {
- const html = `<div><main id="main-content">Main Content</main><aside id="remove">Remove</aside><footer id="footer-content">Footer Content</footer></div>`;
- const options: PageOptions = { onlyIncludeTags: ['#main-content', '#footer-content'] };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('<main id="main-content">Main Content</main>');
- expect(result).toContain('<footer id="footer-content">Footer Content</footer>');
- expect(result).not.toContain('<aside id="remove">');
- });
-
- it('should include specified tags with mixed class and id', () => {
- const html = `<div><main class="main-content">Main Content</main><aside id="remove">Remove</aside><footer id="footer-content">Footer Content</footer></div>`;
- const options: PageOptions = { onlyIncludeTags: ['.main-content', '#footer-content'] };
- const result = removeUnwantedElements(html, options);
- expect(result).toContain('<main class="main-content">Main Content</main>');
- expect(result).toContain('<footer id="footer-content">Footer Content</footer>');
- expect(result).not.toContain('<aside id="remove">');
- });
-
- it("should handle cases when both onlyIncludeTags and removeTags are spec'ed", () => {
- const html = `<html lang="en-us" class="__className_36bd41 scroll-smooth">\n <head>\n <meta charSet="utf-8"/>\n <meta name="viewport" content="width=device-width, initial-scale=1"/>\n <link rel="preload" href="/_next/static/media/a34f9d1faa5f3315-s.p.woff2" as="font" crossorigin="" type="font/woff2"/>\n <link rel="preload" as="image" href="/svgs/icons/hipaa.svg"/>\n <link rel="preload" as="image" href="/svgs/icons/SOC-2.svg"/>\n <link rel="stylesheet" href="/_next/static/css/dc440bb5df3a2a20.css" data-precedence="next"/>\n <link rel="stylesheet" href="/_next/static/css/96453cad2522e5b6.css" data-precedence="next"/>\n <link rel="stylesheet" href="/_next/static/css/7e9926f1811c58a4.css" data-precedence="next"/>\n <link rel="stylesheet" href="/_next/static/css/0e73fdc06d8f3d28.css" data-precedence="next"/>\n <link rel="stylesheet" href="/_next/static/css/5530fdfe2817e998.css" data-precedence="next"/>\n <link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-fd8ac6b7ecd741f2.js"/>\n \x3Cscript src="/_next/static/chunks/fd9d1056-ef540aea688b4954.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/7023-05706eafa5cc37bc.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/main-app-b8d8f654a4c3de8b.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/231-548ba71c7495df73.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/5195-b5571fdee73263ae.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/app/docs/layout-6b3055e085554812.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/8032-14ad89cc247326be.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/6648-ad581d791656821c.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/9955-d2e26e97f84b5584.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/3284-f8e8685fda3e660b.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/1510-58126d0ea9869b6a.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/8279-af16984c3178bf87.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/8546-37038ae8418f7528.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/app/layout-44a592a2d715aed2.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/9448-374795341a6afe73.js" async="">\x3C/script>\n \x3Cscript src="/_next/static/chunks/app/docs/%5B...slug%5D/page-5ec83f06ca67a50a.js" async="">\x3C/script>\n <link rel="preload" href="https://www.googletagmanager.com/gtm.js?id=GTM-N9B6D4H" as="script"/>\n <title>Docker Swarm | SigNoz</title>\n <meta name="description" content="Learn how to operate SigNoz on Docker Swarm"/>\n <meta name="robots" content="index, follow"/>\n <meta name="googlebot" content="index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1"/>\n <link rel="canonical" href="https://signoz.io/docs/operate/docker-swarm/"/>\n <link rel="alternate" type="application/rss+xml" href="https://signoz.io/feed.xml/"/>\n <meta property="og:title" content="Docker Swarm"/>\n <meta property="og:description" content="Learn how to operate SigNoz on Docker Swarm"/>\n <meta property="og:url" content="https://signoz.io/docs/operate/docker-swarm/"/>\n <meta property="og:site_name" content="SigNoz"/>\n <meta property="og:locale" content="en_US"/>\n <meta property="og:type" content="article"/>\n <meta name="twitter:card" content="summary_large_image"/>\n <meta name="twitter:title" content="Docker Swarm"/>\n <meta name="twitter:description" content="Learn how to operate SigNoz on Docker Swarm"/>\n <meta name="next-size-adjust"/>\n <link rel="apple-touch-icon" sizes="76x76" href="/static/favicons/apple-touch-icon.png"/>\n <link rel="icon" type="image/png" sizes="32x32" href="/static/favicons/favicon-32x32.png"/>\n <link rel="icon" type="image/png" sizes="16x16" href="/static/favicons/favicon-16x16.png"/>\n <link rel="manifest" href="/static/favicons/site.webmanifest"/>\n <link rel="mask-icon" href="/static/favicons/safari-pinned-tab.svg" color="#5bbad5"/>\n <meta name="msapplication-TileColor" content="#000000"/>\n <meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/>\n <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000"/>\n <link rel="alternate" type="application/rss+xml" href="/feed.xml"/>\n \x3Cscript src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js" noModule="">\x3C/script>\n </head>\n <body class="pl-[calc(100vw-100%)] text-white antialiased">\n <noscript>\n <iframe src="https://www.googletagmanager.com/ns.html?id=GTM-N9B6D4H" height="0" width="0" style="display:none;visibility:hidden"></iframe>\n </noscript>\n \x3Cscript>\n !function() {\n try {\n var d = document.documentElement\n , c = d.classList;\n c.remove('light', 'dark');\n var e = localStorage.getItem('theme');\n if ('system' === e || (!e && false)) {\n var t = '(prefers-color-scheme: dark)'\n , m = window.matchMedia(t);\n if (m.media !== t || m.matches) {\n d.style.colorScheme = 'dark';\n c.add('dark')\n } else {\n d.style.colorScheme = 'light';\n c.add('light')\n }\n } else if (e) {\n c.add(e || '')\n } else {\n c.add('dark')\n }\n if (e === 'light' || e === 'dark' || !e)\n d.style.colorScheme = e || 'dark'\n } catch (e) {}\n }()\n \x3C/script>\n <section class="relative mx-auto">\n <div class="relative flex h-screen flex-col justify-between ">\n <div class="fixed left-0 right-0 z-30">\n <header class="header-bg mx-auto box-border flex h-[56px] w-full items-center border-b border-signoz_slate-500 px-4 text-signoz_vanilla-100 backdrop-blur-[20px] dark:text-signoz_vanilla-100 md:px-8 lg:px-8">\n <nav class="container flex w-full justify-between text-signoz_vanilla-100 dark:text-signoz_vanilla-100" aria-label="Global">\n <div class="flex justify-start gap-x-6">\n <a class="-m-1.5 flex items-center gap-2 p-1.5" href="/">\n <img alt="" loading="lazy" width="160" height="60" decoding="async" data-nimg="1" class="h-5 w-auto" style="color:transparent" src="/img/SigNozLogo-orange.svg"/>\n <span class="text-[17.111px] font-medium">SigNoz</span>\n </a>\n <div class="hidden items-center gap-x-6 lg:flex">\n <div class="flex items-center">\n <button class="z-10 aria-expanded:scale-[0.97] aria-expanded:opacity-70 subpixel-antialiased truncate px-1.5 py-1 text-sm font-extralight hover:text-signoz_robin-500" data-slot="trigger" aria-haspopup="true" aria-expanded="false" id="react-aria-:R1ipbara:" type="button" data-headlessui-state="">\n <div class="flex items-center">\n Product\n <svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-chevron-down ml-1 transform transition-transform duration-300 ease-in-out rotate-0">\n <path d="m6 9 6 6 6-6"></path>\n </svg>\n </div>\n </button>\n </div>\n <a class="flex items-center truncate px-1.5 py-1 text-sm font-normal hover:text-signoz_robin-500" href="/docs/">Documentation</a>\n <div class="flex items-center">\n <button class="z-10 aria-expanded:scale-[0.97] aria-expanded:opacity-70 subpixel-antialiased truncate px-1.5 py-1 text-sm !font-extralight leading-7 text-signoz_vanilla-100 hover:text-signoz_robin-500" data-slot="trigger" aria-haspopup="true" aria-expanded="false" id="react-aria-:R3ipbara:" type="button" data-headlessui-state="">\n <div class="flex items-center">\n Resources\n <svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-chevron-down ml-1 transform transition-transform duration-300 ease-in-out rotate-0">\n <path d="m6 9 6 6 6-6"></path>\n </svg>\n </div>\n </button>\n </div>\n <a class="flex items-center truncate px-1.5 py-1 text-sm font-normal hover:text-signoz_robin-500" href="/pricing/">Pricing</a>\n <a class="flex items-center truncate px-1.5 py-1 text-sm font-normal hover:text-signoz_robin-500" href="/case-study/">Customer Stories</a>\n </div>\n </div>\n <div class="flex justify-end lg:hidden">\n <button type="button" class="-m-2.5 inline-flex items-center justify-center rounded-md p-2.5">\n <span class="sr-only">Open main menu</span>\n <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="h-6 w-6">\n <path stroke-linecap="round" stroke-linejoin="round" d="M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"></path>\n </svg>\n </button>\n </div>\n <div class="hidden gap-3 lg:flex lg:flex-1 lg:justify-end">\n <button aria-label="Search" class="px-2 py-2 bg-signoz_slate-500 rounded-full">\n <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search">\n <circle cx="11" cy="11" r="8"></circle>\n <path d="m21 21-4.3-4.3"></path>\n </svg>\n </button>\n <button id="btn-get-started-website-navbar" class="start-free-trial-btn mx-2 flex h-8 items-center justify-center gap-1.5 truncate rounded-full px-4 py-2 pl-4 pr-3 text-center text-sm font-medium not-italic leading-5 text-white no-underline outline-none hover:text-white" type="button" data-headlessui-state="">\n <a class="flex-center" href="/teams/">\n Try SigNoz Cloud \n <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right">\n <path d="M5 12h14"></path>\n <path d="m12 5 7 7-7 7"></path>\n </svg>\n </a>\n </button>\n </div>\n </nav>\n <span hidden="" style="position:fixed;top:1px;left:1px;width:1px;height:0;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0;display:none"></span>\n </header>\n </div>\n <main class="mb-auto mt-[48px]">\n <main class="">\n <section class="relative mx-auto">\n <div class="progress-bar fixed left-0 right-0 top-[52px] z-30 w-full">\n <div class="h-1 bg-indigo-500" style="width:0%"></div>\n </div>\n <div class="max-sm:px-4 doc overflow-clip">\n <div class="doc-sidenav border-r border-signoz_slate-500"></div>\n <div class="doc-content md:px-0 lg:px-4">\n <article class="prose prose-slate max-w-none py-6 dark:prose-invert">\n <div class="doc">\n <div class="doc-content">\n <h2 class="mt-2 text-3xl">Docker Swarm</h2>\n <div class="admonition border-l-4 border-green-400 bg-green-900 p-4">\n <div class="flex items-center mb-2">\n <span class="font-bold mr-2 text-xl text-white">πŸ’‘\n \x3C!-- -->\n \x3C!-- -->\n Tip</span>\n </div>\n <div class="text-gray-300">\n <p>The easiest way to run SigNoz is to use SigNoz Cloud - no installation, maintenance, or scaling needed.</p>\n <p>\n New users get 30 days of unlimited access to all features. Click <a href="/teams/">here</a>\n to sign up.\n </p>\n </div>\n </div>\n <p>Once you have successfully installed SigNoz on Docker Swarm, the following sections provide an overview of the activities that are required to successfully operate SigNoz.</p>\n <h2 class="content-header" id="stopstart-signoz-cluster">\n <a href="#stopstart-signoz-cluster" aria-hidden="true" tabindex="-1">\n <span class="content-header-link">\n <svg class="h-5 linkicon w-5" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">\n <path d="M12.232 4.232a2.5 2.5 0 0 1 3.536 3.536l-1.225 1.224a.75.75 0 0 0 1.061 1.06l1.224-1.224a4 4 0 0 0-5.656-5.656l-3 3a4 4 0 0 0 .225 5.865.75.75 0 0 0 .977-1.138 2.5 2.5 0 0 1-.142-3.667l3-3Z"></path>\n <path d="M11.603 7.963a.75.75 0 0 0-.977 1.138 2.5 2.5 0 0 1 .142 3.667l-3 3a2.5 2.5 0 0 1-3.536-3.536l1.225-1.224a.75.75 0 0 0-1.061-1.06l-1.224 1.224a4 4 0 1 0 5.656 5.656l3-3a4 4 0 0 0-.225-5.865Z"></path>\n </svg>\n </span>\n </a>\n Stop/Start SigNoz Cluster\n </h2>\n <p>To stop SigNoz cluster:</p>\n <div class="relative">\n <pre>\n <code class="language-bash code-highlight">\n <span class="code-line">\n <span class="token function">docker</span>\n stack <span class="token function">rm</span>\n signoz\n\n </span>\n </code>\n </pre>\n </div>\n <p>To start/resume SigNoz cluster:</p>\n <div class="relative">\n <pre>\n <code class="language-bash code-highlight">\n <span class="code-line">\n <span class="token function">docker</span>\n stack deploy <span class="token parameter variable">-c</span>\n docker-swarm/clickhouse-setup/docker-compose.yaml signoz\n\n </span>\n </code>\n </pre>\n </div>\n <div class="admonition border-l-4 border-gray-400 bg-gray-900 p-4">\n <div class="flex items-center mb-2">\n <span class="font-bold mr-2 text-xl text-white">πŸ“\n \x3C!-- -->\n \x3C!-- -->\n Note</span>\n </div>\n <div class="text-gray-300">\n <p>The newly created SigNoz cluster should mount on the existing local path in the node.</p>\n </div>\n </div>\n <h2 class="content-header" id="upgrade-signoz-cluster">\n <a href="#upgrade-signoz-cluster" aria-hidden="true" tabindex="-1">\n <span class="content-header-link">\n <svg class="h-5 linkicon w-5" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">\n <path d="M12.232 4.232a2.5 2.5 0 0 1 3.536 3.536l-1.225 1.224a.75.75 0 0 0 1.061 1.06l1.224-1.224a4 4 0 0 0-5.656-5.656l-3 3a4 4 0 0 0 .225 5.865.75.75 0 0 0 .977-1.138 2.5 2.5 0 0 1-.142-3.667l3-3Z"></path>\n <path d="M11.603 7.963a.75.75 0 0 0-.977 1.138 2.5 2.5 0 0 1 .142 3.667l-3 3a2.5 2.5 0 0 1-3.536-3.536l1.225-1.224a.75.75 0 0 0-1.061-1.06l-1.224 1.224a4 4 0 1 0 5.656 5.656l3-3a4 4 0 0 0-.225-5.865Z"></path>\n </svg>\n </span>\n </a>\n Upgrade SigNoz Cluster\n </h2>\n <p>\n To upgrade, you can manually update the image tag for <code>query-service</code>\n , <code>frontend</code>\n and <code>otel-collector</code>\n . And run the command to start the cluster:\n </p>\n <div class="relative">\n <pre>\n <code class="language-bash code-highlight">\n <span class="code-line">\n <span class="token function">docker</span>\n stack deploy <span class="token parameter variable">-c</span>\n docker-swarm/clickhouse-setup/docker-compose.yaml signoz\n\n </span>\n </code>\n </pre>\n </div>\n <div class="admonition border-l-4 border-gray-400 bg-gray-900 p-4">\n <div class="flex items-center mb-2">\n <span class="font-bold mr-2 text-xl text-white">πŸ“\n \x3C!-- -->\n \x3C!-- -->\n Note</span>\n </div>\n <div class="text-gray-300">\n <ul>\n <li>Be careful! There might be configuration changes and version mismatch.</li>\n <li>\n Before upgrading, checkout to the release tag: for example <code>git checkout v0.6.1</code>\n and compare the Docker Compose YAML and config files.\n </li>\n </ul>\n </div>\n </div>\n <h2 class="content-header" id="uninstall-signoz-cluster">\n <a href="#uninstall-signoz-cluster" aria-hidden="true" tabindex="-1">\n <span class="content-header-link">\n <svg class="h-5 linkicon w-5" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">\n <path d="M12.232 4.232a2.5 2.5 0 0 1 3.536 3.536l-1.225 1.224a.75.75 0 0 0 1.061 1.06l1.224-1.224a4 4 0 0 0-5.656-5.656l-3 3a4 4 0 0 0 .225 5.865.75.75 0 0 0 .977-1.138 2.5 2.5 0 0 1-.142-3.667l3-3Z"></path>\n <path d="M11.603 7.963a.75.75 0 0 0-.977 1.138 2.5 2.5 0 0 1 .142 3.667l-3 3a2.5 2.5 0 0 1-3.536-3.536l1.225-1.224a.75.75 0 0 0-1.061-1.06l-1.224 1.224a4 4 0 1 0 5.656 5.656l3-3a4 4 0 0 0-.225-5.865Z"></path>\n </svg>\n </span>\n </a>\n Uninstall SigNoz Cluster\n </h2>\n <p>To delete/uninstall SigNoz cluster:</p>\n <div class="relative">\n <pre>\n <code class="language-bash code-highlight">\n <span class="code-line">\n <span class="token function">docker</span>\n stack <span class="token function">rm</span>\n signoz\n\n </span>\n </code>\n </pre>\n </div>\n <h2 class="content-header" id="scale-up-signoz-cluster">\n <a href="#scale-up-signoz-cluster" aria-hidden="true" tabindex="-1">\n <span class="content-header-link">\n <svg class="h-5 linkicon w-5" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">\n <path d="M12.232 4.232a2.5 2.5 0 0 1 3.536 3.536l-1.225 1.224a.75.75 0 0 0 1.061 1.06l1.224-1.224a4 4 0 0 0-5.656-5.656l-3 3a4 4 0 0 0 .225 5.865.75.75 0 0 0 .977-1.138 2.5 2.5 0 0 1-.142-3.667l3-3Z"></path>\n <path d="M11.603 7.963a.75.75 0 0 0-.977 1.138 2.5 2.5 0 0 1 .142 3.667l-3 3a2.5 2.5 0 0 1-3.536-3.536l1.225-1.224a.75.75 0 0 0-1.061-1.06l-1.224 1.224a4 4 0 1 0 5.656 5.656l3-3a4 4 0 0 0-.225-5.865Z"></path>\n </svg>\n </span>\n </a>\n Scale Up SigNoz Cluster\n </h2>\n <p>\n SigNoz uses the <a target="_blank" rel="noopener noreferrer" href="https://github.com/open-telemetry/opentelemetry-collector">OpenTelemetry Collector</a>\n to ingest data. By default, the instructions in the <a href="/docs/install/docker-swarm/">Install SigNoz on Docker Swarm</a>\n document create three replicas, and each replica can handle 50K spans per second. To handle an increased load, perform the following steps:\n </p>\n <ol>\n <li>\n <p>\n Open the <code>deploy/docker-swarm/clickhouse-setup/docker-compose.yaml</code>\n file in a plain-text editor.\n </p>\n </li>\n <li>\n <p>\n In the <code>services.otel-collector.deploy.replicas</code>\n field, enter the number of replicas you wish to create. The following example creates four replicas:\n </p>\n </li>\n </ol>\n <div>\n <div aria-owns="rmiz-modal-" data-rmiz="">\n <div data-rmiz-content="not-found" style="visibility:visible">\n <img alt="Open Telemetry Collector - Create four replicas" loading="lazy" width="1464" height="762" decoding="async" data-nimg="1" style="color:transparent" srcSet="/_next/image/?url=%2Fimg%2Fscale-up-otel.webp&amp;w=1920&amp;q=75 1x, /_next/image/?url=%2Fimg%2Fscale-up-otel.webp&amp;w=3840&amp;q=75 2x" src="/_next/image/?url=%2Fimg%2Fscale-up-otel.webp&amp;w=3840&amp;q=75"/>\n </div>\n </div>\n </div>\n <ol start="3">\n <li>\n <p>\n Update the <code>signoz</code>\n stack by entering the following command:\n </p>\n <div class="relative">\n <pre>\n <code class="language-bash code-highlight"></code>\n </pre>\n </div>\n </li>\n </ol>\n <p>docker stack deploy -c docker-swarm/clickhouse-setup/docker-compose.yaml signoz</p>\n <div class="relative">\n <pre>\n <code class="code-highlight language-js"></code>\n </pre>\n </div>\n <div class="PageFeedback_feedbackContainer___tGjJ">\n <div class="PageFeedback_separatorLine__DIBW2"></div>\n <h3 class="PageFeedback_title__bvKx6">Was this page helpful?</h3>\n <div class="PageFeedback_buttonGroup__T4R6c">\n <button class="PageFeedback_button__A_6sn">πŸ‘ Yes</button>\n <button class="PageFeedback_button__A_6sn">πŸ‘Ž No</button>\n </div>\n </div>\n <div class="docs-prev-next-nav mt-16 flex items-center justify-between">\n <a class="docs-prev rounded bg-signoz_slate-500 p-2 px-4 no-underline" href="/docs/operate/docker-standalone/">\n <div class="mb-2 text-xs font-bold">Prev</div>\n <div class="flex items-center justify-center gap-1 text-sm font-bold">\n <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-chevrons-left">\n <path d="m11 17-5-5 5-5"></path>\n <path d="m18 17-5-5 5-5"></path>\n </svg>\n \x3C!-- -->\n Docker Standalone\n </div>\n </a>\n <a class="docs-next rounded bg-signoz_slate-500 p-2 px-4 no-underline" href="/docs/operate/kubernetes/">\n <div class="mb-2 flex justify-end text-xs font-bold">Next</div>\n <div class="flex items-center justify-end gap-1 text-sm font-bold">\n Kubernetes\n <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-chevrons-right">\n <path d="m6 17 5-5-5-5"></path>\n <path d="m13 17 5-5-5-5"></path>\n </svg>\n </div>\n </a>\n </div>\n </div>\n <div class="doc-toc">\n <div class="mb-3 text-xs uppercase">On this page </div>\n <div class="doc-toc-items border-l border-signoz_slate-500 pl-3 ">\n <div class="doc-toc-item">\n <a data-level="2" href="#stopstart-signoz-cluster" class="mb-1 line-clamp-2 text-xs">Stop/Start SigNoz Cluster</a>\n </div>\n <div class="doc-toc-item">\n <a data-level="2" href="#upgrade-signoz-cluster" class="mb-1 line-clamp-2 text-xs">Upgrade SigNoz Cluster</a>\n </div>\n <div class="doc-toc-item">\n <a data-level="2" href="#uninstall-signoz-cluster" class="mb-1 line-clamp-2 text-xs">Uninstall SigNoz Cluster</a>\n </div>\n <div class="doc-toc-item">\n <a data-level="2" href="#scale-up-signoz-cluster" class="mb-1 line-clamp-2 text-xs">Scale Up SigNoz Cluster</a>\n </div>\n </div>\n </div>\n </div>\n </article>\n </div>\n </div>\n </section>\n </main>\n </main>\n <div class="flex flex-col justify-center border-t border-solid border-gray-900 z-[10] bg-signoz_ink-500 backdrop-blur-md bg-opacity-70">\n <div class="flex w-full items-center justify-center bg-opacity-70 px-16 py-14 max-md:max-w-full max-md:px-5">\n <div class="container w-full max-w-[1200px] justify-between max-md:max-w-full">\n <div class="flex gap-5 max-md:flex-col max-md:gap-0">\n <div class="flex w-3/12 flex-col max-md:ml-0 max-md:w-full">\n <div class="flex flex-col pb-2.5 text-sm tracking-wide text-stone-300 max-md:mt-10">\n <div class="text-sm font-semibold uppercase leading-5 tracking-wide">Docs</div>\n <a class="mt-5 hover:underline" href="/docs/">Introduction</a>\n <a class="mt-5 hover:underline" href="/docs/contributing/">Contributing</a>\n <div class="mt-5 flex items-center gap-2 pr-7 hover:underline max-md:pr-5">\n <a target="_blank" href="https://knowledgebase.signoz.io/kb">Knowledge Base</a>\n <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-up-right">\n <path d="M7 7h10v10"></path>\n <path d="M7 17 17 7"></path>\n </svg>\n </div>\n <a class="mt-5 hover:underline" href="/api_reference/">SigNoz API</a>\n </div>\n </div>\n <div class="ml-5 flex w-3/12 flex-col max-md:ml-0 max-md:w-full">\n <div class="flex grow flex-col self-stretch pb-20 text-sm tracking-wide text-stone-300 max-md:mt-10">\n <div class="text-sm font-semibold uppercase leading-5 tracking-wide">Community</div>\n <div class="mt-5 flex items-center gap-2 pr-7 hover:underline max-md:pr-5">\n <a href="/support/">Support</a>\n </div>\n <div class="mt-5 flex items-center gap-2 pr-7 hover:underline max-md:pr-5">\n <a target="_blank" href="https://signoz.io/slack">Slack</a>\n <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-up-right">\n <path d="M7 7h10v10"></path>\n <path d="M7 17 17 7"></path>\n </svg>\n </div>\n <div class="mt-5 flex items-center gap-2 whitespace-nowrap pr-8 hover:underline max-md:pr-5">\n <a target="_blank" href="https://twitter.com/SigNozHQ">Twitter</a>\n <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-up-right">\n <path d="M7 7h10v10"></path>\n <path d="M7 17 17 7"></path>\n </svg>\n </div>\n <div class="mt-5 flex items-center gap-2 whitespace-nowrap pr-8 hover:underline max-md:pr-5">\n <a target="_blank" href="https://community-chat.signoz.io/">Community Archive</a>\n <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-up-right">\n <path d="M7 7h10v10"></path>\n <path d="M7 17 17 7"></path>\n </svg>\n </div>\n <div class="mt-5 flex items-center gap-2 pr-7 hover:underline max-md:pr-5">\n <a href="/changelog/">Changelog</a>\n </div>\n </div>\n </div>\n <div class="ml-5 flex w-3/12 flex-col max-md:ml-0 max-md:w-full">\n <div class="flex grow flex-col self-stretch pb-20 text-sm tracking-wide text-stone-300 max-md:mt-10">\n <div class="text-sm font-semibold uppercase leading-5 tracking-wide">More</div>\n <a class="mt-5 hover:underline" href="/product-comparison/signoz-vs-datadog/">SigNoz vs Datadog</a>\n <a class="mt-5 hover:underline" href="/product-comparison/signoz-vs-newrelic/">SigNoz vs New Relic</a>\n <a class="mt-5 hover:underline" href="/product-comparison/signoz-vs-grafana/">SigNoz vs Grafana</a>\n <a class="mt-5 hover:underline" href="/product-comparison/signoz-vs-dynatrace/">SigNoz vs Dynatrace</a>\n <div class="mt-5 flex items-center gap-2 whitespace-nowrap pr-8 hover:underline max-md:pr-5">\n <a target="_blank" href="https://jobs.gem.com/signoz">Careers</a>\n <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-up-right">\n <path d="M7 7h10v10"></path>\n <path d="M7 17 17 7"></path>\n </svg>\n </div>\n <a class="mt-5 hover:underline " href="/about-us/">About</a>\n <a class="mt-5 hover:underline" href="/terms-of-service/">Terms</a>\n <a class="mt-5 hover:underline" href="/privacy/">Privacy</a>\n <a target="_blank" class="mt-5 hover:underline" href="https://trust.signoz.io/">Security &amp;Compliance</a>\n </div>\n </div>\n <div class="ml-5 flex w-3/12 flex-col max-md:ml-0 max-md:w-full">\n <div class="flex flex-col items-end shadow-sm max-md:mt-10">\n <div class="flex items-center justify-between gap-2 self-end whitespace-nowrap text-center text-lg font-medium leading-5 text-white">\n <img alt="" loading="lazy" width="50" height="50" decoding="async" data-nimg="1" class="h-5 w-auto" style="color:transparent" src="/img/SigNozLogo-orange.svg"/>\n <div class="font-satoshi-bold font-medium">SigNoz</div>\n </div>\n <div class="mt-5 items-end justify-center rounded text-sm leading-5 text-emerald-300">All systems operational</div>\n <div class="mt-5 flex items-end justify-between gap-4 py-2 footer-icons">\n <a target="_blank" aria-label="SigNoz Github URL" href="https://github.com/SigNoz">\n <svg xmlns="http://www.w3.org/2000/svg" class="fill-signoz_vanilla-400" viewBox="0 0 496 512">\n <path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3 .3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5 .3-6.2 2.3zm44.2-1.7c-2.9 .7-4.9 2.6-4.6 4.9 .3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3 .7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3 .3 2.9 2.3 3.9 1.6 1 3.6 .7 4.3-.7 .7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3 .7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3 .7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"></path>\n </svg>\n </a>\n <a target="_blank" aria-label="SigNoz LinkedIN URL" href="https://www.linkedin.com/company/signozio/">\n <svg xmlns="http://www.w3.org/2000/svg" class="fill-signoz_vanilla-400" viewBox="0 0 448 512">\n <path fill="#C0C1C3" d="M100.3 448H7.4V148.9h92.9zM53.8 108.1C24.1 108.1 0 83.5 0 53.8a53.8 53.8 0 0 1 107.6 0c0 29.7-24.1 54.3-53.8 54.3zM447.9 448h-92.7V302.4c0-34.7-.7-79.2-48.3-79.2-48.3 0-55.7 37.7-55.7 76.7V448h-92.8V148.9h89.1v40.8h1.3c12.4-23.5 42.7-48.3 87.9-48.3 94 0 111.3 61.9 111.3 142.3V448z"></path>\n </svg>\n </a>\n <a target="_blank" aria-label="SigNoz Slack Connect URL" href="https://signoz.io/slack">\n <svg xmlns="http://www.w3.org/2000/svg" class="fill-signoz_vanilla-400" viewBox="0 0 448 512">\n <path fill="#C0C1C3" d="M94.1 315.1c0 25.9-21.2 47.1-47.1 47.1S0 341 0 315.1c0-25.9 21.2-47.1 47.1-47.1h47.1v47.1zm23.7 0c0-25.9 21.2-47.1 47.1-47.1s47.1 21.2 47.1 47.1v117.8c0 25.9-21.2 47.1-47.1 47.1s-47.1-21.2-47.1-47.1V315.1zm47.1-189c-25.9 0-47.1-21.2-47.1-47.1S139 32 164.9 32s47.1 21.2 47.1 47.1v47.1H164.9zm0 23.7c25.9 0 47.1 21.2 47.1 47.1s-21.2 47.1-47.1 47.1H47.1C21.2 244 0 222.8 0 196.9s21.2-47.1 47.1-47.1H164.9zm189 47.1c0-25.9 21.2-47.1 47.1-47.1 25.9 0 47.1 21.2 47.1 47.1s-21.2 47.1-47.1 47.1h-47.1V196.9zm-23.7 0c0 25.9-21.2 47.1-47.1 47.1-25.9 0-47.1-21.2-47.1-47.1V79.1c0-25.9 21.2-47.1 47.1-47.1 25.9 0 47.1 21.2 47.1 47.1V196.9zM283.1 385.9c25.9 0 47.1 21.2 47.1 47.1 0 25.9-21.2 47.1-47.1 47.1-25.9 0-47.1-21.2-47.1-47.1v-47.1h47.1zm0-23.7c-25.9 0-47.1-21.2-47.1-47.1 0-25.9 21.2-47.1 47.1-47.1h117.8c25.9 0 47.1 21.2 47.1 47.1 0 25.9-21.2 47.1-47.1 47.1H283.1z"></path>\n </svg>\n </a>\n <a target="_blank" aria-label="SigNoz Twitter URL" href="https://twitter.com/SigNozHQ">\n <svg xmlns="http://www.w3.org/2000/svg" class="fill-signoz_vanilla-400" viewBox="0 0 512 512">\n <path fill="#C0C1C3" d="M459.4 151.7c.3 4.5 .3 9.1 .3 13.6 0 138.7-105.6 298.6-298.6 298.6-59.5 0-114.7-17.2-161.1-47.1 8.4 1 16.6 1.3 25.3 1.3 49.1 0 94.2-16.6 130.3-44.8-46.1-1-84.8-31.2-98.1-72.8 6.5 1 13 1.6 19.8 1.6 9.4 0 18.8-1.3 27.6-3.6-48.1-9.7-84.1-52-84.1-103v-1.3c14 7.8 30.2 12.7 47.4 13.3-28.3-18.8-46.8-51-46.8-87.4 0-19.5 5.2-37.4 14.3-53 51.7 63.7 129.3 105.3 216.4 109.8-1.6-7.8-2.6-15.9-2.6-24 0-57.8 46.8-104.9 104.9-104.9 30.2 0 57.5 12.7 76.7 33.1 23.7-4.5 46.5-13.3 66.6-25.3-7.8 24.4-24.4 44.8-46.1 57.8 21.1-2.3 41.6-8.1 60.4-16.2-14.3 20.8-32.2 39.3-52.6 54.3z"></path>\n </svg>\n </a>\n <a target="_blank" aria-label="SigNoz Youtube Channel URL" href="https://www.youtube.com/@signoz">\n <svg xmlns="http://www.w3.org/2000/svg" class="fill-signoz_vanilla-400" viewBox="0 0 576 512">\n <path d="M549.7 124.1c-6.3-23.7-24.8-42.3-48.3-48.6C458.8 64 288 64 288 64S117.2 64 74.6 75.5c-23.5 6.3-42 24.9-48.3 48.6-11.4 42.9-11.4 132.3-11.4 132.3s0 89.4 11.4 132.3c6.3 23.7 24.8 41.5 48.3 47.8C117.2 448 288 448 288 448s170.8 0 213.4-11.5c23.5-6.3 42-24.2 48.3-47.8 11.4-42.9 11.4-132.3 11.4-132.3s0-89.4-11.4-132.3zm-317.5 213.5V175.2l142.7 81.2-142.7 81.2z"></path>\n </svg>\n </a>\n </div>\n <div class="mt-5 flex flex-row gap-8">\n <img class="opacity-60 hover:opacity-100 cursor-pointer" src="/svgs/icons/hipaa.svg" width="90" height="90" alt=""/>\n <img class="shadow-[0px_0_40px_0_rgba(255,255,255,0.27)] rounded-full opacity-60 hover:opacity-100 transition-opacity cursor-pointer" src="/svgs/icons/SOC-2.svg" width="60" height="60" alt=""/>\n </div>\n </div>\n </div>\n </div>\n </div>\n </div>\n </div>\n </div>\n </section>\n \x3Cscript src="/_next/static/chunks/webpack-fd8ac6b7ecd741f2.js" async="">\x3C/script>\n \x3Cscript>\n (self.__next_f = self.__next_f || []).push([0]);\n self.__next_f.push([2, null])\n \x3C/script>\n \x3Cscript>\n self.__next_f.push([1, "1:HL["/_next/static/media/a34f9d1faa5f3315-s.p.woff2","font",{"crossOrigin":"","type":"font/woff2"}]\n2:HL["/_next/static/css/dc440bb5df3a2a20.css","style"]\n3:HL["/_next/static/css/96453cad2522e5b6.css","style"]\n4:HL["/_next/static/css/7e9926f1811c58a4.css","style"]\n5:HL["/_next/static/css/0e73fdc06d8f3d28.css","style"]\n6:HL["/_next/static/css/5530fdfe2817e998.css","style"]\n"])\n \x3C/script>\n \x3Cscript>\n self.__next_f.push([1, "7:I[95751,[],""]\na:I[39275,[],""]\nc:I[61343,[],""]\nd:I[6709,["231","static/chunks/231-548ba71c7495df73.js","5195","static/chunks/5195-b5571fdee73263ae.js","3998","static/chunks/app/docs/layout-6b3055e085554812.js"],"default"]\ne:I[27640,["231","static/chunks/231-548ba71c7495df73.js","8032","static/chunks/8032-14ad89cc247326be.js","6648","static/chunks/6648-ad581d791656821c.js","9955","static/chunks/9955-d2e26e97f84b5584.js","3284","static/chunks/3284-f8e8685fda3e660b.js","1510","static/chunks/1510-58126d0ea9869b6a.js","8279","static/chunks/8279-af16984c3178bf87.js","8546","static/chunks/8546-37038ae8418f7528.js","5195","static/chunks/5195-b5571fdee73263ae.js","3185","static/chunks/app/layout-44a592a2d715aed2.js"],"GoogleTagManager"]\nf:I[8700,["231","static/chunks/231-548ba71c7495df73.js","8032","static/chunks/8032-14ad89cc247326be.js","6648","static/chunks/6648-ad581d791656821c.js","9955","static/chunks/9955-d2e26e97f84b5584.js","3284","static/chunks/3284-f8e8685fda3e660b.js","1510","static/chunks/1510-58126d0ea9869b6a.js","8279","static/chunks/8279-af16984c3178bf87.js","8546","static/chunks/8546-37038ae8418f7528.js","5195","static/chunks/5195-b5571fdee73263ae.js","3185","static/chunks/app/layout-44a592a2d715aed2.js"],"ThemeProviders"]\n10:I[53345,["231","static/chunks/231-548ba71c7495df73.js","8032","static/chunks/8032-14ad89cc247326be.js","6648","static/chunks/6648-ad581d791656821c.js","9955","static/chunks/9955-d2e26e97f84b5584.js","3284","static/chunks/3284-f8e8685fda3e660b.js","1510","static/chunks/1510-58126d0ea9869b6a.js","8279","static/chunks/8279-af16984c3178bf87.js","8546","static/chunks/8546-37038ae8418f7528.js","5195","static/chunks/5195-b5571fdee73263ae.js","3185","static/chunks/app/layout-44a592a2d715aed2.js"],"AlgoliaSearchProvider"]\n11:I[74462,["231","static/chunks/231-548ba71c7495df73.js","8032","static/chunks/8032-14ad89cc247326be.js","6648","static/chunks/6648-ad581d791656821c.js","9955","static/chunks/9955-d2e26e97f84b5584.js","3284","static/chunks/3284-f8e8685fda3e660b.js","1510","static/chunks"])\n \x3C/script>\n \x3Cscript>\n self.__next_f.push([1, "/1510-58126d0ea9869b6a.js","8279","static/chunks/8279-af16984c3178bf87.js","8546","static/chunks/8546-37038ae8418f7528.js","5195","static/chunks/5195-b5571fdee73263ae.js","3185","static/chunks/app/layout-44a592a2d715aed2.js"],"default"]\n12:I[231,["231","static/chunks/231-548ba71c7495df73.js","6648","static/chunks/6648-ad581d791656821c.js","9448","static/chunks/9448-374795341a6afe73.js","5195","static/chunks/5195-b5571fdee73263ae.js","6687","static/chunks/app/docs/%5B...slug%5D/page-5ec83f06ca67a50a.js"],""]\n13:I[31206,["231","static/chunks/231-548ba71c7495df73.js","8032","static/chunks/8032-14ad89cc247326be.js","6648","static/chunks/6648-ad581d791656821c.js","9955","static/chunks/9955-d2e26e97f84b5584.js","3284","static/chunks/3284-f8e8685fda3e660b.js","1510","static/chunks/1510-58126d0ea9869b6a.js","8279","static/chunks/8279-af16984c3178bf87.js","8546","static/chunks/8546-37038ae8418f7528.js","5195","static/chunks/5195-b5571fdee73263ae.js","3185","static/chunks/app/layout-44a592a2d715aed2.js"],"default"]\n15:I[76130,[],""]\nb:["slug","operate/docker-swarm","c"]\n16:[]\n"])\n \x3C/script>\n \x3Cscript>\n self.__next_f.push([1, "0:[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/dc440bb5df3a2a20.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/96453cad2522e5b6.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","2",{"rel":"stylesheet","href":"/_next/static/css/7e9926f1811c58a4.css","precedence":"next","crossOrigin":"$undefined"}]],["$","$L7",null,{"buildId":"HORH3B7AXoX8ufSz6zuhr","assetPrefix":"","initialCanonicalUrl":"/docs/operate/docker-swarm/","initialTree":["",{"children":["docs",{"children":[["slug","operate/docker-swarm","c"],{"children":["__PAGE__?{\\"slug\\":[\\"operate\\",\\"docker-swarm\\"]}",{}]}]}]},"$undefined","$undefined",true],"initialSeedData":["",{"children":["docs",{"children":[["slug","operate/docker-swarm","c"],{"children":["__PAGE__",{},[["$L8","$L9"],null],null]},["$","$La",null,{"parallelRouterKey":"children","segmentPath":["children","docs","children","$b","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$Lc",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","styles":[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/5530fdfe2817e998.css","precedence":"next","crossOrigin":"$undefined"}]]}],null]},[["$","$Ld",null,{"children":["$","$La",null,{"parallelRouterKey":"children","segmentPath":["children","docs","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$Lc",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","styles":null}],"params":{}}],null],null]},[["$","html",null,{"lang":"en-us","className":"__className_36bd41 scroll-smooth","suppressHydrationWarning":true,"children":[["$","$Le",null,{"gtmId":"GTM-N9B6D4H"}],["$","link",null,{"rel":"apple-touch-icon","sizes":"76x76","href":"/static/favicons/apple-touch-icon.png"}],["$","link",null,{"rel":"icon","type":"image/png","sizes":"32x32","href":"/static/favicons/favicon-32x32.png"}],["$","link",null,{"rel":"icon","type":"image/png","sizes":"16x16","href":"/static/favicons/favicon-16x16.png"}],["$","link",null,{"rel":"manifest","href":"/static/favicons/site.webmanifest"}],["$","link",null,{"rel":"mask-icon","href":"/static/favicons/safari-pinned-tab.svg","color":"#5bbad5"}],["$","meta",null,{"name":"msapplication-TileColor","content":"#000000"}],["$","meta",null,{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}],["$","meta",null,{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#000"}],["$","link",null,{"rel":"alternate","type":"application/rss+xml","href":"/feed.xml"}],["$","body",null,{"className":"pl-[calc(100vw-100%)] text-white antialiased","children":[["$","noscript",null,{"children":["$","iframe",null,{"src":"https://www.googletagmanager.com/ns.html?id=GTM-N9B6D4H","height":"0","width":"0","style":{"display":"none","visibility":"hidden"}}]}],["$","$Lf",null,{"children":["$","section",null,{"className":"relative mx-auto","children":["$","div",null,{"className":"relative flex h-screen flex-col justify-between ","children":[["$","$L10",null,{"algoliaConfig":{"appId":"E88QYIQ6K4","apiKey":"1061c173ae001d5f5aaa4f0354e373a0","indexName":"signoz"},"children":[["$","$L11",null,{}],["$","main",null,{"className":"mb-auto mt-[48px]","children":["$","$La",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$Lc",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":["$","div",null,{"className":"flex flex-col items-start justify-start py-16 md:flex-row md:items-center md:justify-center md:space-x-6","children":[["$","div",null,{"className":"space-x-2 pb-8 pt-6 md:space-y-5","children":["$","h1",null,{"className":"text-6xl font-extrabold leading-9 tracking-tight text-gray-900 dark:text-gray-100 md:border-r-2 md:px-6 md:text-8xl md:leading-14","children":"404"}]}],["$","div",null,{"className":"max-w-md","children":[["$","p",null,{"className":"mb-4 text-xl font-bold leading-normal md:text-2xl","children":"Sorry we couldn't find this page."}],["$","p",null,{"className":"mb-8","children":"But don't worry, you can find plenty of other things on our homepage."}],["$","$L12",null,{"href":"/","className":"focus:shadow-outline-blue inline rounded-lg border border-transparent bg-blue-600 px-4 py-2 text-sm font-medium leading-5 text-white shadow transition-colors duration-150 hover:bg-blue-700 focus:outline-none dark:hover:bg-blue-500","children":"Back to homepage"}]]}]]}],"notFoundStyles":[],"styles":[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/0e73fdc06d8f3d28.css","precedence":"next","crossOrigin":"$undefined"}]]}]}]]}],["$","$L13",null,{}]]}]}]}]]}]]}],null],null],"couldBeIntercepted":false,"initialHead":[false,"$L14"],"globalErrorComponent":"$15","missingSlots":"$W16"}]]\n"])\n \x3C/script>\n \x3Cscript>\n self.__next_f.push([1, "17:I[40408,["231","static/chunks/231-548ba71c7495df73.js","6648","static/chunks/6648-ad581d791656821c.js","9448","static/chunks/9448-374795341a6afe73.js","5195","static/chunks/5195-b5571fdee73263ae.js","6687","static/chunks/app/docs/%5B...slug%5D/page-5ec83f06ca67a50a.js"],"default"]\n18:I[20356,["231","static/chunks/231-548ba71c7495df73.js","6648","static/chunks/6648-ad581d791656821c.js","9448","static/chunks/9448-374795341a6afe73.js","5195","static/chunks/5195-b5571fdee73263ae.js","6687","static/chunks/app/docs/%5B...slug%5D/page-5ec83f06ca67a50a.js"],"default"]\n19:I[33943,["231","static/chunks/231-548ba71c7495df73.js","6648","static/chunks/6648-ad581d791656821c.js","9448","static/chunks/9448-374795341a6afe73.js","5195","static/chunks/5195-b5571fdee73263ae.js","6687","static/chunks/app/docs/%5B...slug%5D/page-5ec83f06ca67a50a.js"],"default"]\n1a:I[47502,["231","static/chunks/231-548ba71c7495df73.js","6648","static/chunks/6648-ad581d791656821c.js","9448","static/chunks/9448-374795341a6afe73.js","5195","static/chunks/5195-b5571fdee73263ae.js","6687","static/chunks/app/docs/%5B...slug%5D/page-5ec83f06ca67a50a.js"],"default"]\n"])\n \x3C/script>\n \x3Cscript>\n self.__next_f.push([1, "9:["$","div",null,{"className":"doc","children":[["$","div",null,{"className":"doc-content","children":[["$","h2",null,{"className":"mt-2 text-3xl","children":"Docker Swarm"}],[["$","div",null,{"className":"admonition border-l-4 border-green-400 bg-green-900 p-4","children":[["$","div",null,{"className":"flex items-center mb-2","children":["$","span",null,{"className":"font-bold mr-2 text-xl text-white","children":["πŸ’‘"," ","Tip"]}]}],["$","div",null,{"className":"text-gray-300","children":[["$","p",null,{"children":"The easiest way to run SigNoz is to use SigNoz Cloud - no installation, maintenance, or scaling needed."}],["$","p",null,{"children":["New users get 30 days of unlimited access to all features. Click ",["$","a",null,{"href":"/teams/","children":"here"}]," to sign up."]}]]}]]}],["$","p",null,{"children":"Once you have successfully installed SigNoz on Docker Swarm, the following sections provide an overview of the activities that are required to successfully operate SigNoz."}],["$","h2",null,{"className":"content-header","id":"stopstart-signoz-cluster","children":[["$","a",null,{"href":"#stopstart-signoz-cluster","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"content-header-link","children":["$","svg",null,{"className":"h-5 linkicon w-5","fill":"currentColor","viewBox":"0 0 20 20","xmlns":"http://www.w3.org/2000/svg","children":[["$","path",null,{"d":"M12.232 4.232a2.5 2.5 0 0 1 3.536 3.536l-1.225 1.224a.75.75 0 0 0 1.061 1.06l1.224-1.224a4 4 0 0 0-5.656-5.656l-3 3a4 4 0 0 0 .225 5.865.75.75 0 0 0 .977-1.138 2.5 2.5 0 0 1-.142-3.667l3-3Z"}],["$","path",null,{"d":"M11.603 7.963a.75.75 0 0 0-.977 1.138 2.5 2.5 0 0 1 .142 3.667l-3 3a2.5 2.5 0 0 1-3.536-3.536l1.225-1.224a.75.75 0 0 0-1.061-1.06l-1.224 1.224a4 4 0 1 0 5.656 5.656l3-3a4 4 0 0 0-.225-5.865Z"}]]}]}]}],"Stop/Start SigNoz Cluster"]}],["$","p",null,{"children":"To stop SigNoz cluster:"}],["$","$L17",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"docker"}]," stack ",["$","span",null,{"className":"token function","children":"rm"}]," signoz\\n"]}]}]}],["$","p",null,{"children":"To start/resume SigNoz cluster:"}],["$","$L17",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"docker"}]," stack deploy ",["$","span",null,{"className":"token parameter variable","children":"-c"}]," docker-swarm/clickhouse-setup/docker-compose.yaml signoz\\n"]}]}]}],["$","div",null,{"className":"admonition border-l-4 border-gray-400 bg-gray-900 p-4","children":[["$","div",null,{"className":"flex items-center mb-2","children":["$","span",null,{"className":"font-bold mr-2 text-xl text-white","children":["πŸ“"," ","Note"]}]}],["$","div",null,{"className":"text-gray-300","children":["$","p",null,{"children":"The newly created SigNoz cluster should mount on the existing local path in the node."}]}]]}],["$","h2",null,{"className":"content-header","id":"upgrade-signoz-cluster","children":[["$","a",null,{"href":"#upgrade-signoz-cluster","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"content-header-link","children":["$","svg",null,{"className":"h-5 linkicon w-5","fill":"currentColor","viewBox":"0 0 20 20","xmlns":"http://www.w3.org/2000/svg","children":[["$","path",null,{"d":"M12.232 4.232a2.5 2.5 0 0 1 3.536 3.536l-1.225 1.224a.75.75 0 0 0 1.061 1.06l1.224-1.224a4 4 0 0 0-5.656-5.656l-3 3a4 4 0 0 0 .225 5.865.75.75 0 0 0 .977-1.138 2.5 2.5 0 0 1-.142-3.667l3-3Z"}],["$","path",null,{"d":"M11.603 7.963a.75.75 0 0 0-.977 1.138 2.5 2.5 0 0 1 .142 3.667l-3 3a2.5 2.5 0 0 1-3.536-3.536l1.225-1.224a.75.75 0 0 0-1.061-1.06l-1.224 1.224a4 4 0 1 0 5.656 5.656l3-3a4 4 0 0 0-.225-5.865Z"}]]}]}]}],"Upgrade SigNoz Cluster"]}],["$","p",null,{"children":["To upgrade, you can manually update the image tag for ",["$","code",null,{"children":"query-service"}],", ",["$","code",null,{"children":"frontend"}]," and ",["$","code",null,{"children":"otel-collector"}],". And run the command to start the cluster:"]}],["$","$L17",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"docker"}]," stack deploy ",["$","span",null,{"className":"token parameter variable","children":"-c"}]," docker-swarm/clickhouse-setup/docker-compose.yaml signoz\\n"]}]}]}],["$","div",null,{"className":"admonition border-l-4 border-gray-400 bg-gray-900 p-4","children":[["$","div",null,{"className":"flex items-center mb-2","children":["$","span",null,{"className":"font-bold mr-2 text-xl text-white","children":["πŸ“"," ","Note"]}]}],["$","div",null,{"className":"text-gray-300","children":["$","ul",null,{"children":[["$","li",null,{"children":"Be careful! There might be configuration changes and version mismatch."}],["$","li",null,{"children":["Before upgrading, checkout to the release tag: for example ",["$","code",null,{"children":"git checkout v0.6.1"}]," and compare the Docker Compose YAML and config files."]}]]}]}]]}],["$","h2",null,{"className":"content-header","id":"uninstall-signoz-cluster","children":[["$","a",null,{"href":"#uninstall-signoz-cluster","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"content-header-link","children":["$","svg",null,{"className":"h-5 linkicon w-5","fill":"currentColor","viewBox":"0 0 20 20","xmlns":"http://www.w3.org/2000/svg","children":[["$","path",null,{"d":"M12.232 4.232a2.5 2.5 0 0 1 3.536 3.536l-1.225 1.224a.75.75 0 0 0 1.061 1.06l1.224-1.224a4 4 0 0 0-5.656-5.656l-3 3a4 4 0 0 0 .225 5.865.75.75 0 0 0 .977-1.138 2.5 2.5 0 0 1-.142-3.667l3-3Z"}],["$","path",null,{"d":"M11.603 7.963a.75.75 0 0 0-.977 1.138 2.5 2.5 0 0 1 .142 3.667l-3 3a2.5 2.5 0 0 1-3.536-3.536l1.225-1.224a.75.75 0 0 0-1.061-1.06l-1.224 1.224a4 4 0 1 0 5.656 5.656l3-3a4 4 0 0 0-.225-5.865Z"}]]}]}]}],"Uninstall SigNoz Cluster"]}],["$","p",null,{"children":"To delete/uninstall SigNoz cluster:"}],["$","$L17",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"docker"}]," stack ",["$","span",null,{"className":"token function","children":"rm"}]," signoz\\n"]}]}]}],["$","h2",null,{"className":"content-header","id":"scale-up-signoz-cluster","children":[["$","a",null,{"href":"#scale-up-signoz-cluster","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"content-header-link","children":["$","svg",null,{"className":"h-5 linkicon w-5","fill":"currentColor","viewBox":"0 0 20 20","xmlns":"http://www.w3.org/2000/svg","children":[["$","path",null,{"d":"M12.232 4.232a2.5 2.5 0 0 1 3.536 3.536l-1.225 1.224a.75.75 0 0 0 1.061 1.06l1.224-1.224a4 4 0 0 0-5.656-5.656l-3 3a4 4 0 0 0 .225 5.865.75.75 0 0 0 .977-1.138 2.5 2.5 0 0 1-.142-3.667l3-3Z"}],["$","path",null,{"d":"M11.603 7.963a.75.75 0 0 0-.977 1.138 2.5 2.5 0 0 1 .142 3.667l-3 3a2.5 2.5 0 0 1-3.536-3.536l1.225-1.224a.75.75 0 0 0-1.061-1.06l-1.224 1.224a4 4 0 1 0 5.656 5.656l3-3a4 4 0 0 0-.225-5.865Z"}]]}]}]}],"Scale Up SigNoz Cluster"]}],["$","p",null,{"children":["SigNoz uses the ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://github.com/open-telemetry/opentelemetry-collector","children":"OpenTelemetry Collector"}]," to ingest data. By default, the instructions in the ",["$","$L12",null,{"href":"/docs/install/docker-swarm","children":"Install SigNoz on Docker Swarm"}]," document create three replicas, and each replica can handle 50K spans per second. To handle an increased load, perform the following steps:"]}],["$","ol",null,{"children":[["$","li",null,{"children":["$","p",null,{"children":["Open the ",["$","code",null,{"children":"deploy/docker-swarm/clickhouse-setup/docker-compose.yaml"}]," file in a plain-text editor."]}]}],["$","li",null,{"children":["$","p",null,{"children":["In the ",["$","code",null,{"children":"services.otel-collector.deploy.replicas"}]," field, enter the number of replicas you wish to create. The following example creates four replicas:"]}]}]]}],["$","div",null,{"children":["$","$L18",null,{"alt":"Open Telemetry Collector - Create four replicas","src":"/img/scale-up-otel.webp","width":"1464","height":"762"}]}],["$","ol",null,{"start":"3","children":["$","li",null,{"children":[["$","p",null,{"children":["Update the ",["$","code",null,{"children":"signoz"}]," stack by entering the following command:"]}],["$","$L17",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight"}]}]]}]}],["$","p",null,{"children":"docker stack deploy -c docker-swarm/clickhouse-setup/docker-compose.yaml signoz"}],["$","$L17",null,{"className":"language-js","children":["$","code",null,{"className":"code-highlight language-js"}]}]],["$","$L19",null,{}],["$","$L1a",null,{}]]}],["$","div",null,{"className":"doc-toc","children":[["$","div",null,{"className":"mb-3 text-xs uppercase","children":" On this page "}],["$","div",null,{"className":"doc-toc-items border-l border-signoz_slate-500 pl-3 ","children":[["$","div","#stopstart-signoz-cluster",{"className":"doc-toc-item","children":["$","a",null,{"data-level":2,"href":"#stopstart-signoz-cluster","className":"mb-1 line-clamp-2 text-xs","children":"Stop/Start SigNoz Cluster"}]}],["$","div","#upgrade-signoz-cluster",{"className":"doc-toc-item","children":["$","a",null,{"data-level":2,"href":"#upgrade-signoz-cluster","className":"mb-1 line-clamp-2 text-xs","children":"Upgrade SigNoz Cluster"}]}],["$","div","#uninstall-signoz-cluster",{"className":"doc-toc-item","children":["$","a",null,{"data-level":2,"href":"#uninstall-signoz-cluster","className":"mb-1 line-clamp-2 text-xs","children":"Uninstall SigNoz Cluster"}]}],["$","div","#scale-up-signoz-cluster",{"className":"doc-toc-item","children":["$","a",null,{"data-level":2,"href":"#scale-up-signoz-cluster","className":"mb-1 line-clamp-2 text-xs","children":"Scale Up SigNoz Cluster"}]}]]}]]}]]}]\n"])\n \x3C/script>\n \x3Cscript>\n self.__next_f.push([1, "14:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"Docker Swarm | SigNoz"}],["$","meta","3",{"name":"description","content":"Learn how to operate SigNoz on Docker Swarm"}],["$","meta","4",{"name":"robots","content":"index, follow"}],["$","meta","5",{"name":"googlebot","content":"index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1"}],["$","link","6",{"rel":"canonical","href":"https://signoz.io/docs/operate/docker-swarm/"}],["$","link","7",{"rel":"alternate","type":"application/rss+xml","href":"https://signoz.io/feed.xml/"}],["$","meta","8",{"property":"og:title","content":"Docker Swarm"}],["$","meta","9",{"property":"og:description","content":"Learn how to operate SigNoz on Docker Swarm"}],["$","meta","10",{"property":"og:url","content":"https://signoz.io/docs/operate/docker-swarm/"}],["$","meta","11",{"property":"og:site_name","content":"SigNoz"}],["$","meta","12",{"property":"og:locale","content":"en_US"}],["$","meta","13",{"property":"og:type","content":"article"}],["$","meta","14",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","15",{"name":"twitter:title","content":"Docker Swarm"}],["$","meta","16",{"name":"twitter:description","content":"Learn how to operate SigNoz on Docker Swarm"}],["$","meta","17",{"name":"next-size-adjust"}]]\n8:null\n"])\n \x3C/script>\n </body>\n</html>\n`;
-
- const options: PageOptions = {
- onlyIncludeTags: [".doc-content"],
- removeTags: [
- "nav",
- "img",
- "pre",
- ".PageFeedback_feedbackContainer___tGjJ",
- ],
- };
- const result = removeUnwantedElements(html, options);
- expect(result).not.toContain(
- 'class="PageFeedback_feedbackContainer___tGjJ"',
- );
- });
-});
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/replacePaths.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/replacePaths.test.ts
deleted file mode 100644
index e201926e6..000000000
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/replacePaths.test.ts
+++ /dev/null
@@ -1,127 +0,0 @@
-import { Document } from "../../../../lib/entities";
-import { replacePathsWithAbsolutePaths, replaceImgPathsWithAbsolutePaths } from "../replacePaths";
-
-describe('replacePaths', () => {
- describe('replacePathsWithAbsolutePaths', () => {
- it('should replace relative paths with absolute paths', () => {
- const documents: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'This is a [link](/path/to/resource).',
- markdown: 'This is a [link](/path/to/resource).'
- }];
-
- const expectedDocuments: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'This is a [link](https://example.com/path/to/resource).',
- markdown: 'This is a [link](https://example.com/path/to/resource).'
- }];
-
- const result = replacePathsWithAbsolutePaths(documents);
- expect(result).toEqual(expectedDocuments);
- });
-
- it('should not alter absolute URLs', () => {
- const documents: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'This is an [external link](https://external.com/path).',
- markdown: 'This is an [external link](https://external.com/path).'
- }];
-
- const result = replacePathsWithAbsolutePaths(documents);
- expect(result).toEqual(documents); // Expect no change
- });
-
- it('should not alter data URLs for images', () => {
- const documents: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'This is an image: ![alt text](data:image/png;base64,ABC123==).',
- markdown: 'This is an image: ![alt text](data:image/png;base64,ABC123==).'
- }];
-
- const result = replacePathsWithAbsolutePaths(documents);
- expect(result).toEqual(documents); // Expect no change
- });
-
- it('should handle multiple links and images correctly', () => {
- const documents: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'Here are two links: [link1](/path1) and [link2](/path2).',
- markdown: 'Here are two links: [link1](/path1) and [link2](/path2).'
- }];
-
- const expectedDocuments: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'Here are two links: [link1](https://example.com/path1) and [link2](https://example.com/path2).',
- markdown: 'Here are two links: [link1](https://example.com/path1) and [link2](https://example.com/path2).'
- }];
-
- const result = replacePathsWithAbsolutePaths(documents);
- expect(result).toEqual(expectedDocuments);
- });
-
- it('should correctly handle a mix of absolute and relative paths', () => {
- const documents: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'Mixed paths: [relative](/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).',
- markdown: 'Mixed paths: [relative](/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).'
- }];
-
- const expectedDocuments: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'Mixed paths: [relative](https://example.com/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).',
- markdown: 'Mixed paths: [relative](https://example.com/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).'
- }];
-
- const result = replacePathsWithAbsolutePaths(documents);
- expect(result).toEqual(expectedDocuments);
- });
-
- });
-
- describe('replaceImgPathsWithAbsolutePaths', () => {
- it('should replace relative image paths with absolute paths', () => {
- const documents: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'Here is an image: ![alt text](/path/to/image.jpg).',
- markdown: 'Here is an image: ![alt text](/path/to/image.jpg).'
- }];
-
- const expectedDocuments: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'Here is an image: ![alt text](https://example.com/path/to/image.jpg).',
- markdown: 'Here is an image: ![alt text](https://example.com/path/to/image.jpg).'
- }];
-
- const result = replaceImgPathsWithAbsolutePaths(documents);
- expect(result).toEqual(expectedDocuments);
- });
-
- it('should not alter data:image URLs', () => {
- const documents: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'An image with a data URL: ![alt text](data:image/png;base64,ABC123==).',
- markdown: 'An image with a data URL: ![alt text](data:image/png;base4,ABC123==).'
- }];
-
- const result = replaceImgPathsWithAbsolutePaths(documents);
- expect(result).toEqual(documents); // Expect no change
- });
-
- it('should handle multiple images with a mix of data and relative URLs', () => {
- const documents: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'Multiple images: ![img1](/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](/img3.jpg).',
- markdown: 'Multiple images: ![img1](/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](/img3.jpg).'
- }];
-
- const expectedDocuments: Document[] = [{
- metadata: { sourceURL: 'https://example.com' },
- content: 'Multiple images: ![img1](https://example.com/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](https://example.com/img3.jpg).',
- markdown: 'Multiple images: ![img1](https://example.com/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](https://example.com/img3.jpg).'
- }];
-
- const result = replaceImgPathsWithAbsolutePaths(documents);
- expect(result).toEqual(expectedDocuments);
- });
- });
-});
\ No newline at end of file
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts
deleted file mode 100644
index 3d98fedfb..000000000
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts
+++ /dev/null
@@ -1,66 +0,0 @@
-import { Logger } from '../../../../lib/logger';
-import { isUrlBlocked } from '../blocklist';
-
-describe('isUrlBlocked', () => {
- it('should return true for blocked social media URLs', () => {
- const blockedUrls = [
- 'https://www.facebook.com',
- 'https://twitter.com/someuser',
- 'https://instagram.com/someuser',
- 'https://www.linkedin.com/in/someuser',
- 'https://snapchat.com/someuser',
- 'https://tiktok.com/@someuser',
- 'https://reddit.com/r/somesubreddit',
- 'https://flickr.com/photos/someuser',
- 'https://whatsapp.com/someuser',
- 'https://wechat.com/someuser',
- 'https://telegram.org/someuser',
- ];
-
- blockedUrls.forEach(url => {
- if (!isUrlBlocked(url)) {
- Logger.debug(`URL not blocked: ${url}`);
- }
- expect(isUrlBlocked(url)).toBe(true);
- });
- });
-
- it('should return false for URLs containing allowed keywords', () => {
- const allowedUrls = [
- 'https://www.facebook.com/privacy',
- 'https://twitter.com/terms',
- 'https://instagram.com/legal',
- 'https://www.linkedin.com/help',
- 'https://pinterest.com/about',
- 'https://snapchat.com/support',
- 'https://tiktok.com/contact',
- 'https://reddit.com/user-agreement',
- 'https://tumblr.com/policy',
- 'https://flickr.com/blog',
- 'https://whatsapp.com/press',
- 'https://wechat.com/careers',
- 'https://telegram.org/conditions',
- 'https://wix.com/careers',
- ];
-
- allowedUrls.forEach(url => {
- expect(isUrlBlocked(url)).toBe(false);
- });
- });
-
- it('should return false for non-blocked URLs', () => {
- const nonBlockedUrls = [
- 'https://www.example.com',
- 'https://www.somewebsite.org',
- 'https://subdomain.example.com',
- 'firecrawl.dev',
- 'amazon.com',
- 'wix.com',
- 'https://wix.com'
- ];
-
- nonBlockedUrls.forEach(url => {
- expect(isUrlBlocked(url)).toBe(false);
- });
- });
-});
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 3ed97d0b8..0a7248294 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -1,4 +1,4 @@
-import { Logger } from "../../../lib/logger";
+import { logger } from "../../../lib/logger";
const socialMediaBlocklist = [
'facebook.com',
@@ -68,7 +68,7 @@ export function isUrlBlocked(url: string): boolean {
return isBlocked;
} catch (e) {
// If an error occurs (e.g., invalid URL), return false
- Logger.error(`Error parsing the following URL: ${url}`);
+ logger.error(`Error parsing the following URL: ${url}`);
return false;
}
}
diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
deleted file mode 100644
index 393552cf6..000000000
--- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
+++ /dev/null
@@ -1,198 +0,0 @@
-export const urlSpecificParams = {
-
- "support.greenpay.me":{
- defaultScraper: "fire-engine",
- params: {
- wait_browser: "networkidle2",
- block_resources: false,
- wait: 2000,
-
- },
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
- "sec-fetch-site": "same-origin",
- "sec-fetch-mode": "cors",
- "sec-fetch-dest": "empty",
- referer: "https://www.google.com/",
- "accept-language": "en-US,en;q=0.9",
- "accept-encoding": "gzip, deflate, br",
- accept:
- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- },
- },
- "docs.pdw.co":{
- defaultScraper: "fire-engine",
- params: {
- wait_browser: "networkidle2",
- block_resources: false,
- wait: 3000,
- },
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
- "sec-fetch-site": "same-origin",
- "sec-fetch-mode": "cors",
- "sec-fetch-dest": "empty",
- referer: "https://www.google.com/",
- "accept-language": "en-US,en;q=0.9",
- "accept-encoding": "gzip, deflate, br",
- accept:
- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- },
- },
- "developers.notion.com":{
- defaultScraper: "fire-engine",
- params: {
- wait_browser: "networkidle2",
- block_resources: false,
- wait: 2000,
- },
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
- "sec-fetch-site": "same-origin",
- "sec-fetch-mode": "cors",
- "sec-fetch-dest": "empty",
- referer: "https://www.google.com/",
- "accept-language": "en-US,en;q=0.9",
- "accept-encoding": "gzip, deflate, br",
- accept:
- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- },
- },
- "docs2.hubitat.com":{
- defaultScraper: "fire-engine",
- params: {
- wait_browser: "networkidle2",
- block_resources: false,
- wait: 2000,
- },
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
- "sec-fetch-site": "same-origin",
- "sec-fetch-mode": "cors",
- "sec-fetch-dest": "empty",
- referer: "https://www.google.com/",
- "accept-language": "en-US,en;q=0.9",
- "accept-encoding": "gzip, deflate, br",
- accept:
- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- },
- },
- "scrapethissite.com":{
- defaultScraper: "fetch",
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
- "sec-fetch-site": "same-origin",
- "sec-fetch-mode": "cors",
- "sec-fetch-dest": "empty",
- referer: "https://www.google.com/",
- "accept-language": "en-US,en;q=0.9",
- "accept-encoding": "gzip, deflate, br",
- accept:
- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- },
- },
- "rsseau.fr":{
- defaultScraper: "fetch",
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
- "sec-fetch-site": "same-origin",
- "sec-fetch-mode": "cors",
- "sec-fetch-dest": "empty",
- referer: "https://www.google.com/",
- "accept-language": "en-US,en;q=0.9",
- "accept-encoding": "gzip, deflate, br",
- accept:
- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- },
- },
- "help.salesforce.com":{
- defaultScraper: "fire-engine",
- params: {
- wait_browser: "networkidle2",
- block_resources: false,
- wait: 2000,
- },
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
- "sec-fetch-site": "same-origin",
- "sec-fetch-mode": "cors",
- "sec-fetch-dest": "empty",
- referer: "https://www.google.com/",
- "accept-language": "en-US,en;q=0.9",
- "accept-encoding": "gzip, deflate, br",
- accept:
- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- },
- },
- "ir.veeva.com":{
- defaultScraper: "fire-engine",
- },
- "eonhealth.com":{
- defaultScraper: "fire-engine",
- params:{
- fireEngineOptions:{
- mobileProxy: true,
- method: "get",
- engine: "request",
- },
- },
- },
- "notion.com":{
- defaultScraper: "fire-engine",
- params: {
- wait_browser: "networkidle2",
- block_resources: false,
- wait: 2000,
- engine: "playwright",
- }
- },
- "developer.apple.com":{
- defaultScraper: "fire-engine",
- params:{
- engine: "playwright",
- wait: 2000,
- fireEngineOptions: {
- blockMedia: false,
- }
- },
- },
- "amazon.com":{
- defaultScraper: "fire-engine",
- params:{
- fireEngineOptions:{
- engine: "chrome-cdp",
- },
- },
- },
- "digikey.com":{
- defaultScraper: "fire-engine",
- params:{
- fireEngineOptions:{
- engine: "tlsclient",
- },
- },
- },
- "zoopla.co.uk":{
- defaultScraper: "fire-engine",
- params:{
- fireEngineOptions:{
- engine: "chrome-cdp",
- },
- },
- },
- "lorealparis.hu":{
- defaultScraper: "fire-engine",
- params:{
- fireEngineOptions:{
- engine: "tlsclient",
- },
- },
- }
-};
diff --git a/apps/api/src/scraper/WebScraper/utils/docxProcessor.ts b/apps/api/src/scraper/WebScraper/utils/docxProcessor.ts
deleted file mode 100644
index 8f6dc97cc..000000000
--- a/apps/api/src/scraper/WebScraper/utils/docxProcessor.ts
+++ /dev/null
@@ -1,79 +0,0 @@
-import axios from "axios";
-import fs from "fs";
-import { createWriteStream } from "node:fs";
-import path from "path";
-import os from "os";
-import mammoth from "mammoth";
-import { Logger } from "../../../lib/logger";
-
-export async function fetchAndProcessDocx(url: string): Promise<{ content: string; pageStatusCode: number; pageError: string }> {
- let tempFilePath = '';
- let pageStatusCode = 200;
- let pageError = '';
- let content = '';
-
- try {
- const downloadResult = await downloadDocx(url);
- tempFilePath = downloadResult.tempFilePath;
- pageStatusCode = downloadResult.pageStatusCode;
- pageError = downloadResult.pageError;
- content = await processDocxToText(tempFilePath);
- } catch (error) {
- Logger.error(`Failed to fetch and process DOCX: ${error.message}`);
- pageStatusCode = 500;
- pageError = error.message;
- content = '';
- } finally {
- if (tempFilePath) {
- fs.unlinkSync(tempFilePath); // Clean up the temporary file
- }
- }
-
- return { content, pageStatusCode, pageError };
-}
-
-async function downloadDocx(url: string): Promise<{ tempFilePath: string; pageStatusCode: number; pageError: string }> {
- try {
- const response = await axios({
- url,
- method: "GET",
- responseType: "stream",
- });
-
- const tempFilePath = path.join(os.tmpdir(), `tempDocx-${Date.now()}.docx`);
- const writer = createWriteStream(tempFilePath);
-
- response.data.pipe(writer);
-
- return new Promise((resolve, reject) => {
- writer.on("finish", () => resolve({ tempFilePath, pageStatusCode: response.status, pageError: response.statusText != "OK" ? response.statusText : undefined }));
- writer.on("error", () => {
- Logger.error('Failed to write DOCX file to disk');
- reject(new Error('Failed to write DOCX file to disk'));
- });
- });
- } catch (error) {
- Logger.error(`Failed to download DOCX: ${error.message}`);
- return { tempFilePath: "", pageStatusCode: 500, pageError: error.message };
- }
-}
-
-export async function processDocxToText(filePath: string): Promise<string> {
- try {
- const content = await extractTextFromDocx(filePath);
- return content;
- } catch (error) {
- Logger.error(`Failed to process DOCX to text: ${error.message}`);
- return "";
- }
-}
-
-async function extractTextFromDocx(filePath: string): Promise<string> {
- try {
- const result = await mammoth.extractRawText({ path: filePath });
- return result.value;
- } catch (error) {
- Logger.error(`Failed to extract text from DOCX: ${error.message}`);
- return "";
- }
-}
diff --git a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts b/apps/api/src/scraper/WebScraper/utils/excludeTags.ts
deleted file mode 100644
index 71e1637bb..000000000
--- a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-export const excludeNonMainTags = [
- "header",
- "footer",
- "nav",
- "aside",
- ".top",
- ".navbar",
- ".footer",
- ".bottom",
- "#footer",
- ".sidebar",
- ".side",
- ".aside",
- "#sidebar",
- ".modal",
- ".popup",
- "#modal",
- ".overlay",
- ".ad",
- ".ads",
- ".advert",
- "#ad",
- ".lang-selector",
- ".language",
- "#language-selector",
- ".social",
- ".social-media",
- ".social-links",
- "#social",
- ".menu",
- ".navigation",
- "#nav",
- ".breadcrumbs",
- "#breadcrumbs",
- "#search-form",
- ".search",
- "#search",
- ".share",
- "#share",
- ".cookie",
- "#cookie"
-];
diff --git a/apps/api/src/scraper/WebScraper/utils/imageDescription.ts b/apps/api/src/scraper/WebScraper/utils/imageDescription.ts
deleted file mode 100644
index 318c23404..000000000
--- a/apps/api/src/scraper/WebScraper/utils/imageDescription.ts
+++ /dev/null
@@ -1,89 +0,0 @@
-import Anthropic from '@anthropic-ai/sdk';
-import axios from 'axios';
-import { Logger } from '../../../lib/logger';
-
-export async function getImageDescription(
- imageUrl: string,
- backText: string,
- frontText: string,
- model: string = "gpt-4-turbo"
-): Promise<string> {
- try {
- const prompt = "What's in the image? You need to answer with the content for the alt tag of the image. To help you with the context, the image is in the following text: " +
- backText +
- " and the following text: " +
- frontText +
- ". Be super concise."
-
- switch (model) {
- case 'claude-3-opus': {
- if (!process.env.ANTHROPIC_API_KEY) {
- throw new Error("No Anthropic API key provided");
- }
- const imageRequest = await axios.get(imageUrl, { responseType: 'arraybuffer' });
- const imageMediaType = 'image/png';
- const imageData = Buffer.from(imageRequest.data, 'binary').toString('base64');
-
- const anthropic = new Anthropic();
- const response = await anthropic.messages.create({
- model: "claude-3-opus-20240229",
- max_tokens: 1024,
- messages: [
- {
- role: "user",
- content: [
- {
- type: "image",
- source: {
- type: "base64",
- media_type: imageMediaType,
- data: imageData,
- },
- },
- {
- type: "text",
- text: prompt
- }
- ],
- }
- ]
- });
-
- return response[0].content.text;
- }
- default: {
- if (!process.env.OPENAI_API_KEY) {
- throw new Error("No OpenAI API key provided");
- }
-
- const { OpenAI } = require("openai");
- const openai = new OpenAI();
-
- const response = await openai.chat.completions.create({
- model: "gpt-4-turbo",
- messages: [
- {
- role: "user",
- content: [
- {
- type: "text",
- text: prompt,
- },
- {
- type: "image_url",
- image_url: {
- url: imageUrl,
- },
- },
- ],
- },
- ],
- });
- return response.choices[0].message.content;
- }
- }
- } catch (error) {
- Logger.error(`Error generating image alt text: ${error}`);
- return "";
- }
-}
diff --git a/apps/api/src/scraper/WebScraper/utils/metadata.ts b/apps/api/src/scraper/WebScraper/utils/metadata.ts
deleted file mode 100644
index 531dc17cb..000000000
--- a/apps/api/src/scraper/WebScraper/utils/metadata.ts
+++ /dev/null
@@ -1,185 +0,0 @@
-import { CheerioAPI } from "cheerio";
-import { Logger } from "../../../lib/logger";
-
-interface Metadata {
- title?: string;
- description?: string;
- language?: string;
- keywords?: string;
- robots?: string;
- ogTitle?: string;
- ogDescription?: string;
- ogUrl?: string;
- ogImage?: string;
- ogAudio?: string;
- ogDeterminer?: string;
- ogLocale?: string;
- ogLocaleAlternate?: string[];
- ogSiteName?: string;
- ogVideo?: string;
- dctermsCreated?: string;
- dcDateCreated?: string;
- dcDate?: string;
- dctermsType?: string;
- dcType?: string;
- dctermsAudience?: string;
- dctermsSubject?: string;
- dcSubject?: string;
- dcDescription?: string;
- dctermsKeywords?: string;
- modifiedTime?: string;
- publishedTime?: string;
- articleTag?: string;
- articleSection?: string;
- sourceURL?: string;
- pageStatusCode?: number;
- pageError?: string;
- [key: string]: string | string[] | number | undefined;
-}
-
-export function extractMetadata(soup: CheerioAPI, url: string): Metadata {
- let title: string | null = null;
- let description: string | null = null;
- let language: string | null = null;
- let keywords: string | null = null;
- let robots: string | null = null;
- let ogTitle: string | null = null;
- let ogDescription: string | null = null;
- let ogUrl: string | null = null;
- let ogImage: string | null = null;
- let ogAudio: string | null = null;
- let ogDeterminer: string | null = null;
- let ogLocale: string | null = null;
- let ogLocaleAlternate: string[] | null = null;
- let ogSiteName: string | null = null;
- let ogVideo: string | null = null;
- let dctermsCreated: string | null = null;
- let dcDateCreated: string | null = null;
- let dcDate: string | null = null;
- let dctermsType: string | null = null;
- let dcType: string | null = null;
- let dctermsAudience: string | null = null;
- let dctermsSubject: string | null = null;
- let dcSubject: string | null = null;
- let dcDescription: string | null = null;
- let dctermsKeywords: string | null = null;
- let modifiedTime: string | null = null;
- let publishedTime: string | null = null;
- let articleTag: string | null = null;
- let articleSection: string | null = null;
- let sourceURL: string | null = null;
- let pageStatusCode: number | null = null;
- let pageError: string | null = null;
-
- const customMetadata: Record<string, string | string[]> = {};
-
- try {
- // TODO: remove this as it is redundant with the below implementation
- title = soup("title").text() || null;
- description = soup('meta[name="description"]').attr("content") || null;
-
- language = soup("html").attr("lang") || null;
-
- keywords = soup('meta[name="keywords"]').attr("content") || null;
- robots = soup('meta[name="robots"]').attr("content") || null;
- ogTitle = soup('meta[property="og:title"]').attr("content") || null;
- ogDescription =
- soup('meta[property="og:description"]').attr("content") || null;
- ogUrl = soup('meta[property="og:url"]').attr("content") || null;
- ogImage = soup('meta[property="og:image"]').attr("content") || null;
- ogAudio = soup('meta[property="og:audio"]').attr("content") || null;
- ogDeterminer =
- soup('meta[property="og:determiner"]').attr("content") || null;
- ogLocale = soup('meta[property="og:locale"]').attr("content") || null;
- ogLocaleAlternate =
- soup('meta[property="og:locale:alternate"]')
- .map((i, el) => soup(el).attr("content"))
- .get() || null;
- ogSiteName = soup('meta[property="og:site_name"]').attr("content") || null;
- ogVideo = soup('meta[property="og:video"]').attr("content") || null;
- articleSection =
- soup('meta[name="article:section"]').attr("content") || null;
- articleTag = soup('meta[name="article:tag"]').attr("content") || null;
- publishedTime =
- soup('meta[property="article:published_time"]').attr("content") || null;
- modifiedTime =
- soup('meta[property="article:modified_time"]').attr("content") || null;
- dctermsKeywords =
- soup('meta[name="dcterms.keywords"]').attr("content") || null;
- dcDescription = soup('meta[name="dc.description"]').attr("content") || null;
- dcSubject = soup('meta[name="dc.subject"]').attr("content") || null;
- dctermsSubject =
- soup('meta[name="dcterms.subject"]').attr("content") || null;
- dctermsAudience =
- soup('meta[name="dcterms.audience"]').attr("content") || null;
- dcType = soup('meta[name="dc.type"]').attr("content") || null;
- dctermsType = soup('meta[name="dcterms.type"]').attr("content") || null;
- dcDate = soup('meta[name="dc.date"]').attr("content") || null;
- dcDateCreated =
- soup('meta[name="dc.date.created"]').attr("content") || null;
- dctermsCreated =
- soup('meta[name="dcterms.created"]').attr("content") || null;
-
- try {
- // Extract all meta tags for custom metadata
- soup("meta").each((i, elem) => {
- try {
- const name = soup(elem).attr("name") || soup(elem).attr("property");
- const content = soup(elem).attr("content");
-
- if (name && content) {
- if (customMetadata[name] === undefined) {
- customMetadata[name] = content;
- } else if (Array.isArray(customMetadata[name])) {
- (customMetadata[name] as string[]).push(content);
- } else {
- customMetadata[name] = [customMetadata[name] as string, content];
- }
- }
- } catch (error) {
- Logger.error(`Error extracting custom metadata (in): ${error}`);
- }
- });
- } catch (error) {
- Logger.error(`Error extracting custom metadata: ${error}`);
- }
- } catch (error) {
- Logger.error(`Error extracting metadata: ${error}`);
- }
-
- return {
- ...(title ? { title } : {}),
- ...(description ? { description } : {}),
- ...(language ? { language } : {}),
- ...(keywords ? { keywords } : {}),
- ...(robots ? { robots } : {}),
- ...(ogTitle ? { ogTitle } : {}),
- ...(ogDescription ? { ogDescription } : {}),
- ...(ogUrl ? { ogUrl } : {}),
- ...(ogImage ? { ogImage } : {}),
- ...(ogAudio ? { ogAudio } : {}),
- ...(ogDeterminer ? { ogDeterminer } : {}),
- ...(ogLocale ? { ogLocale } : {}),
- ...(ogLocaleAlternate ? { ogLocaleAlternate } : {}),
- ...(ogSiteName ? { ogSiteName } : {}),
- ...(ogVideo ? { ogVideo } : {}),
- ...(dctermsCreated ? { dctermsCreated } : {}),
- ...(dcDateCreated ? { dcDateCreated } : {}),
- ...(dcDate ? { dcDate } : {}),
- ...(dctermsType ? { dctermsType } : {}),
- ...(dcType ? { dcType } : {}),
- ...(dctermsAudience ? { dctermsAudience } : {}),
- ...(dctermsSubject ? { dctermsSubject } : {}),
- ...(dcSubject ? { dcSubject } : {}),
- ...(dcDescription ? { dcDescription } : {}),
- ...(dctermsKeywords ? { dctermsKeywords } : {}),
- ...(modifiedTime ? { modifiedTime } : {}),
- ...(publishedTime ? { publishedTime } : {}),
- ...(articleTag ? { articleTag } : {}),
- ...(articleSection ? { articleSection } : {}),
- ...(sourceURL ? { sourceURL } : {}),
- ...(pageStatusCode ? { pageStatusCode } : {}),
- ...(pageError ? { pageError } : {}),
- ...customMetadata,
- };
-}
diff --git a/apps/api/src/scraper/WebScraper/utils/parseTable.ts b/apps/api/src/scraper/WebScraper/utils/parseTable.ts
deleted file mode 100644
index 9855650f6..000000000
--- a/apps/api/src/scraper/WebScraper/utils/parseTable.ts
+++ /dev/null
@@ -1,74 +0,0 @@
-import cheerio, { CheerioAPI } from "cheerio";
-
-interface Replacement {
- start: number;
- end: number;
- markdownTable: string;
-}
-
-export const parseTablesToMarkdown = async (html: string): Promise<string> => {
- const soup: CheerioAPI = cheerio.load(html, {
- xmlMode: true,
- withStartIndices: true,
- withEndIndices: true
- });
- let tables = soup("table");
- let replacements: Replacement[] = [];
-
- if (tables.length) {
- tables.each((_, tableElement) => {
- const start: number = tableElement.startIndex;
- const end: number = tableElement.endIndex + 1; // Include the closing tag properly
- let markdownTable: string = convertTableElementToMarkdown(cheerio.load(tableElement));
- const isTableEmpty: boolean = markdownTable.replace(/[|\- \n]/g, '').length === 0;
- if (isTableEmpty) {
- markdownTable = '';
- }
- replacements.push({ start, end, markdownTable });
- });
- }
-
- replacements.sort((a, b) => b.start - a.start);
-
- let modifiedHtml: string = html;
- replacements.forEach(({ start, end, markdownTable }) => {
- modifiedHtml = modifiedHtml.slice(0, start) + `<div>${markdownTable}</div>` + modifiedHtml.slice(end);
- });
-
- return modifiedHtml.trim();
-};
-
-export const convertTableElementToMarkdown = (tableSoup: CheerioAPI): string => {
- let rows: string[] = [];
- let headerRowFound: boolean = false;
- tableSoup("tr").each((i, tr) => {
- const cells: string = tableSoup(tr).find("th, td").map((_, cell) => {
- let cellText: string = tableSoup(cell).text().trim();
- if (tableSoup(cell).is("th") && !headerRowFound) {
- headerRowFound = true;
- }
- return ` ${cellText} |`;
- }).get().join("");
- if (cells) {
- rows.push(`|${cells}`);
- }
- if (headerRowFound && i === 0) { // Header row
- rows.push(createMarkdownDividerRow(tableSoup(tr).find("th, td").length));
- }
- });
-
- return rows.join('\n').trim();
-};
-
-export function convertTableRowElementToMarkdown(rowSoup: CheerioAPI, rowNumber: number): string {
- const cells: string = rowSoup("td, th").map((_, cell) => {
- let cellText: string = rowSoup(cell).text().trim();
- return ` ${cellText} |`;
- }).get().join("");
-
- return `|${cells}`;
-};
-
-export function createMarkdownDividerRow(cellCount: number): string {
- return '| ' + Array(cellCount).fill('---').join(' | ') + ' |';
-}
\ No newline at end of file
diff --git a/apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts b/apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts
deleted file mode 100644
index 4d75df338..000000000
--- a/apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts
+++ /dev/null
@@ -1,140 +0,0 @@
-import axios, { AxiosResponse } from "axios";
-import fs from "fs/promises";
-import { createReadStream, createWriteStream } from "node:fs";
-import FormData from "form-data";
-import dotenv from "dotenv";
-import pdf from "pdf-parse";
-import path from "path";
-import os from "os";
-import { axiosTimeout } from "../../../lib/timeout";
-import { Logger } from "../../../lib/logger";
-
-dotenv.config();
-
-export async function fetchAndProcessPdf(url: string, parsePDF: boolean): Promise<{ content: string, pageStatusCode?: number, pageError?: string }> {
- try {
- const { tempFilePath, pageStatusCode, pageError } = await downloadPdf(url);
- const content = await processPdfToText(tempFilePath, parsePDF);
- await fs.unlink(tempFilePath); // Clean up the temporary file
- return { content, pageStatusCode, pageError };
- } catch (error) {
- Logger.error(`Failed to fetch and process PDF: ${error.message}`);
- return { content: "", pageStatusCode: 500, pageError: error.message };
- }
-}
-
-async function downloadPdf(url: string): Promise<{ tempFilePath: string, pageStatusCode?: number, pageError?: string }> {
- const response = await axios({
- url,
- method: "GET",
- responseType: "stream",
- });
-
- const tempFilePath = path.join(os.tmpdir(), `tempPdf-${Date.now()}.pdf`);
- const writer = createWriteStream(tempFilePath);
-
- response.data.pipe(writer);
-
- return new Promise((resolve, reject) => {
- writer.on("finish", () => resolve({ tempFilePath, pageStatusCode: response.status, pageError: response.statusText != "OK" ? response.statusText : undefined }));
- writer.on("error", reject);
- });
-}
-
-export async function processPdfToText(filePath: string, parsePDF: boolean): Promise<string> {
- let content = "";
-
- if (process.env.LLAMAPARSE_API_KEY && parsePDF) {
- Logger.debug("Processing pdf document w/ LlamaIndex");
- const apiKey = process.env.LLAMAPARSE_API_KEY;
- const headers = {
- Authorization: `Bearer ${apiKey}`,
- };
- const base_url = "https://api.cloud.llamaindex.ai/api/parsing";
- const fileType2 = "application/pdf";
-
- try {
- const formData = new FormData();
- formData.append("file", createReadStream(filePath), {
- filename: filePath,
- contentType: fileType2,
- });
-
- const uploadUrl = `${base_url}/upload`;
- const uploadResponse = await axios.post(uploadUrl, formData, {
- headers: {
- ...headers,
- ...formData.getHeaders(),
- },
- });
-
- const jobId = uploadResponse.data.id;
- const resultType = "text";
- const resultUrl = `${base_url}/job/${jobId}/result/${resultType}`;
-
- let resultResponse: AxiosResponse;
- let attempt = 0;
- const maxAttempts = 10; // Maximum number of attempts
- let resultAvailable = false;
- while (attempt < maxAttempts && !resultAvailable) {
- try {
- resultResponse = await axios.get(resultUrl, { headers, timeout: (axiosTimeout * 2) });
- if (resultResponse.status === 200) {
- resultAvailable = true; // Exit condition met
- } else {
- // If the status code is not 200, increment the attempt counter and wait
- attempt++;
- await new Promise((resolve) => setTimeout(resolve, 500)); // Wait for 0.5 seconds
- }
- } catch (error) {
- Logger.debug("Error fetching result w/ LlamaIndex");
- attempt++;
- if (attempt >= maxAttempts) {
- Logger.error("Max attempts reached, unable to fetch result.");
- break; // Exit the loop if max attempts are reached
- }
- await new Promise((resolve) => setTimeout(resolve, 500)); // Wait for 0.5 seconds before retrying
- // You may want to handle specific errors differently
- }
- }
-
- if (!resultAvailable) {
- try {
- content = await processPdf(filePath);
- } catch (error) {
- Logger.error(`Failed to process PDF: ${error}`);
- content = "";
- }
- }
- content = resultResponse.data[resultType];
- } catch (error) {
- Logger.debug("Error processing pdf document w/ LlamaIndex(2)");
- content = await processPdf(filePath);
- }
- } else if (parsePDF) {
- try {
- content = await processPdf(filePath);
- } catch (error) {
- Logger.error(`Failed to process PDF: ${error}`);
- content = "";
- }
- } else {
- try {
- content = await fs.readFile(filePath, "utf-8");
- } catch (error) {
- Logger.error(`Failed to read PDF file: ${error}`);
- content = "";
- }
- }
- return content;
-}
-
-async function processPdf(file: string) {
- try {
- const fileContent = await fs.readFile(file);
- const data = await pdf(fileContent);
- return data.text;
- } catch (error) {
- throw error;
- }
-}
\ No newline at end of file
diff --git a/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts b/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
deleted file mode 100644
index 1264f89a1..000000000
--- a/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
+++ /dev/null
@@ -1,82 +0,0 @@
-import { AnyNode, Cheerio, load } from "cheerio";
-import { PageOptions } from "../../../lib/entities";
-import { excludeNonMainTags } from "./excludeTags";
-
-export const removeUnwantedElements = (
- html: string,
- pageOptions: PageOptions,
-) => {
- let soup = load(html);
-
- if (
- pageOptions.onlyIncludeTags &&
- pageOptions.onlyIncludeTags.length > 0 &&
- pageOptions.onlyIncludeTags[0] !== ""
- ) {
- if (typeof pageOptions.onlyIncludeTags === "string") {
- pageOptions.onlyIncludeTags = [pageOptions.onlyIncludeTags];
- }
- if (pageOptions.onlyIncludeTags.length !== 0) {
- // Create a new root element to hold the tags to keep
- const newRoot = load("<div></div>")("div");
- pageOptions.onlyIncludeTags.forEach((tag) => {
- soup(tag).each((index, element) => {
- newRoot.append(soup(element).clone());
- });
- });
-
- soup = load(newRoot.html());
- }
- }
-
- soup("script, style, noscript, meta, head").remove();
-
- if (
- pageOptions.removeTags &&
- pageOptions.removeTags.length > 0 &&
- pageOptions.removeTags[0] !== ""
- ) {
- if (typeof pageOptions.removeTags === "string") {
- pageOptions.removeTags = [pageOptions.removeTags];
- }
-
- if (Array.isArray(pageOptions.removeTags)) {
- pageOptions.removeTags.forEach((tag) => {
- let elementsToRemove: Cheerio<AnyNode>;
- if (tag.startsWith("*") && tag.endsWith("*")) {
- let classMatch = false;
-
- const regexPattern = new RegExp(tag.slice(1, -1), "i");
- elementsToRemove = soup("*").filter((i, element) => {
- if (element.type === "tag") {
- const attributes = element.attribs;
- const tagNameMatches = regexPattern.test(element.name);
- const attributesMatch = Object.keys(attributes).some((attr) =>
- regexPattern.test(`${attr}="${attributes[attr]}"`),
- );
- if (tag.startsWith("*.")) {
- classMatch = Object.keys(attributes).some((attr) =>
- regexPattern.test(`class="${attributes[attr]}"`),
- );
- }
- return tagNameMatches || attributesMatch || classMatch;
- }
- return false;
- });
- } else {
- elementsToRemove = soup(tag);
- }
- elementsToRemove.remove();
- });
- }
- }
-
- if (pageOptions.onlyMainContent) {
- excludeNonMainTags.forEach((tag) => {
- const elementsToRemove = soup(tag);
- elementsToRemove.remove();
- });
- }
- const cleanedHtml = soup.html();
- return cleanedHtml;
-};
diff --git a/apps/api/src/scraper/WebScraper/utils/replacePaths.ts b/apps/api/src/scraper/WebScraper/utils/replacePaths.ts
deleted file mode 100644
index f84db63f6..000000000
--- a/apps/api/src/scraper/WebScraper/utils/replacePaths.ts
+++ /dev/null
@@ -1,85 +0,0 @@
-import { Logger } from "../../../lib/logger";
-import { Document } from "../../../lib/entities";
-
-export const replacePathsWithAbsolutePaths = (documents: Document[]): Document[] => {
- try {
- documents.forEach((document) => {
- const baseUrl = new URL(document.metadata.sourceURL).origin;
- const paths =
- document.content.match(
- /!?\[.*?\]\(.*?\)|href=".+?"/g
- ) || [];
-
- paths.forEach((path: string) => {
- try {
- const isImage = path.startsWith("!");
- let matchedUrl = path.match(/\((.*?)\)/) || path.match(/href="([^"]+)"/);
- let url = matchedUrl[1];
-
- if (!url.startsWith("data:") && !url.startsWith("http")) {
- if (url.startsWith("/")) {
- url = url.substring(1);
- }
- url = new URL(url, baseUrl).toString();
- }
-
- const markdownLinkOrImageText = path.match(/(!?\[.*?\])/)[0];
- // Image is handled afterwards
- if (!isImage) {
- document.content = document.content.replace(
- path,
- `${markdownLinkOrImageText}(${url})`
- );
- }
- } catch (error) {
-
- }
- });
- document.markdown = document.content;
- });
-
- return documents;
- } catch (error) {
- Logger.debug(`Error replacing paths with absolute paths: ${error}`);
- return documents;
- }
-};
-
-export const replaceImgPathsWithAbsolutePaths = (documents: Document[]): Document[] => {
- try {
- documents.forEach((document) => {
- const baseUrl = new URL(document.metadata.sourceURL).origin;
- const images =
- document.content.match(
- /!\[.*?\]\(.*?\)/g
- ) || [];
-
- images.forEach((image: string) => {
- let imageUrl = image.match(/\((.*?)\)/)[1];
- let altText = image.match(/\[(.*?)\]/)[1];
-
- if (!imageUrl.startsWith("data:image")) {
- if (!imageUrl.startsWith("http")) {
- if (imageUrl.startsWith("/")) {
- imageUrl = imageUrl.substring(1);
- imageUrl = new URL(imageUrl, baseUrl).toString();
- } else {
- imageUrl = new URL(imageUrl, document.metadata.sourceURL).toString();
- }
- }
- }
-
- document.content = document.content.replace(
- image,
- `![${altText}](${imageUrl})`
- );
- });
- document.markdown = document.content;
- });
-
- return documents;
- } catch (error) {
- Logger.error(`Error replacing img paths with absolute paths: ${error}`);
- return documents;
- }
-};
\ No newline at end of file
diff --git a/apps/api/src/scraper/WebScraper/utils/utils.ts b/apps/api/src/scraper/WebScraper/utils/utils.ts
deleted file mode 100644
index 849d209de..000000000
--- a/apps/api/src/scraper/WebScraper/utils/utils.ts
+++ /dev/null
@@ -1,59 +0,0 @@
-import axios from "axios";
-import * as cheerio from "cheerio";
-import { Logger } from "../../../lib/logger";
-
-
-export async function attemptScrapWithRequests(
- urlToScrap: string
-): Promise<string | null> {
- try {
- const response = await axios.get(urlToScrap, { timeout: 15000 });
-
- if (!response.data) {
- Logger.debug("Failed normal requests as well");
- return null;
- }
-
- return response.data;
- } catch (error) {
- Logger.debug(`Error in attemptScrapWithRequests: ${error}`);
- return null;
- }
-}
-
-export function sanitizeText(text: string): string {
- return text.replace("\u0000", "");
-}
-
-export function extractLinks(html: string, baseUrl: string): string[] {
- const $ = cheerio.load(html);
- const links: string[] = [];
-
- $('a').each((_, element) => {
- const href = $(element).attr('href');
- if (href) {
- try {
- if (href.startsWith('http://') || href.startsWith('https://')) {
- // Absolute URL, add as is
- links.push(href);
- } else if (href.startsWith('/')) {
- // Relative URL starting with '/', append to base URL
- links.push(new URL(href, baseUrl).href);
- } else if (!href.startsWith('#') && !href.startsWith('mailto:')) {
- // Relative URL not starting with '/', append to base URL
- links.push(new URL(href, baseUrl).href);
- } else if (href.startsWith('mailto:')) {
- // mailto: links, add as is
- links.push(href);
- }
- // Fragment-only links (#) are ignored
- } catch (error) {
- // Log the error and continue
- console.error(`Failed to construct URL for href: ${href} with base: ${baseUrl}`, error);
- }
- }
- });
-
- // Remove duplicates and return
- return [...new Set(links)];
-}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/README.md b/apps/api/src/scraper/scrapeURL/README.md
new file mode 100644
index 000000000..462ab4f31
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/README.md
@@ -0,0 +1,25 @@
+# `scrapeURL`
+New URL scraper for Firecrawl
+
+## Signal flow
+```mermaid
+flowchart TD;
+ scrapeURL-.->buildFallbackList;
+ buildFallbackList-.->scrapeURLWithEngine;
+ scrapeURLWithEngine-.->parseMarkdown;
+ parseMarkdown-.->wasScrapeSuccessful{{Was scrape successful?}};
+ wasScrapeSuccessful-."No".->areEnginesLeft{{Are there engines left to try?}};
+ areEnginesLeft-."Yes, try next engine".->scrapeURLWithEngine;
+ areEnginesLeft-."No".->NoEnginesLeftError[/NoEnginesLeftError/]
+ wasScrapeSuccessful-."Yes".->asd;
+```
+
+## Differences from `WebScraperDataProvider`
+ - The job of `WebScraperDataProvider.validateInitialUrl` has been delegated to the zod layer above `scrapeUrl`.
+ - `WebScraperDataProvider.mode` has no equivalent, only `scrape_url` is supported.
+ - You may no longer specify multiple URLs.
+ - Built on `v1` definitons, instead of `v0`.
+ - PDFs are now converted straight to markdown using LlamaParse, instead of converting to just plaintext.
+ - DOCXs are now converted straight to HTML (and then later to markdown) using mammoth, instead of converting to just plaintext.
+ - Using new JSON Schema OpenAI API -- schema fails with LLM Extract will be basically non-existant.
+
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
new file mode 100644
index 000000000..f8196ccdb
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
@@ -0,0 +1,15 @@
+import { Meta } from "../..";
+import { EngineScrapeResult } from "..";
+import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";
+import mammoth from "mammoth";
+
+export async function scrapeDOCX(meta: Meta): Promise<EngineScrapeResult> {
+ const { response, tempFilePath } = await downloadFile(meta.id, meta.url);
+
+ return {
+ url: response.url,
+ statusCode: response.status,
+
+ html: (await mammoth.convertToHtml({ path: tempFilePath })).value,
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
new file mode 100644
index 000000000..2c809901f
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@ -0,0 +1,28 @@
+import { EngineScrapeResult } from "..";
+import { Meta } from "../..";
+import { TimeoutError } from "../../error";
+import { specialtyScrapeCheck } from "../utils/specialtyHandler";
+
+export async function scrapeURLWithFetch(meta: Meta): Promise<EngineScrapeResult> {
+ const timeout = 20000;
+
+ const response = await Promise.race([
+ fetch(meta.url, {
+ redirect: "follow",
+ headers: meta.options.headers,
+ }),
+ (async () => {
+ await new Promise((resolve) => setTimeout(() => resolve(null), timeout));
+ throw new TimeoutError("Fetch was unable to scrape the page before timing out", { cause: { timeout } });
+ })()
+ ]);
+
+ specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFetch/specialtyScrapeCheck" }), Object.fromEntries(response.headers as any));
+
+ return {
+ url: response.url,
+ html: await response.text(),
+ statusCode: response.status,
+ // TODO: error?
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
new file mode 100644
index 000000000..99a19da2f
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -0,0 +1,107 @@
+import { Logger } from "winston";
+import * as Sentry from "@sentry/node";
+import { z } from "zod";
+
+import { robustFetch } from "../../lib/fetch";
+import { EngineError } from "../../error";
+
+const successSchema = z.object({
+ jobId: z.string(),
+ state: z.literal("completed"),
+ processing: z.literal(false),
+
+ // timeTaken: z.number(),
+ content: z.string(),
+ url: z.string().optional(),
+
+ pageStatusCode: z.number(),
+ pageError: z.string().optional(),
+
+ // TODO: this needs to be non-optional, might need fixes on f-e side to ensure reliability
+ responseHeaders: z.record(z.string(), z.string()).optional(),
+
+ // timeTakenCookie: z.number().optional(),
+ // timeTakenRequest: z.number().optional(),
+
+ // legacy: playwright only
+ screenshot: z.string().optional(),
+
+ // new: actions
+ screenshots: z.string().array().optional(),
+ actionContent: z.object({
+ url: z.string(),
+ html: z.string(),
+ }).array().optional(),
+})
+
+export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;
+
+const processingSchema = z.object({
+ jobId: z.string(),
+ state: z.enum(["delayed", "active", "waiting", "waiting-children", "unknown"]),
+ processing: z.boolean(),
+});
+
+const failedSchema = z.object({
+ jobId: z.string(),
+ state: z.literal("failed"),
+ processing: z.literal(false),
+ error: z.string(),
+});
+
+export class StillProcessingError extends Error {
+ constructor(jobId: string) {
+ super("Job is still under processing", { cause: { jobId } })
+ }
+}
+
+export async function fireEngineCheckStatus(logger: Logger, jobId: string): Promise<FireEngineCheckStatusSuccess> {
+ const fireEngineURL = process.env.FIRE_ENGINE_BETA_URL!;
+
+ const status = await Sentry.startSpan({
+ name: "fire-engine: Check status",
+ attributes: {
+ jobId,
+ }
+ }, async span => {
+ return await robustFetch(
+ {
+ url: `${fireEngineURL}/scrape/${jobId}`,
+ method: "GET",
+ logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),
+ headers: {
+ ...(Sentry.isInitialized() ? ({
+ "sentry-trace": Sentry.spanToTraceHeader(span),
+ "baggage": Sentry.spanToBaggageHeader(span),
+ }) : {}),
+ },
+ }
+ )
+ });
+
+ const successParse = successSchema.safeParse(status);
+ const processingParse = processingSchema.safeParse(status);
+ const failedParse = failedSchema.safeParse(status);
+
+ if (successParse.success) {
+ logger.debug("Scrape succeeded!", { jobId });
+ return successParse.data;
+ } else if (processingParse.success) {
+ logger.debug("Scrape is still processing", { jobId });
+ throw new StillProcessingError(jobId);
+ } else if (failedParse.success) {
+ logger.debug("Scrape job failed", { status, jobId });
+ throw new EngineError("Scrape job failed", {
+ cause: {
+ status, jobId
+ }
+ });
+ } else {
+ logger.debug("Check status returned response not matched by any schema", { status, jobId });
+ throw new Error("Check status returned response not matched by any schema", {
+ cause: {
+ status, jobId
+ }
+ });
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts
new file mode 100644
index 000000000..ed07be88e
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts
@@ -0,0 +1,33 @@
+import { Logger } from "winston";
+import * as Sentry from "@sentry/node";
+
+import { robustFetch } from "../../lib/fetch";
+
+export async function fireEngineDelete(logger: Logger, jobId: string) {
+ const fireEngineURL = process.env.FIRE_ENGINE_BETA_URL!;
+
+ await Sentry.startSpan({
+ name: "fire-engine: Delete scrape",
+ attributes: {
+ jobId,
+ }
+ }, async span => {
+ await robustFetch(
+ {
+ url: `${fireEngineURL}/scrape/${jobId}`,
+ method: "DELETE",
+ headers: {
+ ...(Sentry.isInitialized() ? ({
+ "sentry-trace": Sentry.spanToTraceHeader(span),
+ "baggage": Sentry.spanToBaggageHeader(span),
+ }) : {}),
+ },
+ ignoreResponse: true,
+ ignoreFailure: true,
+ logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),
+ }
+ )
+ });
+
+ // We do not care whether this fails or not.
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
new file mode 100644
index 000000000..db2b021f0
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -0,0 +1,198 @@
+import { Logger } from "winston";
+import { Meta } from "../..";
+import { fireEngineScrape, FireEngineScrapeRequestChromeCDP, FireEngineScrapeRequestCommon, FireEngineScrapeRequestPlaywright, FireEngineScrapeRequestTLSClient } from "./scrape";
+import { EngineScrapeResult } from "..";
+import { fireEngineCheckStatus, FireEngineCheckStatusSuccess, StillProcessingError } from "./checkStatus";
+import { EngineError, TimeoutError } from "../../error";
+import * as Sentry from "@sentry/node";
+import { Action } from "../../../../lib/entities";
+import { specialtyScrapeCheck } from "../utils/specialtyHandler";
+
+const defaultTimeout = 20000;
+
+// This function does not take `Meta` on purpose. It may not access any
+// meta values to construct the request -- that must be done by the
+// `scrapeURLWithFireEngine*` functions.
+async function performFireEngineScrape<Engine extends FireEngineScrapeRequestChromeCDP | FireEngineScrapeRequestPlaywright | FireEngineScrapeRequestTLSClient>(
+ logger: Logger,
+ request: FireEngineScrapeRequestCommon & Engine,
+ timeout = defaultTimeout,
+): Promise<FireEngineCheckStatusSuccess> {
+ const scrape = await fireEngineScrape(logger.child({ method: "fireEngineScrape" }), request);
+
+ const startTime = Date.now();
+ const errorLimit = 3;
+ let errors: any[] = [];
+ let status: FireEngineCheckStatusSuccess | undefined = undefined;
+
+ while (status === undefined) {
+ if (errors.length >= errorLimit) {
+ logger.error("Error limit hit.", { errors });
+ throw new Error("Error limit hit. See e.cause.errors for errors.", { cause: { errors } });
+ }
+
+ if (Date.now() - startTime > timeout) {
+ logger.info("Fire-engine was unable to scrape the page before timing out.", { errors, timeout });
+ throw new TimeoutError("Fire-engine was unable to scrape the page before timing out", { cause: { errors, timeout } });
+ }
+
+ try {
+ status = await fireEngineCheckStatus(logger.child({ method: "fireEngineCheckStatus" }), scrape.jobId)
+ } catch (error) {
+ if (error instanceof StillProcessingError) {
+ logger.debug("Scrape is still processing...");
+ } else if (error instanceof EngineError) {
+ logger.debug("Fire-engine scrape job failed.", { error, jobId: scrape.jobId });
+ throw error;
+ } else {
+ Sentry.captureException(error);
+ errors.push(error);
+ logger.debug(`An unexpeceted error occurred while calling checkStatus. Error counter is now at ${errors.length}.`, { error, jobId: scrape.jobId });
+ }
+ }
+
+ await new Promise((resolve) => setTimeout(resolve, 250));
+ }
+
+ return status;
+}
+
+export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise<EngineScrapeResult> {
+ const actions: Action[] = [
+ // Transform waitFor option into an action (unsupported by chrome-cdp)
+ ...(meta.options.waitFor !== 0 ? [{
+ type: "wait" as const,
+ milliseconds: meta.options.waitFor,
+ }] : []),
+
+ // Transform screenshot format into an action (unsupported by chrome-cdp)
+ ...(meta.options.formats.includes("screenshot") || meta.options.formats.includes("screenshot@fullPage") ? [{
+ type: "screenshot" as const,
+ fullPage: meta.options.formats.includes("screenshot@fullPage"),
+ }] : []),
+
+ // Include specified actions
+ ...(meta.options.actions ?? []),
+ ];
+
+ const request: FireEngineScrapeRequestCommon & FireEngineScrapeRequestChromeCDP = {
+ url: meta.url,
+ engine: "chrome-cdp",
+ instantReturn: true,
+ skipTlsVerification: meta.options.skipTlsVerification,
+ headers: meta.options.headers,
+ ...(actions.length > 0 ? ({
+ actions,
+ }) : {}),
+ priority: meta.internalOptions.priority,
+ geolocation: meta.options.geolocation,
+ mobile: meta.options.mobile,
+ // TODO: scrollXPaths
+ };
+
+ let response = await performFireEngineScrape(
+ meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),
+ request,
+ );
+
+ specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck" }), response.responseHeaders);
+
+ if (meta.options.formats.includes("screenshot") || meta.options.formats.includes("screenshot@fullPage")) {
+ meta.logger.debug("Transforming screenshots from actions into screenshot field", { screenshots: response.screenshots });
+ response.screenshot = (response.screenshots ?? [])[0];
+ (response.screenshots ?? []).splice(0, 1);
+ meta.logger.debug("Screenshot transformation done", { screenshots: response.screenshots, screenshot: response.screenshot });
+ }
+
+ if (!response.url) {
+ meta.logger.warn("Fire-engine did not return the response's URL", { response, sourceURL: meta.url });
+ }
+
+ return {
+ url: response.url ?? meta.url,
+
+ html: response.content,
+ error: response.pageError,
+ statusCode: response.pageStatusCode,
+
+ screenshot: response.screenshot,
+ ...(actions.length > 0 ? {
+ actions: {
+ screenshots: response.screenshots ?? [],
+ scrapes: response.actionContent ?? [],
+ }
+ } : {}),
+ };
+}
+
+export async function scrapeURLWithFireEnginePlaywright(meta: Meta): Promise<EngineScrapeResult> {
+ const request: FireEngineScrapeRequestCommon & FireEngineScrapeRequestPlaywright = {
+ url: meta.url,
+ engine: "playwright",
+ instantReturn: true,
+
+ headers: meta.options.headers,
+ priority: meta.internalOptions.priority,
+ screenshot: meta.options.formats.includes("screenshot"),
+ fullPageScreenshot: meta.options.formats.includes("screenshot@fullPage"),
+ wait: meta.options.waitFor,
+ geolocation: meta.options.geolocation,
+ };
+
+ let response = await performFireEngineScrape(
+ meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),
+ request,
+ );
+
+ specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEnginePlaywright/specialtyScrapeCheck" }), response.responseHeaders);
+
+ if (!response.url) {
+ meta.logger.warn("Fire-engine did not return the response's URL", { response, sourceURL: meta.url });
+ }
+
+ return {
+ url: response.url ?? meta.url,
+
+ html: response.content,
+ error: response.pageError,
+ statusCode: response.pageStatusCode,
+
+ ...(response.screenshots !== undefined && response.screenshots.length > 0 ? ({
+ screenshot: response.screenshots[0],
+ }) : {}),
+ };
+}
+
+export async function scrapeURLWithFireEngineTLSClient(meta: Meta): Promise<EngineScrapeResult> {
+ const request: FireEngineScrapeRequestCommon & FireEngineScrapeRequestTLSClient = {
+ url: meta.url,
+ engine: "tlsclient",
+ instantReturn: true,
+
+ headers: meta.options.headers,
+ priority: meta.internalOptions.priority,
+
+ atsv: meta.internalOptions.atsv,
+ geolocation: meta.options.geolocation,
+ disableJsDom: meta.internalOptions.v0DisableJsDom,
+ };
+
+ let response = await performFireEngineScrape(
+ meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),
+ request,
+ );
+
+ specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineTLSClient/specialtyScrapeCheck" }), response.responseHeaders);
+
+ if (!response.url) {
+ meta.logger.warn("Fire-engine did not return the response's URL", { response, sourceURL: meta.url });
+ }
+
+ return {
+ url: response.url ?? meta.url,
+
+ html: response.content,
+ error: response.pageError,
+ statusCode: response.pageStatusCode,
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
new file mode 100644
index 000000000..9332125a6
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -0,0 +1,94 @@
+import { Logger } from "winston";
+import * as Sentry from "@sentry/node";
+import { z } from "zod";
+
+import { Action } from "../../../../lib/entities";
+import { robustFetch } from "../../lib/fetch";
+
+export type FireEngineScrapeRequestCommon = {
+ url: string;
+
+ headers?: { [K: string]: string };
+
+ blockMedia?: boolean; // default: true
+ blockAds?: boolean; // default: true
+ // pageOptions?: any; // unused, .scrollXPaths is considered on FE side
+
+ // useProxy?: boolean; // unused, default: true
+ // customProxy?: string; // unused
+
+ // disableSmartWaitCache?: boolean; // unused, default: false
+ // skipDnsCheck?: boolean; // unused, default: false
+
+ priority?: number; // default: 1
+ // team_id?: string; // unused
+ logRequest?: boolean; // default: true
+ instantReturn?: boolean; // default: false
+ geolocation?: { country?: string; languages?: string[]; };
+}
+
+export type FireEngineScrapeRequestChromeCDP = {
+ engine: "chrome-cdp";
+ skipTlsVerification?: boolean;
+ actions?: Action[];
+ blockMedia?: true; // cannot be false
+ mobile?: boolean;
+};
+
+export type FireEngineScrapeRequestPlaywright = {
+ engine: "playwright";
+ blockAds?: boolean; // default: true
+
+ // mutually exclusive, default: false
+ screenshot?: boolean;
+ fullPageScreenshot?: boolean;
+
+ wait?: number; // default: 0
+};
+
+export type FireEngineScrapeRequestTLSClient = {
+ engine: "tlsclient";
+ atsv?: boolean; // v0 only, default: false
+ disableJsDom?: boolean; // v0 only, default: false
+ // blockAds?: boolean; // default: true
+};
+
+const schema = z.object({
+ jobId: z.string(),
+ processing: z.boolean(),
+});
+
+export async function fireEngineScrape<Engine extends FireEngineScrapeRequestChromeCDP | FireEngineScrapeRequestPlaywright | FireEngineScrapeRequestTLSClient> (
+ logger: Logger,
+ request: FireEngineScrapeRequestCommon & Engine,
+): Promise<z.infer<typeof schema>> {
+ const fireEngineURL = process.env.FIRE_ENGINE_BETA_URL!;
+
+ // TODO: retries
+
+ const scrapeRequest = await Sentry.startSpan({
+ name: "fire-engine: Scrape",
+ attributes: {
+ url: request.url,
+ },
+ }, async span => {
+ return await robustFetch(
+ {
+ url: `${fireEngineURL}/scrape`,
+ method: "POST",
+ headers: {
+ ...(Sentry.isInitialized() ? ({
+ "sentry-trace": Sentry.spanToTraceHeader(span),
+ "baggage": Sentry.spanToBaggageHeader(span),
+ }) : {}),
+ },
+ body: request,
+ logger: logger.child({ method: "fireEngineScrape/robustFetch" }),
+ schema,
+ tryCount: 3,
+ }
+ );
+ });
+
+ return scrapeRequest;
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
new file mode 100644
index 000000000..aadef7fc6
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -0,0 +1,295 @@
+import { ScrapeActionContent } from "../../../lib/entities";
+import { Meta } from "..";
+import { scrapeDOCX } from "./docx";
+import { scrapeURLWithFireEngineChromeCDP, scrapeURLWithFireEnginePlaywright, scrapeURLWithFireEngineTLSClient } from "./fire-engine";
+import { scrapePDF } from "./pdf";
+import { scrapeURLWithScrapingBee } from "./scrapingbee";
+import { scrapeURLWithFetch } from "./fetch";
+import { scrapeURLWithPlaywright } from "./playwright";
+
+export type Engine = "fire-engine;chrome-cdp" | "fire-engine;playwright" | "fire-engine;tlsclient" | "scrapingbee" | "scrapingbeeLoad" | "playwright" | "fetch" | "pdf" | "docx";
+
+const useScrapingBee = process.env.SCRAPING_BEE_API_KEY !== '' && process.env.SCRAPING_BEE_API_KEY !== undefined;
+const useFireEngine = process.env.FIRE_ENGINE_BETA_URL !== '' && process.env.FIRE_ENGINE_BETA_URL !== undefined;
+const usePlaywright = process.env.PLAYWRIGHT_MICROSERVICE_URL !== '' && process.env.PLAYWRIGHT_MICROSERVICE_URL !== undefined;
+
+export const engines: Engine[] = [
+ ...(useFireEngine ? [ "fire-engine;chrome-cdp" as const, "fire-engine;playwright" as const, "fire-engine;tlsclient" as const ] : []),
+ ...(useScrapingBee ? [ "scrapingbee" as const, "scrapingbeeLoad" as const ] : []),
+ ...(usePlaywright ? [ "playwright" as const ] : []),
+ "fetch",
+ "pdf",
+ "docx",
+];
+
+export const featureFlags = [
+ "actions",
+ "waitFor",
+ "screenshot",
+ "screenshot@fullScreen",
+ "pdf",
+ "docx",
+ "atsv",
+ "location",
+ "mobile",
+ "skipTlsVerification",
+ "useFastMode",
+] as const;
+
+export type FeatureFlag = typeof featureFlags[number];
+
+export const featureFlagOptions: {
+ [F in FeatureFlag]: {
+ priority: number;
+ }
+} = {
+ "actions": { priority: 20 },
+ "waitFor": { priority: 1 },
+ "screenshot": { priority: 10 },
+ "screenshot@fullScreen": { priority: 10 },
+ "pdf": { priority: 100 },
+ "docx": { priority: 100 },
+ "atsv": { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if not
+ "useFastMode": { priority: 90 },
+ "location": { priority: 10 },
+ "mobile": { priority: 10 },
+ "skipTlsVerification": { priority: 10 },
+} as const;
+
+export type EngineScrapeResult = {
+ url: string;
+
+ html: string;
+ markdown?: string;
+ statusCode: number;
+ error?: string;
+
+ screenshot?: string;
+ actions?: {
+ screenshots: string[];
+ scrapes: ScrapeActionContent[];
+ };
+}
+
+const engineHandlers: {
+ [E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>
+} = {
+ "fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,
+ "fire-engine;playwright": scrapeURLWithFireEnginePlaywright,
+ "fire-engine;tlsclient": scrapeURLWithFireEngineTLSClient,
+ "scrapingbee": scrapeURLWithScrapingBee("domcontentloaded"),
+ "scrapingbeeLoad": scrapeURLWithScrapingBee("networkidle2"),
+ "playwright": scrapeURLWithPlaywright,
+ "fetch": scrapeURLWithFetch,
+ "pdf": scrapePDF,
+ "docx": scrapeDOCX,
+};
+
+export const engineOptions: {
+ [E in Engine]: {
+ // A list of feature flags the engine supports.
+ features: { [F in FeatureFlag]: boolean },
+
+ // This defines the order of engines in general. The engine with the highest quality will be used the most.
+ // Negative quality numbers are reserved for specialty engines, e.g. PDF and DOCX
+ quality: number,
+ }
+} = {
+ "fire-engine;chrome-cdp": {
+ features: {
+ "actions": true,
+ "waitFor": true, // through actions transform
+ "screenshot": true, // through actions transform
+ "screenshot@fullScreen": true, // through actions transform
+ "pdf": false,
+ "docx": false,
+ "atsv": false,
+ "location": true,
+ "mobile": true,
+ "skipTlsVerification": true,
+ "useFastMode": false,
+ },
+ quality: 50,
+ },
+ "fire-engine;playwright": {
+ features: {
+ "actions": false,
+ "waitFor": true,
+ "screenshot": true,
+ "screenshot@fullScreen": true,
+ "pdf": false,
+ "docx": false,
+ "atsv": false,
+ "location": false,
+ "mobile": false,
+ "skipTlsVerification": false,
+ "useFastMode": false,
+ },
+ quality: 40,
+ },
+ "scrapingbee": {
+ features: {
+ "actions": false,
+ "waitFor": true,
+ "screenshot": true,
+ "screenshot@fullScreen": true,
+ "pdf": false,
+ "docx": false,
+ "atsv": false,
+ "location": false,
+ "mobile": false,
+ "skipTlsVerification": false,
+ "useFastMode": false,
+ },
+ quality: 30,
+ },
+ "scrapingbeeLoad": {
+ features: {
+ "actions": false,
+ "waitFor": true,
+ "screenshot": true,
+ "screenshot@fullScreen": true,
+ "pdf": false,
+ "docx": false,
+ "atsv": false,
+ "location": false,
+ "mobile": false,
+ "skipTlsVerification": false,
+ "useFastMode": false,
+ },
+ quality: 29,
+ },
+ "playwright": {
+ features: {
+ "actions": false,
+ "waitFor": true,
+ "screenshot": false,
+ "screenshot@fullScreen": false,
+ "pdf": false,
+ "docx": false,
+ "atsv": false,
+ "location": false,
+ "mobile": false,
+ "skipTlsVerification": false,
+ "useFastMode": false,
+ },
+ quality: 20,
+ },
+ "fire-engine;tlsclient": {
+ features: {
+ "actions": false,
+ "waitFor": false,
+ "screenshot": false,
+ "screenshot@fullScreen": false,
+ "pdf": false,
+ "docx": false,
+ "atsv": true,
+ "location": true,
+ "mobile": false,
+ "skipTlsVerification": false,
+ "useFastMode": true,
+ },
+ quality: 10,
+ },
+ "fetch": {
+ features: {
+ "actions": false,
+ "waitFor": false,
+ "screenshot": false,
+ "screenshot@fullScreen": false,
+ "pdf": false,
+ "docx": false,
+ "atsv": false,
+ "location": false,
+ "mobile": false,
+ "skipTlsVerification": false,
+ "useFastMode": true,
+ },
+ quality: 5,
+ },
+ "pdf": {
+ features: {
+ "actions": false,
+ "waitFor": false,
+ "screenshot": false,
+ "screenshot@fullScreen": false,
+ "pdf": true,
+ "docx": false,
+ "atsv": false,
+ "location": false,
+ "mobile": false,
+ "skipTlsVerification": false,
+ "useFastMode": true,
+ },
+ quality: -10,
+ },
+ "docx": {
+ features: {
+ "actions": false,
+ "waitFor": false,
+ "screenshot": false,
+ "screenshot@fullScreen": false,
+ "pdf": false,
+ "docx": true,
+ "atsv": false,
+ "location": false,
+ "mobile": false,
+ "skipTlsVerification": false,
+ "useFastMode": true,
+ },
+ quality: -10,
+ },
+};
+
+export function buildFallbackList(meta: Meta): {
+ engine: Engine,
+ unsupportedFeatures: Set<FeatureFlag>,
+}[] {
+ const prioritySum = [...meta.featureFlags].reduce((a, x) => a + featureFlagOptions[x].priority, 0);
+ const priorityThreshold = Math.floor(prioritySum / 2);
+ let selectedEngines: {
+ engine: Engine,
+ supportScore: number,
+ unsupportedFeatures: Set<FeatureFlag>,
+ }[] = [];
+
+ const currentEngines = meta.internalOptions.forceEngine !== undefined ? [meta.internalOptions.forceEngine] : engines;
+
+ for (const engine of currentEngines) {
+ const supportedFlags = new Set([...Object.entries(engineOptions[engine].features).filter(([k, v]) => meta.featureFlags.has(k as FeatureFlag) && v === true).map(([k, _]) => k)]);
+ const supportScore = [...supportedFlags].reduce((a, x) => a + featureFlagOptions[x].priority, 0);
+
+ const unsupportedFeatures = new Set([...meta.featureFlags]);
+ for (const flag of meta.featureFlags) {
+ if (supportedFlags.has(flag)) {
+ unsupportedFeatures.delete(flag);
+ }
+ }
+
+ if (supportScore >= priorityThreshold) {
+ selectedEngines.push({ engine, supportScore, unsupportedFeatures });
+ meta.logger.debug(`Engine ${engine} meets feature priority threshold`, { supportScore, prioritySum, priorityThreshold, featureFlags: [...meta.featureFlags], unsupportedFeatures });
+ } else {
+ meta.logger.debug(`Engine ${engine} does not meet feature priority threshold`, { supportScore, prioritySum, priorityThreshold, featureFlags: [...meta.featureFlags], unsupportedFeatures});
+ }
+ }
+
+ if (selectedEngines.some(x => engineOptions[x.engine].quality > 0)) {
+ selectedEngines = selectedEngines.filter(x => engineOptions[x.engine].quality > 0);
+ }
+
+ selectedEngines.sort((a,b) => b.supportScore - a.supportScore || engineOptions[b.engine].quality - engineOptions[a.engine].quality);
+
+ return selectedEngines;
+}
+
+export async function scrapeURLWithEngine(meta: Meta, engine: Engine): Promise<EngineScrapeResult> {
+ const fn = engineHandlers[engine];
+ const logger = meta.logger.child({ method: fn.name ?? "scrapeURLWithEngine", engine });
+ const _meta = {
+ ...meta,
+ logger,
+ };
+
+ return await fn(_meta);
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
new file mode 100644
index 000000000..ea44b0510
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -0,0 +1,114 @@
+import { createReadStream, promises as fs } from "node:fs";
+import FormData from "form-data";
+import { Meta } from "../..";
+import { EngineScrapeResult } from "..";
+import * as marked from "marked";
+import { robustFetch } from "../../lib/fetch";
+import { z } from "zod";
+import * as Sentry from "@sentry/node";
+import escapeHtml from "escape-html";
+import PdfParse from "pdf-parse";
+import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";
+
+type PDFProcessorResult = {html: string, markdown?: string};
+
+async function scrapePDFWithLlamaParse(meta: Meta, tempFilePath: string): Promise<PDFProcessorResult> {
+ meta.logger.debug("Processing PDF document with LlamaIndex", { tempFilePath });
+
+ const uploadForm = new FormData();
+ uploadForm.append("file", createReadStream(tempFilePath), {
+ filename: tempFilePath,
+ contentType: "application/pdf", // NOTE: request.headers["Content-Type"]?
+ });
+
+ const upload = await robustFetch({
+ url: "https://api.cloud.llamaindex.ai/api/parsing/upload",
+ method: "POST",
+ headers: {
+ "Authorization": `Bearer ${process.env.LLAMAPARSE_API_KEY}`,
+ },
+ body: uploadForm,
+ logger: meta.logger.child({ method: "scrapePDFWithLlamaParse/upload/robustFetch" }),
+ schema: z.object({
+ id: z.string(),
+ }),
+ });
+
+ const jobId = upload.id;
+
+ // TODO: timeout, retries
+ const result = await robustFetch({
+ url: `https://api.cloud.llamaindex.ai/api/parsing/job/${jobId}/result/markdown`,
+ method: "GET",
+ headers: {
+ "Authorization": `Bearer ${process.env.LLAMAPARSE_API_KEY}`,
+ },
+ logger: meta.logger.child({ method: "scrapePDFWithLlamaParse/result/robustFetch" }),
+ schema: z.object({
+ markdown: z.string(),
+ }),
+ });
+
+ return {
+ markdown: result.markdown,
+ html: await marked.parse(result.markdown, { async: true }),
+ };
+}
+
+async function scrapePDFWithParsePDF(meta: Meta, tempFilePath: string): Promise<PDFProcessorResult> {
+ meta.logger.debug("Processing PDF document with parse-pdf", { tempFilePath });
+
+ const result = await PdfParse(await fs.readFile(tempFilePath));
+ const escaped = escapeHtml(result.text);
+
+ return {
+ markdown: escaped,
+ html: escaped,
+ };
+}
+
+export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
+ if (!meta.options.parsePDF) {
+ const file = await fetchFileToBuffer(meta.url);
+ const content = file.buffer.toString("base64");
+ return {
+ url: file.response.url,
+ statusCode: file.response.status,
+
+ html: content,
+ markdown: content,
+ };
+ }
+
+ const { response, tempFilePath } = await downloadFile(meta.id, meta.url);
+
+ let result: PDFProcessorResult | null = null;
+ if (process.env.LLAMAPARSE_API_KEY) {
+ try {
+ result = await scrapePDFWithLlamaParse({
+ ...meta,
+ logger: meta.logger.child({ method: "scrapePDF/scrapePDFWithLlamaParse" }),
+ }, tempFilePath);
+ } catch (error) {
+ meta.logger.warn("LlamaParse failed to parse PDF -- falling back to parse-pdf", { error });
+ Sentry.captureException(error);
+ }
+ }
+
+ if (result === null) {
+ result = await scrapePDFWithParsePDF({
+ ...meta,
+ logger: meta.logger.child({ method: "scrapePDF/scrapePDFWithParsePDF" }),
+ }, tempFilePath);
+ }
+
+ await fs.unlink(tempFilePath);
+
+ return {
+ url: response.url,
+ statusCode: response.status,
+
+ html: result.html,
+ markdown: result.markdown,
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
new file mode 100644
index 000000000..c8e0fe598
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
@@ -0,0 +1,42 @@
+import { z } from "zod";
+import { EngineScrapeResult } from "..";
+import { Meta } from "../..";
+import { TimeoutError } from "../../error";
+import { robustFetch } from "../../lib/fetch";
+
+export async function scrapeURLWithPlaywright(meta: Meta): Promise<EngineScrapeResult> {
+ const timeout = 20000 + meta.options.waitFor;
+
+ const response = await Promise.race([
+ await robustFetch({
+ url: process.env.PLAYWRIGHT_MICROSERVICE_URL!,
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ url: meta.url,
+ wait_after_load: meta.options.waitFor,
+ timeout,
+ headers: meta.options.headers,
+ }),
+ method: "POST",
+ logger: meta.logger.child("scrapeURLWithPlaywright/robustFetch"),
+ schema: z.object({
+ content: z.string(),
+ pageStatusCode: z.number(),
+ pageError: z.string().optional(),
+ }),
+ }),
+ (async () => {
+ await new Promise((resolve) => setTimeout(() => resolve(null), 20000));
+ throw new TimeoutError("Playwright was unable to scrape the page before timing out", { cause: { timeout } });
+ })(),
+ ]);
+
+ return {
+ url: meta.url, // TODO: impove redirect following
+ html: response.content,
+ statusCode: response.pageStatusCode,
+ error: response.pageError,
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts b/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts
new file mode 100644
index 000000000..9b946bf0e
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts
@@ -0,0 +1,66 @@
+import { ScrapingBeeClient } from "scrapingbee";
+import { Meta } from "../..";
+import { EngineScrapeResult } from "..";
+import { specialtyScrapeCheck } from "../utils/specialtyHandler";
+import { AxiosError, type AxiosResponse } from "axios";
+import { EngineError } from "../../error";
+
+const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY!);
+
+export function scrapeURLWithScrapingBee(wait_browser: "domcontentloaded" | "networkidle2"): ((meta: Meta) => Promise<EngineScrapeResult>) {
+ return async (meta: Meta): Promise<EngineScrapeResult> => {
+ let response: AxiosResponse<any>;
+ try {
+ response = await client.get({
+ url: meta.url,
+ params: {
+ timeout: 15000, // TODO: dynamic timeout based on request timeout
+ wait_browser: wait_browser,
+ wait: Math.min(meta.options.waitFor, 35000),
+ transparent_status_code: true,
+ json_response: true,
+ screenshot: meta.options.formats.includes("screenshot"),
+ screenshot_full_page: meta.options.formats.includes("screenshot@fullPage"),
+ },
+ headers: {
+ "ScrapingService-Request": "TRUE", // this is sent to the page, not to ScrapingBee - mogery
+ },
+ });
+ } catch (error) {
+ if (error instanceof AxiosError && error.response !== undefined) {
+ response = error.response;
+ } else {
+ throw error;
+ }
+ }
+
+ const data: Buffer = response.data;
+ const body = JSON.parse(new TextDecoder().decode(data));
+
+ const headers = body.headers ?? {};
+ const isHiddenEngineError = !(headers["Date"] ?? headers["date"] ?? headers["Content-Type"] ?? headers["content-type"]);
+
+ if (body.errors || body.body?.error || isHiddenEngineError) {
+ meta.logger.error("ScrapingBee threw an error", { body: body.body?.error ?? body.errors ?? body.body ?? body });
+ throw new EngineError("Engine error #34", { cause: { body, statusCode: response.status } });
+ }
+
+ if (typeof body.body !== "string") {
+ meta.logger.error("ScrapingBee: Body is not string??", { body });
+ throw new EngineError("Engine error #35", { cause: { body, statusCode: response.status } });
+ }
+
+ specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithScrapingBee/specialtyScrapeCheck" }), body.headers);
+
+ return {
+ url: body["resolved-url"] ?? meta.url,
+
+ html: body.body,
+ error: response.status >= 300 ? response.statusText : undefined,
+ statusCode: response.status,
+ ...(body.screenshot ? ({
+ screenshot: `data:image/png;base64,${body.screenshot}`,
+ }) : {}),
+ };
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
new file mode 100644
index 000000000..8db8892be
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
@@ -0,0 +1,45 @@
+import path from "path";
+import os from "os";
+import { createWriteStream, promises as fs } from "node:fs";
+import { EngineError } from "../../error";
+import { Writable } from "stream";
+import { v4 as uuid } from "uuid";
+
+export async function fetchFileToBuffer(url: string): Promise<{
+ response: Response,
+ buffer: Buffer
+}> {
+ const response = await fetch(url); // TODO: maybe we could use tlsclient for this? for proxying
+ return {
+ response,
+ buffer: Buffer.from(await response.arrayBuffer()),
+ };
+}
+
+export async function downloadFile(id: string, url: string): Promise<{
+ response: Response
+ tempFilePath: string
+}> {
+ const tempFilePath = path.join(os.tmpdir(), `tempFile-${id}--${uuid()}`);
+ const tempFileWrite = createWriteStream(tempFilePath);
+
+ const response = await fetch(url); // TODO: maybe we could use tlsclient for this? for proxying
+
+ // This should never happen in the current state of JS (2024), but let's check anyways.
+ if (response.body === null) {
+ throw new EngineError("Response body was null", { cause: { response } });
+ }
+
+ response.body.pipeTo(Writable.toWeb(tempFileWrite));
+ await new Promise((resolve, reject) => {
+ tempFileWrite.on("finish", () => resolve(null));
+ tempFileWrite.on("error", (error) => {
+ reject(new EngineError("Failed to write to temp file", { cause: { error } }));
+ });
+ })
+
+ return {
+ response,
+ tempFilePath,
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
new file mode 100644
index 000000000..b10657a4d
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -0,0 +1,14 @@
+import { Logger } from "winston";
+import { AddFeatureError } from "../../error";
+
+export function specialtyScrapeCheck(logger: Logger, headers: Record<string, string> | undefined) {
+ const contentType = (Object.entries(headers ?? {}).find(x => x[0].toLowerCase() === "content-type") ?? [])[1];
+
+ if (contentType === undefined) {
+ logger.warn("Failed to check contentType -- was not present in headers", { headers });
+ } else if (contentType === "application/pdf" || contentType.startsWith("application/pdf;")) { // .pdf
+ throw new AddFeatureError(["pdf"]);
+ } else if (contentType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || contentType.startsWith("application/vnd.openxmlformats-officedocument.wordprocessingml.document;")) { // .docx
+ throw new AddFeatureError(["docx"]);
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
new file mode 100644
index 000000000..56ab290d3
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -0,0 +1,34 @@
+import { EngineResultsTracker } from "."
+import { Engine, FeatureFlag } from "./engines"
+
+export class EngineError extends Error {
+ constructor(message?: string, options?: ErrorOptions) {
+ super(message, options)
+ }
+}
+
+export class TimeoutError extends Error {
+ constructor(message?: string, options?: ErrorOptions) {
+ super(message, options)
+ }
+}
+
+export class NoEnginesLeftError extends Error {
+ public fallbackList: Engine[];
+ public results: EngineResultsTracker;
+
+ constructor(fallbackList: Engine[], results: EngineResultsTracker) {
+ super("All scraping engines failed!");
+ this.fallbackList = fallbackList;
+ this.results = results;
+ }
+}
+
+export class AddFeatureError extends Error {
+ public featureFlags: FeatureFlag[];
+
+ constructor(featureFlags: FeatureFlag[]) {
+ super("New feature flags have been discovered: " + featureFlags.join(", "));
+ this.featureFlags = featureFlags;
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
new file mode 100644
index 000000000..738c8380f
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -0,0 +1,320 @@
+import { Logger } from "winston";
+import * as Sentry from "@sentry/node";
+
+import { Document, ScrapeOptions } from "../../controllers/v1/types";
+import { logger, ArrayTransport } from "../../lib/logger";
+import { buildFallbackList, Engine, EngineScrapeResult, FeatureFlag, scrapeURLWithEngine } from "./engines";
+import { parseMarkdown } from "../../lib/html-to-markdown";
+import { AddFeatureError, EngineError, NoEnginesLeftError, TimeoutError } from "./error";
+import { executeTransformers } from "./transformers";
+import { LLMRefusalError } from "./transformers/llmExtract";
+import { urlSpecificParams } from "./lib/urlSpecificParams";
+
+export type ScrapeUrlResponse = ({
+ success: true,
+ document: Document,
+} | {
+ success: false,
+ error: any,
+}) & {
+ logs: any[],
+ engines: EngineResultsTracker,
+}
+
+export type Meta = {
+ id: string;
+ url: string;
+ options: ScrapeOptions;
+ internalOptions: InternalOptions;
+ logger: Logger;
+ logs: any[];
+ featureFlags: Set<FeatureFlag>;
+}
+
+function buildFeatureFlags(url: string, options: ScrapeOptions, internalOptions: InternalOptions): Set<FeatureFlag> {
+ const flags: Set<FeatureFlag> = new Set();
+
+ if (options.actions !== undefined) {
+ flags.add("actions");
+ }
+
+ if (options.formats.includes("screenshot")) {
+ flags.add("screenshot");
+ }
+
+ if (options.formats.includes("screenshot@fullPage")) {
+ flags.add("screenshot@fullScreen");
+ }
+
+ if (options.waitFor !== 0) {
+ flags.add("waitFor");
+ }
+
+ if (internalOptions.atsv) {
+ flags.add("atsv");
+ }
+
+ if (options.location || options.geolocation) {
+ flags.add("location");
+ }
+
+ if (options.mobile) {
+ flags.add("mobile");
+ }
+
+ if (options.skipTlsVerification) {
+ flags.add("skipTlsVerification");
+ }
+
+ if (internalOptions.v0UseFastMode) {
+ flags.add("useFastMode");
+ }
+
+ const urlO = new URL(url);
+
+ if (urlO.pathname.endsWith(".pdf")) {
+ flags.add("pdf");
+ }
+
+ if (urlO.pathname.endsWith(".docx")) {
+ flags.add("docx");
+ }
+
+ return flags;
+}
+
+// The meta object contains all required information to perform a scrape.
+// For example, the scrape ID, URL, options, feature flags, logs that occur while scraping.
+// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)
+// Having a meta object that is treated as immutable helps the code stay clean and easily tracable,
+// while also retaining the benefits that WebScraper had from its OOP design.
+function buildMetaObject(id: string, url: string, options: ScrapeOptions, internalOptions: InternalOptions): Meta {
+ const specParams = urlSpecificParams[new URL(url).hostname.replace(/^www\./, "")];
+ if (specParams !== undefined) {
+ options = Object.assign(options, specParams.scrapeOptions);
+ internalOptions = Object.assign(internalOptions, specParams.internalOptions);
+ }
+
+ const _logger = logger.child({ module: "ScrapeURL", scrapeId: id });
+ const logs: any[] = [];
+ _logger.add(new ArrayTransport({ array: logs, scrapeId: id }));
+
+ return {
+ id, url, options, internalOptions,
+ logger: _logger,
+ logs,
+ featureFlags: buildFeatureFlags(url, options, internalOptions),
+ };
+}
+
+export type InternalOptions = {
+ priority?: number; // Passed along to fire-engine
+ forceEngine?: Engine;
+ atsv?: boolean; // anti-bot solver, beta
+
+ v0CrawlOnlyUrls?: boolean;
+ v0UseFastMode?: boolean;
+ v0DisableJsDom?: boolean;
+};
+
+export type EngineResultsTracker = { [E in Engine]?: ({
+ state: "error",
+ error: any,
+ unexpected: boolean,
+} | {
+ state: "success",
+ result: EngineScrapeResult & { markdown: string },
+ factors: Record<string, boolean>,
+ unsupportedFeatures: Set<FeatureFlag>,
+} | {
+ state: "timeout",
+}) & {
+ startedAt: number,
+ finishedAt: number,
+} };
+
+export type EngineScrapeResultWithContext = {
+ engine: Engine,
+ unsupportedFeatures: Set<FeatureFlag>,
+ result: (EngineScrapeResult & { markdown: string }),
+};
+
+function safeguardCircularError<T>(error: T): T {
+ if (typeof error === "object" && error !== null && (error as any).results) {
+ const newError = structuredClone(error);
+ delete (newError as any).results;
+ return newError;
+ } else {
+ return error;
+ }
+}
+
+async function scrapeURLLoop(
+ meta: Meta
+): Promise<ScrapeUrlResponse> {
+ meta.logger.info(`Scraping URL ${JSON.stringify(meta.url)}...`,);
+
+ // TODO: handle sitemap data, see WebScraper/index.ts:280
+ // TODO: ScrapeEvents
+
+ const fallbackList = buildFallbackList(meta);
+
+ const results: EngineResultsTracker = {};
+ let result: EngineScrapeResultWithContext | null = null;
+
+ for (const { engine, unsupportedFeatures } of fallbackList) {
+ const startedAt = Date.now();
+ try {
+ meta.logger.info("Scraping via " + engine + "...");
+ const _engineResult = await scrapeURLWithEngine(meta, engine);
+ if (_engineResult.markdown === undefined) { // Some engines emit Markdown directly.
+ _engineResult.markdown = await parseMarkdown(_engineResult.html);
+ }
+ const engineResult = _engineResult as EngineScrapeResult & { markdown: string };
+
+ // Success factors
+ const isLongEnough = engineResult.markdown.length >= 20;
+ const isGoodStatusCode = (engineResult.statusCode >= 200 && engineResult.statusCode < 300) || engineResult.statusCode === 304;
+ const hasNoPageError = engineResult.error === undefined;
+
+ results[engine] = {
+ state: "success",
+ result: engineResult,
+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },
+ unsupportedFeatures,
+ startedAt,
+ finishedAt: Date.now(),
+ };
+
+ // NOTE: TODO: what to do when status code is bad is tough...
+ // we cannot just rely on text because error messages can be brief and not hit the limit
+ // should we just use all the fallbacks and pick the one with the longest text? - mogery
+ if (isLongEnough || !isGoodStatusCode) {
+ meta.logger.info("Scrape via " + engine + " deemed successful.", { factors: { isLongEnough, isGoodStatusCode, hasNoPageError }, engineResult });
+ result = {
+ engine,
+ unsupportedFeatures,
+ result: engineResult as EngineScrapeResult & { markdown: string }
+ };
+ break;
+ }
+ } catch (error) {
+ if (error instanceof EngineError) {
+ meta.logger.info("Engine " + engine + " could not scrape the page.", { error });
+ results[engine] = {
+ state: "error",
+ error: safeguardCircularError(error),
+ unexpected: false,
+ startedAt,
+ finishedAt: Date.now(),
+ };
+ } else if (error instanceof TimeoutError) {
+ meta.logger.info("Engine " + engine + " timed out while scraping.", { error });
+ results[engine] = {
+ state: "timeout",
+ startedAt,
+ finishedAt: Date.now(),
+ };
+ } else if (error instanceof AddFeatureError) {
+ throw error;
+ } else if (error instanceof LLMRefusalError) {
+ results[engine] = {
+ state: "error",
+ error: safeguardCircularError(error),
+ unexpected: true,
+ startedAt,
+ finishedAt: Date.now(),
+ }
+ error.results = results;
+ meta.logger.warn("LLM refusal encountered", { error });
+ throw error;
+ } else {
+ Sentry.captureException(error);
+ meta.logger.info("An unexpected error happened while scraping with " + engine + ".", { error });
+ results[engine] = {
+ state: "error",
+ error: safeguardCircularError(error),
+ unexpected: true,
+ startedAt,
+ finishedAt: Date.now(),
+ }
+ }
+ }
+ }
+
+ if (result === null) {
+ throw new NoEnginesLeftError(fallbackList.map(x => x.engine), results);
+ }
+
+ let document: Document = {
+ markdown: result.result.markdown,
+ rawHtml: result.result.html,
+ screenshot: result.result.screenshot,
+ actions: result.result.actions,
+ metadata: {
+ sourceURL: meta.url,
+ url: result.result.url,
+ statusCode: result.result.statusCode,
+ error: result.result.error,
+ },
+ }
+
+ if (result.unsupportedFeatures.size > 0) {
+ const warning = `The engine used does not support the following features: ${[...result.unsupportedFeatures].join(", ")} -- your scrape may be partial.`;
+ meta.logger.warn(warning, { engine: result.engine, unsupportedFeatures: result.unsupportedFeatures });
+ document.warning = document.warning !== undefined ? document.warning + " " + warning : warning;
+ }
+
+ document = await executeTransformers(meta, document);
+
+ return {
+ success: true,
+ document,
+ logs: meta.logs,
+ engines: results,
+ };
+}
+
+export async function scrapeURL(
+ id: string,
+ url: string,
+ options: ScrapeOptions,
+ internalOptions: InternalOptions = {},
+): Promise<ScrapeUrlResponse> {
+ const meta = buildMetaObject(id, url, options, internalOptions);
+ try {
+ while (true) {
+ try {
+ return await scrapeURLLoop(meta);
+ } catch (error) {
+ if (error instanceof AddFeatureError && meta.internalOptions.forceEngine === undefined) {
+ meta.logger.debug("More feature flags requested by scraper: adding " + error.featureFlags.join(", "), { error, existingFlags: meta.featureFlags });
+ meta.featureFlags = new Set([...meta.featureFlags].concat(error.featureFlags));
+ } else {
+ throw error;
+ }
+ }
+ }
+ } catch (error) {
+ let results: EngineResultsTracker = {};
+
+ if (error instanceof NoEnginesLeftError) {
+ meta.logger.warn("scrapeURL: All scraping engines failed!", { error });
+ results = error.results;
+ } else if (error instanceof LLMRefusalError) {
+ meta.logger.warn("scrapeURL: LLM refused to extract content", { error });
+ results = error.results!;
+ } else {
+ Sentry.captureException(error);
+ meta.logger.error("scrapeURL: Unexpected error happened", { error });
+ // TODO: results?
+ }
+
+ return {
+ success: false,
+ error,
+ logs: meta.logs,
+ engines: results,
+ }
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts b/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts
new file mode 100644
index 000000000..484bf725b
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts
@@ -0,0 +1,35 @@
+// TODO: refactor
+import { load } from "cheerio";
+import { logger } from "../../../lib/logger";
+
+export function extractLinks(html: string, baseUrl: string): string[] {
+ const $ = load(html);
+ const links: string[] = [];
+
+ $('a').each((_, element) => {
+ const href = $(element).attr('href');
+ if (href) {
+ try {
+ if (href.startsWith('http://') || href.startsWith('https://')) {
+ // Absolute URL, add as is
+ links.push(href);
+ } else if (href.startsWith('/')) {
+ // Relative URL starting with '/', append to origin
+ links.push(new URL(href, baseUrl).href);
+ } else if (!href.startsWith('#') && !href.startsWith('mailto:')) {
+ // Relative URL not starting with '/', append to base URL
+ links.push(new URL(href, baseUrl).href);
+ } else if (href.startsWith('mailto:')) {
+ // mailto: links, add as is
+ links.push(href);
+ }
+ // Fragment-only links (#) are ignored
+ } catch (error) {
+ logger.error(`Failed to construct URL for href: ${href} with base: ${baseUrl}`, { error });
+ }
+ }
+ });
+
+ // Remove duplicates and return
+ return [...new Set(links)];
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
new file mode 100644
index 000000000..f3fe5a5b1
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
@@ -0,0 +1,132 @@
+import { load } from "cheerio";
+import { Document } from "../../../controllers/v1/types";
+import { Meta } from "..";
+
+export function extractMetadata(meta: Meta, html: string): Document["metadata"] {
+ let title: string | undefined = undefined;
+ let description: string | undefined = undefined;
+ let language: string | undefined = undefined;
+ let keywords: string | undefined = undefined;
+ let robots: string | undefined = undefined;
+ let ogTitle: string | undefined = undefined;
+ let ogDescription: string | undefined = undefined;
+ let ogUrl: string | undefined = undefined;
+ let ogImage: string | undefined = undefined;
+ let ogAudio: string | undefined = undefined;
+ let ogDeterminer: string | undefined = undefined;
+ let ogLocale: string | undefined = undefined;
+ let ogLocaleAlternate: string[] | undefined = undefined;
+ let ogSiteName: string | undefined = undefined;
+ let ogVideo: string | undefined = undefined;
+ let dcTermsCreated: string | undefined = undefined;
+ let dcDateCreated: string | undefined = undefined;
+ let dcDate: string | undefined = undefined;
+ let dcTermsType: string | undefined = undefined;
+ let dcType: string | undefined = undefined;
+ let dcTermsAudience: string | undefined = undefined;
+ let dcTermsSubject: string | undefined = undefined;
+ let dcSubject: string | undefined = undefined;
+ let dcDescription: string | undefined = undefined;
+ let dcTermsKeywords: string | undefined = undefined;
+ let modifiedTime: string | undefined = undefined;
+ let publishedTime: string | undefined = undefined;
+ let articleTag: string | undefined = undefined;
+ let articleSection: string | undefined = undefined;
+ const customMetadata: Record<string, string | string[]> = {};
+
+ const soup = load(html);
+
+ try {
+ title = soup("title").text() || undefined;
+ description = soup('meta[name="description"]').attr("content") || undefined;
+
+ // Assuming the language is part of the URL as per the regex pattern
+ language = soup('html').attr('lang') || undefined;
+
+ keywords = soup('meta[name="keywords"]').attr("content") || undefined;
+ robots = soup('meta[name="robots"]').attr("content") || undefined;
+ ogTitle = soup('meta[property="og:title"]').attr("content") || undefined;
+ ogDescription = soup('meta[property="og:description"]').attr("content") || undefined;
+ ogUrl = soup('meta[property="og:url"]').attr("content") || undefined;
+ ogImage = soup('meta[property="og:image"]').attr("content") || undefined;
+ ogAudio = soup('meta[property="og:audio"]').attr("content") || undefined;
+ ogDeterminer = soup('meta[property="og:determiner"]').attr("content") || undefined;
+ ogLocale = soup('meta[property="og:locale"]').attr("content") || undefined;
+ ogLocaleAlternate = soup('meta[property="og:locale:alternate"]').map((i, el) => soup(el).attr("content")).get() || undefined;
+ ogSiteName = soup('meta[property="og:site_name"]').attr("content") || undefined;
+ ogVideo = soup('meta[property="og:video"]').attr("content") || undefined;
+ articleSection = soup('meta[name="article:section"]').attr("content") || undefined;
+ articleTag = soup('meta[name="article:tag"]').attr("content") || undefined;
+ publishedTime = soup('meta[property="article:published_time"]').attr("content") || undefined;
+ modifiedTime = soup('meta[property="article:modified_time"]').attr("content") || undefined;
+ dcTermsKeywords = soup('meta[name="dcterms.keywords"]').attr("content") || undefined;
+ dcDescription = soup('meta[name="dc.description"]').attr("content") || undefined;
+ dcSubject = soup('meta[name="dc.subject"]').attr("content") || undefined;
+ dcTermsSubject = soup('meta[name="dcterms.subject"]').attr("content") || undefined;
+ dcTermsAudience = soup('meta[name="dcterms.audience"]').attr("content") || undefined;
+ dcType = soup('meta[name="dc.type"]').attr("content") || undefined;
+ dcTermsType = soup('meta[name="dcterms.type"]').attr("content") || undefined;
+ dcDate = soup('meta[name="dc.date"]').attr("content") || undefined;
+ dcDateCreated = soup('meta[name="dc.date.created"]').attr("content") || undefined;
+ dcTermsCreated = soup('meta[name="dcterms.created"]').attr("content") || undefined;
+
+ try {
+ // Extract all meta tags for custom metadata
+ soup("meta").each((i, elem) => {
+ try {
+ const name = soup(elem).attr("name") || soup(elem).attr("property");
+ const content = soup(elem).attr("content");
+
+ if (name && content) {
+ if (customMetadata[name] === undefined) {
+ customMetadata[name] = content;
+ } else if (Array.isArray(customMetadata[name])) {
+ (customMetadata[name] as string[]).push(content);
+ } else {
+ customMetadata[name] = [customMetadata[name] as string, content];
+ }
+ }
+ } catch (error) {
+ meta.logger.error(`Error extracting custom metadata (in)`, { error });
+ }
+ });
+ } catch (error) {
+ meta.logger.error(`Error extracting custom metadata`, { error });
+ }
+ } catch (error) {
+ meta.logger.error(`Error extracting metadata`, { error });
+ }
+
+ return {
+ title,
+ description,
+ language,
+ keywords,
+ robots,
+ ogTitle,
+ ogDescription,
+ ogUrl,
+ ogImage,
+ ogAudio,
+ ogDeterminer,
+ ogLocale,
+ ogLocaleAlternate,
+ ogSiteName,
+ ogVideo,
+ dcTermsCreated,
+ dcDateCreated,
+ dcDate,
+ dcTermsType,
+ dcType,
+ dcTermsAudience,
+ dcTermsSubject,
+ dcSubject,
+ dcDescription,
+ dcTermsKeywords,
+ modifiedTime,
+ publishedTime,
+ articleTag,
+ articleSection,
+ ...customMetadata,
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
new file mode 100644
index 000000000..436cadb1e
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -0,0 +1,144 @@
+import { Logger } from "winston";
+import { z, ZodError } from "zod";
+import { v4 as uuid } from "uuid";
+import * as Sentry from "@sentry/node";
+import FormData from "form-data";
+
+export type RobustFetchParams<Schema extends z.Schema<any>> = {
+ url: string;
+ logger: Logger,
+ method: "GET" | "POST" | "DELETE" | "PUT";
+ body?: any;
+ headers?: Record<string, string>;
+ schema?: Schema;
+ dontParseResponse?: boolean;
+ ignoreResponse?: boolean;
+ ignoreFailure?: boolean;
+ requestId?: string;
+ tryCount?: number;
+};
+
+export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer<Schema>>({
+ url,
+ logger,
+ method = "GET",
+ body,
+ headers,
+ schema,
+ ignoreResponse = false,
+ ignoreFailure = false,
+ requestId = uuid(),
+ tryCount = 1,
+}: RobustFetchParams<Schema>): Promise<Output> {
+ const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount };
+
+ logger.debug("Sending request...", { params, requestId });
+
+ let request: Response;
+ try {
+ request = await fetch(url, {
+ method,
+ headers: {
+ ...(body instanceof FormData
+ ? body.getHeaders()
+ : body !== undefined ? ({
+ "Content-Type": "application/json",
+ }) : {}),
+ ...(headers !== undefined ? headers : {}),
+ },
+ ...(body instanceof FormData ? ({
+ body: body.getBuffer(),
+ }) : body !== undefined ? ({
+ body: JSON.stringify(body),
+ }) : {}),
+ });
+ } catch (error) {
+ if (!ignoreFailure) {
+ Sentry.captureException(error);
+ if (tryCount > 1) {
+ logger.debug("Request failed, trying " + (tryCount - 1) + " more times", { params, error, requestId });
+ return await robustFetch({
+ ...params,
+ requestId,
+ tryCount: tryCount - 1,
+ });
+ } else {
+ logger.debug("Request failed", { params, error, requestId });
+ throw new Error("Request failed", {
+ cause: {
+ params, requestId, error,
+ },
+ });
+ }
+
+ } else {
+ return null as Output;
+ }
+ }
+
+ if (ignoreResponse === true) {
+ return null as Output;
+ }
+
+ const response = {
+ status: request.status,
+ headers: request.headers,
+ body: await request.text(), // NOTE: can this throw an exception?
+ };
+
+ if (request.status >= 300) {
+ if (tryCount > 1) {
+ logger.debug("Request sent failure status, trying " + (tryCount - 1) + " more times", { params, request, response, requestId });
+ return await robustFetch({
+ ...params,
+ requestId,
+ tryCount: tryCount - 1,
+ });
+ } else {
+ logger.debug("Request sent failure status", { params, request, response, requestId });
+ throw new Error("Request sent failure status", {
+ cause: {
+ params, request, response, requestId,
+ },
+ });
+ }
+ }
+
+ let data: Output;
+ try {
+ data = JSON.parse(response.body);
+ } catch (error) {
+ logger.debug("Request sent malformed JSON", { params, request, response, requestId });
+ throw new Error("Request sent malformed JSON", {
+ cause: {
+ params, request, response, requestId,
+ },
+ });
+ }
+
+ if (schema) {
+ try {
+ data = schema.parse(data);
+ } catch (error) {
+ if (error instanceof ZodError) {
+ logger.debug("Response does not match provided schema", { params, request, response, requestId, error, schema });
+ throw new Error("Response does not match provided schema", {
+ cause: {
+ params, request, response, requestId,
+ error, schema,
+ }
+ });
+ } else {
+ logger.debug("Parsing response with provided schema failed", { params, request, response, requestId, error, schema });
+ throw new Error("Parsing response with provided schema failed", {
+ cause: {
+ params, request, response, requestId,
+ error, schema
+ }
+ });
+ }
+ }
+ }
+
+ return data;
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
new file mode 100644
index 000000000..cfed6a646
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -0,0 +1,111 @@
+// TODO: refactor
+
+import { AnyNode, Cheerio, load } from "cheerio";
+import { ScrapeOptions } from "../../../controllers/v1/types";
+
+const excludeNonMainTags = [
+ "header",
+ "footer",
+ "nav",
+ "aside",
+ ".header",
+ ".top",
+ ".navbar",
+ "#header",
+ ".footer",
+ ".bottom",
+ "#footer",
+ ".sidebar",
+ ".side",
+ ".aside",
+ "#sidebar",
+ ".modal",
+ ".popup",
+ "#modal",
+ ".overlay",
+ ".ad",
+ ".ads",
+ ".advert",
+ "#ad",
+ ".lang-selector",
+ ".language",
+ "#language-selector",
+ ".social",
+ ".social-media",
+ ".social-links",
+ "#social",
+ ".menu",
+ ".navigation",
+ "#nav",
+ ".breadcrumbs",
+ "#breadcrumbs",
+ "#search-form",
+ ".search",
+ "#search",
+ ".share",
+ "#share",
+ ".widget",
+ "#widget",
+ ".cookie",
+ "#cookie"
+];
+
+export const removeUnwantedElements = (
+ html: string,
+ scrapeOptions: ScrapeOptions
+) => {
+ const soup = load(html);
+
+ if (scrapeOptions.includeTags && scrapeOptions.includeTags.length > 0) {
+ // Create a new root element to hold the tags to keep
+ const newRoot = load("<div></div>")("div");
+ scrapeOptions.includeTags.forEach((tag) => {
+ soup(tag).each((_, element) => {
+ newRoot.append(soup(element).clone());
+ });
+ });
+ return newRoot.html() ?? "";
+ }
+
+ soup("script, style, noscript, meta, head").remove();
+
+ if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.length > 0) {
+ scrapeOptions.excludeTags.forEach((tag) => {
+ let elementsToRemove: Cheerio<AnyNode>;
+ if (tag.startsWith("*") && tag.endsWith("*")) {
+ let classMatch = false;
+
+ const regexPattern = new RegExp(tag.slice(1, -1), "i");
+ elementsToRemove = soup("*").filter((i, element) => {
+ if (element.type === "tag") {
+ const attributes = element.attribs;
+ const tagNameMatches = regexPattern.test(element.name);
+ const attributesMatch = Object.keys(attributes).some((attr) =>
+ regexPattern.test(`${attr}="${attributes[attr]}"`)
+ );
+ if (tag.startsWith("*.")) {
+ classMatch = Object.keys(attributes).some((attr) =>
+ regexPattern.test(`class="${attributes[attr]}"`)
+ );
+ }
+ return tagNameMatches || attributesMatch || classMatch;
+ }
+ return false;
+ });
+ } else {
+ elementsToRemove = soup(tag);
+ }
+ elementsToRemove.remove();
+ });
+ }
+
+ if (scrapeOptions.onlyMainContent) {
+ excludeNonMainTags.forEach((tag) => {
+ const elementsToRemove = soup(tag);
+ elementsToRemove.remove();
+ });
+ }
+
+ const cleanedHtml = soup.html();
+ return cleanedHtml;
+};
diff --git a/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts b/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts
new file mode 100644
index 000000000..7ce4f66e0
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts
@@ -0,0 +1,51 @@
+import { InternalOptions } from "..";
+import { ScrapeOptions } from "../../../controllers/v1/types";
+
+export type UrlSpecificParams = {
+ scrapeOptions: Partial<ScrapeOptions>,
+ internalOptions: Partial<InternalOptions>,
+};
+
+// const docsParam: UrlSpecificParams = {
+// scrapeOptions: { waitFor: 2000 },
+// internalOptions: {},
+// }
+
+export const urlSpecificParams: Record<string, UrlSpecificParams> = {
+ // "support.greenpay.me": docsParam,
+ // "docs.pdw.co": docsParam,
+ // "developers.notion.com": docsParam,
+ // "docs2.hubitat.com": docsParam,
+ // "rsseau.fr": docsParam,
+ // "help.salesforce.com": docsParam,
+ // "scrapethissite.com": {
+ // scrapeOptions: {},
+ // internalOptions: { forceEngine: "fetch" },
+ // },
+ // "eonhealth.com": {
+ // defaultScraper: "fire-engine",
+ // params: {
+ // fireEngineOptions: {
+ // mobileProxy: true,
+ // method: "get",
+ // engine: "request",
+ // },
+ // },
+ // },
+ // "notion.com": {
+ // scrapeOptions: { waitFor: 2000 },
+ // internalOptions: { forceEngine: "fire-engine;playwright" }
+ // },
+ // "developer.apple.com": {
+ // scrapeOptions: { waitFor: 2000 },
+ // internalOptions: { forceEngine: "fire-engine;playwright" }
+ // },
+ "digikey.com": {
+ scrapeOptions: {},
+ internalOptions: { forceEngine: "fire-engine;tlsclient" }
+ },
+ "lorealparis.hu": {
+ scrapeOptions: {},
+ internalOptions: { forceEngine: "fire-engine;tlsclient" },
+ }
+};
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
new file mode 100644
index 000000000..ba5d5f69a
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -0,0 +1,388 @@
+import "dotenv/config";
+
+import { scrapeURL } from ".";
+import { scrapeOptions } from "../../controllers/v1/types";
+import { Engine } from "./engines";
+
+const testEngines: (Engine | undefined)[] = [
+ undefined,
+ "fire-engine;chrome-cdp",
+ "fire-engine;playwright",
+ "fire-engine;tlsclient",
+ "scrapingbee",
+ "scrapingbeeLoad",
+ "fetch",
+];
+
+const testEnginesScreenshot: (Engine | undefined)[] = [
+ undefined,
+ "fire-engine;chrome-cdp",
+ "fire-engine;playwright",
+ "scrapingbee",
+ "scrapingbeeLoad",
+];
+
+describe("Standalone scrapeURL tests", () => {
+ describe.each(testEngines)("Engine %s", (forceEngine: Engine | undefined) => {
+ it("Basic scrape", async () => {
+ const out = await scrapeURL("test:scrape-basic", "https://www.roastmywebsite.ai/", scrapeOptions.parse({}), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).not.toHaveProperty("content");
+ expect(out.document).toHaveProperty("markdown");
+ expect(out.document).toHaveProperty("metadata");
+ expect(out.document).not.toHaveProperty("html");
+ expect(out.document.markdown).toContain("_Roast_");
+ expect(out.document.metadata.error).toBeUndefined();
+ expect(out.document.metadata.title).toBe("Roast My Website");
+ expect(out.document.metadata.description).toBe(
+ "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌢️"
+ );
+ expect(out.document.metadata.keywords).toBe(
+ "Roast My Website,Roast,Website,GitHub,Firecrawl"
+ );
+ expect(out.document.metadata.robots).toBe("follow, index");
+ expect(out.document.metadata.ogTitle).toBe("Roast My Website");
+ expect(out.document.metadata.ogDescription).toBe(
+ "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌢️"
+ );
+ expect(out.document.metadata.ogUrl).toBe(
+ "https://www.roastmywebsite.ai"
+ );
+ expect(out.document.metadata.ogImage).toBe(
+ "https://www.roastmywebsite.ai/og.png"
+ );
+ expect(out.document.metadata.ogLocaleAlternate).toStrictEqual([]);
+ expect(out.document.metadata.ogSiteName).toBe("Roast My Website");
+ expect(out.document.metadata.sourceURL).toBe(
+ "https://www.roastmywebsite.ai/"
+ );
+ expect(out.document.metadata.statusCode).toBe(200);
+ }
+
+ }, 30000);
+
+ it("Scrape with formats markdown and html", async () => {
+ const out = await scrapeURL("test:scrape-formats-markdown-html", "https://roastmywebsite.ai", scrapeOptions.parse({
+ formats: ["markdown", "html"],
+ }), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty("markdown");
+ expect(out.document).toHaveProperty("html");
+ expect(out.document).toHaveProperty("metadata");
+ expect(out.document.markdown).toContain("_Roast_");
+ expect(out.document.html).toContain("<h1");
+ expect(out.document.metadata.statusCode).toBe(200);
+ expect(out.document.metadata.error).toBeUndefined();
+ }
+
+ }, 30000);
+
+ it("Scrape with onlyMainContent disabled", async () => {
+ const out = await scrapeURL("test:scrape-onlyMainContent-false", "https://www.scrapethissite.com/", scrapeOptions.parse({
+ onlyMainContent: false,
+ }), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty("markdown");
+ expect(out.document).toHaveProperty("metadata");
+ expect(out.document).not.toHaveProperty("html");
+ expect(out.document.markdown).toContain("[FAQ](/faq/)"); // .nav
+ expect(out.document.markdown).toContain("Hartley Brody 2023"); // #footer
+ }
+ }, 30000);
+
+ it("Scrape with excludeTags", async () => {
+ const out = await scrapeURL("test:scrape-excludeTags", "https://www.scrapethissite.com/", scrapeOptions.parse({
+ onlyMainContent: false,
+ excludeTags: ['.nav', '#footer', 'strong'],
+ }), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty("markdown");
+ expect(out.document).toHaveProperty("metadata");
+ expect(out.document).not.toHaveProperty("html");
+ expect(out.document.markdown).not.toContain("Hartley Brody 2023");
+ expect(out.document.markdown).not.toContain("[FAQ](/faq/)");
+ }
+ }, 30000);
+
+ it("Scrape of a page with 400 status code", async () => {
+ const out = await scrapeURL("test:scrape-400", "https://httpstat.us/400", scrapeOptions.parse({}), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('markdown');
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.statusCode).toBe(400);
+ }
+ }, 30000);
+
+ it("Scrape of a page with 401 status code", async () => {
+ const out = await scrapeURL("test:scrape-401", "https://httpstat.us/401", scrapeOptions.parse({}), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('markdown');
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.statusCode).toBe(401);
+ }
+ }, 30000);
+
+ it("Scrape of a page with 403 status code", async () => {
+ const out = await scrapeURL("test:scrape-403", "https://httpstat.us/403", scrapeOptions.parse({}), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('markdown');
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.statusCode).toBe(403);
+ }
+ }, 30000);
+
+ it("Scrape of a page with 404 status code", async () => {
+ const out = await scrapeURL("test:scrape-404", "https://httpstat.us/404", scrapeOptions.parse({}), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('markdown');
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.statusCode).toBe(404);
+ }
+ }, 30000);
+
+ it("Scrape of a page with 405 status code", async () => {
+ const out = await scrapeURL("test:scrape-405", "https://httpstat.us/405", scrapeOptions.parse({}), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('markdown');
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.statusCode).toBe(405);
+ }
+ }, 30000);
+
+ it("Scrape of a page with 500 status code", async () => {
+ const out = await scrapeURL("test:scrape-500", "https://httpstat.us/500", scrapeOptions.parse({}), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('markdown');
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.statusCode).toBe(500);
+ }
+ }, 30000);
+
+ it("Scrape a redirected page", async () => {
+ const out = await scrapeURL("test:scrape-redirect", "https://scrapethissite.com/", scrapeOptions.parse({}), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('markdown');
+ expect(out.document.markdown).toContain("Explore Sandbox");
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.sourceURL).toBe("https://scrapethissite.com/");
+ expect(out.document.metadata.url).toBe("https://www.scrapethissite.com/");
+ expect(out.document.metadata.statusCode).toBe(200);
+ expect(out.document.metadata.error).toBeUndefined();
+ }
+ }, 30000);
+ });
+
+ describe.each(testEnginesScreenshot)("Screenshot on engine %s", (forceEngine: Engine | undefined) => {
+ it("Scrape with screenshot", async () => {
+ const out = await scrapeURL("test:scrape-screenshot", "https://www.scrapethissite.com/", scrapeOptions.parse({
+ formats: ["screenshot"],
+ }), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('screenshot');
+ expect(typeof out.document.screenshot).toBe("string");
+ expect(out.document.screenshot!.startsWith("https://service.firecrawl.dev/storage/v1/object/public/media/"));
+ // TODO: attempt to fetch screenshot
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.statusCode).toBe(200);
+ expect(out.document.metadata.error).toBeUndefined();
+ }
+ }, 30000);
+
+ it("Scrape with full-page screenshot", async () => {
+ const out = await scrapeURL("test:scrape-screenshot-fullPage", "https://www.scrapethissite.com/", scrapeOptions.parse({
+ formats: ["screenshot@fullPage"],
+ }), { forceEngine });
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('screenshot');
+ expect(typeof out.document.screenshot).toBe("string");
+ expect(out.document.screenshot!.startsWith("https://service.firecrawl.dev/storage/v1/object/public/media/"));
+ // TODO: attempt to fetch screenshot
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.statusCode).toBe(200);
+ expect(out.document.metadata.error).toBeUndefined();
+ }
+ }, 30000);
+ });
+
+ it("Scrape of a PDF file", async () => {
+ const out = await scrapeURL("test:scrape-pdf", "https://arxiv.org/pdf/astro-ph/9301001.pdf", scrapeOptions.parse({}));
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.markdown).toContain('Broad Line Radio Galaxy');
+ expect(out.document.metadata.statusCode).toBe(200);
+ expect(out.document.metadata.error).toBeUndefined();
+ }
+ }, 60000);
+
+ it("Scrape a DOCX file", async () => {
+ const out = await scrapeURL("test:scrape-docx", "https://nvca.org/wp-content/uploads/2019/06/NVCA-Model-Document-Stock-Purchase-Agreement.docx", scrapeOptions.parse({}));
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.markdown).toContain('SERIES A PREFERRED STOCK PURCHASE AGREEMENT');
+ expect(out.document.metadata.statusCode).toBe(200);
+ expect(out.document.metadata.error).toBeUndefined();
+ }
+ }, 60000)
+
+ it("LLM extract with prompt and schema", async () => {
+ const out = await scrapeURL("test:llm-extract-prompt-schema", "https://firecrawl.dev", scrapeOptions.parse({
+ formats: ["extract"],
+ extract: {
+ prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
+ schema: {
+ type: "object",
+ properties: {
+ company_mission: { type: "string" },
+ supports_sso: { type: "boolean" },
+ is_open_source: { type: "boolean" },
+ },
+ required: ["company_mission", "supports_sso", "is_open_source"],
+ additionalProperties: false,
+ },
+ },
+ }));
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty("extract");
+ expect(out.document.extract).toHaveProperty("company_mission");
+ expect(out.document.extract).toHaveProperty("supports_sso");
+ expect(out.document.extract).toHaveProperty("is_open_source");
+ expect(typeof out.document.extract.company_mission).toBe("string");
+ expect(out.document.extract.supports_sso).toBe(false);
+ expect(out.document.extract.is_open_source).toBe(true);
+ }
+ }, 120000)
+
+ it("LLM extract with schema only", async () => {
+ const out = await scrapeURL("test:llm-extract-schema", "https://firecrawl.dev", scrapeOptions.parse({
+ formats: ["extract"],
+ extract: {
+ schema: {
+ type: "object",
+ properties: {
+ company_mission: { type: "string" },
+ supports_sso: { type: "boolean" },
+ is_open_source: { type: "boolean" },
+ },
+ required: ["company_mission", "supports_sso", "is_open_source"],
+ additionalProperties: false,
+ },
+ },
+ }));
+
+ expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty("extract");
+ expect(out.document.extract).toHaveProperty("company_mission");
+ expect(out.document.extract).toHaveProperty("supports_sso");
+ expect(out.document.extract).toHaveProperty("is_open_source");
+ expect(typeof out.document.extract.company_mission).toBe("string");
+ expect(out.document.extract.supports_sso).toBe(false);
+ expect(out.document.extract.is_open_source).toBe(true);
+ }
+ }, 120000)
+
+ test.concurrent.each(new Array(100).fill(0).map((_, i) => i))("Concurrent scrape #%i", async (i) => {
+ const url = "https://www.scrapethissite.com/?i=" + i;
+ const id = "test:concurrent:" + url;
+ const out = await scrapeURL(id, url, scrapeOptions.parse({}));
+
+ const replacer = (key: string, value: any) => {
+ if (value instanceof Error) {
+ return {
+ ...value,
+ message: value.message,
+ name: value.name,
+ cause: value.cause,
+ stack: value.stack,
+ }
+ } else {
+ return value;
+ }
+ }
+
+ // verify that log collection works properly while concurrency is happening
+ expect(out.logs.length).toBeGreaterThan(0);
+ const weirdLogs = out.logs.filter(x => x.scrapeId !== id);
+ if (weirdLogs.length > 0) {
+ console.warn(JSON.stringify(weirdLogs, replacer));
+ }
+ expect(weirdLogs.length).toBe(0);
+
+ if (!out.success) console.error(JSON.stringify(out, replacer));
+ expect(out.success).toBe(true);
+
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty('markdown');
+ expect(out.document).toHaveProperty('metadata');
+ expect(out.document.metadata.error).toBeUndefined();
+ expect(out.document.metadata.statusCode).toBe(200);
+ }
+ }, 30000);
+})
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
new file mode 100644
index 000000000..b1d354e61
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -0,0 +1,130 @@
+import { parseMarkdown } from "../../../lib/html-to-markdown";
+import { Meta } from "..";
+import { Document } from "../../../controllers/v1/types";
+import { removeUnwantedElements } from "../lib/removeUnwantedElements";
+import { extractLinks } from "../lib/extractLinks";
+import { extractMetadata } from "../lib/extractMetadata";
+import { performLLMExtract } from "./llmExtract";
+import { uploadScreenshot } from "./uploadScreenshot";
+import { removeBase64Images } from "./removeBase64Images";
+
+export type Transformer = (meta: Meta, document: Document) => Document | Promise<Document>;
+
+export function deriveMetadataFromRawHTML(meta: Meta, document: Document): Document {
+ if (document.rawHtml === undefined) {
+ throw new Error("rawHtml is undefined -- this transformer is being called out of order");
+ }
+
+ document.metadata = {
+ ...extractMetadata(meta, document.rawHtml),
+ ...document.metadata,
+ };
+ return document;
+}
+
+export function deriveHTMLFromRawHTML(meta: Meta, document: Document): Document {
+ if (document.rawHtml === undefined) {
+ throw new Error("rawHtml is undefined -- this transformer is being called out of order");
+ }
+
+ document.html = removeUnwantedElements(document.rawHtml, meta.options);
+ return document;
+}
+
+export async function deriveMarkdownFromHTML(_meta: Meta, document: Document): Promise<Document> {
+ if (document.html === undefined) {
+ throw new Error("html is undefined -- this transformer is being called out of order");
+ }
+
+ document.markdown = await parseMarkdown(document.html);
+ return document;
+}
+
+export function deriveLinksFromHTML(meta: Meta, document: Document): Document {
+ // Only derive if the formats has links
+ if (meta.options.formats.includes("links")) {
+ if (document.html === undefined) {
+ throw new Error("html is undefined -- this transformer is being called out of order");
+ }
+
+ document.links = extractLinks(document.html, meta.url);
+ }
+
+ return document;
+}
+
+export function coerceFieldsToFormats(meta: Meta, document: Document): Document {
+ const formats = new Set(meta.options.formats);
+
+ if (!formats.has("markdown") && document.markdown !== undefined) {
+ delete document.markdown;
+ } else if (formats.has("markdown") && document.markdown === undefined) {
+ meta.logger.warn("Request had format: markdown, but there was no markdown field in the result.");
+ }
+
+ if (!formats.has("rawHtml") && document.rawHtml !== undefined) {
+ delete document.rawHtml;
+ } else if (formats.has("rawHtml") && document.rawHtml === undefined) {
+ meta.logger.warn("Request had format: rawHtml, but there was no rawHtml field in the result.");
+ }
+
+ if (!formats.has("html") && document.html !== undefined) {
+ delete document.html;
+ } else if (formats.has("html") && document.html === undefined) {
+ meta.logger.warn("Request had format: html, but there was no html field in the result.");
+ }
+
+ if (!formats.has("screenshot") && !formats.has("screenshot@fullPage") && document.screenshot !== undefined) {
+ meta.logger.warn("Removed screenshot from Document because it wasn't in formats -- this is very wasteful and indicates a bug.");
+ delete document.screenshot;
+ } else if ((formats.has("screenshot") || formats.has("screenshot@fullPage")) && document.screenshot === undefined) {
+ meta.logger.warn("Request had format: screenshot / screenshot@fullPage, but there was no screenshot field in the result.");
+ }
+
+ if (!formats.has("links") && document.links !== undefined) {
+ meta.logger.warn("Removed links from Document because it wasn't in formats -- this is wasteful and indicates a bug.");
+ delete document.links;
+ } else if (formats.has("links") && document.links === undefined) {
+ meta.logger.warn("Request had format: links, but there was no links field in the result.");
+ }
+
+ if (!formats.has("extract") && document.extract !== undefined) {
+ meta.logger.warn("Removed extract from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.");
+ delete document.extract;
+ } else if (formats.has("extract") && document.extract === undefined) {
+ meta.logger.warn("Request had format: extract, but there was no extract field in the result.");
+ }
+
+ if (meta.options.actions === undefined || meta.options.actions.length === 0) {
+ delete document.actions;
+ }
+
+ return document;
+}
+
+// TODO: allow some of these to run in parallel
+export const transformerStack: Transformer[] = [
+ deriveHTMLFromRawHTML,
+ deriveMarkdownFromHTML,
+ deriveLinksFromHTML,
+ deriveMetadataFromRawHTML,
+ uploadScreenshot,
+ performLLMExtract,
+ coerceFieldsToFormats,
+ removeBase64Images,
+];
+
+export async function executeTransformers(meta: Meta, document: Document): Promise<Document> {
+ for (const transformer of transformerStack) {
+ const _meta = {
+ ...meta,
+ logger: meta.logger.child({ method: "executeTransformers/" + transformer.name }),
+ };
+ meta.logger.debug("Executing transformer " + transformer.name + "...", { document });
+ const start = Date.now();
+ document = await transformer(_meta, document);
+ meta.logger.debug("Finished executing transformer " + transformer.name + " (" + (Date.now() - start) + "ms)", { document });
+ }
+
+ return document;
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
new file mode 100644
index 000000000..e7ffc4b22
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -0,0 +1,131 @@
+import OpenAI from "openai";
+import { encoding_for_model } from "@dqbd/tiktoken";
+import { TiktokenModel } from "@dqbd/tiktoken";
+import { Document, ExtractOptions } from "../../../controllers/v1/types";
+import { Logger } from "winston";
+import { EngineResultsTracker, Meta } from "..";
+
+const maxTokens = 32000;
+const modifier = 4;
+
+export class LLMRefusalError extends Error {
+ public refusal: string;
+ public results: EngineResultsTracker | undefined;
+
+ constructor(refusal: string) {
+ super("LLM refused to extract the website's content")
+ this.refusal = refusal;
+ }
+}
+
+function normalizeSchema(x: any): any {
+ if (x && x.type === "object") {
+ return {
+ ...x,
+ properties: Object.fromEntries(Object.entries(x.properties).map(([k, v]) => [k, normalizeSchema(v)])),
+ additionalProperties: false,
+ }
+ } else {
+ return x;
+ }
+}
+
+async function generateOpenAICompletions(logger: Logger, document: Document, options: ExtractOptions): Promise<Document> {
+ const openai = new OpenAI();
+ const model: TiktokenModel = (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini";
+
+ if (document.markdown === undefined) {
+ throw new Error("document.markdown is undefined -- this is unexpected");
+ }
+
+ let extractionContent = document.markdown;
+
+ // count number of tokens
+ let numTokens = 0;
+ const encoder = encoding_for_model(model as TiktokenModel);
+ try {
+ // Encode the message into tokens
+ const tokens = encoder.encode(extractionContent);
+
+ // Return the number of tokens
+ numTokens = tokens.length;
+ } catch (error) {
+ logger.warn("Calculating num tokens of string failed", { error, extractionContent });
+
+ extractionContent = extractionContent.slice(0, maxTokens * modifier);
+
+ const warning = "Failed to derive number of LLM tokens the extraction might use -- the input has been automatically trimmed to the maximum number of tokens (" + maxTokens + ") we support.";
+ document.warning = document.warning === undefined ? warning : " " + warning;
+ } finally {
+ // Free the encoder resources after use
+ encoder.free();
+ }
+
+ if (numTokens > maxTokens) {
+ // trim the document to the maximum number of tokens, tokens != characters
+ extractionContent = extractionContent.slice(0, maxTokens * modifier);
+
+ const warning = "The extraction content would have used more tokens (" + numTokens + ") than the maximum we allow (" + maxTokens + "). -- the input has been automatically trimmed.";
+ document.warning = document.warning === undefined ? warning : " " + warning;
+ }
+
+ let schema = options.schema;
+ if (schema && schema.type === "array") {
+ schema = {
+ type: "object",
+ properties: {
+ items: options.schema,
+ },
+ required: ["items"],
+ additionalProperties: false,
+ };
+ }
+
+ schema = normalizeSchema(schema);
+
+ const jsonCompletion = await openai.beta.chat.completions.parse({
+ model,
+ messages: [
+ {
+ role: "system",
+ content: options.systemPrompt,
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: extractionContent }],
+ },
+ {
+ role: "user",
+ content: options.prompt !== undefined
+ ? `Transform the above content into structured JSON output based on the following user request: ${options.prompt}`
+ : "Transform the above content into structured JSON output.",
+ },
+ ],
+ response_format: options.schema ? {
+ type: "json_schema",
+ json_schema: {
+ name: "websiteContent",
+ schema: schema,
+ strict: true,
+ }
+ } : { type: "json_object" },
+ });
+
+ if (jsonCompletion.choices[0].message.refusal !== null) {
+ throw new LLMRefusalError(jsonCompletion.choices[0].message.refusal);
+ }
+
+ document.extract = jsonCompletion.choices[0].message.parsed;
+ if (options.schema && options.schema.type === "array") {
+ document.extract = document.extract?.items;
+ }
+ return document;
+}
+
+export async function performLLMExtract(meta: Meta, document: Document): Promise<Document> {
+ if (meta.options.formats.includes("extract")) {
+ document = await generateOpenAICompletions(meta.logger.child({ method: "performLLMExtract/generateOpenAICompletions" }), document, meta.options.extract!);
+ }
+
+ return document;
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/removeBase64Images.ts b/apps/api/src/scraper/scrapeURL/transformers/removeBase64Images.ts
new file mode 100644
index 000000000..92628f8af
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/removeBase64Images.ts
@@ -0,0 +1,11 @@
+import { Meta } from "..";
+import { Document } from "../../../controllers/v1/types";
+
+const regex = /(!\[.*?\])\(data:image\/.*?;base64,.*?\)/g;
+
+export function removeBase64Images(meta: Meta, document: Document): Document {
+ if (meta.options.removeBase64Images && document.markdown !== undefined) {
+ document.markdown = document.markdown.replace(regex, '$1(<Base64-Image-Removed>)');
+ }
+ return document;
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts b/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts
new file mode 100644
index 000000000..4c3fc2b4a
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts
@@ -0,0 +1,26 @@
+// This file is an exception to the "no supabase in scrapeURL" rule,
+// and it makes me sad. - mogery
+
+import { supabase_service } from "../../../services/supabase";
+import { Meta } from "..";
+import { Document } from "../../../controllers/v1/types";
+
+export function uploadScreenshot(meta: Meta, document: Document): Document {
+ if (process.env.USE_DB_AUTHENTICATION === "true" && document.screenshot !== undefined && document.screenshot.startsWith("data:")) {
+ meta.logger.debug("Uploading screenshot to Supabase...");
+
+ const fileName = `screenshot-${crypto.randomUUID()}.png`;
+
+ supabase_service.storage
+ .from("media")
+ .upload(fileName, Buffer.from(document.screenshot.split(",")[1], "base64"), {
+ cacheControl: "3600",
+ upsert: false,
+ contentType: document.screenshot.split(":")[1].split(";")[0],
+ });
+
+ document.screenshot = `https://service.firecrawl.dev/storage/v1/object/public/media/${encodeURIComponent(fileName)}`;
+ }
+
+ return document;
+}
diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.ts
index 1186c8a2c..361a9ebc0 100644
--- a/apps/api/src/search/fireEngine.ts
+++ b/apps/api/src/search/fireEngine.ts
@@ -2,7 +2,7 @@ import axios from "axios";
import dotenv from "dotenv";
import { SearchResult } from "../../src/lib/entities";
import * as Sentry from "@sentry/node";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
dotenv.config();
@@ -53,7 +53,7 @@ export async function fireEngineMap(
return [];
}
} catch (error) {
- Logger.error(error);
+ logger.error(error);
Sentry.captureException(error);
return [];
}
diff --git a/apps/api/src/search/googlesearch.ts b/apps/api/src/search/googlesearch.ts
index 0e247702d..596628290 100644
--- a/apps/api/src/search/googlesearch.ts
+++ b/apps/api/src/search/googlesearch.ts
@@ -2,7 +2,7 @@ import axios from 'axios';
import * as cheerio from 'cheerio';
import * as querystring from 'querystring';
import { SearchResult } from '../../src/lib/entities';
-import { Logger } from '../../src/lib/logger';
+import { logger } from '../../src/lib/logger';
const _useragent_list = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0',
@@ -18,7 +18,7 @@ function get_useragent(): string {
return _useragent_list[Math.floor(Math.random() * _useragent_list.length)];
}
-async function _req(term: string, results: number, lang: string, country: string, start: number, proxies: any, timeout: number, tbs: string = null, filter: string = null) {
+async function _req(term: string, results: number, lang: string, country: string, start: number, proxies: any, timeout: number, tbs: string | undefined = undefined, filter: string | undefined = undefined) {
const params = {
"q": term,
"num": results, // Number of results to return
@@ -52,8 +52,8 @@ async function _req(term: string, results: number, lang: string, country: string
-export async function googleSearch(term: string, advanced = false, num_results = 7, tbs = null, filter = null, lang = "en", country = "us", proxy = null, sleep_interval = 0, timeout = 5000, ) :Promise<SearchResult[]> {
- let proxies = null;
+export async function googleSearch(term: string, advanced = false, num_results = 7, tbs = undefined as string | undefined, filter = undefined as string | undefined, lang = "en", country = "us", proxy = undefined as string | undefined, sleep_interval = 0, timeout = 5000, ) :Promise<SearchResult[]> {
+ let proxies: any = null;
if (proxy) {
if (proxy.startsWith("https")) {
proxies = {"https": proxy};
@@ -97,7 +97,7 @@ export async function googleSearch(term: string, advanced = false, num_results =
await new Promise(resolve => setTimeout(resolve, sleep_interval * 1000));
} catch (error) {
if (error.message === 'Too many requests') {
- Logger.warn('Too many requests, breaking the loop');
+ logger.warn('Too many requests, breaking the loop');
break;
}
throw error;
@@ -108,7 +108,7 @@ export async function googleSearch(term: string, advanced = false, num_results =
}
}
if (attempts >= maxAttempts) {
- Logger.warn('Max attempts reached, breaking the loop');
+ logger.warn('Max attempts reached, breaking the loop');
}
return results
}
diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.ts
index 3bcb85d2b..5899af87d 100644
--- a/apps/api/src/search/index.ts
+++ b/apps/api/src/search/index.ts
@@ -1,4 +1,4 @@
-import { Logger } from "../../src/lib/logger";
+import { logger } from "../../src/lib/logger";
import { SearchResult } from "../../src/lib/entities";
import { googleSearch } from "./googlesearch";
import { fireEngineMap } from "./fireEngine";
@@ -9,12 +9,12 @@ export async function search({
query,
advanced = false,
num_results = 7,
- tbs = null,
- filter = null,
+ tbs = undefined,
+ filter = undefined,
lang = "en",
country = "us",
location = undefined,
- proxy = null,
+ proxy = undefined,
sleep_interval = 0,
timeout = 5000,
}: {
@@ -31,24 +31,24 @@ export async function search({
timeout?: number;
}): Promise<SearchResult[]> {
try {
- if (process.env.SEARCHAPI_API_KEY) {
- return await searchapi_search(query, {
+ if (process.env.SERPER_API_KEY) {
+ return await serper_search(query, {
num_results,
tbs,
filter,
lang,
country,
- location
+ location,
});
}
- if (process.env.SERPER_API_KEY) {
- return await serper_search(query, {
+ if (process.env.SEARCHAPI_API_KEY) {
+ return await searchapi_search(query, {
num_results,
tbs,
filter,
lang,
country,
- location,
+ location
});
}
return await googleSearch(
@@ -64,7 +64,7 @@ export async function search({
timeout
);
} catch (error) {
- Logger.error(`Error in search function: ${error}`);
+ logger.error(`Error in search function: ${error}`);
return [];
}
}
diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts
index cb953e2e8..826bb18ee 100644
--- a/apps/api/src/services/alerts/index.ts
+++ b/apps/api/src/services/alerts/index.ts
@@ -1,4 +1,4 @@
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import { getScrapeQueue } from "../queue-service";
import { sendSlackWebhook } from "./slack";
@@ -10,13 +10,13 @@ export async function checkAlerts() {
process.env.ALERT_NUM_ACTIVE_JOBS &&
process.env.ALERT_NUM_WAITING_JOBS
) {
- Logger.info("Initializing alerts");
+ logger.info("Initializing alerts");
const checkActiveJobs = async () => {
try {
const scrapeQueue = getScrapeQueue();
const activeJobs = await scrapeQueue.getActiveCount();
if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
- Logger.warn(
+ logger.warn(
`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`
);
sendSlackWebhook(
@@ -24,12 +24,12 @@ export async function checkAlerts() {
true
);
} else {
- Logger.info(
+ logger.info(
`Number of active jobs is under ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`
);
}
} catch (error) {
- Logger.error(`Failed to check active jobs: ${error}`);
+ logger.error(`Failed to check active jobs: ${error}`);
}
};
@@ -38,7 +38,7 @@ export async function checkAlerts() {
const waitingJobs = await scrapeQueue.getWaitingCount();
if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
- Logger.warn(
+ logger.warn(
`Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`
);
sendSlackWebhook(
@@ -57,6 +57,6 @@ export async function checkAlerts() {
// setInterval(checkAll, 10000); // Run every
}
} catch (error) {
- Logger.error(`Failed to initialize alerts: ${error}`);
+ logger.error(`Failed to initialize alerts: ${error}`);
}
}
diff --git a/apps/api/src/services/alerts/slack.ts b/apps/api/src/services/alerts/slack.ts
index 0fa75693f..1eac53439 100644
--- a/apps/api/src/services/alerts/slack.ts
+++ b/apps/api/src/services/alerts/slack.ts
@@ -1,5 +1,5 @@
import axios from "axios";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
export async function sendSlackWebhook(
message: string,
@@ -17,8 +17,8 @@ export async function sendSlackWebhook(
"Content-Type": "application/json",
},
});
- Logger.log("Webhook sent successfully:", response.data);
+ logger.info("Webhook sent successfully:", response.data);
} catch (error) {
- Logger.debug(`Error sending webhook: ${error}`);
+ logger.debug(`Error sending webhook: ${error}`);
}
}
diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.ts
index 5f3183216..1659a110e 100644
--- a/apps/api/src/services/billing/auto_charge.ts
+++ b/apps/api/src/services/billing/auto_charge.ts
@@ -1,6 +1,6 @@
// Import necessary dependencies and types
import { AuthCreditUsageChunk } from "../../controllers/v1/types";
-import { getACUC, setCachedACUC } from "../../controllers/auth";
+import { getACUC } from "../../controllers/auth";
import { redlock } from "../redlock";
import { supabase_service } from "../supabase";
import { createPaymentIntent } from "./stripe";
@@ -9,7 +9,7 @@ import { sendNotification } from "../notification/email_notification";
import { NotificationType } from "../../types";
import { deleteKey, getValue, setValue } from "../redis";
import { sendSlackWebhook } from "../alerts/slack";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
// Define the number of credits to be added during auto-recharge
const AUTO_RECHARGE_CREDITS = 1000;
@@ -32,7 +32,7 @@ export async function autoCharge(
// Another check to prevent race conditions, double charging - cool down of 5 minutes
const cooldownValue = await getValue(cooldownKey);
if (cooldownValue) {
- Logger.info(`Auto-recharge for team ${chunk.team_id} is in cooldown period`);
+ logger.info(`Auto-recharge for team ${chunk.team_id} is in cooldown period`);
return {
success: false,
message: "Auto-recharge is in cooldown period",
@@ -59,7 +59,7 @@ export async function autoCharge(
.single();
if (customersError) {
- Logger.error(`Error fetching customer data: ${customersError}`);
+ logger.error(`Error fetching customer data: ${customersError}`);
return {
success: false,
message: "Error fetching customer data",
@@ -124,7 +124,7 @@ export async function autoCharge(
false,
process.env.SLACK_ADMIN_WEBHOOK_URL
).catch((error) => {
- Logger.debug(`Error sending slack notification: ${error}`);
+ logger.debug(`Error sending slack notification: ${error}`);
});
// Set cooldown for 1 hour
@@ -138,7 +138,7 @@ export async function autoCharge(
chunk: {...chunk, remaining_credits: chunk.remaining_credits + AUTO_RECHARGE_CREDITS},
};
} else {
- Logger.error("No Stripe customer ID found for user");
+ logger.error("No Stripe customer ID found for user");
return {
success: false,
message: "No Stripe customer ID found for user",
@@ -147,7 +147,7 @@ export async function autoCharge(
};
}
} else {
- Logger.error("No sub_user_id found in chunk");
+ logger.error("No sub_user_id found in chunk");
return {
success: false,
message: "No sub_user_id found in chunk",
@@ -165,7 +165,7 @@ export async function autoCharge(
});
} catch (error) {
- Logger.error(`Failed to acquire lock for auto-recharge: ${error}`);
+ logger.error(`Failed to acquire lock for auto-recharge: ${error}`);
return {
success: false,
message: "Failed to acquire lock for auto-recharge",
diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts
index 3c43f5a01..8558d8ba6 100644
--- a/apps/api/src/services/billing/credit_billing.ts
+++ b/apps/api/src/services/billing/credit_billing.ts
@@ -2,7 +2,7 @@ import { NotificationType } from "../../types";
import { withAuth } from "../../lib/withAuth";
import { sendNotification } from "../notification/email_notification";
import { supabase_service } from "../supabase";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import * as Sentry from "@sentry/node";
import { AuthCreditUsageChunk } from "../../controllers/v1/types";
import { getACUC, setCachedACUC } from "../../controllers/auth";
@@ -16,22 +16,14 @@ const FREE_CREDITS = 500;
/**
* If you do not know the subscription_id in the current context, pass subscription_id as undefined.
*/
-export async function billTeam(
- team_id: string,
- subscription_id: string | null | undefined,
- credits: number
-) {
- return withAuth(supaBillTeam)(team_id, subscription_id, credits);
+export async function billTeam(team_id: string, subscription_id: string | null | undefined, credits: number) {
+ return withAuth(supaBillTeam, { success: true, message: "No DB, bypassed." })(team_id, subscription_id, credits);
}
-export async function supaBillTeam(
- team_id: string,
- subscription_id: string,
- credits: number
-) {
+export async function supaBillTeam(team_id: string, subscription_id: string | null | undefined, credits: number) {
if (team_id === "preview") {
return { success: true, message: "Preview team, no credits used" };
}
- Logger.info(`Billing team ${team_id} for ${credits} credits`);
+ logger.info(`Billing team ${team_id} for ${credits} credits`);
const { data, error } = await supabase_service.rpc("bill_team", {
_team_id: team_id,
@@ -42,7 +34,7 @@ export async function supaBillTeam(
if (error) {
Sentry.captureException(error);
- Logger.error("Failed to bill team: " + JSON.stringify(error));
+ logger.error("Failed to bill team: " + JSON.stringify(error));
return;
}
@@ -62,33 +54,24 @@ export async function supaBillTeam(
})();
}
-export async function checkTeamCredits(
- chunk: AuthCreditUsageChunk,
- team_id: string,
- credits: number
-): Promise<{ success: boolean; message: string; remainingCredits: number; chunk: AuthCreditUsageChunk }> {
- const result = await withAuth(supaCheckTeamCredits)(chunk, team_id, credits);
- return {
- success: result.success,
- message: result.message,
- remainingCredits: result.remainingCredits,
- chunk: chunk // Ensure chunk is always returned
- };
+export type CheckTeamCreditsResponse = {
+ success: boolean,
+ message: string,
+ remainingCredits: number,
+ chunk?: AuthCreditUsageChunk,
+}
+
+export async function checkTeamCredits(chunk: AuthCreditUsageChunk | null, team_id: string, credits: number): Promise<CheckTeamCreditsResponse> {
+ return withAuth(supaCheckTeamCredits, { success: true, message: "No DB, bypassed", remainingCredits: Infinity })(chunk, team_id, credits);
}
// if team has enough credits for the operation, return true, else return false
-export async function supaCheckTeamCredits(
- chunk: AuthCreditUsageChunk,
- team_id: string,
- credits: number
-) {
+export async function supaCheckTeamCredits(chunk: AuthCreditUsageChunk | null, team_id: string, credits: number): Promise<CheckTeamCreditsResponse> {
// WARNING: chunk will be null if team_id is preview -- do not perform operations on it under ANY circumstances - mogery
if (team_id === "preview") {
- return {
- success: true,
- message: "Preview team, no credits used",
- remainingCredits: Infinity,
- };
+ return { success: true, message: "Preview team, no credits used", remainingCredits: Infinity };
+ } else if (chunk === null) {
+ throw new Error("NULL ACUC passed to supaCheckTeamCredits");
}
const creditsWillBeUsed = chunk.adjusted_credits_used + credits;
diff --git a/apps/api/src/services/billing/issue_credits.ts b/apps/api/src/services/billing/issue_credits.ts
index 6b34b2edc..ce84db1bb 100644
--- a/apps/api/src/services/billing/issue_credits.ts
+++ b/apps/api/src/services/billing/issue_credits.ts
@@ -1,9 +1,9 @@
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import { supabase_service } from "../supabase";
export async function issueCredits(team_id: string, credits: number) {
// Add an entry to supabase coupons
- const { data, error } = await supabase_service.from("coupons").insert({
+ const { error } = await supabase_service.from("coupons").insert({
team_id: team_id,
credits: credits,
status: "active",
@@ -12,7 +12,7 @@ export async function issueCredits(team_id: string, credits: number) {
});
if (error) {
- Logger.error(`Error adding coupon: ${error}`);
+ logger.error(`Error adding coupon: ${error}`);
return false;
}
diff --git a/apps/api/src/services/billing/stripe.ts b/apps/api/src/services/billing/stripe.ts
index e459d3e3c..db482e915 100644
--- a/apps/api/src/services/billing/stripe.ts
+++ b/apps/api/src/services/billing/stripe.ts
@@ -1,4 +1,4 @@
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import Stripe from "stripe";
const stripe = new Stripe(process.env.STRIPE_SECRET_KEY ?? "");
@@ -18,7 +18,7 @@ export async function createPaymentIntent(
try {
const defaultPaymentMethod = await getCustomerDefaultPaymentMethod(customer_id);
if (!defaultPaymentMethod) {
- Logger.error(`No default payment method found for customer: ${customer_id}`);
+ logger.error(`No default payment method found for customer: ${customer_id}`);
return { return_status: "failed", charge_id: "" };
}
const paymentIntent = await stripe.paymentIntents.create({
@@ -33,21 +33,21 @@ export async function createPaymentIntent(
});
if (paymentIntent.status === "succeeded") {
- Logger.info(`Payment succeeded for team: ${team_id}`);
+ logger.info(`Payment succeeded for team: ${team_id}`);
return { return_status: "succeeded", charge_id: paymentIntent.id };
} else if (
paymentIntent.status === "requires_action" ||
paymentIntent.status === "processing" ||
paymentIntent.status === "requires_capture"
) {
- Logger.warn(`Payment requires further action for team: ${team_id}`);
+ logger.warn(`Payment requires further action for team: ${team_id}`);
return { return_status: "requires_action", charge_id: paymentIntent.id };
} else {
- Logger.error(`Payment failed for team: ${team_id}`);
+ logger.error(`Payment failed for team: ${team_id}`);
return { return_status: "failed", charge_id: paymentIntent.id };
}
} catch (error) {
- Logger.error(
+ logger.error(
`Failed to create or confirm PaymentIntent for team: ${team_id}`
);
console.error(error);
diff --git a/apps/api/src/services/idempotency/create.ts b/apps/api/src/services/idempotency/create.ts
index 291e77d91..f29fc70fd 100644
--- a/apps/api/src/services/idempotency/create.ts
+++ b/apps/api/src/services/idempotency/create.ts
@@ -1,6 +1,6 @@
import { Request } from "express";
import { supabase_service } from "../supabase";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
export async function createIdempotencyKey(
req: Request,
@@ -15,7 +15,7 @@ export async function createIdempotencyKey(
.insert({ key: idempotencyKey });
if (error) {
- Logger.error(`Failed to create idempotency key: ${error}`);
+ logger.error(`Failed to create idempotency key: ${error}`);
throw error;
}
diff --git a/apps/api/src/services/idempotency/validate.ts b/apps/api/src/services/idempotency/validate.ts
index 4d58a31d6..ca3acab16 100644
--- a/apps/api/src/services/idempotency/validate.ts
+++ b/apps/api/src/services/idempotency/validate.ts
@@ -1,7 +1,7 @@
import { Request } from "express";
import { supabase_service } from "../supabase";
import { validate as isUuid } from 'uuid';
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
export async function validateIdempotencyKey(
req: Request,
@@ -14,7 +14,7 @@ export async function validateIdempotencyKey(
// Ensure idempotencyKey is treated as a string
const key = Array.isArray(idempotencyKey) ? idempotencyKey[0] : idempotencyKey;
if (!isUuid(key)) {
- Logger.debug("Invalid idempotency key provided in the request headers.");
+ logger.debug("Invalid idempotency key provided in the request headers.");
return false;
}
@@ -24,7 +24,7 @@ export async function validateIdempotencyKey(
.eq("key", idempotencyKey);
if (error) {
- Logger.error(`Error validating idempotency key: ${error}`);
+ logger.error(`Error validating idempotency key: ${error}`);
}
if (!data || data.length === 0) {
diff --git a/apps/api/src/services/logging/crawl_log.ts b/apps/api/src/services/logging/crawl_log.ts
index 3850e05b9..0160828e2 100644
--- a/apps/api/src/services/logging/crawl_log.ts
+++ b/apps/api/src/services/logging/crawl_log.ts
@@ -1,5 +1,5 @@
import { supabase_service } from "../supabase";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import { configDotenv } from "dotenv";
configDotenv();
@@ -16,7 +16,7 @@ export async function logCrawl(job_id: string, team_id: string) {
},
]);
} catch (error) {
- Logger.error(`Error logging crawl job to supabase:\n${error}`);
+ logger.error(`Error logging crawl job to supabase:\n${error}`);
}
}
}
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index c2aedd13a..2a813b21f 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -3,7 +3,7 @@ import { supabase_service } from "../supabase";
import { FirecrawlJob } from "../../types";
import { posthog } from "../posthog";
import "dotenv/config";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import { configDotenv } from "dotenv";
configDotenv();
@@ -16,11 +16,11 @@ export async function logJob(job: FirecrawlJob) {
// Redact any pages that have an authorization header
if (
- job.pageOptions &&
- job.pageOptions.headers &&
- job.pageOptions.headers["Authorization"]
+ job.scrapeOptions &&
+ job.scrapeOptions.headers &&
+ job.scrapeOptions.headers["Authorization"]
) {
- job.pageOptions.headers["Authorization"] = "REDACTED";
+ job.scrapeOptions.headers["Authorization"] = "REDACTED";
job.docs = [{ content: "REDACTED DUE TO AUTHORIZATION HEADER", html: "REDACTED DUE TO AUTHORIZATION HEADER" }];
}
@@ -38,9 +38,8 @@ export async function logJob(job: FirecrawlJob) {
mode: job.mode,
url: job.url,
crawler_options: job.crawlerOptions,
- page_options: job.pageOptions,
+ page_options: job.scrapeOptions,
origin: job.origin,
- extractor_options: job.extractor_options,
num_tokens: job.num_tokens,
retry: !!job.retry,
crawl_id: job.crawl_id,
@@ -63,9 +62,8 @@ export async function logJob(job: FirecrawlJob) {
mode: job.mode,
url: job.url,
crawler_options: job.crawlerOptions,
- page_options: job.pageOptions,
+ page_options: job.scrapeOptions,
origin: job.origin,
- extractor_options: job.extractor_options,
num_tokens: job.num_tokens,
retry: job.retry,
},
@@ -75,9 +73,9 @@ export async function logJob(job: FirecrawlJob) {
}
}
if (error) {
- Logger.error(`Error logging job: ${error.message}`);
+ logger.error(`Error logging job: ${error.message}`);
}
} catch (error) {
- Logger.error(`Error logging job: ${error.message}`);
+ logger.error(`Error logging job: ${error.message}`);
}
}
diff --git a/apps/api/src/services/logging/scrape_log.ts b/apps/api/src/services/logging/scrape_log.ts
index fbe416530..441b3894e 100644
--- a/apps/api/src/services/logging/scrape_log.ts
+++ b/apps/api/src/services/logging/scrape_log.ts
@@ -2,7 +2,7 @@ import "dotenv/config";
import { ScrapeLog } from "../../types";
import { supabase_service } from "../supabase";
import { PageOptions } from "../../lib/entities";
-import { Logger } from "../../lib/logger";
+import { logger } from "../../lib/logger";
import { configDotenv } from "dotenv";
configDotenv();
@@ -12,7 +12,7 @@ export async function logScrape(
) {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
if (!useDbAuthentication) {
- Logger.debug("Skipping logging scrape to Supabase");
+ logger.debug("Skipping logging scrape to Supabase");
return;
}
try {
@@ -47,9 +47,9 @@ export async function logScrape(
]);
if (error) {
- Logger.error(`Error logging proxy:\n${JSON.stringify(error)}`);
+ logger.error(`Error logging proxy:\n${JSON.stringify(error)}`);
}
} catch (error) {
- Logger.error(`Error logging proxy:\n${JSON.stringify(error)}`);
+ logger.error(`Error logging proxy:\n${JSON.stringify(error)}`);
}
}
diff --git a/apps/api/src/services/logtail.ts b/apps/api/src/services/logtail.ts
deleted file mode 100644
index d9af3c7ac..000000000
--- a/apps/api/src/services/logtail.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-import { Logtail } from "@logtail/node";
-import "dotenv/config";
-import { Logger } from "../lib/logger";
-
-// A mock Logtail class to handle cases where LOGTAIL_KEY is not provided
-class MockLogtail {
- info(message: string, context?: Record<string, any>): void {
- Logger.debug(`${message} - ${context}`);
- }
- error(message: string, context: Record<string, any> = {}): void {
- Logger.error(`${message} - ${context}`);
- }
-}
-
-// Using the actual Logtail class if LOGTAIL_KEY exists, otherwise using the mock class
-// Additionally, print a warning to the terminal if LOGTAIL_KEY is not provided
-export const logtail = process.env.LOGTAIL_KEY ? new Logtail(process.env.LOGTAIL_KEY) : (() => {
- Logger.warn("LOGTAIL_KEY is not provided - your events will not be logged. Using MockLogtail as a fallback. see logtail.ts for more.");
- return new MockLogtail();
-})();
diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.ts
index 001f164a3..5aa95b307 100644
--- a/apps/api/src/services/notification/email_notification.ts
+++ b/apps/api/src/services/notification/email_notification.ts
@@ -2,7 +2,7 @@ import { supabase_service } from "../supabase";
import { withAuth } from "../../lib/withAuth";
import { Resend } from "resend";
import { NotificationType } from "../../types";
-import { Logger } from "../../../src/lib/logger";
+import { logger } from "../../../src/lib/logger";
import { sendSlackWebhook } from "../alerts/slack";
import { getNotificationString } from "./notification_string";
import { AuthCreditUsageChunk } from "../../controllers/v1/types";
@@ -37,12 +37,12 @@ const emailTemplates: Record<
export async function sendNotification(
team_id: string,
notificationType: NotificationType,
- startDateString: string,
- endDateString: string,
+ startDateString: string | null,
+ endDateString: string | null,
chunk: AuthCreditUsageChunk,
bypassRecentChecks: boolean = false
) {
- return withAuth(sendNotificationInternal)(
+ return withAuth(sendNotificationInternal, undefined)(
team_id,
notificationType,
startDateString,
@@ -68,11 +68,11 @@ export async function sendEmailNotification(
});
if (error) {
- Logger.debug(`Error sending email: ${error}`);
+ logger.debug(`Error sending email: ${error}`);
return { success: false };
}
} catch (error) {
- Logger.debug(`Error sending email (2): ${error}`);
+ logger.debug(`Error sending email (2): ${error}`);
return { success: false };
}
}
@@ -80,8 +80,8 @@ export async function sendEmailNotification(
export async function sendNotificationInternal(
team_id: string,
notificationType: NotificationType,
- startDateString: string,
- endDateString: string,
+ startDateString: string | null,
+ endDateString: string | null,
chunk: AuthCreditUsageChunk,
bypassRecentChecks: boolean = false
): Promise<{ success: boolean }> {
@@ -101,7 +101,7 @@ export async function sendNotificationInternal(
.gte("sent_date", fifteenDaysAgo.toISOString());
if (error) {
- Logger.debug(`Error fetching notifications: ${error}`);
+ logger.debug(`Error fetching notifications: ${error}`);
return { success: false };
}
@@ -120,7 +120,7 @@ export async function sendNotificationInternal(
.lte("sent_date", endDateString);
if (recentError) {
- Logger.debug(`Error fetching recent notifications: ${recentError.message}`);
+ logger.debug(`Error fetching recent notifications: ${recentError.message}`);
return { success: false };
}
@@ -138,7 +138,7 @@ export async function sendNotificationInternal(
.eq("team_id", team_id);
if (emailsError) {
- Logger.debug(`Error fetching emails: ${emailsError}`);
+ logger.debug(`Error fetching emails: ${emailsError}`);
return { success: false };
}
@@ -162,12 +162,12 @@ export async function sendNotificationInternal(
false,
process.env.SLACK_ADMIN_WEBHOOK_URL
).catch((error) => {
- Logger.debug(`Error sending slack notification: ${error}`);
+ logger.debug(`Error sending slack notification: ${error}`);
});
}
if (insertError) {
- Logger.debug(`Error inserting notification record: ${insertError}`);
+ logger.debug(`Error inserting notification record: ${insertError}`);
return { success: false };
}
diff --git a/apps/api/src/services/posthog.ts b/apps/api/src/services/posthog.ts
index a74198836..e3a01353f 100644
--- a/apps/api/src/services/posthog.ts
+++ b/apps/api/src/services/posthog.ts
@@ -1,9 +1,9 @@
import { PostHog } from 'posthog-node';
import "dotenv/config";
-import { Logger } from '../../src/lib/logger';
+import { logger } from '../../src/lib/logger';
-export default function PostHogClient() {
- const posthogClient = new PostHog(process.env.POSTHOG_API_KEY, {
+export default function PostHogClient(apiKey: string) {
+ const posthogClient = new PostHog(apiKey, {
host: process.env.POSTHOG_HOST,
flushAt: 1,
flushInterval: 0
@@ -18,9 +18,9 @@ class MockPostHog {
// Using the actual PostHog class if POSTHOG_API_KEY exists, otherwise using the mock class
// Additionally, print a warning to the terminal if POSTHOG_API_KEY is not provided
export const posthog = process.env.POSTHOG_API_KEY
- ? PostHogClient()
+ ? PostHogClient(process.env.POSTHOG_API_KEY)
: (() => {
- Logger.warn(
+ logger.warn(
"POSTHOG_API_KEY is not provided - your events will not be logged. Using MockPostHog as a fallback. See posthog.ts for more."
);
return new MockPostHog();
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index e912ef42e..e4a5ace88 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -88,7 +88,7 @@ export async function addScrapeJobs(
await Promise.all(jobs.map(job => addScrapeJob(job.data, job.opts, job.opts.jobId, job.opts.priority)));
}
-export function waitForJob(jobId: string, timeout: number) {
+export function waitForJob<T = unknown>(jobId: string, timeout: number): Promise<T> {
return new Promise((resolve, reject) => {
const start = Date.now();
const int = setInterval(async () => {
@@ -99,7 +99,7 @@ export function waitForJob(jobId: string, timeout: number) {
const state = await getScrapeQueue().getJobState(jobId);
if (state === "completed") {
clearInterval(int);
- resolve((await getScrapeQueue().getJob(jobId)).returnvalue);
+ resolve((await getScrapeQueue().getJob(jobId))!.returnvalue);
} else if (state === "failed") {
// console.log("failed", (await getScrapeQueue().getJob(jobId)).failedReason);
const job = await getScrapeQueue().getJob(jobId);
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 14dddebe2..e6432a3fa 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -1,10 +1,10 @@
import { Queue } from "bullmq";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
import IORedis from "ioredis";
let scrapeQueue: Queue;
-export const redisConnection = new IORedis(process.env.REDIS_URL, {
+export const redisConnection = new IORedis(process.env.REDIS_URL!, {
maxRetriesPerRequest: null,
});
@@ -37,7 +37,7 @@ export function getScrapeQueue() {
// }
// }
);
- Logger.info("Web scraper queue created");
+ logger.info("Web scraper queue created");
}
return scrapeQueue;
}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 0e176ddb9..4b9780dfb 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -7,13 +7,11 @@ import {
redisConnection,
scrapeQueueName,
} from "./queue-service";
-import { logtail } from "./logtail";
import { startWebScraperPipeline } from "../main/runWebScraper";
import { callWebhook } from "./webhook";
import { logJob } from "./logging/log_job";
-import { initSDK } from "@hyperdx/node-opentelemetry";
import { Job, Queue } from "bullmq";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
import { Worker } from "bullmq";
import systemMonitor from "./system-monitor";
import { v4 as uuidv4 } from "uuid";
@@ -28,25 +26,19 @@ import {
} from "../lib/crawl-redis";
import { StoredCrawl } from "../lib/crawl-redis";
import { addScrapeJob } from "./queue-jobs";
-import { supabaseGetJobById } from "../../src/lib/supabase-jobs";
import {
addJobPriority,
deleteJobPriority,
getJobPriority,
} from "../../src/lib/job-priority";
import { PlanType, RateLimiterMode } from "../types";
-import { getJobs } from "../../src/controllers/v1/crawl-status";
+import { getJobs } from "..//controllers/v1/crawl-status";
import { configDotenv } from "dotenv";
+import { scrapeOptions } from "../controllers/v1/types";
import { getRateLimiterPoints } from "./rate-limiter";
import { cleanOldConcurrencyLimitEntries, pushConcurrencyLimitActiveJob, removeConcurrencyLimitActiveJob, takeConcurrencyLimitedJob } from "../lib/concurrency-limit";
configDotenv();
-if (process.env.ENV === "production") {
- initSDK({
- consoleCapture: true,
- additionalInstrumentations: [],
- });
-}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
@@ -63,9 +55,9 @@ const connectionMonitorInterval =
Number(process.env.CONNECTION_MONITOR_INTERVAL) || 10;
const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;
-const processJobInternal = async (token: string, job: Job) => {
+const processJobInternal = async (token: string, job: Job & { id: string }) => {
const extendLockInterval = setInterval(async () => {
- Logger.info(`πŸ‚ Worker extending lock on job ${job.id}`);
+ logger.info(`πŸ‚ Worker extending lock on job ${job.id}`);
await job.extendLock(token, jobLockExtensionTime);
}, jobLockExtendInterval);
@@ -73,13 +65,18 @@ const processJobInternal = async (token: string, job: Job) => {
let err = null;
try {
const result = await processJob(job, token);
- try {
- if (job.data.crawl_id && process.env.USE_DB_AUTHENTICATION === "true") {
- await job.moveToCompleted(null, token, false);
- } else {
- await job.moveToCompleted(result.docs, token, false);
- }
- } catch (e) {}
+ if (result.success) {
+ try {
+ if (job.data.crawl_id && process.env.USE_DB_AUTHENTICATION === "true") {
+ await job.moveToCompleted(null, token, false);
+ } else {
+ await job.moveToCompleted(result.document, token, false);
+ }
+ } catch (e) {}
+ } else {
+ await job.moveToFailed((result as any).error, token, false);
+ }
+
} catch (error) {
console.log("Job failed, error:", error);
Sentry.captureException(error);
@@ -233,8 +230,8 @@ const workerFun = async (
workerFun(getScrapeQueue(), processJobInternal);
-async function processJob(job: Job, token: string) {
- Logger.info(`πŸ‚ Worker taking job ${job.id}`);
+async function processJob(job: Job & { id: string }, token: string) {
+ logger.info(`πŸ‚ Worker taking job ${job.id}`);
// Check if the job URL is researchhub and block it immediately
// TODO: remove this once solve the root issue
@@ -245,15 +242,14 @@ async function processJob(job: Job, token: string) {
job.data.url.includes("youtube.com") ||
job.data.url.includes("microsoft.com"))
) {
- Logger.info(`πŸ‚ Blocking job ${job.id} with URL ${job.data.url}`);
+ logger.info(`πŸ‚ Blocking job ${job.id} with URL ${job.data.url}`);
const data = {
success: false,
- docs: [],
+ document: null,
project_id: job.data.project_id,
error:
"URL is blocked. Suspecious activity detected. Please contact [email protected] if you believe this is an error.",
};
- await job.moveToCompleted(data.docs, token, false);
return data;
}
@@ -266,49 +262,35 @@ async function processJob(job: Job, token: string) {
});
const start = Date.now();
- const { success, message, docs } = await startWebScraperPipeline({
+ const pipeline = await startWebScraperPipeline({
job,
token,
});
- // Better if we throw here so we capture with the correct error
- if (!success) {
- throw new Error(message);
+ if (!pipeline.success) {
+ // TODO: let's Not do this
+ throw pipeline.error;
}
+
const end = Date.now();
const timeTakenInSeconds = (end - start) / 1000;
- const rawHtml = docs[0] ? docs[0].rawHtml : "";
+ const doc = pipeline.document;
+
+ const rawHtml = doc.rawHtml ?? "";
const data = {
- success,
+ success: true,
result: {
- links: docs.map((doc) => {
- return {
- content: doc,
- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
- };
- }),
+ links: [{
+ content: doc,
+ source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",
+ }],
},
project_id: job.data.project_id,
- error: message /* etc... */,
- docs: job.data.concurrencyLimitHit ? docs.map(x => ({
- ...x,
- warning: "This scrape was throttled because you hit you concurrency limit." + (x.warning ? " " + x.warning : ""),
- })) : docs,
+ document: doc,
};
- // No idea what this does and when it is called.
- if (job.data.mode === "crawl" && !job.data.v1) {
- callWebhook(
- job.data.team_id,
- job.id as string,
- data,
- job.data.webhook,
- job.data.v1,
- job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
- );
- }
if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {
await callWebhook(
job.data.team_id,
@@ -322,32 +304,31 @@ async function processJob(job: Job, token: string) {
}
if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+
await logJob({
job_id: job.id as string,
- success: success,
- message: message,
- num_docs: docs.length,
- docs: docs,
+ success: true,
+ num_docs: 1,
+ docs: [doc],
time_taken: timeTakenInSeconds,
team_id: job.data.team_id,
mode: job.data.mode,
url: job.data.url,
- crawlerOptions: job.data.crawlerOptions,
- pageOptions: job.data.pageOptions,
+ crawlerOptions: sc.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
origin: job.data.origin,
crawl_id: job.data.crawl_id,
});
await addCrawlJobDone(job.data.crawl_id, job.id);
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
-
if (!job.data.sitemapped && job.data.crawlerOptions !== null) {
if (!sc.cancelled) {
const crawler = crawlToCrawler(job.data.crawl_id, sc);
const links = crawler.filterLinks(
- crawler.extractLinksFromHTML(rawHtml ?? "", sc.originUrl),
+ crawler.extractLinksFromHTML(rawHtml ?? "", sc.originUrl as string),
Infinity,
sc.crawlerOptions?.maxDepth ?? 10
);
@@ -371,10 +352,10 @@ async function processJob(job: Job, token: string) {
{
url: link,
mode: "single_urls",
- crawlerOptions: sc.crawlerOptions,
team_id: sc.team_id,
+ scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),
+ internalOptions: sc.internalOptions,
plan: job.data.plan,
- pageOptions: sc.pageOptions,
origin: job.data.origin,
crawl_id: job.data.crawl_id,
webhook: job.data.webhook,
@@ -409,15 +390,15 @@ async function processJob(job: Job, token: string) {
await logJob({
job_id: job.data.crawl_id,
success: jobStatus === "completed",
- message: sc.cancelled ? "Cancelled" : message,
+ message: sc.cancelled ? "Cancelled" : undefined,
num_docs: fullDocs.length,
docs: [],
time_taken: (Date.now() - sc.createdAt) / 1000,
team_id: job.data.team_id,
mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
- url: sc.originUrl,
+ url: sc.originUrl!,
+ scrapeOptions: sc.scrapeOptions,
crawlerOptions: sc.crawlerOptions,
- pageOptions: sc.pageOptions,
origin: job.data.origin,
});
@@ -432,7 +413,6 @@ async function processJob(job: Job, token: string) {
}),
},
project_id: job.data.project_id,
- error: message /* etc... */,
docs: fullDocs,
};
@@ -469,57 +449,44 @@ async function processJob(job: Job, token: string) {
await logJob({
job_id: job.data.crawl_id,
success: jobStatus === "completed",
- message: sc.cancelled ? "Cancelled" : message,
+ message: sc.cancelled ? "Cancelled" : undefined,
num_docs: jobIDs.length,
docs: [],
time_taken: (Date.now() - sc.createdAt) / 1000,
team_id: job.data.team_id,
+ scrapeOptions: sc.scrapeOptions,
mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
url: sc?.originUrl ?? (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
crawlerOptions: sc.crawlerOptions,
- pageOptions: sc.pageOptions,
origin: job.data.origin,
});
}
}
}
- Logger.info(`πŸ‚ Job done ${job.id}`);
+ logger.info(`πŸ‚ Job done ${job.id}`);
return data;
} catch (error) {
- Logger.error(`πŸ‚ Job errored ${job.id} - ${error}`);
+ logger.error(`πŸ‚ Job errored ${job.id} - ${error}`);
- if (!(error instanceof Error && error.message.includes("JSON parsing error(s): "))) {
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
- }
+ Sentry.captureException(error, {
+ data: {
+ job: job.id,
+ },
+ });
if (error instanceof CustomError) {
// Here we handle the error, then save the failed job
- Logger.error(error.message); // or any other error handling
-
- logtail.error("Custom error while ingesting", {
- job_id: job.id,
- error: error.message,
- dataIngestionJob: error.dataIngestionJob,
- });
+ logger.error(error.message); // or any other error handling
}
- Logger.error(error);
+ logger.error(error);
if (error.stack) {
- Logger.error(error.stack);
+ logger.error(error.stack);
}
- logtail.error("Overall error ingesting", {
- job_id: job.id,
- error: error.message,
- });
-
const data = {
success: false,
- docs: [],
+ document: null,
project_id: job.data.project_id,
error:
"Something went wrong... Contact [email protected] or try again." /* etc... */,
@@ -547,6 +514,8 @@ async function processJob(job: Job, token: string) {
// }
if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+
await logJob({
job_id: job.id as string,
success: false,
@@ -561,14 +530,12 @@ async function processJob(job: Job, token: string) {
team_id: job.data.team_id,
mode: job.data.mode,
url: job.data.url,
- crawlerOptions: job.data.crawlerOptions,
- pageOptions: job.data.pageOptions,
+ crawlerOptions: sc.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
origin: job.data.origin,
crawl_id: job.data.crawl_id,
});
- const sc = await getCrawl(job.data.crawl_id);
-
await logJob({
job_id: job.data.crawl_id,
success: false,
@@ -583,8 +550,8 @@ async function processJob(job: Job, token: string) {
team_id: job.data.team_id,
mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
url: sc ? sc.originUrl : job.data.url,
- crawlerOptions: sc ? sc.crawlerOptions : job.data.crawlerOptions,
- pageOptions: sc ? sc.pageOptions : job.data.pageOptions,
+ crawlerOptions: sc ? sc.crawlerOptions : undefined,
+ scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions,
origin: job.data.origin,
});
}
diff --git a/apps/api/src/services/rate-limiter.test.ts b/apps/api/src/services/rate-limiter.test.ts
index ba4a0a733..4052bfff4 100644
--- a/apps/api/src/services/rate-limiter.test.ts
+++ b/apps/api/src/services/rate-limiter.test.ts
@@ -345,7 +345,7 @@ describe("Rate Limiter Service", () => {
} catch (error) {
// expect remaining points to be 0
const res = await limiter.get("test-prefix:someToken");
- expect(res.remainingPoints).toBe(0);
+ expect(res?.remainingPoints).toBe(0);
}
});
diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts
index 83729229a..a708f6fb0 100644
--- a/apps/api/src/services/rate-limiter.ts
+++ b/apps/api/src/services/rate-limiter.ts
@@ -86,7 +86,7 @@ const RATE_LIMITS = {
};
export const redisRateLimitClient = new Redis(
- process.env.REDIS_RATE_LIMIT_URL
+ process.env.REDIS_RATE_LIMIT_URL!
)
const createRateLimiter = (keyPrefix, points) =>
diff --git a/apps/api/src/services/redis.ts b/apps/api/src/services/redis.ts
index 608037811..1bd83605a 100644
--- a/apps/api/src/services/redis.ts
+++ b/apps/api/src/services/redis.ts
@@ -1,15 +1,15 @@
import Redis from "ioredis";
import { redisRateLimitClient } from "./rate-limiter";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
// Listen to 'error' events to the Redis connection
redisRateLimitClient.on("error", (error) => {
try {
if (error.message === "ECONNRESET") {
- Logger.error("Connection to Redis Session Rate Limit Store timed out.");
+ logger.error("Connection to Redis Session Rate Limit Store timed out.");
} else if (error.message === "ECONNREFUSED") {
- Logger.error("Connection to Redis Session Rate Limit Store refused!");
- } else Logger.error(error);
+ logger.error("Connection to Redis Session Rate Limit Store refused!");
+ } else logger.error(error);
} catch (error) {}
});
@@ -17,15 +17,15 @@ redisRateLimitClient.on("error", (error) => {
redisRateLimitClient.on("reconnecting", (err) => {
try {
if (redisRateLimitClient.status === "reconnecting")
- Logger.info("Reconnecting to Redis Session Rate Limit Store...");
- else Logger.error("Error reconnecting to Redis Session Rate Limit Store.");
+ logger.info("Reconnecting to Redis Session Rate Limit Store...");
+ else logger.error("Error reconnecting to Redis Session Rate Limit Store.");
} catch (error) {}
});
// Listen to the 'connect' event to Redis
redisRateLimitClient.on("connect", (err) => {
try {
- if (!err) Logger.info("Connected to Redis Session Rate Limit Store!");
+ if (!err) logger.info("Connected to Redis Session Rate Limit Store!");
} catch (error) {}
});
diff --git a/apps/api/src/services/redlock.ts b/apps/api/src/services/redlock.ts
index 4ece058af..921a973a0 100644
--- a/apps/api/src/services/redlock.ts
+++ b/apps/api/src/services/redlock.ts
@@ -4,7 +4,7 @@ import Client from "ioredis";
export const redlock = new Redlock(
// You should have one client for each independent redis node
// or cluster.
- [new Client(process.env.REDIS_RATE_LIMIT_URL)],
+ [new Client(process.env.REDIS_RATE_LIMIT_URL!)],
{
// The expected clock drift; for more details see:
// http://redis.io/topics/distlock
diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts
index 176d3d4b4..072a501e9 100644
--- a/apps/api/src/services/sentry.ts
+++ b/apps/api/src/services/sentry.ts
@@ -1,10 +1,10 @@
// Import with `import * as Sentry from "@sentry/node"` if you are using ESM
import * as Sentry from "@sentry/node";
import { nodeProfilingIntegration } from "@sentry/profiling-node";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
if (process.env.SENTRY_DSN) {
- Logger.info("Setting up Sentry...");
+ logger.info("Setting up Sentry...");
Sentry.init({
dsn: process.env.SENTRY_DSN,
integrations: [
diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.ts
index 7636717e8..61f168367 100644
--- a/apps/api/src/services/supabase.ts
+++ b/apps/api/src/services/supabase.ts
@@ -1,5 +1,5 @@
import { createClient, SupabaseClient } from "@supabase/supabase-js";
-import { Logger } from "../lib/logger";
+import { logger } from "../lib/logger";
import { configDotenv } from "dotenv";
configDotenv();
@@ -14,12 +14,12 @@ class SupabaseService {
// Only initialize the Supabase client if both URL and Service Token are provided.
if (!useDbAuthentication) {
// Warn the user that Authentication is disabled by setting the client to null
- Logger.warn(
+ logger.warn(
"Authentication is disabled. Supabase client will not be initialized."
);
this.client = null;
} else if (!supabaseUrl || !supabaseServiceToken) {
- Logger.error(
+ logger.error(
"Supabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable"
);
} else {
diff --git a/apps/api/src/services/system-monitor.ts b/apps/api/src/services/system-monitor.ts
index c1ebeddfc..6a630701c 100644
--- a/apps/api/src/services/system-monitor.ts
+++ b/apps/api/src/services/system-monitor.ts
@@ -2,7 +2,7 @@ import si from 'systeminformation';
import { Mutex } from "async-mutex";
import os from 'os';
import fs from 'fs';
-import { Logger } from '../lib/logger';
+import { logger } from '../lib/logger';
const IS_KUBERNETES = process.env.IS_KUBERNETES === "true";
@@ -79,7 +79,7 @@ class SystemMonitor {
return memoryUsagePercentage;
} catch (error) {
- Logger.error(`Error calculating memory usage: ${error}`);
+ logger.error(`Error calculating memory usage: ${error}`);
return 0; // Fallback to 0% usage
}
}
@@ -134,7 +134,7 @@ class SystemMonitor {
throw new Error('No CPUs found in cpuset.cpus.effective');
}
} catch (error) {
- Logger.warn(`Unable to read cpuset.cpus.effective, defaulting to OS CPUs: ${error}`);
+ logger.warn(`Unable to read cpuset.cpus.effective, defaulting to OS CPUs: ${error}`);
cpus = os.cpus().map((cpu, index) => index);
}
return cpus.length;
@@ -184,7 +184,7 @@ class SystemMonitor {
return cpuUsagePercentage;
} catch (error) {
- Logger.error(`Error calculating CPU usage: ${error}`);
+ logger.error(`Error calculating CPU usage: ${error}`);
return 0; // Fallback to 0% usage
}
}
diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.ts
index 06e5649d2..620b68329 100644
--- a/apps/api/src/services/webhook.ts
+++ b/apps/api/src/services/webhook.ts
@@ -1,6 +1,5 @@
import axios from "axios";
-import { legacyDocumentConverter } from "../../src/controllers/v1/types";
-import { Logger } from "../../src/lib/logger";
+import { logger } from "../../src/lib/logger";
import { supabase_service } from "./supabase";
import { WebhookEventType } from "../types";
import { configDotenv } from "dotenv";
@@ -32,7 +31,7 @@ export const callWebhook = async (
.eq("team_id", teamId)
.limit(1);
if (error) {
- Logger.error(
+ logger.error(
`Error fetching webhook URL for team ID: ${teamId}, error: ${error.message}`
);
return null;
@@ -45,7 +44,11 @@ export const callWebhook = async (
webhookUrl = webhooksData[0].url;
}
- let dataToSend = [];
+ if (!webhookUrl) {
+ return null;
+ }
+
+ let dataToSend: any[] = [];
if (
data &&
data.result &&
@@ -55,7 +58,7 @@ export const callWebhook = async (
for (let i = 0; i < data.result.links.length; i++) {
if (v1) {
dataToSend.push(
- legacyDocumentConverter(data.result.links[i].content)
+ data.result.links[i].content
);
} else {
dataToSend.push({
@@ -94,7 +97,7 @@ export const callWebhook = async (
}
);
} catch (error) {
- Logger.error(
+ logger.error(
`Axios error (0) sending webhook for team ID: ${teamId}, error: ${error.message}`
);
}
@@ -125,13 +128,13 @@ export const callWebhook = async (
}
)
.catch((error) => {
- Logger.error(
+ logger.error(
`Axios error sending webhook for team ID: ${teamId}, error: ${error.message}`
);
});
}
} catch (error) {
- Logger.debug(
+ logger.debug(
`Error sending webhook for team ID: ${teamId}, error: ${error.message}`
);
}
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index eddaa0175..2da97bd1f 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -1,5 +1,6 @@
-import { AuthCreditUsageChunk } from "./controllers/v1/types";
-import { ExtractorOptions, Document, DocumentUrl } from "./lib/entities";
+import { AuthCreditUsageChunk, ScrapeOptions, Document as V1Document } from "./controllers/v1/types";
+import { ExtractorOptions, Document } from "./lib/entities";
+import { InternalOptions } from "./scraper/scrapeURL";
type Mode = "crawl" | "single_urls" | "sitemap";
@@ -24,9 +25,9 @@ export interface IngestResult {
export interface WebScraperOptions {
url: string;
mode: Mode;
- crawlerOptions: any;
- pageOptions: any;
- extractorOptions?: any;
+ crawlerOptions?: any;
+ scrapeOptions: ScrapeOptions;
+ internalOptions?: InternalOptions;
team_id: string;
plan: string;
origin?: string;
@@ -40,28 +41,28 @@ export interface WebScraperOptions {
export interface RunWebScraperParams {
url: string;
mode: Mode;
- crawlerOptions: any;
- pageOptions?: any;
- extractorOptions?: any;
- inProgress: (progress: any) => void;
- onSuccess: (result: any, mode: string) => void;
- onError: (error: Error) => void;
+ scrapeOptions: ScrapeOptions;
+ internalOptions?: InternalOptions;
+ // onSuccess: (result: V1Document, mode: string) => void;
+ // onError: (error: Error) => void;
team_id: string;
bull_job_id: string;
priority?: number;
is_scrape?: boolean;
}
-export interface RunWebScraperResult {
- success: boolean;
- message: string;
- docs: Document[] | DocumentUrl[];
+export type RunWebScraperResult = {
+ success: false;
+ error: Error;
+} | {
+ success: true;
+ document: V1Document;
}
export interface FirecrawlJob {
job_id?: string;
success: boolean;
- message: string;
+ message?: string;
num_docs: number;
docs: any[];
time_taken: number;
@@ -69,9 +70,8 @@ export interface FirecrawlJob {
mode: string;
url: string;
crawlerOptions?: any;
- pageOptions?: any;
+ scrapeOptions?: any;
origin: string;
- extractor_options?: ExtractorOptions,
num_tokens?: number,
retry?: boolean,
crawl_id?: string;
@@ -115,14 +115,16 @@ export enum RateLimiterMode {
}
-export interface AuthResponse {
- success: boolean;
- team_id?: string;
- error?: string;
- status?: number;
+export type AuthResponse = {
+ success: true;
+ team_id: string;
api_key?: string;
plan?: PlanType;
- chunk?: AuthCreditUsageChunk;
+ chunk: AuthCreditUsageChunk | null;
+} | {
+ success: false;
+ error: string;
+ status: number;
}
diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json
index dd7f0ed27..29093be6f 100644
--- a/apps/api/tsconfig.json
+++ b/apps/api/tsconfig.json
@@ -1,10 +1,10 @@
{
"compilerOptions": {
"rootDir": "./src",
- "lib": ["es6","DOM"],
+ "lib": ["ES2022", "DOM"],
// or higher
- "target": "ES2020",
+ "target": "ES2022",
"module": "commonjs",
"esModuleInterop": true,
@@ -12,6 +12,7 @@
"outDir": "./dist/src",
"moduleResolution": "node",
"baseUrl": ".",
+ "strictNullChecks": true,
"paths": {
"*": ["node_modules/*", "src/types/*"],
diff --git a/examples/kubernetes/cluster-install/secret.yaml b/examples/kubernetes/cluster-install/secret.yaml
index 3cbab7a76..861770c0f 100644
--- a/examples/kubernetes/cluster-install/secret.yaml
+++ b/examples/kubernetes/cluster-install/secret.yaml
@@ -7,7 +7,6 @@ data:
OPENAI_API_KEY: ""
SLACK_WEBHOOK_URL: ""
LLAMAPARSE_API_KEY: ""
- LOGTAIL_KEY: ""
BULL_AUTH_KEY: ""
TEST_API_KEY: ""
POSTHOG_API_KEY: ""
@@ -15,6 +14,5 @@ data:
SCRAPING_BEE_API_KEY: ""
STRIPE_PRICE_ID_STANDARD: ""
STRIPE_PRICE_ID_SCALE: ""
- HYPERDX_API_KEY: ""
FIRE_ENGINE_BETA_URL: ""
REDIS_PASSWORD: ""