Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

queue-worker: move scrape worker to separate threads (ENG-3008) (#1879)

GergΕ‘ MΓ³ricz0f44d947e1dc
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml
index c8d406bad..be890a897 100644
--- a/.github/workflows/test-server-self-host.yml
+++ b/.github/workflows/test-server-self-host.yml
@@ -101,14 +101,17 @@ jobs:
docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng
pnpx wait-on tcp:3434 -t 30s
working-directory: ./
+ - name: Build the application
+ run: npm run build
+ working-directory: ./apps/api
- name: Start server
- run: npm run start:production > api.log 2>&1 &
+ run: npm run start:production:nobuild > api.log 2>&1 &
env:
PORT: 3002
HOST: 0.0.0.0
working-directory: ./apps/api
- name: Start worker
- run: npm run workers > worker.log 2>&1 &
+ run: npm run worker:production > worker.log 2>&1 &
env:
PORT: 3005
HOST: 0.0.0.0
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 6bfc91add..0c7a01c9f 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -102,22 +102,25 @@ jobs:
cargo build --release
chmod +x target/release/libcrawler.so
working-directory: ./apps/api/sharedLibs/crawler
+ - name: Build the application
+ run: npm run build
+ working-directory: ./apps/api
- name: Start the application
- run: npm run start:production > api.log 2>&1 &
+ run: npm run start:production:nobuild > api.log 2>&1 &
env:
PORT: 3002
HOST: 0.0.0.0
working-directory: ./apps/api
id: start_app
- name: Start worker
- run: npm run workers > worker.log 2>&1 &
+ run: npm run worker:production > worker.log 2>&1 &
env:
PORT: 3005
HOST: 0.0.0.0
working-directory: ./apps/api
id: start_workers
- name: Start index worker
- run: npm run index-worker > index-worker.log 2>&1 &
+ run: npm run index-worker:production > index-worker.log 2>&1 &
working-directory: ./apps/api
id: start_index_worker
- name: Wait for API
diff --git a/apps/api/package.json b/apps/api/package.json
index 28b1bc5c9..5d724dc75 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -4,11 +4,12 @@
"description": "",
"main": "index.js",
"scripts": {
- "start": "nodemon --exec ts-node src/index.ts",
+ "start": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"start:production": "tsc && node dist/src/index.js",
+ "start:production:nobuild": "node dist/src/index.js",
"format": "prettier --write \"src/**/*.(js|ts)\"",
"flyio": "node dist/src/index.js",
- "start:dev": "nodemon --exec ts-node src/index.ts",
+ "start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"build": "tsc",
"build:nosentry": "tsc",
"test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'",
@@ -16,9 +17,9 @@
"test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'",
"test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'",
"test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/*.test.ts",
- "workers": "nodemon --exec ts-node src/services/queue-worker.ts",
+ "workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"",
"worker:production": "node dist/src/services/queue-worker.js",
- "index-worker": "nodemon --exec ts-node src/services/indexing/index-worker.ts",
+ "index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",
"index-worker:production": "node dist/src/services/indexing/index-worker.js",
"mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
"mongo-docker-console": "docker exec -it mongodb mongosh",
@@ -44,11 +45,11 @@
"@types/supertest": "^6.0.2",
"jest": "^29.6.3",
"jest-fetch-mock": "^3.0.3",
- "nodemon": "^3.1.10",
"prettier": "^3.4.2",
"supertest": "^6.3.3",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
+ "tsc-watch": "^7.1.1",
"typescript": "^5.8.3"
},
"dependencies": {
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 82854b6c8..1567f780b 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -321,9 +321,6 @@ importers:
jest-fetch-mock:
specifier: ^3.0.3
version: 3.0.3([email protected])
- nodemon:
- specifier: ^3.1.10
- version: 3.1.10
prettier:
specifier: ^3.4.2
version: 3.4.2
@@ -336,6 +333,9 @@ importers:
ts-node:
specifier: ^10.9.1
version: 10.9.2(@types/[email protected])([email protected])
+ tsc-watch:
+ specifier: ^7.1.1
+ version: 7.1.1([email protected])
typescript:
specifier: ^5.8.3
version: 5.8.3
@@ -1964,10 +1964,6 @@ packages:
resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
- resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==}
@@ -2080,10 +2076,6 @@ packages:
resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}
engines: {node: '>= 6'}
- resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}
- engines: {node: '>= 8.10.0'}
-
resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==}
engines: {node: '>=18'}
@@ -2368,6 +2360,9 @@ packages:
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==}
+
resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==}
@@ -2471,6 +2466,9 @@ packages:
resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}
engines: {node: '>= 0.6'}
+ resolution: {integrity: sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==}
+
resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
engines: {node: '>=6'}
@@ -2612,6 +2610,9 @@ packages:
resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==}
engines: {node: '>= 0.6'}
+ resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==}
+
resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
@@ -2663,10 +2664,6 @@ packages:
resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}
engines: {node: '>= 4.8.0'}
- resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
- engines: {node: '>= 6'}
-
resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==}
engines: {node: '>=16 || 14 >=14.18'}
@@ -2806,9 +2803,6 @@ packages:
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
- resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==}
-
resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}
@@ -2856,20 +2850,12 @@ packages:
resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
- resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==}
resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}
- resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
- engines: {node: '>=0.10.0'}
-
resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
engines: {node: '>=8'}
@@ -2878,10 +2864,6 @@ packages:
resolution: {integrity: sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==}
engines: {node: '>=6'}
- resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
- engines: {node: '>=0.10.0'}
-
resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
engines: {node: '>=0.12.0'}
@@ -3249,6 +3231,9 @@ packages:
engines: {node: '>=12.0.0'}
hasBin: true
+ resolution: {integrity: sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==}
+
resolution: {integrity: sha512-f3r0yqpz31VXiDB/wj9GaOB0a2PRLQl6vJmXiFrniNwjkKdvakqJRULhjFKJpxOchlCRiG5fcacoUZY5Xa6PEQ==}
engines: {node: '>= 18'}
@@ -3378,6 +3363,9 @@ packages:
resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}
+ resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==}
+
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
@@ -3408,11 +3396,6 @@ packages:
resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==}
- resolution: {integrity: sha512-WDjw3pJ0/0jMFmyNDp3gvY2YizjLmmOUQo6DEBY+JgdvW/yQ9mEeSw6H5ythl5Ny2ytb7f9C2nIbjSxMNzbJXw==}
- engines: {node: '>=10'}
- hasBin: true
-
resolution: {integrity: sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -3543,6 +3526,9 @@ packages:
resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==}
+ resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==}
+
resolution: {integrity: sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==}
@@ -3643,12 +3629,14 @@ packages:
resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
+ resolution: {integrity: sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==}
+ engines: {node: '>= 0.10'}
+ hasBin: true
+
resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==}
- resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==}
-
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
@@ -3713,10 +3701,6 @@ packages:
resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
- resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
- engines: {node: '>=8.10.0'}
-
resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==}
engines: {node: '>= 0.10'}
@@ -3897,10 +3881,6 @@ packages:
resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
- resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==}
- engines: {node: '>=10'}
-
resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}
@@ -3915,6 +3895,9 @@ packages:
resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
engines: {node: '>=0.10.0'}
+ resolution: {integrity: sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==}
+
resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
@@ -3932,12 +3915,19 @@ packages:
resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
engines: {node: '>= 0.8'}
+ resolution: {integrity: sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==}
+
resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==}
resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==}
+ resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==}
+ engines: {node: '>=0.6.19'}
+
resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==}
engines: {node: '>=10'}
@@ -4049,6 +4039,9 @@ packages:
resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}
+
resolution: {integrity: sha512-AOvV5YYIAFFBfransBzSTyztkc3IMfz5Eq3YluaRiEu55nn43Fzaufx70UqEKYr8BoLCach4q8g/bg6e5+/aFw==}
@@ -4071,10 +4064,6 @@ packages:
resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
engines: {node: '>=0.6'}
- resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==}
- hasBin: true
-
resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==}
engines: {node: '>=16'}
@@ -4128,6 +4117,13 @@ packages:
'@swc/wasm':
optional: true
+ resolution: {integrity: sha512-r6t37Dkk4vK44HwxOe+OzjpE/gDamZAwqXhtcAJD/hPVblcjJK45NxbK0HcDASXG0U4pEnCh640JZbeDVSC6yA==}
+ engines: {node: '>=12.12.0'}
+ hasBin: true
+ peerDependencies:
+ typescript: '*'
+
resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==}
@@ -4157,9 +4153,6 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
- resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==}
-
resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}
@@ -5114,7 +5107,7 @@ snapshots:
'@babel/traverse': 7.24.6
'@babel/types': 7.24.6
convert-source-map: 2.0.0
- debug: 4.4.1([email protected])
+ debug: 4.4.1
gensync: 1.0.0-beta.2
json5: 2.2.3
semver: 6.3.1
@@ -5301,7 +5294,7 @@ snapshots:
'@babel/helper-split-export-declaration': 7.24.6
'@babel/parser': 7.24.6
'@babel/types': 7.24.6
- debug: 4.4.1([email protected])
+ debug: 4.4.1
globals: 11.12.0
transitivePeerDependencies:
- supports-color
@@ -6654,7 +6647,7 @@ snapshots:
dependencies:
- debug: 4.4.1([email protected])
+ debug: 4.4.1
transitivePeerDependencies:
- supports-color
@@ -6816,8 +6809,6 @@ snapshots:
read-cmd-shim: 4.0.0
write-file-atomic: 5.0.1
-
@@ -6958,18 +6949,6 @@ snapshots:
parse5: 7.1.2
parse5-htmlparser2-tree-adapter: 7.0.0
- dependencies:
- anymatch: 3.1.3
- braces: 3.0.3
- glob-parent: 5.1.2
- is-binary-path: 2.1.0
- is-glob: 4.0.3
- normalize-path: 3.0.0
- readdirp: 3.6.0
- optionalDependencies:
- fsevents: 2.3.3
-
@@ -7147,11 +7126,9 @@ snapshots:
dependencies:
ms: 2.1.2
dependencies:
ms: 2.1.3
- optionalDependencies:
- supports-color: 5.5.0
@@ -7233,6 +7210,8 @@ snapshots:
es-errors: 1.3.0
gopd: 1.2.0
+
dependencies:
end-of-stream: 1.4.4
@@ -7319,6 +7298,16 @@ snapshots:
+ dependencies:
+ duplexer: 0.1.2
+ from: 0.1.7
+ map-stream: 0.1.0
+ pause-stream: 0.0.11
+ split: 0.3.3
+ stream-combiner: 0.0.4
+ through: 2.3.8
+
@@ -7499,6 +7488,8 @@ snapshots:
+
@@ -7568,10 +7559,6 @@ snapshots:
shelljs: 0.8.5
shelljs.exec: 1.1.8
- dependencies:
- is-glob: 4.0.3
-
dependencies:
foreground-child: 3.2.1
@@ -7690,7 +7677,7 @@ snapshots:
dependencies:
'@tootallnate/once': 2.0.0
agent-base: 6.0.2
- debug: 4.4.1([email protected])
+ debug: 4.4.1
transitivePeerDependencies:
- supports-color
@@ -7711,7 +7698,7 @@ snapshots:
dependencies:
agent-base: 7.1.3
- debug: 4.4.1([email protected])
+ debug: 4.4.1
transitivePeerDependencies:
- supports-color
@@ -7738,8 +7725,6 @@ snapshots:
-
@@ -7771,7 +7756,7 @@ snapshots:
dependencies:
'@ioredis/commands': 1.2.0
cluster-key-slot: 1.1.2
- debug: 4.4.1([email protected])
+ debug: 4.4.1
denque: 2.1.0
lodash.defaults: 4.2.0
lodash.isarguments: 3.1.0
@@ -7789,26 +7774,16 @@ snapshots:
- dependencies:
- binary-extensions: 2.3.0
-
dependencies:
hasown: 2.0.2
-
- dependencies:
- is-extglob: 2.1.1
-
@@ -7862,7 +7837,7 @@ snapshots:
dependencies:
- debug: 4.4.1([email protected])
+ debug: 4.4.1
istanbul-lib-coverage: 3.2.2
source-map: 0.6.1
transitivePeerDependencies:
@@ -8396,6 +8371,8 @@ snapshots:
underscore: 1.13.6
xmlbuilder: 10.1.1
+
@@ -8498,6 +8475,8 @@ snapshots:
+
@@ -8523,19 +8502,6 @@ snapshots:
- dependencies:
- chokidar: 3.6.0
- debug: 4.4.1([email protected])
- ignore-by-default: 1.0.1
- minimatch: 3.1.2
- pstree.remy: 1.1.8
- semver: 7.6.2
- simple-update-notifier: 2.0.0
- supports-color: 5.5.0
- touch: 3.1.1
- undefsafe: 2.0.5
-
dependencies:
abbrev: 2.0.0
@@ -8650,6 +8616,10 @@ snapshots:
+ dependencies:
+ through: 2.3.8
+
dependencies:
'@pdf-lib/standard-fonts': 1.0.0
@@ -8739,12 +8709,14 @@ snapshots:
+ dependencies:
+ event-stream: 3.3.4
+
dependencies:
punycode: 2.3.1
-
@@ -8818,10 +8790,6 @@ snapshots:
process: 0.11.10
string_decoder: 1.3.0
- dependencies:
- picomatch: 2.3.1
-
dependencies:
resolve: 1.22.8
@@ -8846,7 +8814,7 @@ snapshots:
dependencies:
- debug: 4.4.1([email protected])
+ debug: 4.4.1
module-details-from-path: 1.0.3
resolve: 1.22.8
transitivePeerDependencies:
@@ -9018,10 +8986,6 @@ snapshots:
dependencies:
is-arrayish: 0.3.2
- dependencies:
- semver: 7.6.2
-
@@ -9033,6 +8997,10 @@ snapshots:
+ dependencies:
+ through: 2.3.8
+
@@ -9045,12 +9013,18 @@ snapshots:
+ dependencies:
+ duplexer: 0.1.2
+
dependencies:
stubs: 3.0.0
+
dependencies:
char-regex: 1.0.2
@@ -9184,6 +9158,8 @@ snapshots:
+
@@ -9200,8 +9176,6 @@ snapshots:
-
dependencies:
tldts: 6.1.75
@@ -9250,6 +9224,14 @@ snapshots:
v8-compile-cache-lib: 3.0.1
yn: 3.1.1
+ dependencies:
+ cross-spawn: 7.0.6
+ node-cleanup: 2.1.2
+ ps-tree: 1.2.0
+ string-argv: 0.3.2
+ typescript: 5.8.3
+
@@ -9271,8 +9253,6 @@ snapshots:
-
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index f899d4e31..d3a967de4 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -20,11 +20,9 @@ configDotenv();
export async function startWebScraperPipeline({
job,
- token,
costTracking,
}: {
job: Job<WebScraperOptions> & { id: string };
- token: string;
costTracking: CostTracking;
}) {
return await runWebScraper({
@@ -43,14 +41,6 @@ export async function startWebScraperPipeline({
teamId: job.data.team_id,
...job.data.internalOptions,
},
- // onSuccess: (result, mode) => {
- // logger.debug(`πŸ‚ Job completed ${job.id}`);
- // saveJob(job, result, token, mode);
- // },
- // onError: (error) => {
- // logger.error(`πŸ‚ Job failed ${job.id}`);
- // ScrapeEvents.logJobEvent(job, "failed");
- // },
team_id: job.data.team_id,
bull_job_id: job.id.toString(),
priority: job.opts.priority,
@@ -66,8 +56,6 @@ export async function runWebScraper({
mode,
scrapeOptions,
internalOptions,
- // onSuccess,
- // onError,
team_id,
bull_job_id,
priority,
@@ -128,9 +116,6 @@ export async function runWebScraper({
}
}
- // This is where the returnvalue from the job is set
- // onSuccess(response.document, mode);
-
engines = response.engines;
if (
@@ -205,7 +190,6 @@ export async function runWebScraper({
const saveJob = async (
job: Job,
result: any,
- token: string,
mode: string,
engines?: EngineResultsTracker,
) => {
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
index fee2b45d5..39a4db931 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
@@ -12,7 +12,7 @@ export async function extractMetadataRust(
return {
...fromRust,
...(fromRust.favicon ? {
- favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url)
+ favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url).href
} : {}),
scrapeId: meta.id,
};
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index c2b16778f..ab184efb6 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -72,7 +72,6 @@ export function getExtractQueue() {
},
},
});
- logger.info("Extraction queue created");
}
return extractQueue;
}
@@ -90,7 +89,6 @@ export function getGenerateLlmsTxtQueue() {
},
},
});
- logger.info("LLMs TXT generation queue created");
}
return generateLlmsTxtQueue;
}
@@ -108,7 +106,6 @@ export function getDeepResearchQueue() {
},
},
});
- logger.info("Deep research queue created");
}
return deepResearchQueue;
}
@@ -126,7 +123,6 @@ export function getBillingQueue() {
},
},
});
- logger.info("Billing queue created");
}
return billingQueue;
}
@@ -144,7 +140,6 @@ export function getPrecrawlQueue() {
},
},
});
- logger.info("Precrawl queue created");
}
return precrawlQueue;
}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index de96f194b..53580f652 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -1,99 +1,53 @@
import "dotenv/config";
import "./sentry";
import * as Sentry from "@sentry/node";
-import { CustomError } from "../lib/custom-error";
import {
getScrapeQueue,
getExtractQueue,
getDeepResearchQueue,
redisConnection,
getGenerateLlmsTxtQueue,
- getBillingQueue,
scrapeQueueName,
} from "./queue-service";
-import { startWebScraperPipeline } from "../main/runWebScraper";
-import { callWebhook } from "./webhook";
-import { logJob } from "./logging/log_job";
import { Job, Queue, QueueEvents } from "bullmq";
import { logger as _logger } from "../lib/logger";
import { Worker } from "bullmq";
import systemMonitor from "./system-monitor";
import { v4 as uuidv4 } from "uuid";
import {
- addCrawlJob,
addCrawlJobDone,
- addCrawlJobs,
- crawlToCrawler,
- finishCrawl,
- finishCrawlPre,
- unPreFinishCrawl,
finishCrawlKickoff,
- generateURLPermutations,
getCrawl,
- getCrawlJobs,
- getDoneJobsOrderedLength,
- lockURL,
- lockURLsIndividually,
normalizeURL,
- saveCrawl,
} from "../lib/crawl-redis";
import { StoredCrawl } from "../lib/crawl-redis";
-import { addScrapeJob, addScrapeJobs } from "./queue-jobs";
-import {
- addJobPriority,
- deleteJobPriority,
- getJobPriority,
-} from "../../src/lib/job-priority";
-import { getJobs } from "..//controllers/v1/crawl-status";
import { configDotenv } from "dotenv";
-import { scrapeOptions } from "../controllers/v1/types";
import {
concurrentJobDone,
- pushConcurrencyLimitActiveJob,
} from "../lib/concurrency-limit";
-import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
-import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
-import { Document, TeamFlags } from "../controllers/v1/types";
import {
ExtractResult,
performExtraction,
} from "../lib/extract/extraction-service";
-import { supabase_service } from "../services/supabase";
-import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";
-import { saveExtract, updateExtract } from "../lib/extract/extract-redis";
-import { billTeam } from "./billing/credit_billing";
+import { updateExtract } from "../lib/extract/extract-redis";
import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";
import { performDeepResearch } from "../lib/deep-research/deep-research-service";
import { performGenerateLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-service";
import { updateGeneratedLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-redis";
import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";
-import { CostTracking } from "../lib/extract/extraction-service";
-import { getACUCTeam } from "../controllers/auth";
import Express from "express";
import http from "http";
import https from "https";
import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
-import { RateLimiterMode } from "../types";
-import { calculateCreditsToBeBilled } from "../lib/scrape-billing";
import { redisEvictConnection } from "./redis";
-import { generateURLSplits, queryIndexAtSplitLevel } from "./index";
-import { WebCrawler } from "../scraper/WebScraper/crawler";
-import type { Logger } from "winston";
+import path from "path";
+import { finishCrawlIfNeeded } from "./worker/crawl-logic";
configDotenv();
-class RacedRedirectError extends Error {
- constructor() {
- super("Raced redirect error");
- }
-}
-
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
-const workerStalledCheckInterval =
- Number(process.env.WORKER_STALLED_CHECK_INTERVAL) || 30000;
const jobLockExtendInterval =
Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;
const jobLockExtensionTime =
@@ -111,347 +65,6 @@ const runningJobs: Set<string> = new Set();
cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
-async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
- const logger = _logger.child({
- module: "queue-worker",
- method: "finishCrawlIfNeeded",
- jobId: job.id,
- scrapeId: job.id,
- crawlId: job.data.crawl_id,
- zeroDataRetention: sc.internalOptions.zeroDataRetention,
- });
-
- if (await finishCrawlPre(job.data.crawl_id, logger)) {
- logger.info("Crawl is pre-finished, checking if we need to add more jobs");
- if (
- job.data.crawlerOptions &&
- !(await redisEvictConnection.exists(
- "crawl:" + job.data.crawl_id + ":invisible_urls",
- ))
- ) {
- await redisEvictConnection.set(
- "crawl:" + job.data.crawl_id + ":invisible_urls",
- "done",
- "EX",
- 60 * 60 * 24,
- );
-
- const sc = (await getCrawl(job.data.crawl_id))!;
-
- const visitedUrls = new Set(
- await redisEvictConnection.smembers(
- "crawl:" + job.data.crawl_id + ":visited_unique",
- ),
- );
-
- logger.info("Visited URLs", {
- visitedUrls: visitedUrls.size,
- });
-
- let lastUrls: string[] = [];
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
- if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {
- lastUrls = (
- (
- await supabase_service.rpc("diff_get_last_crawl_urls", {
- i_team_id: job.data.team_id,
- i_url: sc.originUrl!,
- })
- ).data ?? []
- ).map((x) => x.url);
- }
-
- const lastUrlsSet = new Set(lastUrls);
-
- logger.info("Last URLs", {
- lastUrls: lastUrlsSet.size,
- });
-
- const crawler = crawlToCrawler(
- job.data.crawl_id,
- sc,
- (await getACUCTeam(job.data.team_id))?.flags ?? null,
- sc.originUrl!,
- job.data.crawlerOptions,
- );
-
- const univistedUrls = await crawler.filterLinks(
- Array.from(lastUrlsSet).filter((x) => !visitedUrls.has(x)),
- Infinity,
- sc.crawlerOptions.maxDepth ?? 10,
- );
-
- const addableJobCount =
- sc.crawlerOptions.limit === undefined
- ? Infinity
- : sc.crawlerOptions.limit -
- (await getDoneJobsOrderedLength(job.data.crawl_id));
-
- if (univistedUrls.links.length !== 0 && addableJobCount > 0) {
- logger.info("Adding jobs", {
- univistedUrls: univistedUrls.links.length,
- addableJobCount,
- });
-
- const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {
- const uuid = uuidv4();
- return {
- name: uuid,
- data: {
- url,
- mode: "single_urls" as const,
- team_id: job.data.team_id,
- crawlerOptions: {
- ...job.data.crawlerOptions,
- urlInvisibleInCurrentCrawl: true,
- },
- scrapeOptions: job.data.scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- sitemapped: true,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- opts: {
- jobId: uuid,
- priority: 20,
- },
- };
- });
-
- const lockedIds = await lockURLsIndividually(
- job.data.crawl_id,
- sc,
- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
- );
- const lockedJobs = jobs.filter((x) =>
- lockedIds.find((y) => y.id === x.opts.jobId),
- );
- await addCrawlJobs(
- job.data.crawl_id,
- lockedJobs.map((x) => x.opts.jobId),
- logger,
- );
- await addScrapeJobs(lockedJobs);
-
- if (lockedJobs.length > 0) {
- logger.info("Added jobs, not going for the full finish", {
- lockedJobs: lockedJobs.length,
- });
-
- await unPreFinishCrawl(job.data.crawl_id);
- return;
- } else {
- logger.info("No jobs added (all discovered URLs were locked), finishing crawl");
- }
- }
- }
-
- logger.info("Finishing crawl");
- await finishCrawl(job.data.crawl_id, logger);
-
- if (!job.data.v1) {
- const jobIDs = await getCrawlJobs(job.data.crawl_id);
-
- const jobs = (await getJobs(jobIDs)).sort(
- (a, b) => a.timestamp - b.timestamp,
- );
- // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
- const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")
- ? "failed"
- : "completed";
-
- const fullDocs = jobs
- .map((x) =>
- x.returnvalue
- ? Array.isArray(x.returnvalue)
- ? x.returnvalue[0]
- : x.returnvalue
- : null,
- )
- .filter((x) => x !== null);
-
- await logJob({
- job_id: job.data.crawl_id,
- success: jobStatus === "completed",
- message: sc.cancelled ? "Cancelled" : undefined,
- num_docs: fullDocs.length,
- docs: [],
- time_taken: (Date.now() - sc.createdAt) / 1000,
- team_id: job.data.team_id,
- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
- url: sc.originUrl!,
- scrapeOptions: sc.scrapeOptions,
- crawlerOptions: sc.crawlerOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- zeroDataRetention: job.data.zeroDataRetention,
- }, false, job.data.internalOptions?.bypassBilling ?? false);
-
- const data = {
- success: jobStatus !== "failed",
- result: {
- links: fullDocs.map((doc) => {
- return {
- content: doc,
- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
- };
- }),
- },
- project_id: job.data.project_id,
- docs: fullDocs,
- };
-
- // v0 web hooks, call when done with all the data
- if (!job.data.v1) {
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id,
- data,
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: job.data.crawlerOptions !== null
- ? "crawl.completed"
- : "batch_scrape.completed",
- });
- }
- } else {
- const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);
- const jobStatus = sc.cancelled ? "failed" : "completed";
-
- let credits_billed = null;
-
- if (process.env.USE_DB_AUTHENTICATION === "true") {
- const creditsRpc = await supabase_service
- .rpc("credits_billed_by_crawl_id_1", {
- i_crawl_id: job.data.crawl_id,
- });
-
- credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;
-
- if (credits_billed === null) {
- logger.warn("Credits billed is null", {
- error: creditsRpc.error,
- });
- }
- }
-
- await logJob(
- {
- job_id: job.data.crawl_id,
- success: jobStatus === "completed",
- message: sc.cancelled ? "Cancelled" : undefined,
- num_docs,
- docs: [],
- time_taken: (Date.now() - sc.createdAt) / 1000,
- team_id: job.data.team_id,
- scrapeOptions: sc.scrapeOptions,
- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
- url:
- sc?.originUrl ??
- (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
- crawlerOptions: sc.crawlerOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- credits_billed,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- true,
- job.data.internalOptions?.bypassBilling ?? false,
- );
-
-
- // v1 web hooks, call when done with no data, but with event completed
- if (job.data.v1 && job.data.webhook) {
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id,
- data: [],
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: job.data.crawlerOptions !== null
- ? "crawl.completed"
- : "batch_scrape.completed",
- });
- }
- }
- }
-}
-
-const processJobInternal = async (token: string, job: Job & { id: string }) => {
- const logger = _logger.child({
- module: "queue-worker",
- method: "processJobInternal",
- jobId: job.id,
- scrapeId: job.id,
- crawlId: job.data?.crawl_id ?? undefined,
- zeroDataRetention: job.data?.zeroDataRetention ?? false,
- });
-
- const extendLockInterval = setInterval(async () => {
- logger.info(`πŸ‚ Worker extending lock on job ${job.id}`, {
- extendInterval: jobLockExtendInterval,
- extensionTime: jobLockExtensionTime,
- });
-
- if (job.data?.mode !== "kickoff" && job.data?.team_id) {
- await pushConcurrencyLimitActiveJob(job.data.team_id, job.id, 60 * 1000); // 60s lock renew, just like in the queue
- }
-
- await job.extendLock(token, jobLockExtensionTime);
- }, jobLockExtendInterval);
-
- await addJobPriority(job.data.team_id, job.id);
- let err = null;
- try {
- if (job.data?.mode === "kickoff") {
- const result = await processKickoffJob(job, token);
- if (result.success) {
- try {
- await job.moveToCompleted(null, token, false);
- } catch (e) {}
- } else {
- logger.debug("Job failed", { result, mode: job.data.mode });
- await job.moveToFailed((result as any).error, token, false);
- }
- } else {
- const result = await processJob(job, token);
- if (result.success) {
- try {
- if (
- process.env.USE_DB_AUTHENTICATION === "true" &&
- (job.data.crawl_id || process.env.GCS_BUCKET_NAME)
- ) {
- logger.debug(
- "Job succeeded -- putting null in Redis",
- );
- await job.moveToCompleted(null, token, false);
- } else {
- logger.debug("Job succeeded -- putting result in Redis");
- await job.moveToCompleted(result.document, token, false);
- }
- } catch (e) {}
- } else {
- logger.debug("Job failed", { result });
- await job.moveToFailed((result as any).error, token, false);
- }
- }
- } catch (error) {
- logger.debug("Job failed", { error });
- Sentry.captureException(error);
- err = error;
- await job.moveToFailed(error, token, false);
- } finally {
- await deleteJobPriority(job.data.team_id, job.id);
- clearInterval(extendLockInterval);
- }
-
- return err;
-};
-
const processExtractJobInternal = async (
token: string,
job: Job & { id: string },
@@ -714,6 +327,22 @@ process.on("SIGTERM", () => {
let cantAcceptConnectionCount = 0;
+const separateWorkerFun = (
+ queue: Queue,
+ path: string,
+): Worker => {
+ const worker = new Worker(queue.name, path, {
+ connection: redisConnection,
+ lockDuration: 30 * 1000, // 30 seconds
+ stalledInterval: 30 * 1000, // 30 seconds
+ maxStalledCount: 10, // 10 times
+ concurrency: 6, // from k8s setup
+ useWorkerThreads: true,
+ });
+
+ return worker;
+}
+
const workerFun = async (
queue: Queue,
processJobInternal: (token: string, job: Job) => Promise<any>,
@@ -849,741 +478,6 @@ const workerFun = async (
}
};
-async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: string) {
- if (sc.crawlerOptions.ignoreSitemap) {
- return [];
- }
-
- const trimmedURL = new URL(url);
- trimmedURL.search = "";
-
- const index = await queryIndexAtSplitLevel(
- sc.crawlerOptions.allowBackwardCrawling ? generateURLSplits(trimmedURL.href)[0] : trimmedURL.href,
- sc.crawlerOptions.limit ?? 10000,
- );
-
- const validIndexLinksResult = await crawler.filterLinks(
- index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),
- sc.crawlerOptions.limit ?? 10000,
- sc.crawlerOptions.maxDepth ?? 10,
- false,
- );
- const validIndexLinks = validIndexLinksResult.links;
-
- return validIndexLinks;
-}
-
-async function processKickoffJob(job: Job & { id: string }, token: string) {
- const logger = _logger.child({
- module: "queue-worker",
- method: "processKickoffJob",
- jobId: job.id,
- scrapeId: job.id,
- crawlId: job.data?.crawl_id ?? undefined,
- teamId: job.data?.team_id ?? undefined,
- zeroDataRetention: job.data.zeroDataRetention ?? false,
- });
-
- try {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
- const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
-
- logger.debug("Locking URL...");
- await lockURL(job.data.crawl_id, sc, job.data.url);
- const jobId = uuidv4();
- logger.debug("Adding scrape job to Redis...", { jobId });
- await addScrapeJob(
- {
- url: job.data.url,
- mode: "single_urls",
- team_id: job.data.team_id,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- webhook: job.data.webhook,
- v1: job.data.v1,
- isCrawlSourceScrape: true,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- {
- priority: 15,
- },
- jobId,
- );
- logger.debug("Adding scrape job to BullMQ...", { jobId });
- await addCrawlJob(job.data.crawl_id, jobId, logger);
-
- if (job.data.webhook) {
- logger.debug("Calling webhook with crawl.started...", {
- webhook: job.data.webhook,
- });
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id,
- data: null,
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: "crawl.started",
- });
- }
-
- const sitemap = sc.crawlerOptions.ignoreSitemap
- ? 0
- : await crawler.tryGetSitemap(async (urls) => {
- if (urls.length === 0) return;
-
- logger.debug("Using sitemap chunk of length " + urls.length, {
- sitemapLength: urls.length,
- });
-
- let jobPriority = await getJobPriority({
- team_id: job.data.team_id,
- basePriority: 21,
- });
- logger.debug("Using job priority " + jobPriority, { jobPriority });
-
- const jobs = urls.map((url) => {
- const uuid = uuidv4();
- return {
- name: uuid,
- data: {
- url,
- mode: "single_urls" as const,
- team_id: job.data.team_id,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- sitemapped: true,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- opts: {
- jobId: uuid,
- priority: 20,
- },
- };
- });
-
- logger.debug("Locking URLs...");
- const lockedIds = await lockURLsIndividually(
- job.data.crawl_id,
- sc,
- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
- );
- const lockedJobs = jobs.filter((x) =>
- lockedIds.find((y) => y.id === x.opts.jobId),
- );
- logger.debug("Adding scrape jobs to Redis...");
- await addCrawlJobs(
- job.data.crawl_id,
- lockedJobs.map((x) => x.opts.jobId),
- logger,
- );
- logger.debug("Adding scrape jobs to BullMQ...");
- await addScrapeJobs(lockedJobs);
- });
-
- if (sitemap === 0) {
- logger.debug("Sitemap not found or ignored.", {
- ignoreSitemap: sc.crawlerOptions.ignoreSitemap,
- });
- }
-
- const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);
-
- if (indexLinks.length > 0) {
- logger.debug("Using index links of length " + indexLinks.length, {
- indexLinksLength: indexLinks.length,
- });
-
- let jobPriority = await getJobPriority({
- team_id: job.data.team_id,
- basePriority: 21,
- });
- logger.debug("Using job priority " + jobPriority, { jobPriority });
-
- const jobs = indexLinks.map((url) => {
- const uuid = uuidv4();
- return {
- name: uuid,
- data: {
- url,
- mode: "single_urls" as const,
- team_id: job.data.team_id,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- sitemapped: true,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- opts: {
- jobId: uuid,
- priority: 20,
- },
- };
- });
-
- logger.debug("Locking URLs...");
- const lockedIds = await lockURLsIndividually(
- job.data.crawl_id,
- sc,
- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
- );
- const lockedJobs = jobs.filter((x) =>
- lockedIds.find((y) => y.id === x.opts.jobId),
- );
- logger.debug("Adding scrape jobs to Redis...");
- await addCrawlJobs(
- job.data.crawl_id,
- lockedJobs.map((x) => x.opts.jobId),
- logger,
- );
- logger.debug("Adding scrape jobs to BullMQ...");
- await addScrapeJobs(lockedJobs);
- }
-
- logger.debug("Done queueing jobs!");
-
- await finishCrawlKickoff(job.data.crawl_id);
- await finishCrawlIfNeeded(job, sc);
-
- return { success: true };
- } catch (error) {
- logger.error("An error occurred!", { error });
- await finishCrawlKickoff(job.data.crawl_id);
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
- if (sc) {
- await finishCrawlIfNeeded(job, sc);
- }
- return { success: false, error };
- }
-}
-
-async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) {
- let creditsToBeBilled: number | null = null;
-
- if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {
- creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags);
-
- if (
- job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&
- process.env.USE_DB_AUTHENTICATION === "true"
- ) {
- try {
- const billingJobId = uuidv4();
- logger.debug(
- `Adding billing job to queue for team ${job.data.team_id}`,
- {
- billingJobId,
- credits: creditsToBeBilled,
- is_extract: false,
- },
- );
-
- // Add directly to the billing queue - the billing worker will handle the rest
- await getBillingQueue().add(
- "bill_team",
- {
- team_id: job.data.team_id,
- subscription_id: undefined,
- credits: creditsToBeBilled,
- is_extract: false,
- timestamp: new Date().toISOString(),
- originating_job_id: job.id,
- },
- {
- jobId: billingJobId,
- priority: 10,
- },
- );
- return creditsToBeBilled;
- } catch (error) {
- logger.error(
- `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
- { error },
- );
- Sentry.captureException(error);
- return creditsToBeBilled;
- }
- }
- }
-
- return creditsToBeBilled;
-}
-
-async function processJob(job: Job & { id: string }, token: string) {
- const logger = _logger.child({
- module: "queue-worker",
- method: "processJob",
- jobId: job.id,
- scrapeId: job.id,
- crawlId: job.data?.crawl_id ?? undefined,
- teamId: job.data?.team_id ?? undefined,
- zeroDataRetention: job.data?.zeroDataRetention ?? false,
- });
- logger.info(`πŸ‚ Worker taking job ${job.id}`, { url: job.data.url, sitemapped: job.data.sitemapped });
- const start = job.data.startTime ?? Date.now();
- const remainingTime = job.data.scrapeOptions.timeout ? (job.data.scrapeOptions.timeout - (Date.now() - start)) : undefined;
-
- const costTracking = new CostTracking();
-
- try {
- job.updateProgress({
- current: 1,
- total: 100,
- current_step: "SCRAPING",
- current_url: "",
- });
-
- if (remainingTime !== undefined && remainingTime < 0) {
- throw new Error("timeout");
- }
- const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;
-
- if (job.data.crawl_id) {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
- if (sc && sc.cancelled) {
- throw new Error("Parent crawl/batch scrape was cancelled");
- }
- }
-
-
- const pipeline = await Promise.race([
- startWebScraperPipeline({
- job,
- token,
- costTracking,
- }),
- ...(remainingTime !== undefined
- ? [
- (async () => {
- await sleep(remainingTime);
- throw new Error("timeout");
- })(),
- ]
- : []),
- ]);
-
- try {
- signal?.throwIfAborted();
- } catch (e) {
- throw new Error("timeout");
- }
-
- if (!pipeline.success) {
- throw pipeline.error;
- }
-
- const end = Date.now();
- const timeTakenInSeconds = (end - start) / 1000;
-
- const doc = pipeline.document;
-
- const rawHtml = doc.rawHtml ?? "";
-
- if (!job.data.scrapeOptions.formats.includes("rawHtml")) {
- delete doc.rawHtml;
- }
-
- if (job.data.concurrencyLimited) {
- doc.warning =
- "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." +
- (doc.warning ? " " + doc.warning : "");
- }
-
- const data = {
- success: true,
- result: {
- links: [
- {
- content: doc,
- source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",
- id: job.id,
- },
- ],
- },
- project_id: job.data.project_id,
- document: doc,
- };
-
- if (job.data.crawl_id) {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
-
- if (
- doc.metadata.url !== undefined &&
- doc.metadata.sourceURL !== undefined &&
- normalizeURL(doc.metadata.url, sc) !==
- normalizeURL(doc.metadata.sourceURL, sc) &&
- job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
- ) {
- const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
- const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);
- if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {
- const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";
- throw new Error(reason);
- }
-
- // Only re-set originUrl if it's different from the current hostname
- // This is only done on this condition to handle cross-domain redirects
- // If this would be done for non-crossdomain redirects, but also for e.g.
- // redirecting / -> /introduction (like our docs site does), it would
- // break crawling the entire site without allowBackwardsCrawling - mogery
- const isHostnameDifferent =
- normalizeUrlOnlyHostname(doc.metadata.url) !==
- normalizeUrlOnlyHostname(doc.metadata.sourceURL);
- if (job.data.isCrawlSourceScrape && isHostnameDifferent) {
- // TODO: re-fetch sitemap for redirect target domain
- sc.originUrl = doc.metadata.url;
- await saveCrawl(job.data.crawl_id, sc);
- }
-
- if (isUrlBlocked(doc.metadata.url, (await getACUCTeam(job.data.team_id))?.flags ?? null)) {
- throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
- }
-
- const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));
- const p2 = generateURLPermutations(
- normalizeURL(doc.metadata.sourceURL, sc),
- );
-
- if (JSON.stringify(p1) !== JSON.stringify(p2)) {
- logger.debug(
- "Was redirected, removing old URL and locking new URL...",
- { oldUrl: doc.metadata.sourceURL, newUrl: doc.metadata.url },
- );
-
- // Prevent redirect target from being visited in the crawl again
- // See lockURL
- const x = await redisEvictConnection.sadd(
- "crawl:" + job.data.crawl_id + ":visited",
- ...p1.map((x) => x.href),
- );
- const lockRes = x === p1.length;
-
- if (job.data.crawlerOptions !== null && !lockRes) {
- throw new RacedRedirectError();
- }
- }
- }
-
- if (job.data.crawlerOptions !== null) {
- if (!sc.cancelled) {
- const crawler = crawlToCrawler(
- job.data.crawl_id,
- sc,
- (await getACUCTeam(job.data.team_id))?.flags ?? null,
- doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,
- job.data.crawlerOptions,
- );
-
- const links = await crawler.filterLinks(
- await crawler.extractLinksFromHTML(
- rawHtml ?? "",
- doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,
- ),
- Infinity,
- sc.crawlerOptions?.maxDepth ?? 10,
- );
-
- logger.debug("Discovered " + links.links.length + " links...", {
- linksLength: links.links.length,
- });
-
- for (const link of links.links) {
- if (await lockURL(job.data.crawl_id, sc, link)) {
- // This seems to work really welel
- const jobPriority = await getJobPriority({
- team_id: sc.team_id,
- basePriority: job.data.crawl_id ? 20 : 10,
- });
- const jobId = uuidv4();
-
- logger.debug(
- "Determined job priority " +
- jobPriority +
- " for URL " +
- JSON.stringify(link),
- { jobPriority, url: link },
- );
-
- // console.log("team_id: ", sc.team_id)
- // console.log("base priority: ", job.data.crawl_id ? 20 : 10)
- // console.log("job priority: " , jobPriority, "\n\n\n")
-
- await addScrapeJob(
- {
- url: link,
- mode: "single_urls",
- team_id: sc.team_id,
- scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),
- internalOptions: sc.internalOptions,
- crawlerOptions: {
- ...sc.crawlerOptions,
- currentDiscoveryDepth:
- (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,
- },
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- {},
- jobId,
- jobPriority,
- );
-
- await addCrawlJob(job.data.crawl_id, jobId, logger);
- logger.debug("Added job for URL " + JSON.stringify(link), {
- jobPriority,
- url: link,
- newJobId: jobId,
- });
- } else {
- // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!
- // logger.debug("Could not lock URL " + JSON.stringify(link), {
- // url: link,
- // });
- }
- }
-
- // Only run check after adding new jobs for discovery - mogery
- if (job.data.isCrawlSourceScrape) {
- const filterResult = await crawler.filterLinks(
- [doc.metadata.url ?? doc.metadata.sourceURL!],
- 1,
- sc.crawlerOptions?.maxDepth ?? 10,
- );
- if (filterResult.links.length === 0) {
- const url = doc.metadata.url ?? doc.metadata.sourceURL!;
- const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";
- throw new Error(reason);
- }
- }
- }
- }
-
- try {
- signal?.throwIfAborted();
- } catch (e) {
- throw new Error("timeout");
- }
-
- const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
-
- doc.metadata.creditsUsed = credits_billed ?? undefined;
-
- logger.debug("Logging job to DB...");
- await logJob(
- {
- job_id: job.id as string,
- success: true,
- num_docs: 1,
- docs: [doc],
- time_taken: timeTakenInSeconds,
- team_id: job.data.team_id,
- mode: job.data.mode,
- url: job.data.url,
- crawlerOptions: sc.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- cost_tracking: costTracking,
- pdf_num_pages: doc.metadata.numPages,
- credits_billed,
- change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- true,
- job.data.internalOptions?.bypassBilling ?? false,
- );
-
- if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {
- logger.debug("Calling webhook with success...", {
- webhook: job.data.webhook,
- });
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id,
- scrapeId: job.id,
- data,
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
- });
- }
-
- logger.debug("Declaring job as done...");
- await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
-
- await finishCrawlIfNeeded(job, sc);
- } else {
- try {
- signal?.throwIfAborted();
- } catch (e) {
- throw new Error("timeout");
- }
-
- const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
-
- doc.metadata.creditsUsed = credits_billed ?? undefined;
-
- await logJob({
- job_id: job.id,
- success: true,
- message: "Scrape completed",
- num_docs: 1,
- docs: [doc],
- time_taken: timeTakenInSeconds,
- team_id: job.data.team_id,
- mode: "scrape",
- url: job.data.url,
- scrapeOptions: job.data.scrapeOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- num_tokens: 0, // TODO: fix
- cost_tracking: costTracking,
- pdf_num_pages: doc.metadata.numPages,
- credits_billed,
- change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,
- zeroDataRetention: job.data.zeroDataRetention,
- }, false, job.data.internalOptions?.bypassBilling ?? false);
- }
-
- logger.info(`πŸ‚ Job done ${job.id}`);
- return data;
- } catch (error) {
- if (job.data.crawl_id) {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
-
- logger.debug("Declaring job as done...");
- await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);
- await redisEvictConnection.srem(
- "crawl:" + job.data.crawl_id + ":visited_unique",
- normalizeURL(job.data.url, sc),
- );
-
- await finishCrawlIfNeeded(job, sc);
- }
-
- const isEarlyTimeout =
- error instanceof Error && error.message === "timeout";
- const isCancelled =
- error instanceof Error &&
- error.message === "Parent crawl/batch scrape was cancelled";
-
- if (isEarlyTimeout) {
- logger.error(`πŸ‚ Job timed out ${job.id}`);
- } else if (error instanceof RacedRedirectError) {
- logger.warn(`πŸ‚ Job got redirect raced ${job.id}, silently failing`);
- } else if (isCancelled) {
- logger.warn(`πŸ‚ Job got cancelled, silently failing`);
- } else {
- logger.error(`πŸ‚ Job errored ${job.id} - ${error}`, { error });
-
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
-
- if (error instanceof CustomError) {
- // Here we handle the error, then save the failed job
- logger.error(error.message); // or any other error handling
- }
- logger.error(error);
- if (error.stack) {
- logger.error(error.stack);
- }
- }
-
- const data = {
- success: false,
- document: null,
- project_id: job.data.project_id,
- error:
- error instanceof Error
- ? error
- : typeof error === "string"
- ? new Error(error)
- : new Error(JSON.stringify(error)),
- };
-
- if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id ?? (job.id as string),
- scrapeId: job.id,
- data,
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
- });
- }
-
- const end = Date.now();
- const timeTakenInSeconds = (end - start) / 1000;
-
- const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
-
- logger.debug("Logging job to DB...");
- await logJob(
- {
- job_id: job.id as string,
- success: false,
- message:
- typeof error === "string"
- ? error
- : (error.message ??
- "Something went wrong... Contact [email protected]"),
- num_docs: 0,
- docs: [],
- time_taken: timeTakenInSeconds,
- team_id: job.data.team_id,
- mode: job.data.mode,
- url: job.data.url,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- cost_tracking: costTracking,
- credits_billed,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- true,
- job.data.internalOptions?.bypassBilling ?? false,
- );
- return data;
- }
-}
-
-// wsq.process(
-// Math.floor(Number(process.env.NUM_WORKERS_PER_QUEUE ?? 8)),
-// processJob
-// );
-
-// wsq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));
-// wsq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));
-// wsq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));
-// wsq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));
-// wsq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));
-// wsq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));
-
// Start all workers
const app = Express();
@@ -1672,14 +566,18 @@ app.listen(workerPort, () => {
const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });
scrapeQueueEvents.on("failed", failedListener);
- await Promise.all([
- workerFun(getScrapeQueue(), processJobInternal),
+ const results = await Promise.all([
+ separateWorkerFun(getScrapeQueue(), path.join(__dirname, "worker", "scrape-worker.js")),
workerFun(getExtractQueue(), processExtractJobInternal),
workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),
workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),
]);
+
console.log("All workers exited. Waiting for all jobs to finish...");
+ const workerResults = results.filter(x => x instanceof Worker);
+ await Promise.all(workerResults.map(x => x.close()));
+
while (runningJobs.size > 0) {
await new Promise((resolve) => setTimeout(resolve, 500));
}
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
new file mode 100644
index 000000000..9a3d568b3
--- /dev/null
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -0,0 +1,284 @@
+import { logger as _logger } from "../../lib/logger";
+import { Job } from "bullmq";
+import { addCrawlJobs, finishCrawl, getCrawlJobs, getDoneJobsOrderedLength, lockURLsIndividually, StoredCrawl, unPreFinishCrawl } from "../../lib/crawl-redis";
+import { redisEvictConnection } from "../redis";
+import { getCrawl } from "../../lib/crawl-redis";
+import { finishCrawlPre } from "../../lib/crawl-redis";
+import { getACUCTeam } from "../../controllers/auth";
+import { crawlToCrawler } from "../../lib/crawl-redis";
+import { supabase_service } from "../supabase";
+import { v4 as uuidv4 } from "uuid";
+import { addScrapeJobs } from "../queue-jobs";
+import { getJobs } from "../../controllers/v1/crawl-status";
+import { logJob } from "../logging/log_job";
+import { callWebhook } from "../webhook";
+
+export async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "finishCrawlIfNeeded",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data.crawl_id,
+ zeroDataRetention: sc.internalOptions.zeroDataRetention,
+ });
+
+ if (await finishCrawlPre(job.data.crawl_id, logger)) {
+ logger.info("Crawl is pre-finished, checking if we need to add more jobs");
+ if (
+ job.data.crawlerOptions &&
+ !(await redisEvictConnection.exists(
+ "crawl:" + job.data.crawl_id + ":invisible_urls",
+ ))
+ ) {
+ await redisEvictConnection.set(
+ "crawl:" + job.data.crawl_id + ":invisible_urls",
+ "done",
+ "EX",
+ 60 * 60 * 24,
+ );
+
+ const sc = (await getCrawl(job.data.crawl_id))!;
+
+ const visitedUrls = new Set(
+ await redisEvictConnection.smembers(
+ "crawl:" + job.data.crawl_id + ":visited_unique",
+ ),
+ );
+
+ logger.info("Visited URLs", {
+ visitedUrls: visitedUrls.size,
+ });
+
+ let lastUrls: string[] = [];
+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {
+ lastUrls = (
+ (
+ await supabase_service.rpc("diff_get_last_crawl_urls", {
+ i_team_id: job.data.team_id,
+ i_url: sc.originUrl!,
+ })
+ ).data ?? []
+ ).map((x) => x.url);
+ }
+
+ const lastUrlsSet = new Set(lastUrls);
+
+ logger.info("Last URLs", {
+ lastUrls: lastUrlsSet.size,
+ });
+
+ const crawler = crawlToCrawler(
+ job.data.crawl_id,
+ sc,
+ (await getACUCTeam(job.data.team_id))?.flags ?? null,
+ sc.originUrl!,
+ job.data.crawlerOptions,
+ );
+
+ const univistedUrls = await crawler.filterLinks(
+ Array.from(lastUrlsSet).filter((x) => !visitedUrls.has(x)),
+ Infinity,
+ sc.crawlerOptions.maxDepth ?? 10,
+ );
+
+ const addableJobCount =
+ sc.crawlerOptions.limit === undefined
+ ? Infinity
+ : sc.crawlerOptions.limit -
+ (await getDoneJobsOrderedLength(job.data.crawl_id));
+
+ if (univistedUrls.links.length !== 0 && addableJobCount > 0) {
+ logger.info("Adding jobs", {
+ univistedUrls: univistedUrls.links.length,
+ addableJobCount,
+ });
+
+ const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {
+ const uuid = uuidv4();
+ return {
+ name: uuid,
+ data: {
+ url,
+ mode: "single_urls" as const,
+ team_id: job.data.team_id,
+ crawlerOptions: {
+ ...job.data.crawlerOptions,
+ urlInvisibleInCurrentCrawl: true,
+ },
+ scrapeOptions: job.data.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ sitemapped: true,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ opts: {
+ jobId: uuid,
+ priority: 20,
+ },
+ };
+ });
+
+ const lockedIds = await lockURLsIndividually(
+ job.data.crawl_id,
+ sc,
+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
+ );
+ const lockedJobs = jobs.filter((x) =>
+ lockedIds.find((y) => y.id === x.opts.jobId),
+ );
+ await addCrawlJobs(
+ job.data.crawl_id,
+ lockedJobs.map((x) => x.opts.jobId),
+ logger,
+ );
+ await addScrapeJobs(lockedJobs);
+
+ if (lockedJobs.length > 0) {
+ logger.info("Added jobs, not going for the full finish", {
+ lockedJobs: lockedJobs.length,
+ });
+
+ await unPreFinishCrawl(job.data.crawl_id);
+ return;
+ } else {
+ logger.info("No jobs added (all discovered URLs were locked), finishing crawl");
+ }
+ }
+ }
+
+ logger.info("Finishing crawl");
+ await finishCrawl(job.data.crawl_id, logger);
+
+ if (!job.data.v1) {
+ const jobIDs = await getCrawlJobs(job.data.crawl_id);
+
+ const jobs = (await getJobs(jobIDs)).sort(
+ (a, b) => a.timestamp - b.timestamp,
+ );
+ // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
+ const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")
+ ? "failed"
+ : "completed";
+
+ const fullDocs = jobs
+ .map((x) =>
+ x.returnvalue
+ ? Array.isArray(x.returnvalue)
+ ? x.returnvalue[0]
+ : x.returnvalue
+ : null,
+ )
+ .filter((x) => x !== null);
+
+ await logJob({
+ job_id: job.data.crawl_id,
+ success: jobStatus === "completed",
+ message: sc.cancelled ? "Cancelled" : undefined,
+ num_docs: fullDocs.length,
+ docs: [],
+ time_taken: (Date.now() - sc.createdAt) / 1000,
+ team_id: job.data.team_id,
+ mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ url: sc.originUrl!,
+ scrapeOptions: sc.scrapeOptions,
+ crawlerOptions: sc.crawlerOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ zeroDataRetention: job.data.zeroDataRetention,
+ }, false, job.data.internalOptions?.bypassBilling ?? false);
+
+ const data = {
+ success: jobStatus !== "failed",
+ result: {
+ links: fullDocs.map((doc) => {
+ return {
+ content: doc,
+ source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
+ };
+ }),
+ },
+ project_id: job.data.project_id,
+ docs: fullDocs,
+ };
+
+ // v0 web hooks, call when done with all the data
+ if (!job.data.v1) {
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id,
+ data,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: job.data.crawlerOptions !== null
+ ? "crawl.completed"
+ : "batch_scrape.completed",
+ });
+ }
+ } else {
+ const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);
+ const jobStatus = sc.cancelled ? "failed" : "completed";
+
+ let credits_billed = null;
+
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const creditsRpc = await supabase_service
+ .rpc("credits_billed_by_crawl_id_1", {
+ i_crawl_id: job.data.crawl_id,
+ });
+
+ credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;
+
+ if (credits_billed === null) {
+ logger.warn("Credits billed is null", {
+ error: creditsRpc.error,
+ });
+ }
+ }
+
+ await logJob(
+ {
+ job_id: job.data.crawl_id,
+ success: jobStatus === "completed",
+ message: sc.cancelled ? "Cancelled" : undefined,
+ num_docs,
+ docs: [],
+ time_taken: (Date.now() - sc.createdAt) / 1000,
+ team_id: job.data.team_id,
+ scrapeOptions: sc.scrapeOptions,
+ mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ url:
+ sc?.originUrl ??
+ (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
+ crawlerOptions: sc.crawlerOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ credits_billed,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ true,
+ job.data.internalOptions?.bypassBilling ?? false,
+ );
+
+
+ // v1 web hooks, call when done with no data, but with event completed
+ if (job.data.v1 && job.data.webhook) {
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id,
+ data: [],
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: job.data.crawlerOptions !== null
+ ? "crawl.completed"
+ : "batch_scrape.completed",
+ });
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
new file mode 100644
index 000000000..68c9b3a28
--- /dev/null
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -0,0 +1,850 @@
+import { configDotenv } from "dotenv";
+import { Job } from "bullmq";
+import * as Sentry from "@sentry/node";
+import http from "http";
+import https from "https";
+
+import { logger as _logger } from "../../lib/logger";
+import {
+ concurrentJobDone,
+ pushConcurrencyLimitActiveJob,
+} from "../../lib/concurrency-limit";
+import { addJobPriority, deleteJobPriority } from "../../lib/job-priority";
+import { cacheableLookup } from "../../scraper/scrapeURL/lib/cacheableLookup";
+import { v4 as uuidv4 } from "uuid";
+import {
+ addCrawlJob,
+ addCrawlJobs,
+ addCrawlJobDone,
+ crawlToCrawler,
+ finishCrawlKickoff,
+ generateURLPermutations,
+ getCrawl,
+ lockURL,
+ lockURLsIndividually,
+ normalizeURL,
+ saveCrawl,
+ StoredCrawl
+} from "../../lib/crawl-redis";
+import { addScrapeJob, addScrapeJobs } from "../queue-jobs";
+import { getJobPriority } from "../../lib/job-priority";
+import { Document, scrapeOptions, TeamFlags } from "../../controllers/v1/types";
+import { getACUCTeam } from "../../controllers/auth";
+import { callWebhook } from "../webhook";
+import { CustomError } from "../../lib/custom-error";
+import { startWebScraperPipeline } from "../../main/runWebScraper";
+import { CostTracking } from "../../lib/extract/extraction-service";
+import { redisEvictConnection } from "../redis";
+import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";
+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { logJob } from "../logging/log_job";
+import { generateURLSplits, queryIndexAtSplitLevel } from "../index";
+import { WebCrawler } from "../../scraper/WebScraper/crawler";
+import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
+import { getBillingQueue } from "../queue-service";
+import type { Logger } from "winston";
+import { finishCrawlIfNeeded } from "./crawl-logic";
+
+class RacedRedirectError extends Error {
+ constructor() {
+ super("Raced redirect error");
+ }
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+configDotenv();
+
+const jobLockExtendInterval =
+ Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;
+const jobLockExtensionTime =
+ Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;
+
+cacheableLookup.install(http.globalAgent);
+cacheableLookup.install(https.globalAgent);
+
+async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) {
+ let creditsToBeBilled: number | null = null;
+
+ if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {
+ creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags);
+
+ if (
+ job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&
+ process.env.USE_DB_AUTHENTICATION === "true"
+ ) {
+ try {
+ const billingJobId = uuidv4();
+ logger.debug(
+ `Adding billing job to queue for team ${job.data.team_id}`,
+ {
+ billingJobId,
+ credits: creditsToBeBilled,
+ is_extract: false,
+ },
+ );
+
+ // Add directly to the billing queue - the billing worker will handle the rest
+ await getBillingQueue().add(
+ "bill_team",
+ {
+ team_id: job.data.team_id,
+ subscription_id: undefined,
+ credits: creditsToBeBilled,
+ is_extract: false,
+ timestamp: new Date().toISOString(),
+ originating_job_id: job.id,
+ },
+ {
+ jobId: billingJobId,
+ priority: 10,
+ },
+ );
+ return creditsToBeBilled;
+ } catch (error) {
+ logger.error(
+ `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
+ { error },
+ );
+ Sentry.captureException(error);
+ return creditsToBeBilled;
+ }
+ }
+ }
+
+ return creditsToBeBilled;
+}
+
+async function processJob(job: Job & { id: string }) {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "processJob",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data?.crawl_id ?? undefined,
+ teamId: job.data?.team_id ?? undefined,
+ zeroDataRetention: job.data?.zeroDataRetention ?? false,
+ });
+ logger.info(`πŸ‚ Worker taking job ${job.id}`, { url: job.data.url });
+ const start = job.data.startTime ?? Date.now();
+ const remainingTime = job.data.scrapeOptions.timeout ? (job.data.scrapeOptions.timeout - (Date.now() - start)) : undefined;
+
+ const costTracking = new CostTracking();
+
+ try {
+ job.updateProgress({
+ current: 1,
+ total: 100,
+ current_step: "SCRAPING",
+ current_url: "",
+ });
+
+ if (remainingTime !== undefined && remainingTime < 0) {
+ throw new Error("timeout");
+ }
+ const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;
+
+ if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ if (sc && sc.cancelled) {
+ throw new Error("Parent crawl/batch scrape was cancelled");
+ }
+ }
+
+
+ const pipeline = await Promise.race([
+ startWebScraperPipeline({
+ job,
+ costTracking,
+ }),
+ ...(remainingTime !== undefined
+ ? [
+ (async () => {
+ await sleep(remainingTime);
+ throw new Error("timeout");
+ })(),
+ ]
+ : []),
+ ]);
+
+ try {
+ signal?.throwIfAborted();
+ } catch (e) {
+ throw new Error("timeout");
+ }
+
+ if (!pipeline.success) {
+ throw pipeline.error;
+ }
+
+ const end = Date.now();
+ const timeTakenInSeconds = (end - start) / 1000;
+
+ const doc = pipeline.document;
+
+ const rawHtml = doc.rawHtml ?? "";
+
+ if (!job.data.scrapeOptions.formats.includes("rawHtml")) {
+ delete doc.rawHtml;
+ }
+
+ if (job.data.concurrencyLimited) {
+ doc.warning =
+ "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." +
+ (doc.warning ? " " + doc.warning : "");
+ }
+
+ const data = {
+ success: true,
+ result: {
+ links: [
+ {
+ content: doc,
+ source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",
+ id: job.id,
+ },
+ ],
+ },
+ project_id: job.data.project_id,
+ document: doc,
+ };
+
+ if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+
+ if (
+ doc.metadata.url !== undefined &&
+ doc.metadata.sourceURL !== undefined &&
+ normalizeURL(doc.metadata.url, sc) !==
+ normalizeURL(doc.metadata.sourceURL, sc) &&
+ job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
+ ) {
+ const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+ const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);
+ if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {
+ const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";
+ throw new Error(reason);
+ }
+
+ // Only re-set originUrl if it's different from the current hostname
+ // This is only done on this condition to handle cross-domain redirects
+ // If this would be done for non-crossdomain redirects, but also for e.g.
+ // redirecting / -> /introduction (like our docs site does), it would
+ // break crawling the entire site without allowBackwardsCrawling - mogery
+ const isHostnameDifferent =
+ normalizeUrlOnlyHostname(doc.metadata.url) !==
+ normalizeUrlOnlyHostname(doc.metadata.sourceURL);
+ if (job.data.isCrawlSourceScrape && isHostnameDifferent) {
+ // TODO: re-fetch sitemap for redirect target domain
+ sc.originUrl = doc.metadata.url;
+ await saveCrawl(job.data.crawl_id, sc);
+ }
+
+ if (isUrlBlocked(doc.metadata.url, (await getACUCTeam(job.data.team_id))?.flags ?? null)) {
+ throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
+ }
+
+ const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));
+ const p2 = generateURLPermutations(
+ normalizeURL(doc.metadata.sourceURL, sc),
+ );
+
+ if (JSON.stringify(p1) !== JSON.stringify(p2)) {
+ logger.debug(
+ "Was redirected, removing old URL and locking new URL...",
+ { oldUrl: doc.metadata.sourceURL, newUrl: doc.metadata.url },
+ );
+
+ // Prevent redirect target from being visited in the crawl again
+ // See lockURL
+ const x = await redisEvictConnection.sadd(
+ "crawl:" + job.data.crawl_id + ":visited",
+ ...p1.map((x) => x.href),
+ );
+ const lockRes = x === p1.length;
+
+ if (job.data.crawlerOptions !== null && !lockRes) {
+ throw new RacedRedirectError();
+ }
+ }
+ }
+
+ if (job.data.crawlerOptions !== null) {
+ if (!sc.cancelled) {
+ const crawler = crawlToCrawler(
+ job.data.crawl_id,
+ sc,
+ (await getACUCTeam(job.data.team_id))?.flags ?? null,
+ doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,
+ job.data.crawlerOptions,
+ );
+
+ const links = await crawler.filterLinks(
+ await crawler.extractLinksFromHTML(
+ rawHtml ?? "",
+ doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,
+ ),
+ Infinity,
+ sc.crawlerOptions?.maxDepth ?? 10,
+ );
+ logger.debug("Discovered " + links.links.length + " links...", {
+ linksLength: links.links.length,
+ });
+
+ for (const link of links.links) {
+ if (await lockURL(job.data.crawl_id, sc, link)) {
+ // This seems to work really welel
+ const jobPriority = await getJobPriority({
+ team_id: sc.team_id,
+ basePriority: job.data.crawl_id ? 20 : 10,
+ });
+ const jobId = uuidv4();
+
+ logger.debug(
+ "Determined job priority " +
+ jobPriority +
+ " for URL " +
+ JSON.stringify(link),
+ { jobPriority, url: link },
+ );
+
+ await addScrapeJob(
+ {
+ url: link,
+ mode: "single_urls",
+ team_id: sc.team_id,
+ scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),
+ internalOptions: sc.internalOptions,
+ crawlerOptions: {
+ ...sc.crawlerOptions,
+ currentDiscoveryDepth:
+ (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,
+ },
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ {},
+ jobId,
+ jobPriority,
+ );
+
+ await addCrawlJob(job.data.crawl_id, jobId, logger);
+ logger.debug("Added job for URL " + JSON.stringify(link), {
+ jobPriority,
+ url: link,
+ newJobId: jobId,
+ });
+ } else {
+ // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!
+ // logger.debug("Could not lock URL " + JSON.stringify(link), {
+ // url: link,
+ // });
+ }
+ }
+
+ // Only run check after adding new jobs for discovery - mogery
+ if (job.data.isCrawlSourceScrape) {
+ const filterResult = await crawler.filterLinks(
+ [doc.metadata.url ?? doc.metadata.sourceURL!],
+ 1,
+ sc.crawlerOptions?.maxDepth ?? 10,
+ );
+ if (filterResult.links.length === 0) {
+ const url = doc.metadata.url ?? doc.metadata.sourceURL!;
+ const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";
+ throw new Error(reason);
+ }
+ }
+ }
+ }
+
+ try {
+ signal?.throwIfAborted();
+ } catch (e) {
+ throw new Error("timeout");
+ }
+
+ const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+
+ doc.metadata.creditsUsed = credits_billed ?? undefined;
+
+ logger.debug("Logging job to DB...");
+ await logJob(
+ {
+ job_id: job.id as string,
+ success: true,
+ num_docs: 1,
+ docs: [doc],
+ time_taken: timeTakenInSeconds,
+ team_id: job.data.team_id,
+ mode: job.data.mode,
+ url: job.data.url,
+ crawlerOptions: sc.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ cost_tracking: costTracking,
+ pdf_num_pages: doc.metadata.numPages,
+ credits_billed,
+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ true,
+ job.data.internalOptions?.bypassBilling ?? false,
+ );
+
+ if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {
+ logger.debug("Calling webhook with success...", {
+ webhook: job.data.webhook,
+ });
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id,
+ scrapeId: job.id,
+ data,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
+ });
+ }
+
+ logger.debug("Declaring job as done...");
+ await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
+
+ await finishCrawlIfNeeded(job, sc);
+ } else {
+ try {
+ signal?.throwIfAborted();
+ } catch (e) {
+ throw new Error("timeout");
+ }
+
+ const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+
+ doc.metadata.creditsUsed = credits_billed ?? undefined;
+
+ await logJob({
+ job_id: job.id,
+ success: true,
+ message: "Scrape completed",
+ num_docs: 1,
+ docs: [doc],
+ time_taken: timeTakenInSeconds,
+ team_id: job.data.team_id,
+ mode: "scrape",
+ url: job.data.url,
+ scrapeOptions: job.data.scrapeOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ num_tokens: 0, // TODO: fix
+ cost_tracking: costTracking,
+ pdf_num_pages: doc.metadata.numPages,
+ credits_billed,
+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,
+ zeroDataRetention: job.data.zeroDataRetention,
+ }, false, job.data.internalOptions?.bypassBilling ?? false);
+ }
+
+ logger.info(`πŸ‚ Job done ${job.id}`);
+ return data;
+ } catch (error) {
+ if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+
+ logger.debug("Declaring job as done...");
+ await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);
+ await redisEvictConnection.srem(
+ "crawl:" + job.data.crawl_id + ":visited_unique",
+ normalizeURL(job.data.url, sc),
+ );
+
+ await finishCrawlIfNeeded(job, sc);
+ }
+
+ const isEarlyTimeout =
+ error instanceof Error && error.message === "timeout";
+ const isCancelled =
+ error instanceof Error &&
+ error.message === "Parent crawl/batch scrape was cancelled";
+
+ if (isEarlyTimeout) {
+ logger.error(`πŸ‚ Job timed out ${job.id}`);
+ } else if (error instanceof RacedRedirectError) {
+ logger.warn(`πŸ‚ Job got redirect raced ${job.id}, silently failing`);
+ } else if (isCancelled) {
+ logger.warn(`πŸ‚ Job got cancelled, silently failing`);
+ } else {
+ logger.error(`πŸ‚ Job errored ${job.id} - ${error}`, { error });
+
+ Sentry.captureException(error, {
+ data: {
+ job: job.id,
+ },
+ });
+
+ if (error instanceof CustomError) {
+ // Here we handle the error, then save the failed job
+ logger.error(error.message); // or any other error handling
+ }
+ logger.error(error);
+ if (error.stack) {
+ logger.error(error.stack);
+ }
+ }
+
+ const data = {
+ success: false,
+ document: null,
+ project_id: job.data.project_id,
+ error:
+ error instanceof Error
+ ? error
+ : typeof error === "string"
+ ? new Error(error)
+ : new Error(JSON.stringify(error)),
+ };
+
+ if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id ?? (job.id as string),
+ scrapeId: job.id,
+ data,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
+ });
+ }
+
+ const end = Date.now();
+ const timeTakenInSeconds = (end - start) / 1000;
+
+ const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+
+ logger.debug("Logging job to DB...");
+ await logJob(
+ {
+ job_id: job.id as string,
+ success: false,
+ message:
+ typeof error === "string"
+ ? error
+ : (error.message ??
+ "Something went wrong... Contact [email protected]"),
+ num_docs: 0,
+ docs: [],
+ time_taken: timeTakenInSeconds,
+ team_id: job.data.team_id,
+ mode: job.data.mode,
+ url: job.data.url,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ cost_tracking: costTracking,
+ credits_billed,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ true,
+ job.data.internalOptions?.bypassBilling ?? false,
+ );
+ return data;
+ }
+}
+
+async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: string) {
+ if (sc.crawlerOptions.ignoreSitemap) {
+ return [];
+ }
+
+ const trimmedURL = new URL(url);
+ trimmedURL.search = "";
+
+ const index = await queryIndexAtSplitLevel(
+ sc.crawlerOptions.allowBackwardCrawling ? generateURLSplits(trimmedURL.href)[0] : trimmedURL.href,
+ sc.crawlerOptions.limit ?? 10000,
+ );
+
+ const validIndexLinksResult = await crawler.filterLinks(
+ index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),
+ sc.crawlerOptions.limit ?? 10000,
+ sc.crawlerOptions.maxDepth ?? 10,
+ false,
+ );
+ const validIndexLinks = validIndexLinksResult.links;
+
+ return validIndexLinks;
+}
+
+async function processKickoffJob(job: Job & { id: string }) {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "processKickoffJob",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data?.crawl_id ?? undefined,
+ teamId: job.data?.team_id ?? undefined,
+ zeroDataRetention: job.data.zeroDataRetention ?? false,
+ });
+
+ try {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+
+ logger.debug("Locking URL...");
+ await lockURL(job.data.crawl_id, sc, job.data.url);
+ const jobId = uuidv4();
+ logger.debug("Adding scrape job to Redis...", { jobId });
+ await addScrapeJob(
+ {
+ url: job.data.url,
+ mode: "single_urls",
+ team_id: job.data.team_id,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ isCrawlSourceScrape: true,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ {
+ priority: 15,
+ },
+ jobId,
+ );
+ logger.debug("Adding scrape job to BullMQ...", { jobId });
+ await addCrawlJob(job.data.crawl_id, jobId, logger);
+
+ if (job.data.webhook) {
+ logger.debug("Calling webhook with crawl.started...", {
+ webhook: job.data.webhook,
+ });
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id,
+ data: null,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: "crawl.started",
+ });
+ }
+
+ const sitemap = sc.crawlerOptions.ignoreSitemap
+ ? 0
+ : await crawler.tryGetSitemap(async (urls) => {
+ if (urls.length === 0) return;
+
+ logger.debug("Using sitemap chunk of length " + urls.length, {
+ sitemapLength: urls.length,
+ });
+
+ let jobPriority = await getJobPriority({
+ team_id: job.data.team_id,
+ basePriority: 21,
+ });
+ logger.debug("Using job priority " + jobPriority, { jobPriority });
+
+ const jobs = urls.map((url) => {
+ const uuid = uuidv4();
+ return {
+ name: uuid,
+ data: {
+ url,
+ mode: "single_urls" as const,
+ team_id: job.data.team_id,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ sitemapped: true,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ opts: {
+ jobId: uuid,
+ priority: 20,
+ },
+ };
+ });
+
+ logger.debug("Locking URLs...");
+ const lockedIds = await lockURLsIndividually(
+ job.data.crawl_id,
+ sc,
+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
+ );
+ const lockedJobs = jobs.filter((x) =>
+ lockedIds.find((y) => y.id === x.opts.jobId),
+ );
+ logger.debug("Adding scrape jobs to Redis...");
+ await addCrawlJobs(
+ job.data.crawl_id,
+ lockedJobs.map((x) => x.opts.jobId),
+ logger,
+ );
+ logger.debug("Adding scrape jobs to BullMQ...");
+ await addScrapeJobs(lockedJobs);
+ });
+
+ if (sitemap === 0) {
+ logger.debug("Sitemap not found or ignored.", {
+ ignoreSitemap: sc.crawlerOptions.ignoreSitemap,
+ });
+ }
+
+ const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);
+
+ if (indexLinks.length > 0) {
+ logger.debug("Using index links of length " + indexLinks.length, {
+ indexLinksLength: indexLinks.length,
+ });
+
+ let jobPriority = await getJobPriority({
+ team_id: job.data.team_id,
+ basePriority: 21,
+ });
+ logger.debug("Using job priority " + jobPriority, { jobPriority });
+
+ const jobs = indexLinks.map((url) => {
+ const uuid = uuidv4();
+ return {
+ name: uuid,
+ data: {
+ url,
+ mode: "single_urls" as const,
+ team_id: job.data.team_id,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ sitemapped: true,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ opts: {
+ jobId: uuid,
+ priority: 20,
+ },
+ };
+ });
+
+ logger.debug("Locking URLs...");
+ const lockedIds = await lockURLsIndividually(
+ job.data.crawl_id,
+ sc,
+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
+ );
+ const lockedJobs = jobs.filter((x) =>
+ lockedIds.find((y) => y.id === x.opts.jobId),
+ );
+ logger.debug("Adding scrape jobs to Redis...");
+ await addCrawlJobs(
+ job.data.crawl_id,
+ lockedJobs.map((x) => x.opts.jobId),
+ logger,
+ );
+ logger.debug("Adding scrape jobs to BullMQ...");
+ await addScrapeJobs(lockedJobs);
+ }
+
+ logger.debug("Done queueing jobs!");
+
+ await finishCrawlKickoff(job.data.crawl_id);
+ await finishCrawlIfNeeded(job, sc);
+
+ return { success: true };
+ } catch (error) {
+ logger.error("An error occurred!", { error });
+ await finishCrawlKickoff(job.data.crawl_id);
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ if (sc) {
+ await finishCrawlIfNeeded(job, sc);
+ }
+ return { success: false, error };
+ }
+}
+
+export const processJobInternal = async (job: Job & { id: string }) => {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "processJobInternal",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data?.crawl_id ?? undefined,
+ zeroDataRetention: job.data?.zeroDataRetention ?? false,
+ });
+
+ try {
+ try {
+ let extendLockInterval: NodeJS.Timeout | null = null;
+ if (job.data?.mode !== "kickoff" && job.data?.team_id) {
+ extendLockInterval = setInterval(async () => {
+ await pushConcurrencyLimitActiveJob(job.data.team_id, job.id, 60 * 1000); // 60s lock renew, just like in the queue
+ }, jobLockExtendInterval);
+ }
+
+ await addJobPriority(job.data.team_id, job.id);
+ try {
+ if (job.data?.mode === "kickoff") {
+ const result = await processKickoffJob(job);
+ if (result.success) {
+ return null;
+ } else {
+ throw (result as any).error;
+ }
+ } else {
+ const result = await processJob(job);
+ if (result.success) {
+ try {
+ if (
+ process.env.USE_DB_AUTHENTICATION === "true" &&
+ (job.data.crawl_id || process.env.GCS_BUCKET_NAME)
+ ) {
+ logger.debug(
+ "Job succeeded -- putting null in Redis",
+ );
+ return null;
+ } else {
+ console.log("result.document", result.document);
+ logger.debug("Job succeeded -- putting result in Redis", { document: result.document });
+ return result.document;
+ }
+ } catch (e) { }
+ } else {
+ throw (result as any).error;
+ }
+ }
+ } finally {
+ await deleteJobPriority(job.data.team_id, job.id);
+ if (extendLockInterval) {
+ clearInterval(extendLockInterval);
+ }
+ }
+ } finally {
+ await concurrentJobDone(job);
+ }
+ } catch (error) {
+ logger.debug("Job failed", { error });
+ Sentry.captureException(error);
+ throw error;
+ }
+};
+
+module.exports = processJobInternal;
\ No newline at end of file
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index fb21013e3..00c4641bb 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -67,8 +67,6 @@ export interface RunWebScraperParams {
mode: Mode;
scrapeOptions: ScrapeOptions;
internalOptions?: InternalOptions;
- // onSuccess: (result: V1Document, mode: string) => void;
- // onError: (error: Error) => void;
team_id: string;
bull_job_id: string;
priority?: number;