Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
queue-worker: move scrape worker to separate threads (ENG-3008) (#1879)
GergΕ MΓ³ricz0f44d947e1dc
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.ymlindex c8d406bad..be890a897 100644--- a/.github/workflows/test-server-self-host.yml+++ b/.github/workflows/test-server-self-host.yml@@ -101,14 +101,17 @@ jobs:docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxngpnpx wait-on tcp:3434 -t 30sworking-directory: ./+ - name: Build the application+ run: npm run build+ working-directory: ./apps/api- name: Start server- run: npm run start:production > api.log 2>&1 &+ run: npm run start:production:nobuild > api.log 2>&1 &env:PORT: 3002HOST: 0.0.0.0working-directory: ./apps/api- name: Start worker- run: npm run workers > worker.log 2>&1 &+ run: npm run worker:production > worker.log 2>&1 &env:PORT: 3005HOST: 0.0.0.0diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 6bfc91add..0c7a01c9f 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -102,22 +102,25 @@ jobs:cargo build --releasechmod +x target/release/libcrawler.soworking-directory: ./apps/api/sharedLibs/crawler+ - name: Build the application+ run: npm run build+ working-directory: ./apps/api- name: Start the application- run: npm run start:production > api.log 2>&1 &+ run: npm run start:production:nobuild > api.log 2>&1 &env:PORT: 3002HOST: 0.0.0.0working-directory: ./apps/apiid: start_app- name: Start worker- run: npm run workers > worker.log 2>&1 &+ run: npm run worker:production > worker.log 2>&1 &env:PORT: 3005HOST: 0.0.0.0working-directory: ./apps/apiid: start_workers- name: Start index worker- run: npm run index-worker > index-worker.log 2>&1 &+ run: npm run index-worker:production > index-worker.log 2>&1 &working-directory: ./apps/apiid: start_index_worker- name: Wait for APIdiff --git a/apps/api/package.json b/apps/api/package.jsonindex 28b1bc5c9..5d724dc75 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -4,11 +4,12 @@"description": "","main": "index.js","scripts": {- "start": "nodemon --exec ts-node src/index.ts",+ "start": "tsc-watch --onSuccess \"node dist/src/index.js\"","start:production": "tsc && node dist/src/index.js",+ "start:production:nobuild": "node dist/src/index.js","format": "prettier --write \"src/**/*.(js|ts)\"","flyio": "node dist/src/index.js",- "start:dev": "nodemon --exec ts-node src/index.ts",+ "start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"","build": "tsc","build:nosentry": "tsc","test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'",@@ -16,9 +17,9 @@"test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'","test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'","test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/*.test.ts",- "workers": "nodemon --exec ts-node src/services/queue-worker.ts",+ "workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"","worker:production": "node dist/src/services/queue-worker.js",- "index-worker": "nodemon --exec ts-node src/services/indexing/index-worker.ts",+ "index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"","index-worker:production": "node dist/src/services/indexing/index-worker.js","mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest","mongo-docker-console": "docker exec -it mongodb mongosh",@@ -44,11 +45,11 @@"@types/supertest": "^6.0.2","jest": "^29.6.3","jest-fetch-mock": "^3.0.3",- "nodemon": "^3.1.10","prettier": "^3.4.2","supertest": "^6.3.3","ts-jest": "^29.1.1","ts-node": "^10.9.1",+ "tsc-watch": "^7.1.1","typescript": "^5.8.3"},"dependencies": {diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 82854b6c8..1567f780b 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -321,9 +321,6 @@ importers:jest-fetch-mock:specifier: ^3.0.3version: 3.0.3([email protected])- nodemon:- specifier: ^3.1.10- version: 3.1.10prettier:specifier: ^3.4.2version: 3.4.2@@ -336,6 +333,9 @@ importers:ts-node:specifier: ^10.9.1version: 10.9.2(@types/[email protected])([email protected])+ tsc-watch:+ specifier: ^7.1.1+ version: 7.1.1([email protected])typescript:specifier: ^5.8.3version: 5.8.3@@ -1964,10 +1964,6 @@ packages:resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}- resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}- engines: {node: '>=8'}-resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==}@@ -2080,10 +2076,6 @@ packages:resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}engines: {node: '>= 6'}- resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}- engines: {node: '>= 8.10.0'}-resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==}engines: {node: '>=18'}@@ -2368,6 +2360,9 @@ packages:resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==}+resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==}@@ -2471,6 +2466,9 @@ packages:resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}engines: {node: '>= 0.6'}+ resolution: {integrity: sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==}+resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}engines: {node: '>=6'}@@ -2612,6 +2610,9 @@ packages:resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==}engines: {node: '>= 0.6'}+ resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==}+resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}@@ -2663,10 +2664,6 @@ packages:resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}engines: {node: '>= 4.8.0'}- resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}- engines: {node: '>= 6'}-resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==}engines: {node: '>=16 || 14 >=14.18'}@@ -2806,9 +2803,6 @@ packages:resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}- resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==}-resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}@@ -2856,20 +2850,12 @@ packages:resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}- resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}- engines: {node: '>=8'}-resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==}resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}- resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}- engines: {node: '>=0.10.0'}-resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}engines: {node: '>=8'}@@ -2878,10 +2864,6 @@ packages:resolution: {integrity: sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==}engines: {node: '>=6'}- resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}- engines: {node: '>=0.10.0'}-resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}engines: {node: '>=0.12.0'}@@ -3249,6 +3231,9 @@ packages:engines: {node: '>=12.0.0'}hasBin: true+ resolution: {integrity: sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==}+resolution: {integrity: sha512-f3r0yqpz31VXiDB/wj9GaOB0a2PRLQl6vJmXiFrniNwjkKdvakqJRULhjFKJpxOchlCRiG5fcacoUZY5Xa6PEQ==}engines: {node: '>= 18'}@@ -3378,6 +3363,9 @@ packages:resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}+ resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==}+resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}engines: {node: '>=10.5.0'}@@ -3408,11 +3396,6 @@ packages:resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==}- resolution: {integrity: sha512-WDjw3pJ0/0jMFmyNDp3gvY2YizjLmmOUQo6DEBY+JgdvW/yQ9mEeSw6H5ythl5Ny2ytb7f9C2nIbjSxMNzbJXw==}- engines: {node: '>=10'}- hasBin: true-resolution: {integrity: sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}@@ -3543,6 +3526,9 @@ packages:resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==}+ resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==}+resolution: {integrity: sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==}@@ -3643,12 +3629,14 @@ packages:resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}+ resolution: {integrity: sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==}+ engines: {node: '>= 0.10'}+ hasBin: true+resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==}- resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==}-resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}engines: {node: '>=6'}@@ -3713,10 +3701,6 @@ packages:resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==}engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}- resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}- engines: {node: '>=8.10.0'}-resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==}engines: {node: '>= 0.10'}@@ -3897,10 +3881,6 @@ packages:resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}- resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==}- engines: {node: '>=10'}-resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}@@ -3915,6 +3895,9 @@ packages:resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}engines: {node: '>=0.10.0'}+ resolution: {integrity: sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==}+resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}@@ -3932,12 +3915,19 @@ packages:resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}engines: {node: '>= 0.8'}+ resolution: {integrity: sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==}+resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==}resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==}+ resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==}+ engines: {node: '>=0.6.19'}+resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==}engines: {node: '>=10'}@@ -4049,6 +4039,9 @@ packages:resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==}engines: {node: '>=18'}+ resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}+resolution: {integrity: sha512-AOvV5YYIAFFBfransBzSTyztkc3IMfz5Eq3YluaRiEu55nn43Fzaufx70UqEKYr8BoLCach4q8g/bg6e5+/aFw==}@@ -4071,10 +4064,6 @@ packages:resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}engines: {node: '>=0.6'}- resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==}- hasBin: true-resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==}engines: {node: '>=16'}@@ -4128,6 +4117,13 @@ packages:'@swc/wasm':optional: true+ resolution: {integrity: sha512-r6t37Dkk4vK44HwxOe+OzjpE/gDamZAwqXhtcAJD/hPVblcjJK45NxbK0HcDASXG0U4pEnCh640JZbeDVSC6yA==}+ engines: {node: '>=12.12.0'}+ hasBin: true+ peerDependencies:+ typescript: '*'+resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==}@@ -4157,9 +4153,6 @@ packages:engines: {node: '>=14.17'}hasBin: true- resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==}-resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}@@ -5114,7 +5107,7 @@ snapshots:'@babel/traverse': 7.24.6'@babel/types': 7.24.6convert-source-map: 2.0.0- debug: 4.4.1([email protected])+ debug: 4.4.1gensync: 1.0.0-beta.2json5: 2.2.3semver: 6.3.1@@ -5301,7 +5294,7 @@ snapshots:'@babel/helper-split-export-declaration': 7.24.6'@babel/parser': 7.24.6'@babel/types': 7.24.6- debug: 4.4.1([email protected])+ debug: 4.4.1globals: 11.12.0transitivePeerDependencies:- supports-color@@ -6654,7 +6647,7 @@ snapshots:dependencies:- debug: 4.4.1([email protected])+ debug: 4.4.1transitivePeerDependencies:- supports-color@@ -6816,8 +6809,6 @@ snapshots:read-cmd-shim: 4.0.0write-file-atomic: 5.0.1- [email protected]: {}-@@ -6958,18 +6949,6 @@ snapshots:parse5: 7.1.2parse5-htmlparser2-tree-adapter: 7.0.0- dependencies:- anymatch: 3.1.3- braces: 3.0.3- glob-parent: 5.1.2- is-binary-path: 2.1.0- is-glob: 4.0.3- normalize-path: 3.0.0- readdirp: 3.6.0- optionalDependencies:- fsevents: 2.3.3-@@ -7147,11 +7126,9 @@ snapshots:dependencies:ms: 2.1.2dependencies:ms: 2.1.3- optionalDependencies:- supports-color: 5.5.0@@ -7233,6 +7210,8 @@ snapshots:es-errors: 1.3.0gopd: 1.2.0+ [email protected]: {}+dependencies:end-of-stream: 1.4.4@@ -7319,6 +7298,16 @@ snapshots:+ dependencies:+ duplexer: 0.1.2+ from: 0.1.7+ map-stream: 0.1.0+ pause-stream: 0.0.11+ split: 0.3.3+ stream-combiner: 0.0.4+ through: 2.3.8+@@ -7499,6 +7488,8 @@ snapshots:+ [email protected]: {}+@@ -7568,10 +7559,6 @@ snapshots:shelljs: 0.8.5shelljs.exec: 1.1.8- dependencies:- is-glob: 4.0.3-dependencies:foreground-child: 3.2.1@@ -7690,7 +7677,7 @@ snapshots:dependencies:'@tootallnate/once': 2.0.0agent-base: 6.0.2- debug: 4.4.1([email protected])+ debug: 4.4.1transitivePeerDependencies:- supports-color@@ -7711,7 +7698,7 @@ snapshots:dependencies:agent-base: 7.1.3- debug: 4.4.1([email protected])+ debug: 4.4.1transitivePeerDependencies:- supports-color@@ -7738,8 +7725,6 @@ snapshots:- [email protected]: {}-@@ -7771,7 +7756,7 @@ snapshots:dependencies:'@ioredis/commands': 1.2.0cluster-key-slot: 1.1.2- debug: 4.4.1([email protected])+ debug: 4.4.1denque: 2.1.0lodash.defaults: 4.2.0lodash.isarguments: 3.1.0@@ -7789,26 +7774,16 @@ snapshots:- dependencies:- binary-extensions: 2.3.0-dependencies:hasown: 2.0.2- [email protected]: {}-- dependencies:- is-extglob: 2.1.1-@@ -7862,7 +7837,7 @@ snapshots:dependencies:- debug: 4.4.1([email protected])+ debug: 4.4.1istanbul-lib-coverage: 3.2.2source-map: 0.6.1transitivePeerDependencies:@@ -8396,6 +8371,8 @@ snapshots:underscore: 1.13.6xmlbuilder: 10.1.1+ [email protected]: {}+@@ -8498,6 +8475,8 @@ snapshots:+ [email protected]: {}+@@ -8523,19 +8502,6 @@ snapshots:- dependencies:- chokidar: 3.6.0- debug: 4.4.1([email protected])- ignore-by-default: 1.0.1- minimatch: 3.1.2- pstree.remy: 1.1.8- semver: 7.6.2- simple-update-notifier: 2.0.0- supports-color: 5.5.0- touch: 3.1.1- undefsafe: 2.0.5-dependencies:abbrev: 2.0.0@@ -8650,6 +8616,10 @@ snapshots:+ dependencies:+ through: 2.3.8+dependencies:'@pdf-lib/standard-fonts': 1.0.0@@ -8739,12 +8709,14 @@ snapshots:+ dependencies:+ event-stream: 3.3.4+dependencies:punycode: 2.3.1- [email protected]: {}-@@ -8818,10 +8790,6 @@ snapshots:process: 0.11.10string_decoder: 1.3.0- dependencies:- picomatch: 2.3.1-dependencies:resolve: 1.22.8@@ -8846,7 +8814,7 @@ snapshots:dependencies:- debug: 4.4.1([email protected])+ debug: 4.4.1module-details-from-path: 1.0.3resolve: 1.22.8transitivePeerDependencies:@@ -9018,10 +8986,6 @@ snapshots:dependencies:is-arrayish: 0.3.2- dependencies:- semver: 7.6.2-@@ -9033,6 +8997,10 @@ snapshots:+ dependencies:+ through: 2.3.8+@@ -9045,12 +9013,18 @@ snapshots:+ dependencies:+ duplexer: 0.1.2+dependencies:stubs: 3.0.0+ [email protected]: {}+dependencies:char-regex: 1.0.2@@ -9184,6 +9158,8 @@ snapshots:+ [email protected]: {}+@@ -9200,8 +9176,6 @@ snapshots:- [email protected]: {}-dependencies:tldts: 6.1.75@@ -9250,6 +9224,14 @@ snapshots:v8-compile-cache-lib: 3.0.1yn: 3.1.1+ dependencies:+ cross-spawn: 7.0.6+ node-cleanup: 2.1.2+ ps-tree: 1.2.0+ string-argv: 0.3.2+ typescript: 5.8.3+@@ -9271,8 +9253,6 @@ snapshots:- [email protected]: {}-diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex f899d4e31..d3a967de4 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -20,11 +20,9 @@ configDotenv();export async function startWebScraperPipeline({job,- token,costTracking,}: {job: Job<WebScraperOptions> & { id: string };- token: string;costTracking: CostTracking;}) {return await runWebScraper({@@ -43,14 +41,6 @@ export async function startWebScraperPipeline({teamId: job.data.team_id,...job.data.internalOptions,},- // onSuccess: (result, mode) => {- // logger.debug(`π Job completed ${job.id}`);- // saveJob(job, result, token, mode);- // },- // onError: (error) => {- // logger.error(`π Job failed ${job.id}`);- // ScrapeEvents.logJobEvent(job, "failed");- // },team_id: job.data.team_id,bull_job_id: job.id.toString(),priority: job.opts.priority,@@ -66,8 +56,6 @@ export async function runWebScraper({mode,scrapeOptions,internalOptions,- // onSuccess,- // onError,team_id,bull_job_id,priority,@@ -128,9 +116,6 @@ export async function runWebScraper({}}- // This is where the returnvalue from the job is set- // onSuccess(response.document, mode);-engines = response.engines;if (@@ -205,7 +190,6 @@ export async function runWebScraper({const saveJob = async (job: Job,result: any,- token: string,mode: string,engines?: EngineResultsTracker,) => {diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.tsindex fee2b45d5..39a4db931 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts@@ -12,7 +12,7 @@ export async function extractMetadataRust(return {...fromRust,...(fromRust.favicon ? {- favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url)+ favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url).href} : {}),scrapeId: meta.id,};diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex c2b16778f..ab184efb6 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -72,7 +72,6 @@ export function getExtractQueue() {},},});- logger.info("Extraction queue created");}return extractQueue;}@@ -90,7 +89,6 @@ export function getGenerateLlmsTxtQueue() {},},});- logger.info("LLMs TXT generation queue created");}return generateLlmsTxtQueue;}@@ -108,7 +106,6 @@ export function getDeepResearchQueue() {},},});- logger.info("Deep research queue created");}return deepResearchQueue;}@@ -126,7 +123,6 @@ export function getBillingQueue() {},},});- logger.info("Billing queue created");}return billingQueue;}@@ -144,7 +140,6 @@ export function getPrecrawlQueue() {},},});- logger.info("Precrawl queue created");}return precrawlQueue;}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex de96f194b..53580f652 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1,99 +1,53 @@import "dotenv/config";import "./sentry";import * as Sentry from "@sentry/node";-import { CustomError } from "../lib/custom-error";import {getScrapeQueue,getExtractQueue,getDeepResearchQueue,redisConnection,getGenerateLlmsTxtQueue,- getBillingQueue,scrapeQueueName,} from "./queue-service";-import { startWebScraperPipeline } from "../main/runWebScraper";-import { callWebhook } from "./webhook";-import { logJob } from "./logging/log_job";import { Job, Queue, QueueEvents } from "bullmq";import { logger as _logger } from "../lib/logger";import { Worker } from "bullmq";import systemMonitor from "./system-monitor";import { v4 as uuidv4 } from "uuid";import {- addCrawlJob,addCrawlJobDone,- addCrawlJobs,- crawlToCrawler,- finishCrawl,- finishCrawlPre,- unPreFinishCrawl,finishCrawlKickoff,- generateURLPermutations,getCrawl,- getCrawlJobs,- getDoneJobsOrderedLength,- lockURL,- lockURLsIndividually,normalizeURL,- saveCrawl,} from "../lib/crawl-redis";import { StoredCrawl } from "../lib/crawl-redis";-import { addScrapeJob, addScrapeJobs } from "./queue-jobs";-import {- addJobPriority,- deleteJobPriority,- getJobPriority,-} from "../../src/lib/job-priority";-import { getJobs } from "..//controllers/v1/crawl-status";import { configDotenv } from "dotenv";-import { scrapeOptions } from "../controllers/v1/types";import {concurrentJobDone,- pushConcurrencyLimitActiveJob,} from "../lib/concurrency-limit";-import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";-import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";-import { Document, TeamFlags } from "../controllers/v1/types";import {ExtractResult,performExtraction,} from "../lib/extract/extraction-service";-import { supabase_service } from "../services/supabase";-import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";-import { saveExtract, updateExtract } from "../lib/extract/extract-redis";-import { billTeam } from "./billing/credit_billing";+import { updateExtract } from "../lib/extract/extract-redis";import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";import { performDeepResearch } from "../lib/deep-research/deep-research-service";import { performGenerateLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-service";import { updateGeneratedLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-redis";import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";-import { CostTracking } from "../lib/extract/extraction-service";-import { getACUCTeam } from "../controllers/auth";import Express from "express";import http from "http";import https from "https";import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";-import { RateLimiterMode } from "../types";-import { calculateCreditsToBeBilled } from "../lib/scrape-billing";import { redisEvictConnection } from "./redis";-import { generateURLSplits, queryIndexAtSplitLevel } from "./index";-import { WebCrawler } from "../scraper/WebScraper/crawler";-import type { Logger } from "winston";+import path from "path";+import { finishCrawlIfNeeded } from "./worker/crawl-logic";configDotenv();-class RacedRedirectError extends Error {- constructor() {- super("Raced redirect error");- }-}-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));-const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;-const workerStalledCheckInterval =- Number(process.env.WORKER_STALLED_CHECK_INTERVAL) || 30000;const jobLockExtendInterval =Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;const jobLockExtensionTime =@@ -111,347 +65,6 @@ const runningJobs: Set<string> = new Set();cacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);-async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {- const logger = _logger.child({- module: "queue-worker",- method: "finishCrawlIfNeeded",- jobId: job.id,- scrapeId: job.id,- crawlId: job.data.crawl_id,- zeroDataRetention: sc.internalOptions.zeroDataRetention,- });-- if (await finishCrawlPre(job.data.crawl_id, logger)) {- logger.info("Crawl is pre-finished, checking if we need to add more jobs");- if (- job.data.crawlerOptions &&- !(await redisEvictConnection.exists(- "crawl:" + job.data.crawl_id + ":invisible_urls",- ))- ) {- await redisEvictConnection.set(- "crawl:" + job.data.crawl_id + ":invisible_urls",- "done",- "EX",- 60 * 60 * 24,- );-- const sc = (await getCrawl(job.data.crawl_id))!;-- const visitedUrls = new Set(- await redisEvictConnection.smembers(- "crawl:" + job.data.crawl_id + ":visited_unique",- ),- );-- logger.info("Visited URLs", {- visitedUrls: visitedUrls.size,- });-- let lastUrls: string[] = [];- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {- lastUrls = (- (- await supabase_service.rpc("diff_get_last_crawl_urls", {- i_team_id: job.data.team_id,- i_url: sc.originUrl!,- })- ).data ?? []- ).map((x) => x.url);- }-- const lastUrlsSet = new Set(lastUrls);-- logger.info("Last URLs", {- lastUrls: lastUrlsSet.size,- });-- const crawler = crawlToCrawler(- job.data.crawl_id,- sc,- (await getACUCTeam(job.data.team_id))?.flags ?? null,- sc.originUrl!,- job.data.crawlerOptions,- );-- const univistedUrls = await crawler.filterLinks(- Array.from(lastUrlsSet).filter((x) => !visitedUrls.has(x)),- Infinity,- sc.crawlerOptions.maxDepth ?? 10,- );-- const addableJobCount =- sc.crawlerOptions.limit === undefined- ? Infinity- : sc.crawlerOptions.limit -- (await getDoneJobsOrderedLength(job.data.crawl_id));-- if (univistedUrls.links.length !== 0 && addableJobCount > 0) {- logger.info("Adding jobs", {- univistedUrls: univistedUrls.links.length,- addableJobCount,- });-- const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {- const uuid = uuidv4();- return {- name: uuid,- data: {- url,- mode: "single_urls" as const,- team_id: job.data.team_id,- crawlerOptions: {- ...job.data.crawlerOptions,- urlInvisibleInCurrentCrawl: true,- },- scrapeOptions: job.data.scrapeOptions,- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- sitemapped: true,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- },- opts: {- jobId: uuid,- priority: 20,- },- };- });-- const lockedIds = await lockURLsIndividually(- job.data.crawl_id,- sc,- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),- );- const lockedJobs = jobs.filter((x) =>- lockedIds.find((y) => y.id === x.opts.jobId),- );- await addCrawlJobs(- job.data.crawl_id,- lockedJobs.map((x) => x.opts.jobId),- logger,- );- await addScrapeJobs(lockedJobs);-- if (lockedJobs.length > 0) {- logger.info("Added jobs, not going for the full finish", {- lockedJobs: lockedJobs.length,- });-- await unPreFinishCrawl(job.data.crawl_id);- return;- } else {- logger.info("No jobs added (all discovered URLs were locked), finishing crawl");- }- }- }-- logger.info("Finishing crawl");- await finishCrawl(job.data.crawl_id, logger);-- if (!job.data.v1) {- const jobIDs = await getCrawlJobs(job.data.crawl_id);-- const jobs = (await getJobs(jobIDs)).sort(- (a, b) => a.timestamp - b.timestamp,- );- // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));- const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")- ? "failed"- : "completed";-- const fullDocs = jobs- .map((x) =>- x.returnvalue- ? Array.isArray(x.returnvalue)- ? x.returnvalue[0]- : x.returnvalue- : null,- )- .filter((x) => x !== null);-- await logJob({- job_id: job.data.crawl_id,- success: jobStatus === "completed",- message: sc.cancelled ? "Cancelled" : undefined,- num_docs: fullDocs.length,- docs: [],- time_taken: (Date.now() - sc.createdAt) / 1000,- team_id: job.data.team_id,- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",- url: sc.originUrl!,- scrapeOptions: sc.scrapeOptions,- crawlerOptions: sc.crawlerOptions,- origin: job.data.origin,- integration: job.data.integration,- zeroDataRetention: job.data.zeroDataRetention,- }, false, job.data.internalOptions?.bypassBilling ?? false);-- const data = {- success: jobStatus !== "failed",- result: {- links: fullDocs.map((doc) => {- return {- content: doc,- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",- };- }),- },- project_id: job.data.project_id,- docs: fullDocs,- };-- // v0 web hooks, call when done with all the data- if (!job.data.v1) {- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id,- data,- webhook: job.data.webhook,- v1: job.data.v1,- eventType: job.data.crawlerOptions !== null- ? "crawl.completed"- : "batch_scrape.completed",- });- }- } else {- const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);- const jobStatus = sc.cancelled ? "failed" : "completed";-- let credits_billed = null;-- if (process.env.USE_DB_AUTHENTICATION === "true") {- const creditsRpc = await supabase_service- .rpc("credits_billed_by_crawl_id_1", {- i_crawl_id: job.data.crawl_id,- });-- credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;-- if (credits_billed === null) {- logger.warn("Credits billed is null", {- error: creditsRpc.error,- });- }- }-- await logJob(- {- job_id: job.data.crawl_id,- success: jobStatus === "completed",- message: sc.cancelled ? "Cancelled" : undefined,- num_docs,- docs: [],- time_taken: (Date.now() - sc.createdAt) / 1000,- team_id: job.data.team_id,- scrapeOptions: sc.scrapeOptions,- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",- url:- sc?.originUrl ??- (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),- crawlerOptions: sc.crawlerOptions,- origin: job.data.origin,- integration: job.data.integration,- credits_billed,- zeroDataRetention: job.data.zeroDataRetention,- },- true,- job.data.internalOptions?.bypassBilling ?? false,- );--- // v1 web hooks, call when done with no data, but with event completed- if (job.data.v1 && job.data.webhook) {- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id,- data: [],- webhook: job.data.webhook,- v1: job.data.v1,- eventType: job.data.crawlerOptions !== null- ? "crawl.completed"- : "batch_scrape.completed",- });- }- }- }-}--const processJobInternal = async (token: string, job: Job & { id: string }) => {- const logger = _logger.child({- module: "queue-worker",- method: "processJobInternal",- jobId: job.id,- scrapeId: job.id,- crawlId: job.data?.crawl_id ?? undefined,- zeroDataRetention: job.data?.zeroDataRetention ?? false,- });-- const extendLockInterval = setInterval(async () => {- logger.info(`π Worker extending lock on job ${job.id}`, {- extendInterval: jobLockExtendInterval,- extensionTime: jobLockExtensionTime,- });-- if (job.data?.mode !== "kickoff" && job.data?.team_id) {- await pushConcurrencyLimitActiveJob(job.data.team_id, job.id, 60 * 1000); // 60s lock renew, just like in the queue- }-- await job.extendLock(token, jobLockExtensionTime);- }, jobLockExtendInterval);-- await addJobPriority(job.data.team_id, job.id);- let err = null;- try {- if (job.data?.mode === "kickoff") {- const result = await processKickoffJob(job, token);- if (result.success) {- try {- await job.moveToCompleted(null, token, false);- } catch (e) {}- } else {- logger.debug("Job failed", { result, mode: job.data.mode });- await job.moveToFailed((result as any).error, token, false);- }- } else {- const result = await processJob(job, token);- if (result.success) {- try {- if (- process.env.USE_DB_AUTHENTICATION === "true" &&- (job.data.crawl_id || process.env.GCS_BUCKET_NAME)- ) {- logger.debug(- "Job succeeded -- putting null in Redis",- );- await job.moveToCompleted(null, token, false);- } else {- logger.debug("Job succeeded -- putting result in Redis");- await job.moveToCompleted(result.document, token, false);- }- } catch (e) {}- } else {- logger.debug("Job failed", { result });- await job.moveToFailed((result as any).error, token, false);- }- }- } catch (error) {- logger.debug("Job failed", { error });- Sentry.captureException(error);- err = error;- await job.moveToFailed(error, token, false);- } finally {- await deleteJobPriority(job.data.team_id, job.id);- clearInterval(extendLockInterval);- }-- return err;-};-const processExtractJobInternal = async (token: string,job: Job & { id: string },@@ -714,6 +327,22 @@ process.on("SIGTERM", () => {let cantAcceptConnectionCount = 0;+const separateWorkerFun = (+ queue: Queue,+ path: string,+): Worker => {+ const worker = new Worker(queue.name, path, {+ connection: redisConnection,+ lockDuration: 30 * 1000, // 30 seconds+ stalledInterval: 30 * 1000, // 30 seconds+ maxStalledCount: 10, // 10 times+ concurrency: 6, // from k8s setup+ useWorkerThreads: true,+ });++ return worker;+}+const workerFun = async (queue: Queue,processJobInternal: (token: string, job: Job) => Promise<any>,@@ -849,741 +478,6 @@ const workerFun = async (}};-async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: string) {- if (sc.crawlerOptions.ignoreSitemap) {- return [];- }-- const trimmedURL = new URL(url);- trimmedURL.search = "";-- const index = await queryIndexAtSplitLevel(- sc.crawlerOptions.allowBackwardCrawling ? generateURLSplits(trimmedURL.href)[0] : trimmedURL.href,- sc.crawlerOptions.limit ?? 10000,- );-- const validIndexLinksResult = await crawler.filterLinks(- index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),- sc.crawlerOptions.limit ?? 10000,- sc.crawlerOptions.maxDepth ?? 10,- false,- );- const validIndexLinks = validIndexLinksResult.links;-- return validIndexLinks;-}--async function processKickoffJob(job: Job & { id: string }, token: string) {- const logger = _logger.child({- module: "queue-worker",- method: "processKickoffJob",- jobId: job.id,- scrapeId: job.id,- crawlId: job.data?.crawl_id ?? undefined,- teamId: job.data?.team_id ?? undefined,- zeroDataRetention: job.data.zeroDataRetention ?? false,- });-- try {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);-- logger.debug("Locking URL...");- await lockURL(job.data.crawl_id, sc, job.data.url);- const jobId = uuidv4();- logger.debug("Adding scrape job to Redis...", { jobId });- await addScrapeJob(- {- url: job.data.url,- mode: "single_urls",- team_id: job.data.team_id,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- webhook: job.data.webhook,- v1: job.data.v1,- isCrawlSourceScrape: true,- zeroDataRetention: job.data.zeroDataRetention,- },- {- priority: 15,- },- jobId,- );- logger.debug("Adding scrape job to BullMQ...", { jobId });- await addCrawlJob(job.data.crawl_id, jobId, logger);-- if (job.data.webhook) {- logger.debug("Calling webhook with crawl.started...", {- webhook: job.data.webhook,- });- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id,- data: null,- webhook: job.data.webhook,- v1: job.data.v1,- eventType: "crawl.started",- });- }-- const sitemap = sc.crawlerOptions.ignoreSitemap- ? 0- : await crawler.tryGetSitemap(async (urls) => {- if (urls.length === 0) return;-- logger.debug("Using sitemap chunk of length " + urls.length, {- sitemapLength: urls.length,- });-- let jobPriority = await getJobPriority({- team_id: job.data.team_id,- basePriority: 21,- });- logger.debug("Using job priority " + jobPriority, { jobPriority });-- const jobs = urls.map((url) => {- const uuid = uuidv4();- return {- name: uuid,- data: {- url,- mode: "single_urls" as const,- team_id: job.data.team_id,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- sitemapped: true,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- },- opts: {- jobId: uuid,- priority: 20,- },- };- });-- logger.debug("Locking URLs...");- const lockedIds = await lockURLsIndividually(- job.data.crawl_id,- sc,- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),- );- const lockedJobs = jobs.filter((x) =>- lockedIds.find((y) => y.id === x.opts.jobId),- );- logger.debug("Adding scrape jobs to Redis...");- await addCrawlJobs(- job.data.crawl_id,- lockedJobs.map((x) => x.opts.jobId),- logger,- );- logger.debug("Adding scrape jobs to BullMQ...");- await addScrapeJobs(lockedJobs);- });-- if (sitemap === 0) {- logger.debug("Sitemap not found or ignored.", {- ignoreSitemap: sc.crawlerOptions.ignoreSitemap,- });- }-- const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);-- if (indexLinks.length > 0) {- logger.debug("Using index links of length " + indexLinks.length, {- indexLinksLength: indexLinks.length,- });-- let jobPriority = await getJobPriority({- team_id: job.data.team_id,- basePriority: 21,- });- logger.debug("Using job priority " + jobPriority, { jobPriority });-- const jobs = indexLinks.map((url) => {- const uuid = uuidv4();- return {- name: uuid,- data: {- url,- mode: "single_urls" as const,- team_id: job.data.team_id,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- sitemapped: true,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- },- opts: {- jobId: uuid,- priority: 20,- },- };- });-- logger.debug("Locking URLs...");- const lockedIds = await lockURLsIndividually(- job.data.crawl_id,- sc,- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),- );- const lockedJobs = jobs.filter((x) =>- lockedIds.find((y) => y.id === x.opts.jobId),- );- logger.debug("Adding scrape jobs to Redis...");- await addCrawlJobs(- job.data.crawl_id,- lockedJobs.map((x) => x.opts.jobId),- logger,- );- logger.debug("Adding scrape jobs to BullMQ...");- await addScrapeJobs(lockedJobs);- }-- logger.debug("Done queueing jobs!");-- await finishCrawlKickoff(job.data.crawl_id);- await finishCrawlIfNeeded(job, sc);-- return { success: true };- } catch (error) {- logger.error("An error occurred!", { error });- await finishCrawlKickoff(job.data.crawl_id);- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- if (sc) {- await finishCrawlIfNeeded(job, sc);- }- return { success: false, error };- }-}--async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) {- let creditsToBeBilled: number | null = null;-- if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {- creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags);-- if (- job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&- process.env.USE_DB_AUTHENTICATION === "true"- ) {- try {- const billingJobId = uuidv4();- logger.debug(- `Adding billing job to queue for team ${job.data.team_id}`,- {- billingJobId,- credits: creditsToBeBilled,- is_extract: false,- },- );-- // Add directly to the billing queue - the billing worker will handle the rest- await getBillingQueue().add(- "bill_team",- {- team_id: job.data.team_id,- subscription_id: undefined,- credits: creditsToBeBilled,- is_extract: false,- timestamp: new Date().toISOString(),- originating_job_id: job.id,- },- {- jobId: billingJobId,- priority: 10,- },- );- return creditsToBeBilled;- } catch (error) {- logger.error(- `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,- { error },- );- Sentry.captureException(error);- return creditsToBeBilled;- }- }- }-- return creditsToBeBilled;-}--async function processJob(job: Job & { id: string }, token: string) {- const logger = _logger.child({- module: "queue-worker",- method: "processJob",- jobId: job.id,- scrapeId: job.id,- crawlId: job.data?.crawl_id ?? undefined,- teamId: job.data?.team_id ?? undefined,- zeroDataRetention: job.data?.zeroDataRetention ?? false,- });- logger.info(`π Worker taking job ${job.id}`, { url: job.data.url, sitemapped: job.data.sitemapped });- const start = job.data.startTime ?? Date.now();- const remainingTime = job.data.scrapeOptions.timeout ? (job.data.scrapeOptions.timeout - (Date.now() - start)) : undefined;-- const costTracking = new CostTracking();-- try {- job.updateProgress({- current: 1,- total: 100,- current_step: "SCRAPING",- current_url: "",- });-- if (remainingTime !== undefined && remainingTime < 0) {- throw new Error("timeout");- }- const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;-- if (job.data.crawl_id) {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- if (sc && sc.cancelled) {- throw new Error("Parent crawl/batch scrape was cancelled");- }- }--- const pipeline = await Promise.race([- startWebScraperPipeline({- job,- token,- costTracking,- }),- ...(remainingTime !== undefined- ? [- (async () => {- await sleep(remainingTime);- throw new Error("timeout");- })(),- ]- : []),- ]);-- try {- signal?.throwIfAborted();- } catch (e) {- throw new Error("timeout");- }-- if (!pipeline.success) {- throw pipeline.error;- }-- const end = Date.now();- const timeTakenInSeconds = (end - start) / 1000;-- const doc = pipeline.document;-- const rawHtml = doc.rawHtml ?? "";-- if (!job.data.scrapeOptions.formats.includes("rawHtml")) {- delete doc.rawHtml;- }-- if (job.data.concurrencyLimited) {- doc.warning =- "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." +- (doc.warning ? " " + doc.warning : "");- }-- const data = {- success: true,- result: {- links: [- {- content: doc,- source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",- id: job.id,- },- ],- },- project_id: job.data.project_id,- document: doc,- };-- if (job.data.crawl_id) {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;-- if (- doc.metadata.url !== undefined &&- doc.metadata.sourceURL !== undefined &&- normalizeURL(doc.metadata.url, sc) !==- normalizeURL(doc.metadata.sourceURL, sc) &&- job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape- ) {- const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);- const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);- if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {- const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";- throw new Error(reason);- }-- // Only re-set originUrl if it's different from the current hostname- // This is only done on this condition to handle cross-domain redirects- // If this would be done for non-crossdomain redirects, but also for e.g.- // redirecting / -> /introduction (like our docs site does), it would- // break crawling the entire site without allowBackwardsCrawling - mogery- const isHostnameDifferent =- normalizeUrlOnlyHostname(doc.metadata.url) !==- normalizeUrlOnlyHostname(doc.metadata.sourceURL);- if (job.data.isCrawlSourceScrape && isHostnameDifferent) {- // TODO: re-fetch sitemap for redirect target domain- sc.originUrl = doc.metadata.url;- await saveCrawl(job.data.crawl_id, sc);- }-- if (isUrlBlocked(doc.metadata.url, (await getACUCTeam(job.data.team_id))?.flags ?? null)) {- throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking- }-- const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));- const p2 = generateURLPermutations(- normalizeURL(doc.metadata.sourceURL, sc),- );-- if (JSON.stringify(p1) !== JSON.stringify(p2)) {- logger.debug(- "Was redirected, removing old URL and locking new URL...",- { oldUrl: doc.metadata.sourceURL, newUrl: doc.metadata.url },- );-- // Prevent redirect target from being visited in the crawl again- // See lockURL- const x = await redisEvictConnection.sadd(- "crawl:" + job.data.crawl_id + ":visited",- ...p1.map((x) => x.href),- );- const lockRes = x === p1.length;-- if (job.data.crawlerOptions !== null && !lockRes) {- throw new RacedRedirectError();- }- }- }-- if (job.data.crawlerOptions !== null) {- if (!sc.cancelled) {- const crawler = crawlToCrawler(- job.data.crawl_id,- sc,- (await getACUCTeam(job.data.team_id))?.flags ?? null,- doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,- job.data.crawlerOptions,- );-- const links = await crawler.filterLinks(- await crawler.extractLinksFromHTML(- rawHtml ?? "",- doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,- ),- Infinity,- sc.crawlerOptions?.maxDepth ?? 10,- );-- logger.debug("Discovered " + links.links.length + " links...", {- linksLength: links.links.length,- });-- for (const link of links.links) {- if (await lockURL(job.data.crawl_id, sc, link)) {- // This seems to work really welel- const jobPriority = await getJobPriority({- team_id: sc.team_id,- basePriority: job.data.crawl_id ? 20 : 10,- });- const jobId = uuidv4();-- logger.debug(- "Determined job priority " +- jobPriority +- " for URL " +- JSON.stringify(link),- { jobPriority, url: link },- );-- // console.log("team_id: ", sc.team_id)- // console.log("base priority: ", job.data.crawl_id ? 20 : 10)- // console.log("job priority: " , jobPriority, "\n\n\n")-- await addScrapeJob(- {- url: link,- mode: "single_urls",- team_id: sc.team_id,- scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),- internalOptions: sc.internalOptions,- crawlerOptions: {- ...sc.crawlerOptions,- currentDiscoveryDepth:- (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,- },- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- },- {},- jobId,- jobPriority,- );-- await addCrawlJob(job.data.crawl_id, jobId, logger);- logger.debug("Added job for URL " + JSON.stringify(link), {- jobPriority,- url: link,- newJobId: jobId,- });- } else {- // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!- // logger.debug("Could not lock URL " + JSON.stringify(link), {- // url: link,- // });- }- }-- // Only run check after adding new jobs for discovery - mogery- if (job.data.isCrawlSourceScrape) {- const filterResult = await crawler.filterLinks(- [doc.metadata.url ?? doc.metadata.sourceURL!],- 1,- sc.crawlerOptions?.maxDepth ?? 10,- );- if (filterResult.links.length === 0) {- const url = doc.metadata.url ?? doc.metadata.sourceURL!;- const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";- throw new Error(reason);- }- }- }- }-- try {- signal?.throwIfAborted();- } catch (e) {- throw new Error("timeout");- }-- const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);-- doc.metadata.creditsUsed = credits_billed ?? undefined;-- logger.debug("Logging job to DB...");- await logJob(- {- job_id: job.id as string,- success: true,- num_docs: 1,- docs: [doc],- time_taken: timeTakenInSeconds,- team_id: job.data.team_id,- mode: job.data.mode,- url: job.data.url,- crawlerOptions: sc.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- cost_tracking: costTracking,- pdf_num_pages: doc.metadata.numPages,- credits_billed,- change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,- zeroDataRetention: job.data.zeroDataRetention,- },- true,- job.data.internalOptions?.bypassBilling ?? false,- );-- if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {- logger.debug("Calling webhook with success...", {- webhook: job.data.webhook,- });- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id,- scrapeId: job.id,- data,- webhook: job.data.webhook,- v1: job.data.v1,- eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",- });- }-- logger.debug("Declaring job as done...");- await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);-- await finishCrawlIfNeeded(job, sc);- } else {- try {- signal?.throwIfAborted();- } catch (e) {- throw new Error("timeout");- }-- const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);-- doc.metadata.creditsUsed = credits_billed ?? undefined;-- await logJob({- job_id: job.id,- success: true,- message: "Scrape completed",- num_docs: 1,- docs: [doc],- time_taken: timeTakenInSeconds,- team_id: job.data.team_id,- mode: "scrape",- url: job.data.url,- scrapeOptions: job.data.scrapeOptions,- origin: job.data.origin,- integration: job.data.integration,- num_tokens: 0, // TODO: fix- cost_tracking: costTracking,- pdf_num_pages: doc.metadata.numPages,- credits_billed,- change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,- zeroDataRetention: job.data.zeroDataRetention,- }, false, job.data.internalOptions?.bypassBilling ?? false);- }-- logger.info(`π Job done ${job.id}`);- return data;- } catch (error) {- if (job.data.crawl_id) {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;-- logger.debug("Declaring job as done...");- await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);- await redisEvictConnection.srem(- "crawl:" + job.data.crawl_id + ":visited_unique",- normalizeURL(job.data.url, sc),- );-- await finishCrawlIfNeeded(job, sc);- }-- const isEarlyTimeout =- error instanceof Error && error.message === "timeout";- const isCancelled =- error instanceof Error &&- error.message === "Parent crawl/batch scrape was cancelled";-- if (isEarlyTimeout) {- logger.error(`π Job timed out ${job.id}`);- } else if (error instanceof RacedRedirectError) {- logger.warn(`π Job got redirect raced ${job.id}, silently failing`);- } else if (isCancelled) {- logger.warn(`π Job got cancelled, silently failing`);- } else {- logger.error(`π Job errored ${job.id} - ${error}`, { error });-- Sentry.captureException(error, {- data: {- job: job.id,- },- });-- if (error instanceof CustomError) {- // Here we handle the error, then save the failed job- logger.error(error.message); // or any other error handling- }- logger.error(error);- if (error.stack) {- logger.error(error.stack);- }- }-- const data = {- success: false,- document: null,- project_id: job.data.project_id,- error:- error instanceof Error- ? error- : typeof error === "string"- ? new Error(error)- : new Error(JSON.stringify(error)),- };-- if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id ?? (job.id as string),- scrapeId: job.id,- data,- webhook: job.data.webhook,- v1: job.data.v1,- eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",- });- }-- const end = Date.now();- const timeTakenInSeconds = (end - start) / 1000;-- const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);-- logger.debug("Logging job to DB...");- await logJob(- {- job_id: job.id as string,- success: false,- message:- typeof error === "string"- ? error- : (error.message ??- "Something went wrong... Contact [email protected]"),- num_docs: 0,- docs: [],- time_taken: timeTakenInSeconds,- team_id: job.data.team_id,- mode: job.data.mode,- url: job.data.url,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- cost_tracking: costTracking,- credits_billed,- zeroDataRetention: job.data.zeroDataRetention,- },- true,- job.data.internalOptions?.bypassBilling ?? false,- );- return data;- }-}--// wsq.process(-// Math.floor(Number(process.env.NUM_WORKERS_PER_QUEUE ?? 8)),-// processJob-// );--// wsq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));-// wsq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));-// wsq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));-// wsq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));-// wsq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));-// wsq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));-// Start all workersconst app = Express();@@ -1672,14 +566,18 @@ app.listen(workerPort, () => {const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });scrapeQueueEvents.on("failed", failedListener);- await Promise.all([- workerFun(getScrapeQueue(), processJobInternal),+ const results = await Promise.all([+ separateWorkerFun(getScrapeQueue(), path.join(__dirname, "worker", "scrape-worker.js")),workerFun(getExtractQueue(), processExtractJobInternal),workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),]);+console.log("All workers exited. Waiting for all jobs to finish...");+ const workerResults = results.filter(x => x instanceof Worker);+ await Promise.all(workerResults.map(x => x.close()));+while (runningJobs.size > 0) {await new Promise((resolve) => setTimeout(resolve, 500));}diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsnew file mode 100644index 000000000..9a3d568b3--- /dev/null+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -0,0 +1,284 @@+import { logger as _logger } from "../../lib/logger";+import { Job } from "bullmq";+import { addCrawlJobs, finishCrawl, getCrawlJobs, getDoneJobsOrderedLength, lockURLsIndividually, StoredCrawl, unPreFinishCrawl } from "../../lib/crawl-redis";+import { redisEvictConnection } from "../redis";+import { getCrawl } from "../../lib/crawl-redis";+import { finishCrawlPre } from "../../lib/crawl-redis";+import { getACUCTeam } from "../../controllers/auth";+import { crawlToCrawler } from "../../lib/crawl-redis";+import { supabase_service } from "../supabase";+import { v4 as uuidv4 } from "uuid";+import { addScrapeJobs } from "../queue-jobs";+import { getJobs } from "../../controllers/v1/crawl-status";+import { logJob } from "../logging/log_job";+import { callWebhook } from "../webhook";++export async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "finishCrawlIfNeeded",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data.crawl_id,+ zeroDataRetention: sc.internalOptions.zeroDataRetention,+ });++ if (await finishCrawlPre(job.data.crawl_id, logger)) {+ logger.info("Crawl is pre-finished, checking if we need to add more jobs");+ if (+ job.data.crawlerOptions &&+ !(await redisEvictConnection.exists(+ "crawl:" + job.data.crawl_id + ":invisible_urls",+ ))+ ) {+ await redisEvictConnection.set(+ "crawl:" + job.data.crawl_id + ":invisible_urls",+ "done",+ "EX",+ 60 * 60 * 24,+ );++ const sc = (await getCrawl(job.data.crawl_id))!;++ const visitedUrls = new Set(+ await redisEvictConnection.smembers(+ "crawl:" + job.data.crawl_id + ":visited_unique",+ ),+ );++ logger.info("Visited URLs", {+ visitedUrls: visitedUrls.size,+ });++ let lastUrls: string[] = [];+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {+ lastUrls = (+ (+ await supabase_service.rpc("diff_get_last_crawl_urls", {+ i_team_id: job.data.team_id,+ i_url: sc.originUrl!,+ })+ ).data ?? []+ ).map((x) => x.url);+ }++ const lastUrlsSet = new Set(lastUrls);++ logger.info("Last URLs", {+ lastUrls: lastUrlsSet.size,+ });++ const crawler = crawlToCrawler(+ job.data.crawl_id,+ sc,+ (await getACUCTeam(job.data.team_id))?.flags ?? null,+ sc.originUrl!,+ job.data.crawlerOptions,+ );++ const univistedUrls = await crawler.filterLinks(+ Array.from(lastUrlsSet).filter((x) => !visitedUrls.has(x)),+ Infinity,+ sc.crawlerOptions.maxDepth ?? 10,+ );++ const addableJobCount =+ sc.crawlerOptions.limit === undefined+ ? Infinity+ : sc.crawlerOptions.limit -+ (await getDoneJobsOrderedLength(job.data.crawl_id));++ if (univistedUrls.links.length !== 0 && addableJobCount > 0) {+ logger.info("Adding jobs", {+ univistedUrls: univistedUrls.links.length,+ addableJobCount,+ });++ const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ crawlerOptions: {+ ...job.data.crawlerOptions,+ urlInvisibleInCurrentCrawl: true,+ },+ scrapeOptions: job.data.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ opts: {+ jobId: uuid,+ priority: 20,+ },+ };+ });++ const lockedIds = await lockURLsIndividually(+ job.data.crawl_id,+ sc,+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),+ );+ const lockedJobs = jobs.filter((x) =>+ lockedIds.find((y) => y.id === x.opts.jobId),+ );+ await addCrawlJobs(+ job.data.crawl_id,+ lockedJobs.map((x) => x.opts.jobId),+ logger,+ );+ await addScrapeJobs(lockedJobs);++ if (lockedJobs.length > 0) {+ logger.info("Added jobs, not going for the full finish", {+ lockedJobs: lockedJobs.length,+ });++ await unPreFinishCrawl(job.data.crawl_id);+ return;+ } else {+ logger.info("No jobs added (all discovered URLs were locked), finishing crawl");+ }+ }+ }++ logger.info("Finishing crawl");+ await finishCrawl(job.data.crawl_id, logger);++ if (!job.data.v1) {+ const jobIDs = await getCrawlJobs(job.data.crawl_id);++ const jobs = (await getJobs(jobIDs)).sort(+ (a, b) => a.timestamp - b.timestamp,+ );+ // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));+ const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")+ ? "failed"+ : "completed";++ const fullDocs = jobs+ .map((x) =>+ x.returnvalue+ ? Array.isArray(x.returnvalue)+ ? x.returnvalue[0]+ : x.returnvalue+ : null,+ )+ .filter((x) => x !== null);++ await logJob({+ job_id: job.data.crawl_id,+ success: jobStatus === "completed",+ message: sc.cancelled ? "Cancelled" : undefined,+ num_docs: fullDocs.length,+ docs: [],+ time_taken: (Date.now() - sc.createdAt) / 1000,+ team_id: job.data.team_id,+ mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",+ url: sc.originUrl!,+ scrapeOptions: sc.scrapeOptions,+ crawlerOptions: sc.crawlerOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ zeroDataRetention: job.data.zeroDataRetention,+ }, false, job.data.internalOptions?.bypassBilling ?? false);++ const data = {+ success: jobStatus !== "failed",+ result: {+ links: fullDocs.map((doc) => {+ return {+ content: doc,+ source: doc?.metadata?.sourceURL ?? doc?.url ?? "",+ };+ }),+ },+ project_id: job.data.project_id,+ docs: fullDocs,+ };++ // v0 web hooks, call when done with all the data+ if (!job.data.v1) {+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ data,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null+ ? "crawl.completed"+ : "batch_scrape.completed",+ });+ }+ } else {+ const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);+ const jobStatus = sc.cancelled ? "failed" : "completed";++ let credits_billed = null;++ if (process.env.USE_DB_AUTHENTICATION === "true") {+ const creditsRpc = await supabase_service+ .rpc("credits_billed_by_crawl_id_1", {+ i_crawl_id: job.data.crawl_id,+ });++ credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;++ if (credits_billed === null) {+ logger.warn("Credits billed is null", {+ error: creditsRpc.error,+ });+ }+ }++ await logJob(+ {+ job_id: job.data.crawl_id,+ success: jobStatus === "completed",+ message: sc.cancelled ? "Cancelled" : undefined,+ num_docs,+ docs: [],+ time_taken: (Date.now() - sc.createdAt) / 1000,+ team_id: job.data.team_id,+ scrapeOptions: sc.scrapeOptions,+ mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",+ url:+ sc?.originUrl ??+ (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),+ crawlerOptions: sc.crawlerOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ credits_billed,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ true,+ job.data.internalOptions?.bypassBilling ?? false,+ );+++ // v1 web hooks, call when done with no data, but with event completed+ if (job.data.v1 && job.data.webhook) {+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ data: [],+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null+ ? "crawl.completed"+ : "batch_scrape.completed",+ });+ }+ }+ }+}\ No newline at end of filediff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsnew file mode 100644index 000000000..68c9b3a28--- /dev/null+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -0,0 +1,850 @@+import { configDotenv } from "dotenv";+import { Job } from "bullmq";+import * as Sentry from "@sentry/node";+import http from "http";+import https from "https";++import { logger as _logger } from "../../lib/logger";+import {+ concurrentJobDone,+ pushConcurrencyLimitActiveJob,+} from "../../lib/concurrency-limit";+import { addJobPriority, deleteJobPriority } from "../../lib/job-priority";+import { cacheableLookup } from "../../scraper/scrapeURL/lib/cacheableLookup";+import { v4 as uuidv4 } from "uuid";+import {+ addCrawlJob,+ addCrawlJobs,+ addCrawlJobDone,+ crawlToCrawler,+ finishCrawlKickoff,+ generateURLPermutations,+ getCrawl,+ lockURL,+ lockURLsIndividually,+ normalizeURL,+ saveCrawl,+ StoredCrawl+} from "../../lib/crawl-redis";+import { addScrapeJob, addScrapeJobs } from "../queue-jobs";+import { getJobPriority } from "../../lib/job-priority";+import { Document, scrapeOptions, TeamFlags } from "../../controllers/v1/types";+import { getACUCTeam } from "../../controllers/auth";+import { callWebhook } from "../webhook";+import { CustomError } from "../../lib/custom-error";+import { startWebScraperPipeline } from "../../main/runWebScraper";+import { CostTracking } from "../../lib/extract/extraction-service";+import { redisEvictConnection } from "../redis";+import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";+import { logJob } from "../logging/log_job";+import { generateURLSplits, queryIndexAtSplitLevel } from "../index";+import { WebCrawler } from "../../scraper/WebScraper/crawler";+import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";+import { getBillingQueue } from "../queue-service";+import type { Logger } from "winston";+import { finishCrawlIfNeeded } from "./crawl-logic";++class RacedRedirectError extends Error {+ constructor() {+ super("Raced redirect error");+ }+}++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++configDotenv();++const jobLockExtendInterval =+ Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;+const jobLockExtensionTime =+ Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;++cacheableLookup.install(http.globalAgent);+cacheableLookup.install(https.globalAgent);++async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) {+ let creditsToBeBilled: number | null = null;++ if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {+ creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags);++ if (+ job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&+ process.env.USE_DB_AUTHENTICATION === "true"+ ) {+ try {+ const billingJobId = uuidv4();+ logger.debug(+ `Adding billing job to queue for team ${job.data.team_id}`,+ {+ billingJobId,+ credits: creditsToBeBilled,+ is_extract: false,+ },+ );++ // Add directly to the billing queue - the billing worker will handle the rest+ await getBillingQueue().add(+ "bill_team",+ {+ team_id: job.data.team_id,+ subscription_id: undefined,+ credits: creditsToBeBilled,+ is_extract: false,+ timestamp: new Date().toISOString(),+ originating_job_id: job.id,+ },+ {+ jobId: billingJobId,+ priority: 10,+ },+ );+ return creditsToBeBilled;+ } catch (error) {+ logger.error(+ `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,+ { error },+ );+ Sentry.captureException(error);+ return creditsToBeBilled;+ }+ }+ }++ return creditsToBeBilled;+}++async function processJob(job: Job & { id: string }) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "processJob",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data?.crawl_id ?? undefined,+ teamId: job.data?.team_id ?? undefined,+ zeroDataRetention: job.data?.zeroDataRetention ?? false,+ });+ logger.info(`π Worker taking job ${job.id}`, { url: job.data.url });+ const start = job.data.startTime ?? Date.now();+ const remainingTime = job.data.scrapeOptions.timeout ? (job.data.scrapeOptions.timeout - (Date.now() - start)) : undefined;++ const costTracking = new CostTracking();++ try {+ job.updateProgress({+ current: 1,+ total: 100,+ current_step: "SCRAPING",+ current_url: "",+ });++ if (remainingTime !== undefined && remainingTime < 0) {+ throw new Error("timeout");+ }+ const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;++ if (job.data.crawl_id) {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ if (sc && sc.cancelled) {+ throw new Error("Parent crawl/batch scrape was cancelled");+ }+ }+++ const pipeline = await Promise.race([+ startWebScraperPipeline({+ job,+ costTracking,+ }),+ ...(remainingTime !== undefined+ ? [+ (async () => {+ await sleep(remainingTime);+ throw new Error("timeout");+ })(),+ ]+ : []),+ ]);++ try {+ signal?.throwIfAborted();+ } catch (e) {+ throw new Error("timeout");+ }++ if (!pipeline.success) {+ throw pipeline.error;+ }++ const end = Date.now();+ const timeTakenInSeconds = (end - start) / 1000;++ const doc = pipeline.document;++ const rawHtml = doc.rawHtml ?? "";++ if (!job.data.scrapeOptions.formats.includes("rawHtml")) {+ delete doc.rawHtml;+ }++ if (job.data.concurrencyLimited) {+ doc.warning =+ "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." ++ (doc.warning ? " " + doc.warning : "");+ }++ const data = {+ success: true,+ result: {+ links: [+ {+ content: doc,+ source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",+ id: job.id,+ },+ ],+ },+ project_id: job.data.project_id,+ document: doc,+ };++ if (job.data.crawl_id) {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;++ if (+ doc.metadata.url !== undefined &&+ doc.metadata.sourceURL !== undefined &&+ normalizeURL(doc.metadata.url, sc) !==+ normalizeURL(doc.metadata.sourceURL, sc) &&+ job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape+ ) {+ const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);+ const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);+ if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {+ const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";+ throw new Error(reason);+ }++ // Only re-set originUrl if it's different from the current hostname+ // This is only done on this condition to handle cross-domain redirects+ // If this would be done for non-crossdomain redirects, but also for e.g.+ // redirecting / -> /introduction (like our docs site does), it would+ // break crawling the entire site without allowBackwardsCrawling - mogery+ const isHostnameDifferent =+ normalizeUrlOnlyHostname(doc.metadata.url) !==+ normalizeUrlOnlyHostname(doc.metadata.sourceURL);+ if (job.data.isCrawlSourceScrape && isHostnameDifferent) {+ // TODO: re-fetch sitemap for redirect target domain+ sc.originUrl = doc.metadata.url;+ await saveCrawl(job.data.crawl_id, sc);+ }++ if (isUrlBlocked(doc.metadata.url, (await getACUCTeam(job.data.team_id))?.flags ?? null)) {+ throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking+ }++ const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));+ const p2 = generateURLPermutations(+ normalizeURL(doc.metadata.sourceURL, sc),+ );++ if (JSON.stringify(p1) !== JSON.stringify(p2)) {+ logger.debug(+ "Was redirected, removing old URL and locking new URL...",+ { oldUrl: doc.metadata.sourceURL, newUrl: doc.metadata.url },+ );++ // Prevent redirect target from being visited in the crawl again+ // See lockURL+ const x = await redisEvictConnection.sadd(+ "crawl:" + job.data.crawl_id + ":visited",+ ...p1.map((x) => x.href),+ );+ const lockRes = x === p1.length;++ if (job.data.crawlerOptions !== null && !lockRes) {+ throw new RacedRedirectError();+ }+ }+ }++ if (job.data.crawlerOptions !== null) {+ if (!sc.cancelled) {+ const crawler = crawlToCrawler(+ job.data.crawl_id,+ sc,+ (await getACUCTeam(job.data.team_id))?.flags ?? null,+ doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,+ job.data.crawlerOptions,+ );++ const links = await crawler.filterLinks(+ await crawler.extractLinksFromHTML(+ rawHtml ?? "",+ doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,+ ),+ Infinity,+ sc.crawlerOptions?.maxDepth ?? 10,+ );+ logger.debug("Discovered " + links.links.length + " links...", {+ linksLength: links.links.length,+ });++ for (const link of links.links) {+ if (await lockURL(job.data.crawl_id, sc, link)) {+ // This seems to work really welel+ const jobPriority = await getJobPriority({+ team_id: sc.team_id,+ basePriority: job.data.crawl_id ? 20 : 10,+ });+ const jobId = uuidv4();++ logger.debug(+ "Determined job priority " ++ jobPriority ++ " for URL " ++ JSON.stringify(link),+ { jobPriority, url: link },+ );++ await addScrapeJob(+ {+ url: link,+ mode: "single_urls",+ team_id: sc.team_id,+ scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),+ internalOptions: sc.internalOptions,+ crawlerOptions: {+ ...sc.crawlerOptions,+ currentDiscoveryDepth:+ (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,+ },+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ {},+ jobId,+ jobPriority,+ );++ await addCrawlJob(job.data.crawl_id, jobId, logger);+ logger.debug("Added job for URL " + JSON.stringify(link), {+ jobPriority,+ url: link,+ newJobId: jobId,+ });+ } else {+ // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!+ // logger.debug("Could not lock URL " + JSON.stringify(link), {+ // url: link,+ // });+ }+ }++ // Only run check after adding new jobs for discovery - mogery+ if (job.data.isCrawlSourceScrape) {+ const filterResult = await crawler.filterLinks(+ [doc.metadata.url ?? doc.metadata.sourceURL!],+ 1,+ sc.crawlerOptions?.maxDepth ?? 10,+ );+ if (filterResult.links.length === 0) {+ const url = doc.metadata.url ?? doc.metadata.sourceURL!;+ const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";+ throw new Error(reason);+ }+ }+ }+ }++ try {+ signal?.throwIfAborted();+ } catch (e) {+ throw new Error("timeout");+ }++ const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);++ doc.metadata.creditsUsed = credits_billed ?? undefined;++ logger.debug("Logging job to DB...");+ await logJob(+ {+ job_id: job.id as string,+ success: true,+ num_docs: 1,+ docs: [doc],+ time_taken: timeTakenInSeconds,+ team_id: job.data.team_id,+ mode: job.data.mode,+ url: job.data.url,+ crawlerOptions: sc.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ cost_tracking: costTracking,+ pdf_num_pages: doc.metadata.numPages,+ credits_billed,+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ true,+ job.data.internalOptions?.bypassBilling ?? false,+ );++ if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {+ logger.debug("Calling webhook with success...", {+ webhook: job.data.webhook,+ });+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ scrapeId: job.id,+ data,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",+ });+ }++ logger.debug("Declaring job as done...");+ await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);++ await finishCrawlIfNeeded(job, sc);+ } else {+ try {+ signal?.throwIfAborted();+ } catch (e) {+ throw new Error("timeout");+ }++ const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);++ doc.metadata.creditsUsed = credits_billed ?? undefined;++ await logJob({+ job_id: job.id,+ success: true,+ message: "Scrape completed",+ num_docs: 1,+ docs: [doc],+ time_taken: timeTakenInSeconds,+ team_id: job.data.team_id,+ mode: "scrape",+ url: job.data.url,+ scrapeOptions: job.data.scrapeOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ num_tokens: 0, // TODO: fix+ cost_tracking: costTracking,+ pdf_num_pages: doc.metadata.numPages,+ credits_billed,+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,+ zeroDataRetention: job.data.zeroDataRetention,+ }, false, job.data.internalOptions?.bypassBilling ?? false);+ }++ logger.info(`π Job done ${job.id}`);+ return data;+ } catch (error) {+ if (job.data.crawl_id) {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;++ logger.debug("Declaring job as done...");+ await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);+ await redisEvictConnection.srem(+ "crawl:" + job.data.crawl_id + ":visited_unique",+ normalizeURL(job.data.url, sc),+ );++ await finishCrawlIfNeeded(job, sc);+ }++ const isEarlyTimeout =+ error instanceof Error && error.message === "timeout";+ const isCancelled =+ error instanceof Error &&+ error.message === "Parent crawl/batch scrape was cancelled";++ if (isEarlyTimeout) {+ logger.error(`π Job timed out ${job.id}`);+ } else if (error instanceof RacedRedirectError) {+ logger.warn(`π Job got redirect raced ${job.id}, silently failing`);+ } else if (isCancelled) {+ logger.warn(`π Job got cancelled, silently failing`);+ } else {+ logger.error(`π Job errored ${job.id} - ${error}`, { error });++ Sentry.captureException(error, {+ data: {+ job: job.id,+ },+ });++ if (error instanceof CustomError) {+ // Here we handle the error, then save the failed job+ logger.error(error.message); // or any other error handling+ }+ logger.error(error);+ if (error.stack) {+ logger.error(error.stack);+ }+ }++ const data = {+ success: false,+ document: null,+ project_id: job.data.project_id,+ error:+ error instanceof Error+ ? error+ : typeof error === "string"+ ? new Error(error)+ : new Error(JSON.stringify(error)),+ };++ if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id ?? (job.id as string),+ scrapeId: job.id,+ data,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",+ });+ }++ const end = Date.now();+ const timeTakenInSeconds = (end - start) / 1000;++ const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);++ logger.debug("Logging job to DB...");+ await logJob(+ {+ job_id: job.id as string,+ success: false,+ message:+ typeof error === "string"+ ? error+ : (error.message ??+ "Something went wrong... Contact [email protected]"),+ num_docs: 0,+ docs: [],+ time_taken: timeTakenInSeconds,+ team_id: job.data.team_id,+ mode: job.data.mode,+ url: job.data.url,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ cost_tracking: costTracking,+ credits_billed,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ true,+ job.data.internalOptions?.bypassBilling ?? false,+ );+ return data;+ }+}++async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: string) {+ if (sc.crawlerOptions.ignoreSitemap) {+ return [];+ }++ const trimmedURL = new URL(url);+ trimmedURL.search = "";++ const index = await queryIndexAtSplitLevel(+ sc.crawlerOptions.allowBackwardCrawling ? generateURLSplits(trimmedURL.href)[0] : trimmedURL.href,+ sc.crawlerOptions.limit ?? 10000,+ );++ const validIndexLinksResult = await crawler.filterLinks(+ index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),+ sc.crawlerOptions.limit ?? 10000,+ sc.crawlerOptions.maxDepth ?? 10,+ false,+ );+ const validIndexLinks = validIndexLinksResult.links;++ return validIndexLinks;+}++async function processKickoffJob(job: Job & { id: string }) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "processKickoffJob",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data?.crawl_id ?? undefined,+ teamId: job.data?.team_id ?? undefined,+ zeroDataRetention: job.data.zeroDataRetention ?? false,+ });++ try {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);++ logger.debug("Locking URL...");+ await lockURL(job.data.crawl_id, sc, job.data.url);+ const jobId = uuidv4();+ logger.debug("Adding scrape job to Redis...", { jobId });+ await addScrapeJob(+ {+ url: job.data.url,+ mode: "single_urls",+ team_id: job.data.team_id,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ webhook: job.data.webhook,+ v1: job.data.v1,+ isCrawlSourceScrape: true,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ {+ priority: 15,+ },+ jobId,+ );+ logger.debug("Adding scrape job to BullMQ...", { jobId });+ await addCrawlJob(job.data.crawl_id, jobId, logger);++ if (job.data.webhook) {+ logger.debug("Calling webhook with crawl.started...", {+ webhook: job.data.webhook,+ });+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ data: null,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: "crawl.started",+ });+ }++ const sitemap = sc.crawlerOptions.ignoreSitemap+ ? 0+ : await crawler.tryGetSitemap(async (urls) => {+ if (urls.length === 0) return;++ logger.debug("Using sitemap chunk of length " + urls.length, {+ sitemapLength: urls.length,+ });++ let jobPriority = await getJobPriority({+ team_id: job.data.team_id,+ basePriority: 21,+ });+ logger.debug("Using job priority " + jobPriority, { jobPriority });++ const jobs = urls.map((url) => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ opts: {+ jobId: uuid,+ priority: 20,+ },+ };+ });++ logger.debug("Locking URLs...");+ const lockedIds = await lockURLsIndividually(+ job.data.crawl_id,+ sc,+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),+ );+ const lockedJobs = jobs.filter((x) =>+ lockedIds.find((y) => y.id === x.opts.jobId),+ );+ logger.debug("Adding scrape jobs to Redis...");+ await addCrawlJobs(+ job.data.crawl_id,+ lockedJobs.map((x) => x.opts.jobId),+ logger,+ );+ logger.debug("Adding scrape jobs to BullMQ...");+ await addScrapeJobs(lockedJobs);+ });++ if (sitemap === 0) {+ logger.debug("Sitemap not found or ignored.", {+ ignoreSitemap: sc.crawlerOptions.ignoreSitemap,+ });+ }++ const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);++ if (indexLinks.length > 0) {+ logger.debug("Using index links of length " + indexLinks.length, {+ indexLinksLength: indexLinks.length,+ });++ let jobPriority = await getJobPriority({+ team_id: job.data.team_id,+ basePriority: 21,+ });+ logger.debug("Using job priority " + jobPriority, { jobPriority });++ const jobs = indexLinks.map((url) => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ opts: {+ jobId: uuid,+ priority: 20,+ },+ };+ });++ logger.debug("Locking URLs...");+ const lockedIds = await lockURLsIndividually(+ job.data.crawl_id,+ sc,+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),+ );+ const lockedJobs = jobs.filter((x) =>+ lockedIds.find((y) => y.id === x.opts.jobId),+ );+ logger.debug("Adding scrape jobs to Redis...");+ await addCrawlJobs(+ job.data.crawl_id,+ lockedJobs.map((x) => x.opts.jobId),+ logger,+ );+ logger.debug("Adding scrape jobs to BullMQ...");+ await addScrapeJobs(lockedJobs);+ }++ logger.debug("Done queueing jobs!");++ await finishCrawlKickoff(job.data.crawl_id);+ await finishCrawlIfNeeded(job, sc);++ return { success: true };+ } catch (error) {+ logger.error("An error occurred!", { error });+ await finishCrawlKickoff(job.data.crawl_id);+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ if (sc) {+ await finishCrawlIfNeeded(job, sc);+ }+ return { success: false, error };+ }+}++export const processJobInternal = async (job: Job & { id: string }) => {+ const logger = _logger.child({+ module: "queue-worker",+ method: "processJobInternal",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data?.crawl_id ?? undefined,+ zeroDataRetention: job.data?.zeroDataRetention ?? false,+ });++ try {+ try {+ let extendLockInterval: NodeJS.Timeout | null = null;+ if (job.data?.mode !== "kickoff" && job.data?.team_id) {+ extendLockInterval = setInterval(async () => {+ await pushConcurrencyLimitActiveJob(job.data.team_id, job.id, 60 * 1000); // 60s lock renew, just like in the queue+ }, jobLockExtendInterval);+ }++ await addJobPriority(job.data.team_id, job.id);+ try {+ if (job.data?.mode === "kickoff") {+ const result = await processKickoffJob(job);+ if (result.success) {+ return null;+ } else {+ throw (result as any).error;+ }+ } else {+ const result = await processJob(job);+ if (result.success) {+ try {+ if (+ process.env.USE_DB_AUTHENTICATION === "true" &&+ (job.data.crawl_id || process.env.GCS_BUCKET_NAME)+ ) {+ logger.debug(+ "Job succeeded -- putting null in Redis",+ );+ return null;+ } else {+ console.log("result.document", result.document);+ logger.debug("Job succeeded -- putting result in Redis", { document: result.document });+ return result.document;+ }+ } catch (e) { }+ } else {+ throw (result as any).error;+ }+ }+ } finally {+ await deleteJobPriority(job.data.team_id, job.id);+ if (extendLockInterval) {+ clearInterval(extendLockInterval);+ }+ }+ } finally {+ await concurrentJobDone(job);+ }+ } catch (error) {+ logger.debug("Job failed", { error });+ Sentry.captureException(error);+ throw error;+ }+};++module.exports = processJobInternal;\ No newline at end of filediff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex fb21013e3..00c4641bb 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -67,8 +67,6 @@ export interface RunWebScraperParams {mode: Mode;scrapeOptions: ScrapeOptions;internalOptions?: InternalOptions;- // onSuccess: (result: V1Document, mode: string) => void;- // onError: (error: Error) => void;team_id: string;bull_job_id: string;priority?: number;