Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2): parsers + merge w/ main (#1907)
Gergő Móricz7400e10eaca3
diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.ymlindex af6523b9c..01d291245 100644--- a/.github/workflows/deploy-image.yml+++ b/.github/workflows/deploy-image.yml@@ -21,14 +21,21 @@ jobs:- name: 'Checkout GitHub Action'uses: actions/checkout@main+ - name: 'Set up Docker Buildx'+ uses: docker/setup-buildx-action@v3+- name: 'Login to GitHub Container Registry'- uses: docker/login-action@v1+ uses: docker/login-action@v3with:registry: ghcr.iousername: ${{github.actor}}password: ${{secrets.GITHUB_TOKEN}}- - name: 'Build Inventory Image'- run: |- docker build . --tag ghcr.io/mendableai/firecrawl:latest- docker push ghcr.io/mendableai/firecrawl:latest\ No newline at end of file+ - name: 'Build and Push Image'+ uses: docker/build-push-action@v5+ with:+ context: ./apps/api+ push: true+ tags: ghcr.io/mendableai/firecrawl:latest+ cache-from: type=registry,ref=ghcr.io/mendableai/firecrawl:buildcache+ cache-to: type=registry,ref=ghcr.io/mendableai/firecrawl:buildcache,mode=max\ No newline at end of filediff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.ymlindex 77297fe08..c4d27635c 100644--- a/.github/workflows/test-server-self-host.yml+++ b/.github/workflows/test-server-self-host.yml@@ -102,14 +102,17 @@ jobs:docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxngpnpx wait-on tcp:3434 -t 30sworking-directory: ./+ - name: Build the application+ run: npm run build+ working-directory: ./apps/api- name: Start server- run: npm run start:production > api.log 2>&1 &+ run: npm run start:production:nobuild > api.log 2>&1 &env:PORT: 3002HOST: 0.0.0.0working-directory: ./apps/api- name: Start worker- run: npm run workers > worker.log 2>&1 &+ run: npm run worker:production > worker.log 2>&1 &env:PORT: 3005HOST: 0.0.0.0diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 1b903ea1f..06ad0fb08 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -104,22 +104,25 @@ jobs:cargo build --releasechmod +x target/release/libcrawler.soworking-directory: ./apps/api/sharedLibs/crawler+ - name: Build the application+ run: npm run build+ working-directory: ./apps/api- name: Start the application- run: npm run start:production > api.log 2>&1 &+ run: npm run start:production:nobuild > api.log 2>&1 &env:PORT: 3002HOST: 0.0.0.0working-directory: ./apps/apiid: start_app- name: Start worker- run: npm run workers > worker.log 2>&1 &+ run: npm run worker:production > worker.log 2>&1 &env:PORT: 3005HOST: 0.0.0.0working-directory: ./apps/apiid: start_workers- name: Start index worker- run: npm run index-worker > index-worker.log 2>&1 &+ run: npm run index-worker:production > index-worker.log 2>&1 &working-directory: ./apps/apiid: start_index_worker- name: Wait for APIdiff --git a/README.md b/README.mdindex 3948aee29..b042290f5 100644--- a/README.md+++ b/README.md@@ -71,7 +71,7 @@ To use the API, you need to sign up on [Firecrawl](https://firecrawl.dev) and ge- [**Scrape**](#scraping): scrapes a URL and get its content in LLM-ready format (markdown, structured data via [LLM Extract](#llm-extraction-beta), screenshot, html)- [**Crawl**](#crawling): scrapes all the URLs of a web page and return content in LLM-ready format-- [**Map**](#map-alpha): input a website and get all the website urls - extremely fast+- [**Map**](#map): input a website and get all the website urls - extremely fast- [**Search**](#search): search the web and get full content from results- [**Extract**](#extract): get structured data from single page, multiple pages or entire websites with AI.diff --git a/apps/api/Dockerfile b/apps/api/Dockerfileindex 69cb302ae..5a8c62464 100644--- a/apps/api/Dockerfile+++ b/apps/api/Dockerfile@@ -19,7 +19,7 @@ COPY sharedLibs/go-html-to-md /app/sharedLibs/go-html-to-md# Install Go dependencies and build parser libRUN cd /app/sharedLibs/go-html-to-md && \- go mod tidy && \+ go mod download && \go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go && \chmod +x html-to-markdown.so@@ -31,17 +31,17 @@ COPY sharedLibs/crawler /app/sharedLibs/crawler# Install Rust dependencies and build transformer libRUN cd /app/sharedLibs/html-transformer && \- cargo build --release && \+ cargo build --release --locked && \chmod +x target/release/libhtml_transformer.so# Install Rust dependencies and build PDF parser libRUN cd /app/sharedLibs/pdf-parser && \- cargo build --release && \+ cargo build --release --locked && \chmod +x target/release/libpdf_parser.so# Install Rust dependencies and build crawler libRUN cd /app/sharedLibs/crawler && \- cargo build --release && \+ cargo build --release --locked && \chmod +x target/release/libcrawler.soFROM base@@ -53,7 +53,7 @@ COPY --from=rust-base /app/sharedLibs/pdf-parser/target/release/libpdf_parser.soCOPY --from=rust-base /app/sharedLibs/crawler/target/release/libcrawler.so /app/sharedLibs/crawler/target/release/libcrawler.so# Install git-RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*+RUN apt-get update && apt-get install -y git procps && rm -rf /var/lib/apt/lists/*# Start the server by default, this can be overwritten at runtimeEXPOSE 8080diff --git a/apps/api/package.json b/apps/api/package.jsonindex c85393354..5d724dc75 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -4,11 +4,12 @@"description": "","main": "index.js","scripts": {- "start": "nodemon --exec ts-node src/index.ts",+ "start": "tsc-watch --onSuccess \"node dist/src/index.js\"","start:production": "tsc && node dist/src/index.js",+ "start:production:nobuild": "node dist/src/index.js","format": "prettier --write \"src/**/*.(js|ts)\"","flyio": "node dist/src/index.js",- "start:dev": "nodemon --exec ts-node src/index.ts",+ "start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"","build": "tsc","build:nosentry": "tsc","test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'",@@ -16,9 +17,9 @@"test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'","test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'","test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/*.test.ts",- "workers": "nodemon --exec ts-node src/services/queue-worker.ts",+ "workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"","worker:production": "node dist/src/services/queue-worker.js",- "index-worker": "nodemon --exec ts-node src/services/indexing/index-worker.ts",+ "index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"","index-worker:production": "node dist/src/services/indexing/index-worker.js","mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest","mongo-docker-console": "docker exec -it mongodb mongosh",@@ -44,11 +45,11 @@"@types/supertest": "^6.0.2","jest": "^29.6.3","jest-fetch-mock": "^3.0.3",- "nodemon": "^3.1.10","prettier": "^3.4.2","supertest": "^6.3.3","ts-jest": "^29.1.1","ts-node": "^10.9.1",+ "tsc-watch": "^7.1.1","typescript": "^5.8.3"},"dependencies": {@@ -106,7 +107,7 @@"jsdom": "^26.0.0","json-schema-to-zod": "^2.3.0","keyword-extractor": "^0.0.28",- "koffi": "^2.12.3",+ "koffi": "^2.9.0","languagedetect": "^2.0.0","lodash": "^4.17.21","logsnag": "^1.0.0",@@ -151,8 +152,7 @@},"pnpm": {"onlyBuiltDependencies": [- "@sentry-internal/node-cpu-profiler",- "koffi"+ "@sentry-internal/node-cpu-profiler"],"overrides": {"brace-expansion@>=2.0.0 <=2.0.1": ">=2.0.2"diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex d2d1c5d87..1567f780b 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -174,8 +174,8 @@ importers:specifier: ^0.0.28version: 0.0.28koffi:- specifier: ^2.12.3- version: 2.12.3+ specifier: ^2.9.0+ version: 2.9.0languagedetect:specifier: ^2.0.0version: 2.0.0@@ -321,9 +321,6 @@ importers:jest-fetch-mock:specifier: ^3.0.3version: 3.0.3([email protected])- nodemon:- specifier: ^3.1.10- version: 3.1.10prettier:specifier: ^3.4.2version: 3.4.2@@ -336,6 +333,9 @@ importers:ts-node:specifier: ^10.9.1version: 10.9.2(@types/[email protected])([email protected])+ tsc-watch:+ specifier: ^7.1.1+ version: 7.1.1([email protected])typescript:specifier: ^5.8.3version: 5.8.3@@ -1964,10 +1964,6 @@ packages:resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}- resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}- engines: {node: '>=8'}-resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==}@@ -2080,10 +2076,6 @@ packages:resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}engines: {node: '>= 6'}- resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}- engines: {node: '>= 8.10.0'}-resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==}engines: {node: '>=18'}@@ -2368,6 +2360,9 @@ packages:resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==}+resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==}@@ -2471,6 +2466,9 @@ packages:resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}engines: {node: '>= 0.6'}+ resolution: {integrity: sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==}+resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}engines: {node: '>=6'}@@ -2612,6 +2610,9 @@ packages:resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==}engines: {node: '>= 0.6'}+ resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==}+resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}@@ -2663,10 +2664,6 @@ packages:resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}engines: {node: '>= 4.8.0'}- resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}- engines: {node: '>= 6'}-resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==}engines: {node: '>=16 || 14 >=14.18'}@@ -2806,9 +2803,6 @@ packages:resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}- resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==}-resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}@@ -2856,20 +2850,12 @@ packages:resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}- resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}- engines: {node: '>=8'}-resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==}resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}- resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}- engines: {node: '>=0.10.0'}-resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}engines: {node: '>=8'}@@ -2878,10 +2864,6 @@ packages:resolution: {integrity: sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==}engines: {node: '>=6'}- resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}- engines: {node: '>=0.10.0'}-resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}engines: {node: '>=0.12.0'}@@ -3159,8 +3141,8 @@ packages:resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==}engines: {node: '>=6'}- resolution: {integrity: sha512-VdUbreihzsJ6SAV9lm74/UkdvsPqOnkVAzdgtI7+kQlc0UZPtOdo28+dcquIhgSHQx3AO02gEz+3qtiEIhfM3w==}+ resolution: {integrity: sha512-KCsuJ2gM58n6bNdR2Z7gqsh/3TchxxQFbVgax2/UvAjRTgwNSYAJDx9E3jrkBP4jEDHWRCfE47Y2OG+/fiSvEw==}resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==}@@ -3249,6 +3231,9 @@ packages:engines: {node: '>=12.0.0'}hasBin: true+ resolution: {integrity: sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==}+resolution: {integrity: sha512-f3r0yqpz31VXiDB/wj9GaOB0a2PRLQl6vJmXiFrniNwjkKdvakqJRULhjFKJpxOchlCRiG5fcacoUZY5Xa6PEQ==}engines: {node: '>= 18'}@@ -3378,6 +3363,9 @@ packages:resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}+ resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==}+resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}engines: {node: '>=10.5.0'}@@ -3408,11 +3396,6 @@ packages:resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==}- resolution: {integrity: sha512-WDjw3pJ0/0jMFmyNDp3gvY2YizjLmmOUQo6DEBY+JgdvW/yQ9mEeSw6H5ythl5Ny2ytb7f9C2nIbjSxMNzbJXw==}- engines: {node: '>=10'}- hasBin: true-resolution: {integrity: sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}@@ -3543,6 +3526,9 @@ packages:resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==}+ resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==}+resolution: {integrity: sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==}@@ -3643,12 +3629,14 @@ packages:resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}+ resolution: {integrity: sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==}+ engines: {node: '>= 0.10'}+ hasBin: true+resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==}- resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==}-resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}engines: {node: '>=6'}@@ -3713,10 +3701,6 @@ packages:resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==}engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}- resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}- engines: {node: '>=8.10.0'}-resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==}engines: {node: '>= 0.10'}@@ -3897,10 +3881,6 @@ packages:resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}- resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==}- engines: {node: '>=10'}-resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}@@ -3915,6 +3895,9 @@ packages:resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}engines: {node: '>=0.10.0'}+ resolution: {integrity: sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==}+resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}@@ -3932,12 +3915,19 @@ packages:resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}engines: {node: '>= 0.8'}+ resolution: {integrity: sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==}+resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==}resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==}+ resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==}+ engines: {node: '>=0.6.19'}+resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==}engines: {node: '>=10'}@@ -4049,6 +4039,9 @@ packages:resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==}engines: {node: '>=18'}+ resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}+resolution: {integrity: sha512-AOvV5YYIAFFBfransBzSTyztkc3IMfz5Eq3YluaRiEu55nn43Fzaufx70UqEKYr8BoLCach4q8g/bg6e5+/aFw==}@@ -4071,10 +4064,6 @@ packages:resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}engines: {node: '>=0.6'}- resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==}- hasBin: true-resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==}engines: {node: '>=16'}@@ -4128,6 +4117,13 @@ packages:'@swc/wasm':optional: true+ resolution: {integrity: sha512-r6t37Dkk4vK44HwxOe+OzjpE/gDamZAwqXhtcAJD/hPVblcjJK45NxbK0HcDASXG0U4pEnCh640JZbeDVSC6yA==}+ engines: {node: '>=12.12.0'}+ hasBin: true+ peerDependencies:+ typescript: '*'+resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==}@@ -4157,9 +4153,6 @@ packages:engines: {node: '>=14.17'}hasBin: true- resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==}-resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}@@ -5114,7 +5107,7 @@ snapshots:'@babel/traverse': 7.24.6'@babel/types': 7.24.6convert-source-map: 2.0.0- debug: 4.4.1([email protected])+ debug: 4.4.1gensync: 1.0.0-beta.2json5: 2.2.3semver: 6.3.1@@ -5301,7 +5294,7 @@ snapshots:'@babel/helper-split-export-declaration': 7.24.6'@babel/parser': 7.24.6'@babel/types': 7.24.6- debug: 4.4.1([email protected])+ debug: 4.4.1globals: 11.12.0transitivePeerDependencies:- supports-color@@ -6654,7 +6647,7 @@ snapshots:dependencies:- debug: 4.4.1([email protected])+ debug: 4.4.1transitivePeerDependencies:- supports-color@@ -6816,8 +6809,6 @@ snapshots:read-cmd-shim: 4.0.0write-file-atomic: 5.0.1- [email protected]: {}-@@ -6958,18 +6949,6 @@ snapshots:parse5: 7.1.2parse5-htmlparser2-tree-adapter: 7.0.0- dependencies:- anymatch: 3.1.3- braces: 3.0.3- glob-parent: 5.1.2- is-binary-path: 2.1.0- is-glob: 4.0.3- normalize-path: 3.0.0- readdirp: 3.6.0- optionalDependencies:- fsevents: 2.3.3-@@ -7147,11 +7126,9 @@ snapshots:dependencies:ms: 2.1.2dependencies:ms: 2.1.3- optionalDependencies:- supports-color: 5.5.0@@ -7233,6 +7210,8 @@ snapshots:es-errors: 1.3.0gopd: 1.2.0+ [email protected]: {}+dependencies:end-of-stream: 1.4.4@@ -7319,6 +7298,16 @@ snapshots:+ dependencies:+ duplexer: 0.1.2+ from: 0.1.7+ map-stream: 0.1.0+ pause-stream: 0.0.11+ split: 0.3.3+ stream-combiner: 0.0.4+ through: 2.3.8+@@ -7499,6 +7488,8 @@ snapshots:+ [email protected]: {}+@@ -7568,10 +7559,6 @@ snapshots:shelljs: 0.8.5shelljs.exec: 1.1.8- dependencies:- is-glob: 4.0.3-dependencies:foreground-child: 3.2.1@@ -7690,7 +7677,7 @@ snapshots:dependencies:'@tootallnate/once': 2.0.0agent-base: 6.0.2- debug: 4.4.1([email protected])+ debug: 4.4.1transitivePeerDependencies:- supports-color@@ -7711,7 +7698,7 @@ snapshots:dependencies:agent-base: 7.1.3- debug: 4.4.1([email protected])+ debug: 4.4.1transitivePeerDependencies:- supports-color@@ -7738,8 +7725,6 @@ snapshots:- [email protected]: {}-@@ -7771,7 +7756,7 @@ snapshots:dependencies:'@ioredis/commands': 1.2.0cluster-key-slot: 1.1.2- debug: 4.4.1([email protected])+ debug: 4.4.1denque: 2.1.0lodash.defaults: 4.2.0lodash.isarguments: 3.1.0@@ -7789,26 +7774,16 @@ snapshots:- dependencies:- binary-extensions: 2.3.0-dependencies:hasown: 2.0.2- [email protected]: {}-- dependencies:- is-extglob: 2.1.1-@@ -7862,7 +7837,7 @@ snapshots:dependencies:- debug: 4.4.1([email protected])+ debug: 4.4.1istanbul-lib-coverage: 3.2.2source-map: 0.6.1transitivePeerDependencies:@@ -8299,7 +8274,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}@@ -8396,6 +8371,8 @@ snapshots:underscore: 1.13.6xmlbuilder: 10.1.1+ [email protected]: {}+@@ -8498,6 +8475,8 @@ snapshots:+ [email protected]: {}+@@ -8523,19 +8502,6 @@ snapshots:- dependencies:- chokidar: 3.6.0- debug: 4.4.1([email protected])- ignore-by-default: 1.0.1- minimatch: 3.1.2- pstree.remy: 1.1.8- semver: 7.6.2- simple-update-notifier: 2.0.0- supports-color: 5.5.0- touch: 3.1.1- undefsafe: 2.0.5-dependencies:abbrev: 2.0.0@@ -8650,6 +8616,10 @@ snapshots:+ dependencies:+ through: 2.3.8+dependencies:'@pdf-lib/standard-fonts': 1.0.0@@ -8739,12 +8709,14 @@ snapshots:+ dependencies:+ event-stream: 3.3.4+dependencies:punycode: 2.3.1- [email protected]: {}-@@ -8818,10 +8790,6 @@ snapshots:process: 0.11.10string_decoder: 1.3.0- dependencies:- picomatch: 2.3.1-dependencies:resolve: 1.22.8@@ -8846,7 +8814,7 @@ snapshots:dependencies:- debug: 4.4.1([email protected])+ debug: 4.4.1module-details-from-path: 1.0.3resolve: 1.22.8transitivePeerDependencies:@@ -9018,10 +8986,6 @@ snapshots:dependencies:is-arrayish: 0.3.2- dependencies:- semver: 7.6.2-@@ -9033,6 +8997,10 @@ snapshots:+ dependencies:+ through: 2.3.8+@@ -9045,12 +9013,18 @@ snapshots:+ dependencies:+ duplexer: 0.1.2+dependencies:stubs: 3.0.0+ [email protected]: {}+dependencies:char-regex: 1.0.2@@ -9184,6 +9158,8 @@ snapshots:+ [email protected]: {}+@@ -9200,8 +9176,6 @@ snapshots:- [email protected]: {}-dependencies:tldts: 6.1.75@@ -9250,6 +9224,14 @@ snapshots:v8-compile-cache-lib: 3.0.1yn: 3.1.1+ dependencies:+ cross-spawn: 7.0.6+ node-cleanup: 2.1.2+ ps-tree: 1.2.0+ string-argv: 0.3.2+ typescript: 5.8.3+@@ -9271,8 +9253,6 @@ snapshots:- [email protected]: {}-diff --git a/apps/api/sharedLibs/crawler/Cargo.lock b/apps/api/sharedLibs/crawler/Cargo.lockindex 3412e19ce..8a753c488 100644--- a/apps/api/sharedLibs/crawler/Cargo.lock+++ b/apps/api/sharedLibs/crawler/Cargo.lock@@ -11,16 +11,33 @@ dependencies = ["memchr",]+[[package]]+name = "anyhow"+version = "1.0.98"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"++[[package]]+name = "bstr"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"+dependencies = [+ "lazy_static",+ "memchr",+ "regex-automata 0.1.10",+]+[[package]]name = "crawler"version = "0.1.0"dependencies = ["libc","regex",- "robotstxt","roxmltree","serde","serde_json",+ "texting_robots","url",]@@ -157,6 +174,12 @@ version = "1.0.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"+[[package]]+name = "lazy_static"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"+[[package]]name = "libc"version = "0.2.174"@@ -175,6 +198,22 @@ version = "2.7.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"+[[package]]+name = "minimal-lexical"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"++[[package]]+name = "nom"+version = "7.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"+dependencies = [+ "memchr",+ "minimal-lexical",+]+[[package]]name = "percent-encoding"version = "2.3.1"@@ -216,10 +255,16 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"dependencies = ["aho-corasick","memchr",- "regex-automata",+ "regex-automata 0.4.9","regex-syntax",]+[[package]]+name = "regex-automata"+version = "0.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"+[[package]]name = "regex-automata"version = "0.4.9"@@ -237,12 +282,6 @@ version = "0.8.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"-[[package]]-name = "robotstxt"-version = "0.3.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bbc52377db80e3fec3a2c748ca603b8b6cacdd34ff89ff4b742a635361d4b4a7"-[[package]]name = "roxmltree"version = "0.20.0"@@ -321,6 +360,42 @@ dependencies = ["syn",]+[[package]]+name = "texting_robots"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a"+dependencies = [+ "anyhow",+ "bstr",+ "lazy_static",+ "nom",+ "percent-encoding",+ "regex",+ "thiserror",+ "url",+]++[[package]]+name = "thiserror"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"+dependencies = [+ "thiserror-impl",+]++[[package]]+name = "thiserror-impl"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "tinystr"version = "0.8.1"diff --git a/apps/api/sharedLibs/crawler/Cargo.toml b/apps/api/sharedLibs/crawler/Cargo.tomlindex 342789129..1e43b72e9 100644--- a/apps/api/sharedLibs/crawler/Cargo.toml+++ b/apps/api/sharedLibs/crawler/Cargo.toml@@ -9,7 +9,7 @@ serde = { version = "1.0", features = ["derive"] }serde_json = "1.0"url = "2.5.4"regex = "1.11.1"-robotstxt = "0.3.0"+texting_robots = "0.2.2"roxmltree = "0.20"[lib]diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rsindex aec7935a7..0ee510f98 100644--- a/apps/api/sharedLibs/crawler/src/lib.rs+++ b/apps/api/sharedLibs/crawler/src/lib.rs@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};use std::ffi::{CStr, CString};use url::Url;use regex::Regex;-use robotstxt::DefaultMatcher;+use texting_robots::Robot;#[derive(Deserialize)]struct FilterLinksCall {@@ -165,15 +165,27 @@ fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std}if !data.ignore_robots_txt {- let mut matcher = DefaultMatcher::default();- let allowed = matcher.allowed_by_robots(&data.robots_txt, vec![- "FireCrawlAgent",- "FirecrawlAgent",- ], url.as_str());-- if !allowed {- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());- return false;+ match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {+ Ok(robot) => {+ let allowed = robot.allowed(url.as_str());+ if !allowed {+ match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {+ Ok(robot_alt) => {+ let allowed_alt = robot_alt.allowed(url.as_str());+ if !allowed_alt {+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());+ return false;+ }+ }+ Err(_) => {+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());+ return false;+ }+ }+ }+ }+ Err(_) => {+ }}}@@ -513,6 +525,104 @@ mod tests {assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");}+ #[test]+ fn test_filter_links_normal_robots_txt() {+ let data = FilterLinksCall {+ links: vec!["https://example.com/allowed".to_string(), "https://example.com/disallowed".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: "User-agent: *\nDisallow: /disallowed".to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data).unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/allowed");+ assert!(result.denial_reasons.contains_key("https://example.com/disallowed"));+ assert_eq!(result.denial_reasons.get("https://example.com/disallowed").unwrap(), "ROBOTS_TXT");+ }++ #[test]+ fn test_filter_links_malformed_robots_txt() {+ let data = FilterLinksCall {+ links: vec!["https://example.com/test".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: "Invalid robots.txt content with \x00 null bytes and malformed syntax".to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/test");+ }++ #[test]+ fn test_filter_links_non_utf8_robots_txt() {+ let mut non_utf8_bytes = vec![0xFF, 0xFE];+ non_utf8_bytes.extend_from_slice(b"User-agent: *\nDisallow: /blocked");+ let non_utf8_string = String::from_utf8_lossy(&non_utf8_bytes).to_string();++ let data = FilterLinksCall {+ links: vec!["https://example.com/allowed".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: non_utf8_string,+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/allowed");+ }++ #[test]+ fn test_filter_links_char_boundary_issue() {+ let problematic_content = "User-agent: *\nDisallow: /\u{a0}test";++ let data = FilterLinksCall {+ links: vec!["https://example.com/test".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: problematic_content.to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/test");+ }+}/// Frees a string allocated in Rust-land.diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex cccbb7370..588259e49 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -1,5 +1,6 @@import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing, crawlStart, Identity, idmux, scrapeTimeout } from "./lib";import { describe, it, expect } from "@jest/globals";+import { filterLinks } from "../../lib/crawler";let identity: Identity;@@ -272,3 +273,89 @@ describe("Crawl tests", () => {expect(typeof response.body.id).toBe("string");});});++describe("Robots.txt FFI Integration tests", () => {+ it.concurrent("handles normal robots.txt parsing via FFI", async () => {++ const result = await filterLinks({+ links: ['https://example.com/allowed', 'https://example.com/disallowed'],+ limit: 10,+ max_depth: 10,+ base_url: 'https://example.com',+ initial_url: 'https://example.com',+ regex_on_full_url: false,+ excludes: [],+ includes: [],+ allow_backward_crawling: true,+ ignore_robots_txt: false,+ robots_txt: 'User-agent: *\nDisallow: /disallowed'+ });++ expect(result.links).toHaveLength(1);+ expect(result.links[0]).toBe('https://example.com/allowed');+ expect(result.denial_reasons.has('https://example.com/disallowed')).toBe(true);+ expect(result.denial_reasons.get('https://example.com/disallowed')).toBe('ROBOTS_TXT');+ }, 10000);++ it.concurrent("handles malformed robots.txt without crashing via FFI", async () => {++ const result = await filterLinks({+ links: ['https://example.com/test'],+ limit: 10,+ max_depth: 10,+ base_url: 'https://example.com',+ initial_url: 'https://example.com',+ regex_on_full_url: false,+ excludes: [],+ includes: [],+ allow_backward_crawling: true,+ ignore_robots_txt: false,+ robots_txt: 'Invalid robots.txt content with \x00 null bytes and malformed syntax'+ });++ expect(result.links).toHaveLength(1);+ expect(result.links[0]).toBe('https://example.com/test');+ }, 10000);++ it.concurrent("handles non-UTF8 robots.txt content without crashing via FFI", async () => {++ const nonUtf8Content = String.fromCharCode(0xFF, 0xFE) + 'User-agent: *\nDisallow: /blocked';+ const result = await filterLinks({+ links: ['https://example.com/allowed'],+ limit: 10,+ max_depth: 10,+ base_url: 'https://example.com',+ initial_url: 'https://example.com',+ regex_on_full_url: false,+ excludes: [],+ includes: [],+ allow_backward_crawling: true,+ ignore_robots_txt: false,+ robots_txt: nonUtf8Content+ });++ expect(result.links).toHaveLength(1);+ expect(result.links[0]).toBe('https://example.com/allowed');+ }, 10000);++ it.concurrent("handles char boundary issues without crashing via FFI", async () => {++ const problematicContent = 'User-agent: *\nDisallow: /\u{a0}test';+ const result = await filterLinks({+ links: ['https://example.com/safe'],+ limit: 10,+ max_depth: 10,+ base_url: 'https://example.com',+ initial_url: 'https://example.com',+ regex_on_full_url: false,+ excludes: [],+ includes: [],+ allow_backward_crawling: true,+ ignore_robots_txt: false,+ robots_txt: problematicContent+ });++ expect(result.links).toHaveLength(1);+ expect(result.links[0]).toBe('https://example.com/safe');+ }, 10000);+});diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex e8ee156a7..9f6e5b7bb 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -1,7 +1,7 @@import { Request, Response } from "express";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";-import { createRedisConnection, getScrapeQueue } from "../../../src/services/queue-service";+import { getScrapeQueue } from "../../../src/services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";@@ -12,13 +12,11 @@ import { Job } from "bullmq";import { toLegacyDocument } from "../v1/types";import type { DBJob, PseudoJob } from "../v1/crawl-status";import { getJobFromGCS } from "../../lib/gcs-jobs";-import IORedis from "ioredis";configDotenv();-export async function getJobs(crawlId: string, ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {- const queue = getScrapeQueue(conn);+export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -99,10 +97,8 @@ export async function crawlStatusController(req: Request, res: Response) {return res.status(403).json({ error: "Forbidden" });}let jobIDs = await getCrawlJobs(req.params.jobId);- const conn = createRedisConnection();- let jobs = await getJobs(req.params.jobId, jobIDs, conn);+ let jobs = await getJobs(req.params.jobId, jobIDs);let jobStatuses = await Promise.all(jobs.map((x) => x.getState()));- conn.disconnect();// Combine jobs and jobStatuses into a single array of objectslet jobsWithStatuses = jobs.map((job, index) => ({diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 9d0e005bd..d3425dff3 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -94,47 +94,36 @@ export async function scrapeHelper(let doc;- const err = await Sentry.startSpan(- {- name: "Wait for job to finish",- op: "bullmq.wait",- attributes: { job: jobId },- },- async (span) => {- try {- doc = await waitForJob(jobId, timeout);- } catch (e) {- if (- e instanceof Error &&- (e.message.startsWith("Job wait") || e.message === "timeout")- ) {- span.setAttribute("timedOut", true);- return {- success: false,- error: "Request timed out",- returnCode: 408,- };- } else if (- typeof e === "string" &&- (e.includes("Error generating completions: ") ||- e.includes("Invalid schema for function") ||- e.includes(- "LLM extraction did not match the extraction schema you provided.",- ))- ) {- return {- success: false,- error: e,- returnCode: 500,- };- } else {- throw e;- }- }- span.setAttribute("result", JSON.stringify(doc));- return null;- },- );+ try {+ doc = await waitForJob(jobId, timeout);+ } catch (e) {+ if (+ e instanceof Error &&+ (e.message.startsWith("Job wait") || e.message === "timeout")+ ) {+ return {+ success: false,+ error: "Request timed out",+ returnCode: 408,+ };+ } else if (+ typeof e === "string" &&+ (e.includes("Error generating completions: ") ||+ e.includes("Invalid schema for function") ||+ e.includes(+ "LLM extraction did not match the extraction schema you provided.",+ ))+ ) {+ return {+ success: false,+ error: e,+ returnCode: 500,+ };+ } else {+ throw e;+ }+ }+ const err = null;if (err !== null) {return err;diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.tsindex 47ef2fd4f..c68579eac 100644--- a/apps/api/src/controllers/v0/status.ts+++ b/apps/api/src/controllers/v0/status.ts@@ -3,7 +3,6 @@ import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";import { getJobs } from "./crawl-status";import * as Sentry from "@sentry/node";-import { createRedisConnection } from "../../services/queue-service";export async function crawlJobStatusPreviewController(req: Request,@@ -26,12 +25,10 @@ export async function crawlJobStatusPreviewController(// }// }- const conn = createRedisConnection();- const jobs = (await getJobs(req.params.jobId, jobIDs, conn)).sort(+ const jobs = (await getJobs(req.params.jobId, jobIDs)).sort((a, b) => a.timestamp - b.timestamp,);const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));- conn.disconnect();const jobStatus = sc.cancelled? "failed": jobStatuses.every((x) => x === "completed")diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex cee458ab6..190ca5209 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -19,7 +19,7 @@ import {isCrawlFinished,isCrawlFinishedLocked,} from "../../lib/crawl-redis";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import { getJob, getJobs } from "./crawl-status";import * as Sentry from "@sentry/node";import { Job, JobState } from "bullmq";@@ -88,8 +88,7 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);+ const queue = getScrapeQueue();const jobStatuses = await Promise.all(notDoneJobIDs.map(async (x) => [@@ -102,7 +101,7 @@ async function crawlStatusWS(.map((x) => x[0]);const newlyDoneJobs: Job[] = (- await Promise.all(newlyDoneJobIDs.map((x) => getJob(x, conn)))+ await Promise.all(newlyDoneJobIDs.map((x) => getJob(x)))).filter((x) => x !== undefined) as Job[];for (const job of newlyDoneJobs) {@@ -117,7 +116,6 @@ async function crawlStatusWS(}doneJobIDs.push(...newlyDoneJobIDs);- conn.disconnect();setTimeout(loop, 1000);};@@ -127,8 +125,7 @@ async function crawlStatusWS(let jobIDs = await getCrawlJobs(req.params.jobId);- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);+ const queue = getScrapeQueue();let jobStatuses = await Promise.all(jobIDs.map(@@ -163,8 +160,7 @@ async function crawlStatusWS(jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processing- const doneJobs = await getJobs(doneJobIDs, conn);- conn.disconnect();+ const doneJobs = await getJobs(doneJobIDs);const data = doneJobs.map((x) => x.returnvalue);await send(ws, {diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 47e99a7d3..7bca8d6d6 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -13,7 +13,7 @@ import {getDoneJobsOrderedLength,isCrawlKickoffFinished,} from "../../lib/crawl-redis";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import {supabaseGetJobById,supabaseGetJobsById,@@ -24,7 +24,6 @@ import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";import { getJobFromGCS } from "../../lib/gcs-jobs";-import IORedis from "ioredis";configDotenv();export type PseudoJob<T> = {@@ -41,9 +40,9 @@ export type PseudoJob<T> = {export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string }-export async function getJob(id: string, conn: IORedis): Promise<PseudoJob<any> | null> {+export async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue(conn).getJob(id),+ getScrapeQueue().getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);@@ -73,10 +72,9 @@ export async function getJob(id: string, conn: IORedis): Promise<PseudoJob<any>return job;}-export async function getJobs(ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {- const queue = getScrapeQueue(conn);+export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -159,14 +157,11 @@ export async function crawlStatusController(}let jobIDs = await getCrawlJobs(req.params.jobId);- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await queue.getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,),);- conn.disconnect();if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {for (let rangeStart = 0; ; rangeStart += 1000) {@@ -261,7 +256,7 @@ export async function crawlStatusController(const creditsRpc = await supabase_rr_service.rpc("credits_billed_by_crawl_id_1", {i_crawl_id: req.params.jobId,- });+ }, { get: true });creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? (totalCount * (sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")@@ -294,15 +289,12 @@ export async function crawlStatusController(return res.status(403).json({ success: false, error: "Forbidden" });}- const teamIdsExcludedFromExpiry = [- "8f819703-1b85-4f7f-a6eb-e03841ec6617",- "f96ad1a4-8102-4b35-9904-36fd517d3616",- ];+ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;if (crawlJob- && !teamIdsExcludedFromExpiry.includes(crawlJob.team_id)- && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > 24 * 60 * 60 * 1000+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs) {return res.status(404).json({ success: false, error: "Job expired" });}@@ -385,9 +377,8 @@ export async function crawlStatusController(i += factor) {// get current chunk and retrieve jobs- const conn = createRedisConnection();const currentIDs = doneJobsOrder.slice(i, i + factor);- const jobs = await getJobs(currentIDs, conn);+ const jobs = await getJobs(currentIDs);// iterate through jobs and add them one them one to the byte counter// both loops will break once we cross the byte counter@@ -410,7 +401,6 @@ export async function crawlStatusController(doneJobs.push(job);bytes += JSON.stringify(job.returnvalue ?? null).length;}- conn.disconnect();}// if we ran over the bytes limit, remove the last document, except if it's the only document@@ -418,15 +408,13 @@ export async function crawlStatusController(doneJobs.splice(doneJobs.length - 1, 1);}} else {- const conn = createRedisConnection();doneJobs = (await Promise.all(- (await getJobs(doneJobsOrder, conn)).map(async (x) =>+ (await getJobs(doneJobsOrder)).map(async (x) =>(await x.getState()) === "failed" ? null : x,),)).filter((x) => x !== null) as PseudoJob<any>[];- conn.disconnect();}const data = doneJobs.map((x) => x.returnvalue);diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex 472e208eb..525db6b6c 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -75,34 +75,9 @@ export async function deepResearchController(summaries: [],});- if (Sentry.isInitialized()) {- const size = JSON.stringify(jobData).length;- await Sentry.startSpan(- {- name: "Add deep research job",- op: "queue.publish",- attributes: {- "messaging.message.id": researchId,- "messaging.destination.name": getDeepResearchQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await getDeepResearchQueue().add(researchId, {- ...jobData,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- }, { jobId: researchId });- },- );- } else {- await getDeepResearchQueue().add(researchId, jobData, {- jobId: researchId,- });- }+ await getDeepResearchQueue().add(researchId, jobData, {+ jobId: researchId,+ });return res.status(200).json({success: true,diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex e6d356b85..7eba40a60 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -119,34 +119,9 @@ export async function extractController(zeroDataRetention: req.acuc?.flags?.forceZDR,});- if (Sentry.isInitialized()) {- const size = JSON.stringify(jobData).length;- await Sentry.startSpan(- {- name: "Add extract job",- op: "queue.publish",- attributes: {- "messaging.message.id": extractId,- "messaging.destination.name": getExtractQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await getExtractQueue().add(extractId, {- ...jobData,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- }, { jobId: extractId });- },- );- } else {- await getExtractQueue().add(extractId, jobData, {- jobId: extractId,- });- }+ await getExtractQueue().add(extractId, jobData, {+ jobId: extractId,+ });return res.status(200).json({success: true,diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex 4177af00d..6834b819f 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -51,38 +51,9 @@ export async function generateLLMsTextController(fullText: "",});- if (Sentry.isInitialized()) {- const size = JSON.stringify(jobData).length;- await Sentry.startSpan(- {- name: "Add LLMstxt generation job",- op: "queue.publish",- attributes: {- "messaging.message.id": generationId,- "messaging.destination.name": getGenerateLlmsTxtQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await getGenerateLlmsTxtQueue().add(- generationId,- {- ...jobData,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- },- { jobId: generationId },- );- },- );- } else {- await getGenerateLlmsTxtQueue().add(generationId, jobData, {- jobId: generationId,- });- }+ await getGenerateLlmsTxtQueue().add(generationId, jobData, {+ jobId: generationId,+ });return res.status(200).json({success: true,diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.tsindex 802b50a48..78ded113a 100644--- a/apps/api/src/controllers/v1/scrape-status.ts+++ b/apps/api/src/controllers/v1/scrape-status.ts@@ -2,7 +2,6 @@ import { Response } from "express";import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";-import { createRedisConnection } from "../../services/queue-service";export async function scrapeStatusController(req: any, res: any) {const logger = _logger.child({@@ -36,9 +35,7 @@ export async function scrapeStatusController(req: any, res: any) {});}- const conn = createRedisConnection();- const jobData = await getJob(req.params.jobId, conn);- conn.disconnect();+ const jobData = await getJob(req.params.jobId);const data = Array.isArray(jobData?.returnvalue)? jobData?.returnvalue[0]: jobData?.returnvalue;diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex aa7256bf0..c61301930 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -17,7 +17,7 @@ export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,res: Response<ScrapeResponse>,) {- const jobId = uuidv4();+ const jobId: string = uuidv4();const preNormalizedBody = { ...req.body };if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {@@ -60,7 +60,7 @@ export async function scrapeController(const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(req.body, req.body.timeout, req.auth.team_id);- await addScrapeJob(+ const bullJob = await addScrapeJob({url: req.body.url,mode: "single_urls",@@ -85,7 +85,7 @@ export async function scrapeController(jobPriority,isDirectToBullMQ,);- logger.info("Added scrape job now")+ logger.info("Added scrape job now" + (bullJob ? "" : " (to concurrency queue)"));const totalWait =(req.body.waitFor ?? 0) +@@ -96,7 +96,7 @@ export async function scrapeController(let doc: Document;try {- doc = await waitForJob(jobId, timeout + totalWait, logger);+ doc = await waitForJob(bullJob ? bullJob : jobId, timeout + totalWait, logger);} catch (e) {logger.error(`Error in scrapeController`, {startTime,@@ -125,9 +125,7 @@ export async function scrapeController(logger.info("Done with waitForJob");- const conn = createRedisConnection();- await getScrapeQueue(conn).remove(jobId);- conn.disconnect();+ await getScrapeQueue().remove(jobId);logger.info("Removed job from queue");diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 2825a7deb..620820e7f 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1111,6 +1111,7 @@ export type TeamFlags = {zdrCost?: number;checkRobotsOnScrape?: boolean;allowTeammateInvites?: boolean;+ crawlTtlHours?: number;} | null;export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex f7e4a27b9..8153ecdfa 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -200,6 +200,12 @@ export const screenshotFormatWithOptions = z.object({export type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;+export const parsersSchema = z.object({+ pdf: z.boolean().default(true),+}).default({ pdf: true });++export type Parsers = z.infer<typeof parsersSchema>;+const baseScrapeOptions = z.object({formats: z@@ -252,7 +258,7 @@ const baseScrapeOptions = z.max(60000).default(0),mobile: z.boolean().default(false),- parsePDF: z.boolean().default(true),+ parsers: parsersSchema.optional(),actions: actionsSchema.optional(),location: z@@ -1117,6 +1123,7 @@ export function fromV1ScrapeOptions(return x;}}).filter(x => x !== null),+ parsers: v1ScrapeOptions.parsePDF !== undefined ? { pdf: v1ScrapeOptions.parsePDF } : undefined,}),internalOptions: {teamId,diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex d68b4d400..8a806d2f0 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -200,6 +200,10 @@ export async function finishCrawl(id: string, __logger: Logger = _logger) {await redisEvictConnection.srem("crawls_by_team_id:" + crawl.team_id, id);await redisEvictConnection.expire("crawls_by_team_id:" + crawl.team_id, 24 * 60 * 60);}++ // Clear visited sets to save memory+ await redisEvictConnection.del("crawl:" + id + ":visited");+ await redisEvictConnection.del("crawl:" + id + ":visited_unique");}export async function getCrawlJobs(id: string): Promise<string[]> {diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.tsindex aa63a6b39..a474e34e2 100644--- a/apps/api/src/lib/html-to-markdown.ts+++ b/apps/api/src/lib/html-to-markdown.ts@@ -17,6 +17,7 @@ const goExecutablePath = join();class GoMarkdownConverter {+ private static instance: GoMarkdownConverter;private convert: any;private free: any;@@ -29,12 +30,15 @@ class GoMarkdownConverter {}public static async getInstance(): Promise<GoMarkdownConverter> {- try {- await stat(goExecutablePath);- } catch (_) {- throw Error("Go shared library not found");+ if (!GoMarkdownConverter.instance) {+ try {+ await stat(goExecutablePath);+ } catch (_) {+ throw Error("Go shared library not found");+ }+ GoMarkdownConverter.instance = new GoMarkdownConverter();}- return new GoMarkdownConverter();+ return GoMarkdownConverter.instance;}public async convertHTMLToMarkdown(html: string): Promise<string> {diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 468850860..0b601d87c 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -35,7 +35,7 @@ export async function fetchRobotsTxt((x) => x[0].toLowerCase() === "content-type",) ?? [])[1] ?? "";- if (contentType.includes("text/html") ||+ if ((contentType.includes("text/html") && response.data.trim().startsWith("<")) ||contentType.includes("application/json") ||contentType.includes("application/xml")) {return "";diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex e5358d4c2..4c93de040 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -32,7 +32,8 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, internacreditsToBeBilled += (flags?.zdrCost ?? 1);}- if (options.parsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {+ const shouldParsePDF = options.parsers?.pdf ?? true;+ if (shouldParsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1);}diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 81c5eee20..d3a967de4 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -20,11 +20,9 @@ configDotenv();export async function startWebScraperPipeline({job,- token,costTracking,}: {job: Job<WebScraperOptions> & { id: string };- token: string;costTracking: CostTracking;}) {return await runWebScraper({@@ -38,15 +36,11 @@ export async function startWebScraperPipeline({}: {}),},- internalOptions: job.data.internalOptions,- // onSuccess: (result, mode) => {- // logger.debug(`🐂 Job completed ${job.id}`);- // saveJob(job, result, token, mode);- // },- // onError: (error) => {- // logger.error(`🐂 Job failed ${job.id}`);- // ScrapeEvents.logJobEvent(job, "failed");- // },+ internalOptions: {+ crawlId: job.data.crawl_id,+ teamId: job.data.team_id,+ ...job.data.internalOptions,+ },team_id: job.data.team_id,bull_job_id: job.id.toString(),priority: job.opts.priority,@@ -62,8 +56,6 @@ export async function runWebScraper({mode,scrapeOptions,internalOptions,- // onSuccess,- // onError,team_id,bull_job_id,priority,@@ -124,9 +116,6 @@ export async function runWebScraper({}}- // This is where the returnvalue from the job is set- // onSuccess(response.document, mode);-engines = response.engines;if (@@ -201,7 +190,6 @@ export async function runWebScraper({const saveJob = async (job: Job,result: any,- token: string,mode: string,engines?: EngineResultsTracker,) => {diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex d6fcda657..3672097d1 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -353,6 +353,7 @@ export class WebCrawler {};const _urlsHandler = async (urls: string[]) => {+ this.logger.debug("urlsHandler invoked");if (fromMap && onlySitemap) {return await urlsHandler(urls);} else {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 768b63a96..cc1a1e4aa 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -135,35 +135,17 @@ export async function fireEngineCheckStatus(abort?: AbortSignal,production = true,): Promise<FireEngineCheckStatusSuccess> {- let status = await Sentry.startSpan(- {- name: "fire-engine: Check status",- attributes: {- jobId,- },- },- async (span) => {- const x = Date.now();- meta.logger.debug("Checking status (" + x + ")");- const res = await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,- method: "GET",- logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),- headers: {- ...(Sentry.isInitialized()- ? {- "sentry-trace": Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- }- : {}),- },- mock,- abort,- });- meta.logger.debug("Status came back (" + x + ")");- return res;- },- );+ const x = Date.now();+ meta.logger.debug("Checking status (" + x + ")");+ let status = await robustFetch({+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,+ method: "GET",+ logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),+ headers: {},+ mock,+ abort,+ });+ meta.logger.debug("Status came back (" + x + ")");// Fire-engine now saves the content to GCSif (!status.content && status.docUrl) {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.tsindex 9af6627ae..b9185a871 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts@@ -12,33 +12,16 @@ export async function fireEngineDelete(abort?: AbortSignal,production = true,) {- await Sentry.startSpan(- {- name: "fire-engine: Delete scrape",- attributes: {- jobId,- },- },- async (span) => {- await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,- method: "DELETE",- headers: {- ...(Sentry.isInitialized()- ? {- "sentry-trace": Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- }- : {}),- },- ignoreResponse: true,- ignoreFailure: true,- logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),- mock,- abort,- });- },- );+ await robustFetch({+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,+ method: "DELETE",+ headers: {},+ ignoreResponse: true,+ ignoreFailure: true,+ logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),+ mock,+ abort,+ });// We do not care whether this fails or not.}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex f3c566817..fed212dcf 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -79,34 +79,17 @@ export async function fireEngineScrape<abort?: AbortSignal,production = true,): Promise<z.infer<typeof schema>> {- const scrapeRequest = await Sentry.startSpan(- {- name: "fire-engine: Scrape",- attributes: {- url: request.url,- },- },- async (span) => {- return await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,- method: "POST",- headers: {- ...(Sentry.isInitialized()- ? {- "sentry-trace": Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- }- : {}),- },- body: request,- logger: logger.child({ method: "fireEngineScrape/robustFetch" }),- schema,- tryCount: 3,- mock,- abort,- });- },- );+ const scrapeRequest = await robustFetch({+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,+ method: "POST",+ headers: {},+ body: request,+ logger: logger.child({ method: "fireEngineScrape/robustFetch" }),+ schema,+ tryCount: 3,+ mock,+ abort,+ });return scrapeRequest;}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 6649c735d..87b1a2217 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -175,7 +175,9 @@ export async function scrapePDF(): Promise<EngineScrapeResult> {const startTime = Date.now();- if (!meta.options.parsePDF) {+ const shouldParsePDF = meta.options.parsers?.pdf ?? true;++ if (!shouldParsePDF) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {const content = (await readFile(meta.pdfPrefetch.filePath)).toString("base64",diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex ce009eb59..1a477502c 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -187,6 +187,9 @@ async function buildMetaObject(scrapeId: id,scrapeURL: url,zeroDataRetention: internalOptions.zeroDataRetention,+ teamId: internalOptions.teamId,+ team_id: internalOptions.teamId,+ crawlId: internalOptions.crawlId,});const logs: any[] = [];@@ -211,6 +214,7 @@ async function buildMetaObject(export type InternalOptions = {teamId: string;+ crawlId?: string;priority?: number; // Passed along to fire-engineforceEngine?: Engine | Engine[];diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.tsindex fee2b45d5..39a4db931 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts@@ -12,7 +12,7 @@ export async function extractMetadataRust(return {...fromRust,...(fromRust.favicon ? {- favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url)+ favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url).href} : {}),scrapeId: meta.id,};diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex 11d77da22..8dad621db 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -5,7 +5,6 @@ import { getJob } from "../../../controllers/v1/crawl-status";import gitDiff from 'git-diff';import parseDiff from 'parse-diff';import { generateCompletions } from "./llmExtract";-import { createRedisConnection } from "../../../services/queue-service";async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extract: any } | null> {const changeTrackingFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")!;@@ -69,7 +68,6 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume}const start = Date.now();- const conn = createRedisConnection();const res = await supabase_service.rpc("diff_get_last_scrape_4", {i_team_id: meta.internalOptions.teamId,@@ -88,8 +86,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documeconst job: {returnvalue: Document,- } | null = data?.o_job_id ? await getJob(data.o_job_id, conn) : null;- conn.disconnect();+ } | null = data?.o_job_id ? await getJob(data.o_job_id) : null;if (data && job && job?.returnvalue) {const previousMarkdown = job.returnvalue.markdown!;const currentMarkdown = document.markdown!;diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 9c829baa4..ab9b68d41 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -261,30 +261,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =runningJobs.add(job.id);}- if (job.data && job.data.sentry && Sentry.isInitialized()) {- Sentry.continueTrace(- {- sentryTrace: job.data.sentry.trace,- baggage: job.data.sentry.baggage,- },- () => {- Sentry.startSpan(- {- name: "Index job",- attributes: {- job: job.id,- worker: process.env.FLY_MACHINE_ID ?? worker.id,- },- },- async () => {- await jobProcessor(token, job);- },- );- },- );- } else {- await jobProcessor(token, job);- }+ await jobProcessor(token, job);if (job.id) {runningJobs.delete(job.id);diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex a6302ff5f..d6947659a 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,4 +1,4 @@-import { createRedisConnection, getScrapeQueue } from "./queue-service";+import { getScrapeQueue, getScrapeQueueEvents } from "./queue-service";import { v4 as uuidv4 } from "uuid";import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";import * as Sentry from "@sentry/node";@@ -19,6 +19,7 @@ import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs";import { Document } from "../controllers/v1/types";import { getCrawl } from "../lib/crawl-redis";import { Logger } from "winston";+import { Job } from "bullmq";/*** Checks if a job is a crawl or batch scrape based on its options@@ -54,7 +55,7 @@ export async function _addScrapeJobToBullMQ(options: any,jobId: string,jobPriority: number,-) {+): Promise<Job> {if (webScraperOptions &&webScraperOptions.team_id@@ -69,13 +70,11 @@ export async function _addScrapeJobToBullMQ(}}- const conn = createRedisConnection();- await getScrapeQueue(conn).add(jobId, webScraperOptions, {+ return await getScrapeQueue().add(jobId, webScraperOptions, {...options,priority: jobPriority,jobId,});- conn.disconnect();}async function addScrapeJobRaw(@@ -84,7 +83,7 @@ async function addScrapeJobRaw(jobId: string,jobPriority: number,directToBullMQ: boolean = false,-) {+): Promise<Job | null> {let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;let currentActiveConcurrency = 0;let maxConcurrency = 0;@@ -145,8 +144,9 @@ async function addScrapeJobRaw(jobId,jobPriority,);+ return null;} else {- await _addScrapeJobToBullMQ(webScraperOptions, options, jobId, jobPriority);+ return await _addScrapeJobToBullMQ(webScraperOptions, options, jobId, jobPriority);}}@@ -156,39 +156,8 @@ export async function addScrapeJob(jobId: string = uuidv4(),jobPriority: number = 10,directToBullMQ: boolean = false,-) {- if (Sentry.isInitialized()) {- const size = JSON.stringify(webScraperOptions).length;- return await Sentry.startSpan(- {- name: "Add scrape job",- op: "queue.publish",- attributes: {- "messaging.message.id": jobId,- "messaging.destination.name": getScrapeQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await addScrapeJobRaw(- {- ...webScraperOptions,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- },- options,- jobId,- jobPriority,- directToBullMQ,- );- },- );- } else {- await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority, directToBullMQ);- }+): Promise<Job | null> {+ return await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority, directToBullMQ);}export async function addScrapeJobs(@@ -322,33 +291,11 @@ export async function addScrapeJobs(await Promise.all(addToCQ.map(async (job) => {const size = JSON.stringify(job.data).length;- return await Sentry.startSpan(- {- name: "Add scrape job",- op: "queue.publish",- attributes: {- "messaging.message.id": job.opts.jobId,- "messaging.destination.name": getScrapeQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- const jobData = {- ...job.data,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- };-- await _addScrapeJobToConcurrencyQueue(- jobData,- job.opts,- job.opts.jobId,- job.opts.priority,- );- },+ await _addScrapeJobToConcurrencyQueue(+ job.data,+ job.opts,+ job.opts.jobId,+ job.opts.priority,);}),);@@ -356,88 +303,55 @@ export async function addScrapeJobs(await Promise.all(addToBull.map(async (job) => {const size = JSON.stringify(job.data).length;- return await Sentry.startSpan(- {- name: "Add scrape job",- op: "queue.publish",- attributes: {- "messaging.message.id": job.opts.jobId,- "messaging.destination.name": getScrapeQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await _addScrapeJobToBullMQ(- {- ...job.data,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- },- job.opts,- job.opts.jobId,- job.opts.priority,- );- },+ await _addScrapeJobToBullMQ(+ job.data,+ job.opts,+ job.opts.jobId,+ job.opts.priority,);}),);}}-export function waitForJob(- jobId: string,+export async function waitForJob(+ _job: Job | string,timeout: number,logger: Logger = _logger,): Promise<Document> {- return new Promise(async (resolve, reject) => {const start = Date.now();- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);- while (true) {- logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });- if (Date.now() >= start + timeout) {- reject(new Error("Job wait "));- break;- } else {- const state = await queue.getJobState(jobId);- logger.debug("Job in state", { state, scrapeId: jobId, jobId });- if (state === "completed") {- let doc: Document;- const job = (await queue.getJob(jobId))!;- logger.debug("Got job");- doc = job.returnvalue;-- if (!doc) {- const docs = await getJobFromGCS(jobId);- logger.debug("Got job from GCS");- if (!docs || docs.length === 0) {- throw new Error("Job not found in GCS");- }- doc = docs[0];-- if (job.data?.internalOptions?.zeroDataRetention) {- await removeJobFromGCS(jobId);- }- }-- resolve(doc);- break;- } else if (state === "failed") {- const job = await queue.getJob(jobId);- if (job && job.failedReason !== "Concurrency limit hit") {- reject(job.failedReason);- break;- }- }+ const queue = getScrapeQueue();+ let job: Job | undefined = typeof _job == "string" ? await queue.getJob(_job) : _job;+ while (job === undefined) {+ logger.debug("Waiting for job to be created");+ await new Promise(resolve => setTimeout(resolve, 500));+ job = await queue.getJob(_job as string);+ if (Date.now() - start > timeout) {+ throw new Error("Job wait ");}+ }+ let doc: Document = await Promise.race([+ job.waitUntilFinished(getScrapeQueueEvents(), timeout - (Date.now() - start)),+ new Promise((resolve, reject) => {+ setTimeout(() => {+ reject(new Error("Job wait "));+ }, Math.max(0, timeout - (Date.now() - start)));+ }),+ ]);+ logger.debug("Got job");++ if (!doc) {+ const docs = await getJobFromGCS(job.id!);+ logger.debug("Got job from GCS");+ if (!docs || docs.length === 0) {+ throw new Error("Job not found in GCS");+ }+ doc = docs[0];- await new Promise((resolve) => setTimeout(resolve, 750));+ if (job.data?.internalOptions?.zeroDataRetention) {+ await removeJobFromGCS(job.id!);+ }}- conn.disconnect();- return;- });+ return doc;}diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 32ae88c49..4ba840a91 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -1,10 +1,11 @@-import { Queue } from "bullmq";+import { Queue, QueueEvents } from "bullmq";import { logger } from "../lib/logger";import IORedis from "ioredis";export type QueueFunction = () => Queue<any, any, string, any, any, string>;let scrapeQueue: Queue;+let scrapeQueueEvents: QueueEvents;let extractQueue: Queue;let loggingQueue: Queue;let indexQueue: Queue;@@ -13,10 +14,15 @@ let generateLlmsTxtQueue: Queue;let billingQueue: Queue;let precrawlQueue: Queue;-export function createRedisConnection(): IORedis {- return new IORedis(process.env.REDIS_URL!, {+export function createRedisConnection() {+ const connection = new IORedis(process.env.REDIS_URL!, {maxRetriesPerRequest: null,});++ connection.on("reconnecting", () => logger.warn("Redis reconnecting"));+ connection.on("error", (err) => logger.warn("Redis error", { err }));++ return connection;}export const redisConnection = createRedisConnection();@@ -30,27 +36,31 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getScrapeQueue(_redisConnection?: IORedis) {- const _scrapeQueue = !scrapeQueue || _redisConnection !== undefined ? new Queue(scrapeQueueName, {- connection: _redisConnection ?? redisConnection,- defaultJobOptions: {- removeOnComplete: {- age: 3600, // 1 hour- },- removeOnFail: {- age: 3600, // 1 hour+export function getScrapeQueue() {+ if (!scrapeQueue) {+ scrapeQueue = new Queue(scrapeQueueName, {+ connection: createRedisConnection(),+ defaultJobOptions: {+ removeOnComplete: {+ age: 3600, // 1 hour+ },+ removeOnFail: {+ age: 3600, // 1 hour+ },},- },- }) : undefined;+ });+ }+ return scrapeQueue;+}- if (_redisConnection !== undefined) {- return _scrapeQueue!;- } else if (!scrapeQueue) {- scrapeQueue = _scrapeQueue!;- logger.info("Web scraper queue created");+export function getScrapeQueueEvents() {+ if (!scrapeQueueEvents) {+ scrapeQueueEvents = new QueueEvents(scrapeQueueName, {+ connection: createRedisConnection(),+ });}- return scrapeQueue;+ return scrapeQueueEvents;}export function getExtractQueue() {@@ -66,7 +76,6 @@ export function getExtractQueue() {},},});- logger.info("Extraction queue created");}return extractQueue;}@@ -84,7 +93,6 @@ export function getGenerateLlmsTxtQueue() {},},});- logger.info("LLMs TXT generation queue created");}return generateLlmsTxtQueue;}@@ -102,7 +110,6 @@ export function getDeepResearchQueue() {},},});- logger.info("Deep research queue created");}return deepResearchQueue;}@@ -120,7 +127,6 @@ export function getBillingQueue() {},},});- logger.info("Billing queue created");}return billingQueue;}@@ -138,7 +144,6 @@ export function getPrecrawlQueue() {},},});- logger.info("Precrawl queue created");}return precrawlQueue;}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 5e21d0a8f..e3fcdc9be 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1,100 +1,55 @@import "dotenv/config";import "./sentry";import * as Sentry from "@sentry/node";-import { CustomError } from "../lib/custom-error";import {getScrapeQueue,getExtractQueue,getDeepResearchQueue,redisConnection,getGenerateLlmsTxtQueue,- getBillingQueue,scrapeQueueName,createRedisConnection,} from "./queue-service";-import { startWebScraperPipeline } from "../main/runWebScraper";-import { callWebhook } from "./webhook";-import { logJob } from "./logging/log_job";import { Job, Queue, QueueEvents } from "bullmq";import { logger as _logger } from "../lib/logger";import { Worker } from "bullmq";import systemMonitor from "./system-monitor";import { v4 as uuidv4 } from "uuid";import {- addCrawlJob,addCrawlJobDone,- addCrawlJobs,- crawlToCrawler,- finishCrawl,- finishCrawlPre,- unPreFinishCrawl,finishCrawlKickoff,- generateURLPermutations,getCrawl,- getCrawlJobs,- getDoneJobsOrderedLength,- lockURL,- lockURLsIndividually,normalizeURL,- saveCrawl,} from "../lib/crawl-redis";import { StoredCrawl } from "../lib/crawl-redis";-import { addScrapeJob, addScrapeJobs } from "./queue-jobs";-import {- addJobPriority,- deleteJobPriority,- getJobPriority,-} from "../../src/lib/job-priority";-import { getJobs } from "..//controllers/v1/crawl-status";import { configDotenv } from "dotenv";import { scrapeOptions } from "../controllers/v2/types";import {concurrentJobDone,- pushConcurrencyLimitActiveJob,} from "../lib/concurrency-limit";-import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";-import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";-import { Document, TeamFlags } from "../controllers/v1/types";import {ExtractResult,performExtraction,} from "../lib/extract/extraction-service";-import { supabase_service } from "../services/supabase";-import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";-import { saveExtract, updateExtract } from "../lib/extract/extract-redis";-import { billTeam } from "./billing/credit_billing";+import { updateExtract } from "../lib/extract/extract-redis";import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";import { performDeepResearch } from "../lib/deep-research/deep-research-service";import { performGenerateLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-service";import { updateGeneratedLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-redis";import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";-import { CostTracking } from "../lib/extract/extraction-service";-import { getACUCTeam } from "../controllers/auth";import Express from "express";import http from "http";import https from "https";import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";-import { RateLimiterMode } from "../types";-import { calculateCreditsToBeBilled } from "../lib/scrape-billing";import { redisEvictConnection } from "./redis";-import { generateURLSplits, queryIndexAtSplitLevel } from "./index";-import { WebCrawler } from "../scraper/WebScraper/crawler";-import type { Logger } from "winston";+import path from "path";+import { finishCrawlIfNeeded } from "./worker/crawl-logic";configDotenv();-class RacedRedirectError extends Error {- constructor() {- super("Raced redirect error");- }-}-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));-const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;-const workerStalledCheckInterval =- Number(process.env.WORKER_STALLED_CHECK_INTERVAL) || 30000;const jobLockExtendInterval =Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;const jobLockExtensionTime =@@ -112,349 +67,6 @@ const runningJobs: Set<string> = new Set();cacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);-async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {- const logger = _logger.child({- module: "queue-worker",- method: "finishCrawlIfNeeded",- jobId: job.id,- scrapeId: job.id,- crawlId: job.data.crawl_id,- zeroDataRetention: sc.internalOptions.zeroDataRetention,- });-- if (await finishCrawlPre(job.data.crawl_id, logger)) {- logger.info("Crawl is pre-finished, checking if we need to add more jobs");- if (- job.data.crawlerOptions &&- !(await redisEvictConnection.exists(- "crawl:" + job.data.crawl_id + ":invisible_urls",- ))- ) {- await redisEvictConnection.set(- "crawl:" + job.data.crawl_id + ":invisible_urls",- "done",- "EX",- 60 * 60 * 24,- );-- const sc = (await getCrawl(job.data.crawl_id))!;-- const visitedUrls = new Set(- await redisEvictConnection.smembers(- "crawl:" + job.data.crawl_id + ":visited_unique",- ),- );-- logger.info("Visited URLs", {- visitedUrls: visitedUrls.size,- });-- let lastUrls: string[] = [];- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {- lastUrls = (- (- await supabase_service.rpc("diff_get_last_crawl_urls", {- i_team_id: job.data.team_id,- i_url: sc.originUrl!,- })- ).data ?? []- ).map((x) => x.url);- }-- const lastUrlsSet = new Set(lastUrls);-- logger.info("Last URLs", {- lastUrls: lastUrlsSet.size,- });-- const crawler = crawlToCrawler(- job.data.crawl_id,- sc,- (await getACUCTeam(job.data.team_id))?.flags ?? null,- sc.originUrl!,- job.data.crawlerOptions,- );-- const univistedUrls = await crawler.filterLinks(- Array.from(lastUrlsSet).filter((x) => !visitedUrls.has(x)),- Infinity,- sc.crawlerOptions.maxDepth ?? 10,- );-- const addableJobCount =- sc.crawlerOptions.limit === undefined- ? Infinity- : sc.crawlerOptions.limit -- (await getDoneJobsOrderedLength(job.data.crawl_id));-- if (univistedUrls.links.length !== 0 && addableJobCount > 0) {- logger.info("Adding jobs", {- univistedUrls: univistedUrls.links.length,- addableJobCount,- });-- const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {- const uuid = uuidv4();- return {- name: uuid,- data: {- url,- mode: "single_urls" as const,- team_id: job.data.team_id,- crawlerOptions: {- ...job.data.crawlerOptions,- urlInvisibleInCurrentCrawl: true,- },- scrapeOptions: job.data.scrapeOptions,- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- sitemapped: true,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- },- opts: {- jobId: uuid,- priority: 20,- },- };- });-- const lockedIds = await lockURLsIndividually(- job.data.crawl_id,- sc,- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),- );- const lockedJobs = jobs.filter((x) =>- lockedIds.find((y) => y.id === x.opts.jobId),- );- await addCrawlJobs(- job.data.crawl_id,- lockedJobs.map((x) => x.opts.jobId),- logger,- );- await addScrapeJobs(lockedJobs);-- if (lockedJobs.length > 0) {- logger.info("Added jobs, not going for the full finish", {- lockedJobs: lockedJobs.length,- });-- await unPreFinishCrawl(job.data.crawl_id);- return;- } else {- logger.info("No jobs added (all discovered URLs were locked), finishing crawl");- }- }- }-- logger.info("Finishing crawl");- await finishCrawl(job.data.crawl_id, logger);-- if (!job.data.v1) {- const jobIDs = await getCrawlJobs(job.data.crawl_id);-- const conn = createRedisConnection();- const jobs = (await getJobs(jobIDs, conn)).sort(- (a, b) => a.timestamp - b.timestamp,- );- conn.disconnect();- // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));- const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")- ? "failed"- : "completed";-- const fullDocs = jobs- .map((x) =>- x.returnvalue- ? Array.isArray(x.returnvalue)- ? x.returnvalue[0]- : x.returnvalue- : null,- )- .filter((x) => x !== null);-- await logJob({- job_id: job.data.crawl_id,- success: jobStatus === "completed",- message: sc.cancelled ? "Cancelled" : undefined,- num_docs: fullDocs.length,- docs: [],- time_taken: (Date.now() - sc.createdAt) / 1000,- team_id: job.data.team_id,- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",- url: sc.originUrl!,- scrapeOptions: sc.scrapeOptions,- crawlerOptions: sc.crawlerOptions,- origin: job.data.origin,- integration: job.data.integration,- zeroDataRetention: job.data.zeroDataRetention,- }, false, job.data.internalOptions?.bypassBilling ?? false);-- const data = {- success: jobStatus !== "failed",- result: {- links: fullDocs.map((doc) => {- return {- content: doc,- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",- };- }),- },- project_id: job.data.project_id,- docs: fullDocs,- };-- // v0 web hooks, call when done with all the data- if (!job.data.v1) {- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id,- data,- webhook: job.data.webhook,- v1: job.data.v1,- eventType: job.data.crawlerOptions !== null- ? "crawl.completed"- : "batch_scrape.completed",- });- }- } else {- const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);- const jobStatus = sc.cancelled ? "failed" : "completed";-- let credits_billed = null;-- if (process.env.USE_DB_AUTHENTICATION === "true") {- const creditsRpc = await supabase_service- .rpc("credits_billed_by_crawl_id_1", {- i_crawl_id: job.data.crawl_id,- });-- credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;-- if (credits_billed === null) {- logger.warn("Credits billed is null", {- error: creditsRpc.error,- });- }- }-- await logJob(- {- job_id: job.data.crawl_id,- success: jobStatus === "completed",- message: sc.cancelled ? "Cancelled" : undefined,- num_docs,- docs: [],- time_taken: (Date.now() - sc.createdAt) / 1000,- team_id: job.data.team_id,- scrapeOptions: sc.scrapeOptions,- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",- url:- sc?.originUrl ??- (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),- crawlerOptions: sc.crawlerOptions,- origin: job.data.origin,- integration: job.data.integration,- credits_billed,- zeroDataRetention: job.data.zeroDataRetention,- },- true,- job.data.internalOptions?.bypassBilling ?? false,- );--- // v1 web hooks, call when done with no data, but with event completed- if (job.data.v1 && job.data.webhook) {- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id,- data: [],- webhook: job.data.webhook,- v1: job.data.v1,- eventType: job.data.crawlerOptions !== null- ? "crawl.completed"- : "batch_scrape.completed",- });- }- }- }-}--const processJobInternal = async (token: string, job: Job & { id: string }) => {- const logger = _logger.child({- module: "queue-worker",- method: "processJobInternal",- jobId: job.id,- scrapeId: job.id,- crawlId: job.data?.crawl_id ?? undefined,- zeroDataRetention: job.data?.zeroDataRetention ?? false,- });-- const extendLockInterval = setInterval(async () => {- logger.info(`🐂 Worker extending lock on job ${job.id}`, {- extendInterval: jobLockExtendInterval,- extensionTime: jobLockExtensionTime,- });-- if (job.data?.mode !== "kickoff" && job.data?.team_id) {- await pushConcurrencyLimitActiveJob(job.data.team_id, job.id, 60 * 1000); // 60s lock renew, just like in the queue- }-- await job.extendLock(token, jobLockExtensionTime);- }, jobLockExtendInterval);-- await addJobPriority(job.data.team_id, job.id);- let err = null;- try {- if (job.data?.mode === "kickoff") {- const result = await processKickoffJob(job, token);- if (result.success) {- try {- await job.moveToCompleted(null, token, false);- } catch (e) {}- } else {- logger.debug("Job failed", { result, mode: job.data.mode });- await job.moveToFailed((result as any).error, token, false);- }- } else {- const result = await processJob(job, token);- if (result.success) {- try {- if (- process.env.USE_DB_AUTHENTICATION === "true" &&- (job.data.crawl_id || process.env.GCS_BUCKET_NAME)- ) {- logger.debug(- "Job succeeded -- putting null in Redis",- );- await job.moveToCompleted(null, token, false);- } else {- logger.debug("Job succeeded -- putting result in Redis");- await job.moveToCompleted(result.document, token, false);- }- } catch (e) {}- } else {- logger.debug("Job failed", { result });- await job.moveToFailed((result as any).error, token, false);- }- }- } catch (error) {- logger.debug("Job failed", { error });- Sentry.captureException(error);- err = error;- await job.moveToFailed(error, token, false);- } finally {- await deleteJobPriority(job.data.team_id, job.id);- clearInterval(extendLockInterval);- }-- return err;-};-const processExtractJobInternal = async (token: string,job: Job & { id: string },@@ -717,6 +329,22 @@ process.on("SIGTERM", () => {let cantAcceptConnectionCount = 0;+const separateWorkerFun = (+ queue: Queue,+ path: string,+): Worker => {+ const worker = new Worker(queue.name, path, {+ connection: createRedisConnection(),+ lockDuration: 30 * 1000, // 30 seconds+ stalledInterval: 30 * 1000, // 30 seconds+ maxStalledCount: 10, // 10 times+ concurrency: 6, // from k8s setup+ useWorkerThreads: true,+ });++ return worker;+}+const workerFun = async (queue: Queue,processJobInternal: (token: string, job: Job) => Promise<any>,@@ -782,68 +410,7 @@ const workerFun = async (}}- if (job.data && job.data.sentry && Sentry.isInitialized()) {- Sentry.continueTrace(- {- sentryTrace: job.data.sentry.trace,- baggage: job.data.sentry.baggage,- },- () => {- Sentry.startSpan(- {- name: "Scrape job",- attributes: {- job: job.id,- worker: process.env.FLY_MACHINE_ID ?? worker.id,- },- },- async (span) => {- await Sentry.startSpan(- {- name: "Process scrape job",- op: "queue.process",- attributes: {- "messaging.message.id": job.id,- "messaging.destination.name": scrapeQueueName,- "messaging.message.body.size": job.data.sentry.size,- "messaging.message.receive.latency":- Date.now() - (job.processedOn ?? job.timestamp),- "messaging.message.retry.count": job.attemptsMade,- },- },- async () => {- let res;- try {- res = await processJobInternal(token, job);- } finally {- await afterJobDone(job);- }-- if (res !== null) {- span.setStatus({ code: 2 }); // ERROR- } else {- span.setStatus({ code: 1 }); // OK- }- },- );- },- );- },- );- } else {- Sentry.startSpan(- {- name: "Scrape job",- attributes: {- job: job.id,- worker: process.env.FLY_MACHINE_ID ?? worker.id,- },- },- () => {- processJobInternal(token, job).finally(() => afterJobDone(job));- },- );- }+ processJobInternal(token, job).finally(() => afterJobDone(job));await sleep(gotJobInterval);} else {@@ -852,740 +419,6 @@ const workerFun = async (}};-async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: string) {- if (sc.crawlerOptions.ignoreSitemap) {- return [];- }-- const trimmedURL = new URL(url);- trimmedURL.search = "";-- const index = await queryIndexAtSplitLevel(- sc.crawlerOptions.allowBackwardCrawling ? generateURLSplits(trimmedURL.href)[0] : trimmedURL.href,- sc.crawlerOptions.limit ?? 10000,- );-- const validIndexLinksResult = await crawler.filterLinks(- index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),- sc.crawlerOptions.limit ?? 10000,- sc.crawlerOptions.maxDepth ?? 10,- false,- );- const validIndexLinks = validIndexLinksResult.links;-- return validIndexLinks;-}--async function processKickoffJob(job: Job & { id: string }, token: string) {- const logger = _logger.child({- module: "queue-worker",- method: "processKickoffJob",- jobId: job.id,- scrapeId: job.id,- crawlId: job.data?.crawl_id ?? undefined,- teamId: job.data?.team_id ?? undefined,- zeroDataRetention: job.data.zeroDataRetention ?? false,- });-- try {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);-- logger.debug("Locking URL...");- await lockURL(job.data.crawl_id, sc, job.data.url);- const jobId = uuidv4();- logger.debug("Adding scrape job to Redis...", { jobId });- await addScrapeJob(- {- url: job.data.url,- mode: "single_urls",- team_id: job.data.team_id,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- webhook: job.data.webhook,- v1: job.data.v1,- isCrawlSourceScrape: true,- zeroDataRetention: job.data.zeroDataRetention,- },- {- priority: 15,- },- jobId,- );- logger.debug("Adding scrape job to BullMQ...", { jobId });- await addCrawlJob(job.data.crawl_id, jobId, logger);-- if (job.data.webhook) {- logger.debug("Calling webhook with crawl.started...", {- webhook: job.data.webhook,- });- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id,- data: null,- webhook: job.data.webhook,- v1: job.data.v1,- eventType: "crawl.started",- });- }-- const sitemap = sc.crawlerOptions.ignoreSitemap- ? 0- : await crawler.tryGetSitemap(async (urls) => {- if (urls.length === 0) return;-- logger.debug("Using sitemap chunk of length " + urls.length, {- sitemapLength: urls.length,- });-- let jobPriority = await getJobPriority({- team_id: job.data.team_id,- basePriority: 21,- });- logger.debug("Using job priority " + jobPriority, { jobPriority });-- const jobs = urls.map((url) => {- const uuid = uuidv4();- return {- name: uuid,- data: {- url,- mode: "single_urls" as const,- team_id: job.data.team_id,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- sitemapped: true,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- },- opts: {- jobId: uuid,- priority: 20,- },- };- });-- logger.debug("Locking URLs...");- const lockedIds = await lockURLsIndividually(- job.data.crawl_id,- sc,- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),- );- const lockedJobs = jobs.filter((x) =>- lockedIds.find((y) => y.id === x.opts.jobId),- );- logger.debug("Adding scrape jobs to Redis...");- await addCrawlJobs(- job.data.crawl_id,- lockedJobs.map((x) => x.opts.jobId),- logger,- );- logger.debug("Adding scrape jobs to BullMQ...");- await addScrapeJobs(lockedJobs);- });-- if (sitemap === 0) {- logger.debug("Sitemap not found or ignored.", {- ignoreSitemap: sc.crawlerOptions.ignoreSitemap,- });- }-- const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);-- if (indexLinks.length > 0) {- logger.debug("Using index links of length " + indexLinks.length, {- indexLinksLength: indexLinks.length,- });-- let jobPriority = await getJobPriority({- team_id: job.data.team_id,- basePriority: 21,- });- logger.debug("Using job priority " + jobPriority, { jobPriority });-- const jobs = indexLinks.map((url) => {- const uuid = uuidv4();- return {- name: uuid,- data: {- url,- mode: "single_urls" as const,- team_id: job.data.team_id,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- sitemapped: true,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- },- opts: {- jobId: uuid,- priority: 20,- },- };- });-- logger.debug("Locking URLs...");- const lockedIds = await lockURLsIndividually(- job.data.crawl_id,- sc,- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),- );- const lockedJobs = jobs.filter((x) =>- lockedIds.find((y) => y.id === x.opts.jobId),- );- logger.debug("Adding scrape jobs to Redis...");- await addCrawlJobs(- job.data.crawl_id,- lockedJobs.map((x) => x.opts.jobId),- logger,- );- logger.debug("Adding scrape jobs to BullMQ...");- await addScrapeJobs(lockedJobs);- }-- logger.debug("Done queueing jobs!");-- await finishCrawlKickoff(job.data.crawl_id);- await finishCrawlIfNeeded(job, sc);-- return { success: true };- } catch (error) {- logger.error("An error occurred!", { error });- await finishCrawlKickoff(job.data.crawl_id);- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- if (sc) {- await finishCrawlIfNeeded(job, sc);- }- return { success: false, error };- }-}--async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) {- let creditsToBeBilled: number | null = null;-- if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {- creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags);-- if (- job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&- process.env.USE_DB_AUTHENTICATION === "true"- ) {- try {- const billingJobId = uuidv4();- logger.debug(- `Adding billing job to queue for team ${job.data.team_id}`,- {- billingJobId,- credits: creditsToBeBilled,- is_extract: false,- },- );-- // Add directly to the billing queue - the billing worker will handle the rest- await getBillingQueue().add(- "bill_team",- {- team_id: job.data.team_id,- subscription_id: undefined,- credits: creditsToBeBilled,- is_extract: false,- timestamp: new Date().toISOString(),- originating_job_id: job.id,- },- {- jobId: billingJobId,- priority: 10,- },- );- return creditsToBeBilled;- } catch (error) {- logger.error(- `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,- { error },- );- Sentry.captureException(error);- return creditsToBeBilled;- }- }- }-- return creditsToBeBilled;-}--async function processJob(job: Job & { id: string }, token: string) {- const logger = _logger.child({- module: "queue-worker",- method: "processJob",- jobId: job.id,- scrapeId: job.id,- crawlId: job.data?.crawl_id ?? undefined,- teamId: job.data?.team_id ?? undefined,- zeroDataRetention: job.data?.zeroDataRetention ?? false,- });- logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });- const start = job.data.startTime ?? Date.now();- const remainingTime = job.data.scrapeOptions.timeout ? (job.data.scrapeOptions.timeout - (Date.now() - start)) : undefined;-- const costTracking = new CostTracking();-- try {- job.updateProgress({- current: 1,- total: 100,- current_step: "SCRAPING",- current_url: "",- });-- if (remainingTime !== undefined && remainingTime < 0) {- throw new Error("timeout");- }- const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;-- if (job.data.crawl_id) {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- if (sc && sc.cancelled) {- throw new Error("Parent crawl/batch scrape was cancelled");- }- }--- const pipeline = await Promise.race([- startWebScraperPipeline({- job,- token,- costTracking,- }),- ...(remainingTime !== undefined- ? [- (async () => {- await sleep(remainingTime);- throw new Error("timeout");- })(),- ]- : []),- ]);-- try {- signal?.throwIfAborted();- } catch (e) {- throw new Error("timeout");- }-- if (!pipeline.success) {- throw pipeline.error;- }-- const end = Date.now();- const timeTakenInSeconds = (end - start) / 1000;-- const doc = pipeline.document;-- const rawHtml = doc.rawHtml ?? "";-- if (!job.data.scrapeOptions.formats.includes("rawHtml")) {- delete doc.rawHtml;- }-- if (job.data.concurrencyLimited) {- doc.warning =- "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." +- (doc.warning ? " " + doc.warning : "");- }-- const data = {- success: true,- result: {- links: [- {- content: doc,- source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",- id: job.id,- },- ],- },- project_id: job.data.project_id,- document: doc,- };-- if (job.data.crawl_id) {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;-- if (- doc.metadata.url !== undefined &&- doc.metadata.sourceURL !== undefined &&- normalizeURL(doc.metadata.url, sc) !==- normalizeURL(doc.metadata.sourceURL, sc) &&- job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape- ) {- const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);- const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);- if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {- const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";- throw new Error(reason);- }-- // Only re-set originUrl if it's different from the current hostname- // This is only done on this condition to handle cross-domain redirects- // If this would be done for non-crossdomain redirects, but also for e.g.- // redirecting / -> /introduction (like our docs site does), it would- // break crawling the entire site without allowBackwardsCrawling - mogery- const isHostnameDifferent =- normalizeUrlOnlyHostname(doc.metadata.url) !==- normalizeUrlOnlyHostname(doc.metadata.sourceURL);- if (job.data.isCrawlSourceScrape && isHostnameDifferent) {- // TODO: re-fetch sitemap for redirect target domain- sc.originUrl = doc.metadata.url;- await saveCrawl(job.data.crawl_id, sc);- }-- if (isUrlBlocked(doc.metadata.url, (await getACUCTeam(job.data.team_id))?.flags ?? null)) {- throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking- }-- const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));- const p2 = generateURLPermutations(- normalizeURL(doc.metadata.sourceURL, sc),- );-- if (JSON.stringify(p1) !== JSON.stringify(p2)) {- logger.debug(- "Was redirected, removing old URL and locking new URL...",- { oldUrl: doc.metadata.sourceURL, newUrl: doc.metadata.url },- );-- // Prevent redirect target from being visited in the crawl again- // See lockURL- const x = await redisEvictConnection.sadd(- "crawl:" + job.data.crawl_id + ":visited",- ...p1.map((x) => x.href),- );- const lockRes = x === p1.length;-- if (job.data.crawlerOptions !== null && !lockRes) {- throw new RacedRedirectError();- }- }- }-- if (job.data.crawlerOptions !== null) {- if (!sc.cancelled) {- const crawler = crawlToCrawler(- job.data.crawl_id,- sc,- (await getACUCTeam(job.data.team_id))?.flags ?? null,- doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,- job.data.crawlerOptions,- );-- const links = await crawler.filterLinks(- await crawler.extractLinksFromHTML(- rawHtml ?? "",- doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,- ),- Infinity,- sc.crawlerOptions?.maxDepth ?? 10,- );- logger.debug("Discovered " + links.links.length + " links...", {- linksLength: links.links.length,- });-- for (const link of links.links) {- if (await lockURL(job.data.crawl_id, sc, link)) {- // This seems to work really welel- const jobPriority = await getJobPriority({- team_id: sc.team_id,- basePriority: job.data.crawl_id ? 20 : 10,- });- const jobId = uuidv4();-- logger.debug(- "Determined job priority " +- jobPriority +- " for URL " +- JSON.stringify(link),- { jobPriority, url: link },- );-- // console.log("team_id: ", sc.team_id)- // console.log("base priority: ", job.data.crawl_id ? 20 : 10)- // console.log("job priority: " , jobPriority, "\n\n\n")-- await addScrapeJob(- {- url: link,- mode: "single_urls",- team_id: sc.team_id,- scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),- internalOptions: sc.internalOptions,- crawlerOptions: {- ...sc.crawlerOptions,- currentDiscoveryDepth:- (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,- },- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- },- {},- jobId,- jobPriority,- );-- await addCrawlJob(job.data.crawl_id, jobId, logger);- logger.debug("Added job for URL " + JSON.stringify(link), {- jobPriority,- url: link,- newJobId: jobId,- });- } else {- // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!- // logger.debug("Could not lock URL " + JSON.stringify(link), {- // url: link,- // });- }- }-- // Only run check after adding new jobs for discovery - mogery- if (job.data.isCrawlSourceScrape) {- const filterResult = await crawler.filterLinks(- [doc.metadata.url ?? doc.metadata.sourceURL!],- 1,- sc.crawlerOptions?.maxDepth ?? 10,- );- if (filterResult.links.length === 0) {- const url = doc.metadata.url ?? doc.metadata.sourceURL!;- const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";- throw new Error(reason);- }- }- }- }-- try {- signal?.throwIfAborted();- } catch (e) {- throw new Error("timeout");- }-- const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);-- doc.metadata.creditsUsed = credits_billed ?? undefined;-- logger.debug("Logging job to DB...");- await logJob(- {- job_id: job.id as string,- success: true,- num_docs: 1,- docs: [doc],- time_taken: timeTakenInSeconds,- team_id: job.data.team_id,- mode: job.data.mode,- url: job.data.url,- crawlerOptions: sc.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- cost_tracking: costTracking,- pdf_num_pages: doc.metadata.numPages,- credits_billed,- change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,- zeroDataRetention: job.data.zeroDataRetention,- },- true,- job.data.internalOptions?.bypassBilling ?? false,- );-- if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {- logger.debug("Calling webhook with success...", {- webhook: job.data.webhook,- });- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id,- scrapeId: job.id,- data,- webhook: job.data.webhook,- v1: job.data.v1,- eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",- });- }-- logger.debug("Declaring job as done...");- await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);-- await finishCrawlIfNeeded(job, sc);- } else {- try {- signal?.throwIfAborted();- } catch (e) {- throw new Error("timeout");- }-- const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);-- doc.metadata.creditsUsed = credits_billed ?? undefined;-- await logJob({- job_id: job.id,- success: true,- message: "Scrape completed",- num_docs: 1,- docs: [doc],- time_taken: timeTakenInSeconds,- team_id: job.data.team_id,- mode: "scrape",- url: job.data.url,- scrapeOptions: job.data.scrapeOptions,- origin: job.data.origin,- integration: job.data.integration,- num_tokens: 0, // TODO: fix- cost_tracking: costTracking,- pdf_num_pages: doc.metadata.numPages,- credits_billed,- change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,- zeroDataRetention: job.data.zeroDataRetention,- }, false, job.data.internalOptions?.bypassBilling ?? false);- }-- logger.info(`🐂 Job done ${job.id}`);- return data;- } catch (error) {- if (job.data.crawl_id) {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;-- logger.debug("Declaring job as done...");- await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);- await redisEvictConnection.srem(- "crawl:" + job.data.crawl_id + ":visited_unique",- normalizeURL(job.data.url, sc),- );-- await finishCrawlIfNeeded(job, sc);- }-- const isEarlyTimeout =- error instanceof Error && error.message === "timeout";- const isCancelled =- error instanceof Error &&- error.message === "Parent crawl/batch scrape was cancelled";-- if (isEarlyTimeout) {- logger.error(`🐂 Job timed out ${job.id}`);- } else if (error instanceof RacedRedirectError) {- logger.warn(`🐂 Job got redirect raced ${job.id}, silently failing`);- } else if (isCancelled) {- logger.warn(`🐂 Job got cancelled, silently failing`);- } else {- logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });-- Sentry.captureException(error, {- data: {- job: job.id,- },- });-- if (error instanceof CustomError) {- // Here we handle the error, then save the failed job- logger.error(error.message); // or any other error handling- }- logger.error(error);- if (error.stack) {- logger.error(error.stack);- }- }-- const data = {- success: false,- document: null,- project_id: job.data.project_id,- error:- error instanceof Error- ? error- : typeof error === "string"- ? new Error(error)- : new Error(JSON.stringify(error)),- };-- if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {- callWebhook({- teamId: job.data.team_id,- crawlId: job.data.crawl_id ?? (job.id as string),- scrapeId: job.id,- data,- webhook: job.data.webhook,- v1: job.data.v1,- eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",- });- }-- const end = Date.now();- const timeTakenInSeconds = (end - start) / 1000;-- const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);-- logger.debug("Logging job to DB...");- await logJob(- {- job_id: job.id as string,- success: false,- message:- typeof error === "string"- ? error- : (error.message ??- "Something went wrong... Contact [email protected]"),- num_docs: 0,- docs: [],- time_taken: timeTakenInSeconds,- team_id: job.data.team_id,- mode: job.data.mode,- url: job.data.url,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- cost_tracking: costTracking,- credits_billed,- zeroDataRetention: job.data.zeroDataRetention,- },- true,- job.data.internalOptions?.bypassBilling ?? false,- );- return data;- }-}--// wsq.process(-// Math.floor(Number(process.env.NUM_WORKERS_PER_QUEUE ?? 8)),-// processJob-// );--// wsq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));-// wsq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));-// wsq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));-// wsq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));-// wsq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));-// wsq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));-// Start all workersconst app = Express();@@ -1639,9 +472,7 @@ app.listen(workerPort, () => {return;}- const conn = createRedisConnection();- const job = await getScrapeQueue(conn).getJob(args.jobId);- conn.disconnect();+ const job = await getScrapeQueue().getJob(args.jobId);let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener", zeroDataRetention: job?.data.zeroDataRetention });if (job && job.data.crawl_id) {@@ -1676,20 +507,30 @@ app.listen(workerPort, () => {const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });scrapeQueueEvents.on("failed", failedListener);- const conn = createRedisConnection();- await Promise.all([- workerFun(getScrapeQueue(conn), processJobInternal),+ const results = await Promise.all([+ separateWorkerFun(getScrapeQueue(), path.join(__dirname, "worker", "scrape-worker.js")),workerFun(getExtractQueue(), processExtractJobInternal),workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),]);- conn.disconnect();+console.log("All workers exited. Waiting for all jobs to finish...");+ const workerResults = results.filter(x => x instanceof Worker);+ await Promise.all(workerResults.map(x => x.close()));+while (runningJobs.size > 0) {await new Promise((resolve) => setTimeout(resolve, 500));}+ setInterval(async () => {+ _logger.debug("Currently running jobs", {+ jobs: (await Promise.all([...runningJobs].map(async (jobId) => {+ return await getScrapeQueue().getJob(jobId);+ }))).filter(x => x && !x.data?.zeroDataRetention),+ });+ }, 1000);+await scrapeQueueEvents.close();console.log("All jobs finished. Worker out!");process.exit(0);diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsnew file mode 100644index 000000000..9a3d568b3--- /dev/null+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -0,0 +1,284 @@+import { logger as _logger } from "../../lib/logger";+import { Job } from "bullmq";+import { addCrawlJobs, finishCrawl, getCrawlJobs, getDoneJobsOrderedLength, lockURLsIndividually, StoredCrawl, unPreFinishCrawl } from "../../lib/crawl-redis";+import { redisEvictConnection } from "../redis";+import { getCrawl } from "../../lib/crawl-redis";+import { finishCrawlPre } from "../../lib/crawl-redis";+import { getACUCTeam } from "../../controllers/auth";+import { crawlToCrawler } from "../../lib/crawl-redis";+import { supabase_service } from "../supabase";+import { v4 as uuidv4 } from "uuid";+import { addScrapeJobs } from "../queue-jobs";+import { getJobs } from "../../controllers/v1/crawl-status";+import { logJob } from "../logging/log_job";+import { callWebhook } from "../webhook";++export async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "finishCrawlIfNeeded",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data.crawl_id,+ zeroDataRetention: sc.internalOptions.zeroDataRetention,+ });++ if (await finishCrawlPre(job.data.crawl_id, logger)) {+ logger.info("Crawl is pre-finished, checking if we need to add more jobs");+ if (+ job.data.crawlerOptions &&+ !(await redisEvictConnection.exists(+ "crawl:" + job.data.crawl_id + ":invisible_urls",+ ))+ ) {+ await redisEvictConnection.set(+ "crawl:" + job.data.crawl_id + ":invisible_urls",+ "done",+ "EX",+ 60 * 60 * 24,+ );++ const sc = (await getCrawl(job.data.crawl_id))!;++ const visitedUrls = new Set(+ await redisEvictConnection.smembers(+ "crawl:" + job.data.crawl_id + ":visited_unique",+ ),+ );++ logger.info("Visited URLs", {+ visitedUrls: visitedUrls.size,+ });++ let lastUrls: string[] = [];+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {+ lastUrls = (+ (+ await supabase_service.rpc("diff_get_last_crawl_urls", {+ i_team_id: job.data.team_id,+ i_url: sc.originUrl!,+ })+ ).data ?? []+ ).map((x) => x.url);+ }++ const lastUrlsSet = new Set(lastUrls);++ logger.info("Last URLs", {+ lastUrls: lastUrlsSet.size,+ });++ const crawler = crawlToCrawler(+ job.data.crawl_id,+ sc,+ (await getACUCTeam(job.data.team_id))?.flags ?? null,+ sc.originUrl!,+ job.data.crawlerOptions,+ );++ const univistedUrls = await crawler.filterLinks(+ Array.from(lastUrlsSet).filter((x) => !visitedUrls.has(x)),+ Infinity,+ sc.crawlerOptions.maxDepth ?? 10,+ );++ const addableJobCount =+ sc.crawlerOptions.limit === undefined+ ? Infinity+ : sc.crawlerOptions.limit -+ (await getDoneJobsOrderedLength(job.data.crawl_id));++ if (univistedUrls.links.length !== 0 && addableJobCount > 0) {+ logger.info("Adding jobs", {+ univistedUrls: univistedUrls.links.length,+ addableJobCount,+ });++ const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ crawlerOptions: {+ ...job.data.crawlerOptions,+ urlInvisibleInCurrentCrawl: true,+ },+ scrapeOptions: job.data.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ opts: {+ jobId: uuid,+ priority: 20,+ },+ };+ });++ const lockedIds = await lockURLsIndividually(+ job.data.crawl_id,+ sc,+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),+ );+ const lockedJobs = jobs.filter((x) =>+ lockedIds.find((y) => y.id === x.opts.jobId),+ );+ await addCrawlJobs(+ job.data.crawl_id,+ lockedJobs.map((x) => x.opts.jobId),+ logger,+ );+ await addScrapeJobs(lockedJobs);++ if (lockedJobs.length > 0) {+ logger.info("Added jobs, not going for the full finish", {+ lockedJobs: lockedJobs.length,+ });++ await unPreFinishCrawl(job.data.crawl_id);+ return;+ } else {+ logger.info("No jobs added (all discovered URLs were locked), finishing crawl");+ }+ }+ }++ logger.info("Finishing crawl");+ await finishCrawl(job.data.crawl_id, logger);++ if (!job.data.v1) {+ const jobIDs = await getCrawlJobs(job.data.crawl_id);++ const jobs = (await getJobs(jobIDs)).sort(+ (a, b) => a.timestamp - b.timestamp,+ );+ // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));+ const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")+ ? "failed"+ : "completed";++ const fullDocs = jobs+ .map((x) =>+ x.returnvalue+ ? Array.isArray(x.returnvalue)+ ? x.returnvalue[0]+ : x.returnvalue+ : null,+ )+ .filter((x) => x !== null);++ await logJob({+ job_id: job.data.crawl_id,+ success: jobStatus === "completed",+ message: sc.cancelled ? "Cancelled" : undefined,+ num_docs: fullDocs.length,+ docs: [],+ time_taken: (Date.now() - sc.createdAt) / 1000,+ team_id: job.data.team_id,+ mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",+ url: sc.originUrl!,+ scrapeOptions: sc.scrapeOptions,+ crawlerOptions: sc.crawlerOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ zeroDataRetention: job.data.zeroDataRetention,+ }, false, job.data.internalOptions?.bypassBilling ?? false);++ const data = {+ success: jobStatus !== "failed",+ result: {+ links: fullDocs.map((doc) => {+ return {+ content: doc,+ source: doc?.metadata?.sourceURL ?? doc?.url ?? "",+ };+ }),+ },+ project_id: job.data.project_id,+ docs: fullDocs,+ };++ // v0 web hooks, call when done with all the data+ if (!job.data.v1) {+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ data,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null+ ? "crawl.completed"+ : "batch_scrape.completed",+ });+ }+ } else {+ const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);+ const jobStatus = sc.cancelled ? "failed" : "completed";++ let credits_billed = null;++ if (process.env.USE_DB_AUTHENTICATION === "true") {+ const creditsRpc = await supabase_service+ .rpc("credits_billed_by_crawl_id_1", {+ i_crawl_id: job.data.crawl_id,+ });++ credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;++ if (credits_billed === null) {+ logger.warn("Credits billed is null", {+ error: creditsRpc.error,+ });+ }+ }++ await logJob(+ {+ job_id: job.data.crawl_id,+ success: jobStatus === "completed",+ message: sc.cancelled ? "Cancelled" : undefined,+ num_docs,+ docs: [],+ time_taken: (Date.now() - sc.createdAt) / 1000,+ team_id: job.data.team_id,+ scrapeOptions: sc.scrapeOptions,+ mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",+ url:+ sc?.originUrl ??+ (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),+ crawlerOptions: sc.crawlerOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ credits_billed,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ true,+ job.data.internalOptions?.bypassBilling ?? false,+ );+++ // v1 web hooks, call when done with no data, but with event completed+ if (job.data.v1 && job.data.webhook) {+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ data: [],+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null+ ? "crawl.completed"+ : "batch_scrape.completed",+ });+ }+ }+ }+}\ No newline at end of filediff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsnew file mode 100644index 000000000..a80bc4a58--- /dev/null+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -0,0 +1,849 @@+import { configDotenv } from "dotenv";+import { Job } from "bullmq";+import * as Sentry from "@sentry/node";+import http from "http";+import https from "https";++import { logger as _logger } from "../../lib/logger";+import {+ concurrentJobDone,+ pushConcurrencyLimitActiveJob,+} from "../../lib/concurrency-limit";+import { addJobPriority, deleteJobPriority } from "../../lib/job-priority";+import { cacheableLookup } from "../../scraper/scrapeURL/lib/cacheableLookup";+import { v4 as uuidv4 } from "uuid";+import {+ addCrawlJob,+ addCrawlJobs,+ addCrawlJobDone,+ crawlToCrawler,+ finishCrawlKickoff,+ generateURLPermutations,+ getCrawl,+ lockURL,+ lockURLsIndividually,+ normalizeURL,+ saveCrawl,+ StoredCrawl+} from "../../lib/crawl-redis";+import { addScrapeJob, addScrapeJobs } from "../queue-jobs";+import { getJobPriority } from "../../lib/job-priority";+import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";+import { getACUCTeam } from "../../controllers/auth";+import { callWebhook } from "../webhook";+import { CustomError } from "../../lib/custom-error";+import { startWebScraperPipeline } from "../../main/runWebScraper";+import { CostTracking } from "../../lib/extract/extraction-service";+import { redisEvictConnection } from "../redis";+import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";+import { logJob } from "../logging/log_job";+import { generateURLSplits, queryIndexAtSplitLevel } from "../index";+import { WebCrawler } from "../../scraper/WebScraper/crawler";+import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";+import { getBillingQueue } from "../queue-service";+import type { Logger } from "winston";+import { finishCrawlIfNeeded } from "./crawl-logic";++class RacedRedirectError extends Error {+ constructor() {+ super("Raced redirect error");+ }+}++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++configDotenv();++const jobLockExtendInterval =+ Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;+const jobLockExtensionTime =+ Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;++cacheableLookup.install(http.globalAgent);+cacheableLookup.install(https.globalAgent);++async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) {+ let creditsToBeBilled: number | null = null;++ if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {+ creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags);++ if (+ job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&+ process.env.USE_DB_AUTHENTICATION === "true"+ ) {+ try {+ const billingJobId = uuidv4();+ logger.debug(+ `Adding billing job to queue for team ${job.data.team_id}`,+ {+ billingJobId,+ credits: creditsToBeBilled,+ is_extract: false,+ },+ );++ // Add directly to the billing queue - the billing worker will handle the rest+ await getBillingQueue().add(+ "bill_team",+ {+ team_id: job.data.team_id,+ subscription_id: undefined,+ credits: creditsToBeBilled,+ is_extract: false,+ timestamp: new Date().toISOString(),+ originating_job_id: job.id,+ },+ {+ jobId: billingJobId,+ priority: 10,+ },+ );+ return creditsToBeBilled;+ } catch (error) {+ logger.error(+ `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,+ { error },+ );+ Sentry.captureException(error);+ return creditsToBeBilled;+ }+ }+ }++ return creditsToBeBilled;+}++async function processJob(job: Job & { id: string }) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "processJob",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data?.crawl_id ?? undefined,+ teamId: job.data?.team_id ?? undefined,+ zeroDataRetention: job.data?.zeroDataRetention ?? false,+ });+ logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });+ const start = job.data.startTime ?? Date.now();+ const remainingTime = job.data.scrapeOptions.timeout ? (job.data.scrapeOptions.timeout - (Date.now() - start)) : undefined;++ const costTracking = new CostTracking();++ try {+ job.updateProgress({+ current: 1,+ total: 100,+ current_step: "SCRAPING",+ current_url: "",+ });++ if (remainingTime !== undefined && remainingTime < 0) {+ throw new Error("timeout");+ }+ const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;++ if (job.data.crawl_id) {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ if (sc && sc.cancelled) {+ throw new Error("Parent crawl/batch scrape was cancelled");+ }+ }+++ const pipeline = await Promise.race([+ startWebScraperPipeline({+ job,+ costTracking,+ }),+ ...(remainingTime !== undefined+ ? [+ (async () => {+ await sleep(remainingTime);+ throw new Error("timeout");+ })(),+ ]+ : []),+ ]);++ try {+ signal?.throwIfAborted();+ } catch (e) {+ throw new Error("timeout");+ }++ if (!pipeline.success) {+ throw pipeline.error;+ }++ const end = Date.now();+ const timeTakenInSeconds = (end - start) / 1000;++ const doc = pipeline.document;++ const rawHtml = doc.rawHtml ?? "";++ if (!job.data.scrapeOptions.formats.includes("rawHtml")) {+ delete doc.rawHtml;+ }++ if (job.data.concurrencyLimited) {+ doc.warning =+ "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." ++ (doc.warning ? " " + doc.warning : "");+ }++ const data = {+ success: true,+ result: {+ links: [+ {+ content: doc,+ source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",+ id: job.id,+ },+ ],+ },+ project_id: job.data.project_id,+ document: doc,+ };++ if (job.data.crawl_id) {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;++ if (+ doc.metadata.url !== undefined &&+ doc.metadata.sourceURL !== undefined &&+ normalizeURL(doc.metadata.url, sc) !==+ normalizeURL(doc.metadata.sourceURL, sc) &&+ job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape+ ) {+ const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);+ const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);+ if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {+ const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";+ throw new Error(reason);+ }++ // Only re-set originUrl if it's different from the current hostname+ // This is only done on this condition to handle cross-domain redirects+ // If this would be done for non-crossdomain redirects, but also for e.g.+ // redirecting / -> /introduction (like our docs site does), it would+ // break crawling the entire site without allowBackwardsCrawling - mogery+ const isHostnameDifferent =+ normalizeUrlOnlyHostname(doc.metadata.url) !==+ normalizeUrlOnlyHostname(doc.metadata.sourceURL);+ if (job.data.isCrawlSourceScrape && isHostnameDifferent) {+ // TODO: re-fetch sitemap for redirect target domain+ sc.originUrl = doc.metadata.url;+ await saveCrawl(job.data.crawl_id, sc);+ }++ if (isUrlBlocked(doc.metadata.url, (await getACUCTeam(job.data.team_id))?.flags ?? null)) {+ throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking+ }++ const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));+ const p2 = generateURLPermutations(+ normalizeURL(doc.metadata.sourceURL, sc),+ );++ if (JSON.stringify(p1) !== JSON.stringify(p2)) {+ logger.debug(+ "Was redirected, removing old URL and locking new URL...",+ { oldUrl: doc.metadata.sourceURL, newUrl: doc.metadata.url },+ );++ // Prevent redirect target from being visited in the crawl again+ // See lockURL+ const x = await redisEvictConnection.sadd(+ "crawl:" + job.data.crawl_id + ":visited",+ ...p1.map((x) => x.href),+ );+ const lockRes = x === p1.length;++ if (job.data.crawlerOptions !== null && !lockRes) {+ throw new RacedRedirectError();+ }+ }+ }++ if (job.data.crawlerOptions !== null) {+ if (!sc.cancelled) {+ const crawler = crawlToCrawler(+ job.data.crawl_id,+ sc,+ (await getACUCTeam(job.data.team_id))?.flags ?? null,+ doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,+ job.data.crawlerOptions,+ );++ const links = await crawler.filterLinks(+ await crawler.extractLinksFromHTML(+ rawHtml ?? "",+ doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,+ ),+ Infinity,+ sc.crawlerOptions?.maxDepth ?? 10,+ );+ logger.debug("Discovered " + links.links.length + " links...", {+ linksLength: links.links.length,+ });++ for (const link of links.links) {+ if (await lockURL(job.data.crawl_id, sc, link)) {+ // This seems to work really welel+ const jobPriority = await getJobPriority({+ team_id: sc.team_id,+ basePriority: job.data.crawl_id ? 20 : 10,+ });+ const jobId = uuidv4();++ logger.debug(+ "Determined job priority " ++ jobPriority ++ " for URL " ++ JSON.stringify(link),+ { jobPriority, url: link },+ );++ await addScrapeJob(+ {+ url: link,+ mode: "single_urls",+ team_id: sc.team_id,+ scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),+ internalOptions: sc.internalOptions,+ crawlerOptions: {+ ...sc.crawlerOptions,+ currentDiscoveryDepth:+ (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,+ },+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ {},+ jobId,+ jobPriority,+ );++ await addCrawlJob(job.data.crawl_id, jobId, logger);+ logger.debug("Added job for URL " + JSON.stringify(link), {+ jobPriority,+ url: link,+ newJobId: jobId,+ });+ } else {+ // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!+ // logger.debug("Could not lock URL " + JSON.stringify(link), {+ // url: link,+ // });+ }+ }++ // Only run check after adding new jobs for discovery - mogery+ if (job.data.isCrawlSourceScrape) {+ const filterResult = await crawler.filterLinks(+ [doc.metadata.url ?? doc.metadata.sourceURL!],+ 1,+ sc.crawlerOptions?.maxDepth ?? 10,+ );+ if (filterResult.links.length === 0) {+ const url = doc.metadata.url ?? doc.metadata.sourceURL!;+ const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";+ throw new Error(reason);+ }+ }+ }+ }++ try {+ signal?.throwIfAborted();+ } catch (e) {+ throw new Error("timeout");+ }++ const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);++ doc.metadata.creditsUsed = credits_billed ?? undefined;++ logger.debug("Logging job to DB...");+ await logJob(+ {+ job_id: job.id as string,+ success: true,+ num_docs: 1,+ docs: [doc],+ time_taken: timeTakenInSeconds,+ team_id: job.data.team_id,+ mode: job.data.mode,+ url: job.data.url,+ crawlerOptions: sc.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ cost_tracking: costTracking,+ pdf_num_pages: doc.metadata.numPages,+ credits_billed,+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ true,+ job.data.internalOptions?.bypassBilling ?? false,+ );++ if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {+ logger.debug("Calling webhook with success...", {+ webhook: job.data.webhook,+ });+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ scrapeId: job.id,+ data,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",+ });+ }++ logger.debug("Declaring job as done...");+ await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);++ await finishCrawlIfNeeded(job, sc);+ } else {+ try {+ signal?.throwIfAborted();+ } catch (e) {+ throw new Error("timeout");+ }++ const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);++ doc.metadata.creditsUsed = credits_billed ?? undefined;++ await logJob({+ job_id: job.id,+ success: true,+ message: "Scrape completed",+ num_docs: 1,+ docs: [doc],+ time_taken: timeTakenInSeconds,+ team_id: job.data.team_id,+ mode: "scrape",+ url: job.data.url,+ scrapeOptions: job.data.scrapeOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ num_tokens: 0, // TODO: fix+ cost_tracking: costTracking,+ pdf_num_pages: doc.metadata.numPages,+ credits_billed,+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,+ zeroDataRetention: job.data.zeroDataRetention,+ }, false, job.data.internalOptions?.bypassBilling ?? false);+ }++ logger.info(`🐂 Job done ${job.id}`);+ return data;+ } catch (error) {+ if (job.data.crawl_id) {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;++ logger.debug("Declaring job as done...");+ await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);+ await redisEvictConnection.srem(+ "crawl:" + job.data.crawl_id + ":visited_unique",+ normalizeURL(job.data.url, sc),+ );++ await finishCrawlIfNeeded(job, sc);+ }++ const isEarlyTimeout =+ error instanceof Error && error.message === "timeout";+ const isCancelled =+ error instanceof Error &&+ error.message === "Parent crawl/batch scrape was cancelled";++ if (isEarlyTimeout) {+ logger.error(`🐂 Job timed out ${job.id}`);+ } else if (error instanceof RacedRedirectError) {+ logger.warn(`🐂 Job got redirect raced ${job.id}, silently failing`);+ } else if (isCancelled) {+ logger.warn(`🐂 Job got cancelled, silently failing`);+ } else {+ logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });++ Sentry.captureException(error, {+ data: {+ job: job.id,+ },+ });++ if (error instanceof CustomError) {+ // Here we handle the error, then save the failed job+ logger.error(error.message); // or any other error handling+ }+ logger.error(error);+ if (error.stack) {+ logger.error(error.stack);+ }+ }++ const data = {+ success: false,+ document: null,+ project_id: job.data.project_id,+ error:+ error instanceof Error+ ? error+ : typeof error === "string"+ ? new Error(error)+ : new Error(JSON.stringify(error)),+ };++ if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id ?? (job.id as string),+ scrapeId: job.id,+ data,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",+ });+ }++ const end = Date.now();+ const timeTakenInSeconds = (end - start) / 1000;++ const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);++ logger.debug("Logging job to DB...");+ await logJob(+ {+ job_id: job.id as string,+ success: false,+ message:+ typeof error === "string"+ ? error+ : (error.message ??+ "Something went wrong... Contact [email protected]"),+ num_docs: 0,+ docs: [],+ time_taken: timeTakenInSeconds,+ team_id: job.data.team_id,+ mode: job.data.mode,+ url: job.data.url,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ cost_tracking: costTracking,+ credits_billed,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ true,+ job.data.internalOptions?.bypassBilling ?? false,+ );+ return data;+ }+}++async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: string) {+ if (sc.crawlerOptions.ignoreSitemap) {+ return [];+ }++ const trimmedURL = new URL(url);+ trimmedURL.search = "";++ const index = await queryIndexAtSplitLevel(+ sc.crawlerOptions.allowBackwardCrawling ? generateURLSplits(trimmedURL.href)[0] : trimmedURL.href,+ sc.crawlerOptions.limit ?? 10000,+ );++ const validIndexLinksResult = await crawler.filterLinks(+ index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),+ sc.crawlerOptions.limit ?? 10000,+ sc.crawlerOptions.maxDepth ?? 10,+ false,+ );+ const validIndexLinks = validIndexLinksResult.links;++ return validIndexLinks;+}++async function processKickoffJob(job: Job & { id: string }) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "processKickoffJob",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data?.crawl_id ?? undefined,+ teamId: job.data?.team_id ?? undefined,+ zeroDataRetention: job.data.zeroDataRetention ?? false,+ });++ try {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);++ logger.debug("Locking URL...");+ await lockURL(job.data.crawl_id, sc, job.data.url);+ const jobId = uuidv4();+ logger.debug("Adding scrape job to Redis...", { jobId });+ await addScrapeJob(+ {+ url: job.data.url,+ mode: "single_urls",+ team_id: job.data.team_id,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ webhook: job.data.webhook,+ v1: job.data.v1,+ isCrawlSourceScrape: true,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ {+ priority: 15,+ },+ jobId,+ );+ logger.debug("Adding scrape job to BullMQ...", { jobId });+ await addCrawlJob(job.data.crawl_id, jobId, logger);++ if (job.data.webhook) {+ logger.debug("Calling webhook with crawl.started...", {+ webhook: job.data.webhook,+ });+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ data: null,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: "crawl.started",+ });+ }++ const sitemap = sc.crawlerOptions.ignoreSitemap+ ? 0+ : await crawler.tryGetSitemap(async (urls) => {+ if (urls.length === 0) return;++ logger.debug("Using sitemap chunk of length " + urls.length, {+ sitemapLength: urls.length,+ });++ let jobPriority = await getJobPriority({+ team_id: job.data.team_id,+ basePriority: 21,+ });+ logger.debug("Using job priority " + jobPriority, { jobPriority });++ const jobs = urls.map((url) => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ opts: {+ jobId: uuid,+ priority: 20,+ },+ };+ });++ logger.debug("Locking URLs...");+ const lockedIds = await lockURLsIndividually(+ job.data.crawl_id,+ sc,+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),+ );+ const lockedJobs = jobs.filter((x) =>+ lockedIds.find((y) => y.id === x.opts.jobId),+ );+ logger.debug("Adding scrape jobs to Redis...");+ await addCrawlJobs(+ job.data.crawl_id,+ lockedJobs.map((x) => x.opts.jobId),+ logger,+ );+ logger.debug("Adding scrape jobs to BullMQ...");+ await addScrapeJobs(lockedJobs);+ });++ if (sitemap === 0) {+ logger.debug("Sitemap not found or ignored.", {+ ignoreSitemap: sc.crawlerOptions.ignoreSitemap,+ });+ }++ const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);++ if (indexLinks.length > 0) {+ logger.debug("Using index links of length " + indexLinks.length, {+ indexLinksLength: indexLinks.length,+ });++ let jobPriority = await getJobPriority({+ team_id: job.data.team_id,+ basePriority: 21,+ });+ logger.debug("Using job priority " + jobPriority, { jobPriority });++ const jobs = indexLinks.map((url) => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ opts: {+ jobId: uuid,+ priority: 20,+ },+ };+ });++ logger.debug("Locking URLs...");+ const lockedIds = await lockURLsIndividually(+ job.data.crawl_id,+ sc,+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),+ );+ const lockedJobs = jobs.filter((x) =>+ lockedIds.find((y) => y.id === x.opts.jobId),+ );+ logger.debug("Adding scrape jobs to Redis...");+ await addCrawlJobs(+ job.data.crawl_id,+ lockedJobs.map((x) => x.opts.jobId),+ logger,+ );+ logger.debug("Adding scrape jobs to BullMQ...");+ await addScrapeJobs(lockedJobs);+ }++ logger.debug("Done queueing jobs!");++ await finishCrawlKickoff(job.data.crawl_id);+ await finishCrawlIfNeeded(job, sc);++ return { success: true };+ } catch (error) {+ logger.error("An error occurred!", { error });+ await finishCrawlKickoff(job.data.crawl_id);+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ if (sc) {+ await finishCrawlIfNeeded(job, sc);+ }+ return { success: false, error };+ }+}++export const processJobInternal = async (job: Job & { id: string }) => {+ const logger = _logger.child({+ module: "queue-worker",+ method: "processJobInternal",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data?.crawl_id ?? undefined,+ zeroDataRetention: job.data?.zeroDataRetention ?? false,+ });++ try {+ try {+ let extendLockInterval: NodeJS.Timeout | null = null;+ if (job.data?.mode !== "kickoff" && job.data?.team_id) {+ extendLockInterval = setInterval(async () => {+ await pushConcurrencyLimitActiveJob(job.data.team_id, job.id, 60 * 1000); // 60s lock renew, just like in the queue+ }, jobLockExtendInterval);+ }++ await addJobPriority(job.data.team_id, job.id);+ try {+ if (job.data?.mode === "kickoff") {+ const result = await processKickoffJob(job);+ if (result.success) {+ return null;+ } else {+ throw (result as any).error;+ }+ } else {+ const result = await processJob(job);+ if (result.success) {+ try {+ if (+ process.env.USE_DB_AUTHENTICATION === "true" &&+ (job.data.crawl_id || process.env.GCS_BUCKET_NAME)+ ) {+ logger.debug(+ "Job succeeded -- putting null in Redis",+ );+ return null;+ } else {+ logger.debug("Job succeeded -- putting result in Redis");+ return result.document;+ }+ } catch (e) { }+ } else {+ throw (result as any).error;+ }+ }+ } finally {+ await deleteJobPriority(job.data.team_id, job.id);+ if (extendLockInterval) {+ clearInterval(extendLockInterval);+ }+ }+ } finally {+ await concurrentJobDone(job);+ }+ } catch (error) {+ logger.debug("Job failed", { error });+ Sentry.captureException(error);+ throw error;+ }+};++module.exports = processJobInternal;\ No newline at end of filediff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 4f43ff104..d5aa33470 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -67,8 +67,6 @@ export interface RunWebScraperParams {mode: Mode;scrapeOptions: ScrapeOptions;internalOptions?: InternalOptions;- // onSuccess: (result: V1Document, mode: string) => void;- // onError: (error: Error) => void;team_id: string;bull_job_id: string;priority?: number;diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 96ed80ae1..1aa0a0420 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "1.29.2",+ "version": "1.29.3","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",@@ -16,7 +16,8 @@"build": "tsup","build-and-publish": "npm run build && npm publish --access public","publish-beta": "npm run build && npm publish --access public --tag beta",- "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"+ "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"},"repository": {"type": "git",diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.tsindex ffe3ac747..a0a2d345a 100644--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts@@ -297,4 +297,5 @@ describe('FirecrawlApp E2E Tests', () => {const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");});+});diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.tsnew file mode 100644index 000000000..75563aa3b--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts@@ -0,0 +1,154 @@+import FirecrawlApp from '../../../index';+import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';++describe('monitorJobStatus retry logic', () => {+ let app: FirecrawlApp;+ let originalConsoleWarn: typeof console.warn;++ beforeEach(() => {+ app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });+ originalConsoleWarn = console.warn;+ console.warn = jest.fn();+ });++ afterEach(() => {+ console.warn = originalConsoleWarn;+ jest.clearAllMocks();+ });++ test('should retry on socket hang up error', async () => {+ const socketHangUpError = new Error('socket hang up') as any;+ socketHangUpError.code = 'ECONNRESET';++ const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }+ };++ const originalGetRequest = app.getRequest;+ let callCount = 0;++ app.getRequest = async function(url: string, headers: any) {+ callCount++;+ if (callCount === 1) {+ throw socketHangUpError;+ }+ return successResponse;+ };++ const result = await app.monitorJobStatus('test-id', {}, 1);++ expect(callCount).toBe(2);+ expect(result).toEqual(successResponse.data);+ expect(console.warn).toHaveBeenCalledWith(+ expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')+ );+ });++ test('should retry on ETIMEDOUT error', async () => {+ const timeoutError = new Error('timeout') as any;+ timeoutError.code = 'ETIMEDOUT';++ const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }+ };++ const originalGetRequest = app.getRequest;+ let callCount = 0;++ app.getRequest = async function(url: string, headers: any) {+ callCount++;+ if (callCount === 1) {+ throw timeoutError;+ }+ return successResponse;+ };++ const result = await app.monitorJobStatus('test-id', {}, 1);++ expect(callCount).toBe(2);+ expect(result).toEqual(successResponse.data);+ });++ test('should fail after max retries exceeded', async () => {+ const socketHangUpError = new Error('socket hang up') as any;+ socketHangUpError.code = 'ECONNRESET';++ app.getRequest = async function(url: string, headers: any) {+ throw socketHangUpError;+ };++ await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');++ expect(console.warn).toHaveBeenCalledTimes(3);+ }, 15000);++ test('should not retry on non-retryable errors', async () => {+ const authError = new Error('Unauthorized') as any;+ authError.response = { status: 401, data: { error: 'Unauthorized' } };++ app.getRequest = async function(url: string, headers: any) {+ throw authError;+ };++ await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');++ expect(console.warn).not.toHaveBeenCalled();+ });++ test('should retry on HTTP timeout status codes', async () => {+ const timeoutError = new Error('Request timeout') as any;+ timeoutError.response = { status: 408, data: { error: 'Request timeout' } };++ const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }+ };++ const originalGetRequest = app.getRequest;+ let callCount = 0;++ app.getRequest = async function(url: string, headers: any) {+ callCount++;+ if (callCount === 1) {+ throw timeoutError;+ }+ return successResponse;+ };++ const result = await app.monitorJobStatus('test-id', {}, 1);++ expect(callCount).toBe(2);+ expect(result).toEqual(successResponse.data);+ });++ test('should use exponential backoff for retries', async () => {+ const socketHangUpError = new Error('socket hang up') as any;+ socketHangUpError.code = 'ECONNRESET';++ const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }+ };++ const originalGetRequest = app.getRequest;+ let callCount = 0;++ app.getRequest = async function(url: string, headers: any) {+ callCount++;+ if (callCount <= 2) {+ throw socketHangUpError;+ }+ return successResponse;+ };++ const startTime = Date.now();+ const result = await app.monitorJobStatus('test-id', {}, 1);+ const endTime = Date.now();++ expect(callCount).toBe(3);+ expect(result).toEqual(successResponse.data);+ expect(endTime - startTime).toBeGreaterThan(3000);+ });+});diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 139f7a919..f93bb48e3 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -1455,33 +1455,39 @@ export default class FirecrawlApp {headers: AxiosRequestHeaders,checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse> {- try {- let failedTries = 0;- while (true) {+ let failedTries = 0;+ let networkRetries = 0;+ const maxNetworkRetries = 3;++ while (true) {+ try {let statusResponse: AxiosResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`,headers);+if (statusResponse.status === 200) {failedTries = 0;+ networkRetries = 0;let statusData = statusResponse.data;- if (statusData.status === "completed") {- if ("data" in statusData) {- let data = statusData.data;- while (typeof statusData === 'object' && 'next' in statusData) {- if (data.length === 0) {- break- }- statusResponse = await this.getRequest(statusData.next, headers);- statusData = statusResponse.data;- data = data.concat(statusData.data);++ if (statusData.status === "completed") {+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ break}- statusData.data = data;- return statusData;- } else {- throw new FirecrawlError("Crawl job completed but no data was returned", 500);+ statusResponse = await this.getRequest(statusData.next, headers);+ statusData = statusResponse.data;+ data = data.concat(statusData.data);}- } else if (+ statusData.data = data;+ return statusData;+ } else {+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);+ }+ } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {checkInterval = Math.max(checkInterval, 2);@@ -1500,12 +1506,69 @@ export default class FirecrawlApp {this.handleError(statusResponse, "check crawl status");}}+ } catch (error: any) {+ if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {+ networkRetries++;+ const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);++ await new Promise((resolve) => setTimeout(resolve, backoffDelay));+ continue;+ }++ throw new FirecrawlError(error, 500);}- } catch (error: any) {- throw new FirecrawlError(error, 500);}}+ /**+ * Determines if an error is retryable (transient network error)+ * @param error - The error to check+ * @returns True if the error should be retried+ */+ private isRetryableError(error: any): boolean {+ if (error instanceof AxiosError) {+ if (!error.response) {+ const code = error.code;+ const message = error.message?.toLowerCase() || '';++ return (+ code === 'ECONNRESET' ||+ code === 'ETIMEDOUT' ||+ code === 'ENOTFOUND' ||+ code === 'ECONNREFUSED' ||+ message.includes('socket hang up') ||+ message.includes('network error') ||+ message.includes('timeout')+ );+ }++ if (error.response?.status === 408 || error.response?.status === 504) {+ return true;+ }+ }++ if (error && typeof error === 'object') {+ const code = error.code;+ const message = error.message?.toLowerCase() || '';++ if (code === 'ECONNRESET' ||+ code === 'ETIMEDOUT' ||+ code === 'ENOTFOUND' ||+ code === 'ECONNREFUSED' ||+ message.includes('socket hang up') ||+ message.includes('network error') ||+ message.includes('timeout')) {+ return true;+ }++ if (error.response?.status === 408 || error.response?.status === 504) {+ return true;+ }+ }++ return false;+ }+/*** Handles errors from API responses.* @param {AxiosResponse} response - The response from the API.diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 99db69b04..9119facd4 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -13,7 +13,7 @@ import osfrom .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa-__version__ = "2.16.3"+__version__ = "2.16.4"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex df415d4ce..1207c3be7 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -24,12 +24,6 @@ import aiohttpimport asynciofrom pydantic import Field-# Suppress Pydantic warnings about attribute shadowing-warnings.filterwarnings("ignore", message="Field name \"json\" in \"FirecrawlDocument\" shadows an attribute in parent \"BaseModel\"")-warnings.filterwarnings("ignore", message="Field name \"json\" in \"ChangeTrackingData\" shadows an attribute in parent \"BaseModel\"")-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonConfig\" shadows an attribute in parent \"BaseModel\"")-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ExtractParams\" shadows an attribute in parent \"BaseModel\"")-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingOptions\" shadows an attribute in parent \"BaseModel\"")def get_version():try:@@ -106,7 +100,7 @@ class ChangeTrackingData(pydantic.BaseModel):changeStatus: str # "new" | "same" | "changed" | "removed"visibility: str # "visible" | "hidden"diff: Optional[Dict[str, Any]] = None- json: Optional[Any] = None+ json_field: Optional[Any] = pydantic.Field(None, alias='json')class FirecrawlDocument(pydantic.BaseModel, Generic[T]):"""Document retrieved or processed by Firecrawl."""@@ -116,7 +110,7 @@ class FirecrawlDocument(pydantic.BaseModel, Generic[T]):rawHtml: Optional[str] = Nonelinks: Optional[List[str]] = Noneextract: Optional[T] = None- json: Optional[T] = None+ json_field: Optional[T] = pydantic.Field(None, alias='json')screenshot: Optional[str] = Nonemetadata: Optional[Any] = Noneactions: Optional[ActionsResult] = None@@ -139,7 +133,7 @@ class WebhookConfig(pydantic.BaseModel):class ChangeTrackingOptions(pydantic.BaseModel):"""Configuration for change tracking."""modes: Optional[List[Literal["git-diff", "json"]]] = None- schema: Optional[Any] = None+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')prompt: Optional[str] = Nonetag: Optional[str] = None@@ -219,7 +213,7 @@ class ExtractAgent(pydantic.BaseModel):class JsonConfig(pydantic.BaseModel):"""Configuration for extraction."""prompt: Optional[str] = None- schema: Optional[Any] = None+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')systemPrompt: Optional[str] = Noneagent: Optional[ExtractAgent] = None@@ -318,7 +312,7 @@ class MapResponse(pydantic.BaseModel):class ExtractParams(pydantic.BaseModel):"""Parameters for extracting information from URLs."""prompt: Optional[str] = None- schema: Optional[Any] = None+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')systemPrompt: Optional[str] = NoneallowExternalLinks: Optional[bool] = NoneenableWebSearch: Optional[bool] = None@@ -432,7 +426,7 @@ class ExtractParams(pydantic.BaseModel):Parameters for the extract operation."""prompt: Optional[str] = None- schema: Optional[Any] = pydantic.Field(None, alias='schema')+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')system_prompt: Optional[str] = Noneallow_external_links: Optional[bool] = Falseenable_web_search: Optional[bool] = False@@ -548,7 +542,7 @@ class FirecrawlApp:if timeout:scrape_params['timeout'] = timeoutif location:- scrape_params['location'] = location.dict(exclude_none=True)+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)if mobile is not None:scrape_params['mobile'] = mobileif skip_tls_verification is not None:@@ -565,16 +559,16 @@ class FirecrawlApp:extract = self._ensure_schema_dict(extract)if isinstance(extract, dict) and "schema" in extract:extract["schema"] = self._ensure_schema_dict(extract["schema"])- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)if json_options is not None:json_options = self._ensure_schema_dict(json_options)if isinstance(json_options, dict) and "schema" in json_options:json_options["schema"] = self._ensure_schema_dict(json_options["schema"])- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)if actions:- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]if change_tracking_options:- scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(exclude_none=True)+ scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(by_alias=True, exclude_none=True)if max_age is not None:scrape_params['maxAge'] = max_ageif store_in_cache is not None:@@ -671,7 +665,7 @@ class FirecrawlApp:if timeout is not None:search_params['timeout'] = timeoutif scrape_options is not None:- search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)# Add any additional kwargssearch_params.update(kwargs)@@ -679,7 +673,7 @@ class FirecrawlApp:# Create final params objectfinal_params = SearchParams(query=query, **search_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['origin'] = f"python-sdk@{version}"if _integration:@@ -793,7 +787,7 @@ class FirecrawlApp:if ignore_sitemap is not None:crawl_params['ignoreSitemap'] = ignore_sitemapif scrape_options is not None:- crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)if webhook is not None:crawl_params['webhook'] = webhookif deduplicate_similar_urls is not None:@@ -816,7 +810,7 @@ class FirecrawlApp:# Create final params objectfinal_params = CrawlParams(**crawl_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = urlparams_dict['origin'] = f"python-sdk@{version}"@@ -922,7 +916,7 @@ class FirecrawlApp:if ignore_sitemap is not None:crawl_params['ignoreSitemap'] = ignore_sitemapif scrape_options is not None:- crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)if webhook is not None:crawl_params['webhook'] = webhookif deduplicate_similar_urls is not None:@@ -944,7 +938,7 @@ class FirecrawlApp:# Create final params objectfinal_params = CrawlParams(**crawl_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = urlparams_dict['origin'] = f"python-sdk@{version}"@@ -1244,7 +1238,7 @@ class FirecrawlApp:# Create final params objectfinal_params = MapParams(**map_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = urlparams_dict['origin'] = f"python-sdk@{version}"@@ -1356,7 +1350,7 @@ class FirecrawlApp:if timeout is not None:scrape_params['timeout'] = timeoutif location is not None:- scrape_params['location'] = location.dict(exclude_none=True)+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)if mobile is not None:scrape_params['mobile'] = mobileif skip_tls_verification is not None:@@ -1371,16 +1365,16 @@ class FirecrawlApp:extract = self._ensure_schema_dict(extract)if isinstance(extract, dict) and "schema" in extract:extract["schema"] = self._ensure_schema_dict(extract["schema"])- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)if json_options is not None:json_options = self._ensure_schema_dict(json_options)if isinstance(json_options, dict) and "schema" in json_options:json_options["schema"] = self._ensure_schema_dict(json_options["schema"])- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)if actions:- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]if agent is not None:- scrape_params['agent'] = agent.dict(exclude_none=True)+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)if max_concurrency is not None:scrape_params['maxConcurrency'] = max_concurrencyif zero_data_retention is not None:@@ -1391,7 +1385,7 @@ class FirecrawlApp:# Create final params objectfinal_params = ScrapeParams(**scrape_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urlsparams_dict['origin'] = f"python-sdk@{version}"@@ -1497,7 +1491,7 @@ class FirecrawlApp:if timeout is not None:scrape_params['timeout'] = timeoutif location is not None:- scrape_params['location'] = location.dict(exclude_none=True)+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)if mobile is not None:scrape_params['mobile'] = mobileif skip_tls_verification is not None:@@ -1512,16 +1506,16 @@ class FirecrawlApp:extract = self._ensure_schema_dict(extract)if isinstance(extract, dict) and "schema" in extract:extract["schema"] = self._ensure_schema_dict(extract["schema"])- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)if json_options is not None:json_options = self._ensure_schema_dict(json_options)if isinstance(json_options, dict) and "schema" in json_options:json_options["schema"] = self._ensure_schema_dict(json_options["schema"])- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)if actions:- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]if agent is not None:- scrape_params['agent'] = agent.dict(exclude_none=True)+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)if max_concurrency is not None:scrape_params['maxConcurrency'] = max_concurrencyif zero_data_retention is not None:@@ -1532,7 +1526,7 @@ class FirecrawlApp:# Create final params objectfinal_params = ScrapeParams(**scrape_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urlsparams_dict['origin'] = f"python-sdk@{version}"@@ -1633,7 +1627,7 @@ class FirecrawlApp:if timeout is not None:scrape_params['timeout'] = timeoutif location is not None:- scrape_params['location'] = location.dict(exclude_none=True)+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)if mobile is not None:scrape_params['mobile'] = mobileif skip_tls_verification is not None:@@ -1648,16 +1642,16 @@ class FirecrawlApp:extract = self._ensure_schema_dict(extract)if isinstance(extract, dict) and "schema" in extract:extract["schema"] = self._ensure_schema_dict(extract["schema"])- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)if json_options is not None:json_options = self._ensure_schema_dict(json_options)if isinstance(json_options, dict) and "schema" in json_options:json_options["schema"] = self._ensure_schema_dict(json_options["schema"])- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)if actions:- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]if agent is not None:- scrape_params['agent'] = agent.dict(exclude_none=True)+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)if max_concurrency is not None:scrape_params['maxConcurrency'] = max_concurrencyif zero_data_retention is not None:@@ -1668,7 +1662,7 @@ class FirecrawlApp:# Create final params objectfinal_params = ScrapeParams(**scrape_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urlsparams_dict['origin'] = f"python-sdk@{version}"@@ -2097,7 +2091,7 @@ class FirecrawlApp:)headers = self._prepare_headers()- json_data = {'url': url, **params.dict(exclude_none=True)}+ json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}json_data['origin'] = f"python-sdk@{version}"try:@@ -2338,10 +2332,22 @@ class FirecrawlApp:Exception: An exception with a message containing the status code and error details from the response."""try:- error_message = response.json().get('error', 'No error message provided.')- error_details = response.json().get('details', 'No additional error details provided.')+ response_json = response.json()+ error_message = response_json.get('error', 'No error message provided.')+ error_details = response_json.get('details', 'No additional error details provided.')except:- raise requests.exceptions.HTTPError(f'Failed to parse Firecrawl error response as JSON. Status code: {response.status_code}', response=response)+ # If we can't parse JSON, provide a helpful error message with response content+ try:+ response_text = response.text[:500] # Limit to first 500 chars+ if response_text.strip():+ error_message = f"Server returned non-JSON response: {response_text}"+ error_details = f"Full response status: {response.status_code}"+ else:+ error_message = f"Server returned empty response with status {response.status_code}"+ error_details = "No additional details available"++ except ValueError:+ error_message = f"Server returned unreadable response with status {response.status_code}"+ error_details = "No additional details available"message = self._get_error_message(response.status_code, action, error_message, error_details)@@ -2364,7 +2370,7 @@ class FirecrawlApp:if status_code == 402:return f"Payment Required: Failed to {action}. {error_message} - {error_details}"elif status_code == 403:- message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"+ return f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"elif status_code == 408:return f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"elif status_code == 409:@@ -2518,7 +2524,7 @@ class FirecrawlApp:headers = self._prepare_headers()- json_data = {'query': query, **research_params.dict(exclude_none=True)}+ json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}json_data['origin'] = f"python-sdk@{version}"# Handle json options schema if present@@ -3054,7 +3060,7 @@ class AsyncFirecrawlApp(FirecrawlApp):if timeout:scrape_params['timeout'] = timeoutif location:- scrape_params['location'] = location.dict(exclude_none=True)+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)if mobile is not None:scrape_params['mobile'] = mobileif skip_tls_verification is not None:@@ -3071,14 +3077,14 @@ class AsyncFirecrawlApp(FirecrawlApp):extract = self._ensure_schema_dict(extract)if isinstance(extract, dict) and "schema" in extract:extract["schema"] = self._ensure_schema_dict(extract["schema"])- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)if json_options is not None:json_options = self._ensure_schema_dict(json_options)if isinstance(json_options, dict) and "schema" in json_options:json_options["schema"] = self._ensure_schema_dict(json_options["schema"])- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)if actions:- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']:@@ -3182,7 +3188,7 @@ class AsyncFirecrawlApp(FirecrawlApp):if timeout is not None:scrape_params['timeout'] = timeoutif location is not None:- scrape_params['location'] = location.dict(exclude_none=True)+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)if mobile is not None:scrape_params['mobile'] = mobileif skip_tls_verification is not None:@@ -3197,22 +3203,23 @@ class AsyncFirecrawlApp(FirecrawlApp):extract = self._ensure_schema_dict(extract)if isinstance(extract, dict) and "schema" in extract:extract["schema"] = self._ensure_schema_dict(extract["schema"])- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)if json_options is not None:json_options = self._ensure_schema_dict(json_options)if isinstance(json_options, dict) and "schema" in json_options:json_options["schema"] = self._ensure_schema_dict(json_options["schema"])- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)-+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)+ if actions is not None:+ scrape_params['actions'] = [action.dict(by_alias=True, exclude_none=True) for action in actions]if agent is not None:- scrape_params['agent'] = agent.dict(exclude_none=True)+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)# Add any additional kwargsscrape_params.update(kwargs)# Create final params objectfinal_params = ScrapeParams(**scrape_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urlsparams_dict['origin'] = f"python-sdk@{version}"@@ -3321,7 +3328,7 @@ class AsyncFirecrawlApp(FirecrawlApp):if timeout is not None:scrape_params['timeout'] = timeoutif location is not None:- scrape_params['location'] = location.dict(exclude_none=True)+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)if mobile is not None:scrape_params['mobile'] = mobileif skip_tls_verification is not None:@@ -3336,16 +3343,16 @@ class AsyncFirecrawlApp(FirecrawlApp):extract = self._ensure_schema_dict(extract)if isinstance(extract, dict) and "schema" in extract:extract["schema"] = self._ensure_schema_dict(extract["schema"])- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)if json_options is not None:json_options = self._ensure_schema_dict(json_options)if isinstance(json_options, dict) and "schema" in json_options:json_options["schema"] = self._ensure_schema_dict(json_options["schema"])- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)if actions:- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]if agent is not None:- scrape_params['agent'] = agent.dict(exclude_none=True)+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)if zero_data_retention is not None:scrape_params['zeroDataRetention'] = zero_data_retention@@ -3354,7 +3361,7 @@ class AsyncFirecrawlApp(FirecrawlApp):# Create final params objectfinal_params = ScrapeParams(**scrape_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urlsparams_dict['origin'] = f"python-sdk@{version}"@@ -3462,7 +3469,7 @@ class AsyncFirecrawlApp(FirecrawlApp):if ignore_sitemap is not None:crawl_params['ignoreSitemap'] = ignore_sitemapif scrape_options is not None:- crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)if webhook is not None:crawl_params['webhook'] = webhookif deduplicate_similar_urls is not None:@@ -3481,7 +3488,7 @@ class AsyncFirecrawlApp(FirecrawlApp):# Create final params objectfinal_params = CrawlParams(**crawl_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = urlparams_dict['origin'] = f"python-sdk@{version}"# Make request@@ -3577,7 +3584,7 @@ class AsyncFirecrawlApp(FirecrawlApp):if ignore_sitemap is not None:crawl_params['ignoreSitemap'] = ignore_sitemapif scrape_options is not None:- crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)if webhook is not None:crawl_params['webhook'] = webhookif deduplicate_similar_urls is not None:@@ -3596,7 +3603,7 @@ class AsyncFirecrawlApp(FirecrawlApp):# Create final params objectfinal_params = CrawlParams(**crawl_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = urlparams_dict['origin'] = f"python-sdk@{version}"@@ -3762,7 +3769,7 @@ class AsyncFirecrawlApp(FirecrawlApp):"""map_params = {}if params:- map_params.update(params.dict(exclude_none=True))+ map_params.update(params.dict(by_alias=True, exclude_none=True))# Add individual parametersif search is not None:@@ -3780,7 +3787,7 @@ class AsyncFirecrawlApp(FirecrawlApp):# Create final params objectfinal_params = MapParams(**map_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = urlparams_dict['origin'] = f"python-sdk@{version}"@@ -4164,7 +4171,6 @@ class AsyncFirecrawlApp(FirecrawlApp):url,max_urls=max_urls,show_full_text=show_full_text,- cache=cache,experimental_stream=experimental_stream)if not response.get('success') or 'id' not in response:@@ -4228,7 +4234,7 @@ class AsyncFirecrawlApp(FirecrawlApp):)headers = self._prepare_headers()- json_data = {'url': url, **params.dict(exclude_none=True)}+ json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}json_data['origin'] = f"python-sdk@{version}"try:@@ -4413,7 +4419,7 @@ class AsyncFirecrawlApp(FirecrawlApp):headers = self._prepare_headers()- json_data = {'query': query, **research_params.dict(exclude_none=True)}+ json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}json_data['origin'] = f"python-sdk@{version}"try:@@ -4505,7 +4511,7 @@ class AsyncFirecrawlApp(FirecrawlApp):if isinstance(params, dict):search_params.update(params)else:- search_params.update(params.dict(exclude_none=True))+ search_params.update(params.dict(by_alias=True, exclude_none=True))# Add individual parametersif limit is not None:@@ -4523,14 +4529,14 @@ class AsyncFirecrawlApp(FirecrawlApp):if timeout is not None:search_params['timeout'] = timeoutif scrape_options is not None:- search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)# Add any additional kwargssearch_params.update(kwargs)# Create final params objectfinal_params = SearchParams(query=query, **search_params)- params_dict = final_params.dict(exclude_none=True)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['origin'] = f"python-sdk@{version}"return await self._async_post_request(diff --git a/apps/redis/start-redis-server.sh b/apps/redis/start-redis-server.shindex e8a313ce9..8004d1640 100755--- a/apps/redis/start-redis-server.sh+++ b/apps/redis/start-redis-server.sh@@ -19,7 +19,7 @@ fi# Set maxmemory to 80% of RAMMAXMEMORY=$(($FLY_VM_MEMORY_MB*80/100))-mkdir /data/redis+mkdir -p /data/redisredis-server $PW_ARG \--dir /data/redis \