Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(v2): parsers + merge w/ main (#1907)

Gergő Móricz7400e10eaca3
diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index af6523b9c..01d291245 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -21,14 +21,21 @@ jobs:
- name: 'Checkout GitHub Action'
uses: actions/checkout@main
+ - name: 'Set up Docker Buildx'
+ uses: docker/setup-buildx-action@v3
+
- name: 'Login to GitHub Container Registry'
- uses: docker/login-action@v1
+ uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{github.actor}}
password: ${{secrets.GITHUB_TOKEN}}
- - name: 'Build Inventory Image'
- run: |
- docker build . --tag ghcr.io/mendableai/firecrawl:latest
- docker push ghcr.io/mendableai/firecrawl:latest
\ No newline at end of file
+ - name: 'Build and Push Image'
+ uses: docker/build-push-action@v5
+ with:
+ context: ./apps/api
+ push: true
+ tags: ghcr.io/mendableai/firecrawl:latest
+ cache-from: type=registry,ref=ghcr.io/mendableai/firecrawl:buildcache
+ cache-to: type=registry,ref=ghcr.io/mendableai/firecrawl:buildcache,mode=max
\ No newline at end of file
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml
index 77297fe08..c4d27635c 100644
--- a/.github/workflows/test-server-self-host.yml
+++ b/.github/workflows/test-server-self-host.yml
@@ -102,14 +102,17 @@ jobs:
docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng
pnpx wait-on tcp:3434 -t 30s
working-directory: ./
+ - name: Build the application
+ run: npm run build
+ working-directory: ./apps/api
- name: Start server
- run: npm run start:production > api.log 2>&1 &
+ run: npm run start:production:nobuild > api.log 2>&1 &
env:
PORT: 3002
HOST: 0.0.0.0
working-directory: ./apps/api
- name: Start worker
- run: npm run workers > worker.log 2>&1 &
+ run: npm run worker:production > worker.log 2>&1 &
env:
PORT: 3005
HOST: 0.0.0.0
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 1b903ea1f..06ad0fb08 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -104,22 +104,25 @@ jobs:
cargo build --release
chmod +x target/release/libcrawler.so
working-directory: ./apps/api/sharedLibs/crawler
+ - name: Build the application
+ run: npm run build
+ working-directory: ./apps/api
- name: Start the application
- run: npm run start:production > api.log 2>&1 &
+ run: npm run start:production:nobuild > api.log 2>&1 &
env:
PORT: 3002
HOST: 0.0.0.0
working-directory: ./apps/api
id: start_app
- name: Start worker
- run: npm run workers > worker.log 2>&1 &
+ run: npm run worker:production > worker.log 2>&1 &
env:
PORT: 3005
HOST: 0.0.0.0
working-directory: ./apps/api
id: start_workers
- name: Start index worker
- run: npm run index-worker > index-worker.log 2>&1 &
+ run: npm run index-worker:production > index-worker.log 2>&1 &
working-directory: ./apps/api
id: start_index_worker
- name: Wait for API
diff --git a/README.md b/README.md
index 3948aee29..b042290f5 100644
--- a/README.md
+++ b/README.md
@@ -71,7 +71,7 @@ To use the API, you need to sign up on [Firecrawl](https://firecrawl.dev) and ge
- [**Scrape**](#scraping): scrapes a URL and get its content in LLM-ready format (markdown, structured data via [LLM Extract](#llm-extraction-beta), screenshot, html)
- [**Crawl**](#crawling): scrapes all the URLs of a web page and return content in LLM-ready format
-- [**Map**](#map-alpha): input a website and get all the website urls - extremely fast
+- [**Map**](#map): input a website and get all the website urls - extremely fast
- [**Search**](#search): search the web and get full content from results
- [**Extract**](#extract): get structured data from single page, multiple pages or entire websites with AI.
diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
index 69cb302ae..5a8c62464 100644
--- a/apps/api/Dockerfile
+++ b/apps/api/Dockerfile
@@ -19,7 +19,7 @@ COPY sharedLibs/go-html-to-md /app/sharedLibs/go-html-to-md
# Install Go dependencies and build parser lib
RUN cd /app/sharedLibs/go-html-to-md && \
- go mod tidy && \
+ go mod download && \
go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go && \
chmod +x html-to-markdown.so
@@ -31,17 +31,17 @@ COPY sharedLibs/crawler /app/sharedLibs/crawler
# Install Rust dependencies and build transformer lib
RUN cd /app/sharedLibs/html-transformer && \
- cargo build --release && \
+ cargo build --release --locked && \
chmod +x target/release/libhtml_transformer.so
# Install Rust dependencies and build PDF parser lib
RUN cd /app/sharedLibs/pdf-parser && \
- cargo build --release && \
+ cargo build --release --locked && \
chmod +x target/release/libpdf_parser.so
# Install Rust dependencies and build crawler lib
RUN cd /app/sharedLibs/crawler && \
- cargo build --release && \
+ cargo build --release --locked && \
chmod +x target/release/libcrawler.so
FROM base
@@ -53,7 +53,7 @@ COPY --from=rust-base /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so
COPY --from=rust-base /app/sharedLibs/crawler/target/release/libcrawler.so /app/sharedLibs/crawler/target/release/libcrawler.so
# Install git
-RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y git procps && rm -rf /var/lib/apt/lists/*
# Start the server by default, this can be overwritten at runtime
EXPOSE 8080
diff --git a/apps/api/package.json b/apps/api/package.json
index c85393354..5d724dc75 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -4,11 +4,12 @@
"description": "",
"main": "index.js",
"scripts": {
- "start": "nodemon --exec ts-node src/index.ts",
+ "start": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"start:production": "tsc && node dist/src/index.js",
+ "start:production:nobuild": "node dist/src/index.js",
"format": "prettier --write \"src/**/*.(js|ts)\"",
"flyio": "node dist/src/index.js",
- "start:dev": "nodemon --exec ts-node src/index.ts",
+ "start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"build": "tsc",
"build:nosentry": "tsc",
"test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'",
@@ -16,9 +17,9 @@
"test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'",
"test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'",
"test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/*.test.ts",
- "workers": "nodemon --exec ts-node src/services/queue-worker.ts",
+ "workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"",
"worker:production": "node dist/src/services/queue-worker.js",
- "index-worker": "nodemon --exec ts-node src/services/indexing/index-worker.ts",
+ "index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",
"index-worker:production": "node dist/src/services/indexing/index-worker.js",
"mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
"mongo-docker-console": "docker exec -it mongodb mongosh",
@@ -44,11 +45,11 @@
"@types/supertest": "^6.0.2",
"jest": "^29.6.3",
"jest-fetch-mock": "^3.0.3",
- "nodemon": "^3.1.10",
"prettier": "^3.4.2",
"supertest": "^6.3.3",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
+ "tsc-watch": "^7.1.1",
"typescript": "^5.8.3"
},
"dependencies": {
@@ -106,7 +107,7 @@
"jsdom": "^26.0.0",
"json-schema-to-zod": "^2.3.0",
"keyword-extractor": "^0.0.28",
- "koffi": "^2.12.3",
+ "koffi": "^2.9.0",
"languagedetect": "^2.0.0",
"lodash": "^4.17.21",
"logsnag": "^1.0.0",
@@ -151,8 +152,7 @@
},
"pnpm": {
"onlyBuiltDependencies": [
- "@sentry-internal/node-cpu-profiler",
- "koffi"
+ "@sentry-internal/node-cpu-profiler"
],
"overrides": {
"brace-expansion@>=2.0.0 <=2.0.1": ">=2.0.2"
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index d2d1c5d87..1567f780b 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -174,8 +174,8 @@ importers:
specifier: ^0.0.28
version: 0.0.28
koffi:
- specifier: ^2.12.3
- version: 2.12.3
+ specifier: ^2.9.0
+ version: 2.9.0
languagedetect:
specifier: ^2.0.0
version: 2.0.0
@@ -321,9 +321,6 @@ importers:
jest-fetch-mock:
specifier: ^3.0.3
version: 3.0.3([email protected])
- nodemon:
- specifier: ^3.1.10
- version: 3.1.10
prettier:
specifier: ^3.4.2
version: 3.4.2
@@ -336,6 +333,9 @@ importers:
ts-node:
specifier: ^10.9.1
version: 10.9.2(@types/[email protected])([email protected])
+ tsc-watch:
+ specifier: ^7.1.1
+ version: 7.1.1([email protected])
typescript:
specifier: ^5.8.3
version: 5.8.3
@@ -1964,10 +1964,6 @@ packages:
resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
- resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==}
@@ -2080,10 +2076,6 @@ packages:
resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}
engines: {node: '>= 6'}
- resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}
- engines: {node: '>= 8.10.0'}
-
resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==}
engines: {node: '>=18'}
@@ -2368,6 +2360,9 @@ packages:
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==}
+
resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==}
@@ -2471,6 +2466,9 @@ packages:
resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}
engines: {node: '>= 0.6'}
+ resolution: {integrity: sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==}
+
resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
engines: {node: '>=6'}
@@ -2612,6 +2610,9 @@ packages:
resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==}
engines: {node: '>= 0.6'}
+ resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==}
+
resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
@@ -2663,10 +2664,6 @@ packages:
resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}
engines: {node: '>= 4.8.0'}
- resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
- engines: {node: '>= 6'}
-
resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==}
engines: {node: '>=16 || 14 >=14.18'}
@@ -2806,9 +2803,6 @@ packages:
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
- resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==}
-
resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}
@@ -2856,20 +2850,12 @@ packages:
resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
- resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==}
resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}
- resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
- engines: {node: '>=0.10.0'}
-
resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
engines: {node: '>=8'}
@@ -2878,10 +2864,6 @@ packages:
resolution: {integrity: sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==}
engines: {node: '>=6'}
- resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
- engines: {node: '>=0.10.0'}
-
resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
engines: {node: '>=0.12.0'}
@@ -3159,8 +3141,8 @@ packages:
resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==}
engines: {node: '>=6'}
- resolution: {integrity: sha512-VdUbreihzsJ6SAV9lm74/UkdvsPqOnkVAzdgtI7+kQlc0UZPtOdo28+dcquIhgSHQx3AO02gEz+3qtiEIhfM3w==}
+ resolution: {integrity: sha512-KCsuJ2gM58n6bNdR2Z7gqsh/3TchxxQFbVgax2/UvAjRTgwNSYAJDx9E3jrkBP4jEDHWRCfE47Y2OG+/fiSvEw==}
resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==}
@@ -3249,6 +3231,9 @@ packages:
engines: {node: '>=12.0.0'}
hasBin: true
+ resolution: {integrity: sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==}
+
resolution: {integrity: sha512-f3r0yqpz31VXiDB/wj9GaOB0a2PRLQl6vJmXiFrniNwjkKdvakqJRULhjFKJpxOchlCRiG5fcacoUZY5Xa6PEQ==}
engines: {node: '>= 18'}
@@ -3378,6 +3363,9 @@ packages:
resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}
+ resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==}
+
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
@@ -3408,11 +3396,6 @@ packages:
resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==}
- resolution: {integrity: sha512-WDjw3pJ0/0jMFmyNDp3gvY2YizjLmmOUQo6DEBY+JgdvW/yQ9mEeSw6H5ythl5Ny2ytb7f9C2nIbjSxMNzbJXw==}
- engines: {node: '>=10'}
- hasBin: true
-
resolution: {integrity: sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -3543,6 +3526,9 @@ packages:
resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==}
+ resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==}
+
resolution: {integrity: sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==}
@@ -3643,12 +3629,14 @@ packages:
resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
+ resolution: {integrity: sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==}
+ engines: {node: '>= 0.10'}
+ hasBin: true
+
resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==}
- resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==}
-
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
@@ -3713,10 +3701,6 @@ packages:
resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
- resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
- engines: {node: '>=8.10.0'}
-
resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==}
engines: {node: '>= 0.10'}
@@ -3897,10 +3881,6 @@ packages:
resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
- resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==}
- engines: {node: '>=10'}
-
resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}
@@ -3915,6 +3895,9 @@ packages:
resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
engines: {node: '>=0.10.0'}
+ resolution: {integrity: sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==}
+
resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
@@ -3932,12 +3915,19 @@ packages:
resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
engines: {node: '>= 0.8'}
+ resolution: {integrity: sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==}
+
resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==}
resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==}
+ resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==}
+ engines: {node: '>=0.6.19'}
+
resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==}
engines: {node: '>=10'}
@@ -4049,6 +4039,9 @@ packages:
resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}
+
resolution: {integrity: sha512-AOvV5YYIAFFBfransBzSTyztkc3IMfz5Eq3YluaRiEu55nn43Fzaufx70UqEKYr8BoLCach4q8g/bg6e5+/aFw==}
@@ -4071,10 +4064,6 @@ packages:
resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
engines: {node: '>=0.6'}
- resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==}
- hasBin: true
-
resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==}
engines: {node: '>=16'}
@@ -4128,6 +4117,13 @@ packages:
'@swc/wasm':
optional: true
+ resolution: {integrity: sha512-r6t37Dkk4vK44HwxOe+OzjpE/gDamZAwqXhtcAJD/hPVblcjJK45NxbK0HcDASXG0U4pEnCh640JZbeDVSC6yA==}
+ engines: {node: '>=12.12.0'}
+ hasBin: true
+ peerDependencies:
+ typescript: '*'
+
resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==}
@@ -4157,9 +4153,6 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
- resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==}
-
resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}
@@ -5114,7 +5107,7 @@ snapshots:
'@babel/traverse': 7.24.6
'@babel/types': 7.24.6
convert-source-map: 2.0.0
- debug: 4.4.1([email protected])
+ debug: 4.4.1
gensync: 1.0.0-beta.2
json5: 2.2.3
semver: 6.3.1
@@ -5301,7 +5294,7 @@ snapshots:
'@babel/helper-split-export-declaration': 7.24.6
'@babel/parser': 7.24.6
'@babel/types': 7.24.6
- debug: 4.4.1([email protected])
+ debug: 4.4.1
globals: 11.12.0
transitivePeerDependencies:
- supports-color
@@ -6654,7 +6647,7 @@ snapshots:
dependencies:
- debug: 4.4.1([email protected])
+ debug: 4.4.1
transitivePeerDependencies:
- supports-color
@@ -6816,8 +6809,6 @@ snapshots:
read-cmd-shim: 4.0.0
write-file-atomic: 5.0.1
-
@@ -6958,18 +6949,6 @@ snapshots:
parse5: 7.1.2
parse5-htmlparser2-tree-adapter: 7.0.0
- dependencies:
- anymatch: 3.1.3
- braces: 3.0.3
- glob-parent: 5.1.2
- is-binary-path: 2.1.0
- is-glob: 4.0.3
- normalize-path: 3.0.0
- readdirp: 3.6.0
- optionalDependencies:
- fsevents: 2.3.3
-
@@ -7147,11 +7126,9 @@ snapshots:
dependencies:
ms: 2.1.2
dependencies:
ms: 2.1.3
- optionalDependencies:
- supports-color: 5.5.0
@@ -7233,6 +7210,8 @@ snapshots:
es-errors: 1.3.0
gopd: 1.2.0
+
dependencies:
end-of-stream: 1.4.4
@@ -7319,6 +7298,16 @@ snapshots:
+ dependencies:
+ duplexer: 0.1.2
+ from: 0.1.7
+ map-stream: 0.1.0
+ pause-stream: 0.0.11
+ split: 0.3.3
+ stream-combiner: 0.0.4
+ through: 2.3.8
+
@@ -7499,6 +7488,8 @@ snapshots:
+
@@ -7568,10 +7559,6 @@ snapshots:
shelljs: 0.8.5
shelljs.exec: 1.1.8
- dependencies:
- is-glob: 4.0.3
-
dependencies:
foreground-child: 3.2.1
@@ -7690,7 +7677,7 @@ snapshots:
dependencies:
'@tootallnate/once': 2.0.0
agent-base: 6.0.2
- debug: 4.4.1([email protected])
+ debug: 4.4.1
transitivePeerDependencies:
- supports-color
@@ -7711,7 +7698,7 @@ snapshots:
dependencies:
agent-base: 7.1.3
- debug: 4.4.1([email protected])
+ debug: 4.4.1
transitivePeerDependencies:
- supports-color
@@ -7738,8 +7725,6 @@ snapshots:
-
@@ -7771,7 +7756,7 @@ snapshots:
dependencies:
'@ioredis/commands': 1.2.0
cluster-key-slot: 1.1.2
- debug: 4.4.1([email protected])
+ debug: 4.4.1
denque: 2.1.0
lodash.defaults: 4.2.0
lodash.isarguments: 3.1.0
@@ -7789,26 +7774,16 @@ snapshots:
- dependencies:
- binary-extensions: 2.3.0
-
dependencies:
hasown: 2.0.2
-
- dependencies:
- is-extglob: 2.1.1
-
@@ -7862,7 +7837,7 @@ snapshots:
dependencies:
- debug: 4.4.1([email protected])
+ debug: 4.4.1
istanbul-lib-coverage: 3.2.2
source-map: 0.6.1
transitivePeerDependencies:
@@ -8299,7 +8274,7 @@ snapshots:
@@ -8396,6 +8371,8 @@ snapshots:
underscore: 1.13.6
xmlbuilder: 10.1.1
+
@@ -8498,6 +8475,8 @@ snapshots:
+
@@ -8523,19 +8502,6 @@ snapshots:
- dependencies:
- chokidar: 3.6.0
- debug: 4.4.1([email protected])
- ignore-by-default: 1.0.1
- minimatch: 3.1.2
- pstree.remy: 1.1.8
- semver: 7.6.2
- simple-update-notifier: 2.0.0
- supports-color: 5.5.0
- touch: 3.1.1
- undefsafe: 2.0.5
-
dependencies:
abbrev: 2.0.0
@@ -8650,6 +8616,10 @@ snapshots:
+ dependencies:
+ through: 2.3.8
+
dependencies:
'@pdf-lib/standard-fonts': 1.0.0
@@ -8739,12 +8709,14 @@ snapshots:
+ dependencies:
+ event-stream: 3.3.4
+
dependencies:
punycode: 2.3.1
-
@@ -8818,10 +8790,6 @@ snapshots:
process: 0.11.10
string_decoder: 1.3.0
- dependencies:
- picomatch: 2.3.1
-
dependencies:
resolve: 1.22.8
@@ -8846,7 +8814,7 @@ snapshots:
dependencies:
- debug: 4.4.1([email protected])
+ debug: 4.4.1
module-details-from-path: 1.0.3
resolve: 1.22.8
transitivePeerDependencies:
@@ -9018,10 +8986,6 @@ snapshots:
dependencies:
is-arrayish: 0.3.2
- dependencies:
- semver: 7.6.2
-
@@ -9033,6 +8997,10 @@ snapshots:
+ dependencies:
+ through: 2.3.8
+
@@ -9045,12 +9013,18 @@ snapshots:
+ dependencies:
+ duplexer: 0.1.2
+
dependencies:
stubs: 3.0.0
+
dependencies:
char-regex: 1.0.2
@@ -9184,6 +9158,8 @@ snapshots:
+
@@ -9200,8 +9176,6 @@ snapshots:
-
dependencies:
tldts: 6.1.75
@@ -9250,6 +9224,14 @@ snapshots:
v8-compile-cache-lib: 3.0.1
yn: 3.1.1
+ dependencies:
+ cross-spawn: 7.0.6
+ node-cleanup: 2.1.2
+ ps-tree: 1.2.0
+ string-argv: 0.3.2
+ typescript: 5.8.3
+
@@ -9271,8 +9253,6 @@ snapshots:
-
diff --git a/apps/api/sharedLibs/crawler/Cargo.lock b/apps/api/sharedLibs/crawler/Cargo.lock
index 3412e19ce..8a753c488 100644
--- a/apps/api/sharedLibs/crawler/Cargo.lock
+++ b/apps/api/sharedLibs/crawler/Cargo.lock
@@ -11,16 +11,33 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "anyhow"
+version = "1.0.98"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
+
+[[package]]
+name = "bstr"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
+dependencies = [
+ "lazy_static",
+ "memchr",
+ "regex-automata 0.1.10",
+]
+
[[package]]
name = "crawler"
version = "0.1.0"
dependencies = [
"libc",
"regex",
- "robotstxt",
"roxmltree",
"serde",
"serde_json",
+ "texting_robots",
"url",
]
@@ -157,6 +174,12 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
[[package]]
name = "libc"
version = "0.2.174"
@@ -175,6 +198,22 @@ version = "2.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.1"
@@ -216,10 +255,16 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
"aho-corasick",
"memchr",
- "regex-automata",
+ "regex-automata 0.4.9",
"regex-syntax",
]
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+
[[package]]
name = "regex-automata"
version = "0.4.9"
@@ -237,12 +282,6 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
-[[package]]
-name = "robotstxt"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbc52377db80e3fec3a2c748ca603b8b6cacdd34ff89ff4b742a635361d4b4a7"
-
[[package]]
name = "roxmltree"
version = "0.20.0"
@@ -321,6 +360,42 @@ dependencies = [
"syn",
]
+[[package]]
+name = "texting_robots"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a"
+dependencies = [
+ "anyhow",
+ "bstr",
+ "lazy_static",
+ "nom",
+ "percent-encoding",
+ "regex",
+ "thiserror",
+ "url",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "tinystr"
version = "0.8.1"
diff --git a/apps/api/sharedLibs/crawler/Cargo.toml b/apps/api/sharedLibs/crawler/Cargo.toml
index 342789129..1e43b72e9 100644
--- a/apps/api/sharedLibs/crawler/Cargo.toml
+++ b/apps/api/sharedLibs/crawler/Cargo.toml
@@ -9,7 +9,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
url = "2.5.4"
regex = "1.11.1"
-robotstxt = "0.3.0"
+texting_robots = "0.2.2"
roxmltree = "0.20"
[lib]
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rs
index aec7935a7..0ee510f98 100644
--- a/apps/api/sharedLibs/crawler/src/lib.rs
+++ b/apps/api/sharedLibs/crawler/src/lib.rs
@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
use std::ffi::{CStr, CString};
use url::Url;
use regex::Regex;
-use robotstxt::DefaultMatcher;
+use texting_robots::Robot;
#[derive(Deserialize)]
struct FilterLinksCall {
@@ -165,15 +165,27 @@ fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std
}
if !data.ignore_robots_txt {
- let mut matcher = DefaultMatcher::default();
- let allowed = matcher.allowed_by_robots(&data.robots_txt, vec![
- "FireCrawlAgent",
- "FirecrawlAgent",
- ], url.as_str());
-
- if !allowed {
- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
- return false;
+ match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {
+ Ok(robot) => {
+ let allowed = robot.allowed(url.as_str());
+ if !allowed {
+ match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {
+ Ok(robot_alt) => {
+ let allowed_alt = robot_alt.allowed(url.as_str());
+ if !allowed_alt {
+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
+ return false;
+ }
+ }
+ Err(_) => {
+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
+ return false;
+ }
+ }
+ }
+ }
+ Err(_) => {
+ }
}
}
@@ -513,6 +525,104 @@ mod tests {
assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");
}
+ #[test]
+ fn test_filter_links_normal_robots_txt() {
+ let data = FilterLinksCall {
+ links: vec!["https://example.com/allowed".to_string(), "https://example.com/disallowed".to_string()],
+ limit: Some(10),
+ includes: vec![],
+ excludes: vec![],
+ ignore_robots_txt: false,
+ robots_txt: "User-agent: *\nDisallow: /disallowed".to_string(),
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ };
+
+ let result = _filter_links(data).unwrap();
+ assert_eq!(result.links.len(), 1);
+ assert_eq!(result.links[0], "https://example.com/allowed");
+ assert!(result.denial_reasons.contains_key("https://example.com/disallowed"));
+ assert_eq!(result.denial_reasons.get("https://example.com/disallowed").unwrap(), "ROBOTS_TXT");
+ }
+
+ #[test]
+ fn test_filter_links_malformed_robots_txt() {
+ let data = FilterLinksCall {
+ links: vec!["https://example.com/test".to_string()],
+ limit: Some(10),
+ includes: vec![],
+ excludes: vec![],
+ ignore_robots_txt: false,
+ robots_txt: "Invalid robots.txt content with \x00 null bytes and malformed syntax".to_string(),
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ };
+
+ let result = _filter_links(data);
+ assert!(result.is_ok());
+ let result = result.unwrap();
+ assert_eq!(result.links.len(), 1);
+ assert_eq!(result.links[0], "https://example.com/test");
+ }
+
+ #[test]
+ fn test_filter_links_non_utf8_robots_txt() {
+ let mut non_utf8_bytes = vec![0xFF, 0xFE];
+ non_utf8_bytes.extend_from_slice(b"User-agent: *\nDisallow: /blocked");
+ let non_utf8_string = String::from_utf8_lossy(&non_utf8_bytes).to_string();
+
+ let data = FilterLinksCall {
+ links: vec!["https://example.com/allowed".to_string()],
+ limit: Some(10),
+ includes: vec![],
+ excludes: vec![],
+ ignore_robots_txt: false,
+ robots_txt: non_utf8_string,
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ };
+
+ let result = _filter_links(data);
+ assert!(result.is_ok());
+ let result = result.unwrap();
+ assert_eq!(result.links.len(), 1);
+ assert_eq!(result.links[0], "https://example.com/allowed");
+ }
+
+ #[test]
+ fn test_filter_links_char_boundary_issue() {
+ let problematic_content = "User-agent: *\nDisallow: /\u{a0}test";
+
+ let data = FilterLinksCall {
+ links: vec!["https://example.com/test".to_string()],
+ limit: Some(10),
+ includes: vec![],
+ excludes: vec![],
+ ignore_robots_txt: false,
+ robots_txt: problematic_content.to_string(),
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ };
+
+ let result = _filter_links(data);
+ assert!(result.is_ok());
+ let result = result.unwrap();
+ assert_eq!(result.links.len(), 1);
+ assert_eq!(result.links[0], "https://example.com/test");
+ }
+
}
/// Frees a string allocated in Rust-land.
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.ts
index cccbb7370..588259e49 100644
--- a/apps/api/src/__tests__/snips/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/crawl.test.ts
@@ -1,5 +1,6 @@
import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing, crawlStart, Identity, idmux, scrapeTimeout } from "./lib";
import { describe, it, expect } from "@jest/globals";
+import { filterLinks } from "../../lib/crawler";
let identity: Identity;
@@ -272,3 +273,89 @@ describe("Crawl tests", () => {
expect(typeof response.body.id).toBe("string");
});
});
+
+describe("Robots.txt FFI Integration tests", () => {
+ it.concurrent("handles normal robots.txt parsing via FFI", async () => {
+
+ const result = await filterLinks({
+ links: ['https://example.com/allowed', 'https://example.com/disallowed'],
+ limit: 10,
+ max_depth: 10,
+ base_url: 'https://example.com',
+ initial_url: 'https://example.com',
+ regex_on_full_url: false,
+ excludes: [],
+ includes: [],
+ allow_backward_crawling: true,
+ ignore_robots_txt: false,
+ robots_txt: 'User-agent: *\nDisallow: /disallowed'
+ });
+
+ expect(result.links).toHaveLength(1);
+ expect(result.links[0]).toBe('https://example.com/allowed');
+ expect(result.denial_reasons.has('https://example.com/disallowed')).toBe(true);
+ expect(result.denial_reasons.get('https://example.com/disallowed')).toBe('ROBOTS_TXT');
+ }, 10000);
+
+ it.concurrent("handles malformed robots.txt without crashing via FFI", async () => {
+
+ const result = await filterLinks({
+ links: ['https://example.com/test'],
+ limit: 10,
+ max_depth: 10,
+ base_url: 'https://example.com',
+ initial_url: 'https://example.com',
+ regex_on_full_url: false,
+ excludes: [],
+ includes: [],
+ allow_backward_crawling: true,
+ ignore_robots_txt: false,
+ robots_txt: 'Invalid robots.txt content with \x00 null bytes and malformed syntax'
+ });
+
+ expect(result.links).toHaveLength(1);
+ expect(result.links[0]).toBe('https://example.com/test');
+ }, 10000);
+
+ it.concurrent("handles non-UTF8 robots.txt content without crashing via FFI", async () => {
+
+ const nonUtf8Content = String.fromCharCode(0xFF, 0xFE) + 'User-agent: *\nDisallow: /blocked';
+ const result = await filterLinks({
+ links: ['https://example.com/allowed'],
+ limit: 10,
+ max_depth: 10,
+ base_url: 'https://example.com',
+ initial_url: 'https://example.com',
+ regex_on_full_url: false,
+ excludes: [],
+ includes: [],
+ allow_backward_crawling: true,
+ ignore_robots_txt: false,
+ robots_txt: nonUtf8Content
+ });
+
+ expect(result.links).toHaveLength(1);
+ expect(result.links[0]).toBe('https://example.com/allowed');
+ }, 10000);
+
+ it.concurrent("handles char boundary issues without crashing via FFI", async () => {
+
+ const problematicContent = 'User-agent: *\nDisallow: /\u{a0}test';
+ const result = await filterLinks({
+ links: ['https://example.com/safe'],
+ limit: 10,
+ max_depth: 10,
+ base_url: 'https://example.com',
+ initial_url: 'https://example.com',
+ regex_on_full_url: false,
+ excludes: [],
+ includes: [],
+ allow_backward_crawling: true,
+ ignore_robots_txt: false,
+ robots_txt: problematicContent
+ });
+
+ expect(result.links).toHaveLength(1);
+ expect(result.links[0]).toBe('https://example.com/safe');
+ }, 10000);
+});
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index e8ee156a7..9f6e5b7bb 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -1,7 +1,7 @@
import { Request, Response } from "express";
import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
-import { createRedisConnection, getScrapeQueue } from "../../../src/services/queue-service";
+import { getScrapeQueue } from "../../../src/services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { logger } from "../../../src/lib/logger";
import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";
@@ -12,13 +12,11 @@ import { Job } from "bullmq";
import { toLegacyDocument } from "../v1/types";
import type { DBJob, PseudoJob } from "../v1/crawl-status";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-import IORedis from "ioredis";
configDotenv();
-export async function getJobs(crawlId: string, ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {
- const queue = getScrapeQueue(conn);
+export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
@@ -99,10 +97,8 @@ export async function crawlStatusController(req: Request, res: Response) {
return res.status(403).json({ error: "Forbidden" });
}
let jobIDs = await getCrawlJobs(req.params.jobId);
- const conn = createRedisConnection();
- let jobs = await getJobs(req.params.jobId, jobIDs, conn);
+ let jobs = await getJobs(req.params.jobId, jobIDs);
let jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
- conn.disconnect();
// Combine jobs and jobStatuses into a single array of objects
let jobsWithStatuses = jobs.map((job, index) => ({
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 9d0e005bd..d3425dff3 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -94,47 +94,36 @@ export async function scrapeHelper(
let doc;
- const err = await Sentry.startSpan(
- {
- name: "Wait for job to finish",
- op: "bullmq.wait",
- attributes: { job: jobId },
- },
- async (span) => {
- try {
- doc = await waitForJob(jobId, timeout);
- } catch (e) {
- if (
- e instanceof Error &&
- (e.message.startsWith("Job wait") || e.message === "timeout")
- ) {
- span.setAttribute("timedOut", true);
- return {
- success: false,
- error: "Request timed out",
- returnCode: 408,
- };
- } else if (
- typeof e === "string" &&
- (e.includes("Error generating completions: ") ||
- e.includes("Invalid schema for function") ||
- e.includes(
- "LLM extraction did not match the extraction schema you provided.",
- ))
- ) {
- return {
- success: false,
- error: e,
- returnCode: 500,
- };
- } else {
- throw e;
- }
- }
- span.setAttribute("result", JSON.stringify(doc));
- return null;
- },
- );
+ try {
+ doc = await waitForJob(jobId, timeout);
+ } catch (e) {
+ if (
+ e instanceof Error &&
+ (e.message.startsWith("Job wait") || e.message === "timeout")
+ ) {
+ return {
+ success: false,
+ error: "Request timed out",
+ returnCode: 408,
+ };
+ } else if (
+ typeof e === "string" &&
+ (e.includes("Error generating completions: ") ||
+ e.includes("Invalid schema for function") ||
+ e.includes(
+ "LLM extraction did not match the extraction schema you provided.",
+ ))
+ ) {
+ return {
+ success: false,
+ error: e,
+ returnCode: 500,
+ };
+ } else {
+ throw e;
+ }
+ }
+ const err = null;
if (err !== null) {
return err;
diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.ts
index 47ef2fd4f..c68579eac 100644
--- a/apps/api/src/controllers/v0/status.ts
+++ b/apps/api/src/controllers/v0/status.ts
@@ -3,7 +3,6 @@ import { logger } from "../../../src/lib/logger";
import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";
import { getJobs } from "./crawl-status";
import * as Sentry from "@sentry/node";
-import { createRedisConnection } from "../../services/queue-service";
export async function crawlJobStatusPreviewController(
req: Request,
@@ -26,12 +25,10 @@ export async function crawlJobStatusPreviewController(
// }
// }
- const conn = createRedisConnection();
- const jobs = (await getJobs(req.params.jobId, jobIDs, conn)).sort(
+ const jobs = (await getJobs(req.params.jobId, jobIDs)).sort(
(a, b) => a.timestamp - b.timestamp,
);
const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
- conn.disconnect();
const jobStatus = sc.cancelled
? "failed"
: jobStatuses.every((x) => x === "completed")
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index cee458ab6..190ca5209 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -19,7 +19,7 @@ import {
isCrawlFinished,
isCrawlFinishedLocked,
} from "../../lib/crawl-redis";
-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue } from "../../services/queue-service";
import { getJob, getJobs } from "./crawl-status";
import * as Sentry from "@sentry/node";
import { Job, JobState } from "bullmq";
@@ -88,8 +88,7 @@ async function crawlStatusWS(
const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));
- const conn = createRedisConnection();
- const queue = getScrapeQueue(conn);
+ const queue = getScrapeQueue();
const jobStatuses = await Promise.all(
notDoneJobIDs.map(async (x) => [
@@ -102,7 +101,7 @@ async function crawlStatusWS(
.map((x) => x[0]);
const newlyDoneJobs: Job[] = (
- await Promise.all(newlyDoneJobIDs.map((x) => getJob(x, conn)))
+ await Promise.all(newlyDoneJobIDs.map((x) => getJob(x)))
).filter((x) => x !== undefined) as Job[];
for (const job of newlyDoneJobs) {
@@ -117,7 +116,6 @@ async function crawlStatusWS(
}
doneJobIDs.push(...newlyDoneJobIDs);
- conn.disconnect();
setTimeout(loop, 1000);
};
@@ -127,8 +125,7 @@ async function crawlStatusWS(
let jobIDs = await getCrawlJobs(req.params.jobId);
- const conn = createRedisConnection();
- const queue = getScrapeQueue(conn);
+ const queue = getScrapeQueue();
let jobStatuses = await Promise.all(
jobIDs.map(
@@ -163,8 +160,7 @@ async function crawlStatusWS(
jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processing
- const doneJobs = await getJobs(doneJobIDs, conn);
- conn.disconnect();
+ const doneJobs = await getJobs(doneJobIDs);
const data = doneJobs.map((x) => x.returnvalue);
await send(ws, {
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 47e99a7d3..7bca8d6d6 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -13,7 +13,7 @@ import {
getDoneJobsOrderedLength,
isCrawlKickoffFinished,
} from "../../lib/crawl-redis";
-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue } from "../../services/queue-service";
import {
supabaseGetJobById,
supabaseGetJobsById,
@@ -24,7 +24,6 @@ import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-import IORedis from "ioredis";
configDotenv();
export type PseudoJob<T> = {
@@ -41,9 +40,9 @@ export type PseudoJob<T> = {
export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string }
-export async function getJob(id: string, conn: IORedis): Promise<PseudoJob<any> | null> {
+export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([
- getScrapeQueue(conn).getJob(id),
+ getScrapeQueue().getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,
(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
@@ -73,10 +72,9 @@ export async function getJob(id: string, conn: IORedis): Promise<PseudoJob<any>
return job;
}
-export async function getJobs(ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {
- const queue = getScrapeQueue(conn);
+export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
@@ -159,14 +157,11 @@ export async function crawlStatusController(
}
let jobIDs = await getCrawlJobs(req.params.jobId);
- const conn = createRedisConnection();
- const queue = getScrapeQueue(conn);
let jobStatuses = await Promise.all(
jobIDs.map(
- async (x) => [x, await queue.getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
),
);
- conn.disconnect();
if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {
for (let rangeStart = 0; ; rangeStart += 1000) {
@@ -261,7 +256,7 @@ export async function crawlStatusController(
const creditsRpc = await supabase_rr_service
.rpc("credits_billed_by_crawl_id_1", {
i_crawl_id: req.params.jobId,
- });
+ }, { get: true });
creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? (totalCount * (
sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
@@ -294,15 +289,12 @@ export async function crawlStatusController(
return res.status(403).json({ success: false, error: "Forbidden" });
}
- const teamIdsExcludedFromExpiry = [
- "8f819703-1b85-4f7f-a6eb-e03841ec6617",
- "f96ad1a4-8102-4b35-9904-36fd517d3616",
- ];
+ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;
if (
crawlJob
- && !teamIdsExcludedFromExpiry.includes(crawlJob.team_id)
- && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > 24 * 60 * 60 * 1000
+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs
) {
return res.status(404).json({ success: false, error: "Job expired" });
}
@@ -385,9 +377,8 @@ export async function crawlStatusController(
i += factor
) {
// get current chunk and retrieve jobs
- const conn = createRedisConnection();
const currentIDs = doneJobsOrder.slice(i, i + factor);
- const jobs = await getJobs(currentIDs, conn);
+ const jobs = await getJobs(currentIDs);
// iterate through jobs and add them one them one to the byte counter
// both loops will break once we cross the byte counter
@@ -410,7 +401,6 @@ export async function crawlStatusController(
doneJobs.push(job);
bytes += JSON.stringify(job.returnvalue ?? null).length;
}
- conn.disconnect();
}
// if we ran over the bytes limit, remove the last document, except if it's the only document
@@ -418,15 +408,13 @@ export async function crawlStatusController(
doneJobs.splice(doneJobs.length - 1, 1);
}
} else {
- const conn = createRedisConnection();
doneJobs = (
await Promise.all(
- (await getJobs(doneJobsOrder, conn)).map(async (x) =>
+ (await getJobs(doneJobsOrder)).map(async (x) =>
(await x.getState()) === "failed" ? null : x,
),
)
).filter((x) => x !== null) as PseudoJob<any>[];
- conn.disconnect();
}
const data = doneJobs.map((x) => x.returnvalue);
diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.ts
index 472e208eb..525db6b6c 100644
--- a/apps/api/src/controllers/v1/deep-research.ts
+++ b/apps/api/src/controllers/v1/deep-research.ts
@@ -75,34 +75,9 @@ export async function deepResearchController(
summaries: [],
});
- if (Sentry.isInitialized()) {
- const size = JSON.stringify(jobData).length;
- await Sentry.startSpan(
- {
- name: "Add deep research job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": researchId,
- "messaging.destination.name": getDeepResearchQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await getDeepResearchQueue().add(researchId, {
- ...jobData,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- }, { jobId: researchId });
- },
- );
- } else {
- await getDeepResearchQueue().add(researchId, jobData, {
- jobId: researchId,
- });
- }
+ await getDeepResearchQueue().add(researchId, jobData, {
+ jobId: researchId,
+ });
return res.status(200).json({
success: true,
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index e6d356b85..7eba40a60 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -119,34 +119,9 @@ export async function extractController(
zeroDataRetention: req.acuc?.flags?.forceZDR,
});
- if (Sentry.isInitialized()) {
- const size = JSON.stringify(jobData).length;
- await Sentry.startSpan(
- {
- name: "Add extract job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": extractId,
- "messaging.destination.name": getExtractQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await getExtractQueue().add(extractId, {
- ...jobData,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- }, { jobId: extractId });
- },
- );
- } else {
- await getExtractQueue().add(extractId, jobData, {
- jobId: extractId,
- });
- }
+ await getExtractQueue().add(extractId, jobData, {
+ jobId: extractId,
+ });
return res.status(200).json({
success: true,
diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.ts
index 4177af00d..6834b819f 100644
--- a/apps/api/src/controllers/v1/generate-llmstxt.ts
+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts
@@ -51,38 +51,9 @@ export async function generateLLMsTextController(
fullText: "",
});
- if (Sentry.isInitialized()) {
- const size = JSON.stringify(jobData).length;
- await Sentry.startSpan(
- {
- name: "Add LLMstxt generation job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": generationId,
- "messaging.destination.name": getGenerateLlmsTxtQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await getGenerateLlmsTxtQueue().add(
- generationId,
- {
- ...jobData,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- },
- { jobId: generationId },
- );
- },
- );
- } else {
- await getGenerateLlmsTxtQueue().add(generationId, jobData, {
- jobId: generationId,
- });
- }
+ await getGenerateLlmsTxtQueue().add(generationId, jobData, {
+ jobId: generationId,
+ });
return res.status(200).json({
success: true,
diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.ts
index 802b50a48..78ded113a 100644
--- a/apps/api/src/controllers/v1/scrape-status.ts
+++ b/apps/api/src/controllers/v1/scrape-status.ts
@@ -2,7 +2,6 @@ import { Response } from "express";
import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";
import { getJob } from "./crawl-status";
import { logger as _logger } from "../../lib/logger";
-import { createRedisConnection } from "../../services/queue-service";
export async function scrapeStatusController(req: any, res: any) {
const logger = _logger.child({
@@ -36,9 +35,7 @@ export async function scrapeStatusController(req: any, res: any) {
});
}
- const conn = createRedisConnection();
- const jobData = await getJob(req.params.jobId, conn);
- conn.disconnect();
+ const jobData = await getJob(req.params.jobId);
const data = Array.isArray(jobData?.returnvalue)
? jobData?.returnvalue[0]
: jobData?.returnvalue;
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index aa7256bf0..c61301930 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -17,7 +17,7 @@ export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
res: Response<ScrapeResponse>,
) {
- const jobId = uuidv4();
+ const jobId: string = uuidv4();
const preNormalizedBody = { ...req.body };
if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {
@@ -60,7 +60,7 @@ export async function scrapeController(
const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(req.body, req.body.timeout, req.auth.team_id);
- await addScrapeJob(
+ const bullJob = await addScrapeJob(
{
url: req.body.url,
mode: "single_urls",
@@ -85,7 +85,7 @@ export async function scrapeController(
jobPriority,
isDirectToBullMQ,
);
- logger.info("Added scrape job now")
+ logger.info("Added scrape job now" + (bullJob ? "" : " (to concurrency queue)"));
const totalWait =
(req.body.waitFor ?? 0) +
@@ -96,7 +96,7 @@ export async function scrapeController(
let doc: Document;
try {
- doc = await waitForJob(jobId, timeout + totalWait, logger);
+ doc = await waitForJob(bullJob ? bullJob : jobId, timeout + totalWait, logger);
} catch (e) {
logger.error(`Error in scrapeController`, {
startTime,
@@ -125,9 +125,7 @@ export async function scrapeController(
logger.info("Done with waitForJob");
- const conn = createRedisConnection();
- await getScrapeQueue(conn).remove(jobId);
- conn.disconnect();
+ await getScrapeQueue().remove(jobId);
logger.info("Removed job from queue");
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 2825a7deb..620820e7f 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1111,6 +1111,7 @@ export type TeamFlags = {
zdrCost?: number;
checkRobotsOnScrape?: boolean;
allowTeammateInvites?: boolean;
+ crawlTtlHours?: number;
} | null;
export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index f7e4a27b9..8153ecdfa 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -200,6 +200,12 @@ export const screenshotFormatWithOptions = z.object({
export type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;
+export const parsersSchema = z.object({
+ pdf: z.boolean().default(true),
+}).default({ pdf: true });
+
+export type Parsers = z.infer<typeof parsersSchema>;
+
const baseScrapeOptions = z
.object({
formats: z
@@ -252,7 +258,7 @@ const baseScrapeOptions = z
.max(60000)
.default(0),
mobile: z.boolean().default(false),
- parsePDF: z.boolean().default(true),
+ parsers: parsersSchema.optional(),
actions: actionsSchema.optional(),
location: z
@@ -1117,6 +1123,7 @@ export function fromV1ScrapeOptions(
return x;
}
}).filter(x => x !== null),
+ parsers: v1ScrapeOptions.parsePDF !== undefined ? { pdf: v1ScrapeOptions.parsePDF } : undefined,
}),
internalOptions: {
teamId,
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index d68b4d400..8a806d2f0 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -200,6 +200,10 @@ export async function finishCrawl(id: string, __logger: Logger = _logger) {
await redisEvictConnection.srem("crawls_by_team_id:" + crawl.team_id, id);
await redisEvictConnection.expire("crawls_by_team_id:" + crawl.team_id, 24 * 60 * 60);
}
+
+ // Clear visited sets to save memory
+ await redisEvictConnection.del("crawl:" + id + ":visited");
+ await redisEvictConnection.del("crawl:" + id + ":visited_unique");
}
export async function getCrawlJobs(id: string): Promise<string[]> {
diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts
index aa63a6b39..a474e34e2 100644
--- a/apps/api/src/lib/html-to-markdown.ts
+++ b/apps/api/src/lib/html-to-markdown.ts
@@ -17,6 +17,7 @@ const goExecutablePath = join(
);
class GoMarkdownConverter {
+ private static instance: GoMarkdownConverter;
private convert: any;
private free: any;
@@ -29,12 +30,15 @@ class GoMarkdownConverter {
}
public static async getInstance(): Promise<GoMarkdownConverter> {
- try {
- await stat(goExecutablePath);
- } catch (_) {
- throw Error("Go shared library not found");
+ if (!GoMarkdownConverter.instance) {
+ try {
+ await stat(goExecutablePath);
+ } catch (_) {
+ throw Error("Go shared library not found");
+ }
+ GoMarkdownConverter.instance = new GoMarkdownConverter();
}
- return new GoMarkdownConverter();
+ return GoMarkdownConverter.instance;
}
public async convertHTMLToMarkdown(html: string): Promise<string> {
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 468850860..0b601d87c 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -35,7 +35,7 @@ export async function fetchRobotsTxt(
(x) => x[0].toLowerCase() === "content-type",
) ?? [])[1] ?? "";
- if (contentType.includes("text/html") ||
+ if ((contentType.includes("text/html") && response.data.trim().startsWith("<")) ||
contentType.includes("application/json") ||
contentType.includes("application/xml")) {
return "";
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index e5358d4c2..4c93de040 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -32,7 +32,8 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, interna
creditsToBeBilled += (flags?.zdrCost ?? 1);
}
- if (options.parsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {
+ const shouldParsePDF = options.parsers?.pdf ?? true;
+ if (shouldParsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {
creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1);
}
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 81c5eee20..d3a967de4 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -20,11 +20,9 @@ configDotenv();
export async function startWebScraperPipeline({
job,
- token,
costTracking,
}: {
job: Job<WebScraperOptions> & { id: string };
- token: string;
costTracking: CostTracking;
}) {
return await runWebScraper({
@@ -38,15 +36,11 @@ export async function startWebScraperPipeline({
}
: {}),
},
- internalOptions: job.data.internalOptions,
- // onSuccess: (result, mode) => {
- // logger.debug(`🐂 Job completed ${job.id}`);
- // saveJob(job, result, token, mode);
- // },
- // onError: (error) => {
- // logger.error(`🐂 Job failed ${job.id}`);
- // ScrapeEvents.logJobEvent(job, "failed");
- // },
+ internalOptions: {
+ crawlId: job.data.crawl_id,
+ teamId: job.data.team_id,
+ ...job.data.internalOptions,
+ },
team_id: job.data.team_id,
bull_job_id: job.id.toString(),
priority: job.opts.priority,
@@ -62,8 +56,6 @@ export async function runWebScraper({
mode,
scrapeOptions,
internalOptions,
- // onSuccess,
- // onError,
team_id,
bull_job_id,
priority,
@@ -124,9 +116,6 @@ export async function runWebScraper({
}
}
- // This is where the returnvalue from the job is set
- // onSuccess(response.document, mode);
-
engines = response.engines;
if (
@@ -201,7 +190,6 @@ export async function runWebScraper({
const saveJob = async (
job: Job,
result: any,
- token: string,
mode: string,
engines?: EngineResultsTracker,
) => {
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index d6fcda657..3672097d1 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -353,6 +353,7 @@ export class WebCrawler {
};
const _urlsHandler = async (urls: string[]) => {
+ this.logger.debug("urlsHandler invoked");
if (fromMap && onlySitemap) {
return await urlsHandler(urls);
} else {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 768b63a96..cc1a1e4aa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -135,35 +135,17 @@ export async function fireEngineCheckStatus(
abort?: AbortSignal,
production = true,
): Promise<FireEngineCheckStatusSuccess> {
- let status = await Sentry.startSpan(
- {
- name: "fire-engine: Check status",
- attributes: {
- jobId,
- },
- },
- async (span) => {
- const x = Date.now();
- meta.logger.debug("Checking status (" + x + ")");
- const res = await robustFetch({
- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
- method: "GET",
- logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),
- headers: {
- ...(Sentry.isInitialized()
- ? {
- "sentry-trace": Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- }
- : {}),
- },
- mock,
- abort,
- });
- meta.logger.debug("Status came back (" + x + ")");
- return res;
- },
- );
+ const x = Date.now();
+ meta.logger.debug("Checking status (" + x + ")");
+ let status = await robustFetch({
+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
+ method: "GET",
+ logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),
+ headers: {},
+ mock,
+ abort,
+ });
+ meta.logger.debug("Status came back (" + x + ")");
// Fire-engine now saves the content to GCS
if (!status.content && status.docUrl) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts
index 9af6627ae..b9185a871 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts
@@ -12,33 +12,16 @@ export async function fireEngineDelete(
abort?: AbortSignal,
production = true,
) {
- await Sentry.startSpan(
- {
- name: "fire-engine: Delete scrape",
- attributes: {
- jobId,
- },
- },
- async (span) => {
- await robustFetch({
- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
- method: "DELETE",
- headers: {
- ...(Sentry.isInitialized()
- ? {
- "sentry-trace": Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- }
- : {}),
- },
- ignoreResponse: true,
- ignoreFailure: true,
- logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),
- mock,
- abort,
- });
- },
- );
+ await robustFetch({
+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
+ method: "DELETE",
+ headers: {},
+ ignoreResponse: true,
+ ignoreFailure: true,
+ logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),
+ mock,
+ abort,
+ });
// We do not care whether this fails or not.
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index f3c566817..fed212dcf 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -79,34 +79,17 @@ export async function fireEngineScrape<
abort?: AbortSignal,
production = true,
): Promise<z.infer<typeof schema>> {
- const scrapeRequest = await Sentry.startSpan(
- {
- name: "fire-engine: Scrape",
- attributes: {
- url: request.url,
- },
- },
- async (span) => {
- return await robustFetch({
- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,
- method: "POST",
- headers: {
- ...(Sentry.isInitialized()
- ? {
- "sentry-trace": Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- }
- : {}),
- },
- body: request,
- logger: logger.child({ method: "fireEngineScrape/robustFetch" }),
- schema,
- tryCount: 3,
- mock,
- abort,
- });
- },
- );
+ const scrapeRequest = await robustFetch({
+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,
+ method: "POST",
+ headers: {},
+ body: request,
+ logger: logger.child({ method: "fireEngineScrape/robustFetch" }),
+ schema,
+ tryCount: 3,
+ mock,
+ abort,
+ });
return scrapeRequest;
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 6649c735d..87b1a2217 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -175,7 +175,9 @@ export async function scrapePDF(
): Promise<EngineScrapeResult> {
const startTime = Date.now();
- if (!meta.options.parsePDF) {
+ const shouldParsePDF = meta.options.parsers?.pdf ?? true;
+
+ if (!shouldParsePDF) {
if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {
const content = (await readFile(meta.pdfPrefetch.filePath)).toString(
"base64",
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index ce009eb59..1a477502c 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -187,6 +187,9 @@ async function buildMetaObject(
scrapeId: id,
scrapeURL: url,
zeroDataRetention: internalOptions.zeroDataRetention,
+ teamId: internalOptions.teamId,
+ team_id: internalOptions.teamId,
+ crawlId: internalOptions.crawlId,
});
const logs: any[] = [];
@@ -211,6 +214,7 @@ async function buildMetaObject(
export type InternalOptions = {
teamId: string;
+ crawlId?: string;
priority?: number; // Passed along to fire-engine
forceEngine?: Engine | Engine[];
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
index fee2b45d5..39a4db931 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
@@ -12,7 +12,7 @@ export async function extractMetadataRust(
return {
...fromRust,
...(fromRust.favicon ? {
- favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url)
+ favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url).href
} : {}),
scrapeId: meta.id,
};
diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
index 11d77da22..8dad621db 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
@@ -5,7 +5,6 @@ import { getJob } from "../../../controllers/v1/crawl-status";
import gitDiff from 'git-diff';
import parseDiff from 'parse-diff';
import { generateCompletions } from "./llmExtract";
-import { createRedisConnection } from "../../../services/queue-service";
async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extract: any } | null> {
const changeTrackingFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")!;
@@ -69,7 +68,6 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume
}
const start = Date.now();
- const conn = createRedisConnection();
const res = await supabase_service
.rpc("diff_get_last_scrape_4", {
i_team_id: meta.internalOptions.teamId,
@@ -88,8 +86,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume
const job: {
returnvalue: Document,
- } | null = data?.o_job_id ? await getJob(data.o_job_id, conn) : null;
- conn.disconnect();
+ } | null = data?.o_job_id ? await getJob(data.o_job_id) : null;
if (data && job && job?.returnvalue) {
const previousMarkdown = job.returnvalue.markdown!;
const currentMarkdown = document.markdown!;
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 9c829baa4..ab9b68d41 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -261,30 +261,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =
runningJobs.add(job.id);
}
- if (job.data && job.data.sentry && Sentry.isInitialized()) {
- Sentry.continueTrace(
- {
- sentryTrace: job.data.sentry.trace,
- baggage: job.data.sentry.baggage,
- },
- () => {
- Sentry.startSpan(
- {
- name: "Index job",
- attributes: {
- job: job.id,
- worker: process.env.FLY_MACHINE_ID ?? worker.id,
- },
- },
- async () => {
- await jobProcessor(token, job);
- },
- );
- },
- );
- } else {
- await jobProcessor(token, job);
- }
+ await jobProcessor(token, job);
if (job.id) {
runningJobs.delete(job.id);
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index a6302ff5f..d6947659a 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -1,4 +1,4 @@
-import { createRedisConnection, getScrapeQueue } from "./queue-service";
+import { getScrapeQueue, getScrapeQueueEvents } from "./queue-service";
import { v4 as uuidv4 } from "uuid";
import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";
import * as Sentry from "@sentry/node";
@@ -19,6 +19,7 @@ import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs";
import { Document } from "../controllers/v1/types";
import { getCrawl } from "../lib/crawl-redis";
import { Logger } from "winston";
+import { Job } from "bullmq";
/**
* Checks if a job is a crawl or batch scrape based on its options
@@ -54,7 +55,7 @@ export async function _addScrapeJobToBullMQ(
options: any,
jobId: string,
jobPriority: number,
-) {
+): Promise<Job> {
if (
webScraperOptions &&
webScraperOptions.team_id
@@ -69,13 +70,11 @@ export async function _addScrapeJobToBullMQ(
}
}
- const conn = createRedisConnection();
- await getScrapeQueue(conn).add(jobId, webScraperOptions, {
+ return await getScrapeQueue().add(jobId, webScraperOptions, {
...options,
priority: jobPriority,
jobId,
});
- conn.disconnect();
}
async function addScrapeJobRaw(
@@ -84,7 +83,7 @@ async function addScrapeJobRaw(
jobId: string,
jobPriority: number,
directToBullMQ: boolean = false,
-) {
+): Promise<Job | null> {
let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;
let currentActiveConcurrency = 0;
let maxConcurrency = 0;
@@ -145,8 +144,9 @@ async function addScrapeJobRaw(
jobId,
jobPriority,
);
+ return null;
} else {
- await _addScrapeJobToBullMQ(webScraperOptions, options, jobId, jobPriority);
+ return await _addScrapeJobToBullMQ(webScraperOptions, options, jobId, jobPriority);
}
}
@@ -156,39 +156,8 @@ export async function addScrapeJob(
jobId: string = uuidv4(),
jobPriority: number = 10,
directToBullMQ: boolean = false,
-) {
- if (Sentry.isInitialized()) {
- const size = JSON.stringify(webScraperOptions).length;
- return await Sentry.startSpan(
- {
- name: "Add scrape job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": jobId,
- "messaging.destination.name": getScrapeQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await addScrapeJobRaw(
- {
- ...webScraperOptions,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- },
- options,
- jobId,
- jobPriority,
- directToBullMQ,
- );
- },
- );
- } else {
- await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority, directToBullMQ);
- }
+): Promise<Job | null> {
+ return await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority, directToBullMQ);
}
export async function addScrapeJobs(
@@ -322,33 +291,11 @@ export async function addScrapeJobs(
await Promise.all(
addToCQ.map(async (job) => {
const size = JSON.stringify(job.data).length;
- return await Sentry.startSpan(
- {
- name: "Add scrape job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": job.opts.jobId,
- "messaging.destination.name": getScrapeQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- const jobData = {
- ...job.data,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- };
-
- await _addScrapeJobToConcurrencyQueue(
- jobData,
- job.opts,
- job.opts.jobId,
- job.opts.priority,
- );
- },
+ await _addScrapeJobToConcurrencyQueue(
+ job.data,
+ job.opts,
+ job.opts.jobId,
+ job.opts.priority,
);
}),
);
@@ -356,88 +303,55 @@ export async function addScrapeJobs(
await Promise.all(
addToBull.map(async (job) => {
const size = JSON.stringify(job.data).length;
- return await Sentry.startSpan(
- {
- name: "Add scrape job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": job.opts.jobId,
- "messaging.destination.name": getScrapeQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await _addScrapeJobToBullMQ(
- {
- ...job.data,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- },
- job.opts,
- job.opts.jobId,
- job.opts.priority,
- );
- },
+ await _addScrapeJobToBullMQ(
+ job.data,
+ job.opts,
+ job.opts.jobId,
+ job.opts.priority,
);
}),
);
}
}
-export function waitForJob(
- jobId: string,
+export async function waitForJob(
+ _job: Job | string,
timeout: number,
logger: Logger = _logger,
): Promise<Document> {
- return new Promise(async (resolve, reject) => {
const start = Date.now();
- const conn = createRedisConnection();
- const queue = getScrapeQueue(conn);
- while (true) {
- logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });
- if (Date.now() >= start + timeout) {
- reject(new Error("Job wait "));
- break;
- } else {
- const state = await queue.getJobState(jobId);
- logger.debug("Job in state", { state, scrapeId: jobId, jobId });
- if (state === "completed") {
- let doc: Document;
- const job = (await queue.getJob(jobId))!;
- logger.debug("Got job");
- doc = job.returnvalue;
-
- if (!doc) {
- const docs = await getJobFromGCS(jobId);
- logger.debug("Got job from GCS");
- if (!docs || docs.length === 0) {
- throw new Error("Job not found in GCS");
- }
- doc = docs[0];
-
- if (job.data?.internalOptions?.zeroDataRetention) {
- await removeJobFromGCS(jobId);
- }
- }
-
- resolve(doc);
- break;
- } else if (state === "failed") {
- const job = await queue.getJob(jobId);
- if (job && job.failedReason !== "Concurrency limit hit") {
- reject(job.failedReason);
- break;
- }
- }
+ const queue = getScrapeQueue();
+ let job: Job | undefined = typeof _job == "string" ? await queue.getJob(_job) : _job;
+ while (job === undefined) {
+ logger.debug("Waiting for job to be created");
+ await new Promise(resolve => setTimeout(resolve, 500));
+ job = await queue.getJob(_job as string);
+ if (Date.now() - start > timeout) {
+ throw new Error("Job wait ");
}
+ }
+ let doc: Document = await Promise.race([
+ job.waitUntilFinished(getScrapeQueueEvents(), timeout - (Date.now() - start)),
+ new Promise((resolve, reject) => {
+ setTimeout(() => {
+ reject(new Error("Job wait "));
+ }, Math.max(0, timeout - (Date.now() - start)));
+ }),
+ ]);
+ logger.debug("Got job");
+
+ if (!doc) {
+ const docs = await getJobFromGCS(job.id!);
+ logger.debug("Got job from GCS");
+ if (!docs || docs.length === 0) {
+ throw new Error("Job not found in GCS");
+ }
+ doc = docs[0];
- await new Promise((resolve) => setTimeout(resolve, 750));
+ if (job.data?.internalOptions?.zeroDataRetention) {
+ await removeJobFromGCS(job.id!);
+ }
}
- conn.disconnect();
- return;
- });
+ return doc;
}
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 32ae88c49..4ba840a91 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -1,10 +1,11 @@
-import { Queue } from "bullmq";
+import { Queue, QueueEvents } from "bullmq";
import { logger } from "../lib/logger";
import IORedis from "ioredis";
export type QueueFunction = () => Queue<any, any, string, any, any, string>;
let scrapeQueue: Queue;
+let scrapeQueueEvents: QueueEvents;
let extractQueue: Queue;
let loggingQueue: Queue;
let indexQueue: Queue;
@@ -13,10 +14,15 @@ let generateLlmsTxtQueue: Queue;
let billingQueue: Queue;
let precrawlQueue: Queue;
-export function createRedisConnection(): IORedis {
- return new IORedis(process.env.REDIS_URL!, {
+export function createRedisConnection() {
+ const connection = new IORedis(process.env.REDIS_URL!, {
maxRetriesPerRequest: null,
});
+
+ connection.on("reconnecting", () => logger.warn("Redis reconnecting"));
+ connection.on("error", (err) => logger.warn("Redis error", { err }));
+
+ return connection;
}
export const redisConnection = createRedisConnection();
@@ -30,27 +36,31 @@ export const deepResearchQueueName = "{deepResearchQueue}";
export const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-export function getScrapeQueue(_redisConnection?: IORedis) {
- const _scrapeQueue = !scrapeQueue || _redisConnection !== undefined ? new Queue(scrapeQueueName, {
- connection: _redisConnection ?? redisConnection,
- defaultJobOptions: {
- removeOnComplete: {
- age: 3600, // 1 hour
- },
- removeOnFail: {
- age: 3600, // 1 hour
+export function getScrapeQueue() {
+ if (!scrapeQueue) {
+ scrapeQueue = new Queue(scrapeQueueName, {
+ connection: createRedisConnection(),
+ defaultJobOptions: {
+ removeOnComplete: {
+ age: 3600, // 1 hour
+ },
+ removeOnFail: {
+ age: 3600, // 1 hour
+ },
},
- },
- }) : undefined;
+ });
+ }
+ return scrapeQueue;
+}
- if (_redisConnection !== undefined) {
- return _scrapeQueue!;
- } else if (!scrapeQueue) {
- scrapeQueue = _scrapeQueue!;
- logger.info("Web scraper queue created");
+export function getScrapeQueueEvents() {
+ if (!scrapeQueueEvents) {
+ scrapeQueueEvents = new QueueEvents(scrapeQueueName, {
+ connection: createRedisConnection(),
+ });
}
- return scrapeQueue;
+ return scrapeQueueEvents;
}
export function getExtractQueue() {
@@ -66,7 +76,6 @@ export function getExtractQueue() {
},
},
});
- logger.info("Extraction queue created");
}
return extractQueue;
}
@@ -84,7 +93,6 @@ export function getGenerateLlmsTxtQueue() {
},
},
});
- logger.info("LLMs TXT generation queue created");
}
return generateLlmsTxtQueue;
}
@@ -102,7 +110,6 @@ export function getDeepResearchQueue() {
},
},
});
- logger.info("Deep research queue created");
}
return deepResearchQueue;
}
@@ -120,7 +127,6 @@ export function getBillingQueue() {
},
},
});
- logger.info("Billing queue created");
}
return billingQueue;
}
@@ -138,7 +144,6 @@ export function getPrecrawlQueue() {
},
},
});
- logger.info("Precrawl queue created");
}
return precrawlQueue;
}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 5e21d0a8f..e3fcdc9be 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -1,100 +1,55 @@
import "dotenv/config";
import "./sentry";
import * as Sentry from "@sentry/node";
-import { CustomError } from "../lib/custom-error";
import {
getScrapeQueue,
getExtractQueue,
getDeepResearchQueue,
redisConnection,
getGenerateLlmsTxtQueue,
- getBillingQueue,
scrapeQueueName,
createRedisConnection,
} from "./queue-service";
-import { startWebScraperPipeline } from "../main/runWebScraper";
-import { callWebhook } from "./webhook";
-import { logJob } from "./logging/log_job";
import { Job, Queue, QueueEvents } from "bullmq";
import { logger as _logger } from "../lib/logger";
import { Worker } from "bullmq";
import systemMonitor from "./system-monitor";
import { v4 as uuidv4 } from "uuid";
import {
- addCrawlJob,
addCrawlJobDone,
- addCrawlJobs,
- crawlToCrawler,
- finishCrawl,
- finishCrawlPre,
- unPreFinishCrawl,
finishCrawlKickoff,
- generateURLPermutations,
getCrawl,
- getCrawlJobs,
- getDoneJobsOrderedLength,
- lockURL,
- lockURLsIndividually,
normalizeURL,
- saveCrawl,
} from "../lib/crawl-redis";
import { StoredCrawl } from "../lib/crawl-redis";
-import { addScrapeJob, addScrapeJobs } from "./queue-jobs";
-import {
- addJobPriority,
- deleteJobPriority,
- getJobPriority,
-} from "../../src/lib/job-priority";
-import { getJobs } from "..//controllers/v1/crawl-status";
import { configDotenv } from "dotenv";
import { scrapeOptions } from "../controllers/v2/types";
import {
concurrentJobDone,
- pushConcurrencyLimitActiveJob,
} from "../lib/concurrency-limit";
-import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
-import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
-import { Document, TeamFlags } from "../controllers/v1/types";
import {
ExtractResult,
performExtraction,
} from "../lib/extract/extraction-service";
-import { supabase_service } from "../services/supabase";
-import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";
-import { saveExtract, updateExtract } from "../lib/extract/extract-redis";
-import { billTeam } from "./billing/credit_billing";
+import { updateExtract } from "../lib/extract/extract-redis";
import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";
import { performDeepResearch } from "../lib/deep-research/deep-research-service";
import { performGenerateLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-service";
import { updateGeneratedLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-redis";
import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";
-import { CostTracking } from "../lib/extract/extraction-service";
-import { getACUCTeam } from "../controllers/auth";
import Express from "express";
import http from "http";
import https from "https";
import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
-import { RateLimiterMode } from "../types";
-import { calculateCreditsToBeBilled } from "../lib/scrape-billing";
import { redisEvictConnection } from "./redis";
-import { generateURLSplits, queryIndexAtSplitLevel } from "./index";
-import { WebCrawler } from "../scraper/WebScraper/crawler";
-import type { Logger } from "winston";
+import path from "path";
+import { finishCrawlIfNeeded } from "./worker/crawl-logic";
configDotenv();
-class RacedRedirectError extends Error {
- constructor() {
- super("Raced redirect error");
- }
-}
-
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
-const workerStalledCheckInterval =
- Number(process.env.WORKER_STALLED_CHECK_INTERVAL) || 30000;
const jobLockExtendInterval =
Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;
const jobLockExtensionTime =
@@ -112,349 +67,6 @@ const runningJobs: Set<string> = new Set();
cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
-async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
- const logger = _logger.child({
- module: "queue-worker",
- method: "finishCrawlIfNeeded",
- jobId: job.id,
- scrapeId: job.id,
- crawlId: job.data.crawl_id,
- zeroDataRetention: sc.internalOptions.zeroDataRetention,
- });
-
- if (await finishCrawlPre(job.data.crawl_id, logger)) {
- logger.info("Crawl is pre-finished, checking if we need to add more jobs");
- if (
- job.data.crawlerOptions &&
- !(await redisEvictConnection.exists(
- "crawl:" + job.data.crawl_id + ":invisible_urls",
- ))
- ) {
- await redisEvictConnection.set(
- "crawl:" + job.data.crawl_id + ":invisible_urls",
- "done",
- "EX",
- 60 * 60 * 24,
- );
-
- const sc = (await getCrawl(job.data.crawl_id))!;
-
- const visitedUrls = new Set(
- await redisEvictConnection.smembers(
- "crawl:" + job.data.crawl_id + ":visited_unique",
- ),
- );
-
- logger.info("Visited URLs", {
- visitedUrls: visitedUrls.size,
- });
-
- let lastUrls: string[] = [];
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
- if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {
- lastUrls = (
- (
- await supabase_service.rpc("diff_get_last_crawl_urls", {
- i_team_id: job.data.team_id,
- i_url: sc.originUrl!,
- })
- ).data ?? []
- ).map((x) => x.url);
- }
-
- const lastUrlsSet = new Set(lastUrls);
-
- logger.info("Last URLs", {
- lastUrls: lastUrlsSet.size,
- });
-
- const crawler = crawlToCrawler(
- job.data.crawl_id,
- sc,
- (await getACUCTeam(job.data.team_id))?.flags ?? null,
- sc.originUrl!,
- job.data.crawlerOptions,
- );
-
- const univistedUrls = await crawler.filterLinks(
- Array.from(lastUrlsSet).filter((x) => !visitedUrls.has(x)),
- Infinity,
- sc.crawlerOptions.maxDepth ?? 10,
- );
-
- const addableJobCount =
- sc.crawlerOptions.limit === undefined
- ? Infinity
- : sc.crawlerOptions.limit -
- (await getDoneJobsOrderedLength(job.data.crawl_id));
-
- if (univistedUrls.links.length !== 0 && addableJobCount > 0) {
- logger.info("Adding jobs", {
- univistedUrls: univistedUrls.links.length,
- addableJobCount,
- });
-
- const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {
- const uuid = uuidv4();
- return {
- name: uuid,
- data: {
- url,
- mode: "single_urls" as const,
- team_id: job.data.team_id,
- crawlerOptions: {
- ...job.data.crawlerOptions,
- urlInvisibleInCurrentCrawl: true,
- },
- scrapeOptions: job.data.scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- sitemapped: true,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- opts: {
- jobId: uuid,
- priority: 20,
- },
- };
- });
-
- const lockedIds = await lockURLsIndividually(
- job.data.crawl_id,
- sc,
- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
- );
- const lockedJobs = jobs.filter((x) =>
- lockedIds.find((y) => y.id === x.opts.jobId),
- );
- await addCrawlJobs(
- job.data.crawl_id,
- lockedJobs.map((x) => x.opts.jobId),
- logger,
- );
- await addScrapeJobs(lockedJobs);
-
- if (lockedJobs.length > 0) {
- logger.info("Added jobs, not going for the full finish", {
- lockedJobs: lockedJobs.length,
- });
-
- await unPreFinishCrawl(job.data.crawl_id);
- return;
- } else {
- logger.info("No jobs added (all discovered URLs were locked), finishing crawl");
- }
- }
- }
-
- logger.info("Finishing crawl");
- await finishCrawl(job.data.crawl_id, logger);
-
- if (!job.data.v1) {
- const jobIDs = await getCrawlJobs(job.data.crawl_id);
-
- const conn = createRedisConnection();
- const jobs = (await getJobs(jobIDs, conn)).sort(
- (a, b) => a.timestamp - b.timestamp,
- );
- conn.disconnect();
- // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
- const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")
- ? "failed"
- : "completed";
-
- const fullDocs = jobs
- .map((x) =>
- x.returnvalue
- ? Array.isArray(x.returnvalue)
- ? x.returnvalue[0]
- : x.returnvalue
- : null,
- )
- .filter((x) => x !== null);
-
- await logJob({
- job_id: job.data.crawl_id,
- success: jobStatus === "completed",
- message: sc.cancelled ? "Cancelled" : undefined,
- num_docs: fullDocs.length,
- docs: [],
- time_taken: (Date.now() - sc.createdAt) / 1000,
- team_id: job.data.team_id,
- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
- url: sc.originUrl!,
- scrapeOptions: sc.scrapeOptions,
- crawlerOptions: sc.crawlerOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- zeroDataRetention: job.data.zeroDataRetention,
- }, false, job.data.internalOptions?.bypassBilling ?? false);
-
- const data = {
- success: jobStatus !== "failed",
- result: {
- links: fullDocs.map((doc) => {
- return {
- content: doc,
- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
- };
- }),
- },
- project_id: job.data.project_id,
- docs: fullDocs,
- };
-
- // v0 web hooks, call when done with all the data
- if (!job.data.v1) {
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id,
- data,
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: job.data.crawlerOptions !== null
- ? "crawl.completed"
- : "batch_scrape.completed",
- });
- }
- } else {
- const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);
- const jobStatus = sc.cancelled ? "failed" : "completed";
-
- let credits_billed = null;
-
- if (process.env.USE_DB_AUTHENTICATION === "true") {
- const creditsRpc = await supabase_service
- .rpc("credits_billed_by_crawl_id_1", {
- i_crawl_id: job.data.crawl_id,
- });
-
- credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;
-
- if (credits_billed === null) {
- logger.warn("Credits billed is null", {
- error: creditsRpc.error,
- });
- }
- }
-
- await logJob(
- {
- job_id: job.data.crawl_id,
- success: jobStatus === "completed",
- message: sc.cancelled ? "Cancelled" : undefined,
- num_docs,
- docs: [],
- time_taken: (Date.now() - sc.createdAt) / 1000,
- team_id: job.data.team_id,
- scrapeOptions: sc.scrapeOptions,
- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
- url:
- sc?.originUrl ??
- (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
- crawlerOptions: sc.crawlerOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- credits_billed,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- true,
- job.data.internalOptions?.bypassBilling ?? false,
- );
-
-
- // v1 web hooks, call when done with no data, but with event completed
- if (job.data.v1 && job.data.webhook) {
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id,
- data: [],
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: job.data.crawlerOptions !== null
- ? "crawl.completed"
- : "batch_scrape.completed",
- });
- }
- }
- }
-}
-
-const processJobInternal = async (token: string, job: Job & { id: string }) => {
- const logger = _logger.child({
- module: "queue-worker",
- method: "processJobInternal",
- jobId: job.id,
- scrapeId: job.id,
- crawlId: job.data?.crawl_id ?? undefined,
- zeroDataRetention: job.data?.zeroDataRetention ?? false,
- });
-
- const extendLockInterval = setInterval(async () => {
- logger.info(`🐂 Worker extending lock on job ${job.id}`, {
- extendInterval: jobLockExtendInterval,
- extensionTime: jobLockExtensionTime,
- });
-
- if (job.data?.mode !== "kickoff" && job.data?.team_id) {
- await pushConcurrencyLimitActiveJob(job.data.team_id, job.id, 60 * 1000); // 60s lock renew, just like in the queue
- }
-
- await job.extendLock(token, jobLockExtensionTime);
- }, jobLockExtendInterval);
-
- await addJobPriority(job.data.team_id, job.id);
- let err = null;
- try {
- if (job.data?.mode === "kickoff") {
- const result = await processKickoffJob(job, token);
- if (result.success) {
- try {
- await job.moveToCompleted(null, token, false);
- } catch (e) {}
- } else {
- logger.debug("Job failed", { result, mode: job.data.mode });
- await job.moveToFailed((result as any).error, token, false);
- }
- } else {
- const result = await processJob(job, token);
- if (result.success) {
- try {
- if (
- process.env.USE_DB_AUTHENTICATION === "true" &&
- (job.data.crawl_id || process.env.GCS_BUCKET_NAME)
- ) {
- logger.debug(
- "Job succeeded -- putting null in Redis",
- );
- await job.moveToCompleted(null, token, false);
- } else {
- logger.debug("Job succeeded -- putting result in Redis");
- await job.moveToCompleted(result.document, token, false);
- }
- } catch (e) {}
- } else {
- logger.debug("Job failed", { result });
- await job.moveToFailed((result as any).error, token, false);
- }
- }
- } catch (error) {
- logger.debug("Job failed", { error });
- Sentry.captureException(error);
- err = error;
- await job.moveToFailed(error, token, false);
- } finally {
- await deleteJobPriority(job.data.team_id, job.id);
- clearInterval(extendLockInterval);
- }
-
- return err;
-};
-
const processExtractJobInternal = async (
token: string,
job: Job & { id: string },
@@ -717,6 +329,22 @@ process.on("SIGTERM", () => {
let cantAcceptConnectionCount = 0;
+const separateWorkerFun = (
+ queue: Queue,
+ path: string,
+): Worker => {
+ const worker = new Worker(queue.name, path, {
+ connection: createRedisConnection(),
+ lockDuration: 30 * 1000, // 30 seconds
+ stalledInterval: 30 * 1000, // 30 seconds
+ maxStalledCount: 10, // 10 times
+ concurrency: 6, // from k8s setup
+ useWorkerThreads: true,
+ });
+
+ return worker;
+}
+
const workerFun = async (
queue: Queue,
processJobInternal: (token: string, job: Job) => Promise<any>,
@@ -782,68 +410,7 @@ const workerFun = async (
}
}
- if (job.data && job.data.sentry && Sentry.isInitialized()) {
- Sentry.continueTrace(
- {
- sentryTrace: job.data.sentry.trace,
- baggage: job.data.sentry.baggage,
- },
- () => {
- Sentry.startSpan(
- {
- name: "Scrape job",
- attributes: {
- job: job.id,
- worker: process.env.FLY_MACHINE_ID ?? worker.id,
- },
- },
- async (span) => {
- await Sentry.startSpan(
- {
- name: "Process scrape job",
- op: "queue.process",
- attributes: {
- "messaging.message.id": job.id,
- "messaging.destination.name": scrapeQueueName,
- "messaging.message.body.size": job.data.sentry.size,
- "messaging.message.receive.latency":
- Date.now() - (job.processedOn ?? job.timestamp),
- "messaging.message.retry.count": job.attemptsMade,
- },
- },
- async () => {
- let res;
- try {
- res = await processJobInternal(token, job);
- } finally {
- await afterJobDone(job);
- }
-
- if (res !== null) {
- span.setStatus({ code: 2 }); // ERROR
- } else {
- span.setStatus({ code: 1 }); // OK
- }
- },
- );
- },
- );
- },
- );
- } else {
- Sentry.startSpan(
- {
- name: "Scrape job",
- attributes: {
- job: job.id,
- worker: process.env.FLY_MACHINE_ID ?? worker.id,
- },
- },
- () => {
- processJobInternal(token, job).finally(() => afterJobDone(job));
- },
- );
- }
+ processJobInternal(token, job).finally(() => afterJobDone(job));
await sleep(gotJobInterval);
} else {
@@ -852,740 +419,6 @@ const workerFun = async (
}
};
-async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: string) {
- if (sc.crawlerOptions.ignoreSitemap) {
- return [];
- }
-
- const trimmedURL = new URL(url);
- trimmedURL.search = "";
-
- const index = await queryIndexAtSplitLevel(
- sc.crawlerOptions.allowBackwardCrawling ? generateURLSplits(trimmedURL.href)[0] : trimmedURL.href,
- sc.crawlerOptions.limit ?? 10000,
- );
-
- const validIndexLinksResult = await crawler.filterLinks(
- index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),
- sc.crawlerOptions.limit ?? 10000,
- sc.crawlerOptions.maxDepth ?? 10,
- false,
- );
- const validIndexLinks = validIndexLinksResult.links;
-
- return validIndexLinks;
-}
-
-async function processKickoffJob(job: Job & { id: string }, token: string) {
- const logger = _logger.child({
- module: "queue-worker",
- method: "processKickoffJob",
- jobId: job.id,
- scrapeId: job.id,
- crawlId: job.data?.crawl_id ?? undefined,
- teamId: job.data?.team_id ?? undefined,
- zeroDataRetention: job.data.zeroDataRetention ?? false,
- });
-
- try {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
- const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
-
- logger.debug("Locking URL...");
- await lockURL(job.data.crawl_id, sc, job.data.url);
- const jobId = uuidv4();
- logger.debug("Adding scrape job to Redis...", { jobId });
- await addScrapeJob(
- {
- url: job.data.url,
- mode: "single_urls",
- team_id: job.data.team_id,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- webhook: job.data.webhook,
- v1: job.data.v1,
- isCrawlSourceScrape: true,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- {
- priority: 15,
- },
- jobId,
- );
- logger.debug("Adding scrape job to BullMQ...", { jobId });
- await addCrawlJob(job.data.crawl_id, jobId, logger);
-
- if (job.data.webhook) {
- logger.debug("Calling webhook with crawl.started...", {
- webhook: job.data.webhook,
- });
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id,
- data: null,
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: "crawl.started",
- });
- }
-
- const sitemap = sc.crawlerOptions.ignoreSitemap
- ? 0
- : await crawler.tryGetSitemap(async (urls) => {
- if (urls.length === 0) return;
-
- logger.debug("Using sitemap chunk of length " + urls.length, {
- sitemapLength: urls.length,
- });
-
- let jobPriority = await getJobPriority({
- team_id: job.data.team_id,
- basePriority: 21,
- });
- logger.debug("Using job priority " + jobPriority, { jobPriority });
-
- const jobs = urls.map((url) => {
- const uuid = uuidv4();
- return {
- name: uuid,
- data: {
- url,
- mode: "single_urls" as const,
- team_id: job.data.team_id,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- sitemapped: true,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- opts: {
- jobId: uuid,
- priority: 20,
- },
- };
- });
-
- logger.debug("Locking URLs...");
- const lockedIds = await lockURLsIndividually(
- job.data.crawl_id,
- sc,
- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
- );
- const lockedJobs = jobs.filter((x) =>
- lockedIds.find((y) => y.id === x.opts.jobId),
- );
- logger.debug("Adding scrape jobs to Redis...");
- await addCrawlJobs(
- job.data.crawl_id,
- lockedJobs.map((x) => x.opts.jobId),
- logger,
- );
- logger.debug("Adding scrape jobs to BullMQ...");
- await addScrapeJobs(lockedJobs);
- });
-
- if (sitemap === 0) {
- logger.debug("Sitemap not found or ignored.", {
- ignoreSitemap: sc.crawlerOptions.ignoreSitemap,
- });
- }
-
- const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);
-
- if (indexLinks.length > 0) {
- logger.debug("Using index links of length " + indexLinks.length, {
- indexLinksLength: indexLinks.length,
- });
-
- let jobPriority = await getJobPriority({
- team_id: job.data.team_id,
- basePriority: 21,
- });
- logger.debug("Using job priority " + jobPriority, { jobPriority });
-
- const jobs = indexLinks.map((url) => {
- const uuid = uuidv4();
- return {
- name: uuid,
- data: {
- url,
- mode: "single_urls" as const,
- team_id: job.data.team_id,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- sitemapped: true,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- opts: {
- jobId: uuid,
- priority: 20,
- },
- };
- });
-
- logger.debug("Locking URLs...");
- const lockedIds = await lockURLsIndividually(
- job.data.crawl_id,
- sc,
- jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
- );
- const lockedJobs = jobs.filter((x) =>
- lockedIds.find((y) => y.id === x.opts.jobId),
- );
- logger.debug("Adding scrape jobs to Redis...");
- await addCrawlJobs(
- job.data.crawl_id,
- lockedJobs.map((x) => x.opts.jobId),
- logger,
- );
- logger.debug("Adding scrape jobs to BullMQ...");
- await addScrapeJobs(lockedJobs);
- }
-
- logger.debug("Done queueing jobs!");
-
- await finishCrawlKickoff(job.data.crawl_id);
- await finishCrawlIfNeeded(job, sc);
-
- return { success: true };
- } catch (error) {
- logger.error("An error occurred!", { error });
- await finishCrawlKickoff(job.data.crawl_id);
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
- if (sc) {
- await finishCrawlIfNeeded(job, sc);
- }
- return { success: false, error };
- }
-}
-
-async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) {
- let creditsToBeBilled: number | null = null;
-
- if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {
- creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags);
-
- if (
- job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&
- process.env.USE_DB_AUTHENTICATION === "true"
- ) {
- try {
- const billingJobId = uuidv4();
- logger.debug(
- `Adding billing job to queue for team ${job.data.team_id}`,
- {
- billingJobId,
- credits: creditsToBeBilled,
- is_extract: false,
- },
- );
-
- // Add directly to the billing queue - the billing worker will handle the rest
- await getBillingQueue().add(
- "bill_team",
- {
- team_id: job.data.team_id,
- subscription_id: undefined,
- credits: creditsToBeBilled,
- is_extract: false,
- timestamp: new Date().toISOString(),
- originating_job_id: job.id,
- },
- {
- jobId: billingJobId,
- priority: 10,
- },
- );
- return creditsToBeBilled;
- } catch (error) {
- logger.error(
- `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
- { error },
- );
- Sentry.captureException(error);
- return creditsToBeBilled;
- }
- }
- }
-
- return creditsToBeBilled;
-}
-
-async function processJob(job: Job & { id: string }, token: string) {
- const logger = _logger.child({
- module: "queue-worker",
- method: "processJob",
- jobId: job.id,
- scrapeId: job.id,
- crawlId: job.data?.crawl_id ?? undefined,
- teamId: job.data?.team_id ?? undefined,
- zeroDataRetention: job.data?.zeroDataRetention ?? false,
- });
- logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });
- const start = job.data.startTime ?? Date.now();
- const remainingTime = job.data.scrapeOptions.timeout ? (job.data.scrapeOptions.timeout - (Date.now() - start)) : undefined;
-
- const costTracking = new CostTracking();
-
- try {
- job.updateProgress({
- current: 1,
- total: 100,
- current_step: "SCRAPING",
- current_url: "",
- });
-
- if (remainingTime !== undefined && remainingTime < 0) {
- throw new Error("timeout");
- }
- const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;
-
- if (job.data.crawl_id) {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
- if (sc && sc.cancelled) {
- throw new Error("Parent crawl/batch scrape was cancelled");
- }
- }
-
-
- const pipeline = await Promise.race([
- startWebScraperPipeline({
- job,
- token,
- costTracking,
- }),
- ...(remainingTime !== undefined
- ? [
- (async () => {
- await sleep(remainingTime);
- throw new Error("timeout");
- })(),
- ]
- : []),
- ]);
-
- try {
- signal?.throwIfAborted();
- } catch (e) {
- throw new Error("timeout");
- }
-
- if (!pipeline.success) {
- throw pipeline.error;
- }
-
- const end = Date.now();
- const timeTakenInSeconds = (end - start) / 1000;
-
- const doc = pipeline.document;
-
- const rawHtml = doc.rawHtml ?? "";
-
- if (!job.data.scrapeOptions.formats.includes("rawHtml")) {
- delete doc.rawHtml;
- }
-
- if (job.data.concurrencyLimited) {
- doc.warning =
- "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." +
- (doc.warning ? " " + doc.warning : "");
- }
-
- const data = {
- success: true,
- result: {
- links: [
- {
- content: doc,
- source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",
- id: job.id,
- },
- ],
- },
- project_id: job.data.project_id,
- document: doc,
- };
-
- if (job.data.crawl_id) {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
-
- if (
- doc.metadata.url !== undefined &&
- doc.metadata.sourceURL !== undefined &&
- normalizeURL(doc.metadata.url, sc) !==
- normalizeURL(doc.metadata.sourceURL, sc) &&
- job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
- ) {
- const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
- const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);
- if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {
- const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";
- throw new Error(reason);
- }
-
- // Only re-set originUrl if it's different from the current hostname
- // This is only done on this condition to handle cross-domain redirects
- // If this would be done for non-crossdomain redirects, but also for e.g.
- // redirecting / -> /introduction (like our docs site does), it would
- // break crawling the entire site without allowBackwardsCrawling - mogery
- const isHostnameDifferent =
- normalizeUrlOnlyHostname(doc.metadata.url) !==
- normalizeUrlOnlyHostname(doc.metadata.sourceURL);
- if (job.data.isCrawlSourceScrape && isHostnameDifferent) {
- // TODO: re-fetch sitemap for redirect target domain
- sc.originUrl = doc.metadata.url;
- await saveCrawl(job.data.crawl_id, sc);
- }
-
- if (isUrlBlocked(doc.metadata.url, (await getACUCTeam(job.data.team_id))?.flags ?? null)) {
- throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
- }
-
- const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));
- const p2 = generateURLPermutations(
- normalizeURL(doc.metadata.sourceURL, sc),
- );
-
- if (JSON.stringify(p1) !== JSON.stringify(p2)) {
- logger.debug(
- "Was redirected, removing old URL and locking new URL...",
- { oldUrl: doc.metadata.sourceURL, newUrl: doc.metadata.url },
- );
-
- // Prevent redirect target from being visited in the crawl again
- // See lockURL
- const x = await redisEvictConnection.sadd(
- "crawl:" + job.data.crawl_id + ":visited",
- ...p1.map((x) => x.href),
- );
- const lockRes = x === p1.length;
-
- if (job.data.crawlerOptions !== null && !lockRes) {
- throw new RacedRedirectError();
- }
- }
- }
-
- if (job.data.crawlerOptions !== null) {
- if (!sc.cancelled) {
- const crawler = crawlToCrawler(
- job.data.crawl_id,
- sc,
- (await getACUCTeam(job.data.team_id))?.flags ?? null,
- doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,
- job.data.crawlerOptions,
- );
-
- const links = await crawler.filterLinks(
- await crawler.extractLinksFromHTML(
- rawHtml ?? "",
- doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,
- ),
- Infinity,
- sc.crawlerOptions?.maxDepth ?? 10,
- );
- logger.debug("Discovered " + links.links.length + " links...", {
- linksLength: links.links.length,
- });
-
- for (const link of links.links) {
- if (await lockURL(job.data.crawl_id, sc, link)) {
- // This seems to work really welel
- const jobPriority = await getJobPriority({
- team_id: sc.team_id,
- basePriority: job.data.crawl_id ? 20 : 10,
- });
- const jobId = uuidv4();
-
- logger.debug(
- "Determined job priority " +
- jobPriority +
- " for URL " +
- JSON.stringify(link),
- { jobPriority, url: link },
- );
-
- // console.log("team_id: ", sc.team_id)
- // console.log("base priority: ", job.data.crawl_id ? 20 : 10)
- // console.log("job priority: " , jobPriority, "\n\n\n")
-
- await addScrapeJob(
- {
- url: link,
- mode: "single_urls",
- team_id: sc.team_id,
- scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),
- internalOptions: sc.internalOptions,
- crawlerOptions: {
- ...sc.crawlerOptions,
- currentDiscoveryDepth:
- (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,
- },
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- {},
- jobId,
- jobPriority,
- );
-
- await addCrawlJob(job.data.crawl_id, jobId, logger);
- logger.debug("Added job for URL " + JSON.stringify(link), {
- jobPriority,
- url: link,
- newJobId: jobId,
- });
- } else {
- // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!
- // logger.debug("Could not lock URL " + JSON.stringify(link), {
- // url: link,
- // });
- }
- }
-
- // Only run check after adding new jobs for discovery - mogery
- if (job.data.isCrawlSourceScrape) {
- const filterResult = await crawler.filterLinks(
- [doc.metadata.url ?? doc.metadata.sourceURL!],
- 1,
- sc.crawlerOptions?.maxDepth ?? 10,
- );
- if (filterResult.links.length === 0) {
- const url = doc.metadata.url ?? doc.metadata.sourceURL!;
- const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";
- throw new Error(reason);
- }
- }
- }
- }
-
- try {
- signal?.throwIfAborted();
- } catch (e) {
- throw new Error("timeout");
- }
-
- const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
-
- doc.metadata.creditsUsed = credits_billed ?? undefined;
-
- logger.debug("Logging job to DB...");
- await logJob(
- {
- job_id: job.id as string,
- success: true,
- num_docs: 1,
- docs: [doc],
- time_taken: timeTakenInSeconds,
- team_id: job.data.team_id,
- mode: job.data.mode,
- url: job.data.url,
- crawlerOptions: sc.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- cost_tracking: costTracking,
- pdf_num_pages: doc.metadata.numPages,
- credits_billed,
- change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- true,
- job.data.internalOptions?.bypassBilling ?? false,
- );
-
- if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {
- logger.debug("Calling webhook with success...", {
- webhook: job.data.webhook,
- });
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id,
- scrapeId: job.id,
- data,
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
- });
- }
-
- logger.debug("Declaring job as done...");
- await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
-
- await finishCrawlIfNeeded(job, sc);
- } else {
- try {
- signal?.throwIfAborted();
- } catch (e) {
- throw new Error("timeout");
- }
-
- const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
-
- doc.metadata.creditsUsed = credits_billed ?? undefined;
-
- await logJob({
- job_id: job.id,
- success: true,
- message: "Scrape completed",
- num_docs: 1,
- docs: [doc],
- time_taken: timeTakenInSeconds,
- team_id: job.data.team_id,
- mode: "scrape",
- url: job.data.url,
- scrapeOptions: job.data.scrapeOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- num_tokens: 0, // TODO: fix
- cost_tracking: costTracking,
- pdf_num_pages: doc.metadata.numPages,
- credits_billed,
- change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,
- zeroDataRetention: job.data.zeroDataRetention,
- }, false, job.data.internalOptions?.bypassBilling ?? false);
- }
-
- logger.info(`🐂 Job done ${job.id}`);
- return data;
- } catch (error) {
- if (job.data.crawl_id) {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
-
- logger.debug("Declaring job as done...");
- await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);
- await redisEvictConnection.srem(
- "crawl:" + job.data.crawl_id + ":visited_unique",
- normalizeURL(job.data.url, sc),
- );
-
- await finishCrawlIfNeeded(job, sc);
- }
-
- const isEarlyTimeout =
- error instanceof Error && error.message === "timeout";
- const isCancelled =
- error instanceof Error &&
- error.message === "Parent crawl/batch scrape was cancelled";
-
- if (isEarlyTimeout) {
- logger.error(`🐂 Job timed out ${job.id}`);
- } else if (error instanceof RacedRedirectError) {
- logger.warn(`🐂 Job got redirect raced ${job.id}, silently failing`);
- } else if (isCancelled) {
- logger.warn(`🐂 Job got cancelled, silently failing`);
- } else {
- logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });
-
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
-
- if (error instanceof CustomError) {
- // Here we handle the error, then save the failed job
- logger.error(error.message); // or any other error handling
- }
- logger.error(error);
- if (error.stack) {
- logger.error(error.stack);
- }
- }
-
- const data = {
- success: false,
- document: null,
- project_id: job.data.project_id,
- error:
- error instanceof Error
- ? error
- : typeof error === "string"
- ? new Error(error)
- : new Error(JSON.stringify(error)),
- };
-
- if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {
- callWebhook({
- teamId: job.data.team_id,
- crawlId: job.data.crawl_id ?? (job.id as string),
- scrapeId: job.id,
- data,
- webhook: job.data.webhook,
- v1: job.data.v1,
- eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
- });
- }
-
- const end = Date.now();
- const timeTakenInSeconds = (end - start) / 1000;
-
- const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
-
- logger.debug("Logging job to DB...");
- await logJob(
- {
- job_id: job.id as string,
- success: false,
- message:
- typeof error === "string"
- ? error
- : (error.message ??
- "Something went wrong... Contact [email protected]"),
- num_docs: 0,
- docs: [],
- time_taken: timeTakenInSeconds,
- team_id: job.data.team_id,
- mode: job.data.mode,
- url: job.data.url,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- cost_tracking: costTracking,
- credits_billed,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- true,
- job.data.internalOptions?.bypassBilling ?? false,
- );
- return data;
- }
-}
-
-// wsq.process(
-// Math.floor(Number(process.env.NUM_WORKERS_PER_QUEUE ?? 8)),
-// processJob
-// );
-
-// wsq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));
-// wsq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));
-// wsq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));
-// wsq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));
-// wsq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));
-// wsq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));
-
// Start all workers
const app = Express();
@@ -1639,9 +472,7 @@ app.listen(workerPort, () => {
return;
}
- const conn = createRedisConnection();
- const job = await getScrapeQueue(conn).getJob(args.jobId);
- conn.disconnect();
+ const job = await getScrapeQueue().getJob(args.jobId);
let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener", zeroDataRetention: job?.data.zeroDataRetention });
if (job && job.data.crawl_id) {
@@ -1676,20 +507,30 @@ app.listen(workerPort, () => {
const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });
scrapeQueueEvents.on("failed", failedListener);
- const conn = createRedisConnection();
- await Promise.all([
- workerFun(getScrapeQueue(conn), processJobInternal),
+ const results = await Promise.all([
+ separateWorkerFun(getScrapeQueue(), path.join(__dirname, "worker", "scrape-worker.js")),
workerFun(getExtractQueue(), processExtractJobInternal),
workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),
workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),
]);
- conn.disconnect();
+
console.log("All workers exited. Waiting for all jobs to finish...");
+ const workerResults = results.filter(x => x instanceof Worker);
+ await Promise.all(workerResults.map(x => x.close()));
+
while (runningJobs.size > 0) {
await new Promise((resolve) => setTimeout(resolve, 500));
}
+ setInterval(async () => {
+ _logger.debug("Currently running jobs", {
+ jobs: (await Promise.all([...runningJobs].map(async (jobId) => {
+ return await getScrapeQueue().getJob(jobId);
+ }))).filter(x => x && !x.data?.zeroDataRetention),
+ });
+ }, 1000);
+
await scrapeQueueEvents.close();
console.log("All jobs finished. Worker out!");
process.exit(0);
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
new file mode 100644
index 000000000..9a3d568b3
--- /dev/null
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -0,0 +1,284 @@
+import { logger as _logger } from "../../lib/logger";
+import { Job } from "bullmq";
+import { addCrawlJobs, finishCrawl, getCrawlJobs, getDoneJobsOrderedLength, lockURLsIndividually, StoredCrawl, unPreFinishCrawl } from "../../lib/crawl-redis";
+import { redisEvictConnection } from "../redis";
+import { getCrawl } from "../../lib/crawl-redis";
+import { finishCrawlPre } from "../../lib/crawl-redis";
+import { getACUCTeam } from "../../controllers/auth";
+import { crawlToCrawler } from "../../lib/crawl-redis";
+import { supabase_service } from "../supabase";
+import { v4 as uuidv4 } from "uuid";
+import { addScrapeJobs } from "../queue-jobs";
+import { getJobs } from "../../controllers/v1/crawl-status";
+import { logJob } from "../logging/log_job";
+import { callWebhook } from "../webhook";
+
+export async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "finishCrawlIfNeeded",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data.crawl_id,
+ zeroDataRetention: sc.internalOptions.zeroDataRetention,
+ });
+
+ if (await finishCrawlPre(job.data.crawl_id, logger)) {
+ logger.info("Crawl is pre-finished, checking if we need to add more jobs");
+ if (
+ job.data.crawlerOptions &&
+ !(await redisEvictConnection.exists(
+ "crawl:" + job.data.crawl_id + ":invisible_urls",
+ ))
+ ) {
+ await redisEvictConnection.set(
+ "crawl:" + job.data.crawl_id + ":invisible_urls",
+ "done",
+ "EX",
+ 60 * 60 * 24,
+ );
+
+ const sc = (await getCrawl(job.data.crawl_id))!;
+
+ const visitedUrls = new Set(
+ await redisEvictConnection.smembers(
+ "crawl:" + job.data.crawl_id + ":visited_unique",
+ ),
+ );
+
+ logger.info("Visited URLs", {
+ visitedUrls: visitedUrls.size,
+ });
+
+ let lastUrls: string[] = [];
+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {
+ lastUrls = (
+ (
+ await supabase_service.rpc("diff_get_last_crawl_urls", {
+ i_team_id: job.data.team_id,
+ i_url: sc.originUrl!,
+ })
+ ).data ?? []
+ ).map((x) => x.url);
+ }
+
+ const lastUrlsSet = new Set(lastUrls);
+
+ logger.info("Last URLs", {
+ lastUrls: lastUrlsSet.size,
+ });
+
+ const crawler = crawlToCrawler(
+ job.data.crawl_id,
+ sc,
+ (await getACUCTeam(job.data.team_id))?.flags ?? null,
+ sc.originUrl!,
+ job.data.crawlerOptions,
+ );
+
+ const univistedUrls = await crawler.filterLinks(
+ Array.from(lastUrlsSet).filter((x) => !visitedUrls.has(x)),
+ Infinity,
+ sc.crawlerOptions.maxDepth ?? 10,
+ );
+
+ const addableJobCount =
+ sc.crawlerOptions.limit === undefined
+ ? Infinity
+ : sc.crawlerOptions.limit -
+ (await getDoneJobsOrderedLength(job.data.crawl_id));
+
+ if (univistedUrls.links.length !== 0 && addableJobCount > 0) {
+ logger.info("Adding jobs", {
+ univistedUrls: univistedUrls.links.length,
+ addableJobCount,
+ });
+
+ const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {
+ const uuid = uuidv4();
+ return {
+ name: uuid,
+ data: {
+ url,
+ mode: "single_urls" as const,
+ team_id: job.data.team_id,
+ crawlerOptions: {
+ ...job.data.crawlerOptions,
+ urlInvisibleInCurrentCrawl: true,
+ },
+ scrapeOptions: job.data.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ sitemapped: true,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ opts: {
+ jobId: uuid,
+ priority: 20,
+ },
+ };
+ });
+
+ const lockedIds = await lockURLsIndividually(
+ job.data.crawl_id,
+ sc,
+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
+ );
+ const lockedJobs = jobs.filter((x) =>
+ lockedIds.find((y) => y.id === x.opts.jobId),
+ );
+ await addCrawlJobs(
+ job.data.crawl_id,
+ lockedJobs.map((x) => x.opts.jobId),
+ logger,
+ );
+ await addScrapeJobs(lockedJobs);
+
+ if (lockedJobs.length > 0) {
+ logger.info("Added jobs, not going for the full finish", {
+ lockedJobs: lockedJobs.length,
+ });
+
+ await unPreFinishCrawl(job.data.crawl_id);
+ return;
+ } else {
+ logger.info("No jobs added (all discovered URLs were locked), finishing crawl");
+ }
+ }
+ }
+
+ logger.info("Finishing crawl");
+ await finishCrawl(job.data.crawl_id, logger);
+
+ if (!job.data.v1) {
+ const jobIDs = await getCrawlJobs(job.data.crawl_id);
+
+ const jobs = (await getJobs(jobIDs)).sort(
+ (a, b) => a.timestamp - b.timestamp,
+ );
+ // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
+ const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")
+ ? "failed"
+ : "completed";
+
+ const fullDocs = jobs
+ .map((x) =>
+ x.returnvalue
+ ? Array.isArray(x.returnvalue)
+ ? x.returnvalue[0]
+ : x.returnvalue
+ : null,
+ )
+ .filter((x) => x !== null);
+
+ await logJob({
+ job_id: job.data.crawl_id,
+ success: jobStatus === "completed",
+ message: sc.cancelled ? "Cancelled" : undefined,
+ num_docs: fullDocs.length,
+ docs: [],
+ time_taken: (Date.now() - sc.createdAt) / 1000,
+ team_id: job.data.team_id,
+ mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ url: sc.originUrl!,
+ scrapeOptions: sc.scrapeOptions,
+ crawlerOptions: sc.crawlerOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ zeroDataRetention: job.data.zeroDataRetention,
+ }, false, job.data.internalOptions?.bypassBilling ?? false);
+
+ const data = {
+ success: jobStatus !== "failed",
+ result: {
+ links: fullDocs.map((doc) => {
+ return {
+ content: doc,
+ source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
+ };
+ }),
+ },
+ project_id: job.data.project_id,
+ docs: fullDocs,
+ };
+
+ // v0 web hooks, call when done with all the data
+ if (!job.data.v1) {
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id,
+ data,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: job.data.crawlerOptions !== null
+ ? "crawl.completed"
+ : "batch_scrape.completed",
+ });
+ }
+ } else {
+ const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);
+ const jobStatus = sc.cancelled ? "failed" : "completed";
+
+ let credits_billed = null;
+
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const creditsRpc = await supabase_service
+ .rpc("credits_billed_by_crawl_id_1", {
+ i_crawl_id: job.data.crawl_id,
+ });
+
+ credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;
+
+ if (credits_billed === null) {
+ logger.warn("Credits billed is null", {
+ error: creditsRpc.error,
+ });
+ }
+ }
+
+ await logJob(
+ {
+ job_id: job.data.crawl_id,
+ success: jobStatus === "completed",
+ message: sc.cancelled ? "Cancelled" : undefined,
+ num_docs,
+ docs: [],
+ time_taken: (Date.now() - sc.createdAt) / 1000,
+ team_id: job.data.team_id,
+ scrapeOptions: sc.scrapeOptions,
+ mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ url:
+ sc?.originUrl ??
+ (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
+ crawlerOptions: sc.crawlerOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ credits_billed,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ true,
+ job.data.internalOptions?.bypassBilling ?? false,
+ );
+
+
+ // v1 web hooks, call when done with no data, but with event completed
+ if (job.data.v1 && job.data.webhook) {
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id,
+ data: [],
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: job.data.crawlerOptions !== null
+ ? "crawl.completed"
+ : "batch_scrape.completed",
+ });
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
new file mode 100644
index 000000000..a80bc4a58
--- /dev/null
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -0,0 +1,849 @@
+import { configDotenv } from "dotenv";
+import { Job } from "bullmq";
+import * as Sentry from "@sentry/node";
+import http from "http";
+import https from "https";
+
+import { logger as _logger } from "../../lib/logger";
+import {
+ concurrentJobDone,
+ pushConcurrencyLimitActiveJob,
+} from "../../lib/concurrency-limit";
+import { addJobPriority, deleteJobPriority } from "../../lib/job-priority";
+import { cacheableLookup } from "../../scraper/scrapeURL/lib/cacheableLookup";
+import { v4 as uuidv4 } from "uuid";
+import {
+ addCrawlJob,
+ addCrawlJobs,
+ addCrawlJobDone,
+ crawlToCrawler,
+ finishCrawlKickoff,
+ generateURLPermutations,
+ getCrawl,
+ lockURL,
+ lockURLsIndividually,
+ normalizeURL,
+ saveCrawl,
+ StoredCrawl
+} from "../../lib/crawl-redis";
+import { addScrapeJob, addScrapeJobs } from "../queue-jobs";
+import { getJobPriority } from "../../lib/job-priority";
+import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";
+import { getACUCTeam } from "../../controllers/auth";
+import { callWebhook } from "../webhook";
+import { CustomError } from "../../lib/custom-error";
+import { startWebScraperPipeline } from "../../main/runWebScraper";
+import { CostTracking } from "../../lib/extract/extraction-service";
+import { redisEvictConnection } from "../redis";
+import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";
+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { logJob } from "../logging/log_job";
+import { generateURLSplits, queryIndexAtSplitLevel } from "../index";
+import { WebCrawler } from "../../scraper/WebScraper/crawler";
+import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
+import { getBillingQueue } from "../queue-service";
+import type { Logger } from "winston";
+import { finishCrawlIfNeeded } from "./crawl-logic";
+
+class RacedRedirectError extends Error {
+ constructor() {
+ super("Raced redirect error");
+ }
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+configDotenv();
+
+const jobLockExtendInterval =
+ Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;
+const jobLockExtensionTime =
+ Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;
+
+cacheableLookup.install(http.globalAgent);
+cacheableLookup.install(https.globalAgent);
+
+async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) {
+ let creditsToBeBilled: number | null = null;
+
+ if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {
+ creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags);
+
+ if (
+ job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&
+ process.env.USE_DB_AUTHENTICATION === "true"
+ ) {
+ try {
+ const billingJobId = uuidv4();
+ logger.debug(
+ `Adding billing job to queue for team ${job.data.team_id}`,
+ {
+ billingJobId,
+ credits: creditsToBeBilled,
+ is_extract: false,
+ },
+ );
+
+ // Add directly to the billing queue - the billing worker will handle the rest
+ await getBillingQueue().add(
+ "bill_team",
+ {
+ team_id: job.data.team_id,
+ subscription_id: undefined,
+ credits: creditsToBeBilled,
+ is_extract: false,
+ timestamp: new Date().toISOString(),
+ originating_job_id: job.id,
+ },
+ {
+ jobId: billingJobId,
+ priority: 10,
+ },
+ );
+ return creditsToBeBilled;
+ } catch (error) {
+ logger.error(
+ `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
+ { error },
+ );
+ Sentry.captureException(error);
+ return creditsToBeBilled;
+ }
+ }
+ }
+
+ return creditsToBeBilled;
+}
+
+async function processJob(job: Job & { id: string }) {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "processJob",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data?.crawl_id ?? undefined,
+ teamId: job.data?.team_id ?? undefined,
+ zeroDataRetention: job.data?.zeroDataRetention ?? false,
+ });
+ logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });
+ const start = job.data.startTime ?? Date.now();
+ const remainingTime = job.data.scrapeOptions.timeout ? (job.data.scrapeOptions.timeout - (Date.now() - start)) : undefined;
+
+ const costTracking = new CostTracking();
+
+ try {
+ job.updateProgress({
+ current: 1,
+ total: 100,
+ current_step: "SCRAPING",
+ current_url: "",
+ });
+
+ if (remainingTime !== undefined && remainingTime < 0) {
+ throw new Error("timeout");
+ }
+ const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;
+
+ if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ if (sc && sc.cancelled) {
+ throw new Error("Parent crawl/batch scrape was cancelled");
+ }
+ }
+
+
+ const pipeline = await Promise.race([
+ startWebScraperPipeline({
+ job,
+ costTracking,
+ }),
+ ...(remainingTime !== undefined
+ ? [
+ (async () => {
+ await sleep(remainingTime);
+ throw new Error("timeout");
+ })(),
+ ]
+ : []),
+ ]);
+
+ try {
+ signal?.throwIfAborted();
+ } catch (e) {
+ throw new Error("timeout");
+ }
+
+ if (!pipeline.success) {
+ throw pipeline.error;
+ }
+
+ const end = Date.now();
+ const timeTakenInSeconds = (end - start) / 1000;
+
+ const doc = pipeline.document;
+
+ const rawHtml = doc.rawHtml ?? "";
+
+ if (!job.data.scrapeOptions.formats.includes("rawHtml")) {
+ delete doc.rawHtml;
+ }
+
+ if (job.data.concurrencyLimited) {
+ doc.warning =
+ "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." +
+ (doc.warning ? " " + doc.warning : "");
+ }
+
+ const data = {
+ success: true,
+ result: {
+ links: [
+ {
+ content: doc,
+ source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",
+ id: job.id,
+ },
+ ],
+ },
+ project_id: job.data.project_id,
+ document: doc,
+ };
+
+ if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+
+ if (
+ doc.metadata.url !== undefined &&
+ doc.metadata.sourceURL !== undefined &&
+ normalizeURL(doc.metadata.url, sc) !==
+ normalizeURL(doc.metadata.sourceURL, sc) &&
+ job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
+ ) {
+ const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+ const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);
+ if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {
+ const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";
+ throw new Error(reason);
+ }
+
+ // Only re-set originUrl if it's different from the current hostname
+ // This is only done on this condition to handle cross-domain redirects
+ // If this would be done for non-crossdomain redirects, but also for e.g.
+ // redirecting / -> /introduction (like our docs site does), it would
+ // break crawling the entire site without allowBackwardsCrawling - mogery
+ const isHostnameDifferent =
+ normalizeUrlOnlyHostname(doc.metadata.url) !==
+ normalizeUrlOnlyHostname(doc.metadata.sourceURL);
+ if (job.data.isCrawlSourceScrape && isHostnameDifferent) {
+ // TODO: re-fetch sitemap for redirect target domain
+ sc.originUrl = doc.metadata.url;
+ await saveCrawl(job.data.crawl_id, sc);
+ }
+
+ if (isUrlBlocked(doc.metadata.url, (await getACUCTeam(job.data.team_id))?.flags ?? null)) {
+ throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
+ }
+
+ const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));
+ const p2 = generateURLPermutations(
+ normalizeURL(doc.metadata.sourceURL, sc),
+ );
+
+ if (JSON.stringify(p1) !== JSON.stringify(p2)) {
+ logger.debug(
+ "Was redirected, removing old URL and locking new URL...",
+ { oldUrl: doc.metadata.sourceURL, newUrl: doc.metadata.url },
+ );
+
+ // Prevent redirect target from being visited in the crawl again
+ // See lockURL
+ const x = await redisEvictConnection.sadd(
+ "crawl:" + job.data.crawl_id + ":visited",
+ ...p1.map((x) => x.href),
+ );
+ const lockRes = x === p1.length;
+
+ if (job.data.crawlerOptions !== null && !lockRes) {
+ throw new RacedRedirectError();
+ }
+ }
+ }
+
+ if (job.data.crawlerOptions !== null) {
+ if (!sc.cancelled) {
+ const crawler = crawlToCrawler(
+ job.data.crawl_id,
+ sc,
+ (await getACUCTeam(job.data.team_id))?.flags ?? null,
+ doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,
+ job.data.crawlerOptions,
+ );
+
+ const links = await crawler.filterLinks(
+ await crawler.extractLinksFromHTML(
+ rawHtml ?? "",
+ doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,
+ ),
+ Infinity,
+ sc.crawlerOptions?.maxDepth ?? 10,
+ );
+ logger.debug("Discovered " + links.links.length + " links...", {
+ linksLength: links.links.length,
+ });
+
+ for (const link of links.links) {
+ if (await lockURL(job.data.crawl_id, sc, link)) {
+ // This seems to work really welel
+ const jobPriority = await getJobPriority({
+ team_id: sc.team_id,
+ basePriority: job.data.crawl_id ? 20 : 10,
+ });
+ const jobId = uuidv4();
+
+ logger.debug(
+ "Determined job priority " +
+ jobPriority +
+ " for URL " +
+ JSON.stringify(link),
+ { jobPriority, url: link },
+ );
+
+ await addScrapeJob(
+ {
+ url: link,
+ mode: "single_urls",
+ team_id: sc.team_id,
+ scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),
+ internalOptions: sc.internalOptions,
+ crawlerOptions: {
+ ...sc.crawlerOptions,
+ currentDiscoveryDepth:
+ (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,
+ },
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ {},
+ jobId,
+ jobPriority,
+ );
+
+ await addCrawlJob(job.data.crawl_id, jobId, logger);
+ logger.debug("Added job for URL " + JSON.stringify(link), {
+ jobPriority,
+ url: link,
+ newJobId: jobId,
+ });
+ } else {
+ // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!
+ // logger.debug("Could not lock URL " + JSON.stringify(link), {
+ // url: link,
+ // });
+ }
+ }
+
+ // Only run check after adding new jobs for discovery - mogery
+ if (job.data.isCrawlSourceScrape) {
+ const filterResult = await crawler.filterLinks(
+ [doc.metadata.url ?? doc.metadata.sourceURL!],
+ 1,
+ sc.crawlerOptions?.maxDepth ?? 10,
+ );
+ if (filterResult.links.length === 0) {
+ const url = doc.metadata.url ?? doc.metadata.sourceURL!;
+ const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";
+ throw new Error(reason);
+ }
+ }
+ }
+ }
+
+ try {
+ signal?.throwIfAborted();
+ } catch (e) {
+ throw new Error("timeout");
+ }
+
+ const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+
+ doc.metadata.creditsUsed = credits_billed ?? undefined;
+
+ logger.debug("Logging job to DB...");
+ await logJob(
+ {
+ job_id: job.id as string,
+ success: true,
+ num_docs: 1,
+ docs: [doc],
+ time_taken: timeTakenInSeconds,
+ team_id: job.data.team_id,
+ mode: job.data.mode,
+ url: job.data.url,
+ crawlerOptions: sc.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ cost_tracking: costTracking,
+ pdf_num_pages: doc.metadata.numPages,
+ credits_billed,
+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ true,
+ job.data.internalOptions?.bypassBilling ?? false,
+ );
+
+ if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {
+ logger.debug("Calling webhook with success...", {
+ webhook: job.data.webhook,
+ });
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id,
+ scrapeId: job.id,
+ data,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
+ });
+ }
+
+ logger.debug("Declaring job as done...");
+ await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
+
+ await finishCrawlIfNeeded(job, sc);
+ } else {
+ try {
+ signal?.throwIfAborted();
+ } catch (e) {
+ throw new Error("timeout");
+ }
+
+ const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+
+ doc.metadata.creditsUsed = credits_billed ?? undefined;
+
+ await logJob({
+ job_id: job.id,
+ success: true,
+ message: "Scrape completed",
+ num_docs: 1,
+ docs: [doc],
+ time_taken: timeTakenInSeconds,
+ team_id: job.data.team_id,
+ mode: "scrape",
+ url: job.data.url,
+ scrapeOptions: job.data.scrapeOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ num_tokens: 0, // TODO: fix
+ cost_tracking: costTracking,
+ pdf_num_pages: doc.metadata.numPages,
+ credits_billed,
+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,
+ zeroDataRetention: job.data.zeroDataRetention,
+ }, false, job.data.internalOptions?.bypassBilling ?? false);
+ }
+
+ logger.info(`🐂 Job done ${job.id}`);
+ return data;
+ } catch (error) {
+ if (job.data.crawl_id) {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+
+ logger.debug("Declaring job as done...");
+ await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);
+ await redisEvictConnection.srem(
+ "crawl:" + job.data.crawl_id + ":visited_unique",
+ normalizeURL(job.data.url, sc),
+ );
+
+ await finishCrawlIfNeeded(job, sc);
+ }
+
+ const isEarlyTimeout =
+ error instanceof Error && error.message === "timeout";
+ const isCancelled =
+ error instanceof Error &&
+ error.message === "Parent crawl/batch scrape was cancelled";
+
+ if (isEarlyTimeout) {
+ logger.error(`🐂 Job timed out ${job.id}`);
+ } else if (error instanceof RacedRedirectError) {
+ logger.warn(`🐂 Job got redirect raced ${job.id}, silently failing`);
+ } else if (isCancelled) {
+ logger.warn(`🐂 Job got cancelled, silently failing`);
+ } else {
+ logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });
+
+ Sentry.captureException(error, {
+ data: {
+ job: job.id,
+ },
+ });
+
+ if (error instanceof CustomError) {
+ // Here we handle the error, then save the failed job
+ logger.error(error.message); // or any other error handling
+ }
+ logger.error(error);
+ if (error.stack) {
+ logger.error(error.stack);
+ }
+ }
+
+ const data = {
+ success: false,
+ document: null,
+ project_id: job.data.project_id,
+ error:
+ error instanceof Error
+ ? error
+ : typeof error === "string"
+ ? new Error(error)
+ : new Error(JSON.stringify(error)),
+ };
+
+ if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id ?? (job.id as string),
+ scrapeId: job.id,
+ data,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
+ });
+ }
+
+ const end = Date.now();
+ const timeTakenInSeconds = (end - start) / 1000;
+
+ const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+
+ logger.debug("Logging job to DB...");
+ await logJob(
+ {
+ job_id: job.id as string,
+ success: false,
+ message:
+ typeof error === "string"
+ ? error
+ : (error.message ??
+ "Something went wrong... Contact [email protected]"),
+ num_docs: 0,
+ docs: [],
+ time_taken: timeTakenInSeconds,
+ team_id: job.data.team_id,
+ mode: job.data.mode,
+ url: job.data.url,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ cost_tracking: costTracking,
+ credits_billed,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ true,
+ job.data.internalOptions?.bypassBilling ?? false,
+ );
+ return data;
+ }
+}
+
+async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: string) {
+ if (sc.crawlerOptions.ignoreSitemap) {
+ return [];
+ }
+
+ const trimmedURL = new URL(url);
+ trimmedURL.search = "";
+
+ const index = await queryIndexAtSplitLevel(
+ sc.crawlerOptions.allowBackwardCrawling ? generateURLSplits(trimmedURL.href)[0] : trimmedURL.href,
+ sc.crawlerOptions.limit ?? 10000,
+ );
+
+ const validIndexLinksResult = await crawler.filterLinks(
+ index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),
+ sc.crawlerOptions.limit ?? 10000,
+ sc.crawlerOptions.maxDepth ?? 10,
+ false,
+ );
+ const validIndexLinks = validIndexLinksResult.links;
+
+ return validIndexLinks;
+}
+
+async function processKickoffJob(job: Job & { id: string }) {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "processKickoffJob",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data?.crawl_id ?? undefined,
+ teamId: job.data?.team_id ?? undefined,
+ zeroDataRetention: job.data.zeroDataRetention ?? false,
+ });
+
+ try {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
+
+ logger.debug("Locking URL...");
+ await lockURL(job.data.crawl_id, sc, job.data.url);
+ const jobId = uuidv4();
+ logger.debug("Adding scrape job to Redis...", { jobId });
+ await addScrapeJob(
+ {
+ url: job.data.url,
+ mode: "single_urls",
+ team_id: job.data.team_id,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ isCrawlSourceScrape: true,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ {
+ priority: 15,
+ },
+ jobId,
+ );
+ logger.debug("Adding scrape job to BullMQ...", { jobId });
+ await addCrawlJob(job.data.crawl_id, jobId, logger);
+
+ if (job.data.webhook) {
+ logger.debug("Calling webhook with crawl.started...", {
+ webhook: job.data.webhook,
+ });
+ callWebhook({
+ teamId: job.data.team_id,
+ crawlId: job.data.crawl_id,
+ data: null,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ eventType: "crawl.started",
+ });
+ }
+
+ const sitemap = sc.crawlerOptions.ignoreSitemap
+ ? 0
+ : await crawler.tryGetSitemap(async (urls) => {
+ if (urls.length === 0) return;
+
+ logger.debug("Using sitemap chunk of length " + urls.length, {
+ sitemapLength: urls.length,
+ });
+
+ let jobPriority = await getJobPriority({
+ team_id: job.data.team_id,
+ basePriority: 21,
+ });
+ logger.debug("Using job priority " + jobPriority, { jobPriority });
+
+ const jobs = urls.map((url) => {
+ const uuid = uuidv4();
+ return {
+ name: uuid,
+ data: {
+ url,
+ mode: "single_urls" as const,
+ team_id: job.data.team_id,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ sitemapped: true,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ opts: {
+ jobId: uuid,
+ priority: 20,
+ },
+ };
+ });
+
+ logger.debug("Locking URLs...");
+ const lockedIds = await lockURLsIndividually(
+ job.data.crawl_id,
+ sc,
+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
+ );
+ const lockedJobs = jobs.filter((x) =>
+ lockedIds.find((y) => y.id === x.opts.jobId),
+ );
+ logger.debug("Adding scrape jobs to Redis...");
+ await addCrawlJobs(
+ job.data.crawl_id,
+ lockedJobs.map((x) => x.opts.jobId),
+ logger,
+ );
+ logger.debug("Adding scrape jobs to BullMQ...");
+ await addScrapeJobs(lockedJobs);
+ });
+
+ if (sitemap === 0) {
+ logger.debug("Sitemap not found or ignored.", {
+ ignoreSitemap: sc.crawlerOptions.ignoreSitemap,
+ });
+ }
+
+ const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);
+
+ if (indexLinks.length > 0) {
+ logger.debug("Using index links of length " + indexLinks.length, {
+ indexLinksLength: indexLinks.length,
+ });
+
+ let jobPriority = await getJobPriority({
+ team_id: job.data.team_id,
+ basePriority: 21,
+ });
+ logger.debug("Using job priority " + jobPriority, { jobPriority });
+
+ const jobs = indexLinks.map((url) => {
+ const uuid = uuidv4();
+ return {
+ name: uuid,
+ data: {
+ url,
+ mode: "single_urls" as const,
+ team_id: job.data.team_id,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ sitemapped: true,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ opts: {
+ jobId: uuid,
+ priority: 20,
+ },
+ };
+ });
+
+ logger.debug("Locking URLs...");
+ const lockedIds = await lockURLsIndividually(
+ job.data.crawl_id,
+ sc,
+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),
+ );
+ const lockedJobs = jobs.filter((x) =>
+ lockedIds.find((y) => y.id === x.opts.jobId),
+ );
+ logger.debug("Adding scrape jobs to Redis...");
+ await addCrawlJobs(
+ job.data.crawl_id,
+ lockedJobs.map((x) => x.opts.jobId),
+ logger,
+ );
+ logger.debug("Adding scrape jobs to BullMQ...");
+ await addScrapeJobs(lockedJobs);
+ }
+
+ logger.debug("Done queueing jobs!");
+
+ await finishCrawlKickoff(job.data.crawl_id);
+ await finishCrawlIfNeeded(job, sc);
+
+ return { success: true };
+ } catch (error) {
+ logger.error("An error occurred!", { error });
+ await finishCrawlKickoff(job.data.crawl_id);
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ if (sc) {
+ await finishCrawlIfNeeded(job, sc);
+ }
+ return { success: false, error };
+ }
+}
+
+export const processJobInternal = async (job: Job & { id: string }) => {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "processJobInternal",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data?.crawl_id ?? undefined,
+ zeroDataRetention: job.data?.zeroDataRetention ?? false,
+ });
+
+ try {
+ try {
+ let extendLockInterval: NodeJS.Timeout | null = null;
+ if (job.data?.mode !== "kickoff" && job.data?.team_id) {
+ extendLockInterval = setInterval(async () => {
+ await pushConcurrencyLimitActiveJob(job.data.team_id, job.id, 60 * 1000); // 60s lock renew, just like in the queue
+ }, jobLockExtendInterval);
+ }
+
+ await addJobPriority(job.data.team_id, job.id);
+ try {
+ if (job.data?.mode === "kickoff") {
+ const result = await processKickoffJob(job);
+ if (result.success) {
+ return null;
+ } else {
+ throw (result as any).error;
+ }
+ } else {
+ const result = await processJob(job);
+ if (result.success) {
+ try {
+ if (
+ process.env.USE_DB_AUTHENTICATION === "true" &&
+ (job.data.crawl_id || process.env.GCS_BUCKET_NAME)
+ ) {
+ logger.debug(
+ "Job succeeded -- putting null in Redis",
+ );
+ return null;
+ } else {
+ logger.debug("Job succeeded -- putting result in Redis");
+ return result.document;
+ }
+ } catch (e) { }
+ } else {
+ throw (result as any).error;
+ }
+ }
+ } finally {
+ await deleteJobPriority(job.data.team_id, job.id);
+ if (extendLockInterval) {
+ clearInterval(extendLockInterval);
+ }
+ }
+ } finally {
+ await concurrentJobDone(job);
+ }
+ } catch (error) {
+ logger.debug("Job failed", { error });
+ Sentry.captureException(error);
+ throw error;
+ }
+};
+
+module.exports = processJobInternal;
\ No newline at end of file
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 4f43ff104..d5aa33470 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -67,8 +67,6 @@ export interface RunWebScraperParams {
mode: Mode;
scrapeOptions: ScrapeOptions;
internalOptions?: InternalOptions;
- // onSuccess: (result: V1Document, mode: string) => void;
- // onError: (error: Error) => void;
team_id: string;
bull_job_id: string;
priority?: number;
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 96ed80ae1..1aa0a0420 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "1.29.2",
+ "version": "1.29.3",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -16,7 +16,8 @@
"build": "tsup",
"build-and-publish": "npm run build && npm publish --access public",
"publish-beta": "npm run build && npm publish --access public --tag beta",
- "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+ "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
},
"repository": {
"type": "git",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
index ffe3ac747..a0a2d345a 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
@@ -297,4 +297,5 @@ describe('FirecrawlApp E2E Tests', () => {
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");
});
+
});
diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts
new file mode 100644
index 000000000..75563aa3b
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts
@@ -0,0 +1,154 @@
+import FirecrawlApp from '../../../index';
+import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
+
+describe('monitorJobStatus retry logic', () => {
+ let app: FirecrawlApp;
+ let originalConsoleWarn: typeof console.warn;
+
+ beforeEach(() => {
+ app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
+ originalConsoleWarn = console.warn;
+ console.warn = jest.fn();
+ });
+
+ afterEach(() => {
+ console.warn = originalConsoleWarn;
+ jest.clearAllMocks();
+ });
+
+ test('should retry on socket hang up error', async () => {
+ const socketHangUpError = new Error('socket hang up') as any;
+ socketHangUpError.code = 'ECONNRESET';
+
+ const successResponse = {
+ status: 200,
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+ };
+
+ const originalGetRequest = app.getRequest;
+ let callCount = 0;
+
+ app.getRequest = async function(url: string, headers: any) {
+ callCount++;
+ if (callCount === 1) {
+ throw socketHangUpError;
+ }
+ return successResponse;
+ };
+
+ const result = await app.monitorJobStatus('test-id', {}, 1);
+
+ expect(callCount).toBe(2);
+ expect(result).toEqual(successResponse.data);
+ expect(console.warn).toHaveBeenCalledWith(
+ expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
+ );
+ });
+
+ test('should retry on ETIMEDOUT error', async () => {
+ const timeoutError = new Error('timeout') as any;
+ timeoutError.code = 'ETIMEDOUT';
+
+ const successResponse = {
+ status: 200,
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+ };
+
+ const originalGetRequest = app.getRequest;
+ let callCount = 0;
+
+ app.getRequest = async function(url: string, headers: any) {
+ callCount++;
+ if (callCount === 1) {
+ throw timeoutError;
+ }
+ return successResponse;
+ };
+
+ const result = await app.monitorJobStatus('test-id', {}, 1);
+
+ expect(callCount).toBe(2);
+ expect(result).toEqual(successResponse.data);
+ });
+
+ test('should fail after max retries exceeded', async () => {
+ const socketHangUpError = new Error('socket hang up') as any;
+ socketHangUpError.code = 'ECONNRESET';
+
+ app.getRequest = async function(url: string, headers: any) {
+ throw socketHangUpError;
+ };
+
+ await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
+
+ expect(console.warn).toHaveBeenCalledTimes(3);
+ }, 15000);
+
+ test('should not retry on non-retryable errors', async () => {
+ const authError = new Error('Unauthorized') as any;
+ authError.response = { status: 401, data: { error: 'Unauthorized' } };
+
+ app.getRequest = async function(url: string, headers: any) {
+ throw authError;
+ };
+
+ await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
+
+ expect(console.warn).not.toHaveBeenCalled();
+ });
+
+ test('should retry on HTTP timeout status codes', async () => {
+ const timeoutError = new Error('Request timeout') as any;
+ timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
+
+ const successResponse = {
+ status: 200,
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+ };
+
+ const originalGetRequest = app.getRequest;
+ let callCount = 0;
+
+ app.getRequest = async function(url: string, headers: any) {
+ callCount++;
+ if (callCount === 1) {
+ throw timeoutError;
+ }
+ return successResponse;
+ };
+
+ const result = await app.monitorJobStatus('test-id', {}, 1);
+
+ expect(callCount).toBe(2);
+ expect(result).toEqual(successResponse.data);
+ });
+
+ test('should use exponential backoff for retries', async () => {
+ const socketHangUpError = new Error('socket hang up') as any;
+ socketHangUpError.code = 'ECONNRESET';
+
+ const successResponse = {
+ status: 200,
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+ };
+
+ const originalGetRequest = app.getRequest;
+ let callCount = 0;
+
+ app.getRequest = async function(url: string, headers: any) {
+ callCount++;
+ if (callCount <= 2) {
+ throw socketHangUpError;
+ }
+ return successResponse;
+ };
+
+ const startTime = Date.now();
+ const result = await app.monitorJobStatus('test-id', {}, 1);
+ const endTime = Date.now();
+
+ expect(callCount).toBe(3);
+ expect(result).toEqual(successResponse.data);
+ expect(endTime - startTime).toBeGreaterThan(3000);
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts
index 139f7a919..f93bb48e3 100644
--- a/apps/js-sdk/firecrawl/src/index.ts
+++ b/apps/js-sdk/firecrawl/src/index.ts
@@ -1455,33 +1455,39 @@ export default class FirecrawlApp {
headers: AxiosRequestHeaders,
checkInterval: number
): Promise<CrawlStatusResponse | ErrorResponse> {
- try {
- let failedTries = 0;
- while (true) {
+ let failedTries = 0;
+ let networkRetries = 0;
+ const maxNetworkRetries = 3;
+
+ while (true) {
+ try {
let statusResponse: AxiosResponse = await this.getRequest(
`${this.apiUrl}/v1/crawl/${id}`,
headers
);
+
if (statusResponse.status === 200) {
failedTries = 0;
+ networkRetries = 0;
let statusData = statusResponse.data;
- if (statusData.status === "completed") {
- if ("data" in statusData) {
- let data = statusData.data;
- while (typeof statusData === 'object' && 'next' in statusData) {
- if (data.length === 0) {
- break
- }
- statusResponse = await this.getRequest(statusData.next, headers);
- statusData = statusResponse.data;
- data = data.concat(statusData.data);
+
+ if (statusData.status === "completed") {
+ if ("data" in statusData) {
+ let data = statusData.data;
+ while (typeof statusData === 'object' && 'next' in statusData) {
+ if (data.length === 0) {
+ break
}
- statusData.data = data;
- return statusData;
- } else {
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+ statusResponse = await this.getRequest(statusData.next, headers);
+ statusData = statusResponse.data;
+ data = data.concat(statusData.data);
}
- } else if (
+ statusData.data = data;
+ return statusData;
+ } else {
+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+ }
+ } else if (
["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
) {
checkInterval = Math.max(checkInterval, 2);
@@ -1500,12 +1506,69 @@ export default class FirecrawlApp {
this.handleError(statusResponse, "check crawl status");
}
}
+ } catch (error: any) {
+ if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+ networkRetries++;
+ const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);
+
+ await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+ continue;
+ }
+
+ throw new FirecrawlError(error, 500);
}
- } catch (error: any) {
- throw new FirecrawlError(error, 500);
}
}
+ /**
+ * Determines if an error is retryable (transient network error)
+ * @param error - The error to check
+ * @returns True if the error should be retried
+ */
+ private isRetryableError(error: any): boolean {
+ if (error instanceof AxiosError) {
+ if (!error.response) {
+ const code = error.code;
+ const message = error.message?.toLowerCase() || '';
+
+ return (
+ code === 'ECONNRESET' ||
+ code === 'ETIMEDOUT' ||
+ code === 'ENOTFOUND' ||
+ code === 'ECONNREFUSED' ||
+ message.includes('socket hang up') ||
+ message.includes('network error') ||
+ message.includes('timeout')
+ );
+ }
+
+ if (error.response?.status === 408 || error.response?.status === 504) {
+ return true;
+ }
+ }
+
+ if (error && typeof error === 'object') {
+ const code = error.code;
+ const message = error.message?.toLowerCase() || '';
+
+ if (code === 'ECONNRESET' ||
+ code === 'ETIMEDOUT' ||
+ code === 'ENOTFOUND' ||
+ code === 'ECONNREFUSED' ||
+ message.includes('socket hang up') ||
+ message.includes('network error') ||
+ message.includes('timeout')) {
+ return true;
+ }
+
+ if (error.response?.status === 408 || error.response?.status === 504) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
/**
* Handles errors from API responses.
* @param {AxiosResponse} response - The response from the API.
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 99db69b04..9119facd4 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
-__version__ = "2.16.3"
+__version__ = "2.16.4"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py
index df415d4ce..1207c3be7 100644
--- a/apps/python-sdk/firecrawl/firecrawl.py
+++ b/apps/python-sdk/firecrawl/firecrawl.py
@@ -24,12 +24,6 @@ import aiohttp
import asyncio
from pydantic import Field
-# Suppress Pydantic warnings about attribute shadowing
-warnings.filterwarnings("ignore", message="Field name \"json\" in \"FirecrawlDocument\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"json\" in \"ChangeTrackingData\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonConfig\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ExtractParams\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingOptions\" shadows an attribute in parent \"BaseModel\"")
def get_version():
try:
@@ -106,7 +100,7 @@ class ChangeTrackingData(pydantic.BaseModel):
changeStatus: str # "new" | "same" | "changed" | "removed"
visibility: str # "visible" | "hidden"
diff: Optional[Dict[str, Any]] = None
- json: Optional[Any] = None
+ json_field: Optional[Any] = pydantic.Field(None, alias='json')
class FirecrawlDocument(pydantic.BaseModel, Generic[T]):
"""Document retrieved or processed by Firecrawl."""
@@ -116,7 +110,7 @@ class FirecrawlDocument(pydantic.BaseModel, Generic[T]):
rawHtml: Optional[str] = None
links: Optional[List[str]] = None
extract: Optional[T] = None
- json: Optional[T] = None
+ json_field: Optional[T] = pydantic.Field(None, alias='json')
screenshot: Optional[str] = None
metadata: Optional[Any] = None
actions: Optional[ActionsResult] = None
@@ -139,7 +133,7 @@ class WebhookConfig(pydantic.BaseModel):
class ChangeTrackingOptions(pydantic.BaseModel):
"""Configuration for change tracking."""
modes: Optional[List[Literal["git-diff", "json"]]] = None
- schema: Optional[Any] = None
+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
prompt: Optional[str] = None
tag: Optional[str] = None
@@ -219,7 +213,7 @@ class ExtractAgent(pydantic.BaseModel):
class JsonConfig(pydantic.BaseModel):
"""Configuration for extraction."""
prompt: Optional[str] = None
- schema: Optional[Any] = None
+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
systemPrompt: Optional[str] = None
agent: Optional[ExtractAgent] = None
@@ -318,7 +312,7 @@ class MapResponse(pydantic.BaseModel):
class ExtractParams(pydantic.BaseModel):
"""Parameters for extracting information from URLs."""
prompt: Optional[str] = None
- schema: Optional[Any] = None
+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
systemPrompt: Optional[str] = None
allowExternalLinks: Optional[bool] = None
enableWebSearch: Optional[bool] = None
@@ -432,7 +426,7 @@ class ExtractParams(pydantic.BaseModel):
Parameters for the extract operation.
"""
prompt: Optional[str] = None
- schema: Optional[Any] = pydantic.Field(None, alias='schema')
+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
system_prompt: Optional[str] = None
allow_external_links: Optional[bool] = False
enable_web_search: Optional[bool] = False
@@ -548,7 +542,7 @@ class FirecrawlApp:
if timeout:
scrape_params['timeout'] = timeout
if location:
- scrape_params['location'] = location.dict(exclude_none=True)
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
if mobile is not None:
scrape_params['mobile'] = mobile
if skip_tls_verification is not None:
@@ -565,16 +559,16 @@ class FirecrawlApp:
extract = self._ensure_schema_dict(extract)
if isinstance(extract, dict) and "schema" in extract:
extract["schema"] = self._ensure_schema_dict(extract["schema"])
- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
if json_options is not None:
json_options = self._ensure_schema_dict(json_options)
if isinstance(json_options, dict) and "schema" in json_options:
json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
if actions:
- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
if change_tracking_options:
- scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(exclude_none=True)
+ scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(by_alias=True, exclude_none=True)
if max_age is not None:
scrape_params['maxAge'] = max_age
if store_in_cache is not None:
@@ -671,7 +665,7 @@ class FirecrawlApp:
if timeout is not None:
search_params['timeout'] = timeout
if scrape_options is not None:
- search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+ search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
# Add any additional kwargs
search_params.update(kwargs)
@@ -679,7 +673,7 @@ class FirecrawlApp:
# Create final params object
final_params = SearchParams(query=query, **search_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['origin'] = f"python-sdk@{version}"
if _integration:
@@ -793,7 +787,7 @@ class FirecrawlApp:
if ignore_sitemap is not None:
crawl_params['ignoreSitemap'] = ignore_sitemap
if scrape_options is not None:
- crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
if webhook is not None:
crawl_params['webhook'] = webhook
if deduplicate_similar_urls is not None:
@@ -816,7 +810,7 @@ class FirecrawlApp:
# Create final params object
final_params = CrawlParams(**crawl_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
params_dict['origin'] = f"python-sdk@{version}"
@@ -922,7 +916,7 @@ class FirecrawlApp:
if ignore_sitemap is not None:
crawl_params['ignoreSitemap'] = ignore_sitemap
if scrape_options is not None:
- crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
if webhook is not None:
crawl_params['webhook'] = webhook
if deduplicate_similar_urls is not None:
@@ -944,7 +938,7 @@ class FirecrawlApp:
# Create final params object
final_params = CrawlParams(**crawl_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
params_dict['origin'] = f"python-sdk@{version}"
@@ -1244,7 +1238,7 @@ class FirecrawlApp:
# Create final params object
final_params = MapParams(**map_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
params_dict['origin'] = f"python-sdk@{version}"
@@ -1356,7 +1350,7 @@ class FirecrawlApp:
if timeout is not None:
scrape_params['timeout'] = timeout
if location is not None:
- scrape_params['location'] = location.dict(exclude_none=True)
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
if mobile is not None:
scrape_params['mobile'] = mobile
if skip_tls_verification is not None:
@@ -1371,16 +1365,16 @@ class FirecrawlApp:
extract = self._ensure_schema_dict(extract)
if isinstance(extract, dict) and "schema" in extract:
extract["schema"] = self._ensure_schema_dict(extract["schema"])
- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
if json_options is not None:
json_options = self._ensure_schema_dict(json_options)
if isinstance(json_options, dict) and "schema" in json_options:
json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
if actions:
- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
if agent is not None:
- scrape_params['agent'] = agent.dict(exclude_none=True)
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
if max_concurrency is not None:
scrape_params['maxConcurrency'] = max_concurrency
if zero_data_retention is not None:
@@ -1391,7 +1385,7 @@ class FirecrawlApp:
# Create final params object
final_params = ScrapeParams(**scrape_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
params_dict['origin'] = f"python-sdk@{version}"
@@ -1497,7 +1491,7 @@ class FirecrawlApp:
if timeout is not None:
scrape_params['timeout'] = timeout
if location is not None:
- scrape_params['location'] = location.dict(exclude_none=True)
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
if mobile is not None:
scrape_params['mobile'] = mobile
if skip_tls_verification is not None:
@@ -1512,16 +1506,16 @@ class FirecrawlApp:
extract = self._ensure_schema_dict(extract)
if isinstance(extract, dict) and "schema" in extract:
extract["schema"] = self._ensure_schema_dict(extract["schema"])
- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
if json_options is not None:
json_options = self._ensure_schema_dict(json_options)
if isinstance(json_options, dict) and "schema" in json_options:
json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
if actions:
- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
if agent is not None:
- scrape_params['agent'] = agent.dict(exclude_none=True)
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
if max_concurrency is not None:
scrape_params['maxConcurrency'] = max_concurrency
if zero_data_retention is not None:
@@ -1532,7 +1526,7 @@ class FirecrawlApp:
# Create final params object
final_params = ScrapeParams(**scrape_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
params_dict['origin'] = f"python-sdk@{version}"
@@ -1633,7 +1627,7 @@ class FirecrawlApp:
if timeout is not None:
scrape_params['timeout'] = timeout
if location is not None:
- scrape_params['location'] = location.dict(exclude_none=True)
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
if mobile is not None:
scrape_params['mobile'] = mobile
if skip_tls_verification is not None:
@@ -1648,16 +1642,16 @@ class FirecrawlApp:
extract = self._ensure_schema_dict(extract)
if isinstance(extract, dict) and "schema" in extract:
extract["schema"] = self._ensure_schema_dict(extract["schema"])
- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
if json_options is not None:
json_options = self._ensure_schema_dict(json_options)
if isinstance(json_options, dict) and "schema" in json_options:
json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
if actions:
- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
if agent is not None:
- scrape_params['agent'] = agent.dict(exclude_none=True)
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
if max_concurrency is not None:
scrape_params['maxConcurrency'] = max_concurrency
if zero_data_retention is not None:
@@ -1668,7 +1662,7 @@ class FirecrawlApp:
# Create final params object
final_params = ScrapeParams(**scrape_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
params_dict['origin'] = f"python-sdk@{version}"
@@ -2097,7 +2091,7 @@ class FirecrawlApp:
)
headers = self._prepare_headers()
- json_data = {'url': url, **params.dict(exclude_none=True)}
+ json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
json_data['origin'] = f"python-sdk@{version}"
try:
@@ -2338,10 +2332,22 @@ class FirecrawlApp:
Exception: An exception with a message containing the status code and error details from the response.
"""
try:
- error_message = response.json().get('error', 'No error message provided.')
- error_details = response.json().get('details', 'No additional error details provided.')
+ response_json = response.json()
+ error_message = response_json.get('error', 'No error message provided.')
+ error_details = response_json.get('details', 'No additional error details provided.')
except:
- raise requests.exceptions.HTTPError(f'Failed to parse Firecrawl error response as JSON. Status code: {response.status_code}', response=response)
+ # If we can't parse JSON, provide a helpful error message with response content
+ try:
+ response_text = response.text[:500] # Limit to first 500 chars
+ if response_text.strip():
+ error_message = f"Server returned non-JSON response: {response_text}"
+ error_details = f"Full response status: {response.status_code}"
+ else:
+ error_message = f"Server returned empty response with status {response.status_code}"
+ error_details = "No additional details available"
++ except ValueError:
+ error_message = f"Server returned unreadable response with status {response.status_code}"
+ error_details = "No additional details available"
message = self._get_error_message(response.status_code, action, error_message, error_details)
@@ -2364,7 +2370,7 @@ class FirecrawlApp:
if status_code == 402:
return f"Payment Required: Failed to {action}. {error_message} - {error_details}"
elif status_code == 403:
- message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
+ return f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
elif status_code == 408:
return f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
elif status_code == 409:
@@ -2518,7 +2524,7 @@ class FirecrawlApp:
headers = self._prepare_headers()
- json_data = {'query': query, **research_params.dict(exclude_none=True)}
+ json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
json_data['origin'] = f"python-sdk@{version}"
# Handle json options schema if present
@@ -3054,7 +3060,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
if timeout:
scrape_params['timeout'] = timeout
if location:
- scrape_params['location'] = location.dict(exclude_none=True)
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
if mobile is not None:
scrape_params['mobile'] = mobile
if skip_tls_verification is not None:
@@ -3071,14 +3077,14 @@ class AsyncFirecrawlApp(FirecrawlApp):
extract = self._ensure_schema_dict(extract)
if isinstance(extract, dict) and "schema" in extract:
extract["schema"] = self._ensure_schema_dict(extract["schema"])
- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
if json_options is not None:
json_options = self._ensure_schema_dict(json_options)
if isinstance(json_options, dict) and "schema" in json_options:
json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
if actions:
- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:
scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])
if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']:
@@ -3182,7 +3188,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
if timeout is not None:
scrape_params['timeout'] = timeout
if location is not None:
- scrape_params['location'] = location.dict(exclude_none=True)
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
if mobile is not None:
scrape_params['mobile'] = mobile
if skip_tls_verification is not None:
@@ -3197,22 +3203,23 @@ class AsyncFirecrawlApp(FirecrawlApp):
extract = self._ensure_schema_dict(extract)
if isinstance(extract, dict) and "schema" in extract:
extract["schema"] = self._ensure_schema_dict(extract["schema"])
- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
if json_options is not None:
json_options = self._ensure_schema_dict(json_options)
if isinstance(json_options, dict) and "schema" in json_options:
json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
-
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+ if actions is not None:
+ scrape_params['actions'] = [action.dict(by_alias=True, exclude_none=True) for action in actions]
if agent is not None:
- scrape_params['agent'] = agent.dict(exclude_none=True)
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
# Add any additional kwargs
scrape_params.update(kwargs)
# Create final params object
final_params = ScrapeParams(**scrape_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
params_dict['origin'] = f"python-sdk@{version}"
@@ -3321,7 +3328,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
if timeout is not None:
scrape_params['timeout'] = timeout
if location is not None:
- scrape_params['location'] = location.dict(exclude_none=True)
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
if mobile is not None:
scrape_params['mobile'] = mobile
if skip_tls_verification is not None:
@@ -3336,16 +3343,16 @@ class AsyncFirecrawlApp(FirecrawlApp):
extract = self._ensure_schema_dict(extract)
if isinstance(extract, dict) and "schema" in extract:
extract["schema"] = self._ensure_schema_dict(extract["schema"])
- scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
if json_options is not None:
json_options = self._ensure_schema_dict(json_options)
if isinstance(json_options, dict) and "schema" in json_options:
json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
- scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
if actions:
- scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
if agent is not None:
- scrape_params['agent'] = agent.dict(exclude_none=True)
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
if zero_data_retention is not None:
scrape_params['zeroDataRetention'] = zero_data_retention
@@ -3354,7 +3361,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
# Create final params object
final_params = ScrapeParams(**scrape_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
params_dict['origin'] = f"python-sdk@{version}"
@@ -3462,7 +3469,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
if ignore_sitemap is not None:
crawl_params['ignoreSitemap'] = ignore_sitemap
if scrape_options is not None:
- crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
if webhook is not None:
crawl_params['webhook'] = webhook
if deduplicate_similar_urls is not None:
@@ -3481,7 +3488,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
# Create final params object
final_params = CrawlParams(**crawl_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
params_dict['origin'] = f"python-sdk@{version}"
# Make request
@@ -3577,7 +3584,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
if ignore_sitemap is not None:
crawl_params['ignoreSitemap'] = ignore_sitemap
if scrape_options is not None:
- crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
if webhook is not None:
crawl_params['webhook'] = webhook
if deduplicate_similar_urls is not None:
@@ -3596,7 +3603,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
# Create final params object
final_params = CrawlParams(**crawl_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
params_dict['origin'] = f"python-sdk@{version}"
@@ -3762,7 +3769,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
"""
map_params = {}
if params:
- map_params.update(params.dict(exclude_none=True))
+ map_params.update(params.dict(by_alias=True, exclude_none=True))
# Add individual parameters
if search is not None:
@@ -3780,7 +3787,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
# Create final params object
final_params = MapParams(**map_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
params_dict['origin'] = f"python-sdk@{version}"
@@ -4164,7 +4171,6 @@ class AsyncFirecrawlApp(FirecrawlApp):
url,
max_urls=max_urls,
show_full_text=show_full_text,
- cache=cache,
experimental_stream=experimental_stream
)
if not response.get('success') or 'id' not in response:
@@ -4228,7 +4234,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
)
headers = self._prepare_headers()
- json_data = {'url': url, **params.dict(exclude_none=True)}
+ json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
json_data['origin'] = f"python-sdk@{version}"
try:
@@ -4413,7 +4419,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
headers = self._prepare_headers()
- json_data = {'query': query, **research_params.dict(exclude_none=True)}
+ json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
json_data['origin'] = f"python-sdk@{version}"
try:
@@ -4505,7 +4511,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
if isinstance(params, dict):
search_params.update(params)
else:
- search_params.update(params.dict(exclude_none=True))
+ search_params.update(params.dict(by_alias=True, exclude_none=True))
# Add individual parameters
if limit is not None:
@@ -4523,14 +4529,14 @@ class AsyncFirecrawlApp(FirecrawlApp):
if timeout is not None:
search_params['timeout'] = timeout
if scrape_options is not None:
- search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+ search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
# Add any additional kwargs
search_params.update(kwargs)
# Create final params object
final_params = SearchParams(query=query, **search_params)
- params_dict = final_params.dict(exclude_none=True)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['origin'] = f"python-sdk@{version}"
return await self._async_post_request(
diff --git a/apps/redis/start-redis-server.sh b/apps/redis/start-redis-server.sh
index e8a313ce9..8004d1640 100755
--- a/apps/redis/start-redis-server.sh
+++ b/apps/redis/start-redis-server.sh
@@ -19,7 +19,7 @@ fi
# Set maxmemory to 80% of RAM
MAXMEMORY=$(($FLY_VM_MEMORY_MB*80/100))
-mkdir /data/redis
+mkdir -p /data/redis
redis-server $PW_ARG \
--dir /data/redis \