Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: certain redirect responses change request methods (#3296)
Jindřich Bärefac644af2e2
diff --git a/packages/impit-client/package.json b/packages/impit-client/package.jsonindex b6e2cd783..8bd486110 100644--- a/packages/impit-client/package.json+++ b/packages/impit-client/package.json@@ -60,7 +60,7 @@},"dependencies": {"@apify/datastructures": "^2.0.3",- "impit": "^0.7.0",+ "impit": "^0.7.3","tough-cookie": "^6.0.0"}}diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.tsindex 94e4c3052..590f1e1b4 100644--- a/packages/impit-client/src/index.ts+++ b/packages/impit-client/src/index.ts@@ -98,6 +98,17 @@ export class ImpitHttpClient implements BaseHttpClient {return body as any;}+ private shouldRewriteRedirectToGet(httpStatus: number, method: HttpRequest<any>['method']): boolean {+ // See https://github.com/mozilla-firefox/firefox/blob/911b3eec6c5e58a9a49e23aa105e49aa76e00f9c/netwerk/protocol/http/HttpBaseChannel.cpp#L4801+ if ([301, 302].includes(httpStatus)) {+ return method === 'POST';+ }++ if (httpStatus === 303) return method !== 'HEAD';++ return false;+ }+/*** Common implementation for `sendRequest` and `stream` methods.* @param request `HttpRequest` object@@ -141,6 +152,7 @@ export class ImpitHttpClient implements BaseHttpClient {return this.getResponse({...request,+ method: this.shouldRewriteRedirectToGet(response.status, request.method) ? 'GET' : request.method,url: redirectUrl.href,},{diff --git a/yarn.lock b/yarn.lockindex 558cc007b..76db70662 100644--- a/yarn.lock+++ b/yarn.lock@@ -618,7 +618,7 @@ __metadata:dependencies:"@apify/datastructures": "npm:^2.0.3""@crawlee/core": "npm:^3.15.3"- impit: "npm:^0.7.0"+ impit: "npm:^0.7.3"tough-cookie: "npm:^6.0.0"peerDependencies:"@crawlee/core": ^3.12.1@@ -7487,9 +7487,9 @@ __metadata:languageName: nodelinkType: hard-"impit-darwin-arm64@npm:0.7.0":- version: 0.7.0- resolution: "impit-darwin-arm64@npm:0.7.0"+"impit-darwin-arm64@npm:0.7.3":+ version: 0.7.3+ resolution: "impit-darwin-arm64@npm:0.7.3"conditions: os=darwin & cpu=arm64languageName: nodelinkType: hard@@ -7501,9 +7501,9 @@ __metadata:languageName: nodelinkType: hard-"impit-darwin-x64@npm:0.7.0":- version: 0.7.0- resolution: "impit-darwin-x64@npm:0.7.0"+"impit-darwin-x64@npm:0.7.3":+ version: 0.7.3+ resolution: "impit-darwin-x64@npm:0.7.3"conditions: os=darwin & cpu=x64languageName: nodelinkType: hard@@ -7515,9 +7515,9 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-arm64-gnu@npm:0.7.0":- version: 0.7.0- resolution: "impit-linux-arm64-gnu@npm:0.7.0"+"impit-linux-arm64-gnu@npm:0.7.3":+ version: 0.7.3+ resolution: "impit-linux-arm64-gnu@npm:0.7.3"conditions: os=linux & cpu=arm64 & libc=glibclanguageName: nodelinkType: hard@@ -7529,9 +7529,9 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-arm64-musl@npm:0.7.0":- version: 0.7.0- resolution: "impit-linux-arm64-musl@npm:0.7.0"+"impit-linux-arm64-musl@npm:0.7.3":+ version: 0.7.3+ resolution: "impit-linux-arm64-musl@npm:0.7.3"conditions: os=linux & cpu=arm64 & libc=musllanguageName: nodelinkType: hard@@ -7543,9 +7543,9 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-x64-gnu@npm:0.7.0":- version: 0.7.0- resolution: "impit-linux-x64-gnu@npm:0.7.0"+"impit-linux-x64-gnu@npm:0.7.3":+ version: 0.7.3+ resolution: "impit-linux-x64-gnu@npm:0.7.3"conditions: os=linux & cpu=x64 & libc=glibclanguageName: nodelinkType: hard@@ -7557,9 +7557,9 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-x64-musl@npm:0.7.0":- version: 0.7.0- resolution: "impit-linux-x64-musl@npm:0.7.0"+"impit-linux-x64-musl@npm:0.7.3":+ version: 0.7.3+ resolution: "impit-linux-x64-musl@npm:0.7.3"conditions: os=linux & cpu=x64 & libc=musllanguageName: nodelinkType: hard@@ -7571,9 +7571,9 @@ __metadata:languageName: nodelinkType: hard-"impit-win32-arm64-msvc@npm:0.7.0":- version: 0.7.0- resolution: "impit-win32-arm64-msvc@npm:0.7.0"+"impit-win32-arm64-msvc@npm:0.7.3":+ version: 0.7.3+ resolution: "impit-win32-arm64-msvc@npm:0.7.3"conditions: os=win32 & cpu=arm64languageName: nodelinkType: hard@@ -7585,9 +7585,9 @@ __metadata:languageName: nodelinkType: hard-"impit-win32-x64-msvc@npm:0.7.0":- version: 0.7.0- resolution: "impit-win32-x64-msvc@npm:0.7.0"+"impit-win32-x64-msvc@npm:0.7.3":+ version: 0.7.3+ resolution: "impit-win32-x64-msvc@npm:0.7.3"conditions: os=win32 & cpu=x64languageName: nodelinkType: hard@@ -7625,18 +7625,18 @@ __metadata:languageName: nodelinkType: hard-"impit@npm:^0.7.0":- version: 0.7.0- resolution: "impit@npm:0.7.0"- dependencies:- impit-darwin-arm64: "npm:0.7.0"- impit-darwin-x64: "npm:0.7.0"- impit-linux-arm64-gnu: "npm:0.7.0"- impit-linux-arm64-musl: "npm:0.7.0"- impit-linux-x64-gnu: "npm:0.7.0"- impit-linux-x64-musl: "npm:0.7.0"- impit-win32-arm64-msvc: "npm:0.7.0"- impit-win32-x64-msvc: "npm:0.7.0"+"impit@npm:^0.7.3":+ version: 0.7.3+ resolution: "impit@npm:0.7.3"+ dependencies:+ impit-darwin-arm64: "npm:0.7.3"+ impit-darwin-x64: "npm:0.7.3"+ impit-linux-arm64-gnu: "npm:0.7.3"+ impit-linux-arm64-musl: "npm:0.7.3"+ impit-linux-x64-gnu: "npm:0.7.3"+ impit-linux-x64-musl: "npm:0.7.3"+ impit-win32-arm64-msvc: "npm:0.7.3"+ impit-win32-x64-msvc: "npm:0.7.3"dependenciesMeta:impit-darwin-arm64:optional: true@@ -7654,7 +7654,7 @@ __metadata:optional: trueimpit-win32-x64-msvc:optional: true- checksum: 10c0/bd5887db006b68913de1480da47379f4de11cbc7ba9725fd0106faca993eaa16a7a13e9869cc641790880dd044a07f7ce6312848d05555a98b266487e0740f57+ checksum: 10c0/8a7138303b5ea01d59e35c678e8c4a1483db1af722e1f747bf7c3635f86edcc51d49d8949820fd74993a041903d9fd29a57547cfe904a0bb5e9295e9c7269bf3languageName: nodelinkType: hard