Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: certain redirect responses change request methods (#3296)

Jindřich Bärefac644af2e2
diff --git a/packages/impit-client/package.json b/packages/impit-client/package.json
index b6e2cd783..8bd486110 100644
--- a/packages/impit-client/package.json
+++ b/packages/impit-client/package.json
@@ -60,7 +60,7 @@
},
"dependencies": {
"@apify/datastructures": "^2.0.3",
- "impit": "^0.7.0",
+ "impit": "^0.7.3",
"tough-cookie": "^6.0.0"
}
}
diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.ts
index 94e4c3052..590f1e1b4 100644
--- a/packages/impit-client/src/index.ts
+++ b/packages/impit-client/src/index.ts
@@ -98,6 +98,17 @@ export class ImpitHttpClient implements BaseHttpClient {
return body as any;
}
+ private shouldRewriteRedirectToGet(httpStatus: number, method: HttpRequest<any>['method']): boolean {
+ // See https://github.com/mozilla-firefox/firefox/blob/911b3eec6c5e58a9a49e23aa105e49aa76e00f9c/netwerk/protocol/http/HttpBaseChannel.cpp#L4801
+ if ([301, 302].includes(httpStatus)) {
+ return method === 'POST';
+ }
+
+ if (httpStatus === 303) return method !== 'HEAD';
+
+ return false;
+ }
+
/**
* Common implementation for `sendRequest` and `stream` methods.
* @param request `HttpRequest` object
@@ -141,6 +152,7 @@ export class ImpitHttpClient implements BaseHttpClient {
return this.getResponse(
{
...request,
+ method: this.shouldRewriteRedirectToGet(response.status, request.method) ? 'GET' : request.method,
url: redirectUrl.href,
},
{
diff --git a/yarn.lock b/yarn.lock
index 558cc007b..76db70662 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -618,7 +618,7 @@ __metadata:
dependencies:
"@apify/datastructures": "npm:^2.0.3"
"@crawlee/core": "npm:^3.15.3"
- impit: "npm:^0.7.0"
+ impit: "npm:^0.7.3"
tough-cookie: "npm:^6.0.0"
peerDependencies:
"@crawlee/core": ^3.12.1
@@ -7487,9 +7487,9 @@ __metadata:
languageName: node
linkType: hard
-"impit-darwin-arm64@npm:0.7.0":
- version: 0.7.0
- resolution: "impit-darwin-arm64@npm:0.7.0"
+"impit-darwin-arm64@npm:0.7.3":
+ version: 0.7.3
+ resolution: "impit-darwin-arm64@npm:0.7.3"
conditions: os=darwin & cpu=arm64
languageName: node
linkType: hard
@@ -7501,9 +7501,9 @@ __metadata:
languageName: node
linkType: hard
-"impit-darwin-x64@npm:0.7.0":
- version: 0.7.0
- resolution: "impit-darwin-x64@npm:0.7.0"
+"impit-darwin-x64@npm:0.7.3":
+ version: 0.7.3
+ resolution: "impit-darwin-x64@npm:0.7.3"
conditions: os=darwin & cpu=x64
languageName: node
linkType: hard
@@ -7515,9 +7515,9 @@ __metadata:
languageName: node
linkType: hard
-"impit-linux-arm64-gnu@npm:0.7.0":
- version: 0.7.0
- resolution: "impit-linux-arm64-gnu@npm:0.7.0"
+"impit-linux-arm64-gnu@npm:0.7.3":
+ version: 0.7.3
+ resolution: "impit-linux-arm64-gnu@npm:0.7.3"
conditions: os=linux & cpu=arm64 & libc=glibc
languageName: node
linkType: hard
@@ -7529,9 +7529,9 @@ __metadata:
languageName: node
linkType: hard
-"impit-linux-arm64-musl@npm:0.7.0":
- version: 0.7.0
- resolution: "impit-linux-arm64-musl@npm:0.7.0"
+"impit-linux-arm64-musl@npm:0.7.3":
+ version: 0.7.3
+ resolution: "impit-linux-arm64-musl@npm:0.7.3"
conditions: os=linux & cpu=arm64 & libc=musl
languageName: node
linkType: hard
@@ -7543,9 +7543,9 @@ __metadata:
languageName: node
linkType: hard
-"impit-linux-x64-gnu@npm:0.7.0":
- version: 0.7.0
- resolution: "impit-linux-x64-gnu@npm:0.7.0"
+"impit-linux-x64-gnu@npm:0.7.3":
+ version: 0.7.3
+ resolution: "impit-linux-x64-gnu@npm:0.7.3"
conditions: os=linux & cpu=x64 & libc=glibc
languageName: node
linkType: hard
@@ -7557,9 +7557,9 @@ __metadata:
languageName: node
linkType: hard
-"impit-linux-x64-musl@npm:0.7.0":
- version: 0.7.0
- resolution: "impit-linux-x64-musl@npm:0.7.0"
+"impit-linux-x64-musl@npm:0.7.3":
+ version: 0.7.3
+ resolution: "impit-linux-x64-musl@npm:0.7.3"
conditions: os=linux & cpu=x64 & libc=musl
languageName: node
linkType: hard
@@ -7571,9 +7571,9 @@ __metadata:
languageName: node
linkType: hard
-"impit-win32-arm64-msvc@npm:0.7.0":
- version: 0.7.0
- resolution: "impit-win32-arm64-msvc@npm:0.7.0"
+"impit-win32-arm64-msvc@npm:0.7.3":
+ version: 0.7.3
+ resolution: "impit-win32-arm64-msvc@npm:0.7.3"
conditions: os=win32 & cpu=arm64
languageName: node
linkType: hard
@@ -7585,9 +7585,9 @@ __metadata:
languageName: node
linkType: hard
-"impit-win32-x64-msvc@npm:0.7.0":
- version: 0.7.0
- resolution: "impit-win32-x64-msvc@npm:0.7.0"
+"impit-win32-x64-msvc@npm:0.7.3":
+ version: 0.7.3
+ resolution: "impit-win32-x64-msvc@npm:0.7.3"
conditions: os=win32 & cpu=x64
languageName: node
linkType: hard
@@ -7625,18 +7625,18 @@ __metadata:
languageName: node
linkType: hard
-"impit@npm:^0.7.0":
- version: 0.7.0
- resolution: "impit@npm:0.7.0"
- dependencies:
- impit-darwin-arm64: "npm:0.7.0"
- impit-darwin-x64: "npm:0.7.0"
- impit-linux-arm64-gnu: "npm:0.7.0"
- impit-linux-arm64-musl: "npm:0.7.0"
- impit-linux-x64-gnu: "npm:0.7.0"
- impit-linux-x64-musl: "npm:0.7.0"
- impit-win32-arm64-msvc: "npm:0.7.0"
- impit-win32-x64-msvc: "npm:0.7.0"
+"impit@npm:^0.7.3":
+ version: 0.7.3
+ resolution: "impit@npm:0.7.3"
+ dependencies:
+ impit-darwin-arm64: "npm:0.7.3"
+ impit-darwin-x64: "npm:0.7.3"
+ impit-linux-arm64-gnu: "npm:0.7.3"
+ impit-linux-arm64-musl: "npm:0.7.3"
+ impit-linux-x64-gnu: "npm:0.7.3"
+ impit-linux-x64-musl: "npm:0.7.3"
+ impit-win32-arm64-msvc: "npm:0.7.3"
+ impit-win32-x64-msvc: "npm:0.7.3"
dependenciesMeta:
impit-darwin-arm64:
optional: true
@@ -7654,7 +7654,7 @@ __metadata:
optional: true
impit-win32-x64-msvc:
optional: true
- checksum: 10c0/bd5887db006b68913de1480da47379f4de11cbc7ba9725fd0106faca993eaa16a7a13e9869cc641790880dd044a07f7ce6312848d05555a98b266487e0740f57
+ checksum: 10c0/8a7138303b5ea01d59e35c678e8c4a1483db1af722e1f747bf7c3635f86edcc51d49d8949820fd74993a041903d9fd29a57547cfe904a0bb5e9295e9c7269bf3
languageName: node
linkType: hard