Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(http-client): replay the request body on redirects without throwing (#4136)

Suliman Abdulrazzaq3d48f47b454d
diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts
index 2d4c1d0e7..eb98b85df 100644
--- a/packages/http-client/src/base-http-client.ts
+++ b/packages/http-client/src/base-http-client.ts
@@ -183,7 +183,9 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
body: nextBody,
credentials: (currentRequest as any).credentials,
redirect: 'manual',
- });
+ // Node-specific option required for a stream body, which the replayed body always is
+ duplex: 'half',
+ } as RequestInit);
}
/**
diff --git a/test/core/base-http-client.test.ts b/test/core/base-http-client.test.ts
index 2edc199e7..0e12b9171 100644
--- a/test/core/base-http-client.test.ts
+++ b/test/core/base-http-client.test.ts
@@ -1,5 +1,6 @@
import http from 'node:http';
import type { AddressInfo } from 'node:net';
+import { Readable } from 'node:stream';
import type { CustomFetchOptions } from '@crawlee/http-client';
import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
@@ -274,3 +275,60 @@ describe('BaseHttpClient TLS error handling', () => {
expect(warningOnce).not.toHaveBeenCalled();
});
});
+
+describe('BaseHttpClient redirects that keep the request body', () => {
+ let redirectServer: http.Server;
+ let redirectUrl: string;
+
+ beforeAll(async () => {
+ redirectServer = http.createServer(async (req, res) => {
+ const { pathname, searchParams } = new URL(req.url!, 'http://localhost');
+
+ if (pathname === '/redirect') {
+ req.resume();
+ res.writeHead(Number(searchParams.get('status')), { location: '/echo' });
+ res.end();
+ return;
+ }
+
+ let body = '';
+ for await (const chunk of req) body += chunk;
+
+ res.setHeader('content-type', 'application/json');
+ res.end(JSON.stringify({ method: req.method, body }));
+ });
+
+ await new Promise<void>((resolve) => redirectServer.listen(resolve));
+ redirectUrl = `http://127.0.0.1:${(redirectServer.address() as AddressInfo).port}`;
+ });
+
+ afterAll(async () => {
+ await new Promise((resolve) => redirectServer.close(resolve));
+ });
+
+ test.each([
+ [307, 'POST'],
+ [308, 'POST'],
+ [301, 'PUT'],
+ [302, 'PATCH'],
+ ])('resends the body after a %i redirect of a %s request', async (status, method) => {
+ const init: RequestInit = { method, body: 'hello' };
+ const response = await httpClient.sendRequest(new Request(`${redirectUrl}/redirect?status=${status}`, init));
+
+ expect(response.status).toBe(200);
+ expect(await response.json()).toEqual({ method, body: 'hello' });
+ });
+
+ test('resends a streamed body after a 307 redirect', async () => {
+ const response = await httpClient.sendRequest(
+ new Request(`${redirectUrl}/redirect?status=307`, {
+ method: 'POST',
+ body: Readable.toWeb(Readable.from(['hel', 'lo'])) as ReadableStream,
+ duplex: 'half',
+ } as RequestInit),
+ );
+
+ expect(response.status).toBe(200);
+ expect(await response.json()).toEqual({ method: 'POST', body: 'hello' });
+ });
+});