Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(http-client): replay the request body on redirects without throwing (#4136)
Suliman Abdulrazzaq3d48f47b454d
diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.tsindex 2d4c1d0e7..eb98b85df 100644--- a/packages/http-client/src/base-http-client.ts+++ b/packages/http-client/src/base-http-client.ts@@ -183,7 +183,9 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {body: nextBody,credentials: (currentRequest as any).credentials,redirect: 'manual',- });+ // Node-specific option required for a stream body, which the replayed body always is+ duplex: 'half',+ } as RequestInit);}/**diff --git a/test/core/base-http-client.test.ts b/test/core/base-http-client.test.tsindex 2edc199e7..0e12b9171 100644--- a/test/core/base-http-client.test.ts+++ b/test/core/base-http-client.test.ts@@ -1,5 +1,6 @@import http from 'node:http';import type { AddressInfo } from 'node:net';+import { Readable } from 'node:stream';import type { CustomFetchOptions } from '@crawlee/http-client';import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';@@ -274,3 +275,60 @@ describe('BaseHttpClient TLS error handling', () => {expect(warningOnce).not.toHaveBeenCalled();});});++describe('BaseHttpClient redirects that keep the request body', () => {+ let redirectServer: http.Server;+ let redirectUrl: string;++ beforeAll(async () => {+ redirectServer = http.createServer(async (req, res) => {+ const { pathname, searchParams } = new URL(req.url!, 'http://localhost');++ if (pathname === '/redirect') {+ req.resume();+ res.writeHead(Number(searchParams.get('status')), { location: '/echo' });+ res.end();+ return;+ }++ let body = '';+ for await (const chunk of req) body += chunk;++ res.setHeader('content-type', 'application/json');+ res.end(JSON.stringify({ method: req.method, body }));+ });++ await new Promise<void>((resolve) => redirectServer.listen(resolve));+ redirectUrl = `http://127.0.0.1:${(redirectServer.address() as AddressInfo).port}`;+ });++ afterAll(async () => {+ await new Promise((resolve) => redirectServer.close(resolve));+ });++ test.each([+ [307, 'POST'],+ [308, 'POST'],+ [301, 'PUT'],+ [302, 'PATCH'],+ ])('resends the body after a %i redirect of a %s request', async (status, method) => {+ const init: RequestInit = { method, body: 'hello' };+ const response = await httpClient.sendRequest(new Request(`${redirectUrl}/redirect?status=${status}`, init));++ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({ method, body: 'hello' });+ });++ test('resends a streamed body after a 307 redirect', async () => {+ const response = await httpClient.sendRequest(+ new Request(`${redirectUrl}/redirect?status=307`, {+ method: 'POST',+ body: Readable.toWeb(Readable.from(['hel', 'lo'])) as ReadableStream,+ duplex: 'half',+ } as RequestInit),+ );++ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({ method: 'POST', body: 'hello' });+ });+});