Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `AbortSignal` and rename `timeout` → `timeoutMillis` in `SendRequestOptions` (#3457)

Copilotc3e56970a601
diff --git a/packages/basic-crawler/src/internals/send-request.ts b/packages/basic-crawler/src/internals/send-request.ts
index e5aee681a..4e826ef05 100644
--- a/packages/basic-crawler/src/internals/send-request.ts
+++ b/packages/basic-crawler/src/internals/send-request.ts
@@ -38,7 +38,8 @@ export function createSendRequest(
return httpClient.sendRequest(request, {
session,
cookieJar: overrideOptions?.cookieJar ?? session?.cookieJar,
- timeout: overrideOptions.timeout,
+ timeoutMillis: overrideOptions.timeoutMillis,
+ signal: overrideOptions.signal,
});
};
}
diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts
index 94f9e2fc8..2491c432a 100644
--- a/packages/http-client/src/base-http-client.ts
+++ b/packages/http-client/src/base-http-client.ts
@@ -56,16 +56,22 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
private resolveRequestContext(options?: SendRequestOptions): {
proxyUrl?: string;
cookieJar: CookieJar;
- timeout?: number;
+ signal?: AbortSignal;
} {
const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url;
const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? new CookieJar();
- const timeout = options?.timeout;
- return { proxyUrl, cookieJar: cookieJar as CookieJar, timeout };
+ const signal = this.createAbortSignal(options?.signal, options?.timeoutMillis);
+ return { proxyUrl, cookieJar: cookieJar as CookieJar, signal };
}
- private createAbortSignal(timeout?: number): AbortSignal | undefined {
- return timeout ? AbortSignal.timeout(timeout) : undefined;
+ private createAbortSignal(signal?: AbortSignal, timeoutMillis?: number): AbortSignal | undefined {
+ if (signal && timeoutMillis) {
+ return AbortSignal.any([signal, AbortSignal.timeout(timeoutMillis)]);
+ }
+ if (signal) {
+ return signal;
+ }
+ return timeoutMillis ? AbortSignal.timeout(timeoutMillis) : undefined;
}
private isRedirect(response: Response): boolean {
@@ -112,14 +118,14 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
let currentRequest = initialRequest;
let redirectCount = 0;
- const { proxyUrl, cookieJar, timeout } = this.resolveRequestContext(options);
+ const { proxyUrl, cookieJar, signal } = this.resolveRequestContext(options);
currentRequest = initialRequest.clone();
while (true) {
await this.applyCookies(currentRequest, cookieJar);
const response = await this.fetch(currentRequest, {
- signal: this.createAbortSignal(timeout),
+ signal,
proxyUrl,
redirect: 'manual',
});
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index cf2d18825..ad73a5fa1 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -818,7 +818,7 @@ export class HttpCrawler<
} as RequestInit),
{
session,
- timeout: opts.timeout,
+ timeoutMillis: opts.timeout,
},
);
diff --git a/packages/types/src/http-client.ts b/packages/types/src/http-client.ts
index 5ce40556a..5362750f0 100644
--- a/packages/types/src/http-client.ts
+++ b/packages/types/src/http-client.ts
@@ -66,7 +66,10 @@ export type RedirectHandler = (
export interface SendRequestOptions {
session?: ISession;
cookieJar?: CookieJar;
- timeout?: number;
+ /** Timeout for the HTTP request in milliseconds. */
+ timeoutMillis?: number;
+ /** An AbortSignal to cancel the HTTP request. */
+ signal?: AbortSignal;
/**
* Overrides the proxy URL set in the `session` for this request.
*
diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.ts
index bee0e6fab..a91e46662 100644
--- a/packages/utils/src/internals/robots.ts
+++ b/packages/utils/src/internals/robots.ts
@@ -77,9 +77,10 @@ export class RobotsTxtFile {
): Promise<RobotsTxtFile> {
const { proxyUrl, httpClient = new FetchHttpClient() } = options || {};
- const response = await httpClient.sendRequest(new Request(url, { method: 'GET', signal: options?.signal }), {
+ const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
proxyUrl,
- timeout: options?.timeoutMillis,
+ timeoutMillis: options?.timeoutMillis,
+ signal: options?.signal,
});
if (response.status < 200 || response.status >= 300) {
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index de57d9bb0..9067e66c5 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -294,7 +294,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
}),
{
proxyUrl,
- timeout,
+ timeoutMillis: timeout,
},
);
} catch (error: any) {
@@ -596,7 +596,11 @@ export async function* discoverValidSitemaps(
return false;
}
try {
- const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), { proxyUrl });
+ const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
+ proxyUrl,
+ timeoutMillis: requestTimeoutMillis,
+ signal,
+ });
return response.status >= 200 && response.status < 400;
} catch {
return false;
diff --git a/packages/utils/test/mock-http-client.ts b/packages/utils/test/mock-http-client.ts
deleted file mode 100644
index 14673d560..000000000
--- a/packages/utils/test/mock-http-client.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import type { BaseHttpClient, SendRequestOptions, StreamOptions } from '@crawlee/types';
-
-export class FetchHttpClient implements BaseHttpClient {
- async sendRequest(request: Request, options?: SendRequestOptions): Promise<Response> {
- const signal = AbortSignal.timeout(options?.timeout ?? 30000);
- const response = await fetch(request, { signal });
- return response;
- }
-
- async stream(request: Request, options?: StreamOptions): Promise<Response> {
- return this.sendRequest(request, options);
- }
-}
diff --git a/packages/utils/test/robots.test.ts b/packages/utils/test/robots.test.ts
index c7db055f6..4cc72f588 100644
--- a/packages/utils/test/robots.test.ts
+++ b/packages/utils/test/robots.test.ts
@@ -1,8 +1,8 @@
+import { FetchHttpClient } from '@crawlee/http-client';
import nock from 'nock';
import { beforeEach, describe, expect, it } from 'vitest';
import { RobotsTxtFile } from '../src/internals/robots.js';
-import { FetchHttpClient } from './mock-http-client.js';
const httpClient = new FetchHttpClient();
@@ -63,7 +63,7 @@ describe('RobotsTxtFile', () => {
it('respects user-set timeout', async () => {
const start = +Date.now();
- const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', undefined, { timeoutMillis: 200 });
+ const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', { timeoutMillis: 200 });
await expect(robots).rejects.toThrow(/timeout/i);
const end = +Date.now();
@@ -77,7 +77,7 @@ describe('RobotsTxtFile', () => {
setTimeout(() => controller.abort(), 200);
const start = +Date.now();
- const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', undefined, { signal: controller.signal });
+ const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', { signal: controller.signal });
await expect(robots).rejects.toThrow(/aborted/i);
const end = +Date.now();
@@ -90,7 +90,7 @@ describe('RobotsTxtFile', () => {
const controller = new AbortController();
const start = +Date.now();
- const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', undefined, {
+ const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', {
signal: controller.signal,
timeoutMillis: 200,
});
diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.ts
index e952b2000..260c351bc 100644
--- a/packages/utils/test/sitemap.test.ts
+++ b/packages/utils/test/sitemap.test.ts
@@ -1,3 +1,4 @@
+import { FetchHttpClient } from '@crawlee/http-client';
import nock from 'nock';
import { beforeEach, describe, expect, it, vi } from 'vitest';
@@ -5,7 +6,6 @@ import log from '@apify/log';
import type { SitemapUrl } from '../src/internals/sitemap.js';
import { discoverValidSitemaps, parseSitemap, Sitemap } from '../src/internals/sitemap.js';
-import { FetchHttpClient } from './mock-http-client.js';
describe('Sitemap', () => {
beforeEach(() => {
diff --git a/test/core/base-http-client.test.ts b/test/core/base-http-client.test.ts
new file mode 100644
index 000000000..4713b19e4
--- /dev/null
+++ b/test/core/base-http-client.test.ts
@@ -0,0 +1,95 @@
+import http from 'node:http';
+import type { AddressInfo } from 'node:net';
+
+import { FetchHttpClient } from '@crawlee/http-client';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+
+let server: http.Server;
+let url: string;
+
+beforeAll(async () => {
+ server = http.createServer((_req, res) => {
+ res.setHeader('content-type', 'text/plain');
+ res.end('ok');
+ });
+
+ await new Promise<void>((resolve) =>
+ server.listen(() => {
+ url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+ resolve();
+ }),
+ );
+});
+
+afterAll(async () => {
+ await new Promise((resolve) => server.close(resolve));
+});
+
+const httpClient = new FetchHttpClient();
+
+describe('BaseHttpClient signal and timeoutMillis options', () => {
+ test('sends a request without any signal or timeout', async () => {
+ const response = await httpClient.sendRequest(new Request(url));
+ expect(response.status).toBe(200);
+ });
+
+ test('aborts when a pre-aborted signal is passed', async () => {
+ const controller = new AbortController();
+ controller.abort();
+
+ await expect(httpClient.sendRequest(new Request(url), { signal: controller.signal })).rejects.toThrow();
+ });
+
+ test('aborts when the signal is aborted after the request starts', async () => {
+ const controller = new AbortController();
+ setTimeout(() => controller.abort(), 50);
+
+ const slowServer = http.createServer((_req, res) => {
+ setTimeout(() => res.end('late'), 500);
+ });
+
+ await new Promise<void>((r) => slowServer.listen(r));
+ const slowUrl = `http://127.0.0.1:${(slowServer.address() as AddressInfo).port}`;
+
+ try {
+ await expect(httpClient.sendRequest(new Request(slowUrl), { signal: controller.signal })).rejects.toThrow();
+ } finally {
+ await new Promise((r) => slowServer.close(r));
+ }
+ });
+
+ test('aborts when timeoutMillis elapses', async () => {
+ const slowServer = http.createServer((_req, res) => {
+ setTimeout(() => res.end('late'), 500);
+ });
+
+ await new Promise<void>((r) => slowServer.listen(r));
+ const slowUrl = `http://127.0.0.1:${(slowServer.address() as AddressInfo).port}`;
+
+ try {
+ await expect(httpClient.sendRequest(new Request(slowUrl), { timeoutMillis: 50 })).rejects.toThrow();
+ } finally {
+ await new Promise((r) => slowServer.close(r));
+ }
+ });
+
+ test('aborts when both signal and timeoutMillis are provided and the signal fires first', async () => {
+ const slowServer = http.createServer((_req, res) => {
+ setTimeout(() => res.end('late'), 500);
+ });
+
+ await new Promise<void>((r) => slowServer.listen(r));
+ const slowUrl = `http://127.0.0.1:${(slowServer.address() as AddressInfo).port}`;
+
+ const controller = new AbortController();
+ setTimeout(() => controller.abort(), 50);
+
+ try {
+ await expect(
+ httpClient.sendRequest(new Request(slowUrl), { signal: controller.signal, timeoutMillis: 5_000 }),
+ ).rejects.toThrow();
+ } finally {
+ await new Promise((r) => slowServer.close(r));
+ }
+ });
+});