Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: prefer interfaces over types where possible for better API docs

Martin Adámekaad900ab1811
diff --git a/.eslintrc.json b/.eslintrc.json
index c3fc7f95b..563a03e54 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -33,6 +33,7 @@
"@typescript-eslint/consistent-type-imports": ["error", {
"disallowTypeAnnotations": false
}],
+ "@typescript-eslint/consistent-type-definitions": ["error", "interface"],
"@typescript-eslint/member-delimiter-style": ["error", {
"multiline": { "delimiter": "semi", "requireLast": true },
"singleline": { "delimiter": "semi", "requireLast": false }
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 96e6734d3..278018740 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -46,7 +46,7 @@ export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>
}
/** @internal */
-export type BasicCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'requestQueue'>
+export interface BasicCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'requestQueue'> {}
/**
* Since there's no set number of seconds before the container is terminated after
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 3e4990ace..9848f973a 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -15,15 +15,8 @@ import {
resolveBaseUrl,
Configuration,
} from '@crawlee/core';
-import type {
- BasicCrawlerOptions,
- Awaitable,
- Dictionary,
-} from '@crawlee/basic';
-import {
- BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
- BasicCrawler,
-} from '@crawlee/basic';
+import type { BasicCrawlerOptions, Awaitable, Dictionary, RequestHandler, ErrorHandler } from '@crawlee/basic';
+import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler } from '@crawlee/basic';
import type {
BrowserController,
BrowserPlugin,
@@ -33,10 +26,7 @@ import type {
InferBrowserPluginArray,
LaunchContext,
} from '@crawlee/browser-pool';
-import {
- BROWSER_CONTROLLER_EVENTS,
- BrowserPool,
-} from '@crawlee/browser-pool';
+import { BROWSER_CONTROLLER_EVENTS, BrowserPool } from '@crawlee/browser-pool';
import type { GotOptionsInit, Response as GotResponse } from 'got-scraping';
import ow from 'ow';
import { Cookie } from 'tough-cookie';
@@ -57,13 +47,11 @@ export interface BrowserCrawlingContext<
sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;
}
-export type BrowserCrawlerHandleRequest<
- Context extends BrowserCrawlingContext = BrowserCrawlingContext> = (inputs: Context) => Awaitable<void>;
+export type BrowserRequestHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = RequestHandler<Context>;
-export type BrowserCrawlerHandleFailedRequest<
- Context extends BrowserCrawlingContext= BrowserCrawlingContext>= (inputs: Context, error: Error) => Awaitable<void>;
+export type BrowserErrorHandler<Context extends BrowserCrawlingContext= BrowserCrawlingContext>= ErrorHandler<Context>;
-export type BrowserCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'requestQueue' | 'urls'>
+export interface BrowserCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'requestQueue' | 'urls'> {}
export type BrowserHook<
Context = BrowserCrawlingContext,
@@ -116,7 +104,7 @@ export interface BrowserCrawlerOptions<
* The exceptions are logged to the request using the
* {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
*/
- requestHandler?: BrowserCrawlerHandleRequest<Context>;
+ requestHandler?: BrowserRequestHandler<Context>;
/**
* Function that is called to process each request.
@@ -148,7 +136,7 @@ export interface BrowserCrawlerOptions<
* @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.
* @ignore
*/
- handlePageFunction?: BrowserCrawlerHandleRequest<Context>;
+ handlePageFunction?: BrowserRequestHandler<Context>;
/**
* User-provided function that allows modifying the request object before it gets retried by the crawler.
@@ -160,7 +148,7 @@ export interface BrowserCrawlerOptions<
* Second argument is the `Error` instance that
* represents the last error thrown during processing of the request.
*/
- errorHandler?: BrowserCrawlerHandleFailedRequest<Context>;
+ errorHandler?: BrowserErrorHandler<Context>;
/**
* A function to handle requests that failed more than `option.maxRequestRetries` times.
@@ -171,7 +159,7 @@ export interface BrowserCrawlerOptions<
* Second argument is the `Error` instance that
* represents the last error thrown during processing of the request.
*/
- failedRequestHandler?: BrowserCrawlerHandleFailedRequest<Context>;
+ failedRequestHandler?: BrowserErrorHandler<Context>;
/**
* A function to handle requests that failed more than `option.maxRequestRetries` times.
@@ -185,7 +173,7 @@ export interface BrowserCrawlerOptions<
* @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
* @ignore
*/
- handleFailedRequestFunction?: BrowserCrawlerHandleFailedRequest<Context>;
+ handleFailedRequestFunction?: BrowserErrorHandler<Context>;
/**
* Custom options passed to the underlying {@link BrowserPool} constructor.
@@ -314,7 +302,7 @@ export abstract class BrowserCrawler<
launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
- protected userProvidedRequestHandler!: BrowserCrawlerHandleRequest<Context>;
+ protected userProvidedRequestHandler!: BrowserRequestHandler<Context>;
protected navigationTimeoutMillis: number;
protected preNavigationHooks: BrowserHook<Context>[];
protected postNavigationHooks: BrowserHook<Context>[];
diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
index 18774e36c..0b48aea5a 100644
--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
@@ -64,15 +64,15 @@ export interface BrowserPluginOptions<LibraryOptions> {
* Path to a User Data Directory, which stores browser session data like cookies and local storage.
*/
userDataDir?: string;
-};
+}
-export type CreateLaunchContextOptions<
+export interface CreateLaunchContextOptions<
Library extends CommonLibrary,
LibraryOptions = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
-> = Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>>;
+> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {}
/**
* The `BrowserPlugin` serves two purposes. First, it is the base class that
diff --git a/packages/browser-pool/src/fingerprinting/types.ts b/packages/browser-pool/src/fingerprinting/types.ts
index 120a47449..fa3cd77a7 100644
--- a/packages/browser-pool/src/fingerprinting/types.ts
+++ b/packages/browser-pool/src/fingerprinting/types.ts
@@ -4,11 +4,11 @@ export interface FingerprintGenerator {
getFingerprint: (fingerprintGeneratorOptions?: FingerprintGeneratorOptions) => GetFingerprintReturn;
}
-export type GetFingerprintReturn = {
+export interface GetFingerprintReturn {
fingerprint: Fingerprint;
}
-export type FingerprintGeneratorOptions = {
+export interface FingerprintGeneratorOptions {
browsers?: BrowserName[] | BrowserSpecification[];
operatingSystems?: OperatingSystemsName[];
devices?: DeviceCategory[];
@@ -21,7 +21,7 @@ export const enum BrowserName {
safari = 'safari',
}
-export type BrowserSpecification = {
+export interface BrowserSpecification {
name: BrowserName;
minVersion?: number;
maxVersion?: number;
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 8a9ea0e54..f10f078a6 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -1,6 +1,6 @@
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';
-import type { BasicCrawlerOptions, ErrorHandler } from '@crawlee/basic';
+import type { BasicCrawlerOptions, ErrorHandler, RequestHandler } from '@crawlee/basic';
import { BasicCrawler, BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '@crawlee/basic';
import type { CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, Request, RequestQueue, Session } from '@crawlee/core';
import { CrawlerExtension, enqueueLinks, mergeCookies, Router, resolveBaseUrl, validators } from '@crawlee/core';
@@ -267,8 +267,8 @@ export interface CheerioCrawlingContext<JSONData extends Dictionary = Dictionary
sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;
}
-export type CheerioRequestHandler<JSONData = Dictionary> = (inputs: CheerioCrawlingContext<JSONData>) => Awaitable<void>;
-export type CheerioCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'>;
+export type CheerioRequestHandler<JSONData = Dictionary> = RequestHandler<CheerioCrawlingContext<JSONData>>;
+export interface CheerioCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}
/**
* Provides a framework for the parallel crawling of web pages using plain HTTP requests and
diff --git a/packages/core/src/autoscaling/snapshotter.ts b/packages/core/src/autoscaling/snapshotter.ts
index b035d7f81..6ce5a16dd 100644
--- a/packages/core/src/autoscaling/snapshotter.ts
+++ b/packages/core/src/autoscaling/snapshotter.ts
@@ -66,10 +66,10 @@ export interface SnapshotterOptions {
config?: Configuration;
}
-type MemorySnapshot = { createdAt: Date; isOverloaded: boolean; usedBytes?: number };
-type CpuSnapshot = { createdAt: Date; isOverloaded: boolean; usedRatio: number; ticks?: { idle: number; total: number } };
-type EventLoopSnapshot = { createdAt: Date; isOverloaded: boolean; exceededMillis: number };
-type ClientSnapshot = { createdAt: Date; isOverloaded: boolean; rateLimitErrorCount: number };
+interface MemorySnapshot { createdAt: Date; isOverloaded: boolean; usedBytes?: number }
+interface CpuSnapshot { createdAt: Date; isOverloaded: boolean; usedRatio: number; ticks?: { idle: number; total: number } }
+interface EventLoopSnapshot { createdAt: Date; isOverloaded: boolean; exceededMillis: number }
+interface ClientSnapshot { createdAt: Date; isOverloaded: boolean; rateLimitErrorCount: number }
/**
* Creates snapshots of system resources at given intervals and marks the resource
diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.ts
index bd6bd6bfb..033f7a7fa 100644
--- a/packages/core/src/storages/request_list.ts
+++ b/packages/core/src/storages/request_list.ts
@@ -893,5 +893,5 @@ export interface RequestListState {
}
export type Source = string | (Partial<RequestOptions> & { requestsFromUrl?: string; regex?: RegExp }) | Request;
-type InternalSource = { requestsFromUrl: string; regex?: RegExp };
+interface InternalSource { requestsFromUrl: string; regex?: RegExp }
export type RequestListSourcesFunction = () => Promise<Source[]>;
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 9e484f639..77fb796ac 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -1,7 +1,7 @@
import ow from 'ow';
import type { LaunchOptions, Page, Response } from 'playwright';
import type { BrowserPoolOptions, PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserCrawlerHandleRequest, BrowserHook } from '@crawlee/browser';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserRequestHandler, BrowserHook } from '@crawlee/browser';
import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';
import type { Dictionary } from '@crawlee/types';
import { Cookie } from 'tough-cookie';
@@ -10,10 +10,10 @@ import { PlaywrightLauncher } from './playwright-launcher';
import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils';
import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils';
-export type PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> =
- BrowserCrawlingContext<Page, Response, PlaywrightController, UserData> & PlaywrightContextUtils;
-export type PlaywrightHook = BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions>;
-export type PlaywrightRequestHandler = BrowserCrawlerHandleRequest<PlaywrightCrawlingContext>;
+export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends
+ BrowserCrawlingContext<Page, Response, PlaywrightController, UserData>, PlaywrightContextUtils {}
+export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {}
+export interface PlaywrightRequestHandler extends BrowserRequestHandler<PlaywrightCrawlingContext> {}
export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
export type PlaywrightCookie = Parameters<ReturnType<Page['context']>['addCookies']>[0][0];
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index b2ea43cdc..bbc07cd71 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -1,5 +1,5 @@
import type {
- BrowserCrawlerHandleRequest,
+ BrowserRequestHandler,
BrowserCrawlerOptions,
BrowserCrawlingContext,
BrowserHook,
@@ -14,10 +14,10 @@ import { PuppeteerLauncher } from './puppeteer-launcher';
import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils';
import { gotoExtended, registerUtilsToContext } from './utils/puppeteer_utils';
-export type PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> =
- BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, UserData> & PuppeteerContextUtils;
-export type PuppeteerHook = BrowserHook<PuppeteerCrawlingContext, PuppeteerGoToOptions>;
-export type PuppeteerRequestHandler = BrowserCrawlerHandleRequest<PuppeteerCrawlingContext>;
+export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends
+ BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, UserData>, PuppeteerContextUtils {}
+export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, PuppeteerGoToOptions> {}
+export interface PuppeteerRequestHandler extends BrowserRequestHandler<PuppeteerCrawlingContext> {}
export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<