Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type {
Awaitable,
Dictionary,
HttpRequestOptions,
ISession,
ProxyInfo,
SendRequestOptions,
} from '@crawlee/types';
import type { ReadonlyDeep } from 'type-fest';
import type {
AddRequestsBatchedResult,
CrawleeLogger,
Dataset,
KeyValueStore,
Request,
RequestOptions,
Source,
StorageIdentifier,
} from '@crawlee/core';
import type { EnqueueUrlsOptions } from '../enqueue-links/index.js';
import type { CrawlingRequest } from '../crawling-request.js';
/** @internal */
export type IsAny<T> = 0 extends 1 & T ? true : false;
/**
* A request input (URL string, request-options object, or {@apilink Request}) whose `userData` is typed
* according to its `label`, based on a router's route map.
*
* When the route map is open (the default `Record<string, ...>`), this is just the regular loose
* {@apilink Source} input. When the map declares concrete labels, providing a `label` requires the matching
* `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
*/
export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
? string | Source
:
| string
| Request
| ({ requestsFromUrl?: string; regex?: RegExp } & (
| {
[Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
label: Label;
};
}[keyof Routes & string]
| (Omit<Partial<RequestOptions>, 'label'> & { label?: undefined })
));
/**
* The iterable/array of {@apilink LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
* methods of a crawler bound to a typed router.
*/
export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> =
| AsyncIterable<LabeledSource<Routes>>
| Iterable<LabeledSource<Routes>>
| LabeledSource<Routes>[];
/**
* The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
* bound to a typed router. Mirrors {@apilink RestrictedCrawlingContext.addRequests} with typed sources.
*/
export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (
requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>,
options?: ReadonlyDeep<EnqueueUrlsOptions>,
) => Promise<AddRequestsBatchedResult>;
/**
* An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
* declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
* calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
*/
type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
? Options
: Omit<Options, 'label' | 'userData'> &
(
| { [Label in keyof Routes & string]: { label: Label; userData?: Routes[Label] } }[keyof Routes & string]
| { label?: undefined; userData?: Dictionary }
);
/**
* Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
* the router's route map, while preserving everything else about the signature (argument optionality and
* return type, which differ between crawler types).
*/
export type TypedContextEnqueueLinks<
EnqueueLinks,
Routes extends Record<keyof Routes, Dictionary>,
> = EnqueueLinks extends (options?: infer Options) => infer Result
? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result
: EnqueueLinks extends (options: infer Options) => infer Result
? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result
: EnqueueLinks;
/** A {@apilink Request} that has been dispatched, so its `id` and `loadedUrl` are guaranteed to be present. */
// `Required` (homomorphic) strips `undefined`; an inline mapped type over a literal union would only drop the `?`.
export type LoadedRequest<R extends Request> = R & Required<Pick<R, 'id' | 'loadedUrl'>>;
/** @internal */
export type LoadedContext<Context extends RestrictedCrawlingContext> =
IsAny<Context> extends true
? Context
: {
request: LoadedRequest<Context['request']>;
} & Omit<Context, 'request'>;
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
/** @internal */
id: string;
session: ISession;
/**
* An object with information about currently used proxy by the crawler
* and configured by the {@apilink ProxyConfiguration} class.
*/
proxyInfo?: ProxyInfo;
/**
* The request being processed.
*/
request: CrawlingRequest<UserData>;
/**
* This function allows you to push data to a {@apilink Dataset} specified by name, or the one currently used by the crawler.
*
* Shortcut for `crawler.pushData()`.
*
* @param [data] Data to be pushed to the default dataset.
*/
pushData(
data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>,
datasetIdentifier?: string | StorageIdentifier,
): Promise<void>;
/**
* Add requests directly to the request queue currently used by the crawler.
*
* Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
* the same way {@apilink CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
*
* @param requests The requests to add
* @param options Options for the request queue
*/
addRequests: (
requestsLike: ReadonlyDeep<(string | Source)[]>,
options?: ReadonlyDeep<EnqueueUrlsOptions>,
) => Promise<AddRequestsBatchedResult>;
/**
* Returns the state - a piece of mutable persistent data shared across all the request handler runs.
*/
useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
/**
* Get a key-value store with given name or id, or the default one for the crawler.
*/
getKeyValueStore: (
identifier?: string | StorageIdentifier,
) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
/**
* A preconfigured logger for the request handler.
*/
log: CrawleeLogger;
/**
* Stops processing the current request and marks it as handled without counting it as a failure: no retry, no
* `failedRequestHandler`, and no error log. The request's `state` becomes {@apilink RequestState.SKIPPED|`SKIPPED`}
* and {@apilink BasicCrawlerOptions.onSkippedRequest|`onSkippedRequest`} fires with the `'manual'` reason.
*
* Storage writes made for the request so far are rolled back, as with a failure, unless
* {@apilink BasicCrawlerOptions.transactionalStorage|`transactionalStorage`} is disabled.
*
* Works in `errorHandler` too, skipping a request that would otherwise be retried. In `failedRequestHandler` it
* has no effect, since the request has already failed.
*
* ```ts
* preNavigationHooks: [
* async ({ request, skipRequest }) => {
* if (await alreadyScraped(request.url)) skipRequest('already scraped');
* },
* ],
* ```
*
* @param message Passed to `onSkippedRequest`.
*/
skipRequest(message?: string): never;
}
export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
/**
* Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
*
* This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
* Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
* more detailed explanation of how to do that.
*
* ```ts
* async requestHandler({ sendRequest }) {
* const { body } = await sendRequest({
* // override headers only
* headers: { ... },
* });
* },
* ```
*/
sendRequest: (
requestOverrides?: Partial<HttpRequestOptions>,
optionsOverrides?: SendRequestOptions,
) => Promise<Response>;
/**
* Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
*
* The callback runs *outside* the request's storage transaction, so storage writes made here are
* applied immediately and are **not** rolled back when the request fails. In
* {@apilink AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
* here can land more than once for a single request. Push results from the request handler itself.
*/
registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
/**
* Registers `callback` to run once the request's storage writes have been committed, or once
* committing them has failed - the point where a write made in the handler can finally be reacted
* to where it was made. Two things belong here: side effects that must agree with what actually
* landed (result counters, progress reporting), and handling of a write that did not land at all.
*
* ```ts
* async requestHandler({ pushData, afterStorageCommit, useState }) {
* const state = await useState({ itemCount: 0 });
*
* await pushData(item);
* afterStorageCommit((error) => {
* if (error) {
* throw new NonRetryableError('The page is too big to store', { cause: error });
* }
* state.itemCount++;
* });
* }
* ```
*
* `useState()` is deliberately not transactional, which is what makes the counter case necessary:
* incrementing in the handler would keep the increment for a request that pushes an item and then
* fails, while the item itself is rolled back.
*
* Callbacks run in registration order, after the commit and before the request is marked as handled.
* The first one to throw propagates and the rest do not run; on a failed commit its error replaces
* the commit error, so a callback can decide what the request fails with. After a *successful*
* commit, a callback that throws fails the request **without a retry** whatever it throws - the
* writes are already durable and retrying would duplicate them, so anything that is not already a
* {@apilink NonRetryableError} is wrapped in an {@apilink AfterCommitError}.
*
* Callbacks are *not* run when the request handler itself fails - nothing was written, and
* `errorHandler` / `failedRequestHandler` cover that.
*
* Throws when storage is not transactional (`transactionalStorage: false`) - writes are then applied
* as they are made and throw at the call site on their own. In {@apilink AdaptivePlaywrightCrawler}
* the callback belongs to the current request handler attempt, so it runs only for the attempt whose
* writes are the ones being committed.
*/
afterStorageCommit(callback: (error?: Error) => Awaitable<void>): void;
/**
* Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent
* once it is already running - a listing page that turns out to have far more to scroll through than
* usual, say. Prefer `requestHandlerTimeoutSecs`, or a per-route override via
* {@apilink Router.addHandler|`router.addHandler`}, whenever the time needed is known up front.
*
* ```ts
* router.addHandler('LIST', async ({ extendTimeout, page }) => {
* const pageCount = await countPages(page);
* extendTimeout(pageCount * 10);
* await scrapeAllPages(page);
* });
* ```
*
* Extends the request handler's own timeout and the crawler's internal one together, so the extension
* is not immediately undone by the latter. On a locking storage backend the request's lock is
* extended by the same amount, so the extra time is not spent while the queue considers the request
* free to hand out again. Calling it from a handler that has already timed out does nothing.
*/
extendTimeout(secs: number): void;
}