Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import crypto from 'node:crypto';
import type { BaseHttpClient } from '@crawlee/http-client';
import type { Dictionary, StorageBackend } from '@crawlee/types';
import { Configuration } from '../configuration.js';
import type { IProxyConfiguration } from '../proxy-configuration.js';
import { serviceLocator } from '../service-locator.js';
import { KeyValueStore } from './key-value-store.js';
/**
* Options for purging default storage.
*/
interface PurgeDefaultStorageOptions {
/**
* If set to `true`, calling multiple times will only have effect at the first time.
*/
onlyPurgeOnce?: boolean;
configuration?: Configuration;
storageBackend?: StorageBackend;
}
/**
* Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
* Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
* Named storages persist across runs and are not touched.
*
* Purging of storages is happening automatically when we run our crawler (or when we open some storage
* explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}
* option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
*
* This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
* it will call the `purge` method of the underlying storage implementation we are currently using. You can
* make sure the storage is purged only once for a given execution context if you set `onlyPurgeOnce` to `true` in
* the `options` object
*/
export async function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;
/**
* Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
* Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
* Named storages persist across runs and are not touched.
*
* Purging of storages is happening automatically when we run our crawler (or when we open some storage
* explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}
* option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
*
* This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
* it will call the `purge` method of the underlying storage implementation we are currently using.
*/
export async function purgeDefaultStorages(
configuration?: Configuration,
storageBackend?: StorageBackend,
): Promise<void>;
export async function purgeDefaultStorages(
configurationOrOptions?: Configuration | PurgeDefaultStorageOptions,
storageBackend?: StorageBackend,
) {
const options: PurgeDefaultStorageOptions =
configurationOrOptions instanceof Configuration
? {
storageBackend,
configuration: configurationOrOptions,
}
: (configurationOrOptions ?? {});
const { configuration = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
({ storageBackend = serviceLocator.getStorageBackend() } = options);
const casted = storageBackend as StorageBackend & { __purged?: Promise<void> };
const runPurge = async () => {
try {
await casted.purge?.();
} catch (e) {
casted.__purged = undefined;
throw e;
}
};
// if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
if (!onlyPurgeOnce || (configuration.purgeOnStart && !casted.__purged)) {
casted.__purged = runPurge();
}
await casted.__purged;
}
export interface UseStateOptions {
configuration?: Configuration;
/**
* The name of the key-value store you'd like the state to be stored in.
* If not provided, the default store will be used.
*/
keyValueStoreName?: string | null;
}
/**
* Easily create and manage state values. All state values are automatically persisted.
*
* Values can be modified by simply using the assignment operator.
*
* @param name The name of the store to use.
* @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
* @param options An optional object parameter where a custom `keyValueStoreName` and `configuration` can be passed in.
*/
export async function useState<State extends Dictionary = Dictionary>(
name?: string,
defaultValue = {} as State,
options?: UseStateOptions,
) {
const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
configuration: options?.configuration || serviceLocator.getConfiguration(),
});
return kvStore.getAutoSavedValue<State>(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
}
/**
* Helper function that creates ID from uniqueKey for local emulation of request queue.
* It's also used for local cache of remote request queue.
*
* This function may not exactly match how requestId is created server side.
* So we never pass requestId created by this to server and use it only for local cache.
*
* @internal
*/
export function getRequestId(uniqueKey: string) {
const str = crypto.createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, '');
return str.slice(0, 15);
}
/**
* When requesting queue head we always fetch requestsInProgressCount * QUERY_HEAD_BUFFER number of requests.
* @internal
*/
export const QUERY_HEAD_MIN_LENGTH = 100;
/**
* Indicates how long it usually takes for the underlying storage to propagate all writes
* to be available to subsequent reads.
* @internal
*/
export const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
/** @internal */
export const QUERY_HEAD_BUFFER = 3;
/**
* If queue was modified (request added/updated/deleted) before more than API_PROCESSED_REQUESTS_DELAY_MILLIS
* then we assume the get head operation to be consistent.
* @internal
*/
export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10_000;
/**
* How many times we try to get queue head with queueModifiedAt older than API_PROCESSED_REQUESTS_DELAY_MILLIS.
* @internal
*/
export const MAX_QUERIES_FOR_CONSISTENCY = 6;
/** @internal */
export interface DualIterableOptions<TItem, TRawPage> {
/** Factory that returns an async generator yielding pages. */
createPages: () => AsyncGenerator<TRawPage>;
/** Extracts individual items from a page (for iteration). */
extractItems: (page: TRawPage) => TItem[];
}
/**
* Creates an object that is both an `AsyncIterable<TItem>` (for `for await...of`)
* and a `Promise<TItem[]>` (for `await`) from a single async page generator.
*
* - `await result` drains all pages from a fresh generator and returns every
* item as a flat array.
* - `for await (const item of result)` streams all items across all pages,
* yielding them one by one without buffering everything in memory.
*
* Each usage path creates its own generator instance, so `await` and
* `for await...of` never interfere with each other.
*
* @internal
*/
export function createDualIterable<TItem, TRawPage>(
options: DualIterableOptions<TItem, TRawPage>,
): AsyncIterable<TItem> & Promise<TItem[]> {
const { createPages, extractItems } = options;
let cached: Promise<TItem[]> | null = null;
function getOrCreate(): Promise<TItem[]> {
if (!cached) {
cached = (async () => {
const items: TItem[] = [];
for await (const page of createPages()) {
items.push(...extractItems(page));
}
return items;
})();
}
return cached;
}
async function* iterateAll(): AsyncGenerator<TItem> {
for await (const page of createPages()) {
yield* extractItems(page);
}
}
const result = {
[Symbol.asyncIterator]() {
return iterateAll();
},
then<TResult1 = TItem[], TResult2 = never>(
onfulfilled?: ((value: TItem[]) => TResult1 | PromiseLike<TResult1>) | null,
onrejected?: ((reason: any) => TResult2 | PromiseLike<TResult2>) | null,
): Promise<TResult1 | TResult2> {
return getOrCreate().then(onfulfilled, onrejected);
},
catch<TResult = never>(
onrejected?: ((reason: any) => TResult | PromiseLike<TResult>) | null,
): Promise<TItem[] | TResult> {
return getOrCreate().catch(onrejected);
},
finally(onfinally?: (() => void) | null): Promise<TItem[]> {
return getOrCreate().finally(onfinally);
},
[Symbol.toStringTag]: 'DualIterable',
} as AsyncIterable<TItem> & Promise<TItem[]>;
return result;
}
/**
* Options for the static `open()` method on storage classes ({@apilink Dataset}, {@apilink KeyValueStore}, {@apilink RequestQueue}).
*/
export interface StorageOpenOptions {
/**
* SDK configuration instance, defaults to the static register.
*/
configuration?: Configuration;
/**
* Optional storage backend that should be used to open storages.
*/
storageBackend?: StorageBackend;
/**
* Used to pass the proxy configuration for the `requestsFromUrl` objects.
* Takes advantage of the internal address rotation and authentication process.
* If undefined, the `requestsFromUrl` requests will be made without proxy.
*/
proxyConfiguration?: IProxyConfiguration;
/**
* HTTP client to be used to download the list of URLs in `RequestQueue`.
*/
httpClient?: BaseHttpClient;
}