Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { execSync as execSyncOriginal } from 'node:child_process';
import { existsSync } from 'node:fs';
import { readdir, readFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { dirname, join } from 'node:path';
import { setTimeout } from 'node:timers/promises';
import { fileURLToPath } from 'node:url';
import { URL_NO_COMMAS_REGEX } from '@crawlee/utils/internal';
import { Actor, ApifyClient } from 'apify';
import fs from 'fs-extra';
/**
* @param {string} command
* @param {import('node:child_process').ExecSyncOptions} options
*/
function execSync(command, options) {
return execSyncOriginal(command, { ...options, encoding: 'utf-8' });
}
/**
* The index is not always 0: AdaptivePlaywrightCrawler discards the Statistics
* the base crawler created and installs its own, which lands on the next id.
* @param {string} name
*/
const isCrawlerStatisticsKey = (name) => name.startsWith('CRAWLEE_CRAWLER_STATISTICS');
/**
* @param {string} name
*/
const isPrivateEntry = (name) => isCrawlerStatisticsKey(name) || name.startsWith('CRAWLEE_SESSION_POOL_STATE');
export const SKIPPED_TEST_CLOSE_CODE = 404;
/** @type {Record<string, (text: string) => string>} */
export const colors = {
red: (text) => `\x1B[31m${text}\x1B[39m`,
green: (text) => `\x1B[32m${text}\x1B[39m`,
grey: (text) => `\x1B[90m${text}\x1B[39m`,
yellow: (text) => `\x1B[33m${text}\x1B[39m`,
};
/**
* @param {string} dirName
*/
export function getStorage(dirName) {
let folderName;
if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') folderName = 'apify_storage';
if (process.env.STORAGE_IMPLEMENTATION === 'MEMORY') folderName = 'storage';
return join(dirName, folderName);
}
/**
* @param {string} dirName
*/
export async function getStats(dirName) {
// fs-storage writes key-value records extensionless (the filename is the record key).
const dir = join(getStorage(dirName), 'key_value_stores/default');
if (!existsSync(dir)) {
return false;
}
const [statsFile] = (await readdir(dir)).filter((name) => isCrawlerStatisticsKey(name)).sort();
if (!statsFile) {
return false;
}
return fs.readJSON(join(dir, statsFile));
}
/**
* @param {string | URL} url
*/
export function getActorTestDir(url) {
const filename = fileURLToPath(url);
const actorDirName = dirname(filename);
return join(actorDirName, 'actor');
}
export async function pushActor(client, dirName) {
await copyPackages(dirName);
try {
// Capture all stdio (default would inherit stderr and flood the runner
// with the platform's Docker/npm build output on every test). The
// catch block below re-prints stdout/stderr if the push itself fails,
// so failure diagnostics stay intact. maxBuffer bumped because the
// build log can be tens of MB.
execSync('apify push', {
cwd: dirName,
env: { ...process.env, GIT_CEILING_DIRECTORIES: dirname(dirName) },
stdio: 'pipe',
maxBuffer: 50 * 1024 * 1024,
});
} catch (err) {
console.error(colors.red(`Failed to push actor to the Apify platform. (signal ${colors.yellow(err.signal)})`));
if (err.stdout) {
console.log(colors.grey(` STDOUT: `), err.stdout);
}
if (err.stderr) {
console.log(colors.red(` STDERR: `), err.stderr);
}
throw err;
}
const actorName = await getActorName(dirName);
const { items: actors } = await client.actors().list();
const { id } = actors.find((actor) => actor.name === actorName);
return id;
}
export async function startActorOnPlatform(client, id, input, inputContentType = 'application/json', memory = 4096) {
// Do NOT use Apify Client yet!
// See https://github.com/apify/apify-client-js/issues/277
const response = await fetch(`https://api.apify.com/v2/acts/${id}/runs?memory=${memory}`, {
method: 'POST',
headers: {
'content-type': inputContentType,
authorization: `Bearer ${client.token}`,
'user-agent': 'crawlee e2e tests',
},
body: input,
signal: AbortSignal.timeout(10_000),
});
if (!response.ok) {
console.error(
colors.red(`Failed to start actor run on the Apify platform. (code ${colors.yellow(response.status)})`),
);
console.log(colors.grey(` RESPONSE: `), await response.text());
throw new Error(`Failed to start actor run on the Apify platform (HTTP ${response.status})`);
}
const { data } = await response.json();
return data.id;
}
/**
* @param {string} dirName
* @param {number} [memory=4096]
*/
export async function runActor(dirName, memory = 4096) {
let stats;
let datasetItems;
let getKeyValueStoreItems;
let defaultKeyValueStoreItems;
const inputPath = join(dirName, '..', 'INPUT');
const input = fs.existsSync(inputPath) ? fs.readFileSync(inputPath) : undefined;
const contentType = input ? 'application/json' : undefined;
if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {
// Plain client, not Actor.newClient(): the latter's charging-aware `.dataset()`
// throws unless Actor.init() ran, which it never does in this harness process.
const client = new ApifyClient({ token: process.env.APIFY_TOKEN });
const id = await pushActor(client, dirName);
const runId = await startActorOnPlatform(client, id, input, contentType, memory);
const {
defaultKeyValueStoreId,
defaultDatasetId,
startedAt: runStartedAt,
finishedAt: runFinishedAt,
// id: runId,
buildId,
userId,
status: runStatus,
} = await client.run(runId).waitForFinish();
if (runStatus !== 'SUCCEEDED') {
console.log(`[run] actor run finished with status ${runStatus} - the assertions below will likely fail`);
}
getKeyValueStoreItems = async (name) => {
const kvResult = await client.keyValueStore(name ? `${userId}/${name}` : defaultKeyValueStoreId).get();
if (kvResult) {
const { items: keyValueItems } = await client.keyValueStore(kvResult.id).listKeys();
if (keyValueItems.length) {
console.log(`[kv] View storage: https://console.apify.com/storage/key-value/${kvResult.id}`);
}
const entries = await Promise.all(
keyValueItems.map(async ({ key }) => {
const record = await client.keyValueStore(kvResult.id).getRecord(key, { buffer: true });
return {
name: record.key,
raw: record.value,
};
}),
);
// eslint-disable-next-line no-shadow
return entries.filter(({ name }) => !isPrivateEntry(name));
}
return undefined;
};
const { startedAt: buildStartedAt, finishedAt: buildFinishedAt } = await client.build(buildId).get();
const buildTook = (buildFinishedAt.getTime() - buildStartedAt.getTime()) / 1000;
console.log(`[build] View build log: https://api.apify.com/v2/logs/${buildId} [build took ${buildTook}s]`);
const runTook = (runFinishedAt.getTime() - runStartedAt.getTime()) / 1000;
console.log(`[run] View run: https://console.apify.com/view/runs/${runId} [run took ${runTook}s]`);
// The statistics record is persisted during the crawler teardown and the platform KVS is
// eventually consistent, so a single lookup right after the run can race the write and crash
// the whole test on `stats.requestsSucceeded`. Retry briefly and fall back to an empty object,
// so a genuinely missing record fails the assertions instead of throwing a TypeError.
let statsRecord;
for (let attempt = 0; attempt < 10 && !statsRecord; attempt++) {
if (attempt > 0) await setTimeout(3000);
const { items: kvKeys } = await client.keyValueStore(defaultKeyValueStoreId).listKeys();
const [statsKey] = kvKeys
.map(({ key }) => key)
.filter((key) => isCrawlerStatisticsKey(key))
.sort();
statsRecord = statsKey ? await client.keyValueStore(defaultKeyValueStoreId).getRecord(statsKey) : undefined;
}
stats = statsRecord?.value ?? {};
const { items } = await client.dataset(defaultDatasetId).listItems();
datasetItems = items;
defaultKeyValueStoreItems = await getKeyValueStoreItems();
} else {
if (dirName.split('/').at(-2).endsWith('-ts')) {
try {
execSync('tsc', { cwd: dirName });
} catch (/** @type {any} */ e) {
if ('stdout' in e) {
console.error(e.stdout.toString());
} else {
console.log(e);
}
// TODO rethrow or not?
}
}
if (input) {
await Actor.init({
// @ts-ignore installed only optionally run `run.mjs` script
storage:
process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
? new (await import('@apify/storage-local')).ApifyStorageLocal()
: undefined,
});
await Actor.setValue('INPUT', input, { contentType });
}
await import(join(dirName, 'main.js'));
await setTimeout(50);
stats = await getStats(dirName);
datasetItems = await getDatasetItems(dirName);
getKeyValueStoreItems = (name = 'default') => getLocalKeyValueStoreItems(dirName, name);
defaultKeyValueStoreItems = await getKeyValueStoreItems();
}
return {
stats,
datasetItems,
defaultKeyValueStoreItems,
getKeyValueStoreItems,
};
}
/**
* @param {string} dirName
*/
async function getActorName(dirName) {
const actorPackageFile = await fs.readJSON(join(dirName, 'package.json'));
return actorPackageFile.name;
}
/**
* In order to test the most recent 'Crawlee' changes we copy locally built packages,
* push them to the platform together with actor code,
* and install them there from the disk (not from NPM).
* These changes are not merged to 'master' yet and thus not yet published to NPM.
* @param {string} dirName
* @internal
*/
async function copyPackages(dirName) {
const srcPackagesDir = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'packages');
const destPackagesDir = join(dirName, 'packages');
await fs.remove(destPackagesDir);
// `apify push` skips gitignored files, and the copied packages are gitignored.
await fs.writeFile(join(dirName, '.actorignore'), '!packages\n');
// npm (used when building the actor on the platform) understands neither pnpm's `workspace:`
// protocol nor its `catalog:` protocol. Parse the catalog from pnpm-workspace.yaml so
// `catalog:` deps can be rewritten to their pinned versions below.
const workspaceYaml = await fs.readFile(
join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'pnpm-workspace.yaml'),
'utf8',
);
const catalog = new Map();
let inCatalog = false;
for (const line of workspaceYaml.split('\n')) {
if (/^catalog:\s*$/.test(line)) {
inCatalog = true;
continue;
}
if (inCatalog) {
const match = /^\s+([^\s:]+):\s*(\S+)\s*$/.exec(line);
if (!match) {
inCatalog = false;
continue;
}
catalog.set(match[1], match[2]);
}
}
const { dependencies, overrides } = await fs.readJSON(join(dirName, 'package.json'));
if (overrides?.apify) {
Object.assign(dependencies, overrides.apify);
}
// Build a map of `@crawlee/*` package name -> its directory name under `packages/`.
// The two don't always match (e.g. `@crawlee/basic` lives in `basic-crawler`).
const packageNameToDir = new Map();
for (const entry of await readdir(srcPackagesDir, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
try {
const { name } = await fs.readJSON(join(srcPackagesDir, entry.name, 'package.json'));
if (name) packageNameToDir.set(name, entry.name);
} catch {
// ignore directories without a readable package.json
}
}
// Seed the copy queue with the packages referenced by the actor via `file:` deps,
// then transitively pull in every `@crawlee/*` dependency they need.
const queue = [];
for (const dependency of Object.values(dependencies)) {
if (typeof dependency === 'string' && dependency.startsWith('file:')) {
queue.push(dependency.split('/').pop());
}
}
const copied = new Set();
while (queue.length > 0) {
const packageDirName = queue.shift();
if (copied.has(packageDirName)) continue;
copied.add(packageDirName);
const srcDir = join(srcPackagesDir, packageDirName, 'dist');
const destDir = join(destPackagesDir, packageDirName, 'dist');
await fs.copy(srcDir, destDir);
const srcPackageFile = join(srcPackagesDir, packageDirName, 'package.json');
const pkg = await fs.readJSON(srcPackageFile);
// npm (used when building the actor on the platform) does not understand pnpm's
// `workspace:` protocol, so rewrite internal `@crawlee/*` deps to sibling `file:`
// references and make sure those packages get copied as well.
for (const depGroup of ['dependencies', 'optionalDependencies', 'peerDependencies']) {
const deps = pkg[depGroup];
if (!deps) continue;
for (const [depName, depVersion] of Object.entries(deps)) {
if (typeof depVersion !== 'string') {
continue;
}
if (depVersion.startsWith('catalog:')) {
const resolved = catalog.get(depName);
if (!resolved) {
throw new Error(
`Cannot resolve catalog dependency "${depName}" for package "${packageDirName}"`,
);
}
deps[depName] = resolved;
continue;
}
if (!depVersion.startsWith('workspace:')) {
continue;
}
const depDir = packageNameToDir.get(depName);
if (!depDir) {
throw new Error(`Cannot resolve workspace dependency "${depName}" for package "${packageDirName}"`);
}
deps[depName] = `file:../${depDir}`;
queue.push(depDir);
}
}
const destPackageFile = join(destPackagesDir, packageDirName, 'package.json');
await fs.writeJSON(destPackageFile, pkg, { spaces: 4 });
}
}
/**
* @param {string} dirName
*/
export async function clearPackages(dirName) {
const destPackagesDir = join(dirName, 'actor', 'packages');
await fs.remove(destPackagesDir);
await fs.remove(join(dirName, 'actor', '.actorignore'));
}
/**
* @param {string} dirName
*/
export async function clearStorage(dirName) {
let folderName;
if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') folderName = 'apify_storage';
if (process.env.STORAGE_IMPLEMENTATION === 'MEMORY') folderName = 'storage';
const destPackagesDir = join(dirName, 'actor', folderName);
await fs.remove(destPackagesDir);
}
export async function getApifyToken() {
const authPath = join(homedir(), '.apify', 'auth.json');
if (!existsSync(authPath)) {
throw new Error(
'You need to be logged in with your Apify account to run E2E tests. Call "apify login" to fix that.',
);
}
const { token } = await fs.readJSON(authPath);
// Newer CLI versions keep the token in the OS keyring, leaving auth.json without it.
if (!token) {
throw new Error(
'Your Apify token is not stored in auth.json (the CLI likely saved it in the OS keyring). ' +
'Set the "APIFY_TOKEN" environment variable, or re-run "apify login" with "APIFY_DISABLE_KEYRING=1".',
);
}
return token;
}
/**
* @param {string} dirName
*/
export async function getDatasetItems(dirName) {
const dir = getStorage(dirName);
const datasetPath = join(dir, 'datasets/default/');
if (!existsSync(datasetPath)) {
return [];
}
const dirents = await readdir(datasetPath, { withFileTypes: true });
const fileNames = dirents.filter((dirent) => dirent.isFile());
const datasetItems = [];
for (const fileName of fileNames) {
if (fileName.name.includes('__metadata__')) continue;
const filePath = join(datasetPath, fileName.name);
const datasetItem = await fs.readJSON(filePath);
if (!isItemHidden(datasetItem)) {
datasetItems.push(datasetItem);
}
}
return datasetItems;
}
/**
* Gets all items in the local key-value store
* @param {string} dirName
* @param {string} kvName
*/
export async function getLocalKeyValueStoreItems(dirName, kvName) {
const dir = getStorage(dirName);
const storePath = join(dir, 'key_value_stores', kvName);
if (!existsSync(storePath)) {
return undefined;
}
const dirents = await readdir(storePath, { withFileTypes: true });
const fileNames = dirents.filter((dirent) => dirent.isFile());
const keyValueStoreRecords = [];
for (const fileName of fileNames) {
if (fileName.name.includes('__metadata__')) continue;
const filePath = join(storePath, fileName.name);
const buffer = await readFile(filePath);
// fs-storage writes records extensionless — the filename is the record key.
const name = fileName.name;
if (isPrivateEntry(name)) {
continue;
}
keyValueStoreRecords.push({ name, raw: buffer });
}
return keyValueStoreRecords;
}
/**
* @param {string} dirName
*/
export async function initialize(dirName) {
process.env.STORAGE_IMPLEMENTATION ??= 'MEMORY';
if (process.env.STORAGE_IMPLEMENTATION !== 'PLATFORM') {
process.env.APIFY_LOCAL_STORAGE_DIR = getStorage(dirName);
process.env.CRAWLEE_STORAGE_DIR = getStorage(dirName);
process.env.APIFY_HEADLESS = '1'; // run browser in headless mode (default on platform)
process.env.APIFY_TOKEN ??= await getApifyToken();
process.env.APIFY_CONTAINER_URL ??= 'http://127.0.0.1';
process.env.APIFY_CONTAINER_PORT ??= '8000';
}
console.log('[init] Storage directory:', process.env.APIFY_LOCAL_STORAGE_DIR || 'n/a (running on the platform)');
}
/**
* @param {boolean} bool
* @param {string} message
*/
export async function expect(bool, message) {
if (bool) {
console.log(`[assertion] passed: ${message}`);
await setTimeout(10);
} else {
console.log(`[assertion] failed: ${message}`);
await setTimeout(10);
process.exit(1);
}
}
/**
* @param {string} reason
*/
export async function skipTest(reason) {
console.error(`[test skipped] ${reason}`);
process.exit(SKIPPED_TEST_CLOSE_CODE);
}
/**
* @param {Record<string, any>} item
* @param {string} propName
* @returns {boolean}
*/
function checkDatasetItem(item, propName) {
if (!Object.hasOwn(item, propName)) {
return false;
}
switch (propName) {
case 'url':
return item.url.match(URL_NO_COMMAS_REGEX);
case 'modifiedDate':
return !Number.isNaN(Date.parse(item.modifiedDate));
case 'runCount':
return Number.isInteger(item.runCount);
default:
return ['string', 'number', 'boolean'].includes(typeof item[propName]);
}
}
/**
* @param {any[]} items
* @param {string[]} schema
*/
export function validateDataset(items, schema = []) {
for (const item of items) {
for (const propName of schema) {
if (!checkDatasetItem(item, propName)) {
return false;
}
}
}
return true;
}
/**
* @param {Record<PropertyKey, unknown>} item
*/
function isItemHidden(item) {
for (const key of Object.keys(item)) {
if (!key.startsWith('#')) {
return false;
}
}
return true;
}
/**
* @param {any} obj the object to search
* @param {string} keyName the key to search for
* @returns {boolean}
*/
export function hasNestedKey(obj, keyName) {
if (typeof obj !== 'object' || obj === null) {
return false;
}
for (const key of Object.keys(obj)) {
if (key === keyName) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null && hasNestedKey(obj[key], keyName)) {
return true;
}
}
return false;
}