Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { setTimeout } from 'node:timers/promises';
import { Actor } from 'apify';
import { log } from 'crawlee';
import { expect, getActorTestDir, initialize, pushActor, startActorOnPlatform } from '../tools.mjs';
if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
const client = Actor.newClient();
const actorId = await pushActor(client, testActorDirname);
log.info(`Actor build (ID ${actorId})`);
const { id: queueId } = await client.requestQueues().getOrCreate();
const limit = 1000;
const input = JSON.stringify({ limit, queueId });
const runIds = await Promise.all([
startActorOnPlatform(client, actorId, input),
startActorOnPlatform(client, actorId, input),
startActorOnPlatform(client, actorId, input),
startActorOnPlatform(client, actorId, input),
]);
log.info(`Started runs (IDs ${JSON.stringify(runIds)})`);
const stats = await Promise.all(
runIds.map(async (runId) => {
await client.run(runId).waitForFinish();
await setTimeout(6000);
return client.run(runId).get();
}),
);
log.info('Runs finished');
const requestQueue = await client.requestQueue(queueId).get();
await expect(requestQueue.pendingRequestCount === 0, 'No pending requests');
await expect(requestQueue.handledRequestCount === 1000, 'All requests are handled');
const firstActorFinish = Math.min(...stats.map((run) => run.finishedAt.valueOf()));
const lastActorFinish = Math.max(...stats.map((run) => run.finishedAt.valueOf()));
await expect(lastActorFinish - firstActorFinish < 60 * 1000, 'Actors finished at similar times');
}