Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tools.mjs';
// TODO(v4): redesign for the v4 RequestQueue. The test stages a stuck queue via the v3
// client-side `requestQueue.inProgress` set, which no longer exists in the rewritten queue.
await skipTest('The v4 RequestQueue has no client-side `inProgress` set to stage a stuck queue with');
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
const { stats } = await runActor(testActorDirname);
await expect(stats.requestsSucceeded === 3, 'All requests finished');
await expect(
stats.crawlerRuntimeMillis > 30 * 1e3 && stats.crawlerRuntimeMillis < 35 * 1e3,
'RequestQueue triggers auto-reset after being stuck with requests in progress',
);