Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: fix flaky keepAlive and isFinishedFunction override tests (#3344)
Martin Adámeka978ab339622
diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.tsindex 1c2786d38..54ff3cd41 100644--- a/test/browser-pool/browser-plugins/plugins.test.ts+++ b/test/browser-pool/browser-plugins/plugins.test.ts@@ -20,7 +20,9 @@ import { runExampleComServer } from 'test/shared/_helper';import { createProxyServer } from './create-proxy-server';-vitest.setConfig({ testTimeout: 120_000 });+// Firefox browser launch is significantly slower than Chromium/WebKit (~12s vs <1s).+// Under CPU load from parallel tests, it can exceed 2 minutes. Use 5 minute timeout.+vitest.setConfig({ testTimeout: 300_000 });let port: number;let server: Server;diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 6e529cbee..6c576bc59 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -975,9 +975,10 @@ describe('BasicCrawler', () => {requestQueue.fetchNextRequest = async () => queue.pop()!;requestQueue.isEmpty = async () => Promise.resolve(!queue.length);- // Add requests with buffer time for crawler startup- setTimeout(() => queue.push(request0), 150);- setTimeout(() => queue.push(request1), 350);+ // Add requests with buffer time for crawler startup.+ // Use longer delays to avoid flakiness under CPU load from parallel tests.+ setTimeout(() => queue.push(request0), 500);+ setTimeout(() => queue.push(request1), 1000);await basicCrawler.run();@@ -1024,11 +1025,12 @@ describe('BasicCrawler', () => {requestQueue.fetchNextRequest = async () => Promise.resolve(queue.pop()!);requestQueue.isEmpty = async () => Promise.resolve(!queue.length);- setTimeout(() => queue.push(request0), 100);- setTimeout(() => queue.push(request1), 250);+ // Use longer delays to avoid flakiness under CPU load from parallel tests.+ setTimeout(() => queue.push(request0), 500);+ setTimeout(() => queue.push(request1), 1000);setTimeout(() => {void basicCrawler.teardown();- }, 650);+ }, 3000);await basicCrawler.run();