Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix test
Ondra Urbanf5f1b9888a02
diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 3168ffc85..1b9b87e23 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -54,11 +54,13 @@ describe('BasicCrawler', () => {let basicCrawler;let isStopped;+ let resolveAbortPromise;+ const abortPromise = new Promise((res) => { resolveAbortPromise = res; });const processed = [];const requestList = new Apify.RequestList({ sources });const handleRequestFunction = async ({ request }) => {if (request.url.endsWith('200') && !isStopped) {- await basicCrawler.abort();+ await basicCrawler.abort().then(resolveAbortPromise);isStopped = true;} else {await delayPromise(10);@@ -82,6 +84,8 @@ describe('BasicCrawler', () => {expect(await requestList.isFinished()).to.be.eql(false);expect(await requestList.isEmpty()).to.be.eql(false);+ // We need to check if the abort procedure finished before restarting, otherwise everything goes crazy.+ await abortPromise;await basicCrawler.run();expect(processed.length).to.be.within(500, 525);expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));