Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix test

Ondra Urbanf5f1b9888a02
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 3168ffc85..1b9b87e23 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -54,11 +54,13 @@ describe('BasicCrawler', () => {
let basicCrawler;
let isStopped;
+ let resolveAbortPromise;
+ const abortPromise = new Promise((res) => { resolveAbortPromise = res; });
const processed = [];
const requestList = new Apify.RequestList({ sources });
const handleRequestFunction = async ({ request }) => {
if (request.url.endsWith('200') && !isStopped) {
- await basicCrawler.abort();
+ await basicCrawler.abort().then(resolveAbortPromise);
isStopped = true;
} else {
await delayPromise(10);
@@ -82,6 +84,8 @@ describe('BasicCrawler', () => {
expect(await requestList.isFinished()).to.be.eql(false);
expect(await requestList.isEmpty()).to.be.eql(false);
+ // We need to check if the abort procedure finished before restarting, otherwise everything goes crazy.
+ await abortPromise;
await basicCrawler.run();
expect(processed.length).to.be.within(500, 525);
expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));