Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix tests

Ondra Urbana4af91632d23
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 533402f68..45967e953 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -287,6 +287,11 @@ describe('BasicCrawler', () => {
handleRequestFunction,
});
+ sinon.stub(Apify.client.requestQueues, 'getQueue')
+ .returns(Promise.resolve({
+ handledRequestCount: 0,
+ }));
+
// It enqueues all requests from RequestList to RequestQueue.
const mock = sinon.mock(requestQueue);
mock.expects('addRequest')
@@ -378,6 +383,7 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).to.be.eql(true);
mock.verify();
+ sinon.restore();
});
it('should say that task is not ready requestList is not set and requestQueue is empty', async () => {
@@ -398,6 +404,11 @@ describe('BasicCrawler', () => {
const queue = [];
let isFinished = false;
+ sinon.stub(Apify.client.requestQueues, 'getQueue')
+ .returns(Promise.resolve({
+ handledRequestCount: 0,
+ }));
+
const basicCrawler = new Apify.BasicCrawler({
requestQueue,
autoscaledPoolOptions: {
@@ -435,6 +446,7 @@ describe('BasicCrawler', () => {
expect(processed.includes(request0, request1)).to.be.eql(true);
mock.verify();
+ sinon.restore();
});
it('should support maxRequestsPerCrawl parameter', async () => {