Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix tests

Ondra Urbanca28e138c1c5
diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.js
index c63c62b45..7649a9c9d 100644
--- a/test/cheerio_crawler.js
+++ b/test/cheerio_crawler.js
@@ -102,25 +102,25 @@ describe('CheerioCrawler', () => {
const processed = [];
const failed = [];
const requestList = new Apify.RequestList({ sources });
- const requestFunction = async () => {
- await delayPromise(3000);
- return '<html><head></head><body>Body</body></html>';
- };
const handlePageFunction = async ({ request }) => {
processed.push(request);
};
const cheerioCrawler = new Apify.CheerioCrawler({
requestList,
- requestTimeoutSecs: 0.05,
+ requestTimeoutSecs: 5 / 1000,
maxRequestRetries: 1,
minConcurrency: 2,
maxConcurrency: 2,
handlePageFunction,
- requestFunction,
handleFailedRequestFunction: ({ request }) => failed.push(request),
});
+ cheerioCrawler._requestFunction = async () => {
+ await delayPromise(300);
+ return '<html><head></head><body>Body</body></html>';
+ };
+
await requestList.initialize();
await cheerioCrawler.run();
@@ -197,14 +197,15 @@ describe('CheerioCrawler', () => {
const crawler = new Apify.CheerioCrawler({
requestList,
handlePageFunction: async () => {},
- requestFunction: async ({ request }) => {
- const opts = crawler._getRequestOptions(request);
- headers.push(opts.headers);
- // it needs to return something valid
- return 'html';
- },
});
+ crawler._requestFunction = async ({ request }) => {
+ const opts = crawler._getRequestOptions(request);
+ headers.push(opts.headers);
+ // it needs to return something valid
+ return Object.assign({ body: 'html' }, responseMock);
+ };
+
await crawler.run();
headers.forEach(h => expect(h['Accept-Encoding']).to.be.eql('gzip, deflate'));
});
@@ -346,14 +347,15 @@ describe('CheerioCrawler', () => {
const crawler = new Apify.CheerioCrawler({
requestList,
handlePageFunction: async () => {},
- requestFunction: async ({ request }) => {
- const opts = crawler._getRequestOptions(request);
- headers.push(opts.headers);
- // it needs to return something valid
- return 'html';
- },
});
+ crawler._requestFunction = async ({ request }) => {
+ const opts = crawler._getRequestOptions(request);
+ headers.push(opts.headers);
+ // it needs to return something valid
+ return Object.assign({ body: 'html' }, responseMock);
+ };
+
await crawler.run();
headers.forEach(h => expect(h.Accept).to.be.eql('text/html'));
});