Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fixed CheerioCrawler request timeouts.

Petr Patekca9f1d668aa9
diff --git a/docs/api/Apify.md b/docs/api/Apify.md
index 9ea69926d..b4547a57d 100644
--- a/docs/api/Apify.md
+++ b/docs/api/Apify.md
@@ -82,7 +82,7 @@ effect.
<td colspan="3"><p>Idempotency key enables you to ensure that a webhook will not be added multiple times in case of
an actor restart or other situation that would cause the <code>addWebhook()</code> function to be called again.
We suggest using the actor run ID as the idempotency key. You can get the run ID by calling
- [`Apify.getEnv()](apify#module_Apify.getEnv) function.</p>
+ <a href="apify#module_Apify.getEnv">`Apify.getEnv()</a> function.</p>
</td></tr></tbody>
</table>
<a name="module_Apify.call"></a>
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index 60fac93ef..ef61574b4 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -420,6 +420,7 @@ class CheerioCrawler {
return false;
},
+ timeoutSecs: this.requestTimeoutMillis / 1000 + 1,
};
if (/PATCH|POST|PUT/.test(request.method)) mandatoryRequestOptions.payload = request.payload;
diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.js
index 75f9c53e0..49b82772c 100644
--- a/test/crawlers/cheerio_crawler.js
+++ b/test/crawlers/cheerio_crawler.js
@@ -110,6 +110,11 @@ describe('CheerioCrawler', () => {
res.send('DATA');
});
+ app.get('/timeout', async (req, res) => {
+ await delayPromise(32000);
+ res.type('html').send('<div>TEST</div>');
+ });
+
server = await startExpressAppPromise(app, 0);
port = server.address().port; //eslint-disable-line
});
@@ -282,6 +287,38 @@ describe('CheerioCrawler', () => {
});
});
+ describe('should not timeout by the default httpRequest timeoutSecs', () => {
+ it('when requestTimeoutSecs is grater than 30', async function () {
+ this.timeout(5 * 60 * 1000);
+ const sources = [
+ { url: `http://${HOST}:${port}/timeout?a=12` },
+ { url: `http://${HOST}:${port}/timeout?a=23` },
+ ];
+ const processed = [];
+ const failed = [];
+ const requestList = new Apify.RequestList({ sources });
+ const handlePageFunction = async ({ request }) => {
+ processed.push(request);
+ };
+
+ const cheerioCrawler = new Apify.CheerioCrawler({
+ requestList,
+ maxRequestRetries: 1,
+ requestTimeoutSecs: 35,
+ minConcurrency: 2,
+ maxConcurrency: 2,
+ handlePageFunction,
+ handleFailedRequestFunction: ({ request }) => failed.push(request),
+ });
+
+ await requestList.initialize();
+ await cheerioCrawler.run();
+
+ expect(processed).to.have.lengthOf(2);
+ expect(failed).to.have.lengthOf(0);
+ });
+ });
+
describe('should ensure text/html Content-Type', () => {
it('by setting a correct Accept header', async () => {
const headers = [];