Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: add timeout for task function (#1234)
Szymon Marczak33df4391f2f9
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex cbf67a1a0..3d55681c6 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -289,7 +289,7 @@ export class BasicCrawler {const basicCrawlerAutoscaledPoolConfiguration = {minConcurrency,maxConcurrency,- runTaskFunction: this._runTaskFunction.bind(this),+ runTaskFunction: this.timeoutForRunTaskFunction.bind(this),isTaskReadyFunction: async () => {if (isMaxPagesExceeded()) {if (shouldLogMaxPagesExceeded) {@@ -467,6 +467,21 @@ export class BasicCrawler {return nextRequest;}+ /**+ * Wrapper around _runTaskFunction to set a timeout on the entire function.+ * Temporary solution to see whether this causes the 0 concurrency bug.+ * @ignore+ * @protected+ * @interface+ */+ async timeoutForRunTaskFunction() {+ await addTimeoutToPromise(+ this._runTaskFunction(),+ this.handleRequestTimeoutMillis * 2,+ `_runTaskFunction timed out after ${(this.handleRequestTimeoutMillis * 2) / 1000} seconds.`,+ );+ }+/*** Wrapper around handleRequestFunction that fetches requests from RequestList/RequestQueue* then retries them in a case of an error, etc.