Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: improve logging for fetching next request and timeouts (#1292)
Martin Adámek8decf423dc84
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f5a87dba1..1fe64f46a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,9 +1,11 @@-2.2.2 / 2022/01/xx+2.2.2 / 2022/02/xx====================- fix: ensure `request.headers` is set-- Lower RequestQueue timeout to 30 seconds+- fix: lower `RequestQueue` API timeout to 30 seconds+- improve logging for fetching next request and timeouts-- 2.2.1 / 2022/01/03++2.2.1 / 2022/01/03====================- fix: ignore requests that are no longer in progress (#1258)- fix: do not use `tryCancel()` from inside sync callback (#1265)diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex a0137d23d..e1aae1c02 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -2,7 +2,7 @@ import { ACTOR_EVENT_NAMES } from '@apify/consts';import { cryptoRandomObjectId } from '@apify/utilities';import ow, { ArgumentError } from 'ow';import _ from 'underscore';-import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';+import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import AutoscaledPool from '../autoscaling/autoscaled_pool'; // eslint-disable-line import/no-duplicatesimport events from '../events';import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates@@ -482,11 +482,7 @@ export class BasicCrawler {await this._timeoutAndRetry(async () => {- if (this.useSessionPool) {- [request, session] = await Promise.all([this._fetchNextRequest(), this.sessionPool.getSession()]);- } else {- request = await this._fetchNextRequest();- }+ request = await this._fetchNextRequest();},this.internalTimeoutMillis,`Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,@@ -494,6 +490,18 @@ export class BasicCrawler {tryCancel();+ if (this.useSessionPool) {+ await this._timeoutAndRetry(+ async () => {+ session = await this.sessionPool.getSession();+ },+ this.internalTimeoutMillis,+ `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,+ );+ }++ tryCancel();+if (!request) return;// Reset loadedUrl so an old one is not carried over to retries.@@ -515,13 +523,13 @@ export class BasicCrawler {await addTimeoutToPromise(() => this._handleRequestFunction(crawlingContext),this.handleRequestTimeoutMillis,- `handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,+ `handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds (${request.id}).`,);await this._timeoutAndRetry(() => source.markRequestHandled(request),this.internalTimeoutMillis,- `Marking request ${request.url} as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,+ `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,);this.stats.finishJob(statisticsId);@@ -534,7 +542,7 @@ export class BasicCrawler {await this._timeoutAndRetry(() => this._requestFunctionErrorHandler(err, crawlingContext, source),this.internalTimeoutMillis,- `Handling request failure of ${request.url} timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,+ `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,);} catch (secondaryError) {this.log.exception(secondaryError, 'runTaskFunction error handler threw an exception. '@@ -555,13 +563,9 @@ export class BasicCrawler {*/async _timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {try {- await addTimeoutToPromise(- handler,- timeout,- error,- );+ await addTimeoutToPromise(handler, timeout, error);} catch (e) {- if (e instanceof TimeoutError && retried <= maxRetries) {+ if (retried <= maxRetries) { // we retry on any error, not just timeoutthis.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);return this._timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);}diff --git a/src/storages/request_queue.js b/src/storages/request_queue.jsindex 67fb40d88..c4d8e3080 100644--- a/src/storages/request_queue.js+++ b/src/storages/request_queue.js@@ -355,9 +355,9 @@ export class RequestQueue {return null;}- if (!request.handledAt) request.handledAt = new Date();-- const queueOperationInfo = await this.client.updateRequest(request);+ const handledAt = request.handledAt ?? new Date();+ const queueOperationInfo = await this.client.updateRequest({ ...request, handledAt });+ request.handledAt = handledAt;this.inProgress.delete(request.id);this.recentlyHandled.add(request.id, true);@@ -412,12 +412,6 @@ export class RequestQueue {// Wait a little to increase a chance that the next call to fetchNextRequest() will return the request with updated data.// This is to compensate for the limitation of DynamoDB, where writes might not be immediately visible to subsequent reads.setTimeout(() => {- const signal = storage.getStore()?.cancelTask?.signal;-- if (signal?.aborted) {- return;- }-if (!this.inProgress.has(request.id)) {this.log.debug('The request is no longer marked as in progress in the queue?!', { requestId: request.id });return;