Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
save
Petr Patekbd7dcb365ec7
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 312e0e4e9..74edecbf2 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -9,6 +9,7 @@ import { RequestQueue, RequestQueueLocal } from '../request_queue';import events from '../events';import { addTimeoutToPromise } from '../utils';import Statistics from './statistics';+import { openSessionPool } from '../session_pool/session_pool';/*** Since there's no set number of seconds before the container is terminated after@@ -31,6 +32,7 @@ const DEFAULT_OPTIONS = {log.error('BasicCrawler: Request failed and reached maximum retries', details);},autoscaledPoolOptions: {},+ sessionPoolOptions: {},};/**@@ -169,6 +171,7 @@ class BasicCrawler {maxRequestRetries,maxRequestsPerCrawl,autoscaledPoolOptions,+ sessionPoolOptions,// AutoscaledPool shorthandsminConcurrency,@@ -183,6 +186,7 @@ class BasicCrawler {checkParamOrThrow(maxRequestRetries, 'options.maxRequestRetries', 'Number');checkParamOrThrow(maxRequestsPerCrawl, 'options.maxRequestsPerCrawl', 'Maybe Number');checkParamOrThrow(autoscaledPoolOptions, 'options.autoscaledPoolOptions', 'Object');+ checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');if (!requestList && !requestQueue) {throw new Error('At least one of the parameters "options.requestList" and "options.requestQueue" must be provided!');@@ -196,6 +200,7 @@ class BasicCrawler {this.maxRequestRetries = maxRequestRetries;this.handledRequestsCount = 0;this.stats = new Statistics({ logMessage: 'Crawler request statistics:' });+ this.sessionPoolOptions = sessionPoolOptions;let shouldLogMaxPagesExceeded = true;const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;@@ -261,7 +266,7 @@ class BasicCrawler {// so that the caller can get a reference to it before awaiting the promise returned from run()// (otherwise there would be no way)this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);-+ this.sessionPool = await openSessionPool(this.sessionPoolOptions);await this._loadHandledRequestCount();this.isRunningPromise = this.autoscaledPool.run();@@ -338,7 +343,9 @@ class BasicCrawler {async _runTaskFunction() {const source = this.requestQueue || this.requestList;+ // Maybe we can use Promise.all()const request = await this._fetchNextRequest();+ const session = await this.sessionPool.retrieveSession();if (!request) return;// Reset loadedUrl so an old one is not carried over to retries.@@ -348,14 +355,19 @@ class BasicCrawler {this.stats.startJob(statisticsId);try {await addTimeoutToPromise(- this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool }),+ this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool, session }),this.handleRequestTimeoutMillis,`BasicCrawler: handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,);await source.markRequestHandled(request);this.stats.finishJob(statisticsId);this.handledRequestsCount++;++ // reclaim session if request finishes successfully+ session.reclaim();} catch (err) {+ // fail session if request produces an error... Failing maybe should be only manually by the developer in case of the basic crawler+ session.fail();try {await this._requestFunctionErrorHandler(err, request, source);} catch (secondaryError) {diff --git a/test/crawlers/basic_crawler.js b/test/crawlers/basic_crawler.jsindex ab340eb3a..aa2778364 100644--- a/test/crawlers/basic_crawler.js+++ b/test/crawlers/basic_crawler.js@@ -598,4 +598,20 @@ describe('BasicCrawler', () => {expect(results[0].url).to.be.eql(url);results[0].errorMessages.forEach(msg => expect(msg).to.include('handleRequestFunction timed out'));});++ it('should pass session to handleRequestFunction ', async () => {+ const url = 'https://example.com';+ const requestList = new Apify.RequestList({ sources: [{ url }] });+ await requestList.initialize();++ const crawler = new Apify.BasicCrawler({+ requestList,+ handleRequestTimeoutSecs: 0.01,+ maxRequestRetries: 1,+ handleRequestFunction: ({ session }) => expect(session.constructor.name).to.be.eql('Session'),+ handleFailedRequestFunction: ({ request }) => results.push(request),+ });++ await crawler.run();+ });});