Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fixed passing session id to proxyUrl.
Petr Patekd3cae27297a2
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex acaebe0cd..629d359b2 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -12,7 +12,7 @@ import BasicCrawler from './basic_crawler';import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';import { getApifyProxyUrl } from '../actor';import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';-import { requestAsBrowser } from '../utils_request';+import * as utilsRequest from '../utils_request';import { TimeoutError } from '../errors';/**@@ -133,7 +133,8 @@ const DEFAULT_OPTIONS = {* request: Request,* contentType: Object, // Parsed Content-Type header: { type, encoding }* response: Object // An instance of Node's http.IncomingMessage object,- * autoscaledPool: AutoscaledPool+ * autoscaledPool: AutoscaledPool,+ * session: Session* }* ```* Type of `body` depends on web page `Content-Type` header.@@ -191,10 +192,11 @@ const DEFAULT_OPTIONS = {* The function receives the following object as an argument:* ```* {- * request: Request+ * request: Request,+ * session: Session* }* ```- * where the {@link Request} instance corresponds to the initialized request.+ * where the {@link Request} instance corresponds to the initialized request and the {@link Session} instance corresponds to used session.** The function should modify the properties of the passed {@link Request} instance* in place because there are already earlier references to it. Making a copy and returning it from@@ -327,6 +329,10 @@ class CheerioCrawler {throw new Error('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');}+ if (apifyProxySession && useSessionPool) {+ throw new Error('Cannot use "options.apifyProxySession" with "options.useSessionPool"');+ }+this.supportedMimeTypes = new Set(DEFAULT_MIME_TYPES);if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);@@ -450,11 +456,11 @@ class CheerioCrawler {headers.Cookie = session.getCookieString(request.url);}- const opts = this._getRequestOptions(request);+ const opts = this._getRequestOptions(request, session);let responseStream;try {- responseStream = await requestAsBrowser(opts);+ responseStream = await utilsRequest.requestAsBrowser(opts);} catch (e) {if (e instanceof TimeoutError) {this._handleRequestTimeout(session);@@ -494,15 +500,16 @@ class CheerioCrawler {/*** Combines the provided `requestOptions` with mandatory (non-overridable) values.* @param {Request} request+ * @param {Session?} session* @ignore*/- _getRequestOptions(request) {+ _getRequestOptions(request, session) {const mandatoryRequestOptions = {url: request.url,method: request.method,headers: Object.assign({}, request.headers),ignoreSslErrors: this.ignoreSslErrors,- proxyUrl: this._getProxyUrl(),+ proxyUrl: this._getProxyUrl(session),stream: true,useCaseSensitiveHeaders: true,abortFunction: (res) => {@@ -533,13 +540,14 @@ class CheerioCrawler {/*** Enables the use of a proxy by returning a proxy URL* based on configured options or null if no proxy is used.+ * @param {Session?} session* @ignore*/- _getProxyUrl() {+ _getProxyUrl(session = {}) {if (this.useApifyProxy) {return getApifyProxyUrl({groups: this.apifyProxyGroups,- session: this.apifyProxySession,+ session: session.id || this.apifyProxySession,groupsParamName: 'options.apifyProxyGroups',sessionParamName: 'options.apifyProxySession',});diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 74812efeb..ab0529be4 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -7,11 +7,13 @@ import log from 'apify-shared/log';import { ENV_VARS } from 'apify-shared/consts';import express from 'express';import bodyParser from 'body-parser';+import sinon from 'sinon';import Apify from '../../build';import { sleep } from '../../build/utils';import { Session } from '../../build/session_pool/session';import { STATUS_CODES_BLOCKED } from '../../build/constants';import LocalStorageDirEmulator from '../local_storage_dir_emulator';+import * as utilsRequest from '../../build/utils_request';// Add common props to mocked request responses.const responseMock = {@@ -960,5 +962,38 @@ describe('CheerioCrawler', () => {});await cheerioCrawler.run();});++ test('should use sessionId in proxyUrl when the session pool is enabled', async () => {+ const sourcesNew = [+ { url: 'http://example.com/?q=1' },+ ];+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';++ const requestListNew = new Apify.RequestList({ sources: sourcesNew });+ let sessionUsed;+ let requestUsed;+ const handlePageFunction = async ({ session }) => {+ sessionUsed = session;+ };+ const oldCall = utilsRequest.requestAsBrowser;+ const fakeCall = async (opt) => {+ requestUsed = opt;+ return oldCall(opt);+ };+ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);+ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList: requestListNew,+ maxConcurrency: 1,+ handlePageFunction,+ useSessionPool: true,+ useApifyProxy: true,+ });++ await requestListNew.initialize();+ await cheerioCrawler.run();++ expect(requestUsed.proxyUrl.includes(sessionUsed.id)).toBeTruthy();+ stub.restore();+ });});});