Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

improved naming.

Petr Patek5890bcdae857
diff --git a/docs/api/Session.md b/docs/api/Session.md
index 654dc1d24..395d8f113 100644
--- a/docs/api/Session.md
+++ b/docs/api/Session.md
@@ -20,7 +20,7 @@ internal state can be enriched with custom user data for example some authorizat
- [`.retire()`](#Session+retire)
- [`.markBad()`](#Session+markBad)
- [`.checkStatus(statusCode)`](#Session+checkStatus) ⇒ `boolean`
- - [`.setCookiesToJar(response)`](#Session+setCookiesToJar)
+ - [`.putResponse(response)`](#Session+putResponse)
- [`.getCookieString(url)`](#Session+getCookieString) ⇒ `String`
<a name="new_Session_new"></a>
@@ -174,9 +174,9 @@ Retires session based on status code.
<td colspan="3"><p>HTTP status code</p>
</td></tr></tbody>
</table>
-<a name="Session+setCookiesToJar"></a>
+<a name="Session+putResponse"></a>
-## `session.setCookiesToJar(response)`
+## `session.putResponse(response)`
Sets cookies from response to the cookieJar. Parses cookies from `set-cookie` header and sets them to `Session.cookieJar`.
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index 3816a837e..1b82567b3 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -385,7 +385,7 @@ class CheerioCrawler {
* @ignore
*/
async _handleRequestFunction({ request, autoscaledPool, session }) {
- if (this.prepareRequestFunction) await this.prepareRequestFunction({ request });
+ if (this.prepareRequestFunction) await this.prepareRequestFunction({ request, session });
const { dom, isXml, body, contentType, response } = await addTimeoutToPromise(
this._requestFunction({ request, session }),
this.requestTimeoutMillis,
@@ -393,11 +393,11 @@ class CheerioCrawler {
);
if (this.useSessionPool) {
- this._handleBlockedRequest(session, response.statusCode);
+ this._throwOnBlockedRequest(session, response.statusCode);
}
if (this.persistCookiesPerSession) {
- session.setCookiesToJar(response);
+ session.putResponse(response);
}
request.loadedUrl = response.url;
@@ -446,7 +446,7 @@ class CheerioCrawler {
// handle the response based on headers receieved.
if (this.persistCookiesPerSession) {
- const { headers = {} } = request;
+ const { headers } = request;
headers.Cookie = session.getCookieString(request.url);
}
@@ -604,7 +604,7 @@ class CheerioCrawler {
* @param statusCode {Number}
* @private
*/
- _handleBlockedRequest(session, statusCode) {
+ _throwOnBlockedRequest(session, statusCode) {
const isBlocked = session.checkStatus(statusCode);
if (isBlocked) {
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 78fe97323..139dd9813 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -197,7 +197,7 @@ export class Session {
* Parses cookies from `set-cookie` header and sets them to `Session.cookieJar`.
* @param response
*/
- setCookiesToJar(response) {
+ putResponse(response) {
const cookies = getCookiesFromResponse(response).filter(c => c);
for (const cookie of cookies) {
@@ -211,7 +211,6 @@ export class Session {
* @return {String} String representing `Cookie` header.
*/
getCookieString(url) {
- console.log(this.cookieJar);
return this.cookieJar.getCookieStringSync(url, {});
}
}
diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.js
index 50a55b502..0f213be1c 100644
--- a/test/crawlers/cheerio_crawler.test.js
+++ b/test/crawlers/cheerio_crawler.test.js
@@ -213,7 +213,7 @@ describe('CheerioCrawler', () => {
handleFailedRequestFunction: ({ request }) => {
failed = request;
},
- prepareRequestFunction: async ({ request }) => {
+ prepareRequestFunction: async ({ request, session }) => {
request.url = MODIFIED_URL;
return request;
},
@@ -866,8 +866,8 @@ describe('CheerioCrawler', () => {
failed.push(request);
},
});
- const oldCall = crawler._handleBlockedRequest;
- crawler._handleBlockedRequest = (session, statusCode) => {
+ const oldCall = crawler._throwOnBlockedRequest;
+ crawler._throwOnBlockedRequest = (session, statusCode) => {
sessions.push(session);
return oldCall(session, statusCode);
};
@@ -928,5 +928,19 @@ describe('CheerioCrawler', () => {
}
});
});
+
+ test('should pass session to prepareRequestFunction when Session pool is used', async () => {
+ const handlePageFunction = async () => {};
+
+ const cheerioCrawler = new Apify.CheerioCrawler({
+ requestList,
+ handlePageFunction,
+ useSessionPool: true,
+ prepareRequestFunction: async ({ session }) => {
+ expect(session.constructor.name).toEqual('Session');
+ },
+ });
+ await cheerioCrawler.run();
+ });
});
});
diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.js
index 0af4616cb..15b622e75 100644
--- a/test/session_pool/session.test.js
+++ b/test/session_pool/session.test.js
@@ -143,7 +143,7 @@ describe('Session - testing session behaviour ', () => {
});
});
- describe('.setCookiesToJar & .getCookieString', () => {
+ describe('.putResponse & .getCookieString', () => {
test('should set and update cookies from "set-cookie" header', () => {
const headers = {};
@@ -153,13 +153,13 @@ describe('Session - testing session behaviour ', () => {
];
const newSession = new Session({ sessionPool: new SessionPool() });
const url = 'https://example.com';
- newSession.setCookiesToJar({ headers, url });
+ newSession.putResponse({ headers, url });
let cookies = newSession.getCookieString(url);
expect(cookies).toEqual('CSRF=e8b667; id=a3fWa');
const newCookie = 'ABCD=1231231213; Domain=example.com; Secure';
- newSession.setCookiesToJar({ headers: { 'set-cookie': newCookie }, url });
+ newSession.putResponse({ headers: { 'set-cookie': newCookie }, url });
cookies = newSession.getCookieString(url);
expect(cookies).toEqual('CSRF=e8b667; id=a3fWa; ABCD=1231231213');
});
@@ -174,7 +174,7 @@ describe('Session - testing session behaviour ', () => {
];
const newSession = new Session({ sessionPool: new SessionPool() });
const url = 'https://example.com';
- newSession.setCookiesToJar({ headers, url });
+ newSession.putResponse({ headers, url });
const old = newSession.getState();