Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixed retiring of session, when get session is not used per request. (#825)

petrpatekae43c8bdaa06
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 4498bcce4..a9ad7f43f 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -170,6 +170,8 @@ export class Session {
if (this.errorScore > 0) {
this.errorScore -= this.errorScoreDecrement;
}
+
+ this._maybeSelfRetire();
}
/**
@@ -213,6 +215,8 @@ export class Session {
markBad() {
this.errorScore += 1;
this.usageCount += 1;
+
+ this._maybeSelfRetire();
}
/**
@@ -365,4 +369,14 @@ export class Session {
_getDefaultCookieExpirationDate(maxAgeSecs) {
return new Date(Date.now() + (maxAgeSecs * 1000));
}
+
+ /**
+ * Checks if session is not usable. if it is not retires the session.
+ * @private
+ */
+ _maybeSelfRetire() {
+ if (!this.isUsable()) {
+ this.retire();
+ }
+ }
}
diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.js
index 0d8a6f92b..2fdaa3e6b 100644
--- a/test/session_pool/session.test.js
+++ b/test/session_pool/session.test.js
@@ -91,6 +91,36 @@ describe('Session - testing session behaviour ', () => {
expect(session.usageCount).toBe(1);
});
+ test('should retire session after marking bad', () => {
+ jest.spyOn(session, '_maybeSelfRetire');
+ jest.spyOn(session, 'retire');
+ session.markBad();
+ expect(session.retire).toBeCalledTimes(0);
+ session.isUsable = () => false;
+ session.markBad();
+ expect(session.retire).toBeCalledTimes(1);
+ });
+
+ test('should retire session after marking good', () => {
+ jest.spyOn(session, '_maybeSelfRetire');
+ jest.spyOn(session, 'retire');
+
+ session.markGood();
+ expect(session.retire).toBeCalledTimes(0);
+
+ session.isUsable = () => false;
+ session.markGood();
+ expect(session.retire).toBeCalledTimes(1);
+ });
+
+ test('should reevaluate usability of session after marking the session', () => {
+ jest.spyOn(session, '_maybeSelfRetire');
+ session.markGood();
+ expect(session._maybeSelfRetire).toBeCalledTimes(1); // eslint-disable-line
+ session.markBad();
+ expect(session._maybeSelfRetire).toBeCalledTimes(2); // eslint-disable-line
+ });
+
test('should get state', () => {
const state = session.getState();