Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Improved tests.

Petr Patek5f28f02f00f2
diff --git a/docs/api/Session.md b/docs/api/Session.md
index ae5bb8b07..5d4cbc1f1 100644
--- a/docs/api/Session.md
+++ b/docs/api/Session.md
@@ -6,8 +6,8 @@ title: Session
<a name="Session"></a>
Class aggregating data for session. Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy
-sessions. You can think of a session as one specific user. Session internal state can be enriched with custom user data for example some authorization
-tokens and specific headers in general.
+sessions. You can think of a session as about one specific user. Session internal state can be enriched with custom user data for example some
+authorization tokens and specific headers in general.
- [Session](session)
- [`new exports.Session(options)`](#new_Session_new)
@@ -43,11 +43,6 @@ Session configuration.
<tr>
<td colspan="3"><p>Id of session used for generating fingerprints. It is used as proxy session name.</p>
</td></tr><tr>
-<td><code>options.cookies</code></td><td><code>Array</code></td>
-</tr>
-<tr>
-<td colspan="3"><p>Cookies storage per session.</p>
-</td></tr><tr>
<td><code>options.maxAgeSecs</code></td><td><code>Number</code></td>
</tr>
<tr>
diff --git a/docs/api/SessionPool.md b/docs/api/SessionPool.md
index 6fd137cf1..d110662bc 100644
--- a/docs/api/SessionPool.md
+++ b/docs/api/SessionPool.md
@@ -82,7 +82,7 @@ Indicates how many sessions are rotated.</p>
<td><code>options.maxSessionAgeSecs</code></td><td><code>Number</code></td>
</tr>
<tr>
-<td colspan="3"><p>Number of seconds after which the session is marked as expired.</p>
+<td colspan="3"><p>Number of seconds after which the session is considered as expired.</p>
</td></tr><tr>
<td><code>options.maxSessionUsageCount</code></td><td><code>Number</code></td>
</tr>
@@ -103,7 +103,7 @@ It useful, when you know the site rate-limits, so you can retire the session bef
<td><code>options.createSessionFunction</code></td><td><code>function</code></td>
</tr>
<tr>
-<td colspan="3"><p>Custom function that should return Session instance.</p>
+<td colspan="3"><p>Custom function that should return <code>Session</code> instance.</p>
</td></tr></tbody>
</table>
<a name="SessionPool+usableSessionsCount"></a>
@@ -143,4 +143,4 @@ Returns an object representing the internal state of the `SessionPool` instance.
## `sessionPool.persistState()` ⇒ `Promise`
Persists the current state of the `SessionPool` into the default [`KeyValueStore`](keyvaluestore). The state is persisted automatically in regular
-intervals, but calling this method manually
+intervals.
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index ee59abef2..a078f172f 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -1,7 +1,6 @@
import EventEmitter from 'events';
import { cryptoRandomObjectId } from 'apify-shared/utilities';
-import moment from 'moment';
import { checkParamOrThrow } from 'apify-client/build/utils';
import EVENTS from './events';
@@ -18,7 +17,6 @@ export class Session {
* Session configuration.
* @param options
* @param options.id {String} - Id of session used for generating fingerprints. It is used as proxy session name.
- * @param options.cookies {Array} - Cookies storage per session.
* @param options.maxAgeSecs {Number} - Number of seconds after which the session is considered as expired.
* @param options.userData {Object} - Object where custom user data can be stored. For example custom headers.
* @param options.maxErrorScore {number} - Maximum number of failed session usage.
@@ -41,7 +39,7 @@ export class Session {
userData = {},
maxErrorScore = 3,
errorScoreDecrement = 0.5,
- createdAt = moment(),
+ createdAt = new Date(),
usageCount = 0,
errorScore = 0,
maxSessionUsageCount = 50,
@@ -51,16 +49,15 @@ export class Session {
let { expiresAt } = options;
// Validation
- checkParamOrThrow(id, 'options.id', 'Maybe String');
- checkParamOrThrow(cookies, 'options.cookies', 'Maybe Array');
- checkParamOrThrow(maxAgeSecs, 'options.maxAgeSecs', 'Maybe Number');
- checkParamOrThrow(userData, 'options.userData', 'Maybe Object');
- checkParamOrThrow(maxErrorScore, 'options.maxErrorScore', 'Maybe Number');
- checkParamOrThrow(expiresAt, 'options.expiresAt', 'Maybe Object');
- checkParamOrThrow(createdAt, 'options.createdAt', 'Maybe Object');
- checkParamOrThrow(usageCount, 'options.usageCount', 'Maybe Number');
- checkParamOrThrow(errorScore, 'options.errorScore', 'Maybe Number');
- checkParamOrThrow(maxSessionUsageCount, 'options.maxSessionUsageCount', 'Maybe Number');
+ checkParamOrThrow(id, 'options.id', 'String');
+ checkParamOrThrow(maxAgeSecs, 'options.maxAgeSecs', 'Number');
+ checkParamOrThrow(userData, 'options.userData', 'Object');
+ checkParamOrThrow(maxErrorScore, 'options.maxErrorScore', 'Number');
+ checkParamOrThrow(expiresAt, 'options.expiresAt', 'Maybe Date');
+ checkParamOrThrow(createdAt, 'options.createdAt', 'Date');
+ checkParamOrThrow(usageCount, 'options.usageCount', 'Number');
+ checkParamOrThrow(errorScore, 'options.errorScore', 'Number');
+ checkParamOrThrow(maxSessionUsageCount, 'options.maxSessionUsageCount', 'Number');
checkParamOrThrow(sessionPool, 'options.sessionPool', 'Object');
// sessionPool must be at least instance of EvenEmitter.
@@ -70,7 +67,7 @@ export class Session {
}
if (!expiresAt) {
- expiresAt = moment().add(maxAgeSecs, 'seconds');
+ expiresAt = new Date((new Date()).getTime() + (maxAgeSecs * 1000));
}
// Configurable
@@ -106,9 +103,7 @@ export class Session {
* @return {boolean}
*/
isExpired() {
- const now = moment();
-
- return moment(this.expiresAt).isSameOrBefore(now);
+ return this.expiresAt <= new Date();
}
/**
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index 28a4a4716..d0e51bef0 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -2,7 +2,6 @@ import EventEmitter from 'events';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import moment from 'moment';
import { openKeyValueStore } from '../key_value_store';
import { Session } from './session';
import events from '../events';
@@ -82,11 +81,11 @@ export class SessionPool extends EventEmitter {
super();
// Validation
- checkParamOrThrow(maxPoolSize, 'options.maxPoolSize', 'Maybe Number');
- checkParamOrThrow(maxSessionAgeSecs, 'options.maxSessionAgeSecs', 'Maybe Number');
- checkParamOrThrow(maxSessionUsageCount, 'options.maxSessionUsageCount', 'Maybe Number');
+ checkParamOrThrow(maxPoolSize, 'options.maxPoolSize', 'Number');
+ checkParamOrThrow(maxSessionAgeSecs, 'options.maxSessionAgeSecs', 'Number');
+ checkParamOrThrow(maxSessionUsageCount, 'options.maxSessionUsageCount', 'Number');
checkParamOrThrow(persistStateKeyValueStoreId, 'options.persistStateKeyValueStoreId', 'Maybe String');
- checkParamOrThrow(persistStateKey, 'options.persistStateKey', 'Maybe String');
+ checkParamOrThrow(persistStateKey, 'options.persistStateKey', 'String');
checkParamOrThrow(createSessionFunction, 'options.createSessionFunction', 'Maybe Function');
// Pool Configuration
@@ -283,8 +282,8 @@ export class SessionPool extends EventEmitter {
for (const sessionObject of loadedSessionPool.sessions) {
sessionObject.sessionPool = this;
- sessionObject.createdAt = moment(sessionObject.createdAt);
- sessionObject.expiresAt = moment(sessionObject.expiresAt);
+ sessionObject.createdAt = new Date(sessionObject.createdAt);
+ sessionObject.expiresAt = new Date(sessionObject.expiresAt);
const recreatedSession = new Session(sessionObject);
if (recreatedSession.isUsable()) {
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 85d2dfc51..62f824f0c 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -499,7 +499,7 @@ describe('KeyValueStore', () => {
it('should work', async () => {
process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
const defaultStore = await Apify.openKeyValueStore();
-
+ const oldGet = defaultStore.getValue;
// Uses default value.
defaultStore.getValue = async key => expect(key).to.be.eql(KEY_VALUE_STORE_KEYS.INPUT);
await Apify.getInput();
@@ -511,6 +511,8 @@ describe('KeyValueStore', () => {
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
delete process.env[ENV_VARS.INPUT_KEY];
+
+ defaultStore.getValue = oldGet.bind(defaultStore);
});
});
});
diff --git a/test/session_pool/session.js b/test/session_pool/session.js
index 2ebaba3f2..4c9d8a542 100644
--- a/test/session_pool/session.js
+++ b/test/session_pool/session.js
@@ -97,7 +97,7 @@ describe('Session - testing session behaviour ', async () => {
Object.entries(state).forEach(([key, value]) => {
- if (moment.isMoment(session[key])) {
+ if (session[key] instanceof Date) {
expect(session[key].toISOString()).to.be.eql(value);
} else {
expect(session[key]).to.be.eql(value);
diff --git a/test/session_pool/session_pool.js b/test/session_pool/session_pool.js
index bb38ed79c..0f707ef33 100644
--- a/test/session_pool/session_pool.js
+++ b/test/session_pool/session_pool.js
@@ -15,14 +15,11 @@ describe('SessionPool - testing session pool', async () => {
});
beforeEach(async () => {
+ await emptyLocalStorageSubdir('key_value_stores/default');
sessionPool = new SessionPool();
await sessionPool.initialize();
});
- afterEach(async () => {
- await emptyLocalStorageSubdir('key_value_stores/default');
- });
-
it('should initialize with default values for first time', async () => {
expect(sessionPool.sessions.length).to.exist; // eslint-disable-line
expect(sessionPool.maxPoolSize).to.exist; // eslint-disable-line
@@ -118,7 +115,7 @@ describe('SessionPool - testing session pool', async () => {
expect(sessionPoolSaved.sessions.length).to.be.eql(sessionPool.sessions.length);
sessionPoolSaved.sessions.forEach((session, index) => {
Object.entries(session).forEach(([key, value]) => {
- if (moment.isMoment(sessionPool.sessions[index][key])) {
+ if (sessionPool.sessions[index][key] instanceof Date) {
expect(value).to.be.eql(sessionPool.sessions[index][key].toISOString());
} else {
expect(sessionPool.sessions[index][key]).to.be.eql(value);