Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Merge branch 'master' into feature/session-pool-basic-crawler

Petr Patek6b86f6015e1f
diff --cc test/crawlers/basic_crawler.test.js
index d300d4e37,ea9b426bf..70c499de3
--- a/test/crawlers/basic_crawler.test.js
+++ b/test/crawlers/basic_crawler.test.js
@@@ -6,28 -5,22 +5,26 @@@ import { ACTOR_EVENT_NAMES } from 'apif
import * as Apify from '../../build';
import * as keyValueStore from '../../build/key_value_store';
import { RequestQueue, RequestQueueLocal } from '../../build/request_queue';
- import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';
+ import { sleep } from '../../build/utils';
+ import { LOCAL_STORAGE_DIR } from '../_helper';
+import events from '../../build/events';
describe('BasicCrawler', () => {
let logLevel;
- before(() => {
+ beforeAll(() => {
logLevel = log.getLevel();
log.setLevel(log.LEVELS.OFF);
+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
+ emptyLocalStorageSubdir('key_value_stores/default');
});
- after(() => {
+ afterAll(() => {
log.setLevel(logLevel);
+ emptyLocalStorageSubdir('key_value_stores/default');
});
-
- it('should run in parallel thru all the requests', async () => {
- const startedAt = Date.now();
+ test('should run in parallel thru all the requests', async () => {
const sources = _.range(0, 500).map(index => ({ url: `https://example.com/${index}` }));
const processed = [];
@@@ -599,104 -600,8 +604,104 @@@
});
await crawler.run();
- expect(results).to.have.lengthOf(1);
- expect(results[0].url).to.be.eql(url);
- results[0].errorMessages.forEach(msg => expect(msg).to.include('handleRequestFunction timed out'));
+ expect(results).toHaveLength(1);
+ expect(results[0].url).toEqual(url);
+ results[0].errorMessages.forEach(msg => expect(msg).toMatch('handleRequestFunction timed out'));
});
+
+ describe('Uses SessionPool', () => {
+ it('should use SessionPool when useSessionPool is true ', async () => {
+ const url = 'https://example.com';
+ const requestList = new Apify.RequestList({ sources: [{ url }] });
+ await requestList.initialize();
+ const results = [];
+
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ useSessionPool: true,
+ sessionPoolOptions: {
+ maxPoolSize: 10,
+ persistStateKey: 'POOL',
+ },
+ handleRequestFunction: async ({ session }) => {
+ expect(session.constructor.name).to.be.eql('Session');
+ expect(session.id).to.exist // eslint-disable-line
+ },
+ handleFailedRequestFunction: ({ request }) => results.push(request),
+ });
+
+ await crawler.run();
+ expect(crawler.sessionPool).to.exist; //eslint-disable-line
+ expect(results).length(0);
+ });
+
+ it('should use pass options to sessionPool', async () => {
+ const url = 'https://example.com';
+ const requestList = new Apify.RequestList({ sources: [{ url }] });
+ await requestList.initialize();
+
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ useSessionPool: true,
+ sessionPoolOptions: {
+ maxPoolSize: 10,
+ persistStateKey: 'POOL',
+ },
+ handleRequestFunction: async () => {},
+ handleFailedRequestFunction: () => {},
+ });
+ await crawler.run();
+
+ expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);
+ });
+
+ it('should teardown Session pool after it is finished', async () => {
+ const url = 'https://example.com';
+ const requestList = new Apify.RequestList({ sources: [{ url }] });
+ await requestList.initialize();
+ events.removeAllListeners(ACTOR_EVENT_NAMES.PERSIST_STATE);
+
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ useSessionPool: true,
+ sessionPoolOptions: {
+ maxPoolSize: 10,
+ },
+ handleRequestFunction: async () => {},
+ handleFailedRequestFunction: () => {},
+ });
+
+ crawler._loadHandledRequestCount = () => { // eslint-disable-line
+ expect(crawler.sessionPool).to.exist; // eslint-disable-line
+ expect(events.listenerCount(ACTOR_EVENT_NAMES.PERSIST_STATE)).to.be.eql(1);
+ };
+
+ await crawler.run();
+ expect(events.listenerCount(ACTOR_EVENT_NAMES.PERSIST_STATE)).to.be.eql(0);
+ expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);
+ });
+
+ it('should not use SessionPool by default', async () => {
+ const url = 'https://example.com';
+ const requestList = new Apify.RequestList({ sources: [{ url }] });
+ await requestList.initialize();
+
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ handleRequestFunction: async () => {},
+ handleFailedRequestFunction: () => {},
+ });
+ await crawler.run();
+
+ expect(crawler.sessionPool).to.be.undefined; // eslint-disable-line
+ });
+ });
});
diff --cc test/session_pool/session_pool.test.js
index d362fce87,f2f2ebbfd..423ebce9d
--- a/test/session_pool/session_pool.test.js
+++ b/test/session_pool/session_pool.test.js
@@@ -19,20 -18,16 +18,21 @@@ describe('SessionPool - testing sessio
sessionPool = await Apify.openSessionPool();
});
+ afterEach(() => {
+ events.removeAllListeners(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);
+ });
+
- it('should initialize with default values for first time', async () => {
- expect(sessionPool.sessions.length).to.exist; // eslint-disable-line
- expect(sessionPool.maxPoolSize).to.exist; // eslint-disable-line
- expect(sessionPool.maxSessionAgeSecs).to.exist; // eslint-disable-line
- expect(sessionPool.maxSessionUsageCount).to.exist; // eslint-disable-line
- expect(sessionPool.persistStateKey).to.exist; // eslint-disable-line
- expect(sessionPool.createSessionFunction).to.be.eql(sessionPool._defaultCreateSessionFunction); // eslint-disable-line
++ // eslint-disable-line
+ test('should initialize with default values for first time', async () => {
+ expect(sessionPool.sessions.length).toBeDefined(); // eslint-disable-line
+ expect(sessionPool.maxPoolSize).toBeDefined(); // eslint-disable-line
+ expect(sessionPool.maxSessionAgeSecs).toBeDefined(); // eslint-disable-line
+ expect(sessionPool.maxSessionUsageCount).toBeDefined(); // eslint-disable-line
+ expect(sessionPool.persistStateKey).toBeDefined(); // eslint-disable-line
+ expect(sessionPool.createSessionFunction).toEqual(sessionPool._defaultCreateSessionFunction); // eslint-disable-line
});
- it('should override default values', async () => {
+ test('should override default values', async () => {
const opts = {
maxPoolSize: 3000,
maxSessionAgeSecs: 100,
@@@ -46,10 -41,9 +46,10 @@@
};
sessionPool = new SessionPool(opts);
await sessionPool.initialize();
+ sessionPool.teardown();
Object.entries(opts).forEach(([key, value]) => {
- expect(sessionPool[key]).to.be.eql(value);
+ expect(sessionPool[key]).toEqual(value);
});
});
@@@ -66,11 -60,9 +66,11 @@@
};
sessionPool = await openSessionPool(opts);
+ sessionPool.teardown();
+
Object.entries(opts).forEach(([key, value]) => {
- expect(sessionPool[key]).to.be.eql(value);
+ expect(sessionPool[key]).toEqual(value);
});
});
@@@ -152,11 -147,10 +155,11 @@@
const loadedSessionPool = new SessionPool();
await loadedSessionPool.initialize();
- expect(loadedSessionPool).to.have.all.keys(Object.keys(sessionPool));
+ expect(loadedSessionPool).toEqual(expect.objectContaining(sessionPool));
+ sessionPool.teardown();
});
- it('should create only maxPoolSize number of sessions', async () => {
+ test('should create only maxPoolSize number of sessions', async () => {
const max = sessionPool.maxPoolSize;
for (let i = 0; i < max + 100; i++) {
await sessionPool.getSession();
@@@ -180,13 -174,12 +183,13 @@@
afterEach(async () => {
await emptyLocalStorageSubdir(`key_value_stores/${KV_STORE}`);
+ sessionPool.teardown();
});
- it('on persist event', async () => {
+ test('on persist event', async () => {
await sessionPool.getSession();
- expect(sessionPool.sessions.length).to.be.eql(1);
+ expect(sessionPool.sessions.length).toBe(1);
events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);
@@@ -231,11 -224,11 +234,12 @@@
const newSessionPool = new SessionPool();
await newSessionPool.initialize();
- expect(newSessionPool.sessions).to.be.length(10 - invalidSessionsCount);
-
+ expect(newSessionPool.sessions).toHaveLength(10 - invalidSessionsCount);
++
+ newSessionPool.teardown();
});
- it('should createSessionFunction work', async () => {
+ test('should createSessionFunction work', async () => {
let isCalled;
const createSessionFunction = (sessionPool2) => {
isCalled = true;
@@@ -244,13 -237,7 +248,13 @@@
};
const newSessionPool = await Apify.openSessionPool({ createSessionFunction });
const session = await newSessionPool.getSession();
- expect(isCalled).to.be.true; // eslint-disable-line
- expect(session.constructor.name).to.be.eql("Session") // eslint-disable-line
+ expect(isCalled).toBe(true); // eslint-disable-line
+ expect(session.constructor.name).toBe("Session") // eslint-disable-line
});
+
+ it('should remove persist state event listener', () => {
+ expect(events.listenerCount(ACTOR_EVENT_NAMES_EX.PERSIST_STATE)).to.be.eq(1);
+ sessionPool.teardown();
+ expect(events.listenerCount(ACTOR_EVENT_NAMES_EX.PERSIST_STATE)).to.be.eq(0);
+ });
});