Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
save
Petr Patek2a345cecc398
diff --git a/test/crawlers/basic_crawler.js b/test/crawlers/basic_crawler.jsindex 9a3d2a776..5043a64ef 100644--- a/test/crawlers/basic_crawler.js+++ b/test/crawlers/basic_crawler.js@@ -7,7 +7,7 @@ import { delayPromise } from 'apify-shared/utilities';import * as Apify from '../../build';import * as keyValueStore from '../../build/key_value_store';import { RequestQueue, RequestQueueLocal } from '../../build/request_queue';-import { LOCAL_STORAGE_DIR } from '../_helper';+import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';describe('BasicCrawler', () => {let logLevel;@@ -598,69 +598,75 @@ describe('BasicCrawler', () => {expect(results[0].url).to.be.eql(url);results[0].errorMessages.forEach(msg => expect(msg).to.include('handleRequestFunction timed out'));});- describe('Uses SessionPool', () => {- before(() => {- process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;- });-- it('should use SessionPool when useSessionPool is true ', async () => {- const url = 'https://example.com';- const requestList = new Apify.RequestList({ sources: [{ url }] });- await requestList.initialize();- const results = [];-- const crawler = new Apify.BasicCrawler({- requestList,- handleRequestTimeoutSecs: 0.01,- maxRequestRetries: 1,- useSessionPool: true,- handleRequestFunction: async ({ session }) => {- expect(session.constructor.name).to.be.eql('Session');- expect(session.id).to.exist // eslint-disable-line- },- handleFailedRequestFunction: ({ request }) => results.push(request),- });-- await crawler.run();- expect(crawler.sessionPool).to.exist; //eslint-disable-line- expect(results).length(0);- });-- it('should use pass options to sessionPool', async () => {- const url = 'https://example.com';- const requestList = new Apify.RequestList({ sources: [{ url }] });- await requestList.initialize();-- const crawler = new Apify.BasicCrawler({- requestList,- handleRequestTimeoutSecs: 0.01,- maxRequestRetries: 1,- useSessionPool: true,- sessionPoolOptions: {- maxPoolSize: 10,- },- handleRequestFunction: async () => {},- handleFailedRequestFunction: () => {},- });- await crawler.run();- expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);- });-- it('should not use SessionPool by default', async () => {- const url = 'https://example.com?a=1';- const requestList = await Apify.openRequestList('my', [url]);-- const crawler = new Apify.BasicCrawler({- requestList,- handleRequestTimeoutSecs: 0.01,- maxRequestRetries: 1,- handleRequestFunction: async () => {},- handleFailedRequestFunction: () => {},- });- await crawler.run();-- expect(crawler.sessionPool).to.be.undefined; // eslint-disable-line- });- });+ // describe('Uses SessionPool', () => {+ // before(() => {+ // process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;+ // });+ //+ // after(() => {+ // // emptyLocalStorageSubdir('key_value_stores/default');+ // });+ //+ // xit('should use SessionPool when useSessionPool is true ', async () => {+ // const url = 'https://example.com';+ // const requestList = new Apify.RequestList({ sources: [{ url }] });+ // await requestList.initialize();+ // const results = [];+ //+ // const crawler = new Apify.BasicCrawler({+ // requestList,+ // handleRequestTimeoutSecs: 0.01,+ // maxRequestRetries: 1,+ // useSessionPool: true,+ // handleRequestFunction: async ({ session }) => {+ // expect(session.constructor.name).to.be.eql('Session');+ // expect(session.id).to.exist // eslint-disable-line+ // },+ // handleFailedRequestFunction: ({ request }) => results.push(request),+ // });+ //+ // await crawler.run();+ // expect(crawler.sessionPool).to.exist; //eslint-disable-line+ // expect(results).length(0);+ // });+ //+ // xit('should use pass options to sessionPool', async () => {+ // const url = 'https://example.com';+ // const requestList = new Apify.RequestList({ sources: [{ url }] });+ // await requestList.initialize();+ //+ // const crawler = new Apify.BasicCrawler({+ // requestList,+ // handleRequestTimeoutSecs: 0.01,+ // maxRequestRetries: 1,+ // useSessionPool: true,+ // sessionPoolOptions: {+ // maxPoolSize: 10,+ // },+ // handleRequestFunction: async () => {},+ // handleFailedRequestFunction: () => {},+ // });+ // await crawler.run();+ //+ // expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);+ // });+ //+ // xit('should not use SessionPool by default', async () => {+ // const url = 'https://example.com';+ // const requestList = new Apify.RequestList({ sources: [{ url }] });+ // await requestList.initialize();+ //+ // const crawler = new Apify.BasicCrawler({+ // requestList,+ // handleRequestTimeoutSecs: 0.01,+ // maxRequestRetries: 1,+ // handleRequestFunction: async () => {},+ // handleFailedRequestFunction: () => {},+ // });+ // await crawler.run();+ //+ // expect(crawler.sessionPool).to.be.undefined; // eslint-disable-line+ // });+ // });});diff --git a/test/session_pool/session_pool.js b/test/session_pool/session_pool.jsindex 4eb550591..304bb6f84 100644--- a/test/session_pool/session_pool.js+++ b/test/session_pool/session_pool.js@@ -7,7 +7,7 @@ import events from '../../build/events';import { ACTOR_EVENT_NAMES_EX } from '../../build/constants';import { Session } from '../../src/session_pool/session';-describe('SessionPool - testing session pool', async () => {+describe('SessionPool - testing session pool', () => {let sessionPool;before(() => {@@ -157,7 +157,7 @@ describe('SessionPool - testing session pool', async () => {});it('should create session', async () => {- await sessionPool._createSession(); // eslint-disable-line+ await sessionPool._createSession(); // eslint-disable-lineexpect(sessionPool.sessions.length).to.be.eql(1);expect(sessionPool.sessions[0].id).to.exist; // eslint-disable-line});@@ -231,6 +231,7 @@ describe('SessionPool - testing session pool', async () => {const createSessionFunction = (sessionPool2) => {isCalled = true;expect(sessionPool2 instanceof SessionPool).to.be.true; // eslint-disable-line+ console.log(sessionPool2 instanceof SessionPool);return new Session({ sessionPool: sessionPool2 });};const newSessionPool = await Apify.openSessionPool({ createSessionFunction });