Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
save
Petr Pateka1d0878fbd7a
diff --git a/test/crawlers/basic_crawler.js b/test/crawlers/basic_crawler.jsindex 5043a64ef..707cb5e0b 100644--- a/test/crawlers/basic_crawler.js+++ b/test/crawlers/basic_crawler.js@@ -15,12 +15,16 @@ describe('BasicCrawler', () => {before(() => {logLevel = log.getLevel();log.setLevel(log.LEVELS.OFF);+ emptyLocalStorageSubdir('key_value_stores/default');+});after(() => {log.setLevel(logLevel);+ emptyLocalStorageSubdir('key_value_stores/default');});+it('should run in parallel thru all the requests', async () => {const startedAt = Date.now();const sources = _.range(0, 500).map(index => ({ url: `https://example.com/${index}` }));@@ -599,74 +603,71 @@ describe('BasicCrawler', () => {results[0].errorMessages.forEach(msg => expect(msg).to.include('handleRequestFunction timed out'));});- // describe('Uses SessionPool', () => {- // before(() => {- // process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;- // });- //- // after(() => {- // // emptyLocalStorageSubdir('key_value_stores/default');- // });- //- // xit('should use SessionPool when useSessionPool is true ', async () => {- // const url = 'https://example.com';- // const requestList = new Apify.RequestList({ sources: [{ url }] });- // await requestList.initialize();- // const results = [];- //- // const crawler = new Apify.BasicCrawler({- // requestList,- // handleRequestTimeoutSecs: 0.01,- // maxRequestRetries: 1,- // useSessionPool: true,- // handleRequestFunction: async ({ session }) => {- // expect(session.constructor.name).to.be.eql('Session');- // expect(session.id).to.exist // eslint-disable-line- // },- // handleFailedRequestFunction: ({ request }) => results.push(request),- // });- //- // await crawler.run();- // expect(crawler.sessionPool).to.exist; //eslint-disable-line- // expect(results).length(0);- // });- //- // xit('should use pass options to sessionPool', async () => {- // const url = 'https://example.com';- // const requestList = new Apify.RequestList({ sources: [{ url }] });- // await requestList.initialize();- //- // const crawler = new Apify.BasicCrawler({- // requestList,- // handleRequestTimeoutSecs: 0.01,- // maxRequestRetries: 1,- // useSessionPool: true,- // sessionPoolOptions: {- // maxPoolSize: 10,- // },- // handleRequestFunction: async () => {},- // handleFailedRequestFunction: () => {},- // });- // await crawler.run();- //- // expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);- // });- //- // xit('should not use SessionPool by default', async () => {- // const url = 'https://example.com';- // const requestList = new Apify.RequestList({ sources: [{ url }] });- // await requestList.initialize();- //- // const crawler = new Apify.BasicCrawler({- // requestList,- // handleRequestTimeoutSecs: 0.01,- // maxRequestRetries: 1,- // handleRequestFunction: async () => {},- // handleFailedRequestFunction: () => {},- // });- // await crawler.run();- //- // expect(crawler.sessionPool).to.be.undefined; // eslint-disable-line- // });- // });+ describe('Uses SessionPool', () => {+ it('should use SessionPool when useSessionPool is true ', async () => {+ const url = 'https://example.com';+ const requestList = new Apify.RequestList({ sources: [{ url }] });+ await requestList.initialize();+ const results = [];++ const crawler = new Apify.BasicCrawler({+ requestList,+ handleRequestTimeoutSecs: 0.01,+ maxRequestRetries: 1,+ useSessionPool: true,+ sessionPoolOptions: {+ maxPoolSize: 10,+ persistStateKey: 'POOL',+ },+ handleRequestFunction: async ({ session }) => {+ expect(session.constructor.name).to.be.eql('Session');+ expect(session.id).to.exist // eslint-disable-line+ },+ handleFailedRequestFunction: ({ request }) => results.push(request),+ });++ await crawler.run();+ expect(crawler.sessionPool).to.exist; //eslint-disable-line+ expect(results).length(0);+ });++ xit('should use pass options to sessionPool', async () => {+ const url = 'https://example.com';+ const requestList = new Apify.RequestList({ sources: [{ url }] });+ await requestList.initialize();++ const crawler = new Apify.BasicCrawler({+ requestList,+ handleRequestTimeoutSecs: 0.01,+ maxRequestRetries: 1,+ useSessionPool: true,+ sessionPoolOptions: {+ maxPoolSize: 10,+ persistStateKey: 'POOL',+ },+ handleRequestFunction: async () => {},+ handleFailedRequestFunction: () => {},+ });+ await crawler.run();++ expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);+ });++ it('should not use SessionPool by default', async () => {+ const url = 'https://example.com';+ const requestList = new Apify.RequestList({ sources: [{ url }] });+ await requestList.initialize();++ const crawler = new Apify.BasicCrawler({+ requestList,+ handleRequestTimeoutSecs: 0.01,+ maxRequestRetries: 1,+ handleRequestFunction: async () => {},+ handleFailedRequestFunction: () => {},+ });+ await crawler.run();++ expect(crawler.sessionPool).to.be.undefined; // eslint-disable-line+ });+ });});