Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixed tests.
Petr Patek8408ffab1df2
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex c63b79f46..b861ef135 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -2,9 +2,9 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import log from 'apify-shared/log';import _ from 'underscore';import BasicCrawler from './basic_crawler';-import PuppeteerPool from '../puppeteer_pool';+import PuppeteerPool, { BROWSER_SESSION_KEY_NAME } from '../puppeteer_pool';import { addTimeoutToPromise } from '../utils';-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER, BROWSER_SESSION_KEY_NAME } from '../constants';+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';import { gotoExtended } from '../puppeteer_utils';import { openSessionPool } from '../session_pool/session_pool';diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex 27c2c3ee3..534497b41 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -108,11 +108,11 @@ describe('PuppeteerCrawler', () => {const puppeteerCrawler = new Apify.PuppeteerCrawler({requestList,useSessionPool: true,- handlePageFunction: ({ session }) => {+ handlePageFunction: async ({ session }) => {handlePageSessions.push(session);return Promise.resolve();},- gotoFunction: ({ session }) => {+ gotoFunction: async ({ session }) => {goToPageSessions.push(session);return Apify.launchPuppeteer();},diff --git a/test/puppeteer_pool.test.js b/test/puppeteer_pool.test.jsindex 40464109e..6df6fbbb4 100644--- a/test/puppeteer_pool.test.js+++ b/test/puppeteer_pool.test.js@@ -306,7 +306,7 @@ describe('PuppeteerPool', () => {// Check cache dirs were deletedexpect(fs.existsSync(dir1)).toBe(false);expect(fs.existsSync(dir3)).toBe(false);- });+ }, 10000);describe('reuse of browser tabs', () => {xit('should work', async () => {@@ -633,7 +633,7 @@ describe('PuppeteerPool', () => {});afterEach(() => {- emptyLocalStorageSubdir('key_value_stores/default');+ emptyLocalStorageSubdir('key_value_stores');});test('should work', async () => {@@ -641,7 +641,6 @@ describe('PuppeteerPool', () => {await sessionPool.initialize();const pool = new Apify.PuppeteerPool({launchPuppeteerOptions: { headless: true },- useLiveView: true,sessionPool,});expect(pool.sessionPool.constructor.name).toEqual('SessionPool');