Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixed tests.

Petr Patek8408ffab1df2
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index c63b79f46..b861ef135 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -2,9 +2,9 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import log from 'apify-shared/log';
import _ from 'underscore';
import BasicCrawler from './basic_crawler';
-import PuppeteerPool from '../puppeteer_pool';
+import PuppeteerPool, { BROWSER_SESSION_KEY_NAME } from '../puppeteer_pool';
import { addTimeoutToPromise } from '../utils';
-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER, BROWSER_SESSION_KEY_NAME } from '../constants';
+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import { gotoExtended } from '../puppeteer_utils';
import { openSessionPool } from '../session_pool/session_pool';
diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.js
index 27c2c3ee3..534497b41 100644
--- a/test/crawlers/puppeteer_crawler.test.js
+++ b/test/crawlers/puppeteer_crawler.test.js
@@ -108,11 +108,11 @@ describe('PuppeteerCrawler', () => {
const puppeteerCrawler = new Apify.PuppeteerCrawler({
requestList,
useSessionPool: true,
- handlePageFunction: ({ session }) => {
+ handlePageFunction: async ({ session }) => {
handlePageSessions.push(session);
return Promise.resolve();
},
- gotoFunction: ({ session }) => {
+ gotoFunction: async ({ session }) => {
goToPageSessions.push(session);
return Apify.launchPuppeteer();
},
diff --git a/test/puppeteer_pool.test.js b/test/puppeteer_pool.test.js
index 40464109e..6df6fbbb4 100644
--- a/test/puppeteer_pool.test.js
+++ b/test/puppeteer_pool.test.js
@@ -306,7 +306,7 @@ describe('PuppeteerPool', () => {
// Check cache dirs were deleted
expect(fs.existsSync(dir1)).toBe(false);
expect(fs.existsSync(dir3)).toBe(false);
- });
+ }, 10000);
describe('reuse of browser tabs', () => {
xit('should work', async () => {
@@ -633,7 +633,7 @@ describe('PuppeteerPool', () => {
});
afterEach(() => {
- emptyLocalStorageSubdir('key_value_stores/default');
+ emptyLocalStorageSubdir('key_value_stores');
});
test('should work', async () => {
@@ -641,7 +641,6 @@ describe('PuppeteerPool', () => {
await sessionPool.initialize();
const pool = new Apify.PuppeteerPool({
launchPuppeteerOptions: { headless: true },
- useLiveView: true,
sessionPool,
});
expect(pool.sessionPool.constructor.name).toEqual('SessionPool');