Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fixed tests
Petr Patek95e16e4f6300
diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex 534497b41..b3ca6297a 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -1,27 +1,28 @@import log from 'apify-shared/log';import { ENV_VARS } from 'apify-shared/consts';import * as Apify from '../../build';-import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR } from '../_helper';+import LocalStorageDirEmulator from '../local_storage_dir_emulator';describe('PuppeteerCrawler', () => {let prevEnvHeadless;let logLevel;+ let localStorageEmulator;- beforeAll(() => {+ beforeAll(async () => {prevEnvHeadless = process.env[ENV_VARS.HEADLESS];process.env[ENV_VARS.HEADLESS] = '1';logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);- process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;+ localStorageEmulator = new LocalStorageDirEmulator();+ await localStorageEmulator.init();});-- afterEach(() => {- emptyLocalStorageSubdir('key_value_stores/default');+ afterEach(async () => {+ await localStorageEmulator.clean();});-- afterAll(() => {+ afterAll(async () => {log.setLevel(logLevel);process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;+ await localStorageEmulator.destroy();});test('should work', async () => {diff --git a/test/puppeteer_pool.test.js b/test/puppeteer_pool.test.jsindex 6df6fbbb4..12490af6b 100644--- a/test/puppeteer_pool.test.js+++ b/test/puppeteer_pool.test.js@@ -5,9 +5,9 @@ import { ENV_VARS } from 'apify-shared/consts';import * as Apify from '../build/index';import { launchPuppeteer } from '../build/puppeteer';import { SessionPool } from '../build/session_pool/session_pool';-import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from './_helper';import { BROWSER_SESSION_KEY_NAME } from '../build/puppeteer_pool';import { sleep } from '../src/utils';+import LocalStorageDirEmulator from './local_storage_dir_emulator';const shortSleep = (millis = 25) => new Promise(resolve => setTimeout(resolve, millis));@@ -628,12 +628,19 @@ describe('PuppeteerPool', () => {});describe('uses sessionPool', () => {- beforeAll(() => {- process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;+ let localStorageEmulator;++ beforeAll(async () => {+ localStorageEmulator = new LocalStorageDirEmulator();+ await localStorageEmulator.init();+ });++ afterEach(async () => {+ await localStorageEmulator.clean();});- afterEach(() => {- emptyLocalStorageSubdir('key_value_stores');+ afterAll(async () => {+ await localStorageEmulator.destroy();});test('should work', async () => {