Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: fix flakyness of some test that used default storage (#1593)
Martin Adámek38dc277eae9a
diff --git a/package.json b/package.jsonindex af1f71fec..9b0fcbdb0 100644--- a/package.json+++ b/package.json@@ -41,7 +41,7 @@"postbuild": "node ./scripts/typescript_fixes.mjs","ci:build": "turbo run build --cache-dir=\".turbo\"","postci:build": "node ./scripts/typescript_fixes.mjs",- "test": "jest --silent --runInBand",+ "test": "jest --silent","test:e2e": "node test/e2e/run.mjs","coverage": "jest --coverage","publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish",diff --git a/packages/basic-crawler/test/migration.test.ts b/packages/basic-crawler/test/migration.test.tsindex b49668d41..71f4898ae 100644--- a/packages/basic-crawler/test/migration.test.ts+++ b/packages/basic-crawler/test/migration.test.ts@@ -1,7 +1,18 @@import type { Log } from '@apify/log';import log from '@apify/log';+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';import { BasicCrawler, RequestList } from '../src/index';+const localStorageEmulator = new MemoryStorageEmulator();++beforeEach(async () => {+ await localStorageEmulator.init();+});++afterAll(async () => {+ await localStorageEmulator.destroy();+});+describe('Moving from handleRequest* to requestHandler*', () => {let requestList: RequestList;let testLogger: Log;diff --git a/packages/browser-crawler/test/migration.test.ts b/packages/browser-crawler/test/migration.test.tsindex 9497c5c1b..f3b4b208f 100644--- a/packages/browser-crawler/test/migration.test.ts+++ b/packages/browser-crawler/test/migration.test.ts@@ -2,8 +2,19 @@ import type { Log } from '@apify/log';import log from '@apify/log';import { PuppeteerPlugin } from '@crawlee/browser-pool';import puppeteer from 'puppeteer';+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';import { BrowserCrawler, RequestList } from '../src/index';+const localStorageEmulator = new MemoryStorageEmulator();++beforeEach(async () => {+ await localStorageEmulator.init();+});++afterAll(async () => {+ await localStorageEmulator.destroy();+});+const plugin = new PuppeteerPlugin(puppeteer);describe('Moving from handleRequest* to requestHandler*', () => {diff --git a/packages/cheerio-crawler/test/migration.test.ts b/packages/cheerio-crawler/test/migration.test.tsindex ece995f49..36625ce22 100644--- a/packages/cheerio-crawler/test/migration.test.ts+++ b/packages/cheerio-crawler/test/migration.test.ts@@ -1,7 +1,18 @@import type { Log } from '@apify/log';import log from '@apify/log';+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';import { CheerioCrawler, RequestList } from '../src/index';+const localStorageEmulator = new MemoryStorageEmulator();++beforeEach(async () => {+ await localStorageEmulator.init();+});++afterAll(async () => {+ await localStorageEmulator.destroy();+});+describe('Moving from handleRequest* to requestHandler*', () => {let requestList: RequestList;let testLogger: Log;diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.tsindex da958ca4a..8b5f20771 100644--- a/test/core/storages/key_value_store.test.ts+++ b/test/core/storages/key_value_store.test.ts@@ -1,7 +1,17 @@-import { ENV_VARS } from '@apify/consts';import { maybeStringify, Configuration, KeyValueStore } from '@crawlee/core';import type { Dictionary } from '@crawlee/utils';import { PassThrough } from 'stream';+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';++const localStorageEmulator = new MemoryStorageEmulator();++beforeEach(async () => {+ await localStorageEmulator.init();+});++afterAll(async () => {+ await localStorageEmulator.destroy();+});describe('KeyValueStore', () => {const client = Configuration.getStorageClient();