Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: fix flakyness of some test that used default storage (#1593)

Martin Adámek38dc277eae9a
diff --git a/package.json b/package.json
index af1f71fec..9b0fcbdb0 100644
--- a/package.json
+++ b/package.json
@@ -41,7 +41,7 @@
"postbuild": "node ./scripts/typescript_fixes.mjs",
"ci:build": "turbo run build --cache-dir=\".turbo\"",
"postci:build": "node ./scripts/typescript_fixes.mjs",
- "test": "jest --silent --runInBand",
+ "test": "jest --silent",
"test:e2e": "node test/e2e/run.mjs",
"coverage": "jest --coverage",
"publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish",
diff --git a/packages/basic-crawler/test/migration.test.ts b/packages/basic-crawler/test/migration.test.ts
index b49668d41..71f4898ae 100644
--- a/packages/basic-crawler/test/migration.test.ts
+++ b/packages/basic-crawler/test/migration.test.ts
@@ -1,7 +1,18 @@
import type { Log } from '@apify/log';
import log from '@apify/log';
+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
import { BasicCrawler, RequestList } from '../src/index';
+const localStorageEmulator = new MemoryStorageEmulator();
+
+beforeEach(async () => {
+ await localStorageEmulator.init();
+});
+
+afterAll(async () => {
+ await localStorageEmulator.destroy();
+});
+
describe('Moving from handleRequest* to requestHandler*', () => {
let requestList: RequestList;
let testLogger: Log;
diff --git a/packages/browser-crawler/test/migration.test.ts b/packages/browser-crawler/test/migration.test.ts
index 9497c5c1b..f3b4b208f 100644
--- a/packages/browser-crawler/test/migration.test.ts
+++ b/packages/browser-crawler/test/migration.test.ts
@@ -2,8 +2,19 @@ import type { Log } from '@apify/log';
import log from '@apify/log';
import { PuppeteerPlugin } from '@crawlee/browser-pool';
import puppeteer from 'puppeteer';
+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
import { BrowserCrawler, RequestList } from '../src/index';
+const localStorageEmulator = new MemoryStorageEmulator();
+
+beforeEach(async () => {
+ await localStorageEmulator.init();
+});
+
+afterAll(async () => {
+ await localStorageEmulator.destroy();
+});
+
const plugin = new PuppeteerPlugin(puppeteer);
describe('Moving from handleRequest* to requestHandler*', () => {
diff --git a/packages/cheerio-crawler/test/migration.test.ts b/packages/cheerio-crawler/test/migration.test.ts
index ece995f49..36625ce22 100644
--- a/packages/cheerio-crawler/test/migration.test.ts
+++ b/packages/cheerio-crawler/test/migration.test.ts
@@ -1,7 +1,18 @@
import type { Log } from '@apify/log';
import log from '@apify/log';
+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
import { CheerioCrawler, RequestList } from '../src/index';
+const localStorageEmulator = new MemoryStorageEmulator();
+
+beforeEach(async () => {
+ await localStorageEmulator.init();
+});
+
+afterAll(async () => {
+ await localStorageEmulator.destroy();
+});
+
describe('Moving from handleRequest* to requestHandler*', () => {
let requestList: RequestList;
let testLogger: Log;
diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts
index da958ca4a..8b5f20771 100644
--- a/test/core/storages/key_value_store.test.ts
+++ b/test/core/storages/key_value_store.test.ts
@@ -1,7 +1,17 @@
-import { ENV_VARS } from '@apify/consts';
import { maybeStringify, Configuration, KeyValueStore } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
import { PassThrough } from 'stream';
+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
+
+const localStorageEmulator = new MemoryStorageEmulator();
+
+beforeEach(async () => {
+ await localStorageEmulator.init();
+});
+
+afterAll(async () => {
+ await localStorageEmulator.destroy();
+});
describe('KeyValueStore', () => {
const client = Configuration.getStorageClient();