Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: fetch test key value store correctly on all test platforms (#1493)
Vlad Franguee3a8ae6ba05
diff --git a/test/e2e/automatic-persist-value/test.mjs b/test/e2e/automatic-persist-value/test.mjsindex ef9eb0e50..1b101d84c 100644--- a/test/e2e/automatic-persist-value/test.mjs+++ b/test/e2e/automatic-persist-value/test.mjs@@ -1,15 +1,13 @@-import { initialize, expect, getActorTestDir, runActor, getKeyValueStoreItems } from '../tools.mjs';+import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);-await runActor(testActorDirname);+const { keyValueStoreItems } = await runActor(testActorDirname);-const kvsItems = await getKeyValueStoreItems(testActorDirname, 'test');+expect(keyValueStoreItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');-expect(kvsItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');--const [{ name, raw }] = kvsItems;+const [{ name, raw }] = keyValueStoreItems;expect(name === 'crawlee', 'Key-value store auto-saved value is named "crawlee"');diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex dfc80dbce..68ba9c060 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -60,6 +60,7 @@ export function getActorTestDir(url) {export async function runActor(dirName, memory = 4096) {let stats;let datasetItems;+ let keyValueStoreItems;if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {await copyPackages(dirName);@@ -91,6 +92,21 @@ export async function runActor(dirName, memory = 4096) {stats = value;const { items } = await client.dataset(defaultDatasetId).listItems();datasetItems = items;++ try {+ const { items: keyValueItems } = await client.keyValueStore('test').listKeys();++ keyValueStoreItems = await Promise.all(keyValueItems.map(async ({ key }) => {+ const record = await client.keyValueStore('test').getRecord(key, { buffer: true });++ return {+ name: record.key,+ raw: record.value,+ };+ }));+ } catch {+ keyValueStoreItems = [];+ }} else {if (dirName.split('/').at(-2).endsWith('-ts')) {execSync('tsc', { cwd: dirName });@@ -101,9 +117,10 @@ export async function runActor(dirName, memory = 4096) {await setTimeout(50);stats = await getStats(dirName);datasetItems = await getDatasetItems(dirName);+ keyValueStoreItems = await getKeyValueStoreItems(dirName);}- return { stats, datasetItems };+ return { stats, datasetItems, keyValueStoreItems };}/**@@ -211,11 +228,10 @@ export async function getDatasetItems(dirName) {/*** Gets all items in the key-value store, as a Buffer* @param {string} dirName- * @param {string} storeName*/-export async function getKeyValueStoreItems(dirName, storeName) {+export async function getKeyValueStoreItems(dirName) {const dir = getStorage(dirName);- const storePath = join(dir, `key_value_stores/${storeName}`);+ const storePath = join(dir, `key_value_stores/test/`);if (!existsSync(storePath)) {return [];