Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: wait for persist state listeners to run when event manager closes (#1481)
Vlad Franguaa550edf7e01
diff --git a/packages/core/src/events/event_manager.ts b/packages/core/src/events/event_manager.tsindex d1dc3ba6b..bb7833058 100644--- a/packages/core/src/events/event_manager.ts+++ b/packages/core/src/events/event_manager.ts@@ -58,6 +58,9 @@ export abstract class EventManager {// Emit final PERSIST_STATE eventthis.emit(EventType.PERSIST_STATE, { isMigrating: false });++ // Wait for PERSIST_STATE to process+ await this.waitForAllListenersToComplete();}on(event: EventTypeName, listener: (...args: any[]) => any): void {diff --git a/test/e2e/automatic-persist-value/actor/.gitignore b/test/e2e/automatic-persist-value/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/automatic-persist-value/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/automatic-persist-value/actor/Dockerfile b/test/e2e/automatic-persist-value/actor/Dockerfilenew file mode 100644index 000000000..278810c77--- /dev/null+++ b/test/e2e/automatic-persist-value/actor/Dockerfile@@ -0,0 +1,15 @@+FROM apify/actor-node:16++COPY packages ./packages+COPY package*.json ./++RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional \+ && echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++COPY . ./diff --git a/test/e2e/automatic-persist-value/actor/apify.json b/test/e2e/automatic-persist-value/actor/apify.jsonnew file mode 100644index 000000000..944ceeb3d--- /dev/null+++ b/test/e2e/automatic-persist-value/actor/apify.json@@ -0,0 +1,6 @@+{+ "name": "test-automatic-persist-value",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/automatic-persist-value/actor/main.js b/test/e2e/automatic-persist-value/actor/main.jsnew file mode 100644index 000000000..bd5f48b11--- /dev/null+++ b/test/e2e/automatic-persist-value/actor/main.js@@ -0,0 +1,22 @@+import { Actor, KeyValueStore } from 'apify';+import { ApifyStorageLocal } from '@apify/storage-local';+import { BasicCrawler } from '@crawlee/basic';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,+};++await Actor.main(async () => {+ const kv = await KeyValueStore.open('test');++ const crawler = new BasicCrawler({+ async requestHandler() {+ const automaticValue = await kv.getAutoSavedValue('hello');++ automaticValue.crawlee = 'awesome!';+ },+ });++ await crawler.run(['https://example.com']);+}, mainOptions);diff --git a/test/e2e/automatic-persist-value/actor/package.json b/test/e2e/automatic-persist-value/actor/package.jsonnew file mode 100644index 000000000..261549865--- /dev/null+++ b/test/e2e/automatic-persist-value/actor/package.json@@ -0,0 +1,17 @@+{+ "name": "test-automatic-persist-value",+ "version": "0.0.1",+ "description": "Key-Value Store - Automatic Persist Value Test",+ "dependencies": {+ "apify": "^3.0.0-beta.75",+ "@apify/storage-local": "^2.1.0",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils"+ },+ "scripts": { "start": "node main.js" },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/automatic-persist-value/test.mjs b/test/e2e/automatic-persist-value/test.mjsnew file mode 100644index 000000000..ef9eb0e50--- /dev/null+++ b/test/e2e/automatic-persist-value/test.mjs@@ -0,0 +1,19 @@+import { initialize, expect, getActorTestDir, runActor, getKeyValueStoreItems } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++await runActor(testActorDirname);++const kvsItems = await getKeyValueStoreItems(testActorDirname, 'test');++expect(kvsItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');++const [{ name, raw }] = kvsItems;++expect(name === 'crawlee', 'Key-value store auto-saved value is named "crawlee"');++const parsed = JSON.parse(raw.toString());++expect(typeof parsed === 'object' && parsed !== null, 'Key-value store auto-save value is a non-nullable object');+expect(parsed.crawlee === 'awesome!', 'Key-value store auto-save value has a property "crawlee" that is set to "awesome!"');diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.jsonindex 7cf459577..6cb03761d 100644--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json@@ -6,7 +6,6 @@"apify": "^3.0.0-beta.75","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler",- "@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/core": "file:./packages/core","@crawlee/memory-storage": "file:./packages/memory-storage","@crawlee/types": "file:./packages/types",diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex 7b012d7b8..dfc80dbce 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -1,12 +1,13 @@import { dirname, join, resolve } from 'node:path';import { fileURLToPath } from 'node:url';import { existsSync } from 'node:fs';-import { readdir } from 'node:fs/promises';+import { readdir, readFile } from 'node:fs/promises';import { homedir } from 'node:os';import { setTimeout } from 'node:timers/promises';import { execSync } from 'node:child_process';import fs from 'fs-extra';import { Actor } from 'apify';+// eslint-disable-next-line import/no-relative-packagesimport { URL_NO_COMMAS_REGEX } from '../../packages/utils/dist/index.mjs';export const SKIPPED_TEST_CLOSE_CODE = 404;@@ -207,6 +208,35 @@ export async function getDatasetItems(dirName) {return datasetItems;}+/**+ * Gets all items in the key-value store, as a Buffer+ * @param {string} dirName+ * @param {string} storeName+ */+export async function getKeyValueStoreItems(dirName, storeName) {+ const dir = getStorage(dirName);+ const storePath = join(dir, `key_value_stores/${storeName}`);++ if (!existsSync(storePath)) {+ return [];+ }++ const dirents = await readdir(storePath, { withFileTypes: true });+ const fileNames = dirents.filter((dirent) => dirent.isFile());+ const keyValueStoreRecords = [];++ for (const fileName of fileNames) {+ if (fileName.name.includes('__metadata__')) continue;++ const filePath = join(storePath, fileName.name);+ const buffer = await readFile(filePath);++ keyValueStoreRecords.push({ name: fileName.name.split('.').slice(0, -1).join('.'), raw: buffer });+ }++ return keyValueStoreRecords;+}+/*** @param {string} dirName*/