Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: fix migration test

Szymon Marczakcf4176a34fe7
diff --git a/test/e2e/migration/actor/main.js b/test/e2e/migration/actor/main.js
index 440cbe91d..057ca2896 100644
--- a/test/e2e/migration/actor/main.js
+++ b/test/e2e/migration/actor/main.js
@@ -1,3 +1,6 @@
+import { Worker, isMainThread } from 'worker_threads';
+import { URL } from 'url';
+import { once } from 'events';
import { Actor } from 'apify';
import { CheerioCrawler, Configuration, Dataset } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
@@ -9,7 +12,21 @@ const mainOptions = {
storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,
};
-const run = async () => {
+const thisFile = new URL(import.meta.url);
+
+if (isMainThread) {
+ const firstRun = new Worker(thisFile);
+
+ const [firstExitCode] = await once(firstRun, 'exit');
+
+ const secondRun = new Worker(thisFile);
+
+ const [secondExitCode] = await once(secondRun, 'exit');
+
+ if (firstRun !== 0 || secondRun !== 0) {
+ throw new Error(`Unexpected exit code:\nfirst run: ${firstExitCode}\nsecond run: ${secondExitCode}`);
+ }
+} else {
await Actor.main(async () => {
const crawler = new CheerioCrawler({
maxConcurrency: 1,
@@ -29,7 +46,4 @@ const run = async () => {
await crawler.run(['https://apify.com']);
}, mainOptions);
-};
-
-await run();
-await run();
+}