Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: enable migration testing (#1583)

Szymon Marczakee3a68fff1fc
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index b96fc211e..5c9e49033 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -598,6 +598,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
await this._init();
await this.stats.startCapturing();
+ process.once('SIGINT', async () => {
+ this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm run start');
+ await this._pauseOnMigration();
+ await this.autoscaledPool!.abort();
+ });
+
try {
this.log.info('Starting the crawl');
await this.autoscaledPool!.run();
diff --git a/test/e2e/migration/actor/.gitignore b/test/e2e/migration/actor/.gitignore
new file mode 100644
index 000000000..ced7cbfc5
--- /dev/null
+++ b/test/e2e/migration/actor/.gitignore
@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage
diff --git a/test/e2e/migration/actor/Dockerfile b/test/e2e/migration/actor/Dockerfile
new file mode 100644
index 000000000..b5c27460b
--- /dev/null
+++ b/test/e2e/migration/actor/Dockerfile
@@ -0,0 +1,16 @@
+FROM apify/actor-node:16-beta
+
+COPY packages ./packages
+COPY package*.json ./
+
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional \
+ && npm update \
+ && echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
+
+COPY . ./
diff --git a/test/e2e/migration/actor/apify.json b/test/e2e/migration/actor/apify.json
new file mode 100644
index 000000000..f65813559
--- /dev/null
+++ b/test/e2e/migration/actor/apify.json
@@ -0,0 +1,6 @@
+{
+ "name": "test-migration",
+ "version": "0.0",
+ "buildTag": "latest",
+ "env": null
+}
diff --git a/test/e2e/migration/actor/main.js b/test/e2e/migration/actor/main.js
new file mode 100644
index 000000000..440cbe91d
--- /dev/null
+++ b/test/e2e/migration/actor/main.js
@@ -0,0 +1,35 @@
+import { Actor } from 'apify';
+import { CheerioCrawler, Configuration, Dataset } from '@crawlee/cheerio';
+import { ApifyStorageLocal } from '@apify/storage-local';
+
+process.env.CRAWLEE_PURGE_ON_START = '0';
+
+const mainOptions = {
+ exit: Actor.isAtHome(),
+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,
+};
+
+const run = async () => {
+ await Actor.main(async () => {
+ const crawler = new CheerioCrawler({
+ maxConcurrency: 1,
+ maxRequestsPerCrawl: 5,
+ async requestHandler({ enqueueLinks, request }) {
+ const { url } = request;
+ await enqueueLinks({ pseudoUrls: ['https://apify.com[(/[\\w-]+)?]'] });
+
+ await Dataset.pushData({ url });
+
+ process.emit('SIGINT');
+ },
+ });
+
+ // eslint-disable-next-line no-underscore-dangle
+ Configuration.getGlobalConfig().getStorageClient().__purged = false;
+
+ await crawler.run(['https://apify.com']);
+ }, mainOptions);
+};
+
+await run();
+await run();
diff --git a/test/e2e/migration/actor/package.json b/test/e2e/migration/actor/package.json
new file mode 100644
index 000000000..21a09559a
--- /dev/null
+++ b/test/e2e/migration/actor/package.json
@@ -0,0 +1,28 @@
+{
+ "name": "test-migration",
+ "version": "0.0.1",
+ "description": "Migration Test",
+ "dependencies": {
+ "apify": "next",
+ "@apify/storage-local": "^2.1.0",
+ "@crawlee/basic": "file:./packages/basic-crawler",
+ "@crawlee/browser-pool": "file:./packages/browser-pool",
+ "@crawlee/http": "file:./packages/http-crawler",
+ "@crawlee/cheerio": "file:./packages/cheerio-crawler",
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/memory-storage": "file:./packages/memory-storage",
+ "@crawlee/types": "file:./packages/types",
+ "@crawlee/utils": "file:./packages/utils"
+ },
+ "overrides": {
+ "apify": {
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/utils": "file:./packages/utils"
+ }
+ },
+ "scripts": {
+ "start": "node main.js"
+ },
+ "type": "module",
+ "license": "ISC"
+}
diff --git a/test/e2e/migration/test.mjs b/test/e2e/migration/test.mjs
new file mode 100644
index 000000000..a60519eea
--- /dev/null
+++ b/test/e2e/migration/test.mjs
@@ -0,0 +1,10 @@
+import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs';
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { datasetItems } = await runActor(testActorDirname);
+
+await expect(datasetItems.length === 2, 'Number of dataset items');
+await expect(validateDataset(datasetItems, ['url']), 'Dataset items validation');
+await expect(datasetItems[0].url !== datasetItems[1].url, 'Dataset items unique');