Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: enable migration testing (#1583)
Szymon Marczakee3a68fff1fc
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex b96fc211e..5c9e49033 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -598,6 +598,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextawait this._init();await this.stats.startCapturing();+ process.once('SIGINT', async () => {+ this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm run start');+ await this._pauseOnMigration();+ await this.autoscaledPool!.abort();+ });+try {this.log.info('Starting the crawl');await this.autoscaledPool!.run();diff --git a/test/e2e/migration/actor/.gitignore b/test/e2e/migration/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/migration/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/migration/actor/Dockerfile b/test/e2e/migration/actor/Dockerfilenew file mode 100644index 000000000..b5c27460b--- /dev/null+++ b/test/e2e/migration/actor/Dockerfile@@ -0,0 +1,16 @@+FROM apify/actor-node:16-beta++COPY packages ./packages+COPY package*.json ./++RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional \+ && npm update \+ && echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++COPY . ./diff --git a/test/e2e/migration/actor/apify.json b/test/e2e/migration/actor/apify.jsonnew file mode 100644index 000000000..f65813559--- /dev/null+++ b/test/e2e/migration/actor/apify.json@@ -0,0 +1,6 @@+{+ "name": "test-migration",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/migration/actor/main.js b/test/e2e/migration/actor/main.jsnew file mode 100644index 000000000..440cbe91d--- /dev/null+++ b/test/e2e/migration/actor/main.js@@ -0,0 +1,35 @@+import { Actor } from 'apify';+import { CheerioCrawler, Configuration, Dataset } from '@crawlee/cheerio';+import { ApifyStorageLocal } from '@apify/storage-local';++process.env.CRAWLEE_PURGE_ON_START = '0';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,+};++const run = async () => {+ await Actor.main(async () => {+ const crawler = new CheerioCrawler({+ maxConcurrency: 1,+ maxRequestsPerCrawl: 5,+ async requestHandler({ enqueueLinks, request }) {+ const { url } = request;+ await enqueueLinks({ pseudoUrls: ['https://apify.com[(/[\\w-]+)?]'] });++ await Dataset.pushData({ url });++ process.emit('SIGINT');+ },+ });++ // eslint-disable-next-line no-underscore-dangle+ Configuration.getGlobalConfig().getStorageClient().__purged = false;++ await crawler.run(['https://apify.com']);+ }, mainOptions);+};++await run();+await run();diff --git a/test/e2e/migration/actor/package.json b/test/e2e/migration/actor/package.jsonnew file mode 100644index 000000000..21a09559a--- /dev/null+++ b/test/e2e/migration/actor/package.json@@ -0,0 +1,28 @@+{+ "name": "test-migration",+ "version": "0.0.1",+ "description": "Migration Test",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.0",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/http": "file:./packages/http-crawler",+ "@crawlee/cheerio": "file:./packages/cheerio-crawler",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "scripts": {+ "start": "node main.js"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/migration/test.mjs b/test/e2e/migration/test.mjsnew file mode 100644index 000000000..a60519eea--- /dev/null+++ b/test/e2e/migration/test.mjs@@ -0,0 +1,10 @@+import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { datasetItems } = await runActor(testActorDirname);++await expect(datasetItems.length === 2, 'Number of dataset items');+await expect(validateDataset(datasetItems, ['url']), 'Dataset items validation');+await expect(datasetItems[0].url !== datasetItems[1].url, 'Dataset items unique');