Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add update-images script

Ondra Urbane607d715aaee
diff --git a/package.json b/package.json
index e6199c878..fe3f8f117 100644
--- a/package.json
+++ b/package.json
@@ -46,7 +46,8 @@
"prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",
"clean": "rm -rf build",
"lint": "npm run build && ./node_modules/.bin/eslint ./src ./test",
- "lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"
+ "lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix",
+ "update-images": "node ./tools/update_images.js"
},
"dependencies": {
"@apify/ps-tree": "^1.1.3",
@@ -74,6 +75,7 @@
"@babel/core": "^7.0.0",
"@babel/preset-env": "^7.0.0",
"@babel/register": "^7.0.0",
+ "@types/fs-extra": "^7.0.0",
"@types/puppeteer": "^1.12.4",
"@types/socket.io": "^2.1.2",
"basic-auth-parser": "^0.0.2",
@@ -95,6 +97,7 @@
"nyc": "^14.0.0",
"portastic": "^1.0.1",
"proxy": "^0.2.4",
+ "semver": "^6.1.1",
"sinon": "^7.1.1",
"sinon-stub-promise": "^4.0.0",
"socket.io-client": "^2.2.0",
diff --git a/tools/release.js b/tools/release.js
new file mode 100644
index 000000000..1c5a39dec
--- /dev/null
+++ b/tools/release.js
@@ -0,0 +1,34 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { execSync } = require('child_process');
+const semver = require('semver'); // eslint-disable-line import/no-extraneous-dependencies
+
+const VERSION = require('../package.json').version;
+
+log('Checking if current branch is master branch.');
+const currentBranch = execSync('git branch | grep "*"', { encoding: 'utf8' }).substr(2);
+if (currentBranch !== 'master') {
+ log('Release can only be triggered from the master branch.');
+ process.exit(1);
+}
+
+log('Checking if branch is up to date.');
+const upToDate = execSync('git branch | grep "*"', { encoding: 'utf8' }).substr(2);
+
+log('Fetching version of published beta.');
+const betaVersion = fetchPackageJsonPropertyForTag('version', 'beta');
+
+if (semver.gte(betaVersion, VERSION)) {
+ log(`Cannot release version: ${VERSION} with version ${betaVersion} already released.`);
+ process.exit(0);
+}
+
+
+function log(...messages) {
+ /* eslint-disable no-console */
+ console.log('release:', ...messages);
+}
+
+function fetchPackageJsonPropertyForTag(property, tag) {
+ return execSync(`npm show apify@${tag} ${property}`, { encoding: 'utf8' }).trim();
+}
diff --git a/tools/update_images.js b/tools/update_images.js
new file mode 100644
index 000000000..62677c825
--- /dev/null
+++ b/tools/update_images.js
@@ -0,0 +1,113 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { execSync } = require('child_process');
+const semver = require('semver'); // eslint-disable-line import/no-extraneous-dependencies
+
+const TEMP_DIR = path.join(__dirname, '..', 'tmp');
+const IMAGE_REPO_NAME = 'apify-actor-docker';
+const IMAGE_REPO_URL = `https://github.com/apifytech/${IMAGE_REPO_NAME}.git`;
+const IMAGE_REPO_DIR = path.join(TEMP_DIR, IMAGE_REPO_NAME);
+const IMAGES_TO_UPDATE = ['node-basic', 'node-chrome'];
+const PKG_PATHS = IMAGES_TO_UPDATE.map(getPackageJsonPath);
+
+const dryRun = process.argv.includes('--dry-run');
+
+log('Preparing file system.');
+fs.ensureDirSync(TEMP_DIR);
+fs.removeSync(IMAGE_REPO_DIR);
+
+log('Cloning image repository.');
+execSync(`git clone -n ${IMAGE_REPO_URL} --depth 1`, { cwd: TEMP_DIR });
+execGitCommand('reset HEAD');
+
+log('Checking out package.jsons.');
+execGitCommand('checkout HEAD', PKG_PATHS);
+
+log('Loading published Apify versions.');
+const betaVersion = fetchPackageJsonPropertyForTag('version', 'beta');
+
+log('Loading version of puppeteer in apify@beta.');
+const puppeteerVersionInBeta = fetchPackageJsonPropertyForTag('dependencies.puppeteer', 'beta');
+
+let updatedImageCount = 0;
+IMAGES_TO_UPDATE.forEach((imageName) => {
+ log(`Processing image: ${imageName}`);
+ const imagePkgPath = path.join(IMAGE_REPO_DIR, getPackageJsonPath(imageName));
+ const imagePkg = require(imagePkgPath); // eslint-disable-line
+
+ const apifyVersion = imagePkg.dependencies.apify;
+ if (semver.gte(apifyVersion, betaVersion)) {
+ return logSkipMessage(imageName, 'apify', apifyVersion, betaVersion);
+ }
+
+ const newImagePkg = JSON.parse(JSON.stringify(imagePkg));
+ logUpdatingMessage(imageName, 'apify', apifyVersion, betaVersion);
+ newImagePkg.dependencies.apify = betaVersion;
+
+ const puppeteerVersion = imagePkg.dependencies.puppeteer;
+ if (puppeteerVersion && semver.lt(puppeteerVersion, puppeteerVersionInBeta)) {
+ newImagePkg.dependencies.puppeteer = puppeteerVersionInBeta;
+ logUpdatingMessage(imageName, 'puppeteer', puppeteerVersion, puppeteerVersionInBeta);
+ } else if (puppeteerVersion) {
+ logSkipMessage(imageName, 'puppeteer', puppeteerVersion, puppeteerVersionInBeta);
+ }
+
+ log(`${imageName}: Writing new package.json.`);
+ fs.writeFileSync(imagePkgPath, `${JSON.stringify(newImagePkg, null, 4)}\n`);
+ updatedImageCount++;
+});
+
+if (!updatedImageCount) {
+ log('Exiting because no images were updated.');
+ teardown();
+ process.exit(0);
+}
+
+log('Committing changes to package.jsons.');
+execGitCommand('add', PKG_PATHS);
+execGitCommand('commit -m "Update package versions"');
+
+if (dryRun) {
+ log('Exiting process before changes are pushed to remote repository. Temp dir will be kept.');
+ log('Run "git diff origin/master HEAD" to see changes.');
+ process.exit(0);
+}
+
+log('Pushing changes to remote.');
+execGitCommand('git push');
+
+teardown();
+
+function log(...messages) {
+ /* eslint-disable no-console */
+ console.log('update-images:', ...messages);
+}
+
+function execGitCommand(command, filePaths = []) {
+ const fullCommand = filePaths.length
+ ? `git ${command} ${PKG_PATHS.join(' ')}`
+ : `git ${command}`;
+ return execSync(fullCommand, { cwd: IMAGE_REPO_DIR });
+}
+
+function getPackageJsonPath(image) {
+ return `./${image}/package.json`;
+}
+
+function fetchPackageJsonPropertyForTag(property, tag) {
+ return execSync(`npm show apify@${tag} ${property}`, { encoding: 'utf8' }).trim();
+}
+
+function logSkipMessage(imageName, pkg, repoVersion, newVersion) {
+ log(`${imageName}: Cannot update repository version of ${pkg}: ${repoVersion} with version: ${newVersion}`);
+}
+
+function logUpdatingMessage(imageName, pkg, repoVersion, newVersion) {
+ log(`${imageName}: Updating package "${pkg}" from version ${repoVersion} to ${newVersion}`);
+}
+
+function teardown() {
+ log('Cleaning up file system.');
+ fs.removeSync(TEMP_DIR);
+ log('Done.');
+}