Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add a lot of travis configuration

Ondra Urban07a33e1f285d
diff --git a/.travis.yml b/.travis.yml
index 93e78d416..8852c0deb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,14 +1,51 @@
dist: xenial
sudo: false
language: node_js
-
-cache: npm
-
node_js:
- - "12"
- - "10"
- - "8"
-
-script:
- - npm run lint
- - npm test
+- '12'
+- '10'
+- '8'
+services:
+- docker
+cache: npm
+env:
+ global:
+ - DOCKER_USERNAME=mnmkng
+ - secure: Sp706wRWmS2aM86poOhSTngkEBlpbUydvstEXKEQ8Zz/R4HlmWBsb5vX9lk9FkAC504o21qpCJAIyidHs5FXsw9kpeMf2h+zI/Zs2w1RNaitMDeotSofFYo2J76u7QV6JFUhbPzDYjUXKmgJkPAuffCPeLfj0qszjpjRMz+NdbwE4LVWJpaeyM5lOGmdec51//16juKQBbUsdJ9xPD0bXrQ6mmJs7IJEsVMcQpfOkIgqP70IMXNiDTAHBU/15Oi5yipkMW7Wmn4J6GM8wQmADFUXS8t6oLtbVWEr65YtyzfIN/PjWXT3om7KpfXSouf+lOJdRZAn2RhbXe88PwGIIpgMJKgiIil3Sxgch0UtADTMfWvInylrdBldAw7JSM5QMBFP/vOCRNJEVHurfVOvCmQnlJ2NLLe7KzjF37dDI5LHFyY9u0ysXdoMsHMzLptGy+xPgf3kKkiCbT+GfhrYuAAmdb5MJBcxNkbjsQEszSuy0zRgE+Yu9YAgWA9veQtO3JDo6EmUyOyd2vHWvhCE6aF/o2TD/Nf1CTVblG3M1QwLmzvwADBpH6utDCu8L3NOESShWHUA5pAUsdCfMpTKiJR1rnIEOeCRJFGIVv9CmWbKtdL5asN6nJbhkoq3/Lhf7MC5pxFlKH3LUR1a/nwvO824SnF4ZJEjp9f1aeJU+lU=
+ - secure: pgwE9Nxa27nwuC0zI+IrbpY/P/QNPL9oNxdwk+LMxa90Tn/5EUKl5D/LblGzjRYd39i/Z1ucNf1OJoKThX7YQxs/de4fjX00HGC0DjRzBRcLsU02HhbrNSA/6ASpWyR2WTxFGnJG/15CzHgqP1BM6EvLJixFoyhk6tnF0kqpJos/h8yfoSZV1kfdlcaz1TQU/NdoFD2Agxm5BEf3/b4SKtXGF0pGGj/IdPCMc/NwsJpHFHi2lU90Nr0WVnHD3wyZRHyasQlGG9Yu/f8Tjf27LCY5QDgcJ/JhsFbwabUwXe98JtTLssriOCajXECilwdBYFTpctIhIgfs6bmdO5Xy3rUjrvdMbOjhjNhkD0UHAQjQg5SUEAo5RKssEUgTUtL4S7P6falc1QwhehZZO/65Ui13Jh1zhlBZpYAO1RXLB7m8DHj4gRrgQswn4qBjWpjbiwfC9ocIuH0b/3QUXRGA4cPjB3G4tl+OdFMFykSYvuZGdWC4U6x+z/C6OfEEYwqJaapHKNnafp9429LMNPzd8SSS3KhDaY7N2kLnH5YaoFAVsZ0rRIk0GZpP+xYWBfFlELqCarvcuZi9su3QWverzecjOpfNu+LCHpOYdiPQtGvm8Mzr1eZun/VaJ5DZ3ZIX1y6ODSCg9b9ikGtcz6A1n2GkE4cHN/B1YI52NIbRXN0=
+stages:
+- lint
+- test
+- deploy
+- images
+install:
+- npm install --no-package-lock
+jobs:
+ include:
+ - stage: lint
+ node_js: '10'
+ script: npm run lint && node .travis/check-docs.js
+ - stage: deploy
+ node_js: '10'
+ before_deploy: node .travis/before-deploy.js
+ after_deploy: node .travis/after-deploy.js
+ deploy:
+ - provider: npm
+ api_key:
+ secure: p3U5TFS6bhx0DGS15uMak6M09K+k4YCJ4My2jDDuvXrHMTk7KBIiC9yuCB08SGOWYi/bCTaLgDEFXntKB0rTvR+U6JgVWf/8PAYlWJm4ViqQfk8e8KVfGCZrlT3NAXb2rV2DDidf5F11dbx8iNiw/fw1pmyUKHML+l0rip1N1+SM7BThemfFW7Cp2+0VO+wmRYUVTY34TAe7hkWlA7KQEdUPBCdYmitks90nuciJXJObcDhHXrC0vH7ufTtERjCBIU+ilfjNQAFJuOmD0dD1zxF9iIQlaozNf9DCOklGJWkOEQzn+YWyYinZJdX3JvRpwWSaVQy709dOPNKw7FtVF4NuWkUDz9ajGEZRB0+LPlbN/sqeMkodFoQoV5ogSSZVqX3TzDfnbVIUX2A79yoraSjd4kKftAxKclpIaxu4356W5nnuk9Q6n0uBiFZNCDKK5oJKY3IWAyhN9rZXlu8f/W8JccAQXmwPS8W4AHVnRiFsSgFjKytwKSC8NfTcDnSJJXYMAXvLj8PCWfWq38+9iCYi6ByC3PdvS/Hr7VEf4nxnIRStCX8e7h/9IeYiBeqGx1prLw2MLBwRdBCqVNhobEa6zyx4xXoInGS8qhqseqGAdGjOQuMGWDSLVQmZdi2kjoHnf0/Yz1C4caKM6GaMKl9TvbauHAi78c+Av7XnNCI=
+ on:
+ branch: master
+ tag: beta
+ skip_cleanup: true
+ - provider: npm
+ api_key:
+ secure: p3U5TFS6bhx0DGS15uMak6M09K+k4YCJ4My2jDDuvXrHMTk7KBIiC9yuCB08SGOWYi/bCTaLgDEFXntKB0rTvR+U6JgVWf/8PAYlWJm4ViqQfk8e8KVfGCZrlT3NAXb2rV2DDidf5F11dbx8iNiw/fw1pmyUKHML+l0rip1N1+SM7BThemfFW7Cp2+0VO+wmRYUVTY34TAe7hkWlA7KQEdUPBCdYmitks90nuciJXJObcDhHXrC0vH7ufTtERjCBIU+ilfjNQAFJuOmD0dD1zxF9iIQlaozNf9DCOklGJWkOEQzn+YWyYinZJdX3JvRpwWSaVQy709dOPNKw7FtVF4NuWkUDz9ajGEZRB0+LPlbN/sqeMkodFoQoV5ogSSZVqX3TzDfnbVIUX2A79yoraSjd4kKftAxKclpIaxu4356W5nnuk9Q6n0uBiFZNCDKK5oJKY3IWAyhN9rZXlu8f/W8JccAQXmwPS8W4AHVnRiFsSgFjKytwKSC8NfTcDnSJJXYMAXvLj8PCWfWq38+9iCYi6ByC3PdvS/Hr7VEf4nxnIRStCX8e7h/9IeYiBeqGx1prLw2MLBwRdBCqVNhobEa6zyx4xXoInGS8qhqseqGAdGjOQuMGWDSLVQmZdi2kjoHnf0/Yz1C4caKM6GaMKl9TvbauHAi78c+Av7XnNCI=
+ on:
+ tags: true
+ tag: latest
+ skip_cleanup: true
+ - stage: images
+ node_js: '10'
+ script: node .travis/release-images.js
diff --git a/.travis/after-deploy.js b/.travis/after-deploy.js
new file mode 100644
index 000000000..014c3a059
--- /dev/null
+++ b/.travis/after-deploy.js
@@ -0,0 +1,24 @@
+const { execSync } = require('child_process');
+const GIT_TAG = require('../package.json').version;
+
+const { GH_TOKEN } = process.env;
+
+if (process.env.TRAVIS_TAG) {
+ // Only latest releases will have a tag.
+ console.log('after-deploy: Skipping git tag because it\'s a tagged build.');
+ process.exit(0);
+}
+
+console.log('after-deploy: Setting up git.');
+execSync('git config --global user.email "[email protected]"');
+execSync('git config --global user.name "Travis CI"');
+
+console.log('after-deploy: Adding new origin with token.');
+execSync(`git remote add origin-token https://${GH_TOKEN}@github.com/apifytech/apify-js > /dev/null 2>&1`);
+
+console.log(`after-deploy: Tagging commit with tag: ${GIT_TAG}.`);
+execSync(`git tag ${GIT_TAG}`);
+
+console.log('after-deploy: Pushing changes to remote.');
+execSync(`git reset HEAD`);
+execSync(`git push --set-upstream origin-token master ${GIT_TAG}`);
diff --git a/.travis/before-deploy.js b/.travis/before-deploy.js
new file mode 100644
index 000000000..a86c38eed
--- /dev/null
+++ b/.travis/before-deploy.js
@@ -0,0 +1,35 @@
+const path = require('path');
+const fs = require('fs');
+const { execSync } = require('child_process');
+const pkgJson = require('../package.json');
+
+if (process.env.TRAVIS_TAG) {
+ // Only latest releases will have a tag.
+ console.log('before-deploy: Skipping version update, because it\'s a latest release.');
+ process.exit(0);
+}
+
+const PACKAGE_NAME = pkgJson.name;
+const VERSION = pkgJson.version;
+
+const nextVersion = getNextVersion(VERSION);
+console.log(`before-deploy: Setting version to ${nextVersion}`);
+pkgJson.version = nextVersion;
+
+fs.writeFileSync(path.join(__dirname, 'package.json'), JSON.stringify(pkgJson, null, 2) + '\n');
+
+function getNextVersion(version) {
+ const versionString = execSync(`npm show ${PACKAGE_NAME} versions --json`, { encoding: 'utf8'});
+ const versions = JSON.parse(versionString);
+
+ if (versions.some(v => v === VERSION)) {
+ console.error(`before-deploy: A release with version ${VERSION} already exists. Please increment version accordingly.`);
+ process.exit(1);
+ }
+
+ const prereleaseNumbers = versions
+ .filter(v => (v.startsWith(VERSION) && v.includes('-')))
+ .map(v => Number(v.match(/\.(\d+)$/)[1]));
+ const lastPrereleaseNumber = Math.max(-1, ...prereleaseNumbers);
+ return `${version}-beta.${lastPrereleaseNumber + 1}`
+}
diff --git a/.travis/check-docs.js b/.travis/check-docs.js
new file mode 100644
index 000000000..92f05735b
--- /dev/null
+++ b/.travis/check-docs.js
@@ -0,0 +1,13 @@
+const { execSync } = require('child_process');
+
+console.log('Checking if documentation is updated.');
+execSync('npm run build-docs');
+execSync('npm run build-readme');
+
+console.log('Checking if all changes are committed.');
+const gitStatus = execSync('git status --porcelain --branch', { encoding: 'utf8' });
+const statusLines = gitStatus.split('\n');
+if (statusLines.length > 1 && statusLines[1].length) {
+ console.log('You have uncommitted changes in the documentation. Please build the docs, review and commit changes.');
+ process.exit(1);
+}
diff --git a/tools/update_images.js b/.travis/release-images-beta.js
similarity index 91%
rename from tools/update_images.js
rename to .travis/release-images-beta.js
index 5633671a0..ada6990b4 100644
--- a/tools/update_images.js
+++ b/.travis/release-images-beta.js
@@ -20,6 +20,10 @@ log('Cloning image repository.');
execSync(`git clone -n ${IMAGE_REPO_URL} --depth 1`, { cwd: TEMP_DIR });
execGitCommand('reset HEAD');
+log('Setting up git.');
+execGitCommand('config --global user.email "[email protected]"');
+execGitCommand('config --global user.name "Travis CI"');
+
log('Checking out package.jsons.');
execGitCommand('checkout HEAD', PKG_PATHS);
@@ -73,8 +77,11 @@ if (dryRun) {
process.exit(0);
}
+log('Adding new origin with token.');
+execGitCommand(`remote add origin-token https://${GH_TOKEN}@github.com/apifytech/apify-js > /dev/null 2>&1`);
+
log('Pushing changes to remote.');
-execGitCommand('git push');
+execGitCommand(`git push --set-upstream origin-token master`);
teardown();
diff --git a/.travis/release-images-latest.js b/.travis/release-images-latest.js
new file mode 100644
index 000000000..0194029f0
--- /dev/null
+++ b/.travis/release-images-latest.js
@@ -0,0 +1,44 @@
+const { execSync } = require('child_process');
+
+const IMAGES_TO_UPDATE = ['node-basic', 'node-chrome', 'node-chrome-xvfb'];
+const { DOCKER_USERNAME, DOCKER_PASSWORD } = process.env;
+
+const dryRun = process.argv.includes('--dry-run');
+
+log('Checking that docker is running.');
+execSync('docker ps');
+
+log('Logging in to Docker registry.');
+try {
+ execSync(`echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin`);
+} catch (err) {
+ // Swallow the error to prevent printing credentials to console.
+ process.exit(1);
+}
+
+
+if (dryRun) log('DRY RUN: The following operations produce only log messages.');
+
+IMAGES_TO_UPDATE.forEach((imageName) => {
+ log(`Processing image: ${imageName}.`);
+ log(`${imageName}: Removing local copies of image.`);
+ if (!dryRun) execSync(`docker image rm --force apify/actor-${imageName}:beta apify/actor-${imageName}:latest`);
+
+ log(`${imageName}: Pulling remote beta image.`);
+ if (!dryRun) execSync(`docker pull apify/actor-${imageName}:beta`);
+
+ log(`${imageName}: Tagging beta image with the latest tag.`);
+ if (!dryRun) execSync(`docker image tag apify/actor-${imageName}:beta apify/actor-${imageName}:latest`);
+
+ log(`${imageName}: Pushing local version to remote.`);
+ if (!dryRun) execSync(`docker push apify/actor-${imageName}:latest`);
+
+ log(`Docker image was published as apify/actor-${imageName}:latest`);
+});
+
+log('All images were tagged and pushed to remote.');
+
+function log(...messages) {
+ /* eslint-disable no-console */
+ console.log('update-latest-images:', ...messages);
+}
diff --git a/.travis/release-images.js b/.travis/release-images.js
new file mode 100644
index 000000000..30816942b
--- /dev/null
+++ b/.travis/release-images.js
@@ -0,0 +1,17 @@
+const { execSync } = require('child_process');
+
+const TAG = process.env.TRAVIS_TAG;
+const BRANCH = process.env.TRAVIS_BRANCH;
+
+if (TAG && /^v\d+\.\d+\.\d+$/.test(TAG)) {
+ // Latest release
+ console.log('release-images: Triggering release of latest images');
+ execSync('node ./release-images-latest.js --dry-run');
+} else if (BRANCH && /^master$/.test(BRANCH)) {
+ // Beta release
+ console.log('release-images: Triggering release of beta images');
+ execSync('node ./release-images-beta.js --dry-run');
+} else {
+ console.log('release-images: Build is not a release build. Skipping.');
+ process.exit(0);
+}
diff --git a/package.json b/package.json
index fe3f8f117..ac976ee23 100644
--- a/package.json
+++ b/package.json
@@ -47,7 +47,10 @@
"clean": "rm -rf build",
"lint": "npm run build && ./node_modules/.bin/eslint ./src ./test",
"lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix",
- "update-images": "node ./tools/update_images.js"
+ "before-deploy": "node .travis/before-deploy.js",
+ "after-deploy": "node after-deploy.js",
+ "prerelease-check": "npm run lint && node check-docs.js",
+ "release-latest": "node release-latest.js"
},
"dependencies": {
"@apify/ps-tree": "^1.1.3",
diff --git a/tools/release.js b/tools/release.js
deleted file mode 100644
index a101814fa..000000000
--- a/tools/release.js
+++ /dev/null
@@ -1,60 +0,0 @@
-const { execSync } = require('child_process');
-const semver = require('semver'); // eslint-disable-line import/no-extraneous-dependencies
-
-const VERSION = require('../package.json').version;
-
-const dryRun = process.argv.includes('--dry-run');
-
-log('Checking if current branch is master branch.');
-const gitStatus = execSync('git status --porcelain --branch', { encoding: 'utf8' });
-if (!gitStatus.startsWith('## feature/')) { // TODO for testing, dont forget to replace
- log('Release can only be triggered from the master branch. Please checkout master.');
- process.exit(1);
-}
-
-log('Building documentation.');
-execSync('npm run build-docs');
-execSync('npm run build-readme');
-
-log('Checking if all changes are committed.');
-const statusLines = gitStatus.split('\n');
-if (statusLines.length > 1 && statusLines[1].length) {
- log('You have uncommitted changes. Please commit them and run the script again.');
- process.exit(1);
-}
-
-log('Checking if branch is up to date.');
-if (/\[ahead|behind \d+\]/.test(gitStatus)) {
- log('Your local copy of master branch is not up to date with the remote. Please push / pull changes.');
- process.exit(1);
-}
-
-log('Fetching version of published beta.');
-const betaVersion = fetchPackageJsonPropertyForTag('version', 'beta');
-
-if (semver.gte(betaVersion, VERSION)) {
- log(`Cannot release version: ${VERSION} with version ${betaVersion} already released.`);
- process.exit(0);
-}
-
-if (dryRun) log('DRY RUN: The following operations produce only log messages.');
-
-const npmTag = 'beta';
-log(`Publishing version ${VERSION} with the "${npmTag}" tag.`);
-if (!dryRun) execSync(`npm publish --tag ${npmTag}`);
-
-const versionTag = `v${VERSION}`;
-log(`Tagging last commit with "${versionTag}" and "${npmTag}".`);
-if (!dryRun) {
- execSync(`git tag ${versionTag}`);
- execSync(`git tag -f ${npmTag}`);
-}
-
-function log(...messages) {
- /* eslint-disable no-console */
- console.log('release:', ...messages);
-}
-
-function fetchPackageJsonPropertyForTag(property, tag) {
- return execSync(`npm show apify@${tag} ${property}`, { encoding: 'utf8' }).trim();
-}