Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Update travis configuration
Ondra Urban30e6b6d28ddb
diff --git a/.travis.yml b/.travis.ymlindex e83f122a7..683e6c691 100644--- a/.travis.yml+++ b/.travis.yml@@ -59,4 +59,6 @@ jobs:node_js: '10'services:- docker- script: node .travis/release-images.js+ script:+ - set -ex+ - node .travis/release-images.jsdiff --git a/.travis/release-images-beta.js b/.travis/release-images-beta.jsindex d9a68a440..e3b1d523d 100644--- a/.travis/release-images-beta.js+++ b/.travis/release-images-beta.js@@ -3,35 +3,44 @@ const path = require('path');const { execSync } = require('child_process');const semver = require('semver'); // eslint-disable-line import/no-extraneous-dependencies+const TAG = process.env.TRAVIS_TAG;+const BRANCH = process.env.TRAVIS_BRANCH;+const TEMP_DIR = path.join(__dirname, '..', 'tmp');const IMAGE_REPO_NAME = 'apify-actor-docker';const IMAGE_REPO_URL = `https://github.com/apifytech/${IMAGE_REPO_NAME}.git`;const IMAGE_REPO_DIR = path.join(TEMP_DIR, IMAGE_REPO_NAME);-const IMAGES_TO_UPDATE = ['node-basic', 'node-chrome'];+const IMAGES_TO_UPDATE = ['node-chrome', 'node-basic', 'node-chrome-xvfb'];+const IMAGES_TO_BUILD = [...IMAGES_TO_UPDATE, 'node-phantomjs'];const PKG_PATHS = IMAGES_TO_UPDATE.map(getPackageJsonPath);-const dryRun = process.argv.includes('--dry-run');+let RELEASE_TAG;+if (TAG && /^v\d+\.\d+\.\d+$/.test(TAG)) RELEASE_TAG = 'latest';+else if (BRANCH && /^master$/.test(BRANCH)) RELEASE_TAG = 'beta';+else {+ log('Build is not a release build. Skipping.');+ process.exit(0);+}++log(`Triggering release of ${RELEASE_TAG} images.`);log('Preparing file system.');fs.ensureDirSync(TEMP_DIR);fs.removeSync(IMAGE_REPO_DIR);log('Cloning image repository.');-execSync(`git clone -n ${IMAGE_REPO_URL} --depth 1`, { cwd: TEMP_DIR });+execSync(`git clone ${IMAGE_REPO_URL}`, { cwd: TEMP_DIR });execGitCommand('reset HEAD');log('Setting up git.');execGitCommand('config --global user.email "[email protected]"');execGitCommand('config --global user.name "Travis CI"');-log('Checking out package.jsons.');-execGitCommand('checkout master', PKG_PATHS);-log('Loading published Apify versions.');-const betaVersion = fetchPackageJsonPropertyForTag('version', 'beta');+const loadedApifyVersion = fetchPackageJsonPropertyForTag('version', RELEASE_TAG);-log('Loading version of puppeteer in apify@beta.');-const puppeteerVersionInBeta = fetchPackageJsonPropertyForTag('dependencies.puppeteer', 'beta');+log(`Loading version of puppeteer in apify@${RELEASE_TAG}.`);+const loadedPuppeteerVersion = fetchPackageJsonPropertyForTag('dependencies.puppeteer', RELEASE_TAG);let updatedImageCount = 0;IMAGES_TO_UPDATE.forEach((imageName) => {@@ -40,20 +49,20 @@ IMAGES_TO_UPDATE.forEach((imageName) => {const imagePkg = require(imagePkgPath); // eslint-disable-lineconst apifyVersion = imagePkg.dependencies.apify;- if (semver.gte(apifyVersion, betaVersion)) {- return logSkipMessage(imageName, 'apify', apifyVersion, betaVersion);+ if (semver.gte(apifyVersion, loadedApifyVersion)) {+ return logSkipMessage(imageName, 'apify', apifyVersion, loadedApifyVersion);}const newImagePkg = JSON.parse(JSON.stringify(imagePkg));- logUpdatingMessage(imageName, 'apify', apifyVersion, betaVersion);- newImagePkg.dependencies.apify = betaVersion;+ logUpdatingMessage(imageName, 'apify', apifyVersion, loadedApifyVersion);+ newImagePkg.dependencies.apify = loadedApifyVersion;const puppeteerVersion = imagePkg.dependencies.puppeteer;- if (puppeteerVersion && semver.lt(puppeteerVersion, puppeteerVersionInBeta)) {- newImagePkg.dependencies.puppeteer = puppeteerVersionInBeta;- logUpdatingMessage(imageName, 'puppeteer', puppeteerVersion, puppeteerVersionInBeta);+ if (puppeteerVersion && semver.lt(puppeteerVersion, loadedPuppeteerVersion)) {+ newImagePkg.dependencies.puppeteer = loadedPuppeteerVersion;+ logUpdatingMessage(imageName, 'puppeteer', puppeteerVersion, loadedPuppeteerVersion);} else if (puppeteerVersion) {- logSkipMessage(imageName, 'puppeteer', puppeteerVersion, puppeteerVersionInBeta);+ logSkipMessage(imageName, 'puppeteer', puppeteerVersion, loadedPuppeteerVersion);}log(`${imageName}: Writing new package.json.`);@@ -69,19 +78,31 @@ if (!updatedImageCount) {log('Committing changes to package.jsons.');execGitCommand('add', PKG_PATHS);-execGitCommand('commit -m "Update package versions"');--if (dryRun) {- log('DRY RUN: Exiting process before changes are pushed to remote repository. Temp dir will be kept.');- log('Run "git diff origin/master HEAD" to see changes.');- process.exit(0);+let commitMessage = 'Update package versions';+if (isLatest()) {+ log('Skipping CI build of beta images because we\'re running a latest deploy.');+ commitMessage += ' [skip ci]';}+execGitCommand(`commit -m "${commitMessage}"`);log('Adding new origin with token.');execGitCommand(`remote add origin-token https://${process.env.GH_TOKEN}@github.com/apifytech/apify-actor-docker > /dev/null 2>&1`);log('Pushing changes to remote.');-execGitCommand(`push --set-upstream origin-token master`);+execGitCommand('push --set-upstream origin-token master');++if (!isLatest()) teardown();++log('Building latest images.');+IMAGES_TO_BUILD.forEach(imageName => {+ log(`Building image: ${imageName}`);+ const dockerImage = `apify/actor-${imageName}:latest`;+ execSync(`docker build --pull --tag ${dockerImage} --no-cache ./${imageName}/`);+ log(`${imageName}: built. Running a test.`);+ execSync(`docker run ${dockerImage}`);+ log(`${imageName}: test successful. Pushing image to repository.`);+ execSync(`docker push ${dockerImage}`);+});teardown();@@ -118,3 +139,7 @@ function teardown() {fs.removeSync(TEMP_DIR);log('Done.');}++function isLatest() {+ return RELEASE_TAG === 'latest';+}diff --git a/.travis/release-images-latest.js b/.travis/release-images-latest.jsindex 0194029f0..18064015d 100644--- a/.travis/release-images-latest.js+++ b/.travis/release-images-latest.js@@ -1,10 +1,8 @@const { execSync } = require('child_process');-const IMAGES_TO_UPDATE = ['node-basic', 'node-chrome', 'node-chrome-xvfb'];+const IMAGES_TO_UPDATE = ['node-chrome', 'node-basic', 'node-chrome-xvfb'];const { DOCKER_USERNAME, DOCKER_PASSWORD } = process.env;-const dryRun = process.argv.includes('--dry-run');-log('Checking that docker is running.');execSync('docker ps');@@ -17,21 +15,19 @@ try {}-if (dryRun) log('DRY RUN: The following operations produce only log messages.');-IMAGES_TO_UPDATE.forEach((imageName) => {log(`Processing image: ${imageName}.`);log(`${imageName}: Removing local copies of image.`);- if (!dryRun) execSync(`docker image rm --force apify/actor-${imageName}:beta apify/actor-${imageName}:latest`);+ execSync(`docker image rm --force apify/actor-${imageName}:beta apify/actor-${imageName}:latest`);log(`${imageName}: Pulling remote beta image.`);- if (!dryRun) execSync(`docker pull apify/actor-${imageName}:beta`);+ execSync(`docker pull apify/actor-${imageName}:beta`);log(`${imageName}: Tagging beta image with the latest tag.`);- if (!dryRun) execSync(`docker image tag apify/actor-${imageName}:beta apify/actor-${imageName}:latest`);+ execSync(`docker image tag apify/actor-${imageName}:beta apify/actor-${imageName}:latest`);log(`${imageName}: Pushing local version to remote.`);- if (!dryRun) execSync(`docker push apify/actor-${imageName}:latest`);+ execSync(`docker push apify/actor-${imageName}:latest`);log(`Docker image was published as apify/actor-${imageName}:latest`);});diff --git a/CHANGELOG.md b/CHANGELOG.mdindex d39f8957d..6a5775ad3 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,6 @@+NEXT+====================+0.15.2 / 2019-07-11====================- Fix error where Puppeteer would fail to launch when pipes are turned off.diff --git a/package.json b/package.jsonindex 19d4de386..f499e863c 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.15.2",+ "version": "0.15.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=8.14.0"@@ -48,10 +48,6 @@"clean": "rm -rf build","lint": "npm run build && ./node_modules/.bin/eslint ./src ./test","lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix",- "before-deploy": "node .travis/before-deploy.js",- "after-deploy": "node after-deploy.js",- "prerelease-check": "npm run lint && node check-docs.js",- "release-latest": "node release-latest.js"},"dependencies": {"@apify/ps-tree": "^1.1.3",