Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add trivial tests
Ondra Urban13f6c7be9d64
diff --git a/package.json b/package.jsonindex f3c0da3b1..ec122fc59 100644--- a/package.json+++ b/package.json@@ -1,99 +1,101 @@{- "name": "apify",- "version": "0.13.7",- "description": "The scalable web crawling and scraping library for JavaScript. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",- "engines": {- "node": ">=8.14.0"- },- "main": "build/index.js",- "keywords": [- "apify",- "headless",- "chrome",- "puppeteer",- "selenium",- "crawler",- "scraper"- ],- "author": {- "name": "Apify",- "email": "[email protected]",- "url": "https://apify.com"- },- "contributors": [- "Jan Curn <[email protected]>",- "Marek Trunkat <[email protected]>",- "Ondra Urban <[email protected]>"- ],- "license": "Apache-2.0",- "repository": {- "type": "git",- "url": "git+https://github.com/apifytech/apify-js"- },- "bugs": {- "url": "https://github.com/apifytech/apify-js/issues"- },- "homepage": "https://sdk.apify.com/",- "files": [- "build"- ],- "scripts": {- "build": "rm -rf ./build && babel src --out-dir build",- "build-docs": "npm run clean && npm run build && node ./website/tools/build_docs.js",- "build-readme": "node ./tools/build_readme.js",- "test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",- "prepare": "npm run build",- "prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",- "clean": "rm -rf build",- "lint": "npm run build && ./node_modules/.bin/eslint ./src ./test",- "lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"- },- "dependencies": {- "@apify/ps-tree": "^1.1.3",- "apify-client": "^0.5.12",- "apify-shared": "^0.1.23",- "bluebird": "^3.5.3",- "cheerio": "^1.0.0-rc.2",- "content-type": "^1.0.4",- "fs-extra": "^7.0.1",- "jquery": "^3.3.1",- "mime": "^2.3.1",- "proxy-chain": "^0.2.5",- "request": "^2.88.0",- "request-promise-native": "^1.0.5",- "rimraf": "^2.6.2",- "underscore": "^1.9.1",- "ws": "^6.1.2",- "xregexp": "^4.2.4"- },- "devDependencies": {- "@babel/cli": "^7.0.0",- "@babel/core": "^7.0.0",- "@babel/preset-env": "^7.0.0",- "@babel/register": "^7.0.0",- "basic-auth-parser": "^0.0.2",- "chai": "^4.2.0",- "chai-as-promised": "^7.1.1",- "chai-subset": "^1.6.0",- "eslint": "^5.9.0",- "eslint-config-airbnb": "^17.1.0",- "eslint-config-airbnb-base": "^13.1.0",- "eslint-plugin-import": "^2.14.0",- "eslint-plugin-jsx-a11y": "^6.1.2",- "eslint-plugin-promise": "^4.0.1",- "eslint-plugin-react": "^7.11.1",- "jsdoc": "^3.5.5",- "jsdoc-export-default-interop": "^0.3.1",- "mocha": "^5.2.0",- "nyc": "^13.3.0",- "portastic": "^1.0.1",- "proxy": "^0.2.4",- "sinon": "^7.1.1",- "sinon-stub-promise": "^4.0.0",- "why-is-node-running": "^2.0.3"- },- "optionalDependencies": {- "puppeteer": "1.13.0",- "selenium-webdriver": "^3.6.0"- }+ "name": "apify",+ "version": "0.13.7",+ "description": "The scalable web crawling and scraping library for JavaScript. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",+ "engines": {+ "node": ">=8.14.0"+ },+ "main": "build/index.js",+ "keywords": [+ "apify",+ "headless",+ "chrome",+ "puppeteer",+ "selenium",+ "crawler",+ "scraper"+ ],+ "author": {+ "name": "Apify",+ "email": "[email protected]",+ "url": "https://apify.com"+ },+ "contributors": [+ "Jan Curn <[email protected]>",+ "Marek Trunkat <[email protected]>",+ "Ondra Urban <[email protected]>"+ ],+ "license": "Apache-2.0",+ "repository": {+ "type": "git",+ "url": "git+https://github.com/apifytech/apify-js"+ },+ "bugs": {+ "url": "https://github.com/apifytech/apify-js/issues"+ },+ "homepage": "https://sdk.apify.com/",+ "files": [+ "build"+ ],+ "scripts": {+ "build": "rm -rf ./build && babel src --out-dir build",+ "build-docs": "npm run clean && npm run build && node ./website/tools/build_docs.js",+ "build-readme": "node ./tools/build_readme.js",+ "test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",+ "prepare": "npm run build",+ "prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",+ "clean": "rm -rf build",+ "lint": "npm run build && ./node_modules/.bin/eslint ./src ./test",+ "lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"+ },+ "dependencies": {+ "@apify/ps-tree": "^1.1.3",+ "apify-client": "^0.5.12",+ "apify-shared": "^0.1.23",+ "bluebird": "^3.5.3",+ "cheerio": "^1.0.0-rc.2",+ "content-type": "^1.0.4",+ "express": "^4.16.4",+ "fs-extra": "^7.0.1",+ "jquery": "^3.3.1",+ "mime": "^2.3.1",+ "proxy-chain": "^0.2.5",+ "socket.io": "^2.2.0",+ "request": "^2.88.0",+ "request-promise-native": "^1.0.5",+ "rimraf": "^2.6.2",+ "underscore": "^1.9.1",+ "ws": "^6.1.2",+ "xregexp": "^4.2.4"+ },+ "devDependencies": {+ "@babel/cli": "^7.0.0",+ "@babel/core": "^7.0.0",+ "@babel/preset-env": "^7.0.0",+ "@babel/register": "^7.0.0",+ "basic-auth-parser": "^0.0.2",+ "chai": "^4.2.0",+ "chai-as-promised": "^7.1.1",+ "chai-subset": "^1.6.0",+ "eslint": "^5.9.0",+ "eslint-config-airbnb": "^17.1.0",+ "eslint-config-airbnb-base": "^13.1.0",+ "eslint-plugin-import": "^2.14.0",+ "eslint-plugin-jsx-a11y": "^6.1.2",+ "eslint-plugin-promise": "^4.0.1",+ "eslint-plugin-react": "^7.11.1",+ "jsdoc": "^3.5.5",+ "jsdoc-export-default-interop": "^0.3.1",+ "mocha": "^5.2.0",+ "nyc": "^13.3.0",+ "portastic": "^1.0.1",+ "proxy": "^0.2.4",+ "sinon": "^7.1.1",+ "sinon-stub-promise": "^4.0.0",+ "why-is-node-running": "^2.0.3"+ },+ "optionalDependencies": {+ "puppeteer": "1.13.0",+ "selenium-webdriver": "^3.6.0"+ }}diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.jsindex 1b243d717..01b6820dd 100644--- a/src/live_view/live_view_server.js+++ b/src/live_view/live_view_server.js@@ -1,5 +1,5 @@import http from 'http';-import fs from 'fs';+import fs from 'fs-extra';import path from 'path';import { promisify } from 'util';import express from 'express';@@ -14,6 +14,7 @@ const MAX_SCREENSHOT_FILES = 10;const writeFile = promisify(fs.writeFile);const unlink = promisify(fs.unlink);+const ensureDir = promisify(fs.ensureDir);export default class LiveViewServer {constructor(options = {}) {@@ -37,7 +38,7 @@ export default class LiveViewServer {this.screenshotIndexToFilePath = {};// Setup HTTP server and Express router- this.isRunning = false;+ this._isRunning = false;this.httpServer = http.createServer();this.app = express();@@ -84,9 +85,21 @@ export default class LiveViewServer {* Starts the HTTP server.*/async start() {+ await ensureDir(this.screenshotDirectoryPath);await promisifyServerListen(this.httpServer)(this.port);log.info('Live view web server started', { publicUrl: this.liveViewUrl });- this.isRunning = true;+ this._isRunning = true;+ }++ async stop() {+ return new Promise((resolve) => {+ this.httpServer.close((err) => {+ this._isRunning = false;+ if (err) log.exception(err, 'Live view web server could not be stopped.');+ else log.info('Live view web server stopped.');+ resolve();+ });+ });}async serve(page) {@@ -95,6 +108,10 @@ export default class LiveViewServer {await this._pushSnapshot(snapshot);}+ get isRunning() {+ return this._isRunning;+ }+hasClients() {return this.clientCount > 0;}diff --git a/test/live_view/live_view_server.js b/test/live_view/live_view_server.jsindex 56279f8f5..b0fe4bc01 100644--- a/test/live_view/live_view_server.js+++ b/test/live_view/live_view_server.js@@ -1,10 +1,23 @@import path from 'path';import { expect } from 'chai';+import Apify from '../../build/index';import { LOCAL_STORAGE_DIR } from '../_helper';import LiveViewServer from '../../build/live_view/live_view_server';+const { utils: { log } } = Apify;+const LOCAL_STORAGE_SUBDIR = path.join(LOCAL_STORAGE_DIR, 'live_view');+let originalLogLevel;+before(() => {+ originalLogLevel = log.getLevel();+ log.setLevel(log.LEVELS.ERROR);+});++after(() => {+ log.setLevel(originalLogLevel);+});+describe('LiveViewServer', () => {it('should construct', () => {const lvs = new LiveViewServer({@@ -12,4 +25,13 @@ describe('LiveViewServer', () => {});expect(lvs).to.be.instanceOf(LiveViewServer);});+ it('should start and stop', async () => {+ const lvs = new LiveViewServer({+ screenshotDirectoryPath: LOCAL_STORAGE_SUBDIR,+ });+ await lvs.start();+ expect(lvs.isRunning).to.be.eql(true);+ await lvs.stop();+ expect(lvs.isRunning).to.be.eql(false);+ });});