Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Initial development
Jan Curn67f3e58e1f22
diff --git a/.babelrc b/.babelrcnew file mode 100644index 000000000..ec553ce6a--- /dev/null+++ b/.babelrc@@ -0,0 +1,3 @@+{+ presets: ["es2015", "stage-0"]+}\ No newline at end of filediff --git a/.eslintrc b/.eslintrcnew file mode 100644index 000000000..a84546062--- /dev/null+++ b/.eslintrc@@ -0,0 +1,17 @@+root: true++env:+ es6: true+ node: true+ browser: true+ mocha: true++ecmaFeatures:+ modules: true++extends:+ "airbnb"++rules:+ quotes: 0+ func-names: 0diff --git a/.gitignore b/.gitignorenew file mode 100644index 000000000..58bd17fb0--- /dev/null+++ b/.gitignore@@ -0,0 +1,9 @@+node_modules+*.log+*.pid+*.seed+.DS_Store+lib+coverage+logs+pidsdiff --git a/.npmignore b/.npmignorenew file mode 100644index 000000000..f83b1ec13--- /dev/null+++ b/.npmignore@@ -0,0 +1,7 @@+.DS_Store+*.log+node_modules+../babel-npm-boilerplate/src+test+examples+coveragediff --git a/.travis.yml b/.travis.ymlnew file mode 100644index 000000000..eab377c49--- /dev/null+++ b/.travis.yml@@ -0,0 +1,7 @@+language: node_js+node_js:+ - "5"+ - "4"+script:+ - npm run lint+ - npm testdiff --git a/CHANGELOG.md b/CHANGELOG.mdnew file mode 100644index 000000000..7e09e9d34--- /dev/null+++ b/CHANGELOG.md@@ -0,0 +1,3 @@+0.0.x / 2017-01-01+==================+- Initial development, lot of new stuffdiff --git a/README.md b/README.mdindex 5c05daa96..00b086ee9 100644--- a/README.md+++ b/README.md@@ -2,3 +2,42 @@Apifier SDK for JavaScriptThis package is still a work in progress, stay tuned.++++++# babel-npm-boilerplate [](https://travis-ci.org/camsong/babel-npm-boilerplate) [](http://badge.fury.io/js/babel-npm-boilerplate)++A Boilerplate for creating NPM packages using Babel, ESLint, Mocha and Chai+++## What's inside++Minimal structure for a npm package source.++* Babel 6+* Mocha+* ESLint+* Isparta++## Installation++Clone this repo or download using `npm`++```+npm install babel-npm-boilerplate --save+```++## Usage++* `npm run test` to run tests+* `npm run test-cov` to generate test coverage+* `npm run build` to transform es6/es7 to es5 by Babel+* `npm run clean` to clean `build/` directory+* `npm run lint` to lint js using ESLint in Airbnb's Javascript style++## Liscense++MIT+diff --git a/build/index.js b/build/index.jsnew file mode 100644index 000000000..756c7c865--- /dev/null+++ b/build/index.js@@ -0,0 +1,35 @@+"use strict";++Object.defineProperty(exports, "__esModule", {+ value: true+});++var _createClass = (function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; })();++function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }++var Person = (function () {+ function Person(name, age) {+ _classCallCheck(this, Person);++ this.name = name;+ this.age = age;+ }++ _createClass(Person, [{+ key: "getName",+ value: function getName() {+ return this.name;+ }+ }, {+ key: "getAge",+ value: function getAge() {+ return this.age;+ }+ }]);++ return Person;+})();++exports["default"] = Person;+module.exports = exports["default"];\ No newline at end of filediff --git a/index.js b/index.jsindex 2bb50215e..2fa593129 100644--- a/index.js+++ b/index.js@@ -12,3 +12,33 @@console.log("Apifier SDK for JavaScript");console.log("Copyright(c) 2016 Apifier. All rights reserved.");console.log("");++++//exports.++++++const http = require('http');+const fs = require('fs'); // yyyyyy++console.log("Hello world !!!");+console.dir(process.env);++setTimeout( function() {++ const server = http.createServer((req, res) => {+ res.statusCode = 200;+ res.setHeader('Content-Type', 'text/plain');+ res.end('Hello World\n');+ console.log("Received message!!!");+ // console.dir(req);+ });+ server.listen(process.env.APIFIER_INTERNAL_PORT|0, function() {+ console.log("Listening on port : " + (process.env.APIFIER_INTERNAL_PORT|0));+ fs.writeFileSync("/apifier-truncate-when-ready", "");+ });++}, 10000);diff --git a/package.json b/package.jsonindex a2c3d1f15..063ac9dad 100644--- a/package.json+++ b/package.json@@ -1,27 +1,48 @@{- "name": "apifier",- "version": "0.0.2",- "description": "Apifier SDK for JavaScript",- "main": "index.js",- "repository": {- "type": "git",- "url": "https://github.com/apifier/apifier-sdk-js"- },- "keywords": [- "apifier"- ],- "author": {- "name": "Apifier",- "email": "[email protected]",- "url": "https://www.apifier.com"- },- "contributors": [- "Jan Curn <[email protected]>"- ],- "license": "Apache-2.0",- "bugs": {- "url": "https://www.apifier.com"- },- "homepage": "https://www.apifier.com",- "dependencies": {}+ "name": "apifier",+ "version": "0.0.3",+ "description": "Apifier SDK for JavaScript",+ "main": "index.js",+ "keywords": [+ "apifier",+ "crawler",+ "scraper"+ ],+ "author": {+ "name": "Apifier",+ "email": "[email protected]",+ "url": "https://www.apifier.com"+ },+ "contributors": [+ "Jan Curn <[email protected]>"+ ],+ "license": "Apache-2.0",+ "repository": {+ "type": "git",+ "url": "git+https://github.com/apifier/apifier-sdk-js"+ },+ "bugs": {+ "url": "https://github.com/apifier/apifier-sdk-js/issues"+ },+ "homepage": "https://github.com/apifier/apifier-sdk-js#readme",+ "scripts": {+ "test": "mocha --compilers js:babel-core/register --recursive",+ "test-cov": "babel-node isparta cover --report html --report text node_modules/.bin/_mocha",+ "build": "babel src --out-dir build",+ "clean": "rm -rf build",+ "lint": "eslint src test"+ },+ "dependencies": {},+ "devDependencies": {+ "babel-core": "^6.1.21",+ "babel-eslint": "^7.2.2",+ "babel-preset-es2015": "^6.1.18",+ "babel-preset-stage-0": "^6.1.18",+ "chai": "^3.4.1",+ "eslint": "^3.19.0",+ "eslint-config-airbnb": "^14.1.0",+ "eslint-plugin-react": "^6.10.3",+ "isparta": "^4.0.0",+ "mocha": "^3.2.0"+ }}diff --git a/src/index.js b/src/index.jsnew file mode 100644index 000000000..ab6854608--- /dev/null+++ b/src/index.js@@ -0,0 +1,16 @@+class Person {+ constructor(name, age) {+ this.name = name;+ this.age = age;+ }++ getName() {+ return this.name;+ }++ getAge() {+ return this.age;+ }+}++export default Person;diff --git a/test/Counter.spec.js b/test/Counter.spec.jsnew file mode 100644index 000000000..47e041f8b--- /dev/null+++ b/test/Counter.spec.js@@ -0,0 +1,22 @@+import { expect } from 'chai';+import Person from '../src/index.js';++describe("Person", () => {+ let person;++ before(() => {+ person = new Person('Cam', 99);+ });++ after(() => {+ person = undefined;+ });++ it("should return name", function() {+ expect(person.getName()).to.equal('Cam');+ });++ it("should return age", function() {+ expect(person.getAge()).to.equal(99);+ });+});