Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Crawler SDK init
Lubos Tureka9c8d35ede8d
diff --git a/package.json b/package.jsonindex fadb8fcf3..106063dec 100644--- a/package.json+++ b/package.json@@ -34,8 +34,10 @@"lint": "eslint src test"},"dependencies": {+ "bluebird": "^3.5.0","body-parser": "^1.17.1","express": "^4.15.2",+ "request-promise": "^4.2.0","underscore": "^1.8.3"},"devDependencies": {diff --git a/src/actor.js b/src/actor.jsindex 39617271f..22e9528ae 100644--- a/src/actor.js+++ b/src/actor.js@@ -78,4 +78,3 @@ const Actor = {export default Actor;-diff --git a/src/index.js b/src/index.jsindex f1aad83c6..b0a58410e 100644--- a/src/index.js+++ b/src/index.js@@ -1,9 +1,10 @@import _ from 'underscore';import Actor from './actor';+import Crawler from './crawler';const Apifier = {};-_.extend(Apifier, Actor);+_.extend(Apifier, Crawler, Actor);// export this way so the we can import as:// const Apifier = require('apifier');diff --git a/test/actor.js b/test/actor.jsindex be760b8f9..be487fed9 100644--- a/test/actor.js+++ b/test/actor.js@@ -47,7 +47,7 @@ describe('Apifier.main()', () => {describe('Apifier.heyIAmReady()', () => {- it('it works as expected', () => {+ it('works as expected', () => {const tmpobj = tmp.fileSync();const path = tmpobj.name;fs.writeSync(tmpobj.fd, 'bla bla bla bla');