Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add first version of getAllCrawlers and startCrawler
Lubos Turek34b0909f4079
diff --git a/build/index.js b/build/index.jsindex fbde5b8d1..ec377f656 100644--- a/build/index.js+++ b/build/index.js@@ -4,13 +4,17 @@ var _utils = require('./utils');var _actor = require('./actor');-require('./crawler');+var _crawler = require('./crawler');var Apifier = {main: _actor.main,heyIAmReady: _actor.heyIAmReady,setPromisesDependency: _utils.setPromisesDependency,- getPromisesDependency: _utils.getPromisesDependency+ getPromisesDependency: _utils.getPromisesDependency,+ setDefaultToken: _crawler.setDefaultToken,+ setDefaultUserId: _crawler.setDefaultUserId,+ getAllCrawlers: _crawler.getAllCrawlers,+ startCrawler: _crawler.startCrawler};// export this way so that we can import using:diff --git a/build/utils.js b/build/utils.jsindex 978968c84..d0219026d 100644--- a/build/utils.js+++ b/build/utils.js@@ -3,7 +3,13 @@Object.defineProperty(exports, "__esModule", {value: true});+exports.objectToQueryString = exports.newPromise = exports.getPromisesDependency = exports.setPromisesDependency = undefined;+var _underscore = require('underscore');++var _underscore2 = _interopRequireDefault(_underscore);++function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }var promisesDependency = typeof Promise === 'function' ? Promise : null;@@ -41,4 +47,10 @@ var newPromise = exports.newPromise = function newPromise() {}if (typeof Promise === 'function') return Promise.resolve();throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');+};++var objectToQueryString = exports.objectToQueryString = function objectToQueryString(object) {+ return _underscore2.default.chain(object).mapObject(function (val, key) {+ return key + '=' + val;+ }).toArray().join('&');};\ No newline at end of filediff --git a/package.json b/package.jsonindex 9e937e6c6..e4c298436 100644--- a/package.json+++ b/package.json@@ -37,7 +37,6 @@"lint": "eslint src test"},"dependencies": {- "bluebird": "^3.5.0","body-parser": "^1.17.1","express": "^4.15.2","request-promise": "^4.2.0",diff --git a/src/crawler.js b/src/crawler.jsindex e0aff79ce..cfd9bbf6a 100644--- a/src/crawler.js+++ b/src/crawler.js@@ -1,25 +1,71 @@+import _ from 'underscore';++import { newPromise, objectToQueryString } from './utils';+const request = require('request-promise');-// const Promise = require('bluebird');-export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params) => {+const BASE_URL = 'https://api.apifier.com/v1';++let defaultUserId;+let defaultToken;++export const setDefaultToken = (newDefaultToken) => {+ defaultToken = newDefaultToken;+};++export const setDefaultUserId = (newDefaultUserId) => {+ defaultUserId = newDefaultUserId;+};++export const getAllCrawlers = (params, userId, token) => {params = params || {};- params.token = token;+ params.token = token || defaultToken;+ userId = userId || defaultUserId;const queryString = objectToQueryString(params);const requestParams = {- url: `${url}/v1/${userId}/crawlers?${queryString}`,+ url: `${BASE_URL}/${userId}/crawlers?${queryString}`,json: true,resolveWithFullResponse: true,};- return request- .get(requestParams)+ return newPromise().then(() => (request+ .get(requestParams))).then((response) => {const crawlers = response.body;- checkPaginationHeaders(response, params, crawlers.length);- checkResponse(response);- if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));+ console.log('crawlers: ' + crawlers);+ // checkPaginationHeaders(response, params, crawlers.length);+ // checkResponse(response);+ // if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));return crawlers;});};++export const startCrawler = (crawlerId, params, userId, token) => {+ params = params || {};+ params.token = token || defaultToken;+ userId = userId || defaultUserId;++ const queryString = objectToQueryString(params);+ const requestParams = {+ url: `${BASE_URL}/${userId}/crawlers/${crawlerId}/execute?${queryString}`,+ json: true,+ resolveWithFullResponse: true,+ };++ return newPromise().then(() => (request+ .post(requestParams)))+ .then((response) => {+ // checkResponse(response, 201);+ const execution = response.body;++ if (!_.isObject(execution)) return reject('Unknown error of api Start Crawler API request');+ if (execution.type === 'CRAWLING_PROCESS_QUOTA_EXCEEDED') return reject('CRAWLING_PROCESS_QUOTA_EXCEEDED');++ console.log('execution: ' + execution);+ // if (!suppressSchemaCheck) schemaValidator.validate('CrawlerExecution', execution);++ return execution;+ });+};diff --git a/src/index.js b/src/index.jsindex 1bb3b140d..df28e5f77 100644--- a/src/index.js+++ b/src/index.js@@ -1,12 +1,16 @@import { setPromisesDependency, getPromisesDependency } from './utils';import { main, heyIAmReady } from './actor';-import {} from './crawler';+import { setDefaultToken, setDefaultUserId, getAllCrawlers, startCrawler } from './crawler';const Apifier = {main,heyIAmReady,setPromisesDependency,getPromisesDependency,+ setDefaultToken,+ setDefaultUserId,+ getAllCrawlers,+ startCrawler,};// export this way so that we can import using:diff --git a/src/utils.js b/src/utils.jsindex 750334152..672ecdc1d 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,4 +1,4 @@-+import _ from 'underscore';let promisesDependency = typeof Promise === 'function' ? Promise : null;@@ -38,3 +38,10 @@ export const newPromise = () => {if (typeof Promise === 'function') return Promise.resolve();throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');};++export const objectToQueryString = (object) => {+ return _.chain(object)+ .mapObject((val, key) => `${key}=${val}`)+ .toArray()+ .join('&');+};