Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add first version of getAllCrawlers and startCrawler

Lubos Turek34b0909f4079
diff --git a/build/index.js b/build/index.js
index fbde5b8d1..ec377f656 100644
--- a/build/index.js
+++ b/build/index.js
@@ -4,13 +4,17 @@ var _utils = require('./utils');
var _actor = require('./actor');
-require('./crawler');
+var _crawler = require('./crawler');
var Apifier = {
main: _actor.main,
heyIAmReady: _actor.heyIAmReady,
setPromisesDependency: _utils.setPromisesDependency,
- getPromisesDependency: _utils.getPromisesDependency
+ getPromisesDependency: _utils.getPromisesDependency,
+ setDefaultToken: _crawler.setDefaultToken,
+ setDefaultUserId: _crawler.setDefaultUserId,
+ getAllCrawlers: _crawler.getAllCrawlers,
+ startCrawler: _crawler.startCrawler
};
// export this way so that we can import using:
diff --git a/build/utils.js b/build/utils.js
index 978968c84..d0219026d 100644
--- a/build/utils.js
+++ b/build/utils.js
@@ -3,7 +3,13 @@
Object.defineProperty(exports, "__esModule", {
value: true
});
+exports.objectToQueryString = exports.newPromise = exports.getPromisesDependency = exports.setPromisesDependency = undefined;
+var _underscore = require('underscore');
+
+var _underscore2 = _interopRequireDefault(_underscore);
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
var promisesDependency = typeof Promise === 'function' ? Promise : null;
@@ -41,4 +47,10 @@ var newPromise = exports.newPromise = function newPromise() {
}
if (typeof Promise === 'function') return Promise.resolve();
throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');
+};
+
+var objectToQueryString = exports.objectToQueryString = function objectToQueryString(object) {
+ return _underscore2.default.chain(object).mapObject(function (val, key) {
+ return key + '=' + val;
+ }).toArray().join('&');
};
\ No newline at end of file
diff --git a/package.json b/package.json
index 9e937e6c6..e4c298436 100644
--- a/package.json
+++ b/package.json
@@ -37,7 +37,6 @@
"lint": "eslint src test"
},
"dependencies": {
- "bluebird": "^3.5.0",
"body-parser": "^1.17.1",
"express": "^4.15.2",
"request-promise": "^4.2.0",
diff --git a/src/crawler.js b/src/crawler.js
index e0aff79ce..cfd9bbf6a 100644
--- a/src/crawler.js
+++ b/src/crawler.js
@@ -1,25 +1,71 @@
+import _ from 'underscore';
+
+import { newPromise, objectToQueryString } from './utils';
+
const request = require('request-promise');
-// const Promise = require('bluebird');
-export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params) => {
+const BASE_URL = 'https://api.apifier.com/v1';
+
+let defaultUserId;
+let defaultToken;
+
+export const setDefaultToken = (newDefaultToken) => {
+ defaultToken = newDefaultToken;
+};
+
+export const setDefaultUserId = (newDefaultUserId) => {
+ defaultUserId = newDefaultUserId;
+};
+
+export const getAllCrawlers = (params, userId, token) => {
params = params || {};
- params.token = token;
+ params.token = token || defaultToken;
+ userId = userId || defaultUserId;
const queryString = objectToQueryString(params);
const requestParams = {
- url: `${url}/v1/${userId}/crawlers?${queryString}`,
+ url: `${BASE_URL}/${userId}/crawlers?${queryString}`,
json: true,
resolveWithFullResponse: true,
};
- return request
- .get(requestParams)
+ return newPromise().then(() => (request
+ .get(requestParams)))
.then((response) => {
const crawlers = response.body;
- checkPaginationHeaders(response, params, crawlers.length);
- checkResponse(response);
- if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));
+ console.log('crawlers: ' + crawlers);
+ // checkPaginationHeaders(response, params, crawlers.length);
+ // checkResponse(response);
+ // if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));
return crawlers;
});
};
+
+export const startCrawler = (crawlerId, params, userId, token) => {
+ params = params || {};
+ params.token = token || defaultToken;
+ userId = userId || defaultUserId;
+
+ const queryString = objectToQueryString(params);
+ const requestParams = {
+ url: `${BASE_URL}/${userId}/crawlers/${crawlerId}/execute?${queryString}`,
+ json: true,
+ resolveWithFullResponse: true,
+ };
+
+ return newPromise().then(() => (request
+ .post(requestParams)))
+ .then((response) => {
+ // checkResponse(response, 201);
+ const execution = response.body;
+
+ if (!_.isObject(execution)) return reject('Unknown error of api Start Crawler API request');
+ if (execution.type === 'CRAWLING_PROCESS_QUOTA_EXCEEDED') return reject('CRAWLING_PROCESS_QUOTA_EXCEEDED');
+
+ console.log('execution: ' + execution);
+ // if (!suppressSchemaCheck) schemaValidator.validate('CrawlerExecution', execution);
+
+ return execution;
+ });
+};
diff --git a/src/index.js b/src/index.js
index 1bb3b140d..df28e5f77 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,12 +1,16 @@
import { setPromisesDependency, getPromisesDependency } from './utils';
import { main, heyIAmReady } from './actor';
-import {} from './crawler';
+import { setDefaultToken, setDefaultUserId, getAllCrawlers, startCrawler } from './crawler';
const Apifier = {
main,
heyIAmReady,
setPromisesDependency,
getPromisesDependency,
+ setDefaultToken,
+ setDefaultUserId,
+ getAllCrawlers,
+ startCrawler,
};
// export this way so that we can import using:
diff --git a/src/utils.js b/src/utils.js
index 750334152..672ecdc1d 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,4 +1,4 @@
-
+import _ from 'underscore';
let promisesDependency = typeof Promise === 'function' ? Promise : null;
@@ -38,3 +38,10 @@ export const newPromise = () => {
if (typeof Promise === 'function') return Promise.resolve();
throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');
};
+
+export const objectToQueryString = (object) => {
+ return _.chain(object)
+ .mapObject((val, key) => `${key}=${val}`)
+ .toArray()
+ .join('&');
+};