Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Merging develop

Marek Trunkat4f3937e7d5b7
diff --cc package.json
index 7efe32e4f,e4c298436..a7711b420
--- a/package.json
+++ b/package.json
@@@ -39,7 -39,7 +39,8 @@@
"dependencies": {
"body-parser": "^1.17.1",
"express": "^4.15.2",
+ "request": "^2.81.0",
+ "request-promise": "^4.2.0",
"underscore": "^1.8.3"
},
"devDependencies": {
diff --cc src/crawler.js
index 000000000,cfd9bbf6a..402b93190
mode 000000,100644..100644
--- a/src/crawler.js
+++ b/src/crawler.js
@@@ -1,0 -1,71 +1,71 @@@
+ import _ from 'underscore';
+
+ import { newPromise, objectToQueryString } from './utils';
+
+ const request = require('request-promise');
+
+ const BASE_URL = 'https://api.apifier.com/v1';
+
+ let defaultUserId;
+ let defaultToken;
+
+ export const setDefaultToken = (newDefaultToken) => {
+ defaultToken = newDefaultToken;
+ };
+
+ export const setDefaultUserId = (newDefaultUserId) => {
+ defaultUserId = newDefaultUserId;
+ };
+
+ export const getAllCrawlers = (params, userId, token) => {
+ params = params || {};
+ params.token = token || defaultToken;
+ userId = userId || defaultUserId;
+
+ const queryString = objectToQueryString(params);
+ const requestParams = {
- url: `${BASE_URL}/${userId}/crawlers?${queryString}`,
++ url: `${BASE_URL}/${userId}/crawlers${queryString}`,
+ json: true,
+ resolveWithFullResponse: true,
+ };
+
+ return newPromise().then(() => (request
+ .get(requestParams)))
+ .then((response) => {
+ const crawlers = response.body;
+ console.log('crawlers: ' + crawlers);
+ // checkPaginationHeaders(response, params, crawlers.length);
+ // checkResponse(response);
+ // if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));
+
+ return crawlers;
+ });
+ };
+
+ export const startCrawler = (crawlerId, params, userId, token) => {
+ params = params || {};
+ params.token = token || defaultToken;
+ userId = userId || defaultUserId;
+
+ const queryString = objectToQueryString(params);
+ const requestParams = {
- url: `${BASE_URL}/${userId}/crawlers/${crawlerId}/execute?${queryString}`,
++ url: `${BASE_URL}/${userId}/crawlers/${crawlerId}/execute${queryString}`,
+ json: true,
+ resolveWithFullResponse: true,
+ };
+
+ return newPromise().then(() => (request
+ .post(requestParams)))
+ .then((response) => {
+ // checkResponse(response, 201);
+ const execution = response.body;
+
+ if (!_.isObject(execution)) return reject('Unknown error of api Start Crawler API request');
+ if (execution.type === 'CRAWLING_PROCESS_QUOTA_EXCEEDED') return reject('CRAWLING_PROCESS_QUOTA_EXCEEDED');
+
+ console.log('execution: ' + execution);
+ // if (!suppressSchemaCheck) schemaValidator.validate('CrawlerExecution', execution);
+
+ return execution;
+ });
+ };
diff --cc src/index.js
index ae4c9e5fe,df28e5f77..340a97a28
--- a/src/index.js
+++ b/src/index.js
@@@ -1,17 -1,18 +1,20 @@@
import { setPromisesDependency, getPromisesDependency } from './utils';
import { main, heyIAmReady } from './actor';
+import openKeyValueStore from './key-value-store';
-
+ import { setDefaultToken, setDefaultUserId, getAllCrawlers, startCrawler } from './crawler';
const Apifier = {
main,
heyIAmReady,
setPromisesDependency,
getPromisesDependency,
+ openKeyValueStore,
+ setDefaultToken,
+ setDefaultUserId,
+ getAllCrawlers,
+ startCrawler,
};
-
// export this way so that we can import using:
// const Apifier = require('apifier');
module.exports = Apifier;