Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Merge branch 'master' into develop

Lubos Turek8a3f01c409f3
diff --cc src/crawler.js
index 000000000,000000000..5d1ca4b0a
new file mode 100644
--- /dev/null
+++ b/src/crawler.js
@@@ -1,0 -1,0 +1,26 @@@
++const request = require('request-promise');
++const Promise = require('bluebird');
++
++export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params) => {
++ params = params || {};
++ params.token = token;
++
++ const queryString = objectToQueryString(params);
++ const requestParams = {
++ url : `${url}/v1/${userId}/crawlers?${queryString}`,
++ json : true,
++ resolveWithFullResponse : true,
++ };
++
++ return request
++ .get(requestParams)
++ .then((response) => {
++ const crawlers = response.body;
++ checkPaginationHeaders(response, params, crawlers.length);
++ checkResponse(response);
++ if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));
++
++ return crawlers;
++ });
++
++};
diff --cc src/index.js
index b0a58410e,7450b97e4..f13096cdc
--- a/src/index.js
+++ b/src/index.js
@@@ -1,11 -1,14 +1,25 @@@
++<<<<<<< HEAD
+import _ from 'underscore';
+import Actor from './actor';
+import Crawler from './crawler';
++=======
+ import { setPromisesDependency, getPromisesDependency } from './utils';
+ import { main, heyIAmReady } from './actor';
++>>>>>>> master
+ const Apifier = {
+ main,
+ heyIAmReady,
+ setPromisesDependency,
+ getPromisesDependency,
+ };
++<<<<<<< HEAD
+const Apifier = {};
+_.extend(Apifier, Crawler, Actor);
++=======
++>>>>>>> master
- // export this way so the we can import as:
+ // export this way so that we can import using:
// const Apifier = require('apifier');
module.exports = Apifier;