Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Merge branch 'master' into develop
Lubos Turek8a3f01c409f3
diff --cc src/crawler.jsindex 000000000,000000000..5d1ca4b0anew file mode 100644--- /dev/null+++ b/src/crawler.js@@@ -1,0 -1,0 +1,26 @@@++const request = require('request-promise');++const Promise = require('bluebird');++++export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params) => {++ params = params || {};++ params.token = token;++++ const queryString = objectToQueryString(params);++ const requestParams = {++ url : `${url}/v1/${userId}/crawlers?${queryString}`,++ json : true,++ resolveWithFullResponse : true,++ };++++ return request++ .get(requestParams)++ .then((response) => {++ const crawlers = response.body;++ checkPaginationHeaders(response, params, crawlers.length);++ checkResponse(response);++ if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));++++ return crawlers;++ });++++};diff --cc src/index.jsindex b0a58410e,7450b97e4..f13096cdc--- a/src/index.js+++ b/src/index.js@@@ -1,11 -1,14 +1,25 @@@++<<<<<<< HEAD+import _ from 'underscore';+import Actor from './actor';+import Crawler from './crawler';++=======+ import { setPromisesDependency, getPromisesDependency } from './utils';+ import { main, heyIAmReady } from './actor';++>>>>>>> master+ const Apifier = {+ main,+ heyIAmReady,+ setPromisesDependency,+ getPromisesDependency,+ };++<<<<<<< HEAD+const Apifier = {};+_.extend(Apifier, Crawler, Actor);++=======++>>>>>>> master- // export this way so the we can import as:+ // export this way so that we can import using:// const Apifier = require('apifier');module.exports = Apifier;