Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Merging develop
Marek Trunkat4f3937e7d5b7
diff --cc package.jsonindex 7efe32e4f,e4c298436..a7711b420--- a/package.json+++ b/package.json@@@ -39,7 -39,7 +39,8 @@@"dependencies": {"body-parser": "^1.17.1","express": "^4.15.2",+ "request": "^2.81.0",+ "request-promise": "^4.2.0","underscore": "^1.8.3"},"devDependencies": {diff --cc src/crawler.jsindex 000000000,cfd9bbf6a..402b93190mode 000000,100644..100644--- a/src/crawler.js+++ b/src/crawler.js@@@ -1,0 -1,71 +1,71 @@@+ import _ from 'underscore';++ import { newPromise, objectToQueryString } from './utils';++ const request = require('request-promise');++ const BASE_URL = 'https://api.apifier.com/v1';++ let defaultUserId;+ let defaultToken;++ export const setDefaultToken = (newDefaultToken) => {+ defaultToken = newDefaultToken;+ };++ export const setDefaultUserId = (newDefaultUserId) => {+ defaultUserId = newDefaultUserId;+ };++ export const getAllCrawlers = (params, userId, token) => {+ params = params || {};+ params.token = token || defaultToken;+ userId = userId || defaultUserId;++ const queryString = objectToQueryString(params);+ const requestParams = {- url: `${BASE_URL}/${userId}/crawlers?${queryString}`,++ url: `${BASE_URL}/${userId}/crawlers${queryString}`,+ json: true,+ resolveWithFullResponse: true,+ };++ return newPromise().then(() => (request+ .get(requestParams)))+ .then((response) => {+ const crawlers = response.body;+ console.log('crawlers: ' + crawlers);+ // checkPaginationHeaders(response, params, crawlers.length);+ // checkResponse(response);+ // if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));++ return crawlers;+ });+ };++ export const startCrawler = (crawlerId, params, userId, token) => {+ params = params || {};+ params.token = token || defaultToken;+ userId = userId || defaultUserId;++ const queryString = objectToQueryString(params);+ const requestParams = {- url: `${BASE_URL}/${userId}/crawlers/${crawlerId}/execute?${queryString}`,++ url: `${BASE_URL}/${userId}/crawlers/${crawlerId}/execute${queryString}`,+ json: true,+ resolveWithFullResponse: true,+ };++ return newPromise().then(() => (request+ .post(requestParams)))+ .then((response) => {+ // checkResponse(response, 201);+ const execution = response.body;++ if (!_.isObject(execution)) return reject('Unknown error of api Start Crawler API request');+ if (execution.type === 'CRAWLING_PROCESS_QUOTA_EXCEEDED') return reject('CRAWLING_PROCESS_QUOTA_EXCEEDED');++ console.log('execution: ' + execution);+ // if (!suppressSchemaCheck) schemaValidator.validate('CrawlerExecution', execution);++ return execution;+ });+ };diff --cc src/index.jsindex ae4c9e5fe,df28e5f77..340a97a28--- a/src/index.js+++ b/src/index.js@@@ -1,17 -1,18 +1,20 @@@import { setPromisesDependency, getPromisesDependency } from './utils';import { main, heyIAmReady } from './actor';+import openKeyValueStore from './key-value-store';-+ import { setDefaultToken, setDefaultUserId, getAllCrawlers, startCrawler } from './crawler';const Apifier = {main,heyIAmReady,setPromisesDependency,getPromisesDependency,+ openKeyValueStore,+ setDefaultToken,+ setDefaultUserId,+ getAllCrawlers,+ startCrawler,};-// export this way so that we can import using:// const Apifier = require('apifier');module.exports = Apifier;