Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Merge fix
Lubos Turekbaa4a002821f
diff --git a/build/index.js b/build/index.jsindex ff3b6f1a1..fbde5b8d1 100644--- a/build/index.js+++ b/build/index.js@@ -4,6 +4,8 @@ var _utils = require('./utils');var _actor = require('./actor');+require('./crawler');+var Apifier = {main: _actor.main,heyIAmReady: _actor.heyIAmReady,diff --git a/src/crawler.js b/src/crawler.jsindex 5d1ca4b0a..e0aff79ce 100644--- a/src/crawler.js+++ b/src/crawler.js@@ -1,5 +1,5 @@const request = require('request-promise');-const Promise = require('bluebird');+// const Promise = require('bluebird');export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params) => {params = params || {};@@ -7,9 +7,9 @@ export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params)const queryString = objectToQueryString(params);const requestParams = {- url : `${url}/v1/${userId}/crawlers?${queryString}`,- json : true,- resolveWithFullResponse : true,+ url: `${url}/v1/${userId}/crawlers?${queryString}`,+ json: true,+ resolveWithFullResponse: true,};return request@@ -22,5 +22,4 @@ export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params)return crawlers;});-};diff --git a/src/index.js b/src/index.jsindex f13096cdc..1bb3b140d 100644--- a/src/index.js+++ b/src/index.js@@ -1,11 +1,6 @@-<<<<<<< HEAD-import _ from 'underscore';-import Actor from './actor';-import Crawler from './crawler';-=======import { setPromisesDependency, getPromisesDependency } from './utils';import { main, heyIAmReady } from './actor';->>>>>>> master+import {} from './crawler';const Apifier = {main,@@ -14,12 +9,6 @@ const Apifier = {getPromisesDependency,};-<<<<<<< HEAD-const Apifier = {};-_.extend(Apifier, Crawler, Actor);-=======->>>>>>> master-// export this way so that we can import using:// const Apifier = require('apifier');module.exports = Apifier;