Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Merge fix

Lubos Turekbaa4a002821f
diff --git a/build/index.js b/build/index.js
index ff3b6f1a1..fbde5b8d1 100644
--- a/build/index.js
+++ b/build/index.js
@@ -4,6 +4,8 @@ var _utils = require('./utils');
var _actor = require('./actor');
+require('./crawler');
+
var Apifier = {
main: _actor.main,
heyIAmReady: _actor.heyIAmReady,
diff --git a/src/crawler.js b/src/crawler.js
index 5d1ca4b0a..e0aff79ce 100644
--- a/src/crawler.js
+++ b/src/crawler.js
@@ -1,5 +1,5 @@
const request = require('request-promise');
-const Promise = require('bluebird');
+// const Promise = require('bluebird');
export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params) => {
params = params || {};
@@ -7,9 +7,9 @@ export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params)
const queryString = objectToQueryString(params);
const requestParams = {
- url : `${url}/v1/${userId}/crawlers?${queryString}`,
- json : true,
- resolveWithFullResponse : true,
+ url: `${url}/v1/${userId}/crawlers?${queryString}`,
+ json: true,
+ resolveWithFullResponse: true,
};
return request
@@ -22,5 +22,4 @@ export const getAllCrawlers = (url, userId, token, suppressSchemaCheck, params)
return crawlers;
});
-
};
diff --git a/src/index.js b/src/index.js
index f13096cdc..1bb3b140d 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,11 +1,6 @@
-<<<<<<< HEAD
-import _ from 'underscore';
-import Actor from './actor';
-import Crawler from './crawler';
-=======
import { setPromisesDependency, getPromisesDependency } from './utils';
import { main, heyIAmReady } from './actor';
->>>>>>> master
+import {} from './crawler';
const Apifier = {
main,
@@ -14,12 +9,6 @@ const Apifier = {
getPromisesDependency,
};
-<<<<<<< HEAD
-const Apifier = {};
-_.extend(Apifier, Crawler, Actor);
-=======
->>>>>>> master
-
// export this way so that we can import using:
// const Apifier = require('apifier');
module.exports = Apifier;