Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Removing build diorectory from git

Marek Trunkatc2d49fd4ba6b
diff --git a/.gitignore b/.gitignore
index bbde28a78..6e3dcd056 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-builds
+build
node_modules
*.log
*.pid
diff --git a/build/actor.js b/build/actor.js
deleted file mode 100644
index 1b8b78a95..000000000
--- a/build/actor.js
+++ /dev/null
@@ -1,97 +0,0 @@
-'use strict';
-
-Object.defineProperty(exports, "__esModule", {
- value: true
-});
-exports.heyIAmReady = exports.main = undefined;
-
-var _fs = require('fs');
-
-var _fs2 = _interopRequireDefault(_fs);
-
-var _express = require('express');
-
-var _express2 = _interopRequireDefault(_express);
-
-var _bodyParser = require('body-parser');
-
-var _bodyParser2 = _interopRequireDefault(_bodyParser);
-
-var _utils = require('./utils');
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-/* global process */
-
-var main = exports.main = function main(userFunc) {
- if (!userFunc || typeof userFunc !== 'function') {
- throw new Error('Handler function must be provided as a parameter');
- }
- var serverPort = parseInt(process.env.APIFY_INTERNAL_PORT, 10);
- if (!(serverPort > 0 && serverPort < 65536)) {
- throw new Error('APIFY_INTERNAL_PORT environment variable must have a value from 1 to 65535.');
- }
-
- var handler = function handler(req, res) {
- var options = {
- input: {
- body: req.method === 'GET' ? null : req.body,
- method: req.method,
- contentType: req.headers['content-type']
- }
- };
-
- (0, _utils.newPromise)().then(function () {
- return userFunc(options);
- }).then(function () {
- res.statusCode = 200;
- }).catch(function (err) {
- console.error('Main function ended with exception:');
- console.error(err.stack || err);
- res.statusCode = 500;
- }).then(function () {
- res.end(function (err) {
- if (err) {
- console.log('Failed to send HTTP response: ' + err);
- process.exit(2);
- }
- if (res.statusCode === 500) {
- process.exit(1);
- } else {
- console.log('Main function ended successfully');
- process.exit(0);
- }
- });
- }).catch(function (err) {
- console.log('Something went terribly wrong: ' + err);
- process.exit(3);
- });
- };
-
- var app = (0, _express2.default)();
-
- // parse JSON, pass texts and raw data
- app.use(_bodyParser2.default.json({ type: '*/json' }));
- app.use(_bodyParser2.default.text({ type: 'text/*' }));
- app.use(_bodyParser2.default.raw({ type: '*/*' }));
- app.use(handler);
-
- // TODO: handle errors!
- app.listen(serverPort, function () {
- // console.log(`Listening on port ${serverPort}`);
- heyIAmReady();
- });
-};
-
-/**
- * Notifies Apifier runtime that act is listening on port specified by the APIFY_INTERNAL_PORT environment
- * variable and is ready to receive a HTTP request with act input.
- */
-var heyIAmReady = exports.heyIAmReady = function heyIAmReady() {
- var watchFileName = process.env.APIFY_WATCH_FILE;
- if (watchFileName) {
- _fs2.default.writeFile(watchFileName, '', function (err) {
- if (err) console.log('WARNING: Cannot write to watch file ' + watchFileName + ': ' + err);
- });
- }
-};
\ No newline at end of file
diff --git a/build/index.js b/build/index.js
deleted file mode 100644
index 74f6baf60..000000000
--- a/build/index.js
+++ /dev/null
@@ -1,23 +0,0 @@
-'use strict';
-
-var _utils = require('./utils');
-
-var _actor = require('./actor');
-
-var _keyValueStore = require('./key-value-store');
-
-var _keyValueStore2 = _interopRequireDefault(_keyValueStore);
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-var Apifier = {
- main: _actor.main,
- heyIAmReady: _actor.heyIAmReady,
- setPromisesDependency: _utils.setPromisesDependency,
- getPromisesDependency: _utils.getPromisesDependency,
- openKeyValueStore: _keyValueStore2.default
-};
-
-// export this way so that we can import using:
-// const Apifier = require('apifier');
-module.exports = Apifier;
\ No newline at end of file
diff --git a/build/key-value-store.js b/build/key-value-store.js
deleted file mode 100644
index 65fd95309..000000000
--- a/build/key-value-store.js
+++ /dev/null
@@ -1,149 +0,0 @@
-'use strict';
-
-var _underscore = require('underscore');
-
-var _underscore2 = _interopRequireDefault(_underscore);
-
-var _utils = require('./utils');
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-var methods = {
- getStore: function getStore(baseUrl, storeId) {
- return (0, _utils.requestPromise)({
- url: baseUrl + '/' + storeId,
- json: true,
- method: 'GET'
- });
- },
-
- deleteStore: function deleteStore(baseUrl, storeId) {
- return (0, _utils.requestPromise)({
- url: baseUrl + '/' + storeId,
- json: true,
- method: 'DELETE'
- });
- },
-
- getRecordsList: function getRecordsList(baseUrl, storeId) {
- return (0, _utils.requestPromise)({
- url: baseUrl + '/' + storeId + '/records',
- json: true,
- method: 'GET'
- });
- },
-
- getRecord: function getRecord(baseUrl, storeId, recordKey) {
- return (0, _utils.requestPromise)({
- url: baseUrl + '/' + storeId + '/records/' + recordKey,
- json: true,
- method: 'GET'
- });
- },
-
- putRecord: function putRecord(baseUrl, storeId, recordKey, body, contentType) {
- return (0, _utils.requestPromise)({
- url: baseUrl + '/' + storeId + '/records/' + recordKey,
- json: true,
- method: 'PUT',
- body: body,
- headers: {
- 'Content-Type': contentType
- }
- });
- },
-
- deleteRecord: function deleteRecord(baseUrl, storeId, recordKey) {
- return (0, _utils.requestPromise)({
- url: baseUrl + '/' + storeId + '/records/' + recordKey,
- json: true,
- method: 'DELETE'
- });
- }
-};
-
-/**
- * Get or creates key value store and returns it's object.
- * There are 3 possible initializations:
- *
- * 1) If `options.storeId` is provided then store with this id gets opened.
- * 2) If `options.name`, `options.token` and one of `options.ownerUserId` and `options.ownerUser`
- * are provided then store with given name gets opened or created if not exists.
- * 3) If ENV.APIFY_ACT_RUN_ID exists then store with this id (ie. store belonging to this act run)
- * gets opened.
- */
-var createOrGetStore = function createOrGetStore(baseUrl, options) {
- if (options.storeId) return methods.getStore(baseUrl, options.storeId);
-
- if ((options.ownerUserId || options.ownerUser) && options.token && options.name) {
- return (0, _utils.requestPromise)({
- url: baseUrl,
- json: true,
- method: 'POST',
- body: _underscore2.default.pick(options, 'ownerUserId', 'ownerUser', 'token', 'name')
- });
- }
-
- if (process.env.APIFY_ACT_RUN_ID) return methods.getStore(baseUrl, process.env.APIFY_ACT_RUN_ID);
-
- throw new Error('Error: cannot identify store via storeId, credentials or ENV variable.');
-};
-
-/**
- * Opens key value store and returns object containing preconfigured methods for that store.
- * Possible options are:
- * - protocol (http/https)
- * - host
- * - port
- * - basePath
- * - storeId
- * - ownerUser
- * - ownerUserId
- * - token
- * - name
- */
-// export default
-var openKeyValueStore = function openKeyValueStore() {
- var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
-
- if (!_underscore2.default.isObject(options)) throw new Error('Error: `options` must be object or empty.');
-
- // eslint-disable-next-line prefer-template
- var baseUrl = (options.protocol || 'https') + '://' + (options.host || 'api.apifier.com') + (options.port ? ':' + options.port : '') + (options.basePath || '/v2/key-value-stores');
-
- return createOrGetStore(baseUrl, options).then(function (store) {
- return _underscore2.default.mapObject(methods, function (method) {
- return _underscore2.default.partial(method, baseUrl, store._id);
- });
- });
-};
-
-var options = {
- protocol: 'http',
- host: 'localhost',
- port: '3300',
- basePath: '/api/v2/key-value-stores',
- storeId: '9mNnMEBm3LCakrgw3'
-};
-
-var options2 = {
- protocol: 'http',
- host: 'localhost',
- port: '3300',
- basePath: '/api/v2/key-value-stores',
- ownerUser: 'mtrunkat',
- token: '4AqtcLL4aQCtcWj9BSoLLACwJ',
- name: 'new-store'
-};
-
-openKeyValueStore(options2).then(function (store) {
- return store.getStore().then(function (out) {
- return console.log(out);
- }).then(function () {
- return store.putRecord('xxx', '{ "a": 2 }', 'application/json; charset=utf-8');
- }).then(function () {
- return store.getRecord('xxx');
- }).then(function (out) {
- return console.log(out);
- });
-});
\ No newline at end of file
diff --git a/build/utils.js b/build/utils.js
deleted file mode 100644
index 0ef900d98..000000000
--- a/build/utils.js
+++ /dev/null
@@ -1,62 +0,0 @@
-'use strict';
-
-Object.defineProperty(exports, "__esModule", {
- value: true
-});
-exports.requestPromise = exports.newPromise = exports.getPromisesDependency = exports.setPromisesDependency = undefined;
-
-var _request = require('request');
-
-var _request2 = _interopRequireDefault(_request);
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-var PromisesDependency = typeof Promise === 'function' ? Promise : null;
-
-// TODO: add methods to override console.log() and console.error(), add unit tests for that!
-
-/**
- * Sets the promise dependency the SDK will use wherever Promises are returned.
- * Passing `null` will force the SDK to use native Promises if they are available.
- * @param [Constructor] dep A reference to a Promise constructor
- */
-var setPromisesDependency = exports.setPromisesDependency = function setPromisesDependency(dep) {
- if (dep !== null && typeof dep !== 'function') throw new Error('The "dep" parameter must be a function');
- PromisesDependency = dep;
-};
-
-/**
- * Gets the promise dependency set by `Apifier.setPromisesDependency`.
- */
-var getPromisesDependency = exports.getPromisesDependency = function getPromisesDependency() {
- return PromisesDependency;
-};
-
-/**
- * Returns a result of `Promise.resolve()` using promise library set by `setPromisesDependency()`,
- * or using native promises, or throws if no native promises are available.
- * @return {*}
- */
-var newPromise = exports.newPromise = function newPromise() {
- if (PromisesDependency) {
- if (typeof PromisesDependency.resolve !== 'function') {
- throw new Error('The promise dependency set using Apifier.setPromisesDependency() does not define resolve() function.');
- }
- return PromisesDependency.resolve();
- }
- if (typeof Promise === 'function') return Promise.resolve();
- throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');
-};
-
-/**
- * Promised version of request(options) function.
- */
-var requestPromise = exports.requestPromise = function requestPromise(options) {
- return new PromisesDependency(function (resolve, reject) {
- (0, _request2.default)(options, function (error, response, body) {
- if (error) return reject(error);
-
- resolve(body);
- });
- });
-};
\ No newline at end of file