Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added "prebulishOnly" event to package.json to ensure scripts are build and tested

Jan Curn7e0c3bcbb58c
diff --git a/build/actor.js b/build/actor.js
new file mode 100644
index 000000000..6e03c2789
--- /dev/null
+++ b/build/actor.js
@@ -0,0 +1,51 @@
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+
+var _http = require('http');
+
+var _http2 = _interopRequireDefault(_http);
+
+var _fs = require('fs');
+
+var _fs2 = _interopRequireDefault(_fs);
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+exports.default = {
+
+ main: function main(handler) {
+ if (!handler || typeof handler !== 'function') {
+ throw new Error('Handler function must be provided as a parameter');
+ }
+ var serverPort = parseInt(process.env.APIFIER_INTERNAL_PORT, 10);
+ if (!(serverPort > 0 && serverPort < 65536)) {
+ throw new Error('APIFIER_INTERNAL_PORT environment variable must be a value from 1 to 65535.');
+ }
+
+ var watchFileName = process.env.APIFIER_WATCH_FILE;
+
+ var wrappedHandler = function wrappedHandler(req, res) {
+ try {
+ handler(req, res);
+
+ // close response if not yet
+ } catch (e) {
+ console.log('Received message!!!');
+ res.statusCode = 200;
+ res.setHeader('Content-Type', 'text/plain');
+ res.end('Hello World\n');
+ }
+ };
+
+ var server = _http2.default.createServer(wrappedHandler);
+
+ server.listen(serverPort, function () {
+ console.log('Listening on port ' + serverPort);
+ if (watchFileName) _fs2.default.writeFileSync(watchFileName, '');
+ });
+ }
+
+};
\ No newline at end of file
diff --git a/build/index.js b/build/index.js
index f26244dba..4dc5ab943 100644
--- a/build/index.js
+++ b/build/index.js
@@ -1,34 +1,20 @@
-"use strict";
+'use strict';
Object.defineProperty(exports, "__esModule", {
value: true
});
-var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
+var _underscore = require('underscore');
-function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
+var _underscore2 = _interopRequireDefault(_underscore);
-var Person = function () {
- function Person(name, age) {
- _classCallCheck(this, Person);
+var _actor = require('./actor');
- this.name = name;
- this.age = age;
- }
+var _actor2 = _interopRequireDefault(_actor);
- _createClass(Person, [{
- key: "getName",
- value: function getName() {
- return this.name;
- }
- }, {
- key: "getAge",
- value: function getAge() {
- return this.age;
- }
- }]);
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
- return Person;
-}();
+var Apifier = {};
+_underscore2.default.extend(Apifier, Actorxx);
-exports.default = Person;
\ No newline at end of file
+exports.default = Apifier;
\ No newline at end of file
diff --git a/package.json b/package.json
index 9d8e1cc4f..8e5b95f54 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apifier",
- "version": "0.0.4",
+ "version": "0.0.5",
"description": "Apifier SDK for JavaScript",
"main": "build/index.js",
"keywords": [
@@ -29,6 +29,7 @@
"test": "mocha --compilers js:babel-core/register --recursive",
"test-cov": "babel-node isparta cover --report html --report text node_modules/.bin/_mocha",
"build": "babel src --out-dir build",
+ "prepublishOnly": "npm build && npm test",
"clean": "rm -rf build",
"lint": "eslint src test"
},
diff --git a/test/actor.js b/test/actor.js
index 0a97e6c2a..71e46847c 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -3,6 +3,8 @@ import Apifier from '../src/index';
/* global process */
+// TODO: run tests against build scripts too!
+
describe('Apifier.main()', () => {
it('should throw on invalid args', () => {
process.env.APIFIER_INTERNAL_PORT = 1234;