Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/call memory (#114)

Marek Trunkátcace31770e38
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3754a4133..925242250 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxxxxxxxxxxxxxxxxxx
+===================
+- Parameter `memory` added to `Apify.call()`.
+
0.5.37 / 2018-06-07
===================
- `PseudoUrl` class can now contain a template for `Request` object creation and `PseudoUrl.createRequest()` method.
diff --git a/package.json b/package.json
index a8fe51b6a..cd10ad1b6 100644
--- a/package.json
+++ b/package.json
@@ -50,7 +50,7 @@
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
"jquery": "^3.3.1",
- "pidusage": "^2.0.6",
+ "pidusage": "=2.0.9",
"proxy-chain": "^0.1.26",
"ps-tree": "^1.1.0",
"request-promise": "^4.2.2",
diff --git a/src/actor.js b/src/actor.js
index e2ec216c2..b2d0e83ea 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -299,10 +299,12 @@ export const call = (actId, input, opts = {}) => {
if (token) defaultOpts.token = token;
// RunAct() options.
- const { build } = opts;
+ const { build, memory } = opts;
const runActOpts = {};
checkParamOrThrow(build, 'build', 'Maybe String');
+ checkParamOrThrow(memory, 'memory', 'Maybe Number');
if (build) runActOpts.build = build;
+ if (memory) runActOpts.memory = memory;
if (input) {
input = maybeStringify(input, opts);
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 8ebc22539..108fe683a 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -138,7 +138,7 @@ const enqueueRequestsFromClickableElements = async (page, selector, purls, reque
* @param {String} selector CSS selector matching elements to be clicked.
* @param {Array} pseudoUrls An array of `Apify.PseudoUrl` objects matching URL to be enqueued.
* @param {RequestQueue} requestQueue `Apify.RequestQueue` object where URLs will be enqueued.
- * @return {Promise}
+ * @return {Promise} Promise resolves to array of RequestOperationInfo objects.
* @memberof utils.puppeteer
*/
const enqueueLinks = async (page, selector, purls, requestQueue) => {
diff --git a/src/request_queue.js b/src/request_queue.js
index e2eaae5e8..63c43e69a 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -608,7 +608,7 @@ export class RequestQueueLocal {
return writeFilePromised(filePath, JSON.stringify(requestCopy, null, 4))
.then(() => ({
- requestId: request.id,
+ requestId: requestCopy.id,
wasAlreadyHandled: false,
wasAlreadyPresent: false,
}));
diff --git a/test/actor.js b/test/actor.js
index e0fd098b8..0473ec038 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -339,10 +339,11 @@ describe('Apify.call()', () => {
const output = { contentType, body: 'some-output' };
const expected = Object.assign({}, finishedRun, { output });
const build = 'xxx';
+ const memory = 1024;
const actsMock = sinon.mock(Apify.client.acts);
actsMock.expects('runAct')
- .withExactArgs({ token, actId, contentType: `${contentType}; charset=utf-8`, body: input, build })
+ .withExactArgs({ token, actId, contentType: `${contentType}; charset=utf-8`, body: input, build, memory })
.once()
.returns(Promise.resolve(runningRun));
actsMock.expects('getRun')
@@ -361,7 +362,7 @@ describe('Apify.call()', () => {
.returns(Promise.resolve(output));
return Apify
- .call(actId, input, { contentType, token, disableBodyParser: true, build })
+ .call(actId, input, { contentType, token, disableBodyParser: true, build, memory })
.then((callOutput) => {
expect(callOutput).to.be.eql(expected);
keyValueStoresMock.restore();
diff --git a/test/utils.js b/test/utils.js
index a7f4caf28..3a2659a5f 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -295,7 +295,7 @@ describe('utils.isAtHome()', () => {
describe('pidusage NPM package', () => {
it('throws correct error message when process not found', () => {
const NONEXISTING_PID = 9999;
- const promise = Promise.promisify(pidusage)(NONEXISTING_PID);
+ const promise = pidusage(NONEXISTING_PID);
return expect(promise).to.be.rejectedWith(utils.PID_USAGE_NOT_FOUND_ERROR);
});