Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: add caching to ROA signing

Martin Adámek4ec78685abee
diff --git a/website/roa-loader/index.js b/website/roa-loader/index.js
index 970e3d7a6..2208c6c2c 100644
--- a/website/roa-loader/index.js
+++ b/website/roa-loader/index.js
@@ -1,13 +1,25 @@
-const { inspect } = require('util');
+const { createHash } = require('node:crypto');
+const { inspect } = require('node:util');
const { urlToRequest } = require('loader-utils');
const signingUrl = new URL('https://api.apify.com/v2/tools/encode-and-sign');
signingUrl.searchParams.set('token', process.env.APIFY_SIGNING_TOKEN);
const queue = [];
+const cache = {};
let working = false;
+function hash(source) {
+ return createHash('sha1').update(source).digest('hex');
+}
+
async function getHash(source) {
+ const cacheKey = hash(source);
+
+ if (cache[cacheKey]) {
+ return cache[cacheKey];
+ }
+
const memory = source.match(/playwright|puppeteer/i) ? 4096 : 1024;
const res = await (await fetch(signingUrl, {
method: 'POST',
@@ -32,13 +44,14 @@ async function getHash(source) {
const body = await res.json();
- await new Promise((resolve) => setTimeout(resolve, 100));
-
if (!body.data || !body.data.encoded) {
console.error(`Signing failed:' ${inspect(body.error) || 'Unknown error'}`, body);
return 'invalid-token';
}
+ cache[cacheKey] = body.data.encoded;
+ await new Promise((resolve) => setTimeout(resolve, 100));
+
return body.data.encoded;
}