Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: add caching to ROA signing
Martin Adámek4ec78685abee
diff --git a/website/roa-loader/index.js b/website/roa-loader/index.jsindex 970e3d7a6..2208c6c2c 100644--- a/website/roa-loader/index.js+++ b/website/roa-loader/index.js@@ -1,13 +1,25 @@-const { inspect } = require('util');+const { createHash } = require('node:crypto');+const { inspect } = require('node:util');const { urlToRequest } = require('loader-utils');const signingUrl = new URL('https://api.apify.com/v2/tools/encode-and-sign');signingUrl.searchParams.set('token', process.env.APIFY_SIGNING_TOKEN);const queue = [];+const cache = {};let working = false;+function hash(source) {+ return createHash('sha1').update(source).digest('hex');+}+async function getHash(source) {+ const cacheKey = hash(source);++ if (cache[cacheKey]) {+ return cache[cacheKey];+ }+const memory = source.match(/playwright|puppeteer/i) ? 4096 : 1024;const res = await (await fetch(signingUrl, {method: 'POST',@@ -32,13 +44,14 @@ async function getHash(source) {const body = await res.json();- await new Promise((resolve) => setTimeout(resolve, 100));-if (!body.data || !body.data.encoded) {console.error(`Signing failed:' ${inspect(body.error) || 'Unknown error'}`, body);return 'invalid-token';}+ cache[cacheKey] = body.data.encoded;+ await new Promise((resolve) => setTimeout(resolve, 100));+return body.data.encoded;}