Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
const { createHash } = require('node:crypto');
const { inspect } = require('node:util');
const { urlToRequest } = require('loader-utils');
const SIGNING_TOKEN = process.env.APIFY_SIGNING_TOKEN;
const signingUrl = new URL('https://api.apify.com/v2/tools/encode-and-sign');
signingUrl.searchParams.set('token', SIGNING_TOKEN || '');
const queue = [];
const cache = {};
let working = false;
let warnedAboutMissingToken = false;
function hash(source) {
return createHash('sha1').update(source).digest('hex');
}
async function getHash(source) {
const cacheKey = hash(source);
if (cache[cacheKey]) {
return cache[cacheKey];
}
const memory = source.match(/playwright|puppeteer/i) ? 4096 : 1024;
const res = await (await fetch(signingUrl, {
method: 'POST',
body: JSON.stringify({
input: JSON.stringify({ code: source }),
options: {
build: 'latest',
contentType: 'application/json; charset=utf-8',
memory,
timeout: 180,
},
}),
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
}));
if (!res.ok) {
console.error(`Signing failed: ${res.status} ${res.statusText}`, await res.text());
return 'invalid-token';
}
const body = await res.json();
if (!body.data || !body.data.encoded) {
console.error(`Signing failed:' ${inspect(body.error) || 'Unknown error'}`, body);
return 'invalid-token';
}
cache[cacheKey] = body.data.encoded;
await new Promise((resolve) => setTimeout(resolve, 100));
return body.data.encoded;
}
async function encodeAndSign(source) {
if (!process.env.APIFY_SIGNING_TOKEN) {
return 'invalid-token';
}
if (working) {
return new Promise((resolve, reject) => {
queue.push(() => {
return getHash(source).then(resolve, reject);
});
});
}
let res;
try {
working = true;
res = await getHash(source);
while (queue.length) {
await queue.shift()();
}
} finally {
working = false;
}
return res;
}
module.exports = async function (code) {
if (process.env.CRAWLEE_DOCS_FAST) {
return { code, hash: 'fast' };
}
// Skip signing if token is not configured
if (!SIGNING_TOKEN) {
if (!warnedAboutMissingToken) {
console.warn('APIFY_SIGNING_TOKEN is not set, skipping code signing for runnable examples.');
warnedAboutMissingToken = true;
}
return { code, hash: 'unsigned' };
}
console.log(`Signing ${urlToRequest(this.resourcePath)}...`, { working, queue: queue.length });
const codeHash = await encodeAndSign(code);
return { code, hash: codeHash };
};