Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
frequency logging
Gergő Móriczad2f0992fd88
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 18fa3dfca..ace37b9a7 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -509,6 +509,7 @@ export async function scrapeURL(if (shouldRecordFrequency) {(async () => {try {+ meta.logger.info("Recording frequency");const normalizedURL = normalizeURLForIndex(meta.url);const urlHash = hashURL(normalizedURL);@@ -530,15 +531,24 @@ export async function scrapeURL(const domainSplits = generateDomainSplits(new URL(normalizeURLForIndex(meta.url)).hostname);const domainHash = hashURL(domainSplits.slice(-1)[0]);- await addIndexRFInsertJob({+ const out = {domain_hash: domainHash,url: meta.url,age,- });+ };++ await addIndexRFInsertJob(out);+ meta.logger.info("Recorded frequency", { out });} catch (error) {meta.logger.warn("Failed to record frequency", { error });}})();+ } else {+ meta.logger.info("Not recording frequency", {+ useIndex,+ storeInCache: meta.options.storeInCache,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ });}try {