Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): log cache age in request frequency (#1784)
Gergő Móricz6337afeaaf15
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 04e23b43e..18fa3dfca 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -34,7 +34,7 @@ import { LLMRefusalError } from "./transformers/llmExtract";import { urlSpecificParams } from "./lib/urlSpecificParams";import { loadMock, MockState } from "./lib/mock";import { CostTracking } from "../../lib/extract/extraction-service";-import { addIndexRFInsertJob, generateDomainSplits, hashURL, normalizeURLForIndex, useIndex } from "../../services/index";+import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";export type ScrapeUrlResponse = (| {@@ -509,12 +509,31 @@ export async function scrapeURL(if (shouldRecordFrequency) {(async () => {try {+ const normalizedURL = normalizeURLForIndex(meta.url);+ const urlHash = hashURL(normalizedURL);++ let { data, error } = await index_supabase_service+ .from("index")+ .select("id, created_at, status")+ .eq("url_hash", urlHash)+ .order("created_at", { ascending: false })+ .limit(1);++ if (error) {+ meta.logger.warn("Failed to get age data", { error });+ }++ const age = data?.[0]+ ? Date.now() - new Date(data[0].created_at).getTime()+ : -1;+const domainSplits = generateDomainSplits(new URL(normalizeURLForIndex(meta.url)).hostname);const domainHash = hashURL(domainSplits.slice(-1)[0]);await addIndexRFInsertJob({domain_hash: domainHash,url: meta.url,+ age,});} catch (error) {meta.logger.warn("Failed to record frequency", { error });