Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: from cache fixes
Nicolasaef040b41e14
diff --git a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts b/apps/api/src/scraper/scrapeURL/engines/cache/index.tsindex f48806fd3..c0451df4e 100644--- a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/cache/index.ts@@ -10,6 +10,9 @@ export async function scrapeCache(meta: Meta): Promise<EngineScrapeResult> {const entry = await getEntryFromCache(key);if (entry === null) throw new EngineError("Cache missed");+ // Set fromCache flag to indicate this document was retrieved from cache+ meta.internalOptions.fromCache = true;+return {url: entry.url,html: entry.html,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex b13f7d9a8..7f4a76e43 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -154,6 +154,7 @@ export type InternalOptions = {useCache?: boolean;disableSmartWaitCache?: boolean; // Passed along to fire-engineisBackgroundIndex?: boolean;+ fromCache?: boolean; // Indicates if the document was retrieved from cache};export type EngineResultsTracker = {diff --git a/apps/api/src/scraper/scrapeURL/transformers/cache.ts b/apps/api/src/scraper/scrapeURL/transformers/cache.tsindex 4005059f5..f2d7bcf40 100644--- a/apps/api/src/scraper/scrapeURL/transformers/cache.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/cache.ts@@ -19,6 +19,12 @@ export function saveToCache(meta: Meta, document: Document): Document {);}+ // If the document was retrieved from cache, we don't need to save it+ if (meta.internalOptions.fromCache) {+ return document;+ }++const key = cacheKey(meta.url, meta.options, meta.internalOptions);if (key !== null) {