Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: from cache fixes

Nicolasaef040b41e14
diff --git a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts b/apps/api/src/scraper/scrapeURL/engines/cache/index.ts
index f48806fd3..c0451df4e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/cache/index.ts
@@ -10,6 +10,9 @@ export async function scrapeCache(meta: Meta): Promise<EngineScrapeResult> {
const entry = await getEntryFromCache(key);
if (entry === null) throw new EngineError("Cache missed");
+ // Set fromCache flag to indicate this document was retrieved from cache
+ meta.internalOptions.fromCache = true;
+
return {
url: entry.url,
html: entry.html,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index b13f7d9a8..7f4a76e43 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -154,6 +154,7 @@ export type InternalOptions = {
useCache?: boolean;
disableSmartWaitCache?: boolean; // Passed along to fire-engine
isBackgroundIndex?: boolean;
+ fromCache?: boolean; // Indicates if the document was retrieved from cache
};
export type EngineResultsTracker = {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/cache.ts b/apps/api/src/scraper/scrapeURL/transformers/cache.ts
index 4005059f5..f2d7bcf40 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/cache.ts
@@ -19,6 +19,12 @@ export function saveToCache(meta: Meta, document: Document): Document {
);
}
+ // If the document was retrieved from cache, we don't need to save it
+ if (meta.internalOptions.fromCache) {
+ return document;
+ }
+
+
const key = cacheKey(meta.url, meta.options, meta.internalOptions);
if (key !== null) {