Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore(api/extract): remove old dumb caching mechanism

mogery8ef9ee6b8583
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index d0c0715a5..f2a6be45c 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -724,6 +724,7 @@ const extractV1Options = z
__experimental_llmUsage: z.boolean().prefault(false),
__experimental_showSources: z.boolean().prefault(false),
showSources: z.boolean().prefault(false),
+ // These two below don't do anything anymore
__experimental_cacheKey: z.string().optional(),
__experimental_cacheMode: z
.enum(["direct", "save", "load"])
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 1273b92c0..f6b813c84 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -685,6 +685,7 @@ const extractOptions = z
__experimental_llmUsage: z.boolean().prefault(false),
__experimental_showSources: z.boolean().prefault(false),
showSources: z.boolean().prefault(false),
+ // These two below don't do anything anymore
__experimental_cacheKey: z.string().optional(),
__experimental_cacheMode: z
.enum(["direct", "save", "load"])
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index 40ad63342..78fa58c21 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -35,7 +35,6 @@ import { analyzeSchemaAndPrompt } from "./completions/analyzeSchemaAndPrompt";
import { batchExtractPromise } from "./completions/batchExtract";
import { singleAnswerCompletion } from "./completions/singleAnswer";
import { SourceTracker } from "./helpers/source-tracker";
-import { getCachedDocs, saveCachedDocs } from "./helpers/cached-docs";
import { normalizeUrl } from "../canonical-url";
import { search } from "../../search";
import { buildRephraseToSerpPrompt } from "./build-prompts";
@@ -166,27 +165,6 @@ async function performExtraction(
const urls = request.urls || ([] as string[]);
- if (
- request.__experimental_cacheMode == "load" &&
- request.__experimental_cacheKey &&
- urls
- ) {
- logger.debug("Loading cached docs...");
- try {
- const cache = await getCachedDocs(
- urls,
- request.__experimental_cacheKey,
- );
- for (const doc of cache) {
- if (doc.metadata.url) {
- docsMap.set(normalizeUrl(doc.metadata.url), doc);
- }
- }
- } catch (error) {
- logger.error("Error loading cached docs", { error });
- }
- }
-
// Token tracking
let tokenUsage: TokenUsage[] = [];
@@ -1041,21 +1019,6 @@ async function performExtraction(
logger.debug("Done!");
- if (
- request.__experimental_cacheMode == "save" &&
- request.__experimental_cacheKey
- ) {
- logger.debug("Saving cached docs...");
- try {
- await saveCachedDocs(
- [...docsMap.values()],
- request.__experimental_cacheKey,
- );
- } catch (error) {
- logger.error("Error saving cached docs", { error });
- }
- }
-
// fs.writeFile(
// `logs/${request.urls?.[0].replaceAll("https://", "").replaceAll("http://", "").replaceAll("/", "-").replaceAll(".", "-")}-extract-${extractId}.json`,
// JSON.stringify(log, null, 2),
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index 031163b9b..3536f0475 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -15,7 +15,6 @@ import { getErrorContactMessage } from "../../deployment";
import { ExtractStep, updateExtract } from "../extract-redis";
import { CUSTOM_U_TEAMS } from "../config";
-import { getCachedDocs, saveCachedDocs } from "../helpers/cached-docs";
import { normalizeUrl } from "../../canonical-url";
import { search } from "../../../search";
import { buildRephraseToSerpPrompt_F0 } from "./build-prompts-f0";
@@ -142,24 +141,6 @@ export async function performExtraction_F0(
const urls = request.urls || ([] as string[]);
- if (
- request.__experimental_cacheMode == "load" &&
- request.__experimental_cacheKey &&
- urls
- ) {
- logger.debug("Loading cached docs...");
- try {
- const cache = await getCachedDocs(urls, request.__experimental_cacheKey);
- for (const doc of cache) {
- if (doc.metadata.url) {
- docsMap.set(normalizeUrl(doc.metadata.url), doc);
- }
- }
- } catch (error) {
- logger.error("Error loading cached docs", { error });
- }
- }
-
// Token tracking
let tokenUsage: TokenUsage[] = [];
@@ -939,21 +920,6 @@ export async function performExtraction_F0(
logger.debug("Done!");
- if (
- request.__experimental_cacheMode == "save" &&
- request.__experimental_cacheKey
- ) {
- logger.debug("Saving cached docs...");
- try {
- await saveCachedDocs(
- [...docsMap.values()],
- request.__experimental_cacheKey,
- );
- } catch (error) {
- logger.error("Error saving cached docs", { error });
- }
- }
-
return {
success: true,
data: finalResult ?? {},
diff --git a/apps/api/src/lib/extract/helpers/cached-docs.ts b/apps/api/src/lib/extract/helpers/cached-docs.ts
deleted file mode 100644
index e6f6e4b64..000000000
--- a/apps/api/src/lib/extract/helpers/cached-docs.ts
+++ /dev/null
@@ -1,68 +0,0 @@
-import { Document } from "../../../controllers/v1/types";
-import { supabase_service } from "../../../services/supabase";
-import { normalizeUrl } from "../../../lib/canonical-url";
-
-export async function getCachedDocs(
- urls: string[],
- cacheKey: string,
-): Promise<Document[]> {
- const normalizedUrls = urls.map(normalizeUrl);
- const { data, error } = await supabase_service
- .from("cached_scrapes")
- .select("doc")
- .in("url", normalizedUrls)
- .eq("cache_key", cacheKey);
-
- if (error) {
- console.error("Error fetching cached docs:", error);
- return [];
- }
-
- const uniqueDocs = new Map<string, Document>();
- data.forEach((res: any) => {
- const doc = JSON.parse(JSON.stringify(res.doc)) as Document;
- const docKey = `${doc.metadata.url}-${cacheKey}`;
- if (!uniqueDocs.has(docKey)) {
- uniqueDocs.set(docKey, doc);
- }
- });
-
- return Array.from(uniqueDocs.values());
-}
-
-export async function saveCachedDocs(
- docs: Document[],
- cacheKey: string,
-): Promise<void> {
- for (const doc of docs) {
- if (!doc.metadata.url) {
- throw new Error("Document has no URL");
- }
-
- const normalizedUrl = normalizeUrl(doc.metadata.url);
- const { data, error } = await supabase_service
- .from("cached_scrapes")
- .select("url")
- .eq("url", normalizedUrl)
- .eq("cache_key", cacheKey);
-
- if (error) {
- console.error("Error checking existing cached doc:", error);
- continue;
- }
-
- if (data.length === 0) {
- const { error: upsertError } = await supabase_service
- .from("cached_scrapes")
- .upsert({
- url: normalizedUrl,
- doc: doc,
- cache_key: cacheKey,
- });
-
- if (upsertError) {
- console.error("Error saving cached doc:", upsertError);
- }
- }
- }
-}