Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(api/extract): remove old dumb caching mechanism
mogery8ef9ee6b8583
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex d0c0715a5..f2a6be45c 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -724,6 +724,7 @@ const extractV1Options = z__experimental_llmUsage: z.boolean().prefault(false),__experimental_showSources: z.boolean().prefault(false),showSources: z.boolean().prefault(false),+ // These two below don't do anything anymore__experimental_cacheKey: z.string().optional(),__experimental_cacheMode: z.enum(["direct", "save", "load"])diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 1273b92c0..f6b813c84 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -685,6 +685,7 @@ const extractOptions = z__experimental_llmUsage: z.boolean().prefault(false),__experimental_showSources: z.boolean().prefault(false),showSources: z.boolean().prefault(false),+ // These two below don't do anything anymore__experimental_cacheKey: z.string().optional(),__experimental_cacheMode: z.enum(["direct", "save", "load"])diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 40ad63342..78fa58c21 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -35,7 +35,6 @@ import { analyzeSchemaAndPrompt } from "./completions/analyzeSchemaAndPrompt";import { batchExtractPromise } from "./completions/batchExtract";import { singleAnswerCompletion } from "./completions/singleAnswer";import { SourceTracker } from "./helpers/source-tracker";-import { getCachedDocs, saveCachedDocs } from "./helpers/cached-docs";import { normalizeUrl } from "../canonical-url";import { search } from "../../search";import { buildRephraseToSerpPrompt } from "./build-prompts";@@ -166,27 +165,6 @@ async function performExtraction(const urls = request.urls || ([] as string[]);- if (- request.__experimental_cacheMode == "load" &&- request.__experimental_cacheKey &&- urls- ) {- logger.debug("Loading cached docs...");- try {- const cache = await getCachedDocs(- urls,- request.__experimental_cacheKey,- );- for (const doc of cache) {- if (doc.metadata.url) {- docsMap.set(normalizeUrl(doc.metadata.url), doc);- }- }- } catch (error) {- logger.error("Error loading cached docs", { error });- }- }-// Token trackinglet tokenUsage: TokenUsage[] = [];@@ -1041,21 +1019,6 @@ async function performExtraction(logger.debug("Done!");- if (- request.__experimental_cacheMode == "save" &&- request.__experimental_cacheKey- ) {- logger.debug("Saving cached docs...");- try {- await saveCachedDocs(- [...docsMap.values()],- request.__experimental_cacheKey,- );- } catch (error) {- logger.error("Error saving cached docs", { error });- }- }-// fs.writeFile(// `logs/${request.urls?.[0].replaceAll("https://", "").replaceAll("http://", "").replaceAll("/", "-").replaceAll(".", "-")}-extract-${extractId}.json`,// JSON.stringify(log, null, 2),diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 031163b9b..3536f0475 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -15,7 +15,6 @@ import { getErrorContactMessage } from "../../deployment";import { ExtractStep, updateExtract } from "../extract-redis";import { CUSTOM_U_TEAMS } from "../config";-import { getCachedDocs, saveCachedDocs } from "../helpers/cached-docs";import { normalizeUrl } from "../../canonical-url";import { search } from "../../../search";import { buildRephraseToSerpPrompt_F0 } from "./build-prompts-f0";@@ -142,24 +141,6 @@ export async function performExtraction_F0(const urls = request.urls || ([] as string[]);- if (- request.__experimental_cacheMode == "load" &&- request.__experimental_cacheKey &&- urls- ) {- logger.debug("Loading cached docs...");- try {- const cache = await getCachedDocs(urls, request.__experimental_cacheKey);- for (const doc of cache) {- if (doc.metadata.url) {- docsMap.set(normalizeUrl(doc.metadata.url), doc);- }- }- } catch (error) {- logger.error("Error loading cached docs", { error });- }- }-// Token trackinglet tokenUsage: TokenUsage[] = [];@@ -939,21 +920,6 @@ export async function performExtraction_F0(logger.debug("Done!");- if (- request.__experimental_cacheMode == "save" &&- request.__experimental_cacheKey- ) {- logger.debug("Saving cached docs...");- try {- await saveCachedDocs(- [...docsMap.values()],- request.__experimental_cacheKey,- );- } catch (error) {- logger.error("Error saving cached docs", { error });- }- }-return {success: true,data: finalResult ?? {},diff --git a/apps/api/src/lib/extract/helpers/cached-docs.ts b/apps/api/src/lib/extract/helpers/cached-docs.tsdeleted file mode 100644index e6f6e4b64..000000000--- a/apps/api/src/lib/extract/helpers/cached-docs.ts+++ /dev/null@@ -1,68 +0,0 @@-import { Document } from "../../../controllers/v1/types";-import { supabase_service } from "../../../services/supabase";-import { normalizeUrl } from "../../../lib/canonical-url";--export async function getCachedDocs(- urls: string[],- cacheKey: string,-): Promise<Document[]> {- const normalizedUrls = urls.map(normalizeUrl);- const { data, error } = await supabase_service- .from("cached_scrapes")- .select("doc")- .in("url", normalizedUrls)- .eq("cache_key", cacheKey);-- if (error) {- console.error("Error fetching cached docs:", error);- return [];- }-- const uniqueDocs = new Map<string, Document>();- data.forEach((res: any) => {- const doc = JSON.parse(JSON.stringify(res.doc)) as Document;- const docKey = `${doc.metadata.url}-${cacheKey}`;- if (!uniqueDocs.has(docKey)) {- uniqueDocs.set(docKey, doc);- }- });-- return Array.from(uniqueDocs.values());-}--export async function saveCachedDocs(- docs: Document[],- cacheKey: string,-): Promise<void> {- for (const doc of docs) {- if (!doc.metadata.url) {- throw new Error("Document has no URL");- }-- const normalizedUrl = normalizeUrl(doc.metadata.url);- const { data, error } = await supabase_service- .from("cached_scrapes")- .select("url")- .eq("url", normalizedUrl)- .eq("cache_key", cacheKey);-- if (error) {- console.error("Error checking existing cached doc:", error);- continue;- }-- if (data.length === 0) {- const { error: upsertError } = await supabase_service- .from("cached_scrapes")- .upsert({- url: normalizedUrl,- doc: doc,- cache_key: cacheKey,- });-- if (upsertError) {- console.error("Error saving cached doc:", upsertError);- }- }- }-}