Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
added cached scrapes to extract (#1107)
Rafael Miller8d7e8c4f500e
diff --git a/apps/api/requests.http b/apps/api/requests.httpindex f7b7de9cf..c9495cf80 100644--- a/apps/api/requests.http+++ b/apps/api/requests.http@@ -96,3 +96,7 @@ content-type: application/json# @name extractFirecrawlStatusGET {{baseUrl}}/v1/extract/{{extractFirecrawlId}} HTTP/1.1Authorization: Bearer {{$dotenv TEST_API_KEY}}++###+DELETE {{baseUrl}}/v1/crawl/c94136f9-86c1-4a97-966c-1c8e0274778f HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex c3ba3a69b..76cccf7da 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -228,10 +228,12 @@ export const extractV1Options = zenableWebSearch: z.boolean().default(false),origin: z.string().optional().default("api"),urlTrace: z.boolean().default(false),+ timeout: z.number().int().positive().finite().safe().default(60000),__experimental_streamSteps: z.boolean().default(false),__experimental_llmUsage: z.boolean().default(false),__experimental_showSources: z.boolean().default(false),- timeout: z.number().int().positive().finite().safe().default(60000),+ __experimental_cacheKey: z.string().optional(),+ __experimental_cacheMode: z.enum(["direct", "save", "load"]).default("direct").optional()}).strict(strictMessage).transform((obj) => ({diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 3725ff8b1..68213c2c5 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -36,12 +36,16 @@ import { checkShouldExtract } from "./completions/checkShouldExtract";import { batchExtractPromise } from "./completions/batchExtract";import { singleAnswerCompletion } from "./completions/singleAnswer";import { SourceTracker } from "./helpers/source-tracker";+import { getCachedDocs, saveCachedDocs } from "./helpers/cached-docs";+import { normalizeUrl } from "../canonical-url";interface ExtractServiceOptions {request: ExtractRequest;teamId: string;plan: PlanType;subId?: string;+ cacheMode?: "load" | "save" | "direct";+ cacheKey?: string;}interface ExtractResult {@@ -86,6 +90,20 @@ export async function performExtraction(extractId,});+ if (request.__experimental_cacheMode == "load" && request.__experimental_cacheKey) {+ logger.debug("Loading cached docs...");+ try {+ const cache = await getCachedDocs(request.urls, request.__experimental_cacheKey);+ for (const doc of cache) {+ if (doc.metadata.url) {+ docsMap.set(normalizeUrl(doc.metadata.url), doc);+ }+ }+ } catch (error) {+ logger.error("Error loading cached docs", { error });+ }+ }+// Token trackinglet tokenUsage: TokenUsage[] = [];@@ -257,8 +275,9 @@ export async function performExtraction(logger.debug("Starting multi-entity scrape...");let startScrape = Date.now();+const scrapePromises = links.map((url) => {- if (!docsMap.has(url)) {+ if (!docsMap.has(normalizeUrl(url))) {return scrapeDocument({url,@@ -280,7 +299,7 @@ export async function performExtraction(});}- return docsMap.get(url);+ return docsMap.get(normalizeUrl(url));});let multyEntityDocs = (await Promise.all(scrapePromises)).filter(@@ -309,7 +328,7 @@ export async function performExtraction(for (const doc of multyEntityDocs) {if (doc?.metadata?.url) {- docsMap.set(doc.metadata.url, doc);+ docsMap.set(normalizeUrl(doc.metadata.url), doc);}}@@ -519,7 +538,7 @@ export async function performExtraction(],});const scrapePromises = links.map((url) => {- if (!docsMap.has(url)) {+ if (!docsMap.has(normalizeUrl(url))) {return scrapeDocument({url,@@ -537,7 +556,7 @@ export async function performExtraction(}),);}- return docsMap.get(url);+ return docsMap.get(normalizeUrl(url));});try {@@ -545,7 +564,7 @@ export async function performExtraction(for (const doc of results) {if (doc?.metadata?.url) {- docsMap.set(doc.metadata.url, doc);+ docsMap.set(normalizeUrl(doc.metadata.url), doc);}}logger.debug("Updated docsMap.", { docsMapSize: docsMap.size }); // useful for error probing@@ -744,6 +763,15 @@ export async function performExtraction(logger.debug("Done!");+ if (request.__experimental_cacheMode == "save" && request.__experimental_cacheKey) {+ logger.debug("Saving cached docs...");+ try {+ await saveCachedDocs([...docsMap.values()], request.__experimental_cacheKey);+ } catch (error) {+ logger.error("Error saving cached docs", { error });+ }+ }+return {success: true,data: finalResult ?? {},diff --git a/apps/api/src/lib/extract/helpers/cached-docs.ts b/apps/api/src/lib/extract/helpers/cached-docs.tsnew file mode 100644index 000000000..a2cb0ea3d--- /dev/null+++ b/apps/api/src/lib/extract/helpers/cached-docs.ts@@ -0,0 +1,62 @@+import { Document } from "../../../controllers/v1/types";+import { supabase_service } from "../../../services/supabase";+import { normalizeUrl } from "../../../lib/canonical-url";++export async function getCachedDocs(urls: string[], cacheKey: string): Promise<Document[]> {+ const normalizedUrls = urls.map(normalizeUrl);+ const { data, error } = await supabase_service+ .from('cached_scrapes')+ .select('doc')+ .in('url', normalizedUrls)+ .eq('cache_key', cacheKey);++ if (error) {+ console.error('Error fetching cached docs:', error);+ return [];+ }++ const uniqueDocs = new Map<string, Document>();+ data.forEach((res: any) => {+ const doc = JSON.parse(JSON.stringify(res.doc)) as Document;+ const docKey = `${doc.metadata.url}-${cacheKey}`;+ if (!uniqueDocs.has(docKey)) {+ uniqueDocs.set(docKey, doc);+ }+ });++ return Array.from(uniqueDocs.values());+}++export async function saveCachedDocs(docs: Document[], cacheKey: string): Promise<void> {+ for (const doc of docs) {+ if (!doc.metadata.url) {+ throw new Error("Document has no URL");+ }++ const normalizedUrl = normalizeUrl(doc.metadata.url);+ const { data, error } = await supabase_service+ .from('cached_scrapes')+ .select('url')+ .eq('url', normalizedUrl)+ .eq('cache_key', cacheKey);++ if (error) {+ console.error('Error checking existing cached doc:', error);+ continue;+ }++ if (data.length === 0) {+ const { error: upsertError } = await supabase_service+ .from('cached_scrapes')+ .upsert({+ url: normalizedUrl,+ doc: doc,+ cache_key: cacheKey,+ });++ if (upsertError) {+ console.error('Error saving cached doc:', upsertError);+ }+ }+ }+}