Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: implement summary format for v1 and v2 APIs (#1856)
devin-ai-integration[bot]febfa7834d50
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex a28b469ca..577a0ee38 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -944,6 +944,89 @@ describe("Scrape tests", () => {expect(typeof response.json.is_open_source).toBe("boolean");}, scrapeTimeout);});++ describe("Summary format", () => {+ it.concurrent("generates basic summary with no options required", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["summary"],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.summary).toBeDefined();+ expect(typeof response.summary).toBe("string");+ expect(response.summary!.length).toBeGreaterThan(10);+ }, scrapeTimeout);++ it.concurrent("works with markdown format", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", "summary"],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.summary).toBeDefined();+ expect(typeof response.summary).toBe("string");+ expect(response.markdown).toBeDefined();+ }, scrapeTimeout);++ it.concurrent("works alongside extract format", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["summary", "extract"],+ extract: {+ mode: "llm",+ prompt: "Extract the company name",+ schema: {+ type: "object",+ properties: {+ company_name: { type: "string" }+ }+ }+ },+ timeout: scrapeTimeout,+ }, identity);++ expect(response.summary).toBeDefined();+ expect(typeof response.summary).toBe("string");+ expect(response.extract).toBeDefined();+ }, scrapeTimeout);++ it.concurrent("works alongside json format", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["summary", "json"],+ jsonOptions: {+ mode: "llm",+ prompt: "Extract company info as JSON",+ schema: {+ type: "object",+ properties: {+ name: { type: "string" }+ }+ }+ },+ timeout: scrapeTimeout,+ }, identity);++ expect(response.summary).toBeDefined();+ expect(typeof response.summary).toBe("string");+ expect(response.json).toBeDefined();+ }, scrapeTimeout);++ it.concurrent("works with multiple formats", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", "html", "summary"],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.summary).toBeDefined();+ expect(typeof response.summary).toBe("string");+ expect(response.markdown).toBeDefined();+ expect(response.html).toBeDefined();+ }, scrapeTimeout);+ });}it.concurrent("sourceURL stays unnormalized", async () => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex cdf08a345..3cd657c91 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -36,6 +36,7 @@ export type Format =| "screenshot@fullPage"| "extract"| "json"+ | "summary"| "changeTracking";export const url = z.preprocess(@@ -292,6 +293,7 @@ const baseScrapeOptions = z"screenshot@fullPage","extract","json",+ "summary","changeTracking",]).array()@@ -830,6 +832,7 @@ export type Document = {screenshot?: string;extract?: any;json?: any;+ summary?: string;warning?: string;actions?: {screenshots?: string[];diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex e4674bf5d..daf5371b5 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -37,6 +37,7 @@ export type Format =| "screenshot@fullPage"| "extract"| "json"+ | "summary"| "changeTracking";export const url = z.preprocess(@@ -237,6 +238,7 @@ const baseScrapeOptions = z"screenshot@fullPage","extract","json",+ "summary","changeTracking",]).array()@@ -693,6 +695,7 @@ export type Document = {screenshot?: string;extract?: any;json?: any;+ summary?: string;warning?: string;actions?: {screenshots?: string[];diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 907175d37..91299d654 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -4,7 +4,7 @@ import { Document } from "../../../controllers/v1/types";import { htmlTransform } from "../lib/removeUnwantedElements";import { extractLinks } from "../lib/extractLinks";import { extractMetadata } from "../lib/extractMetadata";-import { performLLMExtract } from "./llmExtract";+import { performLLMExtract, performSummary } from "./llmExtract";import { uploadScreenshot } from "./uploadScreenshot";import { removeBase64Images } from "./removeBase64Images";import { performAgent } from "./agent";@@ -185,6 +185,17 @@ export function coerceFieldsToFormats();}+ if (!formats.has("summary") && document.summary !== undefined) {+ meta.logger.warn(+ "Removed summary from Document because it wasn't in formats -- this is wasteful and indicates a bug.",+ );+ delete document.summary;+ } else if (formats.has("summary") && document.summary === undefined) {+ meta.logger.warn(+ "Request had format summary, but there was no summary field in the result.",+ );+ }+if (!formats.has("changeTracking") && document.changeTracking !== undefined) {meta.logger.warn("Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",@@ -230,6 +241,7 @@ export const transformerStack: Transformer[] = [uploadScreenshot,...(useIndex ? [sendDocumentToIndex] : []),performLLMExtract,+ performSummary,performAgent,deriveDiff,coerceFieldsToFormats,diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex d96ec3239..77e564a49 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -786,6 +786,62 @@ export async function performLLMExtract(return document;}+export async function performSummary(+ meta: Meta,+ document: Document,+): Promise<Document> {+ if (meta.options.formats.includes("summary")) {+ if (meta.internalOptions.zeroDataRetention) {+ document.warning = "Summary mode is not supported with zero data retention." + (document.warning ? " " + document.warning : "")+ return document;+ }++ const generationOptions: GenerateCompletionsOptions = {+ logger: meta.logger.child({+ method: "performSummary/generateCompletions",+ }),+ options: {+ mode: "llm",+ systemPrompt: "You are a content summarization expert. Analyze the provided content and create a concise, informative summary that captures the key points, main ideas, and essential information. Focus on clarity and brevity while maintaining accuracy.",+ prompt: "Summarize the main content and key points from this page.",+ },+ markdown: document.markdown,+ previousWarning: document.warning,+ model: getModel("gpt-4o-mini", "openai"),+ retryModel: getModel("gpt-4o", "openai"),+ costTrackingOptions: {+ costTracking: meta.costTracking,+ metadata: {+ module: "scrapeURL",+ method: "performSummary",+ },+ },+ };++ const {+ extract: summaryText,+ warning,+ totalUsage,+ model,+ } = await generateCompletions(generationOptions);++ if (warning) {+ document.warning = warning + (document.warning ? " " + document.warning : "");+ }++ meta.logger.info("LLM summary generation token usage", {+ model: model,+ promptTokens: totalUsage.promptTokens,+ completionTokens: totalUsage.completionTokens,+ totalTokens: totalUsage.totalTokens,+ });++ document.summary = typeof summaryText === 'string' ? summaryText : String(summaryText || '');+ }++ return document;+}+export function removeDefaultProperty(schema: any): any {if (typeof schema !== "object" || schema === null) return schema;