Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: implement summary format for v1 and v2 APIs (#1856)

devin-ai-integration[bot]febfa7834d50
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index a28b469ca..577a0ee38 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -944,6 +944,89 @@ describe("Scrape tests", () => {
expect(typeof response.json.is_open_source).toBe("boolean");
}, scrapeTimeout);
});
+
+ describe("Summary format", () => {
+ it.concurrent("generates basic summary with no options required", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["summary"],
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.summary).toBeDefined();
+ expect(typeof response.summary).toBe("string");
+ expect(response.summary!.length).toBeGreaterThan(10);
+ }, scrapeTimeout);
+
+ it.concurrent("works with markdown format", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["markdown", "summary"],
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.summary).toBeDefined();
+ expect(typeof response.summary).toBe("string");
+ expect(response.markdown).toBeDefined();
+ }, scrapeTimeout);
+
+ it.concurrent("works alongside extract format", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["summary", "extract"],
+ extract: {
+ mode: "llm",
+ prompt: "Extract the company name",
+ schema: {
+ type: "object",
+ properties: {
+ company_name: { type: "string" }
+ }
+ }
+ },
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.summary).toBeDefined();
+ expect(typeof response.summary).toBe("string");
+ expect(response.extract).toBeDefined();
+ }, scrapeTimeout);
+
+ it.concurrent("works alongside json format", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["summary", "json"],
+ jsonOptions: {
+ mode: "llm",
+ prompt: "Extract company info as JSON",
+ schema: {
+ type: "object",
+ properties: {
+ name: { type: "string" }
+ }
+ }
+ },
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.summary).toBeDefined();
+ expect(typeof response.summary).toBe("string");
+ expect(response.json).toBeDefined();
+ }, scrapeTimeout);
+
+ it.concurrent("works with multiple formats", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["markdown", "html", "summary"],
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.summary).toBeDefined();
+ expect(typeof response.summary).toBe("string");
+ expect(response.markdown).toBeDefined();
+ expect(response.html).toBeDefined();
+ }, scrapeTimeout);
+ });
}
it.concurrent("sourceURL stays unnormalized", async () => {
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index cdf08a345..3cd657c91 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -36,6 +36,7 @@ export type Format =
| "screenshot@fullPage"
| "extract"
| "json"
+ | "summary"
| "changeTracking";
export const url = z.preprocess(
@@ -292,6 +293,7 @@ const baseScrapeOptions = z
"screenshot@fullPage",
"extract",
"json",
+ "summary",
"changeTracking",
])
.array()
@@ -830,6 +832,7 @@ export type Document = {
screenshot?: string;
extract?: any;
json?: any;
+ summary?: string;
warning?: string;
actions?: {
screenshots?: string[];
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index e4674bf5d..daf5371b5 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -37,6 +37,7 @@ export type Format =
| "screenshot@fullPage"
| "extract"
| "json"
+ | "summary"
| "changeTracking";
export const url = z.preprocess(
@@ -237,6 +238,7 @@ const baseScrapeOptions = z
"screenshot@fullPage",
"extract",
"json",
+ "summary",
"changeTracking",
])
.array()
@@ -693,6 +695,7 @@ export type Document = {
screenshot?: string;
extract?: any;
json?: any;
+ summary?: string;
warning?: string;
actions?: {
screenshots?: string[];
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 907175d37..91299d654 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -4,7 +4,7 @@ import { Document } from "../../../controllers/v1/types";
import { htmlTransform } from "../lib/removeUnwantedElements";
import { extractLinks } from "../lib/extractLinks";
import { extractMetadata } from "../lib/extractMetadata";
-import { performLLMExtract } from "./llmExtract";
+import { performLLMExtract, performSummary } from "./llmExtract";
import { uploadScreenshot } from "./uploadScreenshot";
import { removeBase64Images } from "./removeBase64Images";
import { performAgent } from "./agent";
@@ -185,6 +185,17 @@ export function coerceFieldsToFormats(
);
}
+ if (!formats.has("summary") && document.summary !== undefined) {
+ meta.logger.warn(
+ "Removed summary from Document because it wasn't in formats -- this is wasteful and indicates a bug.",
+ );
+ delete document.summary;
+ } else if (formats.has("summary") && document.summary === undefined) {
+ meta.logger.warn(
+ "Request had format summary, but there was no summary field in the result.",
+ );
+ }
+
if (!formats.has("changeTracking") && document.changeTracking !== undefined) {
meta.logger.warn(
"Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
@@ -230,6 +241,7 @@ export const transformerStack: Transformer[] = [
uploadScreenshot,
...(useIndex ? [sendDocumentToIndex] : []),
performLLMExtract,
+ performSummary,
performAgent,
deriveDiff,
coerceFieldsToFormats,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index d96ec3239..77e564a49 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -786,6 +786,62 @@ export async function performLLMExtract(
return document;
}
+export async function performSummary(
+ meta: Meta,
+ document: Document,
+): Promise<Document> {
+ if (meta.options.formats.includes("summary")) {
+ if (meta.internalOptions.zeroDataRetention) {
+ document.warning = "Summary mode is not supported with zero data retention." + (document.warning ? " " + document.warning : "")
+ return document;
+ }
+
+ const generationOptions: GenerateCompletionsOptions = {
+ logger: meta.logger.child({
+ method: "performSummary/generateCompletions",
+ }),
+ options: {
+ mode: "llm",
+ systemPrompt: "You are a content summarization expert. Analyze the provided content and create a concise, informative summary that captures the key points, main ideas, and essential information. Focus on clarity and brevity while maintaining accuracy.",
+ prompt: "Summarize the main content and key points from this page.",
+ },
+ markdown: document.markdown,
+ previousWarning: document.warning,
+ model: getModel("gpt-4o-mini", "openai"),
+ retryModel: getModel("gpt-4o", "openai"),
+ costTrackingOptions: {
+ costTracking: meta.costTracking,
+ metadata: {
+ module: "scrapeURL",
+ method: "performSummary",
+ },
+ },
+ };
+
+ const {
+ extract: summaryText,
+ warning,
+ totalUsage,
+ model,
+ } = await generateCompletions(generationOptions);
+
+ if (warning) {
+ document.warning = warning + (document.warning ? " " + document.warning : "");
+ }
+
+ meta.logger.info("LLM summary generation token usage", {
+ model: model,
+ promptTokens: totalUsage.promptTokens,
+ completionTokens: totalUsage.completionTokens,
+ totalTokens: totalUsage.totalTokens,
+ });
+
+ document.summary = typeof summaryText === 'string' ? summaryText : String(summaryText || '');
+ }
+
+ return document;
+}
+
export function removeDefaultProperty(schema: any): any {
if (typeof schema !== "object" || schema === null) return schema;