Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrape): split question and highlights formats

mogery05416c7562f5
diff --git a/apps/api/src/__tests__/snips/v2/scrape-query.test.ts b/apps/api/src/__tests__/snips/v2/scrape-query.test.ts
index b088edc1e..22b0356d7 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-query.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-query.test.ts
@@ -38,6 +38,25 @@ describe("Query format", () => {
scrapeTimeout,
);
+ concurrentIf(TEST_PRODUCTION || HAS_AI)(
+ "returns a non-empty answer for a valid question",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ formats: [{ type: "question", question: "What is Firecrawl?" }],
+ },
+ identity,
+ );
+
+ expect(response.answer).toBeDefined();
+ expect(typeof response.answer).toBe("string");
+ expect(response.answer!.length).toBeGreaterThan(0);
+ expect(response.markdown).toBeUndefined();
+ },
+ scrapeTimeout,
+ );
+
concurrentIf(TEST_PRODUCTION || HAS_AI)(
"returns both answer and markdown when formats include markdown and query",
async () => {
@@ -61,6 +80,25 @@ describe("Query format", () => {
scrapeTimeout,
);
+ concurrentIf(TEST_PRODUCTION || HAS_FIREWORKS)(
+ "returns non-empty highlights for a valid highlights query",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ formats: [{ type: "highlights", query: "What is Firecrawl?" }],
+ },
+ identity,
+ );
+
+ expect(response.highlights).toBeDefined();
+ expect(typeof response.highlights).toBe("string");
+ expect(response.highlights!.length).toBeGreaterThan(0);
+ expect(response.answer).toBeUndefined();
+ },
+ scrapeTimeout,
+ );
+
concurrentIf(TEST_PRODUCTION || HAS_FIREWORKS)(
"returns a direct quote answer when query mode is directQuote",
async () => {
@@ -118,4 +156,38 @@ describe("Query format", () => {
},
scrapeTimeout,
);
+
+ it(
+ "rejects question over 10000 characters",
+ async () => {
+ const response = await scrapeWithFailure(
+ {
+ url: "https://firecrawl.dev",
+ formats: [{ type: "question", question: "a".repeat(10001) }],
+ } as any,
+ identity,
+ );
+
+ expect(response.success).toBe(false);
+ expect(response.error).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "rejects highlights query over 10000 characters",
+ async () => {
+ const response = await scrapeWithFailure(
+ {
+ url: "https://firecrawl.dev",
+ formats: [{ type: "highlights", query: "a".repeat(10001) }],
+ } as any,
+ identity,
+ );
+
+ expect(response.success).toBe(false);
+ expect(response.error).toBeDefined();
+ },
+ scrapeTimeout,
+ );
});
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index 7aa29e744..aef9c2067 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -114,6 +114,62 @@ describe("V2 Types Validation", () => {
]);
});
+ it("should accept question format", () => {
+ const input: ScrapeRequestInput = {
+ url: "https://example.com",
+ formats: [{ type: "question", question: "What is Firecrawl?" }],
+ };
+
+ const result = scrapeRequestSchema.parse(input);
+ expect(result.formats).toEqual([
+ { type: "question", question: "What is Firecrawl?" },
+ ]);
+ });
+
+ it("should accept highlights format", () => {
+ const input: ScrapeRequestInput = {
+ url: "https://example.com",
+ formats: [{ type: "highlights", query: "What is Firecrawl?" }],
+ };
+
+ const result = scrapeRequestSchema.parse(input);
+ expect(result.formats).toEqual([
+ { type: "highlights", query: "What is Firecrawl?" },
+ ]);
+ });
+
+ it("should reject invalid question and highlights fields", () => {
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com",
+ formats: [{ type: "question", question: "" }],
+ } satisfies ScrapeRequestInput),
+ ).toThrow();
+
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com",
+ formats: [{ type: "question", prompt: "What is Firecrawl?" } as any],
+ }),
+ ).toThrow();
+
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com",
+ formats: [{ type: "highlights", query: "" }],
+ } satisfies ScrapeRequestInput),
+ ).toThrow();
+
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com",
+ formats: [
+ { type: "highlights", prompt: "What is Firecrawl?" } as any,
+ ],
+ }),
+ ).toThrow();
+ });
+
it("should accept valid scrape request with changeTracking format", () => {
const input: ScrapeRequestInput = {
url: "https://example.com",
@@ -1030,6 +1086,24 @@ describe("V2 Types Validation", () => {
]);
});
+ it("should accept search scrapeOptions with question and highlights formats", () => {
+ const input: SearchRequestInput = {
+ query: "test",
+ scrapeOptions: {
+ formats: [
+ { type: "question", question: "What is Firecrawl?" },
+ { type: "highlights", query: "What is Firecrawl?" },
+ ],
+ },
+ };
+
+ const result = searchRequestSchema.parse(input);
+ expect(result.scrapeOptions?.formats).toEqual([
+ { type: "question", question: "What is Firecrawl?" },
+ { type: "highlights", query: "What is Firecrawl?" },
+ ]);
+ });
+
it("should reject search scrapeOptions query format with invalid mode", () => {
const input: SearchRequestInput = {
query: "test",
@@ -1074,6 +1148,26 @@ describe("V2 Types Validation", () => {
expect(() => searchRequestSchema.parse(input)).toThrow();
});
+
+ it("should reject search scrapeOptions question and highlights values over 10000 characters", () => {
+ expect(() =>
+ searchRequestSchema.parse({
+ query: "test",
+ scrapeOptions: {
+ formats: [{ type: "question", question: "a".repeat(10001) }],
+ },
+ } satisfies SearchRequestInput),
+ ).toThrow();
+
+ expect(() =>
+ searchRequestSchema.parse({
+ query: "test",
+ scrapeOptions: {
+ formats: [{ type: "highlights", query: "a".repeat(10001) }],
+ },
+ } satisfies SearchRequestInput),
+ ).toThrow();
+ });
});
describe("Type inference", () => {
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index d6c7e5f2c..a10fdfd10 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -632,6 +632,8 @@ export async function parseController(
let usedLlm =
!!hasFormatOfType(req.body.formats, "json") ||
!!hasFormatOfType(req.body.formats, "summary") ||
+ !!hasFormatOfType(req.body.formats, "question") ||
+ !!hasFormatOfType(req.body.formats, "highlights") ||
!!hasFormatOfType(req.body.formats, "query");
if (!usedLlm) {
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 1aa301e4d..a96140144 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -422,6 +422,8 @@ export async function scrapeController(
!!hasFormatOfType(req.body.formats, "json") ||
!!hasFormatOfType(req.body.formats, "summary") ||
!!hasFormatOfType(req.body.formats, "branding") ||
+ !!hasFormatOfType(req.body.formats, "question") ||
+ !!hasFormatOfType(req.body.formats, "highlights") ||
!!hasFormatOfType(req.body.formats, "query");
if (!usedLlm) {
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index ecfe24d5f..2b838f087 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -400,6 +400,21 @@ const attributesFormatWithOptions = z.strictObject({
type AttributesFormatWithOptions = z.output<typeof attributesFormatWithOptions>;
+const questionFormatWithOptions = z.strictObject({
+ type: z.literal("question"),
+ question: z.string().min(1).max(10000),
+});
+
+type QuestionFormatWithOptions = z.output<typeof questionFormatWithOptions>;
+
+const highlightsFormatWithOptions = z.strictObject({
+ type: z.literal("highlights"),
+ query: z.string().min(1).max(10000),
+});
+
+type HighlightsFormatWithOptions = z.output<typeof highlightsFormatWithOptions>;
+
+/** @deprecated Use `question` or `highlights` format instead. */
const queryFormatWithOptions = z.strictObject({
type: z.literal("query"),
prompt: z.string().max(10000),
@@ -419,6 +434,8 @@ export type FormatObject =
| ChangeTrackingFormatWithOptions
| ScreenshotFormatWithOptions
| AttributesFormatWithOptions
+ | QuestionFormatWithOptions
+ | HighlightsFormatWithOptions
| QueryFormatWithOptions
| { type: "branding" }
| { type: "audio" };
@@ -529,6 +546,8 @@ const baseScrapeOptions = z.strictObject({
screenshotFormatWithOptions,
attributesFormatWithOptions,
z.strictObject({ type: z.literal("branding") }),
+ questionFormatWithOptions,
+ highlightsFormatWithOptions,
queryFormatWithOptions,
z.strictObject({ type: z.literal("audio") }),
])
@@ -1082,6 +1101,7 @@ export type Document = {
json?: any;
summary?: string;
answer?: string;
+ highlights?: string;
branding?: BrandingProfile;
warning?: string;
attributes?: {
@@ -1784,6 +1804,8 @@ export const searchRequestSchema = z
z.strictObject({ type: z.literal("images") }),
z.strictObject({ type: z.literal("summary") }),
jsonFormatWithOptions,
+ questionFormatWithOptions,
+ highlightsFormatWithOptions,
queryFormatWithOptions,
screenshotFormatWithOptions,
])
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 3e2f2515c..0d5dac8ee 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -122,6 +122,7 @@ interface ImageSearchResult {
url?: string;
position?: number;
answer?: string;
+ highlights?: string;
}
interface NewsSearchResult {
@@ -140,6 +141,7 @@ interface NewsSearchResult {
screenshot?: string;
metadata?: Record<string, any>;
answer?: string;
+ highlights?: string;
}
export interface WebSearchResult {
@@ -156,6 +158,7 @@ export interface WebSearchResult {
screenshot?: string;
metadata?: Record<string, any>;
answer?: string;
+ highlights?: string;
}
export type SearchResultType = "web" | "images" | "news";
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 440e03930..166a91eab 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -75,7 +75,14 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled = Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);
}
- if (hasFormatOfType(options.formats, "query")) {
+ const hasQuestionFormat =
+ hasFormatOfType(options.formats, "question") ||
+ hasFormatOfType(options.formats, "query");
+ if (hasQuestionFormat) {
+ creditsToBeBilled += 4;
+ }
+
+ if (hasFormatOfType(options.formats, "highlights")) {
creditsToBeBilled += 4;
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index eac798f5e..0b3a02f61 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -520,9 +520,17 @@ async function scrapeURLLoopIter(
);
const hasJson = hasFormatOfType(meta.options.formats, "json");
const hasSummary = hasFormatOfType(meta.options.formats, "summary");
+ const hasQuestion = hasFormatOfType(meta.options.formats, "question");
+ const hasHighlights = hasFormatOfType(meta.options.formats, "highlights");
const hasQuery = hasFormatOfType(meta.options.formats, "query");
const needsMarkdown =
- hasMarkdown || hasChangeTracking || hasJson || hasSummary || hasQuery;
+ hasMarkdown ||
+ hasChangeTracking ||
+ hasJson ||
+ hasSummary ||
+ hasQuestion ||
+ hasHighlights ||
+ hasQuery;
let checkMarkdown: string;
const htmlSize = engineResult.html?.length ?? 0;
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 45fd933d7..883ec5873 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -83,7 +83,7 @@ async function deriveMarkdownFromHTML(
// - changeTracking requires markdown
// - json format requires markdown (for LLM extraction)
// - summary format requires markdown (for summarization)
- // - query format requires markdown (for page-level answers)
+ // - question/highlights/query formats require markdown (for page-level answers)
const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");
const hasChangeTracking = hasFormatOfType(
meta.options.formats,
@@ -91,12 +91,16 @@ async function deriveMarkdownFromHTML(
);
const hasJson = hasFormatOfType(meta.options.formats, "json");
const hasSummary = hasFormatOfType(meta.options.formats, "summary");
+ const hasQuestion = hasFormatOfType(meta.options.formats, "question");
+ const hasHighlights = hasFormatOfType(meta.options.formats, "highlights");
const hasQuery = hasFormatOfType(meta.options.formats, "query");
if (
!hasMarkdown &&
!hasChangeTracking &&
!hasJson &&
!hasSummary &&
+ !hasQuestion &&
+ !hasHighlights &&
!hasQuery &&
!meta.options.onlyCleanContent
) {
@@ -317,7 +321,13 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {
const hasScreenshot = hasFormatOfType(meta.options.formats, "screenshot");
const hasSummary = hasFormatOfType(meta.options.formats, "summary");
const hasBranding = hasFormatOfType(meta.options.formats, "branding");
- const hasQueryFormat = hasFormatOfType(meta.options.formats, "query");
+ const hasQuestionFormat = hasFormatOfType(meta.options.formats, "question");
+ const hasHighlightsFormat = hasFormatOfType(
+ meta.options.formats,
+ "highlights",
+ );
+ const hasLegacyQueryFormat = hasFormatOfType(meta.options.formats, "query");
+ const hasAnswerFormat = hasQuestionFormat || hasLegacyQueryFormat;
if (!hasMarkdown && document.markdown !== undefined) {
delete document.markdown;
@@ -432,14 +442,25 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {
);
}
- if (!hasQueryFormat && document.answer !== undefined) {
+ if (!hasAnswerFormat && document.answer !== undefined) {
meta.logger.warn(
- "Removed answer from Document because query wasn't in formats -- this is wasteful and indicates a bug.",
+ "Removed answer from Document because question/query wasn't in formats -- this is wasteful and indicates a bug.",
);
delete document.answer;
- } else if (hasQueryFormat && document.answer === undefined) {
+ } else if (hasAnswerFormat && document.answer === undefined) {
+ meta.logger.warn(
+ "Request had format question/query, but there was no answer field in the result.",
+ );
+ }
+
+ if (!hasHighlightsFormat && document.highlights !== undefined) {
+ meta.logger.warn(
+ "Removed highlights from Document because highlights wasn't in formats -- this is wasteful and indicates a bug.",
+ );
+ delete document.highlights;
+ } else if (hasHighlightsFormat && document.highlights === undefined) {
meta.logger.warn(
- "Request had format query, but there was no answer field in the result.",
+ "Request had format highlights, but there was no highlights field in the result.",
);
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/query.ts b/apps/api/src/scraper/scrapeURL/transformers/query.ts
index b6cb0d735..ca09d6981 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/query.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/query.ts
@@ -1,7 +1,7 @@
import { generateText } from "ai";
import * as marked from "marked";
import { decode as decodeHtmlEntities } from "he";
-import { Document } from "../../../controllers/v2/types";
+import { Document, FormatObject } from "../../../controllers/v2/types";
import { Meta } from "..";
import { getModel } from "../../../lib/generic-ai";
import { hasFormatOfType } from "../../../lib/format-utils";
@@ -475,8 +475,12 @@ export async function performQuery(
meta: Meta,
document: Document,
): Promise<Document> {
- const queryFormat = hasFormatOfType(meta.options.formats, "query");
- if (!queryFormat) {
+ const answerFormat = meta.options.formats?.find(
+ (format): format is Extract<FormatObject, { type: "question" | "query" }> =>
+ format.type === "question" || format.type === "query",
+ );
+ const highlightsFormat = hasFormatOfType(meta.options.formats, "highlights");
+ if (!answerFormat && !highlightsFormat) {
return document;
}
@@ -505,30 +509,40 @@ export async function performQuery(
const pageUrl = meta.url ?? document.metadata?.sourceURL ?? "";
- let answer: string | null;
+ if (answerFormat) {
+ const prompt =
+ answerFormat.type === "question"
+ ? answerFormat.question
+ : answerFormat.prompt;
+ const answer =
+ answerFormat.type === "query" && answerFormat.mode === "directQuote"
+ ? await performDirectQuoteQuery(meta, document, prompt, markdown)
+ : await performFreeformQuery(meta, prompt, markdown, pageUrl);
+
+ if (answer !== null) {
+ document.answer = answer;
+ } else {
+ document.warning =
+ "Query generation failed after all models." +
+ (document.warning ? " " + document.warning : "");
+ }
+ }
- if (queryFormat.mode === "directQuote") {
- answer = await performDirectQuoteQuery(
+ if (highlightsFormat) {
+ const highlights = await performDirectQuoteQuery(
meta,
document,
- queryFormat.prompt,
- markdown,
- );
- } else {
- answer = await performFreeformQuery(
- meta,
- queryFormat.prompt,
+ highlightsFormat.query,
markdown,
- pageUrl,
);
- }
- if (answer !== null) {
- document.answer = answer;
- } else {
- document.warning =
- "Query generation failed after all models." +
- (document.warning ? " " + document.warning : "");
+ if (highlights !== null) {
+ document.highlights = highlights;
+ } else {
+ document.warning =
+ "Highlights generation failed after all models." +
+ (document.warning ? " " + document.warning : "");
+ }
}
return document;
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
index 7c1f20ec5..492f4c22d 100644
--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
@@ -259,6 +259,27 @@ public class ModelsTests
Assert.Contains("\"mode\":\"directQuote\"", json);
}
+ [Fact]
+ public void QuestionAndHighlightsFormats_SerializeCorrectly()
+ {
+ var question = new QuestionFormat
+ {
+ Question = "What is Firecrawl?"
+ };
+ var highlights = new HighlightsFormat
+ {
+ Query = "What is Firecrawl?"
+ };
+
+ var questionJson = JsonSerializer.Serialize(question, JsonOptions);
+ Assert.Contains("\"type\":\"question\"", questionJson);
+ Assert.Contains("\"question\":\"What is Firecrawl?\"", questionJson);
+
+ var highlightsJson = JsonSerializer.Serialize(highlights, JsonOptions);
+ Assert.Contains("\"type\":\"highlights\"", highlightsJson);
+ Assert.Contains("\"query\":\"What is Firecrawl?\"", highlightsJson);
+ }
+
[Fact]
public void WebhookConfig_SerializesCorrectly()
{
diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
index 4962ae390..efdeaea3d 100644
--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
@@ -8,7 +8,7 @@
<!-- NuGet package metadata -->
<PackageId>firecrawl-sdk</PackageId>
- <Version>1.3.0</Version>
+ <Version>1.3.1</Version>
<Authors>Firecrawl</Authors>
<Company>Firecrawl</Company>
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
diff --git a/apps/dot-net-sdk/Firecrawl/Models/Document.cs b/apps/dot-net-sdk/Firecrawl/Models/Document.cs
index 754ad665a..5dd68eb44 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/Document.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/Document.cs
@@ -40,6 +40,12 @@ public class Document
[JsonPropertyName("actions")]
public object? Actions { get; set; }
+ [JsonPropertyName("answer")]
+ public string? Answer { get; set; }
+
+ [JsonPropertyName("highlights")]
+ public string? Highlights { get; set; }
+
[JsonPropertyName("warning")]
public string? Warning { get; set; }
diff --git a/apps/dot-net-sdk/Firecrawl/Models/HighlightsFormat.cs b/apps/dot-net-sdk/Firecrawl/Models/HighlightsFormat.cs
new file mode 100644
index 000000000..ee40ceddc
--- /dev/null
+++ b/apps/dot-net-sdk/Firecrawl/Models/HighlightsFormat.cs
@@ -0,0 +1,15 @@
+using System.Text.Json.Serialization;
+
+namespace Firecrawl.Models;
+
+/// <summary>
+/// Highlights format specification for use in ScrapeOptions.Formats.
+/// </summary>
+public class HighlightsFormat
+{
+ [JsonPropertyName("type")]
+ public string Type { get; } = "highlights";
+
+ [JsonPropertyName("query")]
+ public required string Query { get; set; }
+}
diff --git a/apps/dot-net-sdk/Firecrawl/Models/QueryFormat.cs b/apps/dot-net-sdk/Firecrawl/Models/QueryFormat.cs
index 44cd93920..9aa46e16a 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/QueryFormat.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/QueryFormat.cs
@@ -3,8 +3,9 @@ using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
-/// Query format specification for use in ScrapeOptions.Formats.
+/// Deprecated query format specification for use in ScrapeOptions.Formats.
/// </summary>
+[Obsolete("Use QuestionFormat or HighlightsFormat instead.")]
public class QueryFormat
{
public const string FreeformMode = "freeform";
diff --git a/apps/dot-net-sdk/Firecrawl/Models/QuestionFormat.cs b/apps/dot-net-sdk/Firecrawl/Models/QuestionFormat.cs
new file mode 100644
index 000000000..070e9d7f1
--- /dev/null
+++ b/apps/dot-net-sdk/Firecrawl/Models/QuestionFormat.cs
@@ -0,0 +1,15 @@
+using System.Text.Json.Serialization;
+
+namespace Firecrawl.Models;
+
+/// <summary>
+/// Question format specification for use in ScrapeOptions.Formats.
+/// </summary>
+public class QuestionFormat
+{
+ [JsonPropertyName("type")]
+ public string Type { get; } = "question";
+
+ [JsonPropertyName("question")]
+ public required string Question { get; set; }
+}
diff --git a/apps/dot-net-sdk/Firecrawl/Models/SearchData.cs b/apps/dot-net-sdk/Firecrawl/Models/SearchData.cs
index 7c44b2a38..d2092220c 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/SearchData.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/SearchData.cs
@@ -42,6 +42,9 @@ public class WebSearchHit
[JsonPropertyName("answer")]
public string? Answer { get; set; }
+
+ [JsonPropertyName("highlights")]
+ public string? Highlights { get; set; }
}
/// <summary>
@@ -90,6 +93,9 @@ public class NewsSearchHit
[JsonPropertyName("answer")]
public string? Answer { get; set; }
+
+ [JsonPropertyName("highlights")]
+ public string? Highlights { get; set; }
}
/// <summary>
@@ -117,6 +123,9 @@ public class ImageSearchHit
[JsonPropertyName("answer")]
public string? Answer { get; set; }
+
+ [JsonPropertyName("highlights")]
+ public string? Highlights { get; set; }
}
/// <summary>
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index a3ee2b40f..ca35773f2 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -1028,7 +1028,7 @@ defmodule Firecrawl do
actions: [type: {:list, :any}, doc: "Actions to perform on the page before grabbing the content"],
block_ads: [type: :boolean, doc: "Enables ad-blocking and cookie popup blocking."],
exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],
- formats: [type: {:list, :any}, doc: "Output formats to include in the response. You can specify one or more formats, either as strings (e.g., `'markdown'`) or as objects with additional options (e.g., `{ type: 'json', schema: {...} }`). Some formats require specific options to be set. Example: `['markdown', { type: 'json', schema: {...} }]`."],
+ formats: [type: {:list, :any}, doc: "Output formats to include in the response. You can specify one or more formats, either as strings (e.g., `'markdown'`) or as objects with additional options (e.g., `{ type: 'json', schema: {...} }`, `{ type: 'question', question: '...' }`, `{ type: 'highlights', query: '...' }`). The legacy `{ type: 'query', prompt: '...', mode: 'freeform' | 'directQuote' }` format is deprecated."],
headers: [type: :any, doc: "Headers to send with the request. Can be used to send cookies, user-agent, etc."],
include_tags: [type: {:list, :string}, doc: "Tags to include in the output."],
location: [type: :keyword_list, doc: "Location settings for the request. When specified, this will use an appropriate proxy if available and emulate the corresponding language and timezone settings. Defaults to 'US' if not specified."],
@@ -1093,7 +1093,7 @@ defmodule Firecrawl do
actions: [type: {:list, :any}, doc: "Actions to perform on the page before grabbing the content"],
block_ads: [type: :boolean, doc: "Enables ad-blocking and cookie popup blocking."],
exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],
- formats: [type: {:list, :any}, doc: "Output formats to include in the response. You can specify one or more formats, either as strings (e.g., `'markdown'`) or as objects with additional options (e.g., `{ type: 'json', schema: {...} }`). Some formats require specific options to be set. Example: `['markdown', { type: 'json', schema: {...} }]`."],
+ formats: [type: {:list, :any}, doc: "Output formats to include in the response. You can specify one or more formats, either as strings (e.g., `'markdown'`) or as objects with additional options (e.g., `{ type: 'json', schema: {...} }`, `{ type: 'question', question: '...' }`, `{ type: 'highlights', query: '...' }`). The legacy `{ type: 'query', prompt: '...', mode: 'freeform' | 'directQuote' }` format is deprecated."],
headers: [type: :any, doc: "Headers to send with the request. Can be used to send cookies, user-agent, etc."],
include_tags: [type: {:list, :string}, doc: "Tags to include in the output."],
location: [type: :keyword_list, doc: "Location settings for the request. When specified, this will use an appropriate proxy if available and emulate the corresponding language and timezone settings. Defaults to 'US' if not specified."],
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index e15468af2..f66938714 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.3.0"
+ @version "1.3.1"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go
index 5877eadea..a4c14b0ce 100644
--- a/apps/go-sdk/models.go
+++ b/apps/go-sdk/models.go
@@ -16,6 +16,8 @@ type Document struct {
Audio string `json:"audio,omitempty"`
Attributes []map[string]interface{} `json:"attributes,omitempty"`
Actions map[string]interface{} `json:"actions,omitempty"`
+ Answer string `json:"answer,omitempty"`
+ Highlights string `json:"highlights,omitempty"`
Warning string `json:"warning,omitempty"`
ChangeTracking map[string]interface{} `json:"changeTracking,omitempty"`
Branding map[string]interface{} `json:"branding,omitempty"`
diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.go
index 4f610553d..5f1116535 100644
--- a/apps/go-sdk/options.go
+++ b/apps/go-sdk/options.go
@@ -2,7 +2,7 @@ package firecrawl
import "encoding/json"
-// QueryFormatMode selects how query answers are generated.
+// QueryFormatMode selects how deprecated query answers are generated.
type QueryFormatMode string
const (
@@ -10,7 +10,45 @@ const (
QueryModeDirectQuote QueryFormatMode = "directQuote"
)
+// QuestionFormat asks a question about page content.
+type QuestionFormat struct {
+ Question string `json:"question"`
+}
+
+// MarshalJSON always emits the API-required question format type.
+func (q QuestionFormat) MarshalJSON() ([]byte, error) {
+ type questionFormat struct {
+ Type string `json:"type"`
+ Question string `json:"question"`
+ }
+
+ return json.Marshal(questionFormat{
+ Type: "question",
+ Question: q.Question,
+ })
+}
+
+// HighlightsFormat extracts direct highlights from page content.
+type HighlightsFormat struct {
+ Query string `json:"query"`
+}
+
+// MarshalJSON always emits the API-required highlights format type.
+func (h HighlightsFormat) MarshalJSON() ([]byte, error) {
+ type highlightsFormat struct {
+ Type string `json:"type"`
+ Query string `json:"query"`
+ }
+
+ return json.Marshal(highlightsFormat{
+ Type: "highlights",
+ Query: h.Query,
+ })
+}
+
// QueryFormat asks a question about page content.
+//
+// Deprecated: use QuestionFormat or HighlightsFormat instead.
type QueryFormat struct {
Prompt string `json:"prompt"`
Mode QueryFormatMode `json:"mode,omitempty"`
@@ -56,7 +94,7 @@ type ScrapeOptions struct {
JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
}
-// MarshalJSON preserves string formats while allowing object formats such as QueryFormat.
+// MarshalJSON preserves string formats while allowing object formats such as QuestionFormat.
func (o ScrapeOptions) MarshalJSON() ([]byte, error) {
type scrapeOptions ScrapeOptions
payload := struct {
diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.go
index dbb4f0b5c..9416ec487 100644
--- a/apps/go-sdk/options_test.go
+++ b/apps/go-sdk/options_test.go
@@ -29,6 +29,28 @@ func TestScrapeOptionsSerializesQueryFormatMode(t *testing.T) {
}
}
+func TestScrapeOptionsSerializesQuestionAndHighlightsFormats(t *testing.T) {
+ payload, err := json.Marshal(ScrapeOptions{
+ FormatOptions: []interface{}{
+ QuestionFormat{Question: "What is Firecrawl?"},
+ HighlightsFormat{Query: "What is Firecrawl?"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Marshal ScrapeOptions: %v", err)
+ }
+
+ jsonBody := string(payload)
+ for _, want := range []string{
+ `{"type":"question","question":"What is Firecrawl?"}`,
+ `{"type":"highlights","query":"What is Firecrawl?"}`,
+ } {
+ if !strings.Contains(jsonBody, want) {
+ t.Fatalf("serialized formats = %s, want to contain %s", jsonBody, want)
+ }
+ }
+}
+
func TestScrapeOptionsPreservesStringFormats(t *testing.T) {
payload, err := json.Marshal(ScrapeOptions{
Formats: []string{"markdown"},
diff --git a/apps/go-sdk/parse.go b/apps/go-sdk/parse.go
index 6329f2f7a..8ccd4071a 100644
--- a/apps/go-sdk/parse.go
+++ b/apps/go-sdk/parse.go
@@ -73,7 +73,7 @@ type ParseOptions struct {
JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
}
-// MarshalJSON preserves string formats while allowing object formats such as QueryFormat.
+// MarshalJSON preserves string formats while allowing object formats such as QuestionFormat.
func (o ParseOptions) MarshalJSON() ([]byte, error) {
type parseOptions ParseOptions
payload := struct {
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index afafa732e..702528172 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.2.1"
+const Version = "1.2.2"
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index 3479bb1e7..8df199421 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.5.0"
+version = "1.5.1"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
index 0fc75145d..b61670d2a 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
@@ -22,6 +22,8 @@ public class Document {
private String audio;
private List<Map<String, Object>> attributes;
private Map<String, Object> actions;
+ private String answer;
+ private String highlights;
private String warning;
private Map<String, Object> changeTracking;
private Map<String, Object> branding;
@@ -38,6 +40,8 @@ public class Document {
public String getAudio() { return audio; }
public List<Map<String, Object>> getAttributes() { return attributes; }
public Map<String, Object> getActions() { return actions; }
+ public String getAnswer() { return answer; }
+ public String getHighlights() { return highlights; }
public String getWarning() { return warning; }
public Map<String, Object> getChangeTracking() { return changeTracking; }
public Map<String, Object> getBranding() { return branding; }
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/HighlightsFormat.java b/apps/java-sdk/src/main/java/com/firecrawl/models/HighlightsFormat.java
new file mode 100644
index 000000000..5bdc8def1
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/HighlightsFormat.java
@@ -0,0 +1,35 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+/**
+ * Highlights format for extracting direct highlights from page content.
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class HighlightsFormat {
+
+ private final String type = "highlights";
+ private String query;
+
+ private HighlightsFormat() {}
+
+ public String getType() { return type; }
+ public String getQuery() { return query; }
+
+ public static Builder builder() { return new Builder(); }
+
+ public static final class Builder {
+ private String query;
+
+ private Builder() {}
+
+ /** Query used to select highlights from the page content. */
+ public Builder query(String query) { this.query = query; return this; }
+
+ public HighlightsFormat build() {
+ HighlightsFormat f = new HighlightsFormat();
+ f.query = this.query;
+ return f;
+ }
+ }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/QueryFormat.java b/apps/java-sdk/src/main/java/com/firecrawl/models/QueryFormat.java
index 195281897..6c5773dee 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/QueryFormat.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/QueryFormat.java
@@ -4,8 +4,11 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonValue;
/**
- * Query format for asking a question about page content.
+ * Deprecated query format for asking a question about page content.
+ *
+ * @deprecated Use {@link QuestionFormat} or {@link HighlightsFormat} instead.
*/
+@Deprecated
@JsonInclude(JsonInclude.Include.NON_NULL)
public class QueryFormat {
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/QuestionFormat.java b/apps/java-sdk/src/main/java/com/firecrawl/models/QuestionFormat.java
new file mode 100644
index 000000000..bcf19e346
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/QuestionFormat.java
@@ -0,0 +1,35 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+/**
+ * Question format for asking a question about page content.
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class QuestionFormat {
+
+ private final String type = "question";
+ private String question;
+
+ private QuestionFormat() {}
+
+ public String getType() { return type; }
+ public String getQuestion() { return question; }
+
+ public static Builder builder() { return new Builder(); }
+
+ public static final class Builder {
+ private String question;
+
+ private Builder() {}
+
+ /** Question to answer from the page content. */
+ public Builder question(String question) { this.question = question; return this; }
+
+ public QuestionFormat build() {
+ QuestionFormat f = new QuestionFormat();
+ f.question = this.question;
+ return f;
+ }
+ }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
index 440af580d..700e29179 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
@@ -109,7 +109,7 @@ public class ScrapeOptions {
/**
* Output formats to request. Accepts strings like "markdown", "html", "rawHtml",
* "links", "screenshot", "json", "audio", etc., or format configuration maps/objects for
- * advanced formats (e.g., JsonFormat, QueryFormat).
+ * advanced formats (e.g., JsonFormat, QuestionFormat, HighlightsFormat).
*/
public Builder formats(List<Object> formats) { this.formats = formats; return this; }
diff --git a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java
index 04a68737e..1d7cc74af 100644
--- a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java
+++ b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java
@@ -80,6 +80,26 @@ class FirecrawlClientTest {
assertFalse(options.getMobile());
}
+ @Test
+ void testQuestionAndHighlightsFormats() {
+ QuestionFormat questionFormat = QuestionFormat.builder()
+ .question("What is Firecrawl?")
+ .build();
+ HighlightsFormat highlightsFormat = HighlightsFormat.builder()
+ .query("What is Firecrawl?")
+ .build();
+
+ ScrapeOptions options = ScrapeOptions.builder()
+ .formats(List.of(questionFormat, highlightsFormat))
+ .build();
+
+ assertEquals(List.of(questionFormat, highlightsFormat), options.getFormats());
+ assertEquals("question", questionFormat.getType());
+ assertEquals("What is Firecrawl?", questionFormat.getQuestion());
+ assertEquals("highlights", highlightsFormat.getType());
+ assertEquals("What is Firecrawl?", highlightsFormat.getQuery());
+ }
+
@Test
void testCrawlOptionsBuilder() {
CrawlOptions options = CrawlOptions.builder()
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index f88ddd72b..dc9fd677d 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.22.0",
+ "version": "4.22.1",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
index 83a35546a..83b8795fb 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
@@ -56,6 +56,27 @@ describe("v2 utils: validation", () => {
expect((formats[0] as any).viewport).toEqual({ width: 800, height: 600 });
});
+ test("ensureValidFormats: accepts question, highlights, and deprecated query formats", () => {
+ const formats: FormatOption[] = [
+ { type: "question", question: "What is Firecrawl?" },
+ { type: "highlights", query: "What is Firecrawl?" },
+ { type: "query", prompt: "What is Firecrawl?", mode: "directQuote" },
+ ];
+ expect(() => ensureValidFormats(formats)).not.toThrow();
+ });
+
+ test("ensureValidFormats: validates question, highlights, and deprecated query fields", () => {
+ expect(() =>
+ ensureValidFormats([{ type: "question", question: "" } as any]),
+ ).toThrow(/question format requires/i);
+ expect(() =>
+ ensureValidFormats([{ type: "highlights", query: "" } as any]),
+ ).toThrow(/highlights format requires/i);
+ expect(() =>
+ ensureValidFormats([{ type: "query", prompt: "p", mode: "quoted" } as any]),
+ ).toThrow(/query format mode/i);
+ });
+
test("ensureValidScrapeOptions: leaves parsers untouched", () => {
const options = { parsers: ["pdf", "images"] as string[] } as any;
const before = [...options.parsers];
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index a07232dc9..0b38774ad 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -52,6 +52,17 @@ export interface AttributesFormat extends Format {
}>;
}
+export interface QuestionFormat {
+ type: 'question';
+ question: string;
+}
+
+export interface HighlightsFormat {
+ type: 'highlights';
+ query: string;
+}
+
+/** @deprecated Use QuestionFormat or HighlightsFormat instead. */
export interface QueryFormat {
type: 'query';
prompt: string;
@@ -65,6 +76,8 @@ export type FormatOption =
| ChangeTrackingFormat
| ScreenshotFormat
| AttributesFormat
+ | QuestionFormat
+ | HighlightsFormat
| QueryFormat;
export type ParseFormatString = Exclude<
@@ -81,6 +94,8 @@ export type ParseFormatOption =
| ParseFormat
| JsonFormat
| AttributesFormat
+ | QuestionFormat
+ | HighlightsFormat
| QueryFormat;
export interface LocationConfig {
@@ -450,6 +465,7 @@ export interface Document {
}>;
actions?: Record<string, unknown>;
answer?: string;
+ highlights?: string;
warning?: string;
changeTracking?: Record<string, unknown>;
branding?: BrandingProfile;
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
index afb91a8cc..b8082268f 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
@@ -4,6 +4,9 @@ import {
type JsonFormat,
type ParseFormatOption,
type ParseOptions,
+ type QuestionFormat,
+ type HighlightsFormat,
+ type QueryFormat,
type ScrapeOptions,
type ScreenshotFormat,
} from "../types";
@@ -49,6 +52,30 @@ export function ensureValidFormats(formats?: FormatOption[]): void {
}
continue;
}
+ if ((fmt as QuestionFormat).type === "question") {
+ const q = fmt as QuestionFormat;
+ if (typeof q.question !== "string" || q.question.trim().length === 0) {
+ throw new Error("question format requires a non-empty 'question' string");
+ }
+ continue;
+ }
+ if ((fmt as HighlightsFormat).type === "highlights") {
+ const h = fmt as HighlightsFormat;
+ if (typeof h.query !== "string" || h.query.trim().length === 0) {
+ throw new Error("highlights format requires a non-empty 'query' string");
+ }
+ continue;
+ }
+ if ((fmt as QueryFormat).type === "query") {
+ const q = fmt as QueryFormat;
+ if (typeof q.prompt !== "string" || q.prompt.trim().length === 0) {
+ throw new Error("query format requires a non-empty 'prompt' string");
+ }
+ if (q.mode != null && q.mode !== "freeform" && q.mode !== "directQuote") {
+ throw new Error("query format mode must be 'freeform' or 'directQuote'");
+ }
+ continue;
+ }
if ((fmt as ScreenshotFormat).type === "screenshot") {
// no-op; already camelCase; validate numeric fields if present
const s = fmt as ScreenshotFormat;
@@ -116,6 +143,31 @@ export function ensureValidParseFormats(formats?: ParseFormatOption[]): void {
"The SDK will automatically convert Zod schemas to JSON Schema format."
);
}
+ continue;
+ }
+
+ if ((fmt as QuestionFormat).type === "question") {
+ const q = fmt as QuestionFormat;
+ if (typeof q.question !== "string" || q.question.trim().length === 0) {
+ throw new Error("question format requires a non-empty 'question' string");
+ }
+ continue;
+ }
+ if ((fmt as HighlightsFormat).type === "highlights") {
+ const h = fmt as HighlightsFormat;
+ if (typeof h.query !== "string" || h.query.trim().length === 0) {
+ throw new Error("highlights format requires a non-empty 'query' string");
+ }
+ continue;
+ }
+ if ((fmt as QueryFormat).type === "query") {
+ const q = fmt as QueryFormat;
+ if (typeof q.prompt !== "string" || q.prompt.trim().length === 0) {
+ throw new Error("query format requires a non-empty 'prompt' string");
+ }
+ if (q.mode != null && q.mode !== "freeform" && q.mode !== "directQuote") {
+ throw new Error("query format mode must be 'freeform' or 'directQuote'");
+ }
}
}
}
diff --git a/apps/php-sdk/src/Models/Document.php b/apps/php-sdk/src/Models/Document.php
index ad9f021b0..994de0661 100644
--- a/apps/php-sdk/src/Models/Document.php
+++ b/apps/php-sdk/src/Models/Document.php
@@ -28,6 +28,8 @@ final class Document
private readonly ?string $audio = null,
private readonly ?array $attributes = null,
private readonly ?array $actions = null,
+ private readonly ?string $answer = null,
+ private readonly ?string $highlights = null,
private readonly ?string $warning = null,
private readonly ?array $changeTracking = null,
private readonly ?array $branding = null,
@@ -49,6 +51,8 @@ final class Document
audio: $data['audio'] ?? null,
attributes: $data['attributes'] ?? null,
actions: $data['actions'] ?? null,
+ answer: $data['answer'] ?? null,
+ highlights: $data['highlights'] ?? null,
warning: $data['warning'] ?? null,
changeTracking: $data['changeTracking'] ?? null,
branding: $data['branding'] ?? null,
@@ -125,6 +129,16 @@ final class Document
return $this->warning;
}
+ public function getAnswer(): ?string
+ {
+ return $this->answer;
+ }
+
+ public function getHighlights(): ?string
+ {
+ return $this->highlights;
+ }
+
/** @return array<string, mixed>|null */
public function getChangeTracking(): ?array
{
diff --git a/apps/php-sdk/src/Models/HighlightsFormat.php b/apps/php-sdk/src/Models/HighlightsFormat.php
new file mode 100644
index 000000000..fd47f348c
--- /dev/null
+++ b/apps/php-sdk/src/Models/HighlightsFormat.php
@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Firecrawl\Models;
+
+final class HighlightsFormat
+{
+ private function __construct(
+ private readonly string $query,
+ ) {}
+
+ public static function with(string $query): self
+ {
+ return new self($query);
+ }
+
+ /** @return array<string, mixed> */
+ public function toArray(): array
+ {
+ return [
+ 'type' => 'highlights',
+ 'query' => $this->query,
+ ];
+ }
+
+ public function getQuery(): string
+ {
+ return $this->query;
+ }
+}
diff --git a/apps/php-sdk/src/Models/ParseOptions.php b/apps/php-sdk/src/Models/ParseOptions.php
index ec1d458aa..89e03faff 100644
--- a/apps/php-sdk/src/Models/ParseOptions.php
+++ b/apps/php-sdk/src/Models/ParseOptions.php
@@ -23,7 +23,7 @@ final class ParseOptions
];
/**
- * @param list<string|JsonFormat|QueryFormat>|null $formats
+ * @param list<string|JsonFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats
* @param array<string, string>|null $headers
* @param list<string>|null $includeTags
* @param list<string>|null $excludeTags
@@ -45,7 +45,7 @@ final class ParseOptions
) {}
/**
- * @param list<string|JsonFormat|QueryFormat>|null $formats
+ * @param list<string|JsonFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats
* @param array<string, string>|null $headers
* @param list<string>|null $includeTags
* @param list<string>|null $excludeTags
@@ -105,8 +105,13 @@ final class ParseOptions
if ($this->formats !== null) {
$data['formats'] = array_map(
- fn (string|JsonFormat|QueryFormat $f): string|array =>
- $f instanceof JsonFormat || $f instanceof QueryFormat ? $f->toArray() : $f,
+ fn (string|JsonFormat|QuestionFormat|HighlightsFormat|QueryFormat $f): string|array =>
+ (
+ $f instanceof JsonFormat
+ || $f instanceof QuestionFormat
+ || $f instanceof HighlightsFormat
+ || $f instanceof QueryFormat
+ ) ? $f->toArray() : $f,
$this->formats,
);
}
@@ -142,13 +147,22 @@ final class ParseOptions
if ($fmt instanceof JsonFormat) {
return 'json';
}
+ if ($fmt instanceof QuestionFormat) {
+ return 'question';
+ }
+ if ($fmt instanceof HighlightsFormat) {
+ return 'highlights';
+ }
+ if ($fmt instanceof QueryFormat) {
+ return 'query';
+ }
if (is_array($fmt) && isset($fmt['type']) && is_string($fmt['type'])) {
return $fmt['type'];
}
return null;
}
- /** @return list<string|JsonFormat>|null */
+ /** @return list<string|JsonFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null */
public function getFormats(): ?array
{
return $this->formats;
diff --git a/apps/php-sdk/src/Models/QueryFormat.php b/apps/php-sdk/src/Models/QueryFormat.php
index 70daa7920..de95f57f0 100644
--- a/apps/php-sdk/src/Models/QueryFormat.php
+++ b/apps/php-sdk/src/Models/QueryFormat.php
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace Firecrawl\Models;
+/** @deprecated Use QuestionFormat or HighlightsFormat instead. */
final class QueryFormat
{
public const MODE_FREEFORM = 'freeform';
diff --git a/apps/php-sdk/src/Models/QuestionFormat.php b/apps/php-sdk/src/Models/QuestionFormat.php
new file mode 100644
index 000000000..6d076e185
--- /dev/null
+++ b/apps/php-sdk/src/Models/QuestionFormat.php
@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Firecrawl\Models;
+
+final class QuestionFormat
+{
+ private function __construct(
+ private readonly string $question,
+ ) {}
+
+ public static function with(string $question): self
+ {
+ return new self($question);
+ }
+
+ /** @return array<string, mixed> */
+ public function toArray(): array
+ {
+ return [
+ 'type' => 'question',
+ 'question' => $this->question,
+ ];
+ }
+
+ public function getQuestion(): string
+ {
+ return $this->question;
+ }
+}
diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.php
index 7694d70ca..b0d8d50d7 100644
--- a/apps/php-sdk/src/Models/ScrapeOptions.php
+++ b/apps/php-sdk/src/Models/ScrapeOptions.php
@@ -7,7 +7,7 @@ namespace Firecrawl\Models;
final class ScrapeOptions
{
/**
- * @param list<string|JsonFormat|ScreenshotFormat|QueryFormat>|null $formats
+ * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats
* @param array<string, string>|null $headers
* @param list<string>|null $includeTags
* @param list<string>|null $excludeTags
@@ -41,7 +41,7 @@ final class ScrapeOptions
) {}
/**
- * @param list<string|JsonFormat|ScreenshotFormat|QueryFormat>|null $formats
+ * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats
* @param array<string, string>|null $headers
* @param list<string>|null $includeTags
* @param list<string>|null $excludeTags
@@ -89,8 +89,14 @@ final class ScrapeOptions
if ($this->formats !== null) {
$data['formats'] = array_map(
- fn (string|JsonFormat|ScreenshotFormat|QueryFormat $f): string|array =>
- $f instanceof JsonFormat || $f instanceof ScreenshotFormat || $f instanceof QueryFormat ? $f->toArray() : $f,
+ fn (string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat $f): string|array =>
+ (
+ $f instanceof JsonFormat
+ || $f instanceof ScreenshotFormat
+ || $f instanceof QuestionFormat
+ || $f instanceof HighlightsFormat
+ || $f instanceof QueryFormat
+ ) ? $f->toArray() : $f,
$this->formats,
);
}
@@ -128,7 +134,7 @@ final class ScrapeOptions
return $data;
}
- /** @return list<string|JsonFormat|ScreenshotFormat|QueryFormat>|null */
+ /** @return list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null */
public function getFormats(): ?array
{
return $this->formats;
diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php
index 8fb71f9b4..aefc62765 100644
--- a/apps/php-sdk/src/Version.php
+++ b/apps/php-sdk/src/Version.php
@@ -6,5 +6,5 @@ namespace Firecrawl;
final class Version
{
- public const SDK_VERSION = '1.2.0';
+ public const SDK_VERSION = '1.2.1';
}
diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.php
index 59b2592bb..b36a04d18 100644
--- a/apps/php-sdk/tests/Unit/ModelsTest.php
+++ b/apps/php-sdk/tests/Unit/ModelsTest.php
@@ -6,7 +6,9 @@ use Firecrawl\Models\CreditUsage;
use Firecrawl\Models\MapData;
use Firecrawl\Models\BatchScrapeJob;
use Firecrawl\Models\CrawlJob;
+use Firecrawl\Models\HighlightsFormat;
use Firecrawl\Models\QueryFormat;
+use Firecrawl\Models\QuestionFormat;
use Firecrawl\Models\ScrapeOptions;
it('hydrates CreditUsage from nested data key', function (): void {
@@ -147,6 +149,26 @@ it('serializes query format mode in ScrapeOptions', function (): void {
]);
});
+it('serializes question and highlights formats in ScrapeOptions', function (): void {
+ $options = ScrapeOptions::with(
+ formats: [
+ QuestionFormat::with('What is Firecrawl?'),
+ HighlightsFormat::with('What is Firecrawl?'),
+ ],
+ );
+
+ expect($options->toArray()['formats'])->toMatchArray([
+ [
+ 'type' => 'question',
+ 'question' => 'What is Firecrawl?',
+ ],
+ [
+ 'type' => 'highlights',
+ 'query' => 'What is Firecrawl?',
+ ],
+ ]);
+});
+
it('rejects invalid query format mode', function (): void {
QueryFormat::with('What is Firecrawl?', 'quoted');
})->throws(InvalidArgumentException::class, "query mode must be 'freeform' or 'directQuote'");
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 8abe1b757..90f63b1cd 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.25.0"
+__version__ = "4.25.1"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
index 41b205b1a..2988d3972 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
@@ -1,5 +1,12 @@
import pytest
-from firecrawl.v2.types import JsonFormat, QueryFormat, ScrapeOptions, PDFParser
+from firecrawl.v2.types import (
+ HighlightsFormat,
+ JsonFormat,
+ QueryFormat,
+ QuestionFormat,
+ ScrapeOptions,
+ PDFParser,
+)
from firecrawl.v2.utils.validation import validate_scrape_options, prepare_scrape_options
@@ -200,6 +207,33 @@ class TestPrepareScrapeOptions:
{"type": "query", "prompt": "What is Firecrawl?", "mode": "directQuote"}
]
+ def test_prepare_question_and_highlights_formats(self):
+ """Test question and highlights formats are preserved."""
+ options = ScrapeOptions(
+ formats=[
+ QuestionFormat(question="What is Firecrawl?"),
+ HighlightsFormat(query="What is Firecrawl?"),
+ ]
+ )
+ result = prepare_scrape_options(options)
+
+ assert result["formats"] == [
+ {"type": "question", "question": "What is Firecrawl?"},
+ {"type": "highlights", "query": "What is Firecrawl?"},
+ ]
+
+ def test_prepare_question_and_highlights_reject_empty_values(self):
+ """Test question and highlights validation."""
+ with pytest.raises(ValueError, match="question format requires"):
+ prepare_scrape_options(
+ ScrapeOptions(formats=[{"type": "question", "question": ""}])
+ )
+
+ with pytest.raises(ValueError, match="highlights format requires"):
+ prepare_scrape_options(
+ ScrapeOptions(formats=[{"type": "highlights", "query": ""}])
+ )
+
def test_prepare_query_format_rejects_direct_quote_boolean(self):
"""Test old query directQuote layout is rejected."""
options = ScrapeOptions(
diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.py
index 2f5a43a73..4bb832409 100644
--- a/apps/python-sdk/firecrawl/types.py
+++ b/apps/python-sdk/firecrawl/types.py
@@ -46,6 +46,9 @@ from .v2.types import (
SourceOption,
Format,
JsonFormat,
+ QuestionFormat,
+ HighlightsFormat,
+ QueryFormat,
FormatOption,
SearchRequest,
SearchResultWeb,
@@ -127,6 +130,9 @@ __all__ = [
'SourceOption',
'Format',
'JsonFormat',
+ 'QuestionFormat',
+ 'HighlightsFormat',
+ 'QueryFormat',
'FormatOption',
'SearchRequest',
'SearchResultWeb',
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index a2fe2368f..2ace0eb62 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -277,6 +277,7 @@ class Document(BaseModel):
audio: Optional[str] = None
actions: Optional[Dict[str, Any]] = None
answer: Optional[str] = None
+ highlights: Optional[str] = None
warning: Optional[str] = None
change_tracking: Optional[Dict[str, Any]] = None
branding: Optional[BrandingProfile] = None
@@ -446,8 +447,22 @@ class AttributesFormat(Format):
selectors: List[AttributeSelector]
+class QuestionFormat(Format):
+ """Configuration for question format - ask a question about the page content."""
+
+ type: Literal["question"] = "question"
+ question: str
+
+
+class HighlightsFormat(Format):
+ """Configuration for highlights format - extract direct highlights from page content."""
+
+ type: Literal["highlights"] = "highlights"
+ query: str
+
+
class QueryFormat(Format):
- """Configuration for query format - ask a question about the page content."""
+ """Deprecated query format. Use QuestionFormat or HighlightsFormat instead."""
type: Literal["query"] = "query"
prompt: str
@@ -461,6 +476,8 @@ FormatOption = Union[
ChangeTrackingFormat,
ScreenshotFormat,
AttributesFormat,
+ QuestionFormat,
+ HighlightsFormat,
QueryFormat,
Format,
]
@@ -495,10 +512,16 @@ class ScrapeFormats(BaseModel):
raise ValueError("query format must be an object with 'type' and 'prompt' fields")
normalized_formats.append(Format(type=format_item))
elif isinstance(format_item, dict):
- # Reject query dicts missing prompt early
+ fmt_type = format_item.get('type')
prompt = format_item.get('prompt')
- if format_item.get('type') == 'query' and (not isinstance(prompt, str) or not prompt.strip()):
+ question = format_item.get('question')
+ query = format_item.get('query')
+ if fmt_type == 'query' and (not isinstance(prompt, str) or not prompt.strip()):
raise ValueError("query format requires a non-empty 'prompt' string")
+ if fmt_type == 'question' and (not isinstance(question, str) or not question.strip()):
+ raise ValueError("question format requires a non-empty 'question' string")
+ if fmt_type == 'highlights' and (not isinstance(query, str) or not query.strip()):
+ raise ValueError("highlights format requires a non-empty 'query' string")
# Preserve dicts as-is to avoid dropping custom fields like 'schema'
normalized_formats.append(format_item)
elif isinstance(format_item, Format):
diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py
index 8568bec50..11f427f2c 100644
--- a/apps/python-sdk/firecrawl/v2/utils/validation.py
+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py
@@ -464,6 +464,28 @@ def _validate_query_format(format_obj: Any) -> Dict[str, Any]:
return format_obj
+def _validate_question_format(format_obj: Any) -> Dict[str, Any]:
+ """Validate and prepare question format object."""
+ if not isinstance(format_obj, dict):
+ raise ValueError("question format must be an object with 'type' and 'question' fields")
+
+ if not isinstance(format_obj.get('question'), str) or not format_obj['question'].strip():
+ raise ValueError("question format requires a non-empty 'question' string")
+
+ return format_obj
+
+
+def _validate_highlights_format(format_obj: Any) -> Dict[str, Any]:
+ """Validate and prepare highlights format object."""
+ if not isinstance(format_obj, dict):
+ raise ValueError("highlights format must be an object with 'type' and 'query' fields")
+
+ if not isinstance(format_obj.get('query'), str) or not format_obj['query'].strip():
+ raise ValueError("highlights format requires a non-empty 'query' string")
+
+ return format_obj
+
+
def validate_scrape_options(options: Optional[ScrapeOptions]) -> Optional[ScrapeOptions]:
"""
Validate and normalize scrape options.
@@ -573,12 +595,20 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
if fmt == "query":
raise ValueError("query format must be an object with 'type' and 'prompt' fields")
+ if fmt == "question":
+ raise ValueError("question format must be an object with 'type' and 'question' fields")
+ if fmt == "highlights":
+ raise ValueError("highlights format must be an object with 'type' and 'query' fields")
converted_formats.append(_convert_format_string(fmt))
elif isinstance(fmt, dict):
fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None
if fmt_type == 'json':
validated_json = _validate_json_format({**fmt, 'type': 'json'})
converted_formats.append(validated_json)
+ elif fmt_type == 'question':
+ converted_formats.append(_validate_question_format(fmt))
+ elif fmt_type == 'highlights':
+ converted_formats.append(_validate_highlights_format(fmt))
elif fmt_type == 'query':
converted_formats.append(_validate_query_format(fmt))
elif fmt_type == 'screenshot':
@@ -598,6 +628,10 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
elif hasattr(fmt, 'type'):
if fmt.type == 'json':
converted_formats.append(_validate_json_format(fmt.model_dump()))
+ elif fmt.type == 'question':
+ converted_formats.append(_validate_question_format(fmt.model_dump(exclude_none=True)))
+ elif fmt.type == 'highlights':
+ converted_formats.append(_validate_highlights_format(fmt.model_dump(exclude_none=True)))
elif fmt.type == 'query':
converted_formats.append(_validate_query_format(fmt.model_dump(exclude_none=True)))
else:
@@ -630,12 +664,20 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
if fmt == "query":
raise ValueError("query format must be an object with 'type' and 'prompt' fields")
+ if fmt == "question":
+ raise ValueError("question format must be an object with 'type' and 'question' fields")
+ if fmt == "highlights":
+ raise ValueError("highlights format must be an object with 'type' and 'query' fields")
converted_formats.append(_convert_format_string(fmt))
elif isinstance(fmt, dict):
fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None
if fmt_type == 'json':
validated_json = _validate_json_format({**fmt, 'type': 'json'})
converted_formats.append(validated_json)
+ elif fmt_type == 'question':
+ converted_formats.append(_validate_question_format(fmt))
+ elif fmt_type == 'highlights':
+ converted_formats.append(_validate_highlights_format(fmt))
elif fmt_type == 'query':
converted_formats.append(_validate_query_format(fmt))
elif fmt_type == 'screenshot':
@@ -653,6 +695,10 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
elif hasattr(fmt, 'type'):
if fmt.type == 'json':
converted_formats.append(_validate_json_format(fmt.model_dump()))
+ elif fmt.type == 'question':
+ converted_formats.append(_validate_question_format(fmt.model_dump(exclude_none=True)))
+ elif fmt.type == 'highlights':
+ converted_formats.append(_validate_highlights_format(fmt.model_dump(exclude_none=True)))
elif fmt.type == 'screenshot':
normalized = {'type': 'screenshot'}
if getattr(fmt, 'full_page', None) is not None:
diff --git a/apps/ruby-sdk/lib/firecrawl/models/document.rb b/apps/ruby-sdk/lib/firecrawl/models/document.rb
index 931911af3..578c095b9 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/document.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/document.rb
@@ -6,7 +6,8 @@ module Firecrawl
class Document
attr_reader :markdown, :html, :raw_html, :json, :summary,
:metadata, :links, :images, :screenshot, :audio,
- :attributes, :actions, :warning, :change_tracking, :branding
+ :attributes, :actions, :answer, :highlights, :warning,
+ :change_tracking, :branding
def initialize(data)
@markdown = data["markdown"]
@@ -21,6 +22,8 @@ module Firecrawl
@audio = data["audio"]
@attributes = data["attributes"]
@actions = data["actions"]
+ @answer = data["answer"]
+ @highlights = data["highlights"]
@warning = data["warning"]
@change_tracking = data["changeTracking"]
@branding = data["branding"]
diff --git a/apps/ruby-sdk/lib/firecrawl/models/query_format.rb b/apps/ruby-sdk/lib/firecrawl/models/query_format.rb
index 5706ff36b..bf9ad3dac 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/query_format.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/query_format.rb
@@ -2,7 +2,47 @@
module Firecrawl
module Models
- # Query format for asking a question about page content.
+ # Question format for asking a question about page content.
+ class QuestionFormat
+ attr_reader :question
+
+ def initialize(question:)
+ @question = question
+ end
+
+ def to_h
+ {
+ "type" => "question",
+ "question" => question,
+ }
+ end
+
+ def type
+ "question"
+ end
+ end
+
+ # Highlights format for extracting direct highlights from page content.
+ class HighlightsFormat
+ attr_reader :query
+
+ def initialize(query:)
+ @query = query
+ end
+
+ def to_h
+ {
+ "type" => "highlights",
+ "query" => query,
+ }
+ end
+
+ def type
+ "highlights"
+ end
+ end
+
+ # Deprecated query format for asking a question about page content.
class QueryFormat
MODE_FREEFORM = "freeform"
MODE_DIRECT_QUOTE = "directQuote"
diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb
index 02eba3e87..7b806711f 100644
--- a/apps/ruby-sdk/lib/firecrawl/version.rb
+++ b/apps/ruby-sdk/lib/firecrawl/version.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
module Firecrawl
- VERSION = "1.4.0"
+ VERSION = "1.4.1"
end
diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rb
index 184d5505b..7b4997d2c 100644
--- a/apps/ruby-sdk/test/firecrawl/client_test.rb
+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb
@@ -471,6 +471,20 @@ class ClientTest < Minitest::Test
)
end
+ def test_question_and_highlights_format_to_h
+ question = Firecrawl::Models::QuestionFormat.new(question: "What is Firecrawl?")
+ highlights = Firecrawl::Models::HighlightsFormat.new(query: "What is Firecrawl?")
+ opts = Firecrawl::Models::ScrapeOptions.new(formats: [question, highlights])
+
+ assert_equal(
+ [
+ { "type" => "question", "question" => "What is Firecrawl?" },
+ { "type" => "highlights", "query" => "What is Firecrawl?" },
+ ],
+ opts.to_h["formats"]
+ )
+ end
+
def test_query_format_rejects_invalid_mode
assert_raises(ArgumentError) do
Firecrawl::Models::QueryFormat.new(prompt: "What is Firecrawl?", mode: "quoted")
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index a0f539d34..48cda122f 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "2.4.0"
+version = "2.4.1"
dependencies = [
"mockito",
"reqwest",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 509e1dcdc..8e6bb84c0 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "firecrawl"
-version = "2.4.0"
+version = "2.4.1"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs
index 43ec1b86d..36d213a05 100644
--- a/apps/rust-sdk/src/scrape.rs
+++ b/apps/rust-sdk/src/scrape.rs
@@ -436,7 +436,7 @@ impl Client {
#[cfg(test)]
mod tests {
use super::*;
- use crate::{QueryFormat, QueryFormatMode};
+ use crate::{HighlightsFormat, QueryFormat, QueryFormatMode, QuestionFormat};
use serde_json::json;
#[test]
@@ -460,6 +460,37 @@ mod tests {
);
}
+ #[test]
+ fn test_question_and_highlights_formats_serialize() {
+ let options = ScrapeOptions {
+ formats: Some(vec![
+ Format::Question(QuestionFormat {
+ question: "What is Firecrawl?".to_string(),
+ }),
+ Format::Highlights(HighlightsFormat {
+ query: "What is Firecrawl?".to_string(),
+ }),
+ ]),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ assert_eq!(
+ payload["formats"][0],
+ json!({
+ "type": "question",
+ "question": "What is Firecrawl?"
+ })
+ );
+ assert_eq!(
+ payload["formats"][1],
+ json!({
+ "type": "highlights",
+ "query": "What is Firecrawl?"
+ })
+ );
+ }
+
#[tokio::test]
async fn test_scrape_with_mock() {
let mut server = mockito::Server::new_async().await;
diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rs
index bcc7879ee..5cb1cdea0 100644
--- a/apps/rust-sdk/src/types.rs
+++ b/apps/rust-sdk/src/types.rs
@@ -33,7 +33,11 @@ pub enum Format {
Branding,
/// Audio extraction (MP3) from YouTube videos.
Audio,
- /// Query answer generated from the page content.
+ /// Question answer generated from the page content.
+ Question(QuestionFormat),
+ /// Direct highlights selected from the page content.
+ Highlights(HighlightsFormat),
+ /// Deprecated query answer generated from the page content.
Query(QueryFormat),
}
@@ -55,6 +59,8 @@ impl Serialize for Format {
Format::Attributes => serializer.serialize_str("attributes"),
Format::Branding => serializer.serialize_str("branding"),
Format::Audio => serializer.serialize_str("audio"),
+ Format::Question(question) => question.serialize(serializer),
+ Format::Highlights(highlights) => highlights.serialize(serializer),
Format::Query(query) => query.serialize(serializer),
}
}
@@ -82,15 +88,116 @@ impl<'de> Deserialize<'de> for Format {
"audio" => Ok(Format::Audio),
_ => Err(de::Error::custom(format!("unknown format: {}", format))),
},
- Value::Object(_) => QueryFormat::deserialize(value)
- .map(Format::Query)
- .map_err(de::Error::custom),
+ Value::Object(_) => match value.get("type").and_then(Value::as_str) {
+ Some("question") => QuestionFormat::deserialize(value)
+ .map(Format::Question)
+ .map_err(de::Error::custom),
+ Some("highlights") => HighlightsFormat::deserialize(value)
+ .map(Format::Highlights)
+ .map_err(de::Error::custom),
+ Some("query") => QueryFormat::deserialize(value)
+ .map(Format::Query)
+ .map_err(de::Error::custom),
+ Some(format_type) => Err(de::Error::custom(format!(
+ "unknown object format: {}",
+ format_type
+ ))),
+ None => Err(de::Error::custom("object format must have a type")),
+ },
_ => Err(de::Error::custom("format must be a string or object")),
}
}
}
-/// Query format for asking a question about page content.
+/// Question format for asking a question about page content.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct QuestionFormat {
+ pub question: String,
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct QuestionFormatWire {
+ #[serde(rename = "type")]
+ format_type: String,
+ question: String,
+}
+
+impl Serialize for QuestionFormat {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ QuestionFormatWire {
+ format_type: "question".to_string(),
+ question: self.question.clone(),
+ }
+ .serialize(serializer)
+ }
+}
+
+impl<'de> Deserialize<'de> for QuestionFormat {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let wire = QuestionFormatWire::deserialize(deserializer)?;
+ if wire.format_type != "question" {
+ return Err(de::Error::custom(
+ "question format object must have type question",
+ ));
+ }
+
+ Ok(Self {
+ question: wire.question,
+ })
+ }
+}
+
+/// Highlights format for selecting direct highlights from page content.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct HighlightsFormat {
+ pub query: String,
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct HighlightsFormatWire {
+ #[serde(rename = "type")]
+ format_type: String,
+ query: String,
+}
+
+impl Serialize for HighlightsFormat {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ HighlightsFormatWire {
+ format_type: "highlights".to_string(),
+ query: self.query.clone(),
+ }
+ .serialize(serializer)
+ }
+}
+
+impl<'de> Deserialize<'de> for HighlightsFormat {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let wire = HighlightsFormatWire::deserialize(deserializer)?;
+ if wire.format_type != "highlights" {
+ return Err(de::Error::custom(
+ "highlights format object must have type highlights",
+ ));
+ }
+
+ Ok(Self { query: wire.query })
+ }
+}
+
+/// Deprecated query format for asking a question about page content.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct QueryFormat {
pub prompt: String,
@@ -555,6 +662,10 @@ pub struct Document {
pub attributes: Option<Vec<AttributeResult>>,
/// Action results.
pub actions: Option<HashMap<String, Value>>,
+ /// Answer generated by the question or deprecated query format.
+ pub answer: Option<String>,
+ /// Highlights generated by the highlights format.
+ pub highlights: Option<String>,
/// Warning message.
pub warning: Option<String>,
/// Change tracking data.