Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Feat/product service client (#3818)
Kris Oeidf88fb6d5e6a
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 249be61fc..c4b8a0818 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -71,6 +71,13 @@ STRIPE_PRICE_ID_GROWTH_YEARLY=# set if you'd like to use the fire engine closed betaFIRE_ENGINE_BETA_URL=+# set to enable the `product` scrape format. The product format extracts a+# structured product from a page's HTML via an external product-extraction+# service; point this at that service's base URL. If unset, requesting the+# `product` format returns a warning and no product (same pattern as the+# audio/video formats and AVGRAB_SERVICE_URL).+PRODUCT_EXTRACTION_SERVICE_URL=+# Proxy Settings for Playwright (Alternative you can use a proxy service like oxylabs, which rotates IPs for you on every request)PROXY_SERVER=PROXY_USERNAME=diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex b92b4be94..78f2c2ca9 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -26,6 +26,7 @@ export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;export const HAS_PROXY = !!config.PROXY_SERVER;+export const HAS_PRODUCT_SERVICE = !!config.PRODUCT_EXTRACTION_SERVICE_URL;export const HAS_SEARCH = TEST_PRODUCTION || !!config.SEARXNG_ENDPOINT;diff --git a/apps/api/src/__tests__/snips/v1/product.test.ts b/apps/api/src/__tests__/snips/v1/product.test.tsnew file mode 100644index 000000000..4f591e32d--- /dev/null+++ b/apps/api/src/__tests__/snips/v1/product.test.ts@@ -0,0 +1,52 @@+import {+ ALLOW_TEST_SUITE_WEBSITE,+ describeIf,+ TEST_SUITE_WEBSITE,+} from "../lib";+import { scrape, scrapeTimeout, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "product-v1",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++// The `product` scrape format is registered for both v1 and v2. The v2 suite+// covers the positive extraction paths; here we only need to prove the+// additive-only contract on v1: a scrape that does NOT request the `product`+// format is byte-identical in shape to a pre-feature v1 scrape (no `product`,+// no spurious `warning`, markdown + metadata intact). The product transformer+// is shared across v1/v2, so this guards the v1 format gate specifically.+describeIf(ALLOW_TEST_SUITE_WEBSITE)(+ "Product scrape format (v1 back-compat)",+ () => {+ const base = TEST_SUITE_WEBSITE;+ const productUrl = `${base}/product`;++ it.concurrent(+ "does not populate product when the product format is not requested (v1)",+ async () => {+ const response = await scrape(+ {+ url: productUrl,+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.product).toBeUndefined();+ expect(response.warning).toBeUndefined();+ expect(response.markdown).toBeDefined();+ expect(typeof response.markdown).toBe("string");+ expect(response.markdown?.length).toBeGreaterThan(0);+ expect(response.metadata).toBeDefined();+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );+ },+);diff --git a/apps/api/src/__tests__/snips/v2/product.test.ts b/apps/api/src/__tests__/snips/v2/product.test.tsnew file mode 100644index 000000000..5fc958996--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/product.test.ts@@ -0,0 +1,110 @@+import {+ ALLOW_TEST_SUITE_WEBSITE,+ describeIf,+ HAS_PRODUCT_SERVICE,+ TEST_SUITE_WEBSITE,+} from "../lib";+import { scrape, scrapeTimeout, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "product",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++// The `product` scrape format extracts a structured product from a page's HTML+// (JSON-LD / microdata / etc.). The extraction itself runs in the product-search+// Rust service, called over HTTP when PRODUCT_EXTRACTION_SERVICE_URL is set+// (the audio/video → AVGRAB_SERVICE_URL pattern). These tests target the static+// test-site product fixture (`apps/test-site/src/pages/product.astro`), which+// embeds a schema.org JSON-LD Product, and a plain non-product page (`/about`).+// A local test-site fetch needs neither fire-engine nor AI, so we gate on+// ALLOW_TEST_SUITE_WEBSITE; and because the format now requires the external+// extractor, we also gate on HAS_PRODUCT_SERVICE so the suite only runs where+// the service is configured (mirroring how avgrab-dependent snips are gated).+describeIf(ALLOW_TEST_SUITE_WEBSITE && HAS_PRODUCT_SERVICE)(+ "Product scrape format",+ () => {+ const base = TEST_SUITE_WEBSITE;+ const productUrl = `${base}/product`;+ const nonProductUrl = `${base}/about`;++ it.concurrent(+ "extracts a structured product from a JSON-LD product page",+ async () => {+ const response = await scrape(+ {+ url: productUrl,+ formats: [{ type: "product" }],+ },+ identity,+ );++ expect(response.product).toBeDefined();+ expect(response.product?.title).toBe("Firecrawl Test Widget");+ // Canonical shape: price/availability live on variants, not top-level.+ const variant = response.product?.variants?.[0];+ expect(variant).toBeDefined();+ expect(variant?.price).toBeDefined();+ expect(variant?.price?.amount).toBe(49.99);+ expect(variant?.price?.currency).toBe("USD");+ expect(variant?.availability).toBeDefined();+ expect(variant?.availability?.inStock).toBe(true);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "sets a no-product warning for a non-product page",+ async () => {+ const response = await scrape(+ {+ url: nonProductUrl,+ formats: [{ type: "product" }],+ },+ identity,+ );++ expect(response.product).toBeUndefined();+ expect(response.warning).toBeDefined();+ expect(response.warning).toContain("No product found");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "does not populate product when the product format is not requested",+ async () => {+ const response = await scrape(+ {+ url: productUrl,+ formats: ["markdown"],+ },+ identity,+ );++ // Additive-only contract: a scrape that does not request the `product`+ // format must be byte-identical in shape to a pre-feature scrape.+ // No `product` field...+ expect(response.product).toBeUndefined();+ // ...and no spurious `warning` (the no-product warning is only emitted+ // when the format is actually requested).+ expect(response.warning).toBeUndefined();+ // ...while the rest of the document is present and unchanged.+ expect(response.markdown).toBeDefined();+ expect(typeof response.markdown).toBe("string");+ expect(response.markdown?.length).toBeGreaterThan(0);+ expect(response.metadata).toBeDefined();+ expect(response.metadata.sourceURL ?? response.metadata.url).toBe(+ productUrl,+ );+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );+ },+);diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex a3a88d9ac..86eb7346a 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -327,6 +327,9 @@ const configSchema = z.object({// Audio (avgrab)AVGRAB_SERVICE_URL: z.string().optional(),+ // Product extraction (product-search Rust service)+ PRODUCT_EXTRACTION_SERVICE_URL: z.string().optional(),+// PII Redaction (fire-privacy)FIRE_PRIVACY_URL: z.string().optional(),FIRE_PRIVACY_TIMEOUT_MS: z.coerce.number().int().positive().default(5000),diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 4b4bd5c9a..8ae4f31ad 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -17,6 +17,7 @@ import { integrationSchema } from "../../utils/integration";import { includesFormat } from "../../lib/format-utils";import { webhookSchema } from "../../services/webhook/schema";import { BrandingProfile } from "../../types/branding";+import { ProductProfile } from "../../types/product";type Format =| "markdown"@@ -29,7 +30,8 @@ type Format =| "json"| "summary"| "changeTracking"- | "branding";+ | "branding"+ | "product";export const url = z.preprocess(x => {@@ -429,6 +431,7 @@ const baseScrapeOptions = z.strictObject({"summary","changeTracking","branding",+ "product",]).array().optional()@@ -992,6 +995,7 @@ export type Document = {json?: any;summary?: string;branding?: BrandingProfile;+ product?: ProductProfile;warning?: string;actions?: {screenshots?: string[];@@ -1558,6 +1562,7 @@ export const searchRequestSchema = z"screenshot@fullPage","extract","json",+ "product",]),).prefault([]),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 1f9e3e9ef..a398e4fc7 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -26,6 +26,7 @@ import {createWebhookSchema,} from "../../services/webhook/schema";import { BrandingProfile } from "../../types/branding";+import { ProductProfile } from "../../types/product";// Base URL schema with common validation logicexport const URL = z.preprocess(@@ -458,6 +459,7 @@ export type FormatObject =| HighlightsFormatWithOptions| QueryFormatWithOptions| { type: "branding" }+ | { type: "product" }| { type: "audio" }| { type: "video" };@@ -629,6 +631,7 @@ const baseScrapeOptions = z.strictObject({screenshotFormatWithOptions,attributesFormatWithOptions,z.strictObject({ type: z.literal("branding") }),+ z.strictObject({ type: z.literal("product") }),questionFormatWithOptions,highlightsFormatWithOptions,queryFormatWithOptions,@@ -1204,6 +1207,7 @@ export type Document = {answer?: string;highlights?: string;branding?: BrandingProfile;+ product?: ProductProfile;warning?: string;attributes?: {selector: string;@@ -1773,6 +1777,8 @@ export function fromV1ScrapeOptions(return { type: "screenshot" as const, fullPage: true };} else if (x === "branding") {return { type: "branding" as const };+ } else if (x === "product") {+ return { type: "product" as const };} else {return x;}@@ -1933,6 +1939,7 @@ export const searchRequestSchema = zz.strictObject({ type: z.literal("links") }),z.strictObject({ type: z.literal("images") }),z.strictObject({ type: z.literal("summary") }),+ z.strictObject({ type: z.literal("product") }),jsonFormatWithOptions,questionFormatWithOptions,highlightsFormatWithOptions,diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.tsnew file mode 100644index 000000000..b501884a2--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts@@ -0,0 +1,130 @@+import { fetchProduct } from "../product";+import { config } from "../../../../config";++describe("fetchProduct", () => {+ const originalFetch = global.fetch;+ const originalServiceUrl = config.PRODUCT_EXTRACTION_SERVICE_URL;++ afterEach(() => {+ global.fetch = originalFetch;+ config.PRODUCT_EXTRACTION_SERVICE_URL = originalServiceUrl;+ vi.clearAllMocks();+ });++ function baseMeta(formats: any[] = [{ type: "product" }]) {+ return {+ url: "https://shop.test/p",+ rewrittenUrl: "https://shop.test/p",+ options: { formats },+ logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn() },+ } as any;+ }++ it("warns and yields no product when the service is not configured", async () => {+ config.PRODUCT_EXTRACTION_SERVICE_URL = undefined;+ const fetchSpy = vi.fn();+ global.fetch = fetchSpy as any;+ const document: any = {+ rawHtml: "<html></html>",+ metadata: { url: "https://shop.test/p" },+ };++ const out = await fetchProduct(baseMeta(), document);++ expect(fetchSpy).not.toHaveBeenCalled();+ expect(out.product).toBeUndefined();+ expect(out.warning).toMatch(/not available/i);+ });++ it("posts rawHtml to the service and sets document.product", async () => {+ config.PRODUCT_EXTRACTION_SERVICE_URL = "https://product.internal";+ const product = { title: "X", url: "https://shop.test/p", variants: [] };+ const fetchSpy = vi.fn(async (_url: string, _init?: RequestInit) => ({+ ok: true,+ json: async () => ({ product }),+ }));+ global.fetch = fetchSpy as any;+ const document: any = {+ rawHtml: "<html>jsonld</html>",+ metadata: { url: "https://shop.test/p" },+ };++ const out = await fetchProduct(baseMeta(), document);++ expect(out.product).toEqual(product);+ const [calledUrl, init] = fetchSpy.mock.calls[0];+ expect(calledUrl).toBe("https://product.internal/v1/extract-product");+ expect(init!.method).toBe("POST");+ expect(JSON.parse(init!.body as string)).toEqual({+ html: "<html>jsonld</html>",+ url: "https://shop.test/p",+ });+ });++ it("warns (no product) when the service reports no product", async () => {+ config.PRODUCT_EXTRACTION_SERVICE_URL = "https://product.internal";+ global.fetch = vi.fn(async () => ({+ ok: true,+ json: async () => ({ product: null }),+ })) as any;+ const document: any = {+ rawHtml: "<p>about</p>",+ metadata: { url: "https://shop.test/a" },+ };++ const out = await fetchProduct(baseMeta(), document);++ expect(out.product).toBeUndefined();+ expect(out.warning).toMatch(/no product found/i);+ });++ it("throws when the service returns a non-JSON 200 response", async () => {+ config.PRODUCT_EXTRACTION_SERVICE_URL = "https://product.internal";+ global.fetch = vi.fn(async () => ({+ ok: true,+ json: async () => {+ throw new SyntaxError("Unexpected token < in JSON");+ },+ })) as any;+ const document: any = {+ rawHtml: "<html></html>",+ metadata: { url: "https://shop.test/p" },+ };++ await expect(fetchProduct(baseMeta(), document)).rejects.toThrow(+ /Product extraction failed/i,+ );+ expect(document.product).toBeUndefined();+ expect(document.warning).toBeUndefined();+ });++ it("throws when the service 200 response omits the product key", async () => {+ config.PRODUCT_EXTRACTION_SERVICE_URL = "https://product.internal";+ global.fetch = vi.fn(async () => ({+ ok: true,+ json: async () => ({ unexpected: "shape" }),+ })) as any;+ const document: any = {+ rawHtml: "<html></html>",+ metadata: { url: "https://shop.test/p" },+ };++ await expect(fetchProduct(baseMeta(), document)).rejects.toThrow(+ /unexpected response/i,+ );+ expect(document.product).toBeUndefined();+ expect(document.warning).toBeUndefined();+ });++ it("early-returns when the product format isn't requested", async () => {+ const fetchSpy = vi.fn();+ global.fetch = fetchSpy as any;+ const document: any = { rawHtml: "<html></html>", metadata: {} };++ const out = await fetchProduct(baseMeta([{ type: "markdown" }]), document);++ expect(fetchSpy).not.toHaveBeenCalled();+ expect(out.product).toBeUndefined();+ expect(out.warning).toBeUndefined();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 1a9b35d82..cb0a96017 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -18,6 +18,7 @@ import { performAttributes } from "./performAttributes";import { deriveDiff } from "./diff";import { fetchAudio } from "./audio";+import { fetchProduct } from "./product";import { fetchVideo } from "./video";import { performRedactPII } from "./redactPII";import { useIndex, useSearchIndex } from "../../../services/index";@@ -334,6 +335,7 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {const hasScreenshot = hasFormatOfType(meta.options.formats, "screenshot");const hasSummary = hasFormatOfType(meta.options.formats, "summary");const hasBranding = hasFormatOfType(meta.options.formats, "branding");+ const hasProduct = hasFormatOfType(meta.options.formats, "product");const hasQuestionFormat = hasFormatOfType(meta.options.formats, "question");const hasHighlightsFormat = hasFormatOfType(meta.options.formats,@@ -488,6 +490,13 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {);}+ if (!hasProduct && document.product !== undefined) {+ meta.logger.warn(+ "Removed product from Document because it wasn't in formats -- this indicates the engine returned unexpected data.",+ );+ delete document.product;+ }+const hasAudio = hasFormatOfType(meta.options.formats, "audio");if (!hasAudio && document.audio !== undefined) {delete document.audio;@@ -577,6 +586,7 @@ const transformerStack: Transformer[] = [deriveImagesFromHTML,deriveBrandingFromActions,deriveMetadataFromRawHTML,+ fetchProduct,...(useIndex ? [sendDocumentToIndex] : []),...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time searchperformLLMExtract,diff --git a/apps/api/src/scraper/scrapeURL/transformers/product.ts b/apps/api/src/scraper/scrapeURL/transformers/product.tsnew file mode 100644index 000000000..f74537349--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/product.ts@@ -0,0 +1,78 @@+import { Meta } from "..";+import { Document } from "../../../controllers/v2/types";+import { config } from "../../../config";+import { hasFormatOfType } from "../../../lib/format-utils";++export async function fetchProduct(+ meta: Meta,+ document: Document,+): Promise<Document> {+ if (!hasFormatOfType(meta.options.formats, "product")) {+ return document;+ }++ if (!config.PRODUCT_EXTRACTION_SERVICE_URL) {+ meta.logger.warn("PRODUCT_EXTRACTION_SERVICE_URL is not configured");+ document.warning =+ "Product format is not available (service not configured)." ++ (document.warning ? " " + document.warning : "");+ return document;+ }++ if (document.rawHtml === undefined) {+ throw new Error(+ "rawHtml is undefined -- fetchProduct must run after rawHtml is available",+ );+ }++ const url =+ document.metadata.url ??+ document.metadata.sourceURL ??+ meta.rewrittenUrl ??+ meta.url;++ const response = await fetch(+ `${config.PRODUCT_EXTRACTION_SERVICE_URL}/v1/extract-product`,+ {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ html: document.rawHtml, url }),+ },+ );++ if (!response.ok) {+ const error = await response+ .json()+ .catch(() => ({ detail: "Unknown error" }));+ throw new Error(`Product extraction failed: ${error.detail}`);+ }++ let data: any;+ try {+ data = await response.json();+ } catch {+ throw new Error(+ "Product extraction failed: service returned a non-JSON response",+ );+ }++ // The service contract is a 200 with a body of `{ product: Product | null }`.+ // A body that isn't an object, or is missing the `product` key entirely, is a+ // malformed response -- surface it as a service failure rather than silently+ // reporting "no product found" (which is reserved for an explicit `null`).+ if (typeof data !== "object" || data === null || !("product" in data)) {+ throw new Error(+ "Product extraction failed: service returned an unexpected response shape (missing 'product')",+ );+ }++ if (data.product) {+ document.product = data.product;+ } else {+ // `product` is null: the page loaded but is not a product page.+ document.warning =+ "No product found on this page; it does not appear to be a product page." ++ (document.warning ? " " + document.warning : "");+ }+ return document;+}diff --git a/apps/api/src/types/product.ts b/apps/api/src/types/product.tsnew file mode 100644index 000000000..9ec23b6ce--- /dev/null+++ b/apps/api/src/types/product.ts@@ -0,0 +1,42 @@+import { z } from "zod";++// Canonical product shape, mirroring the product-extraction service's output+// (product-search `Product`). Everything that can differ by size, color, store+// selection, or stock state lives on a variant — there is no top-level price,+// availability, or images. A single-SKU product still has exactly one variant.+const productPriceSchema = z.object({+ amount: z.number(),+ currency: z.string().optional(),+ formatted: z.string().optional(),+});+const productSaleSchema = z.object({+ originalPrice: productPriceSchema,+});+const productAvailabilitySchema = z.object({+ inStock: z.boolean(),+ text: z.string().optional(),+});+const productImageSchema = z.object({+ url: z.string(),+ alt: z.string().optional(),+});+const productVariantSchema = z.object({+ id: z.string().optional(),+ sku: z.string().optional(),+ title: z.string().optional(),+ values: z.record(z.string(), z.unknown()).optional(),+ price: productPriceSchema.optional(),+ sale: productSaleSchema.optional(),+ availability: productAvailabilitySchema,+ images: z.array(productImageSchema).optional(),+});+const productProfileSchema = z.object({+ title: z.string(),+ brand: z.string().optional(),+ category: z.string().optional(),+ url: z.string(),+ description: z.string().optional(),+ variants: z.array(productVariantSchema),+});++export type ProductProfile = z.infer<typeof productProfileSchema>;diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.csindex b7283134b..32aed1092 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs@@ -139,6 +139,59 @@ public class ModelsTestsAssert.Null(doc.Warning);}+ [Fact]+ public void Document_DeserializesProductCorrectly()+ {+ var json = """+ {+ "markdown": "# Product",+ "product": {+ "title": "Test Sneaker",+ "brand": "Acme",+ "category": "Shoes",+ "url": "https://example.com/product/1",+ "description": "A great sneaker",+ "variants": [+ {+ "id": "v1",+ "sku": "SKU-1",+ "title": "Size 10",+ "values": { "size": "10" },+ "price": { "amount": 99.99, "currency": "USD", "formatted": "$99.99" },+ "sale": { "originalPrice": { "amount": 129.99, "currency": "USD" } },+ "availability": { "inStock": true, "text": "In stock" },+ "images": [ { "url": "https://example.com/v1.jpg", "alt": "Front" } ]+ }+ ]+ }+ }+ """;++ var doc = JsonSerializer.Deserialize<Document>(json, JsonOptions);+ Assert.NotNull(doc);+ Assert.NotNull(doc.Product);+ Assert.Equal("Test Sneaker", doc.Product.Title);+ Assert.Equal("Acme", doc.Product.Brand);+ Assert.Equal("https://example.com/product/1", doc.Product.Url);+ Assert.NotNull(doc.Product.Variants);+ Assert.Single(doc.Product.Variants);+ var variant = doc.Product.Variants[0];+ Assert.Equal("v1", variant.Id);+ Assert.Equal("SKU-1", variant.Sku);+ Assert.NotNull(variant.Values);+ Assert.Equal("10", variant.Values["size"].GetString());+ Assert.NotNull(variant.Price);+ Assert.Equal(99.99, variant.Price.Amount);+ Assert.Equal("USD", variant.Price.Currency);+ Assert.NotNull(variant.Sale);+ Assert.Equal(129.99, variant.Sale.OriginalPrice.Amount);+ Assert.NotNull(variant.Availability);+ Assert.True(variant.Availability.InStock);+ Assert.NotNull(variant.Images);+ Assert.Single(variant.Images);+ Assert.Equal("Front", variant.Images[0].Alt);+ }+[Fact]public void Document_IgnoresUnknownProperties(){diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.csindex 616b63224..2ebba52c4 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs@@ -79,6 +79,18 @@ public class ParseTestsAssert.Contains("screenshot", ex.Message);}+ [Fact]+ public void ParseOptions_Validate_RejectsProductFormat()+ {+ var options = new ParseOptions+ {+ Formats = new List<object> { "markdown", "product" }+ };++ var ex = Assert.Throws<ArgumentException>(() => options.Validate());+ Assert.Contains("product", ex.Message);+ }+[Fact]public void ParseOptions_Validate_RejectsUnsupportedProxy(){diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex 168f9fe80..1b26b813d 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.8.0</Version>+ <Version>1.8.1</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.csindex 1b5b4b005..1f0e3a460 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs@@ -259,7 +259,7 @@ public class FirecrawlClient/// </summary>/// <param name="file">The file to upload.</param>/// <param name="options">Optional parse options. Browser-only formats- /// (changeTracking, screenshot, branding), actions, waitFor, location,+ /// (changeTracking, screenshot, branding, product), actions, waitFor, location,/// and mobile are rejected.</param>/// <param name="cancellationToken">Cancellation token.</param>public async Task<Document> ParseAsync(diff --git a/apps/dot-net-sdk/Firecrawl/Models/Document.cs b/apps/dot-net-sdk/Firecrawl/Models/Document.csindex ba601fbf2..9054c8db1 100644--- a/apps/dot-net-sdk/Firecrawl/Models/Document.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/Document.cs@@ -57,4 +57,7 @@ public class Document[JsonPropertyName("branding")]public object? Branding { get; set; }++ [JsonPropertyName("product")]+ public ProductProfile? Product { get; set; }}diff --git a/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.csindex 00f3b7e82..920ff62b4 100644--- a/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs@@ -8,7 +8,7 @@ namespace Firecrawl.Models;////// <para>/// Parse does not support browser-rendering formats/options such as change-/// tracking, screenshot, branding, actions, waitFor, location, or mobile.+/// tracking, screenshot, branding, product, actions, waitFor, location, or mobile./// These are rejected client-side in <see cref="Validate"/>./// </para>/// </summary>@@ -21,6 +21,7 @@ public class ParseOptions"screenshot","screenshot@fullPage","branding",+ "product",};internal static readonly HashSet<string> SupportedProxies = new(StringComparer.OrdinalIgnoreCase)diff --git a/apps/dot-net-sdk/Firecrawl/Models/ProductProfile.cs b/apps/dot-net-sdk/Firecrawl/Models/ProductProfile.csnew file mode 100644index 000000000..6bd6fc0ea--- /dev/null+++ b/apps/dot-net-sdk/Firecrawl/Models/ProductProfile.cs@@ -0,0 +1,121 @@+using System.Text.Json;+using System.Text.Json.Serialization;++namespace Firecrawl.Models;++/// <summary>+/// Structured product information extracted via the <c>product</c> scrape format.+/// </summary>+public class ProductProfile+{+ [JsonPropertyName("title")]+ public string Title { get; set; } = string.Empty;++ [JsonPropertyName("brand")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Brand { get; set; }++ [JsonPropertyName("category")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Category { get; set; }++ [JsonPropertyName("url")]+ public string Url { get; set; } = string.Empty;++ [JsonPropertyName("description")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Description { get; set; }++ [JsonPropertyName("variants")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public List<ProductVariant>? Variants { get; set; }+}++/// <summary>+/// An image associated with a product or product variant.+/// </summary>+public class ProductImage+{+ [JsonPropertyName("url")]+ public string Url { get; set; } = string.Empty;++ [JsonPropertyName("alt")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Alt { get; set; }+}++/// <summary>+/// A monetary price for a product or product variant.+/// </summary>+public class ProductPrice+{+ [JsonPropertyName("amount")]+ public double Amount { get; set; }++ [JsonPropertyName("currency")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Currency { get; set; }++ [JsonPropertyName("formatted")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Formatted { get; set; }+}++/// <summary>+/// Availability information for a product or product variant.+/// </summary>+public class ProductAvailability+{+ [JsonPropertyName("inStock")]+ public bool InStock { get; set; }++ [JsonPropertyName("text")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Text { get; set; }+}++/// <summary>+/// A purchasable variant of a product (e.g. a specific size/color combination).+/// </summary>+public class ProductVariant+{+ [JsonPropertyName("id")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Id { get; set; }++ [JsonPropertyName("sku")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Sku { get; set; }++ [JsonPropertyName("title")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Title { get; set; }++ [JsonPropertyName("values")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public Dictionary<string, JsonElement>? Values { get; set; }++ [JsonPropertyName("price")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public ProductPrice? Price { get; set; }++ [JsonPropertyName("sale")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public ProductSale? Sale { get; set; }++ [JsonPropertyName("availability")]+ public ProductAvailability Availability { get; set; } = new();++ [JsonPropertyName("images")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public List<ProductImage>? Images { get; set; }+}++/// <summary>+/// Sale information for a product variant (the price before the current sale).+/// </summary>+public class ProductSale+{+ [JsonPropertyName("originalPrice")]+ public ProductPrice OriginalPrice { get; set; } = new();+}diff --git a/apps/dot-net-sdk/README.md b/apps/dot-net-sdk/README.mdindex 3c61ee7fe..9a894b272 100644--- a/apps/dot-net-sdk/README.md+++ b/apps/dot-net-sdk/README.md@@ -84,6 +84,20 @@ var doc = await client.ScrapeAsync("https://www.youtube.com/watch?v=dQw4w9WgXcQ"Console.WriteLine(doc.Video);```+### Product Extraction++Use the `product` format on product pages to get structured product extraction+(title, brand, and variants, each with their own price and availability) on the+document's `Product` property. It is the deterministic counterpart to the LLM-based+`json` format.++```csharp+var doc = await client.ScrapeAsync("https://example.com/product/123",+ new ScrapeOptions { Formats = new List<object> { "product" } });++Console.WriteLine(doc.Product);+```+### Crawl```csharpdiff --git a/apps/elixir-sdk/README.md b/apps/elixir-sdk/README.mdindex 8a1cfd2aa..29a1c9206 100644--- a/apps/elixir-sdk/README.md+++ b/apps/elixir-sdk/README.md@@ -49,6 +49,15 @@ All params are passed as keyword lists with snake_case keys. Invalid keys, missiformats: ["video"])+# Structured product extraction from product pages+# `product` pulls title, variants, and per-variant price/availability. It's+# the deterministic counterpart to the LLM `json` format. Read the result+# from the response body's "product" key.+{:ok, response} = Firecrawl.scrape_and_extract_from_url(+ url: "https://example.com/products/widget",+ formats: ["product"]+)+# Crawl a site{:ok, response} = Firecrawl.crawl_urls(url: "https://example.com",diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex f4e59ee41..c0c007082 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.7.0"+ @version "1.7.1"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.mdindex 99661dec8..e42dbb633 100644--- a/apps/go-sdk/README.md+++ b/apps/go-sdk/README.md@@ -108,6 +108,22 @@ if err != nil {fmt.Println(doc.Video)```+### Product Extraction++Use the `product` format on product pages for structured product extraction+(title, price, availability, variants). The result is returned on the document's+`Product` field. This is the deterministic counterpart to the LLM-based `json` format.++```go+doc, err := client.Scrape(ctx, "https://example.com/products/widget", &firecrawl.ScrapeOptions{+ Formats: []string{"product"},+})+if err != nil {+ return err+}+fmt.Println(doc.Product)+```+#### Interactive BrowserExecute code in a scrape-bound browser session:@@ -126,7 +142,7 @@ deleteResp, err := client.StopInteractiveBrowser(ctx, scrapeJobID)Upload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data andparse it synchronously. Parse options intentionally exclude browser-only-features such as change tracking, screenshot, branding, audio, video, actions,+features such as change tracking, screenshot, branding, product, audio, video, actions,waitFor, location, and mobile. The `Proxy` option only accepts `"auto"` or `"basic"`.```godiff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.goindex 51d3a1541..67c5c487d 100644--- a/apps/go-sdk/models.go+++ b/apps/go-sdk/models.go@@ -22,6 +22,54 @@ type Document struct {Warning string `json:"warning,omitempty"`ChangeTracking map[string]interface{} `json:"changeTracking,omitempty"`Branding map[string]interface{} `json:"branding,omitempty"`+ Product *ProductProfile `json:"product,omitempty"`+}++// ProductProfile represents structured product data extracted from a page+// via the `product` scrape format.+type ProductProfile struct {+ Title string `json:"title"`+ Brand string `json:"brand,omitempty"`+ Category string `json:"category,omitempty"`+ URL string `json:"url"`+ Description string `json:"description,omitempty"`+ Variants []ProductVariant `json:"variants,omitempty"`+}++// ProductImage is a single product image.+type ProductImage struct {+ URL string `json:"url"`+ Alt string `json:"alt,omitempty"`+}++// ProductPrice represents a price for a product or variant.+type ProductPrice struct {+ Amount float64 `json:"amount"`+ Currency string `json:"currency,omitempty"`+ Formatted string `json:"formatted,omitempty"`+}++// ProductAvailability represents stock availability for a product or variant.+type ProductAvailability struct {+ InStock bool `json:"inStock"`+ Text string `json:"text,omitempty"`+}++// ProductSale represents sale information for a product variant.+type ProductSale struct {+ OriginalPrice ProductPrice `json:"originalPrice"`+}++// ProductVariant represents a single purchasable variant of a product.+type ProductVariant struct {+ ID string `json:"id,omitempty"`+ SKU string `json:"sku,omitempty"`+ Title string `json:"title,omitempty"`+ Values map[string]any `json:"values,omitempty"`+ Price *ProductPrice `json:"price,omitempty"`+ Sale *ProductSale `json:"sale,omitempty"`+ Availability ProductAvailability `json:"availability"`+ Images []ProductImage `json:"images,omitempty"`}// PaperResult represents a ranked research paper result.diff --git a/apps/go-sdk/parse.go b/apps/go-sdk/parse.goindex 529b1220d..4b3e3632e 100644--- a/apps/go-sdk/parse.go+++ b/apps/go-sdk/parse.go@@ -54,7 +54,7 @@ func NewParseFileFromBytes(filename string, content []byte) *ParseFile {// ParseOptions configures a parse request.//// Parse does not support browser-rendering features (actions, waitFor, location,-// mobile) nor the screenshot, branding, or changeTracking formats. The proxy+// mobile) nor the screenshot, branding, product, or changeTracking formats. The proxy// field only accepts "auto" or "basic".type ParseOptions struct {Formats []string `json:"-"`diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 7c0127dce..1cdf6c43e 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.7.0"+const Version = "1.7.1"diff --git a/apps/java-sdk/README.md b/apps/java-sdk/README.mdindex 0a83d6576..436a7f4c6 100644--- a/apps/java-sdk/README.md+++ b/apps/java-sdk/README.md@@ -204,6 +204,19 @@ Document doc = client.scrape("https://example.com/product",System.out.println(doc.getJson());```+#### Product Extraction++Use the `product` format to get structured product data (title, brand, and variants, each with their own price and availability) from product pages, available on the document's `getProduct()`. It is the deterministic counterpart to the LLM-powered `json` format.++```java+Document doc = client.scrape("https://example.com/product",+ ScrapeOptions.builder()+ .formats(List.of("product"))+ .build());++System.out.println(doc.getProduct());+```+#### Scrape-Bound Interactive SessionRun browser automation against the page context captured by a scrape job:diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex 5f76edbfc..ba1573566 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.10.0"+version = "1.10.1"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.javaindex 6d4eee5c9..90b136f87 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java@@ -28,6 +28,7 @@ public class Document {private String warning;private Map<String, Object> changeTracking;private Map<String, Object> branding;+ private Product product;public String getMarkdown() { return markdown; }public String getHtml() { return html; }@@ -47,6 +48,7 @@ public class Document {public String getWarning() { return warning; }public Map<String, Object> getChangeTracking() { return changeTracking; }public Map<String, Object> getBranding() { return branding; }+ public Product getProduct() { return product; }@Overridepublic String toString() {diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.javaindex 59a419206..c1b68876c 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java@@ -12,7 +12,7 @@ import java.util.Map;* Options for parsing uploaded files via /v2/parse.** <p>Parse does not support browser-rendering formats/options such as- * change tracking, screenshot, branding, audio, video, actions, waitFor, location, or mobile.+ * change tracking, screenshot, branding, product, audio, video, actions, waitFor, location, or mobile.*/@JsonInclude(JsonInclude.Include.NON_NULL)public class ParseOptions {@@ -93,6 +93,7 @@ public class ParseOptions {|| normalized.equals("screenshot")|| normalized.equals("screenshot@fullPage")|| normalized.equals("branding")+ || normalized.equals("product")|| normalized.equals("audio")|| normalized.equals("video");}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Product.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Product.javanew file mode 100644index 000000000..08efe70ef--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Product.java@@ -0,0 +1,30 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import java.util.List;++/**+ * Structured product information extracted by the {@code product} scrape format.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class Product {++ private String title;+ private String brand;+ private String category;+ private String url;+ private String description;+ private List<ProductVariant> variants;++ public String getTitle() { return title; }+ public String getBrand() { return brand; }+ public String getCategory() { return category; }+ public String getUrl() { return url; }+ public String getDescription() { return description; }+ public List<ProductVariant> getVariants() { return variants; }++ @Override+ public String toString() {+ return "Product{title=" + title + ", url=" + url + "}";+ }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ProductAvailability.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductAvailability.javanew file mode 100644index 000000000..a083b1fe3--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductAvailability.java@@ -0,0 +1,21 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++/**+ * Availability information for a product extracted by the {@code product} scrape format.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class ProductAvailability {++ private boolean inStock;+ private String text;++ public boolean isInStock() { return inStock; }+ public String getText() { return text; }++ @Override+ public String toString() {+ return "ProductAvailability{inStock=" + inStock + "}";+ }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ProductImage.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductImage.javanew file mode 100644index 000000000..65b502373--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductImage.java@@ -0,0 +1,21 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++/**+ * An image associated with a product extracted by the {@code product} scrape format.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class ProductImage {++ private String url;+ private String alt;++ public String getUrl() { return url; }+ public String getAlt() { return alt; }++ @Override+ public String toString() {+ return "ProductImage{url=" + url + "}";+ }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ProductPrice.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductPrice.javanew file mode 100644index 000000000..eb14384ba--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductPrice.java@@ -0,0 +1,23 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++/**+ * A price associated with a product extracted by the {@code product} scrape format.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class ProductPrice {++ private double amount;+ private String currency;+ private String formatted;++ public double getAmount() { return amount; }+ public String getCurrency() { return currency; }+ public String getFormatted() { return formatted; }++ @Override+ public String toString() {+ return "ProductPrice{formatted=" + formatted + "}";+ }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ProductSale.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductSale.javanew file mode 100644index 000000000..6852ad368--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductSale.java@@ -0,0 +1,19 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++/**+ * Sale information for a product variant extracted by the {@code product} scrape format.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class ProductSale {++ private ProductPrice originalPrice;++ public ProductPrice getOriginalPrice() { return originalPrice; }++ @Override+ public String toString() {+ return "ProductSale{originalPrice=" + originalPrice + "}";+ }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ProductVariant.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductVariant.javanew file mode 100644index 000000000..979e4d2ac--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ProductVariant.java@@ -0,0 +1,35 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import java.util.List;+import java.util.Map;++/**+ * A variant of a product extracted by the {@code product} scrape format.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class ProductVariant {++ private String id;+ private String sku;+ private String title;+ private Map<String, Object> values;+ private ProductPrice price;+ private ProductSale sale;+ private ProductAvailability availability;+ private List<ProductImage> images;++ public String getId() { return id; }+ public String getSku() { return sku; }+ public String getTitle() { return title; }+ public Map<String, Object> getValues() { return values; }+ public ProductPrice getPrice() { return price; }+ public ProductSale getSale() { return sale; }+ public ProductAvailability getAvailability() { return availability; }+ public List<ProductImage> getImages() { return images; }++ @Override+ public String toString() {+ return "ProductVariant{id=" + id + ", title=" + title + "}";+ }+}diff --git a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.javaindex d7727253d..37ac688fe 100644--- a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java+++ b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java@@ -243,6 +243,15 @@ class FirecrawlClientTest {);}+ @Test+ void testParseOptionsRejectsProductFormat() {+ assertThrows(IllegalArgumentException.class, () ->+ ParseOptions.builder()+ .formats(List.of("product"))+ .build()+ );+ }+@Testvoid testParseOptionsBuilderSupportsRedactPII() {ParseOptions options = ParseOptions.builder()diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex 04ec490a1..f4f2a5f1e 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -58,6 +58,18 @@ const doc = await app.scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', {console.log(doc.video);```+### Product extraction++Use the `product` format to deterministically pull a product (title, price, availability, variants) from product pages — the deterministic counterpart to the LLM-based `json` format.++```js+const doc = await app.scrape('https://example.com/product/123', {+ formats: ['product'],+});++console.log(doc.product);+```+### Parsing uploaded filesUse `parse` to upload a file (`html`, `pdf`, `docx`, etc.) as multipart form data and process it through the same parsing pipeline.diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 9c4b50d99..9ad1639c5 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.28.0",+ "version": "4.28.1","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/product.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/product.test.tsnew file mode 100644index 000000000..f3747de27--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/product.test.ts@@ -0,0 +1,162 @@+import { describe, test, expect, jest } from "@jest/globals";+import { scrape } from "../../../v2/methods/scrape";++describe("JS SDK v2 product format", () => {+ function makeHttp(postImpl: (url: string, data: any) => any) {+ return { post: jest.fn(async (u: string, d: any) => postImpl(u, d)) } as any;+ }++ test("scrape with product format returns product data", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ markdown: "# Example Product",+ product: {+ title: "Acme Running Shoe",+ brand: "Acme",+ category: "Footwear",+ url: "https://example.com/shoe",+ description: "A lightweight running shoe.",+ variants: [+ {+ id: "default",+ images: [{ url: "https://example.com/shoe.jpg", alt: "Acme shoe" }],+ price: { amount: 89.99, currency: "USD", formatted: "$89.99" },+ sale: { originalPrice: { amount: 129.99, currency: "USD", formatted: "$129.99" } },+ availability: { inStock: true, text: "In stock" }+ }+ ]+ }+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["product"] });++ expect(result.product).toBeDefined();+ expect(result.product?.title).toBe("Acme Running Shoe");+ expect(result.product?.brand).toBe("Acme");+ expect(result.product?.variants?.[0]?.price?.amount).toBe(89.99);+ expect(result.product?.variants?.[0]?.price?.currency).toBe("USD");+ expect(result.product?.variants?.[0]?.sale?.originalPrice?.amount).toBe(129.99);+ expect(result.product?.variants?.[0]?.availability?.inStock).toBe(true);+ expect(result.product?.variants?.[0]?.images?.[0]?.url).toBe("https://example.com/shoe.jpg");+ });++ test("scrape with product and markdown formats returns both", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ markdown: "# Example Content",+ product: {+ title: "Acme Mug",+ url: "https://example.com/mug",+ variants: [+ {+ price: { amount: 12.5, currency: "USD" },+ availability: { inStock: true }+ }+ ]+ }+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["markdown", "product"] });++ expect(result.markdown).toBe("# Example Content");+ expect(result.product).toBeDefined();+ expect(result.product?.title).toBe("Acme Mug");+ expect(result.product?.variants?.[0]?.price?.amount).toBe(12.5);+ });++ test("scrape without product format does not return product", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ markdown: "# Example"+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["markdown"] });++ expect(result.markdown).toBe("# Example");+ expect(result.product).toBeUndefined();+ });++ test("non-product page scraped with product format yields a warning and no product", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ markdown: "# Blog Post",+ warning: "No product found on this page."+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["product"] });++ expect(result.product).toBeUndefined();+ expect(result.warning).toContain("No product found");+ });++ test("product format with variants populated", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ product: {+ title: "Acme T-Shirt",+ brand: "Acme",+ url: "https://example.com/tshirt",+ variants: [+ {+ id: "v1",+ sku: "TSHIRT-S-RED",+ title: "Small / Red",+ values: { size: "S", color: "Red" },+ price: { amount: 19.0, currency: "USD" },+ sale: { originalPrice: { amount: 24.0, currency: "USD" } },+ availability: { inStock: true },+ images: [{ url: "https://example.com/tshirt-red.jpg" }]+ },+ {+ id: "v2",+ sku: "TSHIRT-L-BLUE",+ title: "Large / Blue",+ values: { size: "L", color: "Blue" },+ availability: { inStock: false, text: "Sold out" }+ }+ ]+ }+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["product"] });++ expect(result.product).toBeDefined();+ expect(result.product?.variants).toHaveLength(2);+ expect(result.product?.variants?.[0]?.sku).toBe("TSHIRT-S-RED");+ expect(result.product?.variants?.[0]?.values?.color).toBe("Red");+ expect(result.product?.variants?.[0]?.images?.[0]?.url).toBe("https://example.com/tshirt-red.jpg");+ expect(result.product?.variants?.[0]?.sale?.originalPrice?.amount).toBe(24.0);+ expect(result.product?.variants?.[1]?.availability?.inStock).toBe(false);+ expect(result.product?.variants?.[1]?.availability?.text).toBe("Sold out");+ });+});diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 175486687..e005ce1d6 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -13,6 +13,7 @@ export type FormatString =| "json"| "attributes"| "branding"+ | "product"| "audio"| "video";@@ -427,6 +428,46 @@ export interface BrandingProfile {[key: string]: unknown;}+export interface ProductPrice {+ amount: number;+ currency?: string;+ formatted?: string;+}++export interface ProductAvailability {+ inStock: boolean;+ text?: string;+}++export interface ProductImage {+ url: string;+ alt?: string;+}++export interface ProductSale {+ originalPrice: ProductPrice;+}++export interface ProductVariant {+ id?: string;+ sku?: string;+ title?: string;+ values?: Record<string, unknown>;+ price?: ProductPrice;+ sale?: ProductSale;+ availability: ProductAvailability;+ images?: ProductImage[];+}++export interface ProductProfile {+ title: string;+ brand?: string;+ category?: string;+ url: string;+ description?: string;+ variants: ProductVariant[];+}+export interface DocumentMetadata {// Common metadata fieldstitle?: string;@@ -509,6 +550,7 @@ export interface Document {warning?: string;changeTracking?: Record<string, unknown>;branding?: BrandingProfile;+ product?: ProductProfile;}// Pagination configuration for auto-fetching pages from v2 endpoints that return a `next` URLdiff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.mdindex a36fb6301..2b1bea1b9 100644--- a/apps/php-sdk/README.md+++ b/apps/php-sdk/README.md@@ -83,6 +83,21 @@ $doc = $client->scrape('https://example.com/product', ScrapeOptions::with(echo $doc->getJson(); // Structured data```+### Product Extraction++Use the `product` format on product pages for structured product extraction+(title, brand, category, and per-variant price, availability, and images).+It is the deterministic counterpart to the LLM-based `json` format, returned+on the document's `getProduct()`.++```php+$doc = $client->scrape('https://example.com/product', ScrapeOptions::with(+ formats: ['product'],+));++echo $doc->getProduct(); // Structured product data+```+### Video ExtractionUse the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.@@ -99,7 +114,7 @@ echo $doc->getVideo();Upload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data andparse it synchronously. Parse options intentionally exclude browser-only-features such as change tracking, screenshot, branding, audio, video, actions,+features such as change tracking, screenshot, branding, product, audio, video, actions,waitFor, location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.```phpdiff --git a/apps/php-sdk/src/Models/Document.php b/apps/php-sdk/src/Models/Document.phpindex 5b4f02c98..3a257c57a 100644--- a/apps/php-sdk/src/Models/Document.php+++ b/apps/php-sdk/src/Models/Document.php@@ -34,6 +34,7 @@ final class Documentprivate readonly ?string $warning = null,private readonly ?array $changeTracking = null,private readonly ?array $branding = null,+ private readonly ?Product $product = null,) {}/** @param array<string, mixed> $data */@@ -58,6 +59,9 @@ final class Documentwarning: $data['warning'] ?? null,changeTracking: $data['changeTracking'] ?? null,branding: $data['branding'] ?? null,+ product: isset($data['product']) && is_array($data['product'])+ ? Product::fromArray($data['product'])+ : null,);}@@ -157,4 +161,9 @@ final class Document{return $this->branding;}++ public function getProduct(): ?Product+ {+ return $this->product;+ }}diff --git a/apps/php-sdk/src/Models/ParseOptions.php b/apps/php-sdk/src/Models/ParseOptions.phpindex d4de1530a..1377a2845 100644--- a/apps/php-sdk/src/Models/ParseOptions.php+++ b/apps/php-sdk/src/Models/ParseOptions.php@@ -10,7 +10,7 @@ use Firecrawl\Exceptions\FirecrawlException;* Options for parsing uploaded files via `/v2/parse`.** Parse does not support browser-rendering features (actions, waitFor,- * location, mobile) nor the screenshot, branding, audio, video, or changeTracking formats.+ * location, mobile) nor the screenshot, branding, product, audio, video, or changeTracking formats.* The proxy field only accepts "auto" or "basic".*/final class ParseOptions@@ -20,6 +20,7 @@ final class ParseOptions'screenshot','screenshot@fullPage','branding',+ 'product','audio','video',];diff --git a/apps/php-sdk/src/Models/Product.php b/apps/php-sdk/src/Models/Product.phpnew file mode 100644index 000000000..5e82fd2a0--- /dev/null+++ b/apps/php-sdk/src/Models/Product.php@@ -0,0 +1,180 @@+<?php++declare(strict_types=1);++namespace Firecrawl\Models;++/**+ * Structured product information extracted via the `product` scrape format.+ *+ * Pricing, availability, and images live on individual variants; the+ * top-level product carries identity and descriptive fields only.+ */+final class Product+{+ /**+ * @param list<array<string, mixed>> $variants+ */+ public function __construct(+ private readonly string $title,+ private readonly string $url,+ private readonly ?string $brand = null,+ private readonly ?string $category = null,+ private readonly ?string $description = null,+ private readonly array $variants = [],+ ) {}++ /** @param array<string, mixed> $data */+ public static function fromArray(array $data): self+ {+ return new self(+ title: (string) ($data['title'] ?? ''),+ url: (string) ($data['url'] ?? ''),+ brand: ($data['brand'] ?? null) !== null ? (string) $data['brand'] : null,+ category: ($data['category'] ?? null) !== null ? (string) $data['category'] : null,+ description: ($data['description'] ?? null) !== null ? (string) $data['description'] : null,+ variants: self::normalizeVariants($data['variants'] ?? []),+ );+ }++ /**+ * @param mixed $images+ * @return list<array{url: string, alt?: string|null}>+ */+ private static function normalizeImages(mixed $images): array+ {+ if (!is_array($images)) {+ return [];+ }++ $result = [];+ foreach ($images as $image) {+ if (!is_array($image) || !isset($image['url'])) {+ continue;+ }+ $entry = ['url' => (string) $image['url']];+ if (array_key_exists('alt', $image)) {+ $entry['alt'] = $image['alt'] !== null ? (string) $image['alt'] : null;+ }+ $result[] = $entry;+ }++ return $result;+ }++ /**+ * @param mixed $price+ * @return array{amount: float, currency?: string|null, formatted?: string|null}|null+ */+ private static function normalizePrice(mixed $price): ?array+ {+ if (!is_array($price) || !isset($price['amount'])) {+ return null;+ }++ $entry = ['amount' => (float) $price['amount']];+ if (array_key_exists('currency', $price)) {+ $entry['currency'] = $price['currency'] !== null ? (string) $price['currency'] : null;+ }+ if (array_key_exists('formatted', $price)) {+ $entry['formatted'] = $price['formatted'] !== null ? (string) $price['formatted'] : null;+ }++ return $entry;+ }++ /**+ * @param mixed $availability+ * @return array{inStock: bool, text?: string|null}+ */+ private static function normalizeAvailability(mixed $availability): array+ {+ if (!is_array($availability)) {+ return ['inStock' => false];+ }++ $entry = ['inStock' => (bool) ($availability['inStock'] ?? false)];+ if (array_key_exists('text', $availability)) {+ $entry['text'] = $availability['text'] !== null ? (string) $availability['text'] : null;+ }++ return $entry;+ }++ /**+ * @param mixed $variants+ * @return list<array<string, mixed>>+ */+ private static function normalizeVariants(mixed $variants): array+ {+ if (!is_array($variants)) {+ return [];+ }++ $result = [];+ foreach ($variants as $variant) {+ if (!is_array($variant)) {+ continue;+ }++ $entry = [];+ foreach (['id', 'sku', 'title'] as $key) {+ if (isset($variant[$key])) {+ $entry[$key] = (string) $variant[$key];+ }+ }+ if (isset($variant['values']) && is_array($variant['values'])) {+ $entry['values'] = $variant['values'];+ }+ if (($price = self::normalizePrice($variant['price'] ?? null)) !== null) {+ $entry['price'] = $price;+ }+ if (+ is_array($variant['sale'] ?? null)+ && ($originalPrice = self::normalizePrice($variant['sale']['originalPrice'] ?? null)) !== null+ ) {+ $entry['sale'] = ['originalPrice' => $originalPrice];+ }+ // Availability is always present on a variant.+ $entry['availability'] = self::normalizeAvailability($variant['availability'] ?? null);+ if (isset($variant['images'])) {+ $entry['images'] = self::normalizeImages($variant['images']);+ }++ $result[] = $entry;+ }++ return $result;+ }++ public function getTitle(): string+ {+ return $this->title;+ }++ public function getUrl(): string+ {+ return $this->url;+ }++ public function getBrand(): ?string+ {+ return $this->brand;+ }++ public function getCategory(): ?string+ {+ return $this->category;+ }++ public function getDescription(): ?string+ {+ return $this->description;+ }++ /** @return list<array<string, mixed>> */+ public function getVariants(): array+ {+ return $this->variants;+ }+}diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.phpindex 9cdde7df4..5fb943b39 100644--- a/apps/php-sdk/src/Version.php+++ b/apps/php-sdk/src/Version.php@@ -6,5 +6,5 @@ namespace Firecrawl;final class Version{- public const SDK_VERSION = '1.7.0';+ public const SDK_VERSION = '1.7.1';}diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.phpindex e715e081c..221e2e372 100644--- a/apps/php-sdk/tests/Unit/ModelsTest.php+++ b/apps/php-sdk/tests/Unit/ModelsTest.php@@ -4,6 +4,7 @@ declare(strict_types=1);use Firecrawl\Models\CreditUsage;use Firecrawl\Models\Document;+use Firecrawl\Models\Product;use Firecrawl\Models\MapData;use Firecrawl\Models\BatchScrapeJob;use Firecrawl\Models\CrawlJob;@@ -104,6 +105,85 @@ it('hydrates video URL in Document', function (): void {expect($doc->getVideo())->toBe('https://storage.googleapis.com/firecrawl/video.mp4');});+it('hydrates product into Product model in Document', function (): void {+ $doc = Document::fromArray([+ 'markdown' => '# Product',+ 'product' => [+ 'title' => 'Running Shoe',+ 'brand' => 'Acme',+ 'category' => 'Footwear',+ 'url' => 'https://example.com/shoe',+ 'description' => 'A fast running shoe.',+ 'variants' => [+ [+ 'id' => 'v1',+ 'sku' => 'SHOE-RED-42',+ 'title' => 'Red / 42',+ 'values' => ['color' => 'red', 'size' => '42'],+ 'price' => ['amount' => 99.99, 'currency' => 'USD', 'formatted' => '$99.99'],+ 'sale' => ['originalPrice' => ['amount' => 129.99, 'currency' => 'USD']],+ 'availability' => ['inStock' => false, 'text' => 'Sold out'],+ 'images' => [['url' => 'https://example.com/shoe-red.jpg']],+ ],+ [+ 'id' => 'v2',+ 'title' => 'Blue / 42',+ 'values' => ['color' => 'blue', 'size' => '42', 'limited' => true],+ ],+ ],+ ],+ ]);++ $product = $doc->getProduct();++ expect($product)->toBeInstanceOf(Product::class);+ expect($product->getTitle())->toBe('Running Shoe');+ expect($product->getBrand())->toBe('Acme');+ expect($product->getCategory())->toBe('Footwear');+ expect($product->getUrl())->toBe('https://example.com/shoe');+ expect($product->getDescription())->toBe('A fast running shoe.');+ expect($product->getVariants())->toHaveCount(2);++ $v1 = $product->getVariants()[0];+ expect($v1['id'])->toBe('v1');+ expect($v1['values'])->toBe(['color' => 'red', 'size' => '42']);+ expect($v1['price'])->toBe(['amount' => 99.99, 'currency' => 'USD', 'formatted' => '$99.99']);+ expect($v1['sale'])->toBe(['originalPrice' => ['amount' => 129.99, 'currency' => 'USD']]);+ expect($v1['availability'])->toBe(['inStock' => false, 'text' => 'Sold out']);+ expect($v1['images'])->toBe([['url' => 'https://example.com/shoe-red.jpg']]);++ // Availability is always present, even when omitted from the payload.+ $v2 = $product->getVariants()[1];+ expect($v2['values'])->toBe(['color' => 'blue', 'size' => '42', 'limited' => true]);+ expect($v2['availability'])->toBe(['inStock' => false]);+ expect(array_key_exists('sale', $v2))->toBeFalse();+ expect(array_key_exists('price', $v2))->toBeFalse();+});++it('returns null product when absent in Document', function (): void {+ $doc = Document::fromArray(['markdown' => '# No product']);++ expect($doc->getProduct())->toBeNull();+});++it('coerces non-string scalar identity fields without a TypeError under strict_types', function (): void {+ // Defensive: upstream data could carry a non-string scalar (e.g. a numeric+ // brand). Under declare(strict_types=1) these must be cast, not passed raw.+ $product = Product::fromArray([+ 'title' => 'Widget',+ 'url' => 'https://example.com/widget',+ 'brand' => 1234,+ 'category' => 56.7,+ 'description' => true,+ 'variants' => [],+ ]);++ expect($product->getBrand())->toBe('1234');+ expect($product->getCategory())->toBe('56.7');+ expect($product->getDescription())->toBe('1');+ expect($product->getVariants())->toBe([]);+});+it('preserves positional integration in ScrapeOptions::with', function (): void {$options = ScrapeOptions::with(null,diff --git a/apps/php-sdk/tests/Unit/ParseTest.php b/apps/php-sdk/tests/Unit/ParseTest.phpindex 832a25c1a..9fa6a6067 100644--- a/apps/php-sdk/tests/Unit/ParseTest.php+++ b/apps/php-sdk/tests/Unit/ParseTest.php@@ -45,6 +45,10 @@ it('rejects video parse format', function (): void {ParseOptions::with(formats: ['video']);})->throws(FirecrawlException::class);+it('rejects product parse format', function (): void {+ ParseOptions::with(formats: ['product']);+})->throws(FirecrawlException::class);+it('rejects invalid proxy values', function (): void {ParseOptions::with(proxy: 'stealth');})->throws(FirecrawlException::class);diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex 38ea388a2..eb12cfb79 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -58,6 +58,15 @@ doc = firecrawl.scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', formats=['print(doc.video)```+### Product extraction++Use the `product` format on product pages to deterministically pull structured product data (title, price, availability, variants). It is the deterministic counterpart to the LLM-based `json` format.++```python+doc = firecrawl.scrape('https://firecrawl.dev', formats=['product'])+print(doc.product)+```+### Parsing uploaded filesUse `parse` to upload local bytes/files (`html`, `pdf`, `docx`, etc.) as multipart form data and return the parsed document.diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 9db7941a2..f2030b575 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.30.0"+__version__ = "4.30.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_product.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_product.pynew file mode 100644index 000000000..c7ca606f0--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_product.py@@ -0,0 +1,171 @@+import pytest+from unittest.mock import Mock+from firecrawl.v2.methods.scrape import scrape+from firecrawl.v2.types import ScrapeOptions+++class TestProductFormat:+ """Unit tests for product format support."""++ def test_scrape_with_product_format_returns_product_data(self):+ """Test that scraping with product format returns product data."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "markdown": "# Example Product",+ "product": {+ "title": "Acme Running Shoe",+ "brand": "Acme",+ "category": "Footwear",+ "url": "https://example.com/shoe",+ "description": "A lightweight running shoe.",+ "variants": [+ {+ "id": "default",+ "values": {"size": "10"},+ "price": {"amount": 89.99, "currency": "USD", "formatted": "$89.99"},+ "sale": {"originalPrice": {"amount": 129.99, "currency": "USD"}},+ "availability": {"inStock": True, "text": "In stock"},+ "images": [{"url": "https://example.com/shoe.jpg", "alt": "Acme shoe"}],+ }+ ]+ }+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["product"]))++ assert result.product is not None+ assert result.product.title == "Acme Running Shoe"+ assert result.product.brand == "Acme"+ assert result.product.category == "Footwear"+ variant = result.product.variants[0]+ assert variant.price.amount == 89.99+ assert variant.price.currency == "USD"+ assert variant.sale.original_price.amount == 129.99+ assert variant.sale.original_price.currency == "USD"+ assert variant.availability.in_stock is True+ assert variant.availability.text == "In stock"+ assert variant.images[0].url == "https://example.com/shoe.jpg"++ def test_scrape_with_product_and_markdown_formats_returns_both(self):+ """Test that scraping with both product and markdown formats returns both."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "markdown": "# Example Content",+ "product": {+ "title": "Acme Mug",+ "url": "https://example.com/mug",+ "variants": [+ {+ "price": {"amount": 12.5, "currency": "USD"},+ "availability": {"inStock": True}+ }+ ]+ }+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown", "product"]))++ assert result.markdown == "# Example Content"+ assert result.product is not None+ assert result.product.title == "Acme Mug"+ assert result.product.variants[0].price.amount == 12.5++ def test_scrape_without_product_format_does_not_return_product(self):+ """Test that scraping without product format does not return product."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "markdown": "# Example"+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown"]))++ assert result.markdown == "# Example"+ assert result.product is None++ def test_non_product_page_yields_warning_and_no_product(self):+ """Test that a non-product page scraped with product format yields a warning and no product."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "markdown": "# Blog Post",+ "warning": "No product found on this page."+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["product"]))++ assert result.product is None+ assert "No product found" in result.warning++ def test_product_format_with_variants_populated(self):+ """Test product format with variants populated, including camelCase aliasing."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "product": {+ "title": "Acme T-Shirt",+ "brand": "Acme",+ "url": "https://example.com/tshirt",+ "variants": [+ {+ "id": "v1",+ "sku": "TSHIRT-S-RED",+ "title": "Small / Red",+ "values": {"size": "S", "color": "Red"},+ "price": {"amount": 24.0, "currency": "USD"},+ "availability": {"inStock": True},+ "images": [{"url": "https://example.com/tshirt-red.jpg"}]+ },+ {+ "id": "v2",+ "sku": "TSHIRT-L-BLUE",+ "title": "Large / Blue",+ "values": {"size": "L", "color": "Blue"},+ "availability": {"inStock": False, "text": "Sold out"}+ }+ ]+ }+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["product"]))++ assert result.product is not None+ assert len(result.product.variants) == 2+ assert result.product.variants[0].sku == "TSHIRT-S-RED"+ assert result.product.variants[0].values["color"] == "Red"+ assert result.product.variants[0].availability.in_stock is True+ assert result.product.variants[0].images[0].url == "https://example.com/tshirt-red.jpg"+ assert result.product.variants[1].availability.in_stock is False+ assert result.product.variants[1].availability.text == "Sold out"diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 55b24b297..742a2f68e 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -262,6 +262,70 @@ class BrandingProfile(BaseModel):personality: Optional[Dict[str, Any]] = None+class ProductPrice(BaseModel):+ """A monetary price for a product or variant."""++ model_config = {"extra": "allow", "populate_by_name": True}++ amount: float+ currency: Optional[str] = None+ formatted: Optional[str] = None+++class ProductAvailability(BaseModel):+ """Availability information for a product or variant."""++ model_config = {"extra": "allow", "populate_by_name": True}++ in_stock: bool = Field(alias="inStock")+ text: Optional[str] = None+++class ProductImage(BaseModel):+ """An image associated with a product or variant."""++ model_config = {"extra": "allow", "populate_by_name": True}++ url: str+ alt: Optional[str] = None+++class ProductSale(BaseModel):+ """Sale information for a variant, holding the pre-sale price."""++ model_config = {"extra": "allow", "populate_by_name": True}++ original_price: ProductPrice = Field(alias="originalPrice")+++class ProductVariant(BaseModel):+ """A purchasable variant of a product (e.g. a size/color combination)."""++ model_config = {"extra": "allow", "populate_by_name": True}++ id: Optional[str] = None+ sku: Optional[str] = None+ title: Optional[str] = None+ values: Optional[Dict[str, Any]] = None+ price: Optional[ProductPrice] = None+ sale: Optional[ProductSale] = None+ availability: ProductAvailability+ images: Optional[List[ProductImage]] = None+++class ProductProfile(BaseModel):+ """Structured product information extracted from a website."""++ model_config = {"extra": "allow", "populate_by_name": True}++ title: str+ brand: Optional[str] = None+ category: Optional[str] = None+ url: str+ description: Optional[str] = None+ variants: List[ProductVariant] = Field(default_factory=list)++RedactPIIEntity = Literal["PERSON","EMAIL",@@ -292,6 +356,7 @@ class Document(BaseModel):warning: Optional[str] = Nonechange_tracking: Optional[Dict[str, Any]] = Nonebranding: Optional[BrandingProfile] = None+ product: Optional[ProductProfile] = None@propertydef metadata_typed(self) -> DocumentMetadata:@@ -415,6 +480,7 @@ FormatString = Literal["json","attributes","branding",+ "product","query","audio","video",diff --git a/apps/ruby-sdk/README.md b/apps/ruby-sdk/README.mdindex 032ccaa53..6b8876b05 100644--- a/apps/ruby-sdk/README.md+++ b/apps/ruby-sdk/README.md@@ -74,11 +74,25 @@ doc = client.scrape("https://www.youtube.com/watch?v=dQw4w9WgXcQ",puts doc.video```+### Product Extraction++Use the `product` format on product pages to get structured product data+(title, brand, category, and per-variant price, availability, and images).+It is the deterministic counterpart to the LLM-based `json` format. The+returned `product` field contains the extracted fields.++```ruby+doc = client.scrape("https://example.com/products/widget",+ Firecrawl::Models::ScrapeOptions.new(formats: ["product"]))++puts doc.product+```+### ParseUpload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data andparse it synchronously. Parse options intentionally exclude browser-only-features such as change tracking, screenshot, branding, audio, video, actions,+features such as change tracking, screenshot, branding, audio, video, product, actions,wait_for, location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.```rubydiff --git a/apps/ruby-sdk/lib/firecrawl.rb b/apps/ruby-sdk/lib/firecrawl.rbindex 64cd6dd5a..101d23d24 100644--- a/apps/ruby-sdk/lib/firecrawl.rb+++ b/apps/ruby-sdk/lib/firecrawl.rb@@ -4,6 +4,7 @@ require_relative "firecrawl/version"require_relative "firecrawl/errors"require_relative "firecrawl/http_client"require_relative "firecrawl/models/query_format"+require_relative "firecrawl/models/product_profile"require_relative "firecrawl/models/document"require_relative "firecrawl/models/scrape_options"require_relative "firecrawl/models/crawl_options"diff --git a/apps/ruby-sdk/lib/firecrawl/models/document.rb b/apps/ruby-sdk/lib/firecrawl/models/document.rbindex 0b0200dc7..00f2d38f7 100644--- a/apps/ruby-sdk/lib/firecrawl/models/document.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/document.rb@@ -7,7 +7,7 @@ module Firecrawlattr_reader :markdown, :html, :raw_html, :json, :summary,:metadata, :links, :images, :screenshot, :audio,:video, :attributes, :actions, :answer, :highlights,- :warning, :change_tracking, :branding+ :warning, :change_tracking, :branding, :productdef initialize(data)@markdown = data["markdown"]@@ -28,6 +28,7 @@ module Firecrawl@warning = data["warning"]@change_tracking = data["changeTracking"]@branding = data["branding"]+ @product = data["product"] && ProductProfile.new(data["product"])enddef to_sdiff --git a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rbindex ddfadfee4..ca2c4f874 100644--- a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb@@ -5,10 +5,10 @@ module Firecrawl# Options for parsing uploaded files via `/v2/parse`.## Parse does not support browser-rendering features (actions, waitFor,- # location, mobile) nor screenshot/branding/audio/video/changeTracking formats. The+ # location, mobile) nor screenshot/branding/product/audio/video/changeTracking formats. The# proxy field only accepts "auto" or "basic".class ParseOptions- UNSUPPORTED_FORMATS = %w[changeTracking screenshot screenshot@fullPage branding audio video].freeze+ UNSUPPORTED_FORMATS = %w[changeTracking screenshot screenshot@fullPage branding product audio video].freezeFIELDS = %i[formats headers include_tags exclude_tags only_main_contentdiff --git a/apps/ruby-sdk/lib/firecrawl/models/product_profile.rb b/apps/ruby-sdk/lib/firecrawl/models/product_profile.rbnew file mode 100644index 000000000..9552968b3--- /dev/null+++ b/apps/ruby-sdk/lib/firecrawl/models/product_profile.rb@@ -0,0 +1,83 @@+# frozen_string_literal: true++module Firecrawl+ module Models+ # Structured product information extracted from a product page via the+ # `product` scrape format.+ class ProductProfile+ # An image associated with a product or variant.+ class Image+ attr_reader :url, :alt++ def initialize(data)+ @url = data["url"]+ @alt = data["alt"]+ end+ end++ # A monetary value with an optional currency and formatted string.+ class Price+ attr_reader :amount, :currency, :formatted++ def initialize(data)+ @amount = data["amount"]+ @currency = data["currency"]+ @formatted = data["formatted"]+ end+ end++ # Stock availability information for a variant. Always present.+ class Availability+ attr_reader :in_stock, :text++ def initialize(data)+ data ||= {}+ @in_stock = data["inStock"] || false+ @text = data["text"]+ end+ end++ # Sale pricing for a variant, carrying the pre-sale original price.+ class Sale+ attr_reader :original_price++ def initialize(data)+ @original_price = data["originalPrice"] && Price.new(data["originalPrice"])+ end+ end++ # A purchasable variant of a product. Pricing, availability, and images+ # live here rather than on the top-level product.+ class Variant+ attr_reader :id, :sku, :title, :values, :price, :sale,+ :availability, :images++ def initialize(data)+ @id = data["id"]+ @sku = data["sku"]+ @title = data["title"]+ @values = data["values"]+ @price = data["price"] && Price.new(data["price"])+ @sale = data["sale"] && Sale.new(data["sale"])+ @availability = Availability.new(data["availability"])+ @images = (data["images"] || []).map { |img| Image.new(img) }+ end+ end++ attr_reader :title, :brand, :category, :url, :description, :variants++ def initialize(data)+ @title = data["title"]+ @brand = data["brand"]+ @category = data["category"]+ @url = data["url"]+ @description = data["description"]+ @variants = (data["variants"] || []).map { |variant| Variant.new(variant) }+ end++ def to_s+ "ProductProfile{title=#{title || 'untitled'}, url=#{url || 'unknown'}}"+ end+ end+ end+enddiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex 23865bfe8..e762a2a78 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.9.0"+ VERSION = "1.9.1"enddiff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rbindex 9581408db..c2ca91934 100644--- a/apps/ruby-sdk/test/firecrawl/client_test.rb+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb@@ -104,6 +104,93 @@ class ClientTest < Minitest::Testassert_raises(ArgumentError) { @client.scrape(nil) }end+ def test_scrape_with_product_format+ stub_request(:post, "#{BASE_URL}/v2/scrape")+ .to_return(+ status: 200,+ body: JSON.generate(+ data: {+ markdown: "# Widget",+ product: {+ title: "Acme Widget",+ brand: "Acme",+ category: "Gadgets",+ url: "https://example.com/widget",+ description: "A fine widget.",+ variants: [+ {+ id: "v1",+ sku: "ACME-1",+ title: "Large",+ values: { size: "L" },+ price: { amount: 2199, currency: "USD" },+ sale: { originalPrice: { amount: 2999, currency: "USD", formatted: "$29.99" } },+ availability: { inStock: true, text: "In stock" },+ images: [{ url: "https://example.com/v1.jpg", alt: "Widget" }],+ },+ {+ id: "v2",+ title: "Small",+ values: { size: "S" },+ },+ ],+ },+ }+ ),+ headers: { "Content-Type" => "application/json" }+ )++ doc = @client.scrape("https://example.com/widget")+ product = doc.product+ assert_instance_of Firecrawl::Models::ProductProfile, product+ assert_equal "Acme Widget", product.title+ assert_equal "Acme", product.brand+ assert_equal "Gadgets", product.category+ assert_equal "https://example.com/widget", product.url+ assert_equal "A fine widget.", product.description++ assert_equal 2, product.variants.size+ variant = product.variants.first+ assert_equal "v1", variant.id+ assert_equal "ACME-1", variant.sku+ assert_equal "Large", variant.title+ assert_equal({ "size" => "L" }, variant.values)++ assert_equal 2199, variant.price.amount+ assert_equal "USD", variant.price.currency++ assert_equal 2999, variant.sale.original_price.amount+ assert_equal "$29.99", variant.sale.original_price.formatted++ assert_equal true, variant.availability.in_stock+ assert_equal "In stock", variant.availability.text++ assert_equal 1, variant.images.size+ assert_equal "https://example.com/v1.jpg", variant.images.first.url+ assert_equal "Widget", variant.images.first.alt++ # Availability is always present; sale/price/images are optional.+ bare = product.variants.last+ assert_equal "v2", bare.id+ assert_nil bare.price+ assert_nil bare.sale+ refute_nil bare.availability+ assert_equal false, bare.availability.in_stock+ assert_empty bare.images+ end++ def test_scrape_without_product_format_leaves_product_nil+ stub_request(:post, "#{BASE_URL}/v2/scrape")+ .to_return(+ status: 200,+ body: JSON.generate(data: { markdown: "# Hi" }),+ headers: { "Content-Type" => "application/json" }+ )++ doc = @client.scrape("https://example.com")+ assert_nil doc.product+ end+# ================================================================# CRAWL# ================================================================@@ -688,6 +775,12 @@ class ClientTest < Minitest::Testendend+ def test_parse_options_rejects_product_format+ assert_raises(ArgumentError) do+ Firecrawl::Models::ParseOptions.new(formats: ["product"])+ end+ end+def test_parse_options_rejects_invalid_proxyassert_raises(ArgumentError) doFirecrawl::Models::ParseOptions.new(proxy: "stealth")diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex eb499e5eb..a78b1a047 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.9.0"+version = "2.9.1"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 9f4eec34e..fbb6b9a87 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.9.0"+version = "2.9.1"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.mdindex eba032b56..023c108a7 100644--- a/apps/rust-sdk/README.md+++ b/apps/rust-sdk/README.md@@ -58,6 +58,24 @@ let doc = clientprintln!("{:?}", doc.video);```+### Product extraction++Use `Format::Product` on product pages for structured product extraction (title, price, availability, variants). The result is returned on the document's `product` field. This is the deterministic counterpart to the LLM-based `json` format.++```rust+use firecrawl::{Format, ScrapeOptions};++let options = ScrapeOptions {+ formats: Some(vec![Format::Product]),+ ..Default::default()+};++let doc = client+ .scrape("https://www.example.com/product/123", options)+ .await?;+println!("{:?}", doc.product);+```+### Parsing uploaded files (v2)Use the v2 client `parse` method to upload local files (`html`, `pdf`, `docx`, etc.) as multipart form data.diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rsindex 4e8d03cde..acf42854a 100644--- a/apps/rust-sdk/src/types.rs+++ b/apps/rust-sdk/src/types.rs@@ -31,6 +31,8 @@ pub enum Format {Attributes,/// Brand analysis of the page.Branding,+ /// Product extraction from the page.+ Product,/// Audio extraction (MP3) from YouTube videos.Audio,/// Video extraction from supported video URLs.@@ -60,6 +62,7 @@ impl Serialize for Format {Format::Json => serializer.serialize_str("json"),Format::Attributes => serializer.serialize_str("attributes"),Format::Branding => serializer.serialize_str("branding"),+ Format::Product => serializer.serialize_str("product"),Format::Audio => serializer.serialize_str("audio"),Format::Video => serializer.serialize_str("video"),Format::Question(question) => question.serialize(serializer),@@ -88,6 +91,7 @@ impl<'de> Deserialize<'de> for Format {"json" => Ok(Format::Json),"attributes" => Ok(Format::Attributes),"branding" => Ok(Format::Branding),+ "product" => Ok(Format::Product),"audio" => Ok(Format::Audio),"video" => Ok(Format::Video),_ => Err(de::Error::custom(format!("unknown format: {}", format))),@@ -678,6 +682,96 @@ pub struct Document {pub change_tracking: Option<Value>,/// Branding analysis.pub branding: Option<Value>,+ /// Product extraction result.+ pub product: Option<Product>,+}++/// Product extraction result for a page.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct Product {+ /// Product title.+ pub title: String,+ /// Brand name.+ pub brand: Option<String>,+ /// Product category.+ pub category: Option<String>,+ /// Product URL.+ pub url: String,+ /// Product description.+ pub description: Option<String>,+ /// Product variants.+ #[serde(default)]+ pub variants: Vec<ProductVariant>,+}++/// An image associated with a product.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ProductImage {+ /// Image URL.+ pub url: String,+ /// Alternative text for the image.+ pub alt: Option<String>,+}++/// Price information for a product.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ProductPrice {+ /// Numeric price amount.+ pub amount: f64,+ /// Currency code.+ pub currency: Option<String>,+ /// Human-readable formatted price.+ pub formatted: Option<String>,+}++/// Availability information for a product.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ProductAvailability {+ /// Whether the product is in stock.+ #[serde(rename = "inStock")]+ pub in_stock: bool,+ /// Human-readable availability text.+ pub text: Option<String>,+}++/// A variant of a product.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ProductVariant {+ /// Variant identifier.+ pub id: Option<String>,+ /// Stock keeping unit.+ pub sku: Option<String>,+ /// Variant title.+ pub title: Option<String>,+ /// Variant option values (e.g. size, color).+ pub values: Option<HashMap<String, serde_json::Value>>,+ /// Variant price.+ pub price: Option<ProductPrice>,+ /// Sale information, present when the variant is discounted.+ pub sale: Option<ProductSale>,+ /// Variant availability information (always present).+ pub availability: ProductAvailability,+ /// Variant images.+ pub images: Option<Vec<ProductImage>>,+}++/// Sale information for a product variant.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ProductSale {+ /// Original price before the discount.+ pub original_price: ProductPrice,}/// Job status types for crawl and batch operations.diff --git a/apps/test-site/src/pages/product.astro b/apps/test-site/src/pages/product.astronew file mode 100644index 000000000..b6e14c241--- /dev/null+++ b/apps/test-site/src/pages/product.astro@@ -0,0 +1,50 @@+---+import BaseHead from '../components/BaseHead.astro';+import Footer from '../components/Footer.astro';+import Header from '../components/Header.astro';++// Static, deterministic JSON-LD Product fixture used by the API `product`+// scrape-format e2e (snips) test. Keep the title/price/currency/availability+// stable — `apps/api/src/__tests__/snips/v2/product.test.ts` asserts on them.+const productJsonLd = {+ '@context': 'https://schema.org',+ '@type': 'Product',+ name: 'Firecrawl Test Widget',+ description: 'A deterministic sample product used for end-to-end scrape testing.',+ sku: 'FC-WIDGET-001',+ brand: { '@type': 'Brand', name: 'Firecrawl' },+ image: 'https://firecrawl.dev/firecrawl-logo.png',+ offers: {+ '@type': 'Offer',+ price: '49.99',+ priceCurrency: 'USD',+ availability: 'https://schema.org/InStock',+ },+};+---++<!doctype html>+<html lang="en">+ <head>+ <BaseHead title="Firecrawl Test Widget" description="A deterministic sample product used for end-to-end scrape testing." />+ <script type="application/ld+json" is:inline set:html={JSON.stringify(productJsonLd)}></script>+ </head>+ <body>+ <Header />+ <main>+ <h1>Firecrawl Test Widget</h1>+ <p>+ This page is a deterministic sample product page. It embeds a+ schema.org <code>Product</code> as JSON-LD so the Firecrawl+ <code>product</code> scrape format has a stable target to extract.+ </p>+ <ul>+ <li>Title: Firecrawl Test Widget</li>+ <li>Brand: Firecrawl</li>+ <li>Price: $49.99 USD</li>+ <li>Availability: In stock</li>+ </ul>+ </main>+ <Footer />+ </body>+</html>