Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add branding format support to JS and Python SDKs (#2360)
devin-ai-integration[bot]dabf991ca9c4
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex f6a3fc6f7..f700ac4c7 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.4.1",+ "version": "4.5.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/branding.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/branding.test.tsnew file mode 100644index 000000000..c18a76e80--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/branding.test.ts@@ -0,0 +1,186 @@+import { describe, test, expect, jest } from "@jest/globals";+import { scrape } from "../../../v2/methods/scrape";++describe("JS SDK v2 branding format", () => {+ function makeHttp(postImpl: (url: string, data: any) => any) {+ return { post: jest.fn(async (u: string, d: any) => postImpl(u, d)) } as any;+ }++ test("scrape with branding format returns branding data", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ markdown: "# Example",+ branding: {+ colorScheme: "light",+ colors: {+ primary: "#E11D48",+ secondary: "#3B82F6",+ accent: "#F59E0B"+ },+ typography: {+ fontFamilies: {+ primary: "Inter",+ heading: "Poppins"+ },+ fontSizes: {+ h1: "2.5rem",+ body: "1rem"+ }+ },+ spacing: {+ baseUnit: 8+ },+ components: {+ buttonPrimary: {+ background: "#E11D48",+ textColor: "#FFFFFF",+ borderRadius: "0.5rem"+ }+ }+ }+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["branding"] });++ expect(result.branding).toBeDefined();+ expect(result.branding?.colorScheme).toBe("light");+ expect(result.branding?.colors?.primary).toBe("#E11D48");+ expect(result.branding?.typography?.fontFamilies?.primary).toBe("Inter");+ expect(result.branding?.spacing?.baseUnit).toBe(8);+ expect(result.branding?.components?.buttonPrimary?.background).toBe("#E11D48");+ });++ test("scrape with branding and markdown formats returns both", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ markdown: "# Example Content",+ branding: {+ colorScheme: "dark",+ colors: {+ primary: "#10B981"+ },+ typography: {+ fontFamilies: {+ primary: "Roboto"+ }+ }+ }+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["markdown", "branding"] });++ expect(result.markdown).toBe("# Example Content");+ expect(result.branding).toBeDefined();+ expect(result.branding?.colorScheme).toBe("dark");+ expect(result.branding?.colors?.primary).toBe("#10B981");+ });++ test("scrape without branding format does not return branding", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ markdown: "# Example"+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["markdown"] });++ expect(result.markdown).toBe("# Example");+ expect(result.branding).toBeUndefined();+ });++ test("branding format with all nested fields", async () => {+ const mockResponse = {+ status: 200,+ data: {+ success: true,+ data: {+ branding: {+ colorScheme: "light",+ logo: "https://example.com/logo.png",+ fonts: [+ { family: "Inter", weight: 400 },+ { family: "Poppins", weight: 700 }+ ],+ colors: {+ primary: "#E11D48",+ background: "#FFFFFF"+ },+ typography: {+ fontFamilies: { primary: "Inter" },+ fontStacks: { body: ["Inter", "sans-serif"] },+ fontSizes: { h1: "2.5rem" },+ lineHeights: { body: 1.5 },+ fontWeights: { regular: 400 }+ },+ spacing: {+ baseUnit: 8,+ padding: { sm: 8, md: 16 }+ },+ components: {+ buttonPrimary: {+ background: "#E11D48",+ textColor: "#FFFFFF"+ }+ },+ icons: {+ style: "outline",+ primaryColor: "#E11D48"+ },+ images: {+ logo: "https://example.com/logo.png",+ favicon: "https://example.com/favicon.ico"+ },+ animations: {+ transitionDuration: "200ms",+ easing: "ease-in-out"+ },+ layout: {+ grid: { columns: 12, maxWidth: "1200px" },+ headerHeight: "64px"+ },+ tone: {+ voice: "professional",+ emojiUsage: "minimal"+ },+ personality: {+ tone: "professional",+ energy: "medium",+ targetAudience: "developers"+ }+ }+ }+ }+ };++ const http = makeHttp(() => mockResponse);+ const result = await scrape(http, "https://example.com", { formats: ["branding"] });++ expect(result.branding).toBeDefined();+ expect(result.branding?.logo).toBe("https://example.com/logo.png");+ expect(result.branding?.fonts).toHaveLength(2);+ expect(result.branding?.typography?.fontStacks?.body).toEqual(["Inter", "sans-serif"]);+ expect(result.branding?.spacing?.padding).toEqual({ sm: 8, md: 16 });+ expect(result.branding?.icons?.style).toBe("outline");+ expect(result.branding?.images?.favicon).toBe("https://example.com/favicon.ico");+ expect(result.branding?.animations?.easing).toBe("ease-in-out");+ expect(result.branding?.layout?.grid?.columns).toBe(12);+ expect(result.branding?.personality?.tone).toBe("professional");+ });+});diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 13c58cd0e..fb7733c84 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -11,7 +11,8 @@ export type FormatString =| 'summary'| 'changeTracking'| 'json'- | 'attributes';+ | 'attributes'+ | 'branding';export interface Viewport {width: number;@@ -165,6 +166,141 @@ export interface WebhookConfig {events?: Array<'completed' | 'failed' | 'page' | 'started'>;}+export interface BrandingProfile {+ colorScheme?: 'light' | 'dark';+ logo?: string | null;+ fonts?: Array<{+ family: string;+ [key: string]: unknown;+ }>;+ colors?: {+ primary?: string;+ secondary?: string;+ accent?: string;+ background?: string;+ textPrimary?: string;+ textSecondary?: string;+ link?: string;+ success?: string;+ warning?: string;+ error?: string;+ [key: string]: string | undefined;+ };+ typography?: {+ fontFamilies?: {+ primary?: string;+ heading?: string;+ code?: string;+ [key: string]: string | undefined;+ };+ fontStacks?: {+ primary?: string[];+ heading?: string[];+ body?: string[];+ paragraph?: string[];+ [key: string]: string[] | undefined;+ };+ fontSizes?: {+ h1?: string;+ h2?: string;+ h3?: string;+ body?: string;+ small?: string;+ [key: string]: string | undefined;+ };+ lineHeights?: {+ heading?: number;+ body?: number;+ [key: string]: number | undefined;+ };+ fontWeights?: {+ light?: number;+ regular?: number;+ medium?: number;+ bold?: number;+ [key: string]: number | undefined;+ };+ };+ spacing?: {+ baseUnit?: number;+ padding?: Record<string, number>;+ margins?: Record<string, number>;+ gridGutter?: number;+ borderRadius?: string;+ [key: string]: number | string | Record<string, number> | undefined;+ };+ components?: {+ buttonPrimary?: {+ background?: string;+ textColor?: string;+ borderColor?: string;+ borderRadius?: string;+ [key: string]: string | undefined;+ };+ buttonSecondary?: {+ background?: string;+ textColor?: string;+ borderColor?: string;+ borderRadius?: string;+ [key: string]: string | undefined;+ };+ input?: {+ borderColor?: string;+ focusBorderColor?: string;+ borderRadius?: string;+ [key: string]: string | undefined;+ };+ [key: string]: unknown;+ };+ icons?: {+ style?: string;+ primaryColor?: string;+ [key: string]: string | undefined;+ };+ images?: {+ logo?: string | null;+ favicon?: string | null;+ ogImage?: string | null;+ [key: string]: string | null | undefined;+ };+ animations?: {+ transitionDuration?: string;+ easing?: string;+ [key: string]: string | undefined;+ };+ layout?: {+ grid?: {+ columns?: number;+ maxWidth?: string;+ [key: string]: number | string | undefined;+ };+ headerHeight?: string;+ footerHeight?: string;+ [key: string]:+ | number+ | string+ | Record<string, number | string | undefined>+ | undefined;+ };+ tone?: {+ voice?: string;+ emojiUsage?: string;+ [key: string]: string | undefined;+ };+ personality?: {+ tone:+ | 'professional'+ | 'playful'+ | 'modern'+ | 'traditional'+ | 'minimalist'+ | 'bold';+ energy: 'low' | 'medium' | 'high';+ targetAudience: string;+ };+ [key: string]: unknown;+}+export interface DocumentMetadata {// Common metadata fieldstitle?: string;@@ -239,6 +375,7 @@ export interface Document {actions?: Record<string, unknown>;warning?: string;changeTracking?: Record<string, unknown>;+ branding?: BrandingProfile;}// Pagination configuration for auto-fetching pages from v2 endpoints that return a `next` URLdiff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex ce904ecd5..d73378b82 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.5.0"+__version__ = "4.6.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_branding.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_branding.pynew file mode 100644index 000000000..e4da89adf--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_branding.py@@ -0,0 +1,214 @@+import pytest+from unittest.mock import Mock, MagicMock+from firecrawl.v2.methods.scrape import scrape+from firecrawl.v2.types import ScrapeOptions, Document+++class TestBrandingFormat:+ """Unit tests for branding format support."""++ def test_scrape_with_branding_format_returns_branding_data(self):+ """Test that scraping with branding format returns branding data."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "markdown": "# Example",+ "branding": {+ "colorScheme": "light",+ "colors": {+ "primary": "#E11D48",+ "secondary": "#3B82F6",+ "accent": "#F59E0B"+ },+ "typography": {+ "fontFamilies": {+ "primary": "Inter",+ "heading": "Poppins"+ },+ "fontSizes": {+ "h1": "2.5rem",+ "body": "1rem"+ }+ },+ "spacing": {+ "baseUnit": 8+ },+ "components": {+ "buttonPrimary": {+ "background": "#E11D48",+ "textColor": "#FFFFFF",+ "borderRadius": "0.5rem"+ }+ }+ }+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))++ assert result.branding is not None+ assert result.branding.color_scheme == "light"+ assert result.branding.colors["primary"] == "#E11D48"+ assert result.branding.typography["fontFamilies"]["primary"] == "Inter"+ assert result.branding.spacing["baseUnit"] == 8+ assert result.branding.components["buttonPrimary"]["background"] == "#E11D48"++ def test_scrape_with_branding_and_markdown_formats_returns_both(self):+ """Test that scraping with both branding and markdown formats returns both."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "markdown": "# Example Content",+ "branding": {+ "colorScheme": "dark",+ "colors": {+ "primary": "#10B981"+ },+ "typography": {+ "fontFamilies": {+ "primary": "Roboto"+ }+ }+ }+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown", "branding"]))++ assert result.markdown == "# Example Content"+ assert result.branding is not None+ assert result.branding.color_scheme == "dark"+ assert result.branding.colors["primary"] == "#10B981"++ def test_scrape_without_branding_format_does_not_return_branding(self):+ """Test that scraping without branding format does not return branding."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "markdown": "# Example"+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown"]))++ assert result.markdown == "# Example"+ assert result.branding is None++ def test_branding_format_with_all_nested_fields(self):+ """Test branding format with all nested fields populated."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "branding": {+ "colorScheme": "light",+ "logo": "https://example.com/logo.png",+ "fonts": [+ {"family": "Inter", "weight": 400},+ {"family": "Poppins", "weight": 700}+ ],+ "colors": {+ "primary": "#E11D48",+ "background": "#FFFFFF"+ },+ "typography": {+ "fontFamilies": {"primary": "Inter"},+ "fontStacks": {"body": ["Inter", "sans-serif"]},+ "fontSizes": {"h1": "2.5rem"},+ "lineHeights": {"body": 1.5},+ "fontWeights": {"regular": 400}+ },+ "spacing": {+ "baseUnit": 8,+ "padding": {"sm": 8, "md": 16}+ },+ "components": {+ "buttonPrimary": {+ "background": "#E11D48",+ "textColor": "#FFFFFF"+ }+ },+ "icons": {+ "style": "outline",+ "primaryColor": "#E11D48"+ },+ "images": {+ "logo": "https://example.com/logo.png",+ "favicon": "https://example.com/favicon.ico"+ },+ "animations": {+ "transitionDuration": "200ms",+ "easing": "ease-in-out"+ },+ "layout": {+ "grid": {"columns": 12, "maxWidth": "1200px"},+ "headerHeight": "64px"+ },+ "tone": {+ "voice": "professional",+ "emojiUsage": "minimal"+ },+ "personality": {+ "tone": "professional",+ "energy": "medium",+ "targetAudience": "developers"+ }+ }+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))++ assert result.branding is not None+ assert result.branding.color_scheme == "light"+ assert result.branding.logo == "https://example.com/logo.png"+ assert len(result.branding.fonts) == 2+ assert result.branding.typography["fontStacks"]["body"] == ["Inter", "sans-serif"]+ assert result.branding.spacing["padding"] == {"sm": 8, "md": 16}+ assert result.branding.icons["style"] == "outline"+ assert result.branding.images["favicon"] == "https://example.com/favicon.ico"+ assert result.branding.animations["easing"] == "ease-in-out"+ assert result.branding.layout["grid"]["columns"] == 12+ assert result.branding.personality["tone"] == "professional"++ def test_branding_colorscheme_normalization(self):+ """Test that colorScheme is normalized to color_scheme."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "data": {+ "branding": {+ "colorScheme": "dark",+ "colors": {"primary": "#000000"}+ }+ }+ }++ mock_client = Mock()+ mock_client.post.return_value = mock_response++ result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))++ assert result.branding is not None+ assert result.branding.color_scheme == "dark"+ assert not hasattr(result.branding, "colorScheme")diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex f8980c2c4..90c085bcd 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -124,6 +124,24 @@ class AttributeResult(BaseModel):attribute: strvalues: List[str]+class BrandingProfile(BaseModel):+ """Branding information extracted from a website."""+ model_config = {"extra": "allow"}++ color_scheme: Optional[Literal["light", "dark"]] = None+ logo: Optional[str] = None+ fonts: Optional[List[Dict[str, Any]]] = None+ colors: Optional[Dict[str, str]] = None+ typography: Optional[Dict[str, Any]] = None+ spacing: Optional[Dict[str, Any]] = None+ components: Optional[Dict[str, Any]] = None+ icons: Optional[Dict[str, str]] = None+ images: Optional[Dict[str, Optional[str]]] = None+ animations: Optional[Dict[str, str]] = None+ layout: Optional[Dict[str, Any]] = None+ tone: Optional[Dict[str, str]] = None+ personality: Optional[Dict[str, Any]] = None+class Document(BaseModel):"""A scraped document."""markdown: Optional[str] = None@@ -138,6 +156,7 @@ class Document(BaseModel):actions: Optional[Dict[str, Any]] = Nonewarning: Optional[str] = Nonechange_tracking: Optional[Dict[str, Any]] = None+ branding: Optional[BrandingProfile] = None@propertydef metadata_typed(self) -> DocumentMetadata:@@ -199,7 +218,7 @@ CategoryOption = Union[str, Category]FormatString = Literal[# camelCase versions (API format)- "markdown", "html", "rawHtml", "links", "images", "screenshot", "summary", "changeTracking", "json", "attributes",+ "markdown", "html", "rawHtml", "links", "images", "screenshot", "summary", "changeTracking", "json", "attributes", "branding",# snake_case versions (user-friendly)"raw_html", "change_tracking"]diff --git a/apps/python-sdk/firecrawl/v2/utils/normalize.py b/apps/python-sdk/firecrawl/v2/utils/normalize.pyindex 54415b555..b821310d1 100644--- a/apps/python-sdk/firecrawl/v2/utils/normalize.py+++ b/apps/python-sdk/firecrawl/v2/utils/normalize.py@@ -83,6 +83,7 @@ def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:Normalize a raw Document dict from the API into the Python SDK's expected shape:- Convert top-level keys rawHtml->raw_html, changeTracking->change_tracking- Convert metadata keys from camelCase to snake_case+ - Convert branding.colorScheme to branding.color_scheme"""normalized = dict(doc)@@ -102,6 +103,12 @@ def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:# Fallback to mapped dict if model construction fails for any reasonnormalized["metadata"] = mapped+ # Normalize branding top-level camelCase keys+ branding = normalized.get("branding")+ if isinstance(branding, dict):+ if "colorScheme" in branding and "color_scheme" not in branding:+ branding["color_scheme"] = branding.pop("colorScheme")+return normalized