Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): declare the source kind on fire-pdf requests and type the image parser (#4678)
Abimael Martell28d3337b3177
diff --git a/apps/api/src/scraper/scrapeURL/engines/image/index.ts b/apps/api/src/scraper/scrapeURL/engines/image/index.tsindex d07282cfa..7623bfb1f 100644--- a/apps/api/src/scraper/scrapeURL/engines/image/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/image/index.ts@@ -152,6 +152,10 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {undefined,1,"ocr",+ false,+ false,+ false,+ "image",);} catch (error) {// FirePDF answers 400 when it cannot open the bytes (truncated ordiff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.test.tsnew file mode 100644index 000000000..8b1f6f4bf--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.test.ts@@ -0,0 +1,54 @@+import { describe, expect, it } from "vitest";+import type { Meta } from "../../..";+import { buildFirePdfRequestMetadata } from "./request-metadata";++function metaWith(internalOptions: Record<string, unknown>): Meta {+ return {+ url: "https://example.com/doc.pdf",+ rewrittenUrl: undefined,+ options: {},+ internalOptions,+ } as unknown as Meta;+}++describe("buildFirePdfRequestMetadata", () => {+ it("declares a PDF source unless told otherwise", () => {+ expect(buildFirePdfRequestMetadata(metaWith({}))).toEqual({+ source_endpoint: "scrape",+ source_request_context: "default",+ source_kind: "pdf",+ url: "https://example.com/doc.pdf",+ });+ });++ it("carries the image kind on scrape and parse requests", () => {+ expect(buildFirePdfRequestMetadata(metaWith({}), "image")).toMatchObject({+ source_endpoint: "scrape",+ source_kind: "image",+ url: "https://example.com/doc.pdf",+ });+ expect(+ buildFirePdfRequestMetadata(+ metaWith({ uploadedFile: { kind: "image" } }),+ "image",+ ),+ ).toEqual({+ source_endpoint: "parse",+ source_request_context: "default",+ source_kind: "image",+ });+ });++ it("keeps the kind while dropping the URL for zero-data-retention scrapes", () => {+ expect(+ buildFirePdfRequestMetadata(+ metaWith({ zeroDataRetention: true }),+ "image",+ ),+ ).toEqual({+ source_endpoint: "scrape",+ source_request_context: "default",+ source_kind: "image",+ });+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.tsindex 9b3d954a9..d67ed2d5a 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts@@ -1,9 +1,16 @@import type { Meta } from "../../..";import { hasCustomRequestContext } from "../../../lib/request-context";-export function buildFirePdfRequestMetadata(meta: Meta): {+/** What we hand fire-pdf: a PDF, or a raster image wrapped as a one-page document. */+export type FirePdfSourceKind = "pdf" | "image";++export function buildFirePdfRequestMetadata(+ meta: Meta,+ sourceKind: FirePdfSourceKind = "pdf",+): {source_endpoint: "scrape" | "parse";source_request_context: "default" | "custom";+ source_kind: FirePdfSourceKind;url?: string;} {const isParse =@@ -14,15 +21,19 @@ export function buildFirePdfRequestMetadata(meta: Meta): {const source_request_context = hasCustomRequestContext(meta.options)? "custom": "default";+ // fire-pdf labels its metrics and logs with this so image OCR traffic can+ // be measured — and, if needed, routed — separately from PDF parsing.+ const source_kind = sourceKind;// Upload requests use synthetic URLs, and ZDR requests omit URL metadata.if (isParse || meta.internalOptions.zeroDataRetention === true) {- return { source_endpoint, source_request_context };+ return { source_endpoint, source_request_context, source_kind };}return {source_endpoint,source_request_context,+ source_kind,url: meta.rewrittenUrl ?? meta.url,};}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex 9ebc4c937..cb3a6b83b 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -8,7 +8,10 @@ import { safeMarkdownToHtml } from "./markdownToHtml";import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";import { maybeSaveResult, tryGetCached } from "./fire-pdf/cache";import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";-import { buildFirePdfRequestMetadata } from "./fire-pdf/request-metadata";+import {+ buildFirePdfRequestMetadata,+ type FirePdfSourceKind,+} from "./fire-pdf/request-metadata";/*** Reconcile an existing page count with what fire-pdf reported.@@ -49,6 +52,7 @@ export async function scrapePDFWithFirePDF(includePageMarkdown = false,includeBlocks = false,pageMarkers = false,+ sourceKind: FirePdfSourceKind = "pdf",): Promise<PDFProcessorResult> {const logger = meta.logger;@@ -139,7 +143,7 @@ export async function scrapePDFWithFirePDF(...(meta.internalOptions.crawlId && {crawl_id: meta.internalOptions.crawlId,}),- ...buildFirePdfRequestMetadata(meta),+ ...buildFirePdfRequestMetadata(meta, sourceKind),pdf_sha256: pdfSha256,source: "firecrawl",zdr,diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 6ead58ef2..5b8cf7ab3 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -188,6 +188,16 @@ export interface AuditMetadata {username: string;}+/**+ * OCR raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP, WebP, AVIF) as+ * one-page documents. Part of the default parsers list next to "pdf"; takes+ * no options, so the string "image" is equivalent. Omit it from an explicit+ * list to keep image URLs failing as unsupported files.+ */+export type ImageParser = {+ type: "image";+};+export type PDFParser = {type: "pdf";mode?: "fast" | "auto" | "ocr";@@ -249,7 +259,7 @@ export interface ScrapeOptions {timeout?: number;waitFor?: number;mobile?: boolean;- parsers?: Array<string | PDFParser>;+ parsers?: Array<string | PDFParser | ImageParser>;actions?: ActionOption[];location?: LocationConfig;skipTlsVerification?: boolean;diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pyindex 956e4ce2b..34b635193 100644--- a/apps/python-sdk/firecrawl/types.py+++ b/apps/python-sdk/firecrawl/types.py@@ -81,6 +81,7 @@ from .v2.types import (ExecuteJavascriptAction,PDFAction,PDFParser,+ ImageParser,# Usage typesQueueStatusResponse,@@ -180,6 +181,7 @@ __all__ = ['ExecuteJavascriptAction','PDFAction','PDFParser',+ 'ImageParser',# Usage types'QueueStatusResponse',diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 0cec695de..ad8b6e4e1 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -26,6 +26,7 @@ from .types import (CrawlJob,CrawlParamsRequest,PDFParser,+ ImageParser,CrawlParamsData,WebhookConfig,AgentWebhookConfig,@@ -166,7 +167,7 @@ class FirecrawlClient:timeout: Optional[int] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,- parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser, ImageParser]]]] = None,actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,location: Optional['Location'] = None,skip_tls_verification: Optional[bool] = None,@@ -590,7 +591,7 @@ class FirecrawlClient:only_main_content: Optional[bool] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,- parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser, ImageParser]]]] = None,actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,location: Optional['Location'] = None,skip_tls_verification: Optional[bool] = None,@@ -752,7 +753,7 @@ class FirecrawlClient:timeout: Optional[int] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,- parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser, ImageParser]]]] = None,actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,location: Optional['Location'] = None,skip_tls_verification: Optional[bool] = None,@@ -1273,7 +1274,7 @@ class FirecrawlClient:timeout: Optional[int] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,- parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser, ImageParser]]]] = None,actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,location: Optional['Location'] = None,skip_tls_verification: Optional[bool] = None,@@ -1786,7 +1787,7 @@ class FirecrawlClient:timeout: Optional[int] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,- parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser, ImageParser]]]] = None,actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,location: Optional['Location'] = None,skip_tls_verification: Optional[bool] = None,diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 4ef19366e..9d367ee27 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -886,7 +886,7 @@ class ScrapeOptions(BaseModel):timeout: Optional[int] = Nonewait_for: Optional[int] = Nonemobile: Optional[bool] = None- parsers: Optional[Union[List[str], List[Union[str, "PDFParser"]]]] = None+ parsers: Optional[Union[List[str], List[Union[str, "PDFParser", "ImageParser"]]]] = Noneactions: Optional[List[Union[@@ -2177,6 +2177,15 @@ class PDFParser(BaseModel):return folded+class ImageParser(BaseModel):+ """Image parser: OCR raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP,+ WebP, AVIF) as one-page documents. Part of the default parsers list next to+ "pdf"; takes no options, so the string "image" is equivalent. Omit it from+ an explicit list to keep image URLs failing as unsupported files."""++ type: Literal["image"] = "image"++# Location typesclass Location(BaseModel):"""Location configuration for scraping."""