Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(sdks)feat/added concurrency info to metadata (#2502)
Rafael Millerd496714d4443
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 0fc9799f0..b6cedfdc1 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -53,6 +53,9 @@ export interface FirecrawlDocumentMetadata {proxyUsed?: "basic" | "stealth";cacheState?: "miss" | "hit";cachedAt?: string;+ creditsUsed?: number;+ concurrencyLimited?: boolean;+ concurrencyQueueDurationMs?: number;[key: string]: any; // Allows for additional metadata properties not explicitly defined.}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 2556b63c7..bb251d047 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -351,6 +351,8 @@ export interface DocumentMetadata {cacheState?: 'hit' | 'miss';cachedAt?: string;creditsUsed?: number;+ concurrencyLimited?: boolean;+ concurrencyQueueDurationMs?: number;// Error informationerror?: string;diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.pyindex 8105ff41d..46e870c0a 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py@@ -63,6 +63,19 @@ class TestDocumentMetadataExtras:assert dumped["metadata"]["title"] == "Page"assert dumped["metadata"]["twitter:site"] == "@site"+ def test_concurrency_fields_are_mapped(self):+ raw = {+ "markdown": "# Queue info",+ "metadata": {+ "concurrencyLimited": True,+ "concurrencyQueueDurationMs": 1234,+ },+ }+ doc = Document(**normalize_document_input(raw))+ md = doc.metadata_typed+ assert md.concurrency_limited is True+ assert md.concurrency_queue_duration_ms == 1234+def test_unknown_list_metadata_preserved(self):raw = {"markdown": "# Body",diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 73c5760c5..78f6ac010 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -120,6 +120,8 @@ class DocumentMetadata(BaseModel):cache_state: Optional[Literal["hit", "miss"]] = Nonecached_at: Optional[str] = Nonecredits_used: Optional[int] = None+ concurrency_limited: Optional[bool] = None+ concurrency_queue_duration_ms: Optional[int] = None# Error informationerror: Optional[str] = Nonediff --git a/apps/python-sdk/firecrawl/v2/utils/normalize.py b/apps/python-sdk/firecrawl/v2/utils/normalize.pyindex 296933a7c..19e849034 100644--- a/apps/python-sdk/firecrawl/v2/utils/normalize.py+++ b/apps/python-sdk/firecrawl/v2/utils/normalize.py@@ -48,6 +48,8 @@ def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:"cacheState": "cache_state","cachedAt": "cached_at","creditsUsed": "credits_used",+ "concurrencyLimited": "concurrency_limited",+ "concurrencyQueueDurationMs": "concurrency_queue_duration_ms",}out: Dict[str, Any] = {}