Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(sdks)feat/added concurrency info to metadata (#2502)

Rafael Millerd496714d4443
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index 0fc9799f0..b6cedfdc1 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -53,6 +53,9 @@ export interface FirecrawlDocumentMetadata {
proxyUsed?: "basic" | "stealth";
cacheState?: "miss" | "hit";
cachedAt?: string;
+ creditsUsed?: number;
+ concurrencyLimited?: boolean;
+ concurrencyQueueDurationMs?: number;
[key: string]: any; // Allows for additional metadata properties not explicitly defined.
}
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 2556b63c7..bb251d047 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -351,6 +351,8 @@ export interface DocumentMetadata {
cacheState?: 'hit' | 'miss';
cachedAt?: string;
creditsUsed?: number;
+ concurrencyLimited?: boolean;
+ concurrencyQueueDurationMs?: number;
// Error information
error?: string;
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py
index 8105ff41d..46e870c0a 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py
@@ -63,6 +63,19 @@ class TestDocumentMetadataExtras:
assert dumped["metadata"]["title"] == "Page"
assert dumped["metadata"]["twitter:site"] == "@site"
+ def test_concurrency_fields_are_mapped(self):
+ raw = {
+ "markdown": "# Queue info",
+ "metadata": {
+ "concurrencyLimited": True,
+ "concurrencyQueueDurationMs": 1234,
+ },
+ }
+ doc = Document(**normalize_document_input(raw))
+ md = doc.metadata_typed
+ assert md.concurrency_limited is True
+ assert md.concurrency_queue_duration_ms == 1234
+
def test_unknown_list_metadata_preserved(self):
raw = {
"markdown": "# Body",
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 73c5760c5..78f6ac010 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -120,6 +120,8 @@ class DocumentMetadata(BaseModel):
cache_state: Optional[Literal["hit", "miss"]] = None
cached_at: Optional[str] = None
credits_used: Optional[int] = None
+ concurrency_limited: Optional[bool] = None
+ concurrency_queue_duration_ms: Optional[int] = None
# Error information
error: Optional[str] = None
diff --git a/apps/python-sdk/firecrawl/v2/utils/normalize.py b/apps/python-sdk/firecrawl/v2/utils/normalize.py
index 296933a7c..19e849034 100644
--- a/apps/python-sdk/firecrawl/v2/utils/normalize.py
+++ b/apps/python-sdk/firecrawl/v2/utils/normalize.py
@@ -48,6 +48,8 @@ def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:
"cacheState": "cache_state",
"cachedAt": "cached_at",
"creditsUsed": "credits_used",
+ "concurrencyLimited": "concurrency_limited",
+ "concurrencyQueueDurationMs": "concurrency_queue_duration_ms",
}
out: Dict[str, Any] = {}