Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

further crawl-errors improvements

Gergő Móricz356b04fb65ff
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index 029561a2c..eee0397ba 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -417,6 +417,7 @@ export interface CrawlErrorsResponse {
id: string,
timestamp?: string,
url: string,
+ code?: string,
error: string,
}[];
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 7ae910e9f..88d983165 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -263,7 +263,13 @@ export interface TokenUsage {
}
export interface CrawlErrorsResponse {
- errors: Array<Record<string, string>>;
+ errors: {
+ id: string;
+ timestamp?: string;
+ url: string;
+ code?: string;
+ error: string;
+ }[];
robotsBlocked: string[];
}
diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py
index ea2be9d00..52e328119 100644
--- a/apps/python-sdk/firecrawl/v1/client.py
+++ b/apps/python-sdk/firecrawl/v1/client.py
@@ -287,9 +287,17 @@ class V1CrawlStatusResponse(pydantic.BaseModel):
next: Optional[str] = None
data: List[V1FirecrawlDocument]
+class V1CrawlError(pydantic.BaseModel):
+ """A crawl error."""
+ id: str
+ timestamp: Optional[datetime] = None
+ url: str
+ code: Optional[str] = None
+ error: str
+
class V1CrawlErrorsResponse(pydantic.BaseModel):
"""Response from crawl/batch scrape error monitoring."""
- errors: List[Dict[str, str]] # {id: str, timestamp: str, url: str, error: str}
+ errors: List[V1CrawlError]
robotsBlocked: List[str]
class V1MapParams(pydantic.BaseModel):
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index eff1008df..f4fa0cab9 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -502,9 +502,17 @@ class JobStatus(BaseModel):
completed_at: Optional[datetime] = None
expires_at: Optional[datetime] = None
+class CrawlError(BaseModel):
+ """A crawl error."""
+ id: str
+ timestamp: Optional[datetime] = None
+ url: str
+ code: Optional[str] = None
+ error: str
+
class CrawlErrorsResponse(BaseModel):
"""Response from crawl error monitoring."""
- errors: List[Dict[str, str]]
+ errors: List[CrawlError]
robots_blocked: List[str]
class ActiveCrawl(BaseModel):