Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
added timezone to metadata response (#2526)
Rafael Milleraecce165b5f7
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex fcac784bd..3b759eeac 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -2505,6 +2505,10 @@"type": "integer","description": "The status code of the page"},+ "timezone": {+ "type": "string",+ "description": "Timezone inferred by the scraping engine, when available"+ },"error": {"type": "string","nullable": true,@@ -2704,6 +2708,10 @@"type": "integer","description": "The status code of the page"},+ "timezone": {+ "type": "string",+ "description": "Timezone inferred by the scraping engine, when available"+ },"error": {"type": "string","nullable": true,@@ -2837,6 +2845,10 @@"type": "integer","description": "The status code of the page"},+ "timezone": {+ "type": "string",+ "description": "Timezone inferred by the scraping engine, when available"+ },"error": {"type": "string","nullable": true,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ad360009f..ff1efb7cc 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1046,6 +1046,7 @@ export type Document = {error?: string;numPages?: number;contentType?: string;+ timezone?: string;proxyUsed: "basic" | "stealth";cacheState?: "hit" | "miss";cachedAt?: string;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex c7805f02a..4f2eac8e3 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -984,6 +984,7 @@ export type Document = {error?: string;numPages?: number;contentType?: string;+ timezone?: string;proxyUsed: "basic" | "stealth";cacheState?: "hit" | "miss";cachedAt?: string;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex cc477dcdf..5cfebe5f9 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -102,6 +102,7 @@ const successSchema = z.object({usedMobileProxy: z.boolean().optional(),youtubeTranscriptContent: z.any().optional(),+ timezone: z.string().optional(),});export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex a5bd8aa66..b18ffee60 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -411,6 +411,7 @@ export async function scrapeURLWithFireEngineChromeCDP(proxyUsed: response.usedMobileProxy ? "stealth" : "basic",youtubeTranscriptContent: response.youtubeTranscriptContent,+ timezone: response.timezone,};});}@@ -487,6 +488,7 @@ export async function scrapeURLWithFireEnginePlaywright(: {}),proxyUsed: response.usedMobileProxy ? "stealth" : "basic",+ timezone: response.timezone,};});}@@ -552,6 +554,7 @@ export async function scrapeURLWithFireEngineTLSClient() ?? [])[1] ?? undefined,proxyUsed: response.usedMobileProxy ? "stealth" : "basic",+ timezone: response.timezone,};});}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 1d91851e8..eff8c8ca6 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -155,6 +155,7 @@ const successSchema = z.object({usedMobileProxy: z.boolean().optional(),youtubeTranscriptContent: z.any().optional(),+ timezone: z.string().optional(),});type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 7a9dda79d..151e711db 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -137,6 +137,7 @@ export type EngineScrapeResult = {postprocessorsUsed?: string[];proxyUsed: "basic" | "stealth";+ timezone?: string;};const engineHandlers: {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 2320a85f0..c0b858af3 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -756,6 +756,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {? { title: engineResult.pdfMetadata.title }: {}),contentType: engineResult.contentType,+ timezone: engineResult.timezone,proxyUsed: engineResult.proxyUsed ?? "basic",...(fallbackList.find(x =>["index", "index;documents"].includes(x.engine),diff --git a/apps/api/v1-openapi.json b/apps/api/v1-openapi.jsonindex 5176e5472..077d7eed3 100644--- a/apps/api/v1-openapi.json+++ b/apps/api/v1-openapi.json@@ -2505,6 +2505,10 @@"type": "integer","description": "The status code of the page"},+ "timezone": {+ "type": "string",+ "description": "Timezone inferred by the scraping engine, when available"+ },"error": {"type": "string","nullable": true,@@ -2704,6 +2708,10 @@"type": "integer","description": "The status code of the page"},+ "timezone": {+ "type": "string",+ "description": "Timezone inferred by the scraping engine, when available"+ },"error": {"type": "string","nullable": true,@@ -2837,6 +2845,10 @@"type": "integer","description": "The status code of the page"},+ "timezone": {+ "type": "string",+ "description": "Timezone inferred by the scraping engine, when available"+ },"error": {"type": "string","nullable": true,diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex b6cedfdc1..21e935f65 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -49,6 +49,7 @@ export interface FirecrawlDocumentMetadata {articleSection?: string;sourceURL?: string;statusCode?: number;+ timezone?: string;error?: string;proxyUsed?: "basic" | "stealth";cacheState?: "miss" | "hit";diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex bb251d047..ffdd18641 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -347,6 +347,7 @@ export interface DocumentMetadata {scrapeId?: string;numPages?: number;contentType?: string;+ timezone?: string;proxyUsed?: 'basic' | 'stealth';cacheState?: 'hit' | 'miss';cachedAt?: string;diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 78f6ac010..3f0c3ad03 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -117,6 +117,7 @@ class DocumentMetadata(BaseModel):num_pages: Optional[int] = Nonecontent_type: Optional[str] = Noneproxy_used: Optional[Literal["basic", "stealth"]] = None+ timezone: Optional[str] = Nonecache_state: Optional[Literal["hit", "miss"]] = Nonecached_at: Optional[str] = Nonecredits_used: Optional[int] = None@@ -193,6 +194,7 @@ class DocumentMetadata(BaseModel):"content_type","cached_at","error",+ "timezone",}for k, v in list(data.items()):if isinstance(v, list) and k in single_str_fields: