Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixes
Nicolase34ad51635ef
diff --git a/apps/api/src/services/monitoring/diff.test.ts b/apps/api/src/services/monitoring/diff.test.tsindex a39fdbbc7..df707ec0b 100644--- a/apps/api/src/services/monitoring/diff.test.ts+++ b/apps/api/src/services/monitoring/diff.test.ts@@ -137,6 +137,53 @@ describe("diffMonitorJson", () => {});});+ it("ignores whitespace-only differences in string values", () => {+ // Regression: LLM extractions of the same content often differ only+ // in incidental whitespace (extra spaces, trailing newlines, NBSP+ // from HTML rendering, etc.). Those shouldn't be reported as field+ // changes β they drown out real signal.+ const previous = {+ headline: "Power AI agents with clean web data",+ cta: "Start for free\n",+ // Non-breaking space (U+00A0) instead of a regular space β common+ // when scraping rendered HTML.+ subhead: "The API\u00a0to scrape the web",+ // BOM at the start of the string.+ tagline: "\ufeffNo more brittle scrapers",+ };+ const current = {+ headline: "Power AI agents with clean web data",+ cta: "Start for free",+ subhead: "The API to scrape the web",+ tagline: "No more brittle scrapers",+ };++ expect(diffMonitorJson(previous, current)).toEqual({+ kind: "json",+ status: "same",+ });+ });++ it("still detects real content changes after whitespace normalization", () => {+ // Sanity check: the whitespace normalization shouldn't make every+ // string look equal.+ expect(+ diffMonitorJson(+ { headline: "Power AI agents with clean web data" },+ { headline: "Power AI agents with structured web data" },+ ),+ ).toEqual({+ kind: "json",+ status: "changed",+ json: {+ headline: {+ previous: "Power AI agents with clean web data",+ current: "Power AI agents with structured web data",+ },+ },+ });+ });+it("handles undefined inputs as empty objects", () => {expect(diffMonitorJson(undefined, undefined)).toEqual({kind: "json",diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.tsindex 30b9aa703..d5696a85d 100644--- a/apps/api/src/services/monitoring/diff.ts+++ b/apps/api/src/services/monitoring/diff.ts@@ -118,7 +118,29 @@ export function diffMonitorMarkdown(}function normalizeJsonValue(value: unknown): unknown {- if (typeof value === "string") return value.normalize("NFC");+ if (typeof value === "string") {+ // For JSON-extraction monitor diffs we treat strings as semantically+ // equal when they differ only in whitespace. LLM extractions are+ // routinely inconsistent about incidental whitespace between runs+ // β extra spaces, trailing newlines, NBSP (U+00A0), BOM, zero-width+ // characters from HTML rendering β and reporting those as changes+ // drowns out the signal from real content changes (price, headline,+ // status flag, etc.). Users who need whitespace fidelity should+ // monitor markdown, not JSON.+ //+ // Steps:+ // 1. NFC normalize so `Γ©` and `e\u0301` compare equal.+ // 2. Strip zero-width chars that `\s` doesn't match (ZWSP, ZWNJ,+ // ZWJ, BOM).+ // 3. Collapse every run of whitespace (incl. \n, \t, NBSP) to a+ // single space.+ // 4. Trim.+ return value+ .normalize("NFC")+ .replace(/[\u200b-\u200d\ufeff]/g, "")+ .replace(/\s+/g, " ")+ .trim();+ }return value;}diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex dc8c03cf1..15a375460 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.24.0",+ "version": "4.24.1","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex baa69a431..a7ba5394e 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -41,7 +41,12 @@ export interface ScreenshotFormat {export interface ChangeTrackingFormat extends Format {type: "changeTracking";modes: ("git-diff" | "json")[];- schema?: Record<string, unknown>;+ /**+ * Either a JSON Schema object or a Zod schema. Zod schemas are+ * auto-converted to JSON Schema by the SDK before being sent β see+ * `utils/validation.ts`.+ */+ schema?: Record<string, unknown> | ZodTypeAny;prompt?: string;tag?: string;}@@ -631,8 +636,21 @@ export interface MapOptions {location?: LocationConfig;}+/**+ * Schedule for a monitor.+ *+ * On create/update, provide exactly one of `cron` or `text`:+ * - `cron`: a 5-field cron expression (e.g. `"*\u002F30 * * * *"`).+ * - `text`: a natural-language schedule (e.g. `"every 30 minutes"`,+ * `"hourly"`, `"daily at 9:00"`). Firecrawl normalizes this to a cron+ * expression server-side.+ *+ * On read, the API always returns the normalized `cron` value, so `cron`+ * is populated in responses even when the monitor was created with `text`.+ */export interface MonitorSchedule {- cron: string;+ cron?: string;+ text?: string;timezone?: string;}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 804a56270..893aa34c5 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.27.0"+__version__ = "4.27.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 96558fdb4..c10a62020 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -24,6 +24,7 @@ from .types import (CrawlParamsData,WebhookConfig,AgentWebhookConfig,+ MonitorWebhookConfig,CrawlErrorsResponse,ActiveCrawlsResponse,MapOptions,@@ -675,7 +676,7 @@ class FirecrawlClient:schedule: Union[MonitorSchedule, Dict[str, Any]],targets: List[Union[MonitorTarget, Dict[str, Any]]],*,- webhook: Optional[WebhookConfig] = None,+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,notification: Optional[MonitorNotification] = None,retention_days: Optional[int] = None,) -> Monitor:@@ -712,7 +713,7 @@ class FirecrawlClient:name: Optional[str] = None,status: Optional[Literal["active", "paused"]] = None,schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,- webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None,+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,retention_days: Optional[int] = None,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 834ac4407..6f6f5fbf8 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -13,6 +13,7 @@ from .types import (CrawlRequest,WebhookConfig,AgentWebhookConfig,+ MonitorWebhookConfig,SearchRequest,SearchData,SourceOption,@@ -347,7 +348,7 @@ class AsyncFirecrawlClient:schedule: Union[MonitorSchedule, Dict[str, Any]],targets: List[Union[MonitorTarget, Dict[str, Any]]],*,- webhook: Optional[WebhookConfig] = None,+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,notification: Optional[MonitorNotification] = None,retention_days: Optional[int] = None,) -> Monitor:@@ -385,7 +386,7 @@ class AsyncFirecrawlClient:name: Optional[str] = None,status: Optional[Literal["active", "paused"]] = None,schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,- webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None,+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,retention_days: Optional[int] = None,diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex f5e2f46ab..372ab6566 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -341,6 +341,23 @@ class AgentWebhookConfig(BaseModel):events: Optional[List[Literal["started", "action", "completed", "failed", "cancelled"]]] = None+class MonitorWebhookConfig(BaseModel):+ """Configuration for monitor webhooks.++ Monitor webhooks support different events than crawl webhooks:+ - monitor.page: One event per scraped URL as it finishes, with the+ page-level diff status (`same` | `changed` | `new` | `removed` |+ `error`).+ - monitor.check.completed: A summary event sent after the full+ monitor check is reconciled.+ """++ url: str+ headers: Optional[Dict[str, str]] = None+ metadata: Optional[Dict[str, str]] = None+ events: Optional[List[Literal["monitor.page", "monitor.check.completed"]]] = None++class WebhookData(BaseModel):"""Data sent to webhooks."""@@ -824,9 +841,22 @@ class MapResponse(BaseResponse[MapData]):# Monitor typesclass MonitorSchedule(BaseModel):- """Cron schedule for a monitor."""+ """Schedule for a monitor.++ On create / update you provide exactly one of `cron` or `text`:++ - `cron`: a 5-field cron expression (e.g. ``"*/30 * * * *"``).+ - `text`: a natural-language schedule (e.g. ``"every 30 minutes"``,+ ``"hourly"``, ``"daily at 9:00"``). Firecrawl normalizes this to a+ cron expression server-side.- cron: str+ On read, the API always returns the normalized ``cron`` value, so+ `cron` is populated in responses even when the monitor was created+ with `text`.+ """++ cron: Optional[str] = None+ text: Optional[str] = Nonetimezone: str = "UTC"@@ -860,7 +890,7 @@ class MonitorCreateRequest(BaseModel):name: strschedule: MonitorSchedule- webhook: Optional[WebhookConfig] = None+ webhook: Optional[MonitorWebhookConfig] = Nonenotification: Optional[MonitorNotification] = Nonetargets: List[Union[MonitorTarget, Dict[str, Any]]]retention_days: Optional[int] = Field(default=None, alias="retentionDays")@@ -872,7 +902,7 @@ class MonitorUpdateRequest(BaseModel):name: Optional[str] = Nonestatus: Optional[Literal["active", "paused"]] = Noneschedule: Optional[MonitorSchedule] = None- webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = Nonenotification: Optional[Union[MonitorNotification, Dict[str, Any]]] = Nonetargets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = Noneretention_days: Optional[int] = Field(default=None, alias="retentionDays")