Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Nick: fixes

Nicolase34ad51635ef
diff --git a/apps/api/src/services/monitoring/diff.test.ts b/apps/api/src/services/monitoring/diff.test.ts
index a39fdbbc7..df707ec0b 100644
--- a/apps/api/src/services/monitoring/diff.test.ts
+++ b/apps/api/src/services/monitoring/diff.test.ts
@@ -137,6 +137,53 @@ describe("diffMonitorJson", () => {
});
});
+ it("ignores whitespace-only differences in string values", () => {
+ // Regression: LLM extractions of the same content often differ only
+ // in incidental whitespace (extra spaces, trailing newlines, NBSP
+ // from HTML rendering, etc.). Those shouldn't be reported as field
+ // changes β€” they drown out real signal.
+ const previous = {
+ headline: "Power AI agents with clean web data",
+ cta: "Start for free\n",
+ // Non-breaking space (U+00A0) instead of a regular space β€” common
+ // when scraping rendered HTML.
+ subhead: "The API\u00a0to scrape the web",
+ // BOM at the start of the string.
+ tagline: "\ufeffNo more brittle scrapers",
+ };
+ const current = {
+ headline: "Power AI agents with clean web data",
+ cta: "Start for free",
+ subhead: "The API to scrape the web",
+ tagline: "No more brittle scrapers",
+ };
+
+ expect(diffMonitorJson(previous, current)).toEqual({
+ kind: "json",
+ status: "same",
+ });
+ });
+
+ it("still detects real content changes after whitespace normalization", () => {
+ // Sanity check: the whitespace normalization shouldn't make every
+ // string look equal.
+ expect(
+ diffMonitorJson(
+ { headline: "Power AI agents with clean web data" },
+ { headline: "Power AI agents with structured web data" },
+ ),
+ ).toEqual({
+ kind: "json",
+ status: "changed",
+ json: {
+ headline: {
+ previous: "Power AI agents with clean web data",
+ current: "Power AI agents with structured web data",
+ },
+ },
+ });
+ });
+
it("handles undefined inputs as empty objects", () => {
expect(diffMonitorJson(undefined, undefined)).toEqual({
kind: "json",
diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.ts
index 30b9aa703..d5696a85d 100644
--- a/apps/api/src/services/monitoring/diff.ts
+++ b/apps/api/src/services/monitoring/diff.ts
@@ -118,7 +118,29 @@ export function diffMonitorMarkdown(
}
function normalizeJsonValue(value: unknown): unknown {
- if (typeof value === "string") return value.normalize("NFC");
+ if (typeof value === "string") {
+ // For JSON-extraction monitor diffs we treat strings as semantically
+ // equal when they differ only in whitespace. LLM extractions are
+ // routinely inconsistent about incidental whitespace between runs
+ // β€” extra spaces, trailing newlines, NBSP (U+00A0), BOM, zero-width
+ // characters from HTML rendering β€” and reporting those as changes
+ // drowns out the signal from real content changes (price, headline,
+ // status flag, etc.). Users who need whitespace fidelity should
+ // monitor markdown, not JSON.
+ //
+ // Steps:
+ // 1. NFC normalize so `Γ©` and `e\u0301` compare equal.
+ // 2. Strip zero-width chars that `\s` doesn't match (ZWSP, ZWNJ,
+ // ZWJ, BOM).
+ // 3. Collapse every run of whitespace (incl. \n, \t, NBSP) to a
+ // single space.
+ // 4. Trim.
+ return value
+ .normalize("NFC")
+ .replace(/[\u200b-\u200d\ufeff]/g, "")
+ .replace(/\s+/g, " ")
+ .trim();
+ }
return value;
}
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index dc8c03cf1..15a375460 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.24.0",
+ "version": "4.24.1",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index baa69a431..a7ba5394e 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -41,7 +41,12 @@ export interface ScreenshotFormat {
export interface ChangeTrackingFormat extends Format {
type: "changeTracking";
modes: ("git-diff" | "json")[];
- schema?: Record<string, unknown>;
+ /**
+ * Either a JSON Schema object or a Zod schema. Zod schemas are
+ * auto-converted to JSON Schema by the SDK before being sent β€” see
+ * `utils/validation.ts`.
+ */
+ schema?: Record<string, unknown> | ZodTypeAny;
prompt?: string;
tag?: string;
}
@@ -631,8 +636,21 @@ export interface MapOptions {
location?: LocationConfig;
}
+/**
+ * Schedule for a monitor.
+ *
+ * On create/update, provide exactly one of `cron` or `text`:
+ * - `cron`: a 5-field cron expression (e.g. `"*\u002F30 * * * *"`).
+ * - `text`: a natural-language schedule (e.g. `"every 30 minutes"`,
+ * `"hourly"`, `"daily at 9:00"`). Firecrawl normalizes this to a cron
+ * expression server-side.
+ *
+ * On read, the API always returns the normalized `cron` value, so `cron`
+ * is populated in responses even when the monitor was created with `text`.
+ */
export interface MonitorSchedule {
- cron: string;
+ cron?: string;
+ text?: string;
timezone?: string;
}
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 804a56270..893aa34c5 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.27.0"
+__version__ = "4.27.1"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 96558fdb4..c10a62020 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -24,6 +24,7 @@ from .types import (
CrawlParamsData,
WebhookConfig,
AgentWebhookConfig,
+ MonitorWebhookConfig,
CrawlErrorsResponse,
ActiveCrawlsResponse,
MapOptions,
@@ -675,7 +676,7 @@ class FirecrawlClient:
schedule: Union[MonitorSchedule, Dict[str, Any]],
targets: List[Union[MonitorTarget, Dict[str, Any]]],
*,
- webhook: Optional[WebhookConfig] = None,
+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,
notification: Optional[MonitorNotification] = None,
retention_days: Optional[int] = None,
) -> Monitor:
@@ -712,7 +713,7 @@ class FirecrawlClient:
name: Optional[str] = None,
status: Optional[Literal["active", "paused"]] = None,
schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,
- webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None,
+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,
notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,
targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,
retention_days: Optional[int] = None,
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 834ac4407..6f6f5fbf8 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -13,6 +13,7 @@ from .types import (
CrawlRequest,
WebhookConfig,
AgentWebhookConfig,
+ MonitorWebhookConfig,
SearchRequest,
SearchData,
SourceOption,
@@ -347,7 +348,7 @@ class AsyncFirecrawlClient:
schedule: Union[MonitorSchedule, Dict[str, Any]],
targets: List[Union[MonitorTarget, Dict[str, Any]]],
*,
- webhook: Optional[WebhookConfig] = None,
+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,
notification: Optional[MonitorNotification] = None,
retention_days: Optional[int] = None,
) -> Monitor:
@@ -385,7 +386,7 @@ class AsyncFirecrawlClient:
name: Optional[str] = None,
status: Optional[Literal["active", "paused"]] = None,
schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,
- webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None,
+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,
notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,
targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,
retention_days: Optional[int] = None,
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index f5e2f46ab..372ab6566 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -341,6 +341,23 @@ class AgentWebhookConfig(BaseModel):
events: Optional[List[Literal["started", "action", "completed", "failed", "cancelled"]]] = None
+class MonitorWebhookConfig(BaseModel):
+ """Configuration for monitor webhooks.
+
+ Monitor webhooks support different events than crawl webhooks:
+ - monitor.page: One event per scraped URL as it finishes, with the
+ page-level diff status (`same` | `changed` | `new` | `removed` |
+ `error`).
+ - monitor.check.completed: A summary event sent after the full
+ monitor check is reconciled.
+ """
+
+ url: str
+ headers: Optional[Dict[str, str]] = None
+ metadata: Optional[Dict[str, str]] = None
+ events: Optional[List[Literal["monitor.page", "monitor.check.completed"]]] = None
+
+
class WebhookData(BaseModel):
"""Data sent to webhooks."""
@@ -824,9 +841,22 @@ class MapResponse(BaseResponse[MapData]):
# Monitor types
class MonitorSchedule(BaseModel):
- """Cron schedule for a monitor."""
+ """Schedule for a monitor.
+
+ On create / update you provide exactly one of `cron` or `text`:
+
+ - `cron`: a 5-field cron expression (e.g. ``"*/30 * * * *"``).
+ - `text`: a natural-language schedule (e.g. ``"every 30 minutes"``,
+ ``"hourly"``, ``"daily at 9:00"``). Firecrawl normalizes this to a
+ cron expression server-side.
- cron: str
+ On read, the API always returns the normalized ``cron`` value, so
+ `cron` is populated in responses even when the monitor was created
+ with `text`.
+ """
+
+ cron: Optional[str] = None
+ text: Optional[str] = None
timezone: str = "UTC"
@@ -860,7 +890,7 @@ class MonitorCreateRequest(BaseModel):
name: str
schedule: MonitorSchedule
- webhook: Optional[WebhookConfig] = None
+ webhook: Optional[MonitorWebhookConfig] = None
notification: Optional[MonitorNotification] = None
targets: List[Union[MonitorTarget, Dict[str, Any]]]
retention_days: Optional[int] = Field(default=None, alias="retentionDays")
@@ -872,7 +902,7 @@ class MonitorUpdateRequest(BaseModel):
name: Optional[str] = None
status: Optional[Literal["active", "paused"]] = None
schedule: Optional[MonitorSchedule] = None
- webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None
+ webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None
notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None
targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None
retention_days: Optional[int] = Field(default=None, alias="retentionDays")