Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): monitoring json format (#3536)

Developers Digestf3a47df25ffc
diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.ts
index 9c677507a..caba1621c 100644
--- a/apps/api/src/__tests__/snips/v2/monitor.test.ts
+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts
@@ -139,4 +139,190 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
},
2 * scrapeTimeout,
);
+
+ it(
+ "runs a JSON-mode monitor and surfaces a snapshot",
+ async () => {
+ const jsonSchema = {
+ type: "object",
+ additionalProperties: false,
+ required: ["title"],
+ properties: {
+ title: {
+ type: "string",
+ description: "The page title, verbatim.",
+ },
+ },
+ };
+
+ const create = await monitorCreateRaw(
+ {
+ name: "json-mode monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl()],
+ scrapeOptions: {
+ formats: [
+ "markdown",
+ {
+ type: "json",
+ prompt: "Extract the page title verbatim.",
+ schema: jsonSchema,
+ },
+ ],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+
+ const monitorId = create.body.data.id;
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ const checkId = run.body.id;
+
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, checkId, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (["completed", "partial", "failed"].includes(check.status)) break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ expect(["completed", "partial"]).toContain(check.status);
+ // First run: status is "new" (no previous scrape to diff against), no
+ // snapshot persisted to GCS. We can't assert snapshot.json here
+ // without a mutating fixture; a second run with a changed page would
+ // be needed. The contract assertion is: when JSON mode is requested,
+ // the monitor doesn't fall through the markdown path and crash.
+ expect(check.pages[0].status).toBe("new");
+
+ await monitorDeleteRaw(monitorId, identity);
+ },
+ 2 * scrapeTimeout,
+ );
+
+ it(
+ "accepts changeTracking-json format on a monitor target",
+ async () => {
+ const jsonSchema = {
+ type: "object",
+ additionalProperties: false,
+ required: ["title"],
+ properties: {
+ title: { type: "string", description: "The page title." },
+ },
+ };
+
+ const create = await monitorCreateRaw(
+ {
+ name: "ct-json monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl()],
+ scrapeOptions: {
+ formats: [
+ "markdown",
+ {
+ type: "changeTracking",
+ modes: ["json"],
+ prompt: "Extract the title.",
+ schema: jsonSchema,
+ },
+ ],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+
+ const monitorId = create.body.data.id;
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ const checkId = run.body.id;
+
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, checkId, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (["completed", "partial", "failed"].includes(check.status)) break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ expect(["completed", "partial"]).toContain(check.status);
+ expect(check.pages[0].status).toBe("new");
+
+ await monitorDeleteRaw(monitorId, identity);
+ },
+ 2 * scrapeTimeout,
+ );
+
+ it(
+ "accepts mixed json + git-diff changeTracking modes",
+ async () => {
+ const jsonSchema = {
+ type: "object",
+ additionalProperties: false,
+ required: ["title"],
+ properties: {
+ title: { type: "string", description: "The page title." },
+ },
+ };
+
+ const create = await monitorCreateRaw(
+ {
+ name: "ct-mixed monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl()],
+ scrapeOptions: {
+ formats: [
+ "markdown",
+ {
+ type: "changeTracking",
+ modes: ["json", "git-diff"],
+ prompt: "Extract the title.",
+ schema: jsonSchema,
+ },
+ ],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+
+ const monitorId = create.body.data.id;
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ const checkId = run.body.id;
+
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, checkId, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (["completed", "partial", "failed"].includes(check.status)) break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ expect(["completed", "partial"]).toContain(check.status);
+ // First run is always "new"; the assertion here is that mixed-mode
+ // configuration is accepted end-to-end without erroring.
+ expect(check.pages[0].status).toBe("new");
+
+ await monitorDeleteRaw(monitorId, identity);
+ },
+ 2 * scrapeTimeout,
+ );
});
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
index 29d0a2180..571f5386a 100644
--- a/apps/api/src/controllers/v2/monitor.ts
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -389,19 +389,38 @@ export async function getMonitorCheckController(
]);
const pagesWithDiffs = await Promise.all(
- pages.map(async page => ({
- id: page.id,
- targetId: page.target_id,
- url: page.url,
- status: page.status,
- previousScrapeId: page.previous_scrape_id,
- currentScrapeId: page.current_scrape_id,
- statusCode: page.status_code,
- error: page.error,
- metadata: page.metadata,
- diff: await getMonitorDiffArtifact(page.diff_gcs_key),
- createdAt: page.created_at,
- })),
+ pages.map(async page => {
+ const artifact = await getMonitorDiffArtifact(page.diff_gcs_key);
+ const base = {
+ id: page.id,
+ targetId: page.target_id,
+ url: page.url,
+ status: page.status,
+ previousScrapeId: page.previous_scrape_id,
+ currentScrapeId: page.current_scrape_id,
+ statusCode: page.status_code,
+ error: page.error,
+ metadata: page.metadata,
+ createdAt: page.created_at,
+ };
+ if (!artifact) {
+ return { ...base, diff: null };
+ }
+ if (artifact.kind === "json") {
+ return {
+ ...base,
+ diff: {
+ json: artifact.json,
+ ...(artifact.markdown ? { text: artifact.markdown.text } : {}),
+ },
+ snapshot: { json: artifact.snapshot },
+ };
+ }
+ return {
+ ...base,
+ diff: { text: artifact.text, json: artifact.json },
+ };
+ }),
);
const nextSkip = skip + pagesWithDiffs.length;
const next = (() => {
diff --git a/apps/api/src/lib/gcs-monitoring.ts b/apps/api/src/lib/gcs-monitoring.ts
index 36b3e9e12..a40373316 100644
--- a/apps/api/src/lib/gcs-monitoring.ts
+++ b/apps/api/src/lib/gcs-monitoring.ts
@@ -1,15 +1,37 @@
import { config } from "../config";
import { storage } from "./gcs-jobs";
-type MonitorDiffArtifact = {
+type MonitorDiffArtifactBase = {
url: string;
previousScrapeId: string | null;
currentScrapeId: string | null;
- text: string;
- json: unknown;
generatedAt: string;
};
+export type MonitorDiffArtifact =
+ | (MonitorDiffArtifactBase & {
+ kind: "markdown";
+ text: string;
+ json: unknown;
+ })
+ | (MonitorDiffArtifactBase & {
+ kind: "json";
+ /** Per-field {previous, current} diff. */
+ json: Record<string, { previous: unknown; current: unknown }>;
+ /** Full current JSON extraction (the snapshot at this run). */
+ snapshot: Record<string, unknown>;
+ /**
+ * Optional markdown diff sidecar. Populated only when the monitor's
+ * formats requested both `"json"` and `"git-diff"` change-tracking
+ * modes — in that case we run both diffs and report `changed` if
+ * either path saw a change.
+ */
+ markdown?: {
+ text: string;
+ json: unknown;
+ };
+ });
+
const contentType = "application/json";
export function monitorDiffGcsKey(params: {
@@ -21,16 +43,29 @@ export function monitorDiffGcsKey(params: {
return `monitors/${params.teamId}/${params.monitorId}/${params.checkId}/${params.pageId}.diff.json`;
}
+function artifactBytes(artifact: MonitorDiffArtifact): {
+ textBytes: number;
+ jsonBytes: number;
+} {
+ const jsonBytes = Buffer.byteLength(JSON.stringify(artifact.json ?? null));
+ let textBytes = 0;
+ if (artifact.kind === "markdown") {
+ textBytes = Buffer.byteLength(artifact.text);
+ } else if (artifact.kind === "json" && artifact.markdown) {
+ // Sidecar markdown diff (mixed-mode monitor) — count it so storage
+ // accounting stays honest.
+ textBytes = Buffer.byteLength(artifact.markdown.text);
+ }
+ return { textBytes, jsonBytes };
+}
+
export async function saveMonitorDiffArtifact(
key: string,
artifact: MonitorDiffArtifact,
): Promise<{ textBytes: number; jsonBytes: number }> {
const payload = JSON.stringify(artifact);
if (!config.GCS_BUCKET_NAME) {
- return {
- textBytes: Buffer.byteLength(artifact.text),
- jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),
- };
+ return artifactBytes(artifact);
}
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
@@ -39,10 +74,7 @@ export async function saveMonitorDiffArtifact(
resumable: false,
});
- return {
- textBytes: Buffer.byteLength(artifact.text),
- jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),
- };
+ return artifactBytes(artifact);
}
export async function getMonitorDiffArtifact(
@@ -53,7 +85,26 @@ export async function getMonitorDiffArtifact(
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
try {
const [contents] = await bucket.file(key).download();
- return JSON.parse(contents.toString()) as MonitorDiffArtifact;
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(contents.toString());
+ } catch {
+ // Corrupt or truncated artifact — surface as "no diff" instead of
+ // letting JSON.parse throw and break the entire check response.
+ return null;
+ }
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+ // An unexpected payload shape (e.g. number, array, null) was written
+ // here; treat as missing rather than risk reading kind off a non-object.
+ return null;
+ }
+ const asPartial = parsed as Partial<MonitorDiffArtifact>;
+ // Backwards compat: historical artifacts predate the `kind` field and
+ // are always markdown.
+ if (!asPartial.kind) {
+ return { ...(asPartial as any), kind: "markdown" } as MonitorDiffArtifact;
+ }
+ return asPartial as MonitorDiffArtifact;
} catch (error) {
const maybeGcsError = error as { code?: number; statusCode?: number };
if (maybeGcsError.code === 404 || maybeGcsError.statusCode === 404) {
diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.ts
new file mode 100644
index 000000000..e9a4fd6e5
--- /dev/null
+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts
@@ -0,0 +1,190 @@
+import { v7 as uuidv7 } from "uuid";
+import { getJobFromGCS } from "../../lib/gcs-jobs";
+import {
+ MonitorDiffArtifact,
+ monitorDiffGcsKey,
+ saveMonitorDiffArtifact,
+} from "../../lib/gcs-monitoring";
+import {
+ diffMonitorJson,
+ diffMonitorMarkdown,
+ formatsRequestGitDiff,
+ formatsRequestJsonExtraction,
+} from "./diff";
+
+type MonitorPageDiffStatus = "same" | "new" | "changed";
+
+type MonitorPageDiffResult = {
+ status: MonitorPageDiffStatus;
+ diffGcsKey: string | null;
+ diffTextBytes: number | null;
+ diffJsonBytes: number | null;
+};
+
+type PreviousPageRef = {
+ last_scrape_id: string | null;
+ is_removed: boolean | null;
+};
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Compute the page diff for a single monitored URL and persist the diff
+ * artifact to GCS when content changed. Branches on whether the monitor
+ * asked for JSON extraction (in which case the diff is over `doc.json`) or
+ * the default markdown text diff.
+ *
+ * Returns the new status + GCS key/sizes for the caller to write into the
+ * monitor pages row.
+ */
+export async function computeAndPersistPageDiff(params: {
+ teamId: string;
+ monitorId: string;
+ checkId: string;
+ url: string;
+ scrapeId: string;
+ doc: any;
+ previous: PreviousPageRef | null;
+ formats: unknown;
+}): Promise<MonitorPageDiffResult> {
+ const { teamId, monitorId, checkId, url, scrapeId, doc, previous, formats } =
+ params;
+
+ const wantsJson = formatsRequestJsonExtraction(formats);
+ const wantsGitDiff = formatsRequestGitDiff(formats);
+
+ if (!previous?.last_scrape_id || previous.is_removed) {
+ return {
+ status: "new",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];
+
+ if (wantsJson) {
+ const previousJson = isPlainObject(previousDoc?.json)
+ ? (previousDoc!.json as Record<string, unknown>)
+ : undefined;
+ const currentJson = isPlainObject(doc?.json)
+ ? (doc.json as Record<string, unknown>)
+ : undefined;
+
+ // If the current scrape didn't produce a JSON document we can't
+ // compute a JSON diff; treat as `changed` to be safe (matches the
+ // markdown-missing branch's behavior).
+ if (!currentJson) {
+ return {
+ status: "changed",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const result = diffMonitorJson(previousJson, currentJson);
+
+ // Mixed-mode: also compute the markdown diff so a monitor that asked
+ // for both ["json","git-diff"] modes reports `changed` whenever either
+ // surface changed, with both diffs persisted side by side.
+ let markdownSidecar: { text: string; json: unknown } | undefined;
+ if (wantsGitDiff) {
+ const previousMarkdown = previousDoc?.markdown;
+ const currentMarkdown = doc?.markdown;
+ if (previousMarkdown && currentMarkdown) {
+ const md = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
+ if (md.status === "changed") {
+ markdownSidecar = { text: md.text, json: md.json };
+ }
+ }
+ }
+
+ if (result.status === "same" && !markdownSidecar) {
+ return {
+ status: "same",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const diffGcsKey = monitorDiffGcsKey({
+ teamId,
+ monitorId,
+ checkId,
+ pageId: uuidv7(),
+ });
+ const artifact: MonitorDiffArtifact = {
+ kind: "json",
+ url,
+ previousScrapeId: previous.last_scrape_id,
+ currentScrapeId: scrapeId,
+ generatedAt: new Date().toISOString(),
+ // When only markdown changed, the per-field JSON diff is empty.
+ json: result.status === "changed" ? result.json : {},
+ snapshot: currentJson,
+ ...(markdownSidecar ? { markdown: markdownSidecar } : {}),
+ };
+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, artifact);
+ return {
+ status: "changed",
+ diffGcsKey,
+ diffTextBytes: sizes.textBytes,
+ diffJsonBytes: sizes.jsonBytes,
+ };
+ }
+
+ // Markdown path (existing behavior).
+ const previousMarkdown = previousDoc?.markdown;
+ const currentMarkdown = doc?.markdown;
+
+ // Only treat genuinely missing markdown as the fallback "changed" case.
+ // An empty string is a valid scrape result (e.g. a page that renders no
+ // textual content) and should still flow through diffMonitorMarkdown so
+ // we can correctly report "same" when both runs returned "".
+ if (previousMarkdown == null || currentMarkdown == null) {
+ return {
+ status: "changed",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
+ if (diff.status === "same") {
+ return {
+ status: "same",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const diffGcsKey = monitorDiffGcsKey({
+ teamId,
+ monitorId,
+ checkId,
+ pageId: uuidv7(),
+ });
+ const artifact: MonitorDiffArtifact = {
+ kind: "markdown",
+ url,
+ previousScrapeId: previous.last_scrape_id,
+ currentScrapeId: scrapeId,
+ generatedAt: new Date().toISOString(),
+ text: diff.text,
+ json: diff.json,
+ };
+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, artifact);
+ return {
+ status: "changed",
+ diffGcsKey,
+ diffTextBytes: sizes.textBytes,
+ diffJsonBytes: sizes.jsonBytes,
+ };
+}
diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.ts
index 3197367d3..b4a719eab 100644
--- a/apps/api/src/services/monitoring/diff.ts
+++ b/apps/api/src/services/monitoring/diff.ts
@@ -1,13 +1,15 @@
import gitDiff from "git-diff";
import parseDiff from "parse-diff";
-type MonitoringDiffResult =
+type MonitorMarkdownDiffResult =
| {
+ kind: "markdown";
status: "same";
text?: undefined;
json?: undefined;
}
| {
+ kind: "markdown";
status: "changed";
text: string;
json: {
@@ -31,6 +33,16 @@ type MonitoringDiffResult =
};
};
+type MonitorJsonDiffResult =
+ | { kind: "json"; status: "same"; json?: undefined }
+ | {
+ kind: "json";
+ status: "changed";
+ json: Record<string, { previous: unknown; current: unknown }>;
+ };
+
+type MonitoringDiffResult = MonitorMarkdownDiffResult;
+
function normalizeMarkdownForChangeTracking(markdown: string): string {
return [...markdown.replace(/\s+/g, "").replace(/\[iframe\]\(.+?\)/g, "")]
.sort()
@@ -45,7 +57,7 @@ export function diffMonitorMarkdown(
normalizeMarkdownForChangeTracking(previousMarkdown) ===
normalizeMarkdownForChangeTracking(currentMarkdown)
) {
- return { status: "same" };
+ return { kind: "markdown", status: "same" };
}
const text = gitDiff(previousMarkdown, currentMarkdown, {
@@ -55,6 +67,7 @@ export function diffMonitorMarkdown(
const structured = parseDiff(text);
return {
+ kind: "markdown",
status: "changed",
text,
json: {
@@ -103,3 +116,155 @@ export function diffMonitorMarkdown(
},
};
}
+
+function normalizeJsonValue(value: unknown): unknown {
+ if (typeof value === "string") return value.normalize("NFC");
+ return value;
+}
+
+function jsonValuesEqual(a: unknown, b: unknown): boolean {
+ const na = normalizeJsonValue(a);
+ const nb = normalizeJsonValue(b);
+ if (na === nb) return true;
+ if (typeof na !== typeof nb) return false;
+ if (na && nb && typeof na === "object") {
+ // Deep, key-order-independent equality. Using JSON.stringify here would
+ // report a spurious diff whenever the LLM returned the same fields in a
+ // different order between runs.
+ if (Array.isArray(na) || Array.isArray(nb)) {
+ if (!Array.isArray(na) || !Array.isArray(nb)) return false;
+ if (na.length !== nb.length) return false;
+ for (let i = 0; i < na.length; i++) {
+ if (!jsonValuesEqual(na[i], nb[i])) return false;
+ }
+ return true;
+ }
+ const aKeys = Object.keys(na as Record<string, unknown>);
+ const bKeys = Object.keys(nb as Record<string, unknown>);
+ if (aKeys.length !== bKeys.length) return false;
+ for (const k of aKeys) {
+ if (!Object.prototype.hasOwnProperty.call(nb, k)) return false;
+ if (
+ !jsonValuesEqual(
+ (na as Record<string, unknown>)[k],
+ (nb as Record<string, unknown>)[k],
+ )
+ ) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+}
+
+/**
+ * Diff two JSON snapshots (current vs previous scrape `doc.json`) for a
+ * monitor. Returns `same` if every field is unchanged after NFC
+ * normalization, otherwise `changed` with a per-field {previous, current}
+ * map containing only the fields that differ.
+ */
+export function diffMonitorJson(
+ previous: Record<string, unknown> | undefined,
+ current: Record<string, unknown> | undefined,
+): MonitorJsonDiffResult {
+ const prev = previous ?? {};
+ const curr = current ?? {};
+ const keys = new Set([...Object.keys(prev), ...Object.keys(curr)]);
+ const diff: Record<string, { previous: unknown; current: unknown }> = {};
+
+ for (const key of keys) {
+ if (!jsonValuesEqual(prev[key], curr[key])) {
+ diff[key] = { previous: prev[key], current: curr[key] };
+ }
+ }
+
+ if (Object.keys(diff).length === 0) {
+ return { kind: "json", status: "same" };
+ }
+ return { kind: "json", status: "changed", json: diff };
+}
+
+/**
+ * True iff the monitor's scrapeOptions.formats requests a git-diff style
+ * markdown diff via a `{type:"changeTracking", modes:[...,"git-diff",...]}`
+ * format. Plain `"markdown"` does not count — that's the default scrape
+ * output and the markdown-diff path runs whenever JSON extraction wasn't
+ * requested.
+ */
+export function formatsRequestGitDiff(formats: unknown): boolean {
+ if (!Array.isArray(formats)) return false;
+ for (const entry of formats) {
+ if (entry && typeof entry === "object") {
+ const obj = entry as { type?: unknown; modes?: unknown };
+ if (obj.type === "changeTracking") {
+ const modes = Array.isArray(obj.modes) ? obj.modes : [];
+ if (modes.includes("git-diff")) return true;
+ }
+ }
+ }
+ return false;
+}
+
+/**
+ * True iff the monitor's scrapeOptions.formats requests JSON extraction —
+ * either as a plain `{type:"json"}` format or as a change-tracking format
+ * with `"json"` in its modes.
+ */
+export function formatsRequestJsonExtraction(formats: unknown): boolean {
+ if (!Array.isArray(formats)) return false;
+ for (const entry of formats) {
+ if (typeof entry === "string" && entry === "json") return true;
+ if (entry && typeof entry === "object") {
+ const obj = entry as { type?: unknown; modes?: unknown };
+ if (obj.type === "json") return true;
+ if (obj.type === "changeTracking") {
+ const modes = Array.isArray(obj.modes) ? obj.modes : [];
+ if (modes.includes("json")) return true;
+ }
+ }
+ }
+ return false;
+}
+
+/**
+ * Collapse monitor-level format shorthand into what the scrape engine
+ * understands. Today the runner owns history; we always want the scrape to
+ * return current values via the plain `{type:"json"}` format. A
+ * `{type:"changeTracking", modes:["json"], schema, prompt}` entry becomes
+ * `{type:"json", schema, prompt}`; if the user asked for `git-diff` mode in
+ * addition, that mode is dropped (markdown is already requested elsewhere).
+ */
+export function normalizeMonitorFormats(formats: unknown): unknown[] {
+ if (!Array.isArray(formats)) return [];
+ const out: unknown[] = [];
+ for (const entry of formats) {
+ if (
+ entry &&
+ typeof entry === "object" &&
+ (entry as { type?: unknown }).type === "changeTracking"
+ ) {
+ const obj = entry as {
+ type: "changeTracking";
+ modes?: unknown;
+ schema?: unknown;
+ prompt?: unknown;
+ tag?: unknown;
+ };
+ const modes = Array.isArray(obj.modes) ? obj.modes : [];
+ if (modes.includes("json")) {
+ out.push({
+ type: "json",
+ ...(typeof obj.prompt === "string" ? { prompt: obj.prompt } : {}),
+ ...(obj.schema && typeof obj.schema === "object"
+ ? { schema: obj.schema }
+ : {}),
+ });
+ }
+ // git-diff mode is implicit via the markdown format already added.
+ continue;
+ }
+ out.push(entry);
+ }
+ return out;
+}
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.ts
index 2c06706f1..fbd82dc0f 100644
--- a/apps/api/src/services/monitoring/results.ts
+++ b/apps/api/src/services/monitoring/results.ts
@@ -1,14 +1,8 @@
-import { v7 as uuidv7 } from "uuid";
import { NuQJob } from "../worker/nuq";
import { ScrapeJobData } from "../../types";
-import { getJobFromGCS } from "../../lib/gcs-jobs";
-import {
- monitorDiffGcsKey,
- saveMonitorDiffArtifact,
-} from "../../lib/gcs-monitoring";
import { logger as _logger } from "../../lib/logger";
import { createWebhookSender, WebhookEvent } from "../webhook";
-import { diffMonitorMarkdown } from "./diff";
+import { computeAndPersistPageDiff } from "./diff-orchestrator";
import {
getMonitorForUpdate,
getMonitorPage,
@@ -88,42 +82,32 @@ export async function recordMonitorScrapeSuccess(
url,
});
- let status: "same" | "new" | "changed" = "new";
- let diffGcsKey: string | null = null;
- let diffTextBytes: number | null = null;
- let diffJsonBytes: number | null = null;
-
- if (previous?.last_scrape_id && !previous.is_removed) {
- const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];
- const previousMarkdown = previousDoc?.markdown;
- const currentMarkdown = doc?.markdown;
-
- if (previousMarkdown && currentMarkdown) {
- const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
- status = diff.status;
-
- if (diff.status === "changed") {
- diffGcsKey = monitorDiffGcsKey({
- teamId: job.data.team_id,
- monitorId: monitoring.monitorId,
- checkId: monitoring.checkId,
- pageId: uuidv7(),
- });
- const sizes = await saveMonitorDiffArtifact(diffGcsKey, {
- url,
- previousScrapeId: previous.last_scrape_id,
- currentScrapeId: job.id,
- text: diff.text,
- json: diff.json,
- generatedAt: new Date().toISOString(),
- });
- diffTextBytes = sizes.textBytes;
- diffJsonBytes = sizes.jsonBytes;
- }
- } else {
- status = "changed";
- }
- }
+ // The monitor row's target carries the canonical formats; fetch it to
+ // decide whether this run is a JSON-extraction monitor.
+ const monitorForRun = await getMonitorForUpdate(
+ job.data.team_id,
+ monitoring.monitorId,
+ );
+ const targetFormats = monitorForRun?.targets?.find(
+ (t: any) => t.id === monitoring.targetId,
+ )?.scrapeOptions?.formats;
+
+ const { status, diffGcsKey, diffTextBytes, diffJsonBytes } =
+ await computeAndPersistPageDiff({
+ teamId: job.data.team_id,
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ url,
+ scrapeId: job.id,
+ doc,
+ previous: previous
+ ? {
+ last_scrape_id: previous.last_scrape_id,
+ is_removed: previous.is_removed,
+ }
+ : null,
+ formats: targetFormats,
+ });
await upsertMonitorPage({
monitorId: monitoring.monitorId,
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index ed110dd18..45ed76835 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -6,11 +6,8 @@ import { processJobInternal } from "../worker/scrape-worker";
import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";
import { ScrapeJobData } from "../../types";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-import {
- monitorDiffGcsKey,
- saveMonitorDiffArtifact,
-} from "../../lib/gcs-monitoring";
-import { diffMonitorMarkdown } from "./diff";
+import { computeAndPersistPageDiff } from "./diff-orchestrator";
+import { normalizeMonitorFormats } from "./diff";
import { autumnService } from "../autumn/autumn.service";
import { getBillingQueue } from "../queue-service";
import {
@@ -115,9 +112,15 @@ function recoverScrapeTargetRunsFromMonitor(
function withMonitorScrapeDefaults(
options: Record<string, unknown>,
): ScrapeOptions {
+ // Monitor owns history: rewrite changeTracking-json to plain json before
+ // handing the formats array to the scrape engine, so we always get back
+ // current values to diff against the previous run.
+ const formats = Array.isArray(options.formats)
+ ? normalizeMonitorFormats(options.formats)
+ : options.formats;
return {
maxAge: 0,
- ...withMarkdownFormat(options),
+ ...withMarkdownFormat({ ...options, formats }),
};
}
@@ -217,42 +220,22 @@ async function diffAndPersistPage(params: {
url: params.url,
});
- let status: PageResult["status"] = "new";
- let diffGcsKey: string | null = null;
- let diffTextBytes: number | null = null;
- let diffJsonBytes: number | null = null;
-
- if (previous?.last_scrape_id && !previous.is_removed) {
- const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];
- const previousMarkdown = previousDoc?.markdown;
- const currentMarkdown = params.doc?.markdown;
-
- if (previousMarkdown && currentMarkdown) {
- const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
- status = diff.status;
-
- if (diff.status === "changed") {
- diffGcsKey = monitorDiffGcsKey({
- teamId: params.monitor.team_id,
- monitorId: params.monitor.id,
- checkId: params.check.id,
- pageId: uuidv7(),
- });
- const sizes = await saveMonitorDiffArtifact(diffGcsKey, {
- url: params.url,
- previousScrapeId: previous.last_scrape_id,
- currentScrapeId: params.scrapeId,
- text: diff.text,
- json: diff.json,
- generatedAt: new Date().toISOString(),
- });
- diffTextBytes = sizes.textBytes;
- diffJsonBytes = sizes.jsonBytes;
- }
- } else {
- status = "changed";
- }
- }
+ const { status, diffGcsKey, diffTextBytes, diffJsonBytes } =
+ await computeAndPersistPageDiff({
+ teamId: params.monitor.team_id,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ url: params.url,
+ scrapeId: params.scrapeId,
+ doc: params.doc,
+ previous: previous
+ ? {
+ last_scrape_id: previous.last_scrape_id,
+ is_removed: previous.is_removed,
+ }
+ : null,
+ formats: params.target.scrapeOptions?.formats,
+ });
await upsertMonitorPage({
monitorId: params.monitor.id,
diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
index da297be8f..d9279e2a1 100644
--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
@@ -8,7 +8,7 @@
<!-- NuGet package metadata -->
<PackageId>firecrawl-sdk</PackageId>
- <Version>1.4.0</Version>
+ <Version>1.5.0</Version>
<Authors>Firecrawl</Authors>
<Company>Firecrawl</Company>
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
diff --git a/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs b/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs
index ed5c7ef83..bb71069f5 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs
@@ -179,6 +179,61 @@ public class MonitorCheck
public string? UpdatedAt { get; set; }
}
+/// <summary>
+/// Per-field diff entry returned for monitors that requested JSON
+/// extraction. The keys on <see cref="MonitorPageDiff.Json"/> (when used
+/// in JSON or mixed mode) are field paths in the extracted JSON; the
+/// values describe what changed between the previous and current run.
+/// </summary>
+public class MonitorJsonFieldDiff
+{
+ [JsonPropertyName("previous")]
+ public object? Previous { get; set; }
+
+ [JsonPropertyName("current")]
+ public object? Current { get; set; }
+}
+
+/// <summary>
+/// Diff payload returned alongside a monitor page when its scrape
+/// produced a change. The shape depends on what the monitor's formats
+/// asked for:
+/// <list type="bullet">
+/// <item>Markdown-only monitors: <see cref="Text"/> holds the unified
+/// diff and <see cref="Json"/> holds the parseDiff AST
+/// (<c>{ "files": [...] }</c>).</item>
+/// <item>JSON-extraction monitors: <see cref="Json"/> holds the
+/// per-field <see cref="MonitorJsonFieldDiff"/> map and
+/// <see cref="Text"/> is null.</item>
+/// <item>Mixed (JSON + git-diff) monitors: both fields are populated:
+/// <see cref="Json"/> is the per-field diff and <see cref="Text"/>
+/// is the markdown sidecar.</item>
+/// </list>
+/// <see cref="Json"/> is exposed as <see cref="object"/> because its
+/// concrete shape depends on the monitor mode; callers should
+/// re-deserialize with <c>System.Text.Json</c> into either a
+/// <c>Dictionary&lt;string, MonitorJsonFieldDiff&gt;</c> (JSON / mixed
+/// mode) or a wrapper containing the <c>files</c> array (markdown mode).
+/// </summary>
+public class MonitorPageDiff
+{
+ [JsonPropertyName("text")]
+ public string? Text { get; set; }
+
+ [JsonPropertyName("json")]
+ public object? Json { get; set; }
+}
+
+/// <summary>
+/// Snapshot of the current JSON extraction at this run. Present on JSON
+/// and mixed-mode monitors; absent for markdown-only monitors.
+/// </summary>
+public class MonitorPageSnapshot
+{
+ [JsonPropertyName("json")]
+ public Dictionary<string, object>? Json { get; set; }
+}
+
public class MonitorCheckPage
{
[JsonPropertyName("id")]
@@ -209,7 +264,10 @@ public class MonitorCheckPage
public object? Metadata { get; set; }
[JsonPropertyName("diff")]
- public object? Diff { get; set; }
+ public MonitorPageDiff? Diff { get; set; }
+
+ [JsonPropertyName("snapshot")]
+ public MonitorPageSnapshot? Snapshot { get; set; }
[JsonPropertyName("createdAt")]
public string? CreatedAt { get; set; }
diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go
index 33b1df63e..5d435c4bd 100644
--- a/apps/go-sdk/models.go
+++ b/apps/go-sdk/models.go
@@ -180,19 +180,54 @@ type MonitorCheck struct {
UpdatedAt string `json:"updatedAt,omitempty"`
}
+// MonitorJsonFieldDiff is a single field-level diff returned for monitors
+// that requested JSON extraction. Keys are field paths in the extracted
+// JSON; values describe what changed between the previous and current run.
+type MonitorJsonFieldDiff struct {
+ Previous interface{} `json:"previous"`
+ Current interface{} `json:"current"`
+}
+
+// MonitorPageDiff is the diff payload returned alongside a monitor page
+// when its scrape produced a change. The shape depends on what the
+// monitor's formats asked for:
+//
+// - markdown-only monitors → Text holds the unified diff and JSON
+// holds the parseDiff AST (a {"files": [...]} object).
+// - JSON-extraction monitors → JSON holds the per-field
+// map[string]MonitorJsonFieldDiff and Text is empty.
+// - mixed (JSON + git-diff) monitors → both fields are populated:
+// JSON is the per-field diff and Text is the markdown sidecar.
+//
+// JSON is left as interface{} so callers can decode into either of the
+// two possible shapes; use json.Unmarshal with a concrete target when
+// the monitor's mode is known.
+type MonitorPageDiff struct {
+ Text string `json:"text,omitempty"`
+ JSON interface{} `json:"json,omitempty"`
+}
+
+// MonitorPageSnapshot is the snapshot of the current JSON extraction at
+// this run. It is present on JSON and mixed-mode monitors and absent
+// for markdown-only monitors.
+type MonitorPageSnapshot struct {
+ JSON map[string]interface{} `json:"json,omitempty"`
+}
+
// MonitorCheckPage is a single page result in a monitor check.
type MonitorCheckPage struct {
- ID string `json:"id"`
- TargetID string `json:"targetId"`
- URL string `json:"url"`
- Status string `json:"status"`
- PreviousScrapeID string `json:"previousScrapeId,omitempty"`
- CurrentScrapeID string `json:"currentScrapeId,omitempty"`
- StatusCode *int `json:"statusCode,omitempty"`
- Error string `json:"error,omitempty"`
- Metadata interface{} `json:"metadata,omitempty"`
- Diff interface{} `json:"diff,omitempty"`
- CreatedAt string `json:"createdAt,omitempty"`
+ ID string `json:"id"`
+ TargetID string `json:"targetId"`
+ URL string `json:"url"`
+ Status string `json:"status"`
+ PreviousScrapeID string `json:"previousScrapeId,omitempty"`
+ CurrentScrapeID string `json:"currentScrapeId,omitempty"`
+ StatusCode *int `json:"statusCode,omitempty"`
+ Error string `json:"error,omitempty"`
+ Metadata interface{} `json:"metadata,omitempty"`
+ Diff *MonitorPageDiff `json:"diff,omitempty"`
+ Snapshot *MonitorPageSnapshot `json:"snapshot,omitempty"`
+ CreatedAt string `json:"createdAt,omitempty"`
}
// MonitorCheckDetail includes paginated page results and inline diffs.
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index a6ab7a966..82c6fbcf5 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.3.0"
+const Version = "1.4.0"
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index b0a7ffab0..c179499e1 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.6.0"
+version = "1.7.0"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java
index 231d3788e..ed547021f 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java
@@ -13,7 +13,8 @@ public class MonitorCheckPage {
private Integer statusCode;
private String error;
private Object metadata;
- private Object diff;
+ private MonitorPageDiff diff;
+ private MonitorPageSnapshot snapshot;
private String createdAt;
public String getId() { return id; }
@@ -25,6 +26,7 @@ public class MonitorCheckPage {
public Integer getStatusCode() { return statusCode; }
public String getError() { return error; }
public Object getMetadata() { return metadata; }
- public Object getDiff() { return diff; }
+ public MonitorPageDiff getDiff() { return diff; }
+ public MonitorPageSnapshot getSnapshot() { return snapshot; }
public String getCreatedAt() { return createdAt; }
}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorJsonFieldDiff.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorJsonFieldDiff.java
new file mode 100644
index 000000000..fa5c001f9
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorJsonFieldDiff.java
@@ -0,0 +1,19 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * Per-field diff entry returned for monitors that requested JSON
+ * extraction. The keys on {@link MonitorPageDiff#getJson()} (when used in
+ * JSON or mixed mode) are field paths in the extracted JSON; the values
+ * are instances of this class describing what changed between the
+ * previous and current run.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MonitorJsonFieldDiff {
+ private Object previous;
+ private Object current;
+
+ public Object getPrevious() { return previous; }
+ public Object getCurrent() { return current; }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageDiff.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageDiff.java
new file mode 100644
index 000000000..cf4ef3a70
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageDiff.java
@@ -0,0 +1,35 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * Diff payload returned alongside a monitor page when its scrape
+ * produced a change. The shape depends on what the monitor's formats
+ * asked for:
+ *
+ * <ul>
+ * <li>Markdown-only monitors: {@code text} holds the unified diff
+ * and {@code json} holds the {@code parseDiff} AST (a
+ * {@code {"files": [...]}} object).</li>
+ * <li>JSON-extraction monitors: {@code json} holds the per-field
+ * map of {@link MonitorJsonFieldDiff} entries and {@code text} is
+ * absent.</li>
+ * <li>Mixed (JSON + git-diff) monitors: both fields are populated:
+ * {@code json} is the per-field diff map and {@code text} is the
+ * markdown sidecar.</li>
+ * </ul>
+ *
+ * <p>{@code json} is exposed as {@link Object} because its concrete
+ * shape depends on the monitor mode; callers should cast or re-deserialize
+ * with Jackson into either a {@code Map<String, MonitorJsonFieldDiff>}
+ * (JSON / mixed mode) or a {@code Map<String, Object>} containing the
+ * {@code files} key (markdown mode).
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MonitorPageDiff {
+ private String text;
+ private Object json;
+
+ public String getText() { return text; }
+ public Object getJson() { return json; }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageSnapshot.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageSnapshot.java
new file mode 100644
index 000000000..72081b671
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageSnapshot.java
@@ -0,0 +1,15 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import java.util.Map;
+
+/**
+ * Snapshot of the current JSON extraction at this run. Present on JSON
+ * and mixed-mode monitors; absent for markdown-only monitors.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MonitorPageSnapshot {
+ private Map<String, Object> json;
+
+ public Map<String, Object> getJson() { return json; }
+}
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 3a6f436c6..dc8c03cf1 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.23.0",
+ "version": "4.24.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 2ceae804d..baa69a431 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -1,20 +1,20 @@
-import type { ZodTypeAny } from 'zod';
+import type { ZodTypeAny } from "zod";
// Public types for Firecrawl JS/TS SDK v2 (camelCase only)
export type FormatString =
- | 'markdown'
- | 'html'
- | 'rawHtml'
- | 'links'
- | 'images'
- | 'screenshot'
- | 'summary'
- | 'changeTracking'
- | 'json'
- | 'attributes'
- | 'branding'
- | 'audio'
- | 'video';
+ | "markdown"
+ | "html"
+ | "rawHtml"
+ | "links"
+ | "images"
+ | "screenshot"
+ | "summary"
+ | "changeTracking"
+ | "json"
+ | "attributes"
+ | "branding"
+ | "audio"
+ | "video";
export interface Viewport {
width: number;
@@ -26,27 +26,27 @@ export interface Format {
}
export interface JsonFormat extends Format {
- type: 'json';
+ type: "json";
prompt?: string;
schema?: Record<string, unknown> | ZodTypeAny;
}
export interface ScreenshotFormat {
- type: 'screenshot';
+ type: "screenshot";
fullPage?: boolean;
quality?: number;
viewport?: Viewport | { width: number; height: number };
}
export interface ChangeTrackingFormat extends Format {
- type: 'changeTracking';
- modes: ('git-diff' | 'json')[];
+ type: "changeTracking";
+ modes: ("git-diff" | "json")[];
schema?: Record<string, unknown>;
prompt?: string;
tag?: string;
}
export interface AttributesFormat extends Format {
- type: 'attributes';
+ type: "attributes";
selectors: Array<{
selector: string;
attribute: string;
@@ -54,20 +54,20 @@ export interface AttributesFormat extends Format {
}
export interface QuestionFormat {
- type: 'question';
+ type: "question";
question: string;
}
export interface HighlightsFormat {
- type: 'highlights';
+ type: "highlights";
query: string;
}
/** @deprecated Use QuestionFormat or HighlightsFormat instead. */
export interface QueryFormat {
- type: 'query';
+ type: "query";
prompt: string;
- mode?: 'freeform' | 'directQuote';
+ mode?: "freeform" | "directQuote";
}
export type FormatOption =
@@ -83,7 +83,7 @@ export type FormatOption =
export type ParseFormatString = Exclude<
FormatString,
- 'screenshot' | 'changeTracking' | 'branding' | 'audio' | 'video'
+ "screenshot" | "changeTracking" | "branding" | "audio" | "video"
>;
export interface ParseFormat {
@@ -105,62 +105,62 @@ export interface LocationConfig {
}
export interface WaitAction {
- type: 'wait';
+ type: "wait";
milliseconds?: number;
selector?: string;
}
export interface ScreenshotAction {
- type: 'screenshot';
+ type: "screenshot";
fullPage?: boolean;
quality?: number;
viewport?: Viewport | { width: number; height: number };
}
export interface ClickAction {
- type: 'click';
+ type: "click";
selector: string;
}
export interface WriteAction {
- type: 'write';
+ type: "write";
text: string;
}
export interface PressAction {
- type: 'press';
+ type: "press";
key: string;
}
export interface ScrollAction {
- type: 'scroll';
- direction: 'up' | 'down';
+ type: "scroll";
+ direction: "up" | "down";
selector?: string;
}
export interface ScrapeAction {
- type: 'scrape';
+ type: "scrape";
}
export interface ExecuteJavascriptAction {
- type: 'executeJavascript';
+ type: "executeJavascript";
script: string;
}
export interface PDFAction {
- type: 'pdf';
+ type: "pdf";
format?:
- | 'A0'
- | 'A1'
- | 'A2'
- | 'A3'
- | 'A4'
- | 'A5'
- | 'A6'
- | 'Letter'
- | 'Legal'
- | 'Tabloid'
- | 'Ledger';
+ | "A0"
+ | "A1"
+ | "A2"
+ | "A3"
+ | "A4"
+ | "A5"
+ | "A6"
+ | "Letter"
+ | "Legal"
+ | "Tabloid"
+ | "Ledger";
landscape?: boolean;
scale?: number;
}
@@ -185,7 +185,9 @@ export interface ScrapeOptions {
timeout?: number;
waitFor?: number;
mobile?: boolean;
- parsers?: Array<string | { type: 'pdf'; mode?: 'fast' | 'auto' | 'ocr'; maxPages?: number }>;
+ parsers?: Array<
+ string | { type: "pdf"; mode?: "fast" | "auto" | "ocr"; maxPages?: number }
+ >;
actions?: ActionOption[];
location?: LocationConfig;
skipTlsVerification?: boolean;
@@ -193,7 +195,7 @@ export interface ScrapeOptions {
fastMode?: boolean;
useMock?: string;
blockAds?: boolean;
- proxy?: 'basic' | 'stealth' | 'enhanced' | 'auto' | string;
+ proxy?: "basic" | "stealth" | "enhanced" | "auto" | string;
maxAge?: number;
minAge?: number;
storeInCache?: boolean;
@@ -222,30 +224,35 @@ export interface ParseFile {
export type ParseOptions = Omit<
ScrapeOptions,
- | 'formats'
- | 'waitFor'
- | 'mobile'
- | 'actions'
- | 'location'
- | 'maxAge'
- | 'minAge'
- | 'storeInCache'
- | 'lockdown'
- | 'proxy'
+ | "formats"
+ | "waitFor"
+ | "mobile"
+ | "actions"
+ | "location"
+ | "maxAge"
+ | "minAge"
+ | "storeInCache"
+ | "lockdown"
+ | "proxy"
> & {
formats?: ParseFormatOption[];
- proxy?: 'basic' | 'auto';
+ proxy?: "basic" | "auto";
};
export interface WebhookConfig {
url: string;
headers?: Record<string, string>;
metadata?: Record<string, string>;
- events?: Array<'completed' | 'failed' | 'page' | 'started'>;
+ events?: Array<"completed" | "failed" | "page" | "started">;
}
// Agent webhook events differ from crawl: has 'action' and 'cancelled', no 'page'
-export type AgentWebhookEvent = 'started' | 'action' | 'completed' | 'failed' | 'cancelled';
+export type AgentWebhookEvent =
+ | "started"
+ | "action"
+ | "completed"
+ | "failed"
+ | "cancelled";
export interface AgentWebhookConfig {
url: string;
@@ -255,7 +262,7 @@ export interface AgentWebhookConfig {
}
export interface BrandingProfile {
- colorScheme?: 'light' | 'dark';
+ colorScheme?: "light" | "dark";
logo?: string | null;
fonts?: Array<{
family: string;
@@ -377,13 +384,13 @@ export interface BrandingProfile {
};
personality?: {
tone:
- | 'professional'
- | 'playful'
- | 'modern'
- | 'traditional'
- | 'minimalist'
- | 'bold';
- energy: 'low' | 'medium' | 'high';
+ | "professional"
+ | "playful"
+ | "modern"
+ | "traditional"
+ | "minimalist"
+ | "bold";
+ energy: "low" | "medium" | "high";
targetAudience: string;
};
[key: string]: unknown;
@@ -435,8 +442,8 @@ export interface DocumentMetadata {
numPages?: number;
contentType?: string;
timezone?: string;
- proxyUsed?: 'basic' | 'stealth';
- cacheState?: 'hit' | 'miss';
+ proxyUsed?: "basic" | "stealth";
+ cacheState?: "hit" | "miss";
cachedAt?: string;
creditsUsed?: number;
concurrencyLimited?: boolean;
@@ -518,15 +525,15 @@ export interface SearchData {
}
export interface CategoryOption {
- type: 'github' | 'research' | 'pdf';
+ type: "github" | "research" | "pdf";
}
export interface SearchRequest {
query: string;
sources?: Array<
- 'web' | 'news' | 'images' | { type: 'web' | 'news' | 'images' }
+ "web" | "news" | "images" | { type: "web" | "news" | "images" }
>;
- categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;
+ categories?: Array<"github" | "research" | "pdf" | CategoryOption>;
includeDomains?: string[];
excludeDomains?: string[];
limit?: number;
@@ -544,7 +551,7 @@ export interface CrawlOptions {
excludePaths?: string[] | null;
includePaths?: string[] | null;
maxDiscoveryDepth?: number | null;
- sitemap?: 'skip' | 'include' | 'only';
+ sitemap?: "skip" | "include" | "only";
ignoreQueryParameters?: boolean;
deduplicateSimilarURLs?: boolean;
limit?: number | null;
@@ -570,7 +577,7 @@ export interface CrawlResponse {
export interface CrawlJob {
id: string;
- status: 'scraping' | 'completed' | 'failed' | 'cancelled';
+ status: "scraping" | "completed" | "failed" | "cancelled";
total: number;
completed: number;
creditsUsed?: number;
@@ -599,7 +606,7 @@ export interface BatchScrapeResponse {
export interface BatchScrapeJob {
id: string;
- status: 'scraping' | 'completed' | 'failed' | 'cancelled';
+ status: "scraping" | "completed" | "failed" | "cancelled";
completed: number;
total: number;
creditsUsed?: number;
@@ -614,7 +621,7 @@ export interface MapData {
export interface MapOptions {
search?: string;
- sitemap?: 'only' | 'include' | 'skip';
+ sitemap?: "only" | "include" | "skip";
includeSubdomains?: boolean;
ignoreQueryParameters?: boolean;
limit?: number;
@@ -648,14 +655,14 @@ export interface MonitorWebhookConfig {
export interface MonitorScrapeTarget {
id?: string;
- type: 'scrape';
+ type: "scrape";
urls: string[];
scrapeOptions?: ScrapeOptions;
}
export interface MonitorCrawlTarget {
id?: string;
- type: 'crawl';
+ type: "crawl";
url: string;
crawlOptions?: CrawlOptions;
scrapeOptions?: ScrapeOptions;
@@ -674,7 +681,7 @@ export interface CreateMonitorRequest {
export interface UpdateMonitorRequest {
name?: string;
- status?: 'active' | 'paused';
+ status?: "active" | "paused";
schedule?: MonitorSchedule;
webhook?: MonitorWebhookConfig | null;
notification?: MonitorNotification | null;
@@ -694,7 +701,7 @@ export interface MonitorSummary {
export interface Monitor {
id: string;
name: string;
- status: 'active' | 'paused' | 'deleted';
+ status: "active" | "paused" | "deleted";
schedule: MonitorSchedule;
nextRunAt?: string | null;
lastRunAt?: string | null;
@@ -713,13 +720,13 @@ export interface MonitorCheck {
id: string;
monitorId: string;
status:
- | 'queued'
- | 'running'
- | 'completed'
- | 'failed'
- | 'partial'
- | 'skipped_overlap';
- trigger: 'scheduled' | 'manual';
+ | "queued"
+ | "running"
+ | "completed"
+ | "failed"
+ | "partial"
+ | "skipped_overlap";
+ trigger: "scheduled" | "manual";
scheduledFor?: string | null;
startedAt?: string | null;
finishedAt?: string | null;
@@ -727,11 +734,11 @@ export interface MonitorCheck {
reservedCredits?: number | null;
actualCredits?: number | null;
billingStatus:
- | 'not_applicable'
- | 'reserved'
- | 'confirmed'
- | 'released'
- | 'failed';
+ | "not_applicable"
+ | "reserved"
+ | "confirmed"
+ | "released"
+ | "failed";
summary: MonitorSummary;
targetResults?: unknown;
notificationStatus?: unknown;
@@ -740,17 +747,48 @@ export interface MonitorCheck {
updatedAt: string;
}
+/** Per-field diff for monitors that requested JSON extraction. */
+export interface MonitorJsonFieldDiff {
+ [field: string]: { previous: unknown; current: unknown };
+}
+
+/**
+ * Diff payload returned alongside a monitor page when its scrape produced
+ * a change. The shape depends on what the monitor's formats asked for:
+ *
+ * - markdown-only monitors → `{ text, json }` where `json` is the
+ * `parseDiff` AST (a `{ files: [...] }` object).
+ * - JSON-extraction monitors → `{ json }` where `json` is the per-field
+ * `{ previous, current }` map.
+ * - Mixed (JSON + git-diff) monitors → both `text` (markdown sidecar)
+ * and `json` (field-level diff) are present.
+ */
+export interface MonitorPageDiff {
+ text?: string;
+ /** Markdown variants: parseDiff AST. JSON variants: per-field diff. */
+ json?: MonitorJsonFieldDiff | { files: unknown[] };
+}
+
+/**
+ * Snapshot of the current JSON extraction at this run. Present on JSON
+ * and mixed-mode monitors; absent for markdown-only.
+ */
+export interface MonitorPageSnapshot {
+ json?: Record<string, unknown>;
+}
+
export interface MonitorCheckPage {
id: string;
targetId: string;
url: string;
- status: 'same' | 'new' | 'changed' | 'removed' | 'error';
+ status: "same" | "new" | "changed" | "removed" | "error";
previousScrapeId?: string | null;
currentScrapeId?: string | null;
statusCode?: number | null;
error?: string | null;
metadata?: unknown;
- diff?: unknown;
+ diff?: MonitorPageDiff | null;
+ snapshot?: MonitorPageSnapshot | null;
createdAt: string;
}
@@ -775,7 +813,7 @@ export type GetMonitorCheckOptions = PaginationConfig & {
export interface ExtractResponse {
success?: boolean;
id?: string;
- status?: 'processing' | 'completed' | 'failed' | 'cancelled';
+ status?: "processing" | "completed" | "failed" | "cancelled";
data?: unknown;
error?: string;
warning?: string;
@@ -794,16 +832,16 @@ export interface AgentResponse {
export interface AgentStatusResponse {
success: boolean;
- status: 'processing' | 'completed' | 'failed';
+ status: "processing" | "completed" | "failed";
error?: string;
data?: unknown;
- model?: 'spark-1-pro' | 'spark-1-mini';
+ model?: "spark-1-pro" | "spark-1-mini";
expiresAt: string;
creditsUsed?: number;
}
export interface AgentOptions {
- model: 'FIRE-1' | 'v3-beta';
+ model: "FIRE-1" | "v3-beta";
}
export interface ConcurrencyCheck {
@@ -889,10 +927,10 @@ export class SdkError extends Error {
status?: number,
code?: string,
details?: unknown,
- jobId?: string
+ jobId?: string,
) {
super(message);
- this.name = 'FirecrawlSdkError';
+ this.name = "FirecrawlSdkError";
this.status = status;
this.code = code;
this.details = details;
@@ -902,16 +940,20 @@ export class SdkError extends Error {
export class JobTimeoutError extends SdkError {
timeoutSeconds: number;
- constructor(jobId: string, timeoutSeconds: number, jobType: 'batch' | 'crawl' = 'batch') {
- const jobTypeLabel = jobType === 'batch' ? 'batch scrape' : 'crawl';
+ constructor(
+ jobId: string,
+ timeoutSeconds: number,
+ jobType: "batch" | "crawl" = "batch",
+ ) {
+ const jobTypeLabel = jobType === "batch" ? "batch scrape" : "crawl";
super(
`${jobTypeLabel.charAt(0).toUpperCase() + jobTypeLabel.slice(1)} job ${jobId} did not complete within ${timeoutSeconds} seconds`,
undefined,
- 'JOB_TIMEOUT',
+ "JOB_TIMEOUT",
undefined,
- jobId
+ jobId,
);
- this.name = 'JobTimeoutError';
+ this.name = "JobTimeoutError";
this.timeoutSeconds = timeoutSeconds;
}
}
diff --git a/apps/php-sdk/src/Models/MonitorCheckDetail.php b/apps/php-sdk/src/Models/MonitorCheckDetail.php
index f6709c81e..8f75538aa 100644
--- a/apps/php-sdk/src/Models/MonitorCheckDetail.php
+++ b/apps/php-sdk/src/Models/MonitorCheckDetail.php
@@ -7,7 +7,15 @@ namespace Firecrawl\Models;
final class MonitorCheckDetail extends MonitorCheck
{
/**
- * @param list<array<string, mixed>> $pages
+ * @param list<array<string, mixed>> $pages Each page array contains
+ * keys: id, targetId, url, status, previousScrapeId,
+ * currentScrapeId, statusCode, error, metadata, diff, snapshot,
+ * createdAt. The `diff` value (when present) is an array with
+ * `text` (markdown unified diff) and/or `json` (parseDiff AST
+ * for markdown monitors, or a per-field
+ * `{previous, current}` map for JSON-extraction monitors). The
+ * `snapshot` value (JSON / mixed mode only) is an array with a
+ * `json` key holding the current JSON extraction at this run.
*/
public function __construct(
?string $id = null,
@@ -80,7 +88,12 @@ final class MonitorCheckDetail extends MonitorCheck
return $check;
}
- /** @return list<array<string, mixed>> */
+ /**
+ * @return list<array<string, mixed>> Each page array contains the
+ * standard monitor page fields plus the `diff` and `snapshot`
+ * payloads when present. See the constructor PHPDoc for the
+ * shape of those nested values.
+ */
public function getPages(): array { return $this->pages; }
public function getNext(): ?string { return $this->next; }
}
diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php
index 1ed1826b6..ac8491217 100644
--- a/apps/php-sdk/src/Version.php
+++ b/apps/php-sdk/src/Version.php
@@ -6,5 +6,5 @@ namespace Firecrawl;
final class Version
{
- public const SDK_VERSION = '1.3.0';
+ public const SDK_VERSION = '1.4.0';
}
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index a5e0b906f..804a56270 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.26.1"
+__version__ = "4.27.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 0ef08b9e2..f5e2f46ab 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -931,6 +931,28 @@ class MonitorCheck(BaseModel):
updated_at: str = Field(alias="updatedAt")
+class MonitorPageDiff(BaseModel):
+ """Diff payload returned alongside a monitor page.
+
+ Markdown-only monitors populate both `text` (unified diff) and `json`
+ (the parseDiff AST). JSON-extraction monitors populate `json` only,
+ where `json` is the per-field `{previous, current}` map. Mixed-mode
+ monitors (JSON + git-diff) populate both `json` (field diff) and
+ `text` (markdown sidecar).
+ """
+ model_config = {"populate_by_name": True, "extra": "allow"}
+
+ text: Optional[str] = None
+ json: Optional[Any] = None # markdown→parseDiff AST | json→field diff
+
+
+class MonitorPageSnapshot(BaseModel):
+ """Current JSON extraction at this run. JSON / mixed mode only."""
+ model_config = {"populate_by_name": True, "extra": "allow"}
+
+ json: Optional[Dict[str, Any]] = None
+
+
class MonitorCheckPage(BaseModel):
model_config = {"populate_by_name": True, "extra": "allow"}
@@ -943,7 +965,8 @@ class MonitorCheckPage(BaseModel):
status_code: Optional[int] = Field(default=None, alias="statusCode")
error: Optional[str] = None
metadata: Optional[Any] = None
- diff: Optional[Any] = None
+ diff: Optional[MonitorPageDiff] = None
+ snapshot: Optional[MonitorPageSnapshot] = None
created_at: str = Field(alias="createdAt")
diff --git a/apps/ruby-sdk/lib/firecrawl/models/monitor.rb b/apps/ruby-sdk/lib/firecrawl/models/monitor.rb
index 8948baf23..5245aeeb0 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/monitor.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/monitor.rb
@@ -55,6 +55,19 @@ module Firecrawl
end
end
+ # A monitor check with paginated page results inlined.
+ #
+ # Each entry in {#pages} is a {Hash} with the standard monitor page
+ # fields (id, targetId, url, status, previousScrapeId,
+ # currentScrapeId, statusCode, error, metadata, createdAt) plus:
+ #
+ # * +"diff"+ – when the page changed. A hash with +"text"+ (markdown
+ # unified diff) and/or +"json"+ (parseDiff AST for markdown
+ # monitors, or a per-field +{ "previous", "current" }+ map for
+ # JSON-extraction monitors).
+ # * +"snapshot"+ – present on JSON / mixed-mode monitors. A hash
+ # with a +"json"+ key holding the current JSON extraction at this
+ # run.
class MonitorCheckDetail < MonitorCheck
attr_accessor :pages, :next_url
diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb
index 7bb66e07a..7cffb0bd0 100644
--- a/apps/ruby-sdk/lib/firecrawl/version.rb
+++ b/apps/ruby-sdk/lib/firecrawl/version.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
module Firecrawl
- VERSION = "1.5.0"
+ VERSION = "1.6.0"
end
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index 2c3cb7cc3..1eaf8660e 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "2.5.0"
+version = "2.6.0"
dependencies = [
"mockito",
"reqwest",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 2c48e2edd..da9d860d1 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "firecrawl"
-version = "2.5.0"
+version = "2.6.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/monitor.rs b/apps/rust-sdk/src/monitor.rs
index 2b7e1b493..1273c4420 100644
--- a/apps/rust-sdk/src/monitor.rs
+++ b/apps/rust-sdk/src/monitor.rs
@@ -91,6 +91,42 @@ pub struct MonitorCheck {
pub updated_at: String,
}
+/// Per-field diff entry returned for monitors that requested JSON extraction.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct MonitorJsonFieldDiff {
+ pub previous: Value,
+ pub current: Value,
+}
+
+/// Diff payload returned alongside a monitor page when its scrape produced
+/// a change. The shape depends on what the monitor's formats asked for:
+///
+/// - markdown-only monitors → `text` is the unified diff and `json` is
+/// the `parseDiff` AST (a `{ "files": [...] }` object).
+/// - JSON-extraction monitors → `json` is the per-field
+/// `{ previous, current }` map and `text` is absent.
+/// - mixed (JSON + git-diff) monitors → both `text` (markdown sidecar)
+/// and `json` (field-level diff) are present.
+///
+/// `json` is kept as a raw [`serde_json::Value`] so callers can decode it
+/// into either shape (`HashMap<String, MonitorJsonFieldDiff>` for the
+/// field-diff case, or a `{ files: [...] }` struct for the parseDiff AST).
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct MonitorPageDiff {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub text: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub json: Option<Value>,
+}
+
+/// Snapshot of the current JSON extraction at this run. Present on JSON
+/// and mixed-mode monitors; absent for markdown-only.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct MonitorPageSnapshot {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub json: Option<Value>,
+}
+
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct MonitorCheckPage {
@@ -103,7 +139,8 @@ pub struct MonitorCheckPage {
pub status_code: Option<u16>,
pub error: Option<String>,
pub metadata: Option<Value>,
- pub diff: Option<Value>,
+ pub diff: Option<MonitorPageDiff>,
+ pub snapshot: Option<MonitorPageSnapshot>,
pub created_at: String,
}