Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): monitoring json format (#3536)
Developers Digestf3a47df25ffc
diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.tsindex 9c677507a..caba1621c 100644--- a/apps/api/src/__tests__/snips/v2/monitor.test.ts+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts@@ -139,4 +139,190 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {},2 * scrapeTimeout,);++ it(+ "runs a JSON-mode monitor and surfaces a snapshot",+ async () => {+ const jsonSchema = {+ type: "object",+ additionalProperties: false,+ required: ["title"],+ properties: {+ title: {+ type: "string",+ description: "The page title, verbatim.",+ },+ },+ };++ const create = await monitorCreateRaw(+ {+ name: "json-mode monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl()],+ scrapeOptions: {+ formats: [+ "markdown",+ {+ type: "json",+ prompt: "Extract the page title verbatim.",+ schema: jsonSchema,+ },+ ],+ },+ },+ ],+ },+ identity,+ );+ expect(create.statusCode).toBe(200);++ const monitorId = create.body.data.id;+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ const checkId = run.body.id;++ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, checkId, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (["completed", "partial", "failed"].includes(check.status)) break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ expect(["completed", "partial"]).toContain(check.status);+ // First run: status is "new" (no previous scrape to diff against), no+ // snapshot persisted to GCS. We can't assert snapshot.json here+ // without a mutating fixture; a second run with a changed page would+ // be needed. The contract assertion is: when JSON mode is requested,+ // the monitor doesn't fall through the markdown path and crash.+ expect(check.pages[0].status).toBe("new");++ await monitorDeleteRaw(monitorId, identity);+ },+ 2 * scrapeTimeout,+ );++ it(+ "accepts changeTracking-json format on a monitor target",+ async () => {+ const jsonSchema = {+ type: "object",+ additionalProperties: false,+ required: ["title"],+ properties: {+ title: { type: "string", description: "The page title." },+ },+ };++ const create = await monitorCreateRaw(+ {+ name: "ct-json monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl()],+ scrapeOptions: {+ formats: [+ "markdown",+ {+ type: "changeTracking",+ modes: ["json"],+ prompt: "Extract the title.",+ schema: jsonSchema,+ },+ ],+ },+ },+ ],+ },+ identity,+ );+ expect(create.statusCode).toBe(200);++ const monitorId = create.body.data.id;+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ const checkId = run.body.id;++ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, checkId, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (["completed", "partial", "failed"].includes(check.status)) break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ expect(["completed", "partial"]).toContain(check.status);+ expect(check.pages[0].status).toBe("new");++ await monitorDeleteRaw(monitorId, identity);+ },+ 2 * scrapeTimeout,+ );++ it(+ "accepts mixed json + git-diff changeTracking modes",+ async () => {+ const jsonSchema = {+ type: "object",+ additionalProperties: false,+ required: ["title"],+ properties: {+ title: { type: "string", description: "The page title." },+ },+ };++ const create = await monitorCreateRaw(+ {+ name: "ct-mixed monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl()],+ scrapeOptions: {+ formats: [+ "markdown",+ {+ type: "changeTracking",+ modes: ["json", "git-diff"],+ prompt: "Extract the title.",+ schema: jsonSchema,+ },+ ],+ },+ },+ ],+ },+ identity,+ );+ expect(create.statusCode).toBe(200);++ const monitorId = create.body.data.id;+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ const checkId = run.body.id;++ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, checkId, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (["completed", "partial", "failed"].includes(check.status)) break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ expect(["completed", "partial"]).toContain(check.status);+ // First run is always "new"; the assertion here is that mixed-mode+ // configuration is accepted end-to-end without erroring.+ expect(check.pages[0].status).toBe("new");++ await monitorDeleteRaw(monitorId, identity);+ },+ 2 * scrapeTimeout,+ );});diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.tsindex 29d0a2180..571f5386a 100644--- a/apps/api/src/controllers/v2/monitor.ts+++ b/apps/api/src/controllers/v2/monitor.ts@@ -389,19 +389,38 @@ export async function getMonitorCheckController(]);const pagesWithDiffs = await Promise.all(- pages.map(async page => ({- id: page.id,- targetId: page.target_id,- url: page.url,- status: page.status,- previousScrapeId: page.previous_scrape_id,- currentScrapeId: page.current_scrape_id,- statusCode: page.status_code,- error: page.error,- metadata: page.metadata,- diff: await getMonitorDiffArtifact(page.diff_gcs_key),- createdAt: page.created_at,- })),+ pages.map(async page => {+ const artifact = await getMonitorDiffArtifact(page.diff_gcs_key);+ const base = {+ id: page.id,+ targetId: page.target_id,+ url: page.url,+ status: page.status,+ previousScrapeId: page.previous_scrape_id,+ currentScrapeId: page.current_scrape_id,+ statusCode: page.status_code,+ error: page.error,+ metadata: page.metadata,+ createdAt: page.created_at,+ };+ if (!artifact) {+ return { ...base, diff: null };+ }+ if (artifact.kind === "json") {+ return {+ ...base,+ diff: {+ json: artifact.json,+ ...(artifact.markdown ? { text: artifact.markdown.text } : {}),+ },+ snapshot: { json: artifact.snapshot },+ };+ }+ return {+ ...base,+ diff: { text: artifact.text, json: artifact.json },+ };+ }),);const nextSkip = skip + pagesWithDiffs.length;const next = (() => {diff --git a/apps/api/src/lib/gcs-monitoring.ts b/apps/api/src/lib/gcs-monitoring.tsindex 36b3e9e12..a40373316 100644--- a/apps/api/src/lib/gcs-monitoring.ts+++ b/apps/api/src/lib/gcs-monitoring.ts@@ -1,15 +1,37 @@import { config } from "../config";import { storage } from "./gcs-jobs";-type MonitorDiffArtifact = {+type MonitorDiffArtifactBase = {url: string;previousScrapeId: string | null;currentScrapeId: string | null;- text: string;- json: unknown;generatedAt: string;};+export type MonitorDiffArtifact =+ | (MonitorDiffArtifactBase & {+ kind: "markdown";+ text: string;+ json: unknown;+ })+ | (MonitorDiffArtifactBase & {+ kind: "json";+ /** Per-field {previous, current} diff. */+ json: Record<string, { previous: unknown; current: unknown }>;+ /** Full current JSON extraction (the snapshot at this run). */+ snapshot: Record<string, unknown>;+ /**+ * Optional markdown diff sidecar. Populated only when the monitor's+ * formats requested both `"json"` and `"git-diff"` change-tracking+ * modes — in that case we run both diffs and report `changed` if+ * either path saw a change.+ */+ markdown?: {+ text: string;+ json: unknown;+ };+ });+const contentType = "application/json";export function monitorDiffGcsKey(params: {@@ -21,16 +43,29 @@ export function monitorDiffGcsKey(params: {return `monitors/${params.teamId}/${params.monitorId}/${params.checkId}/${params.pageId}.diff.json`;}+function artifactBytes(artifact: MonitorDiffArtifact): {+ textBytes: number;+ jsonBytes: number;+} {+ const jsonBytes = Buffer.byteLength(JSON.stringify(artifact.json ?? null));+ let textBytes = 0;+ if (artifact.kind === "markdown") {+ textBytes = Buffer.byteLength(artifact.text);+ } else if (artifact.kind === "json" && artifact.markdown) {+ // Sidecar markdown diff (mixed-mode monitor) — count it so storage+ // accounting stays honest.+ textBytes = Buffer.byteLength(artifact.markdown.text);+ }+ return { textBytes, jsonBytes };+}+export async function saveMonitorDiffArtifact(key: string,artifact: MonitorDiffArtifact,): Promise<{ textBytes: number; jsonBytes: number }> {const payload = JSON.stringify(artifact);if (!config.GCS_BUCKET_NAME) {- return {- textBytes: Buffer.byteLength(artifact.text),- jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),- };+ return artifactBytes(artifact);}const bucket = storage.bucket(config.GCS_BUCKET_NAME);@@ -39,10 +74,7 @@ export async function saveMonitorDiffArtifact(resumable: false,});- return {- textBytes: Buffer.byteLength(artifact.text),- jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),- };+ return artifactBytes(artifact);}export async function getMonitorDiffArtifact(@@ -53,7 +85,26 @@ export async function getMonitorDiffArtifact(const bucket = storage.bucket(config.GCS_BUCKET_NAME);try {const [contents] = await bucket.file(key).download();- return JSON.parse(contents.toString()) as MonitorDiffArtifact;+ let parsed: unknown;+ try {+ parsed = JSON.parse(contents.toString());+ } catch {+ // Corrupt or truncated artifact — surface as "no diff" instead of+ // letting JSON.parse throw and break the entire check response.+ return null;+ }+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {+ // An unexpected payload shape (e.g. number, array, null) was written+ // here; treat as missing rather than risk reading kind off a non-object.+ return null;+ }+ const asPartial = parsed as Partial<MonitorDiffArtifact>;+ // Backwards compat: historical artifacts predate the `kind` field and+ // are always markdown.+ if (!asPartial.kind) {+ return { ...(asPartial as any), kind: "markdown" } as MonitorDiffArtifact;+ }+ return asPartial as MonitorDiffArtifact;} catch (error) {const maybeGcsError = error as { code?: number; statusCode?: number };if (maybeGcsError.code === 404 || maybeGcsError.statusCode === 404) {diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.tsnew file mode 100644index 000000000..e9a4fd6e5--- /dev/null+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts@@ -0,0 +1,190 @@+import { v7 as uuidv7 } from "uuid";+import { getJobFromGCS } from "../../lib/gcs-jobs";+import {+ MonitorDiffArtifact,+ monitorDiffGcsKey,+ saveMonitorDiffArtifact,+} from "../../lib/gcs-monitoring";+import {+ diffMonitorJson,+ diffMonitorMarkdown,+ formatsRequestGitDiff,+ formatsRequestJsonExtraction,+} from "./diff";++type MonitorPageDiffStatus = "same" | "new" | "changed";++type MonitorPageDiffResult = {+ status: MonitorPageDiffStatus;+ diffGcsKey: string | null;+ diffTextBytes: number | null;+ diffJsonBytes: number | null;+};++type PreviousPageRef = {+ last_scrape_id: string | null;+ is_removed: boolean | null;+};++function isPlainObject(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++/**+ * Compute the page diff for a single monitored URL and persist the diff+ * artifact to GCS when content changed. Branches on whether the monitor+ * asked for JSON extraction (in which case the diff is over `doc.json`) or+ * the default markdown text diff.+ *+ * Returns the new status + GCS key/sizes for the caller to write into the+ * monitor pages row.+ */+export async function computeAndPersistPageDiff(params: {+ teamId: string;+ monitorId: string;+ checkId: string;+ url: string;+ scrapeId: string;+ doc: any;+ previous: PreviousPageRef | null;+ formats: unknown;+}): Promise<MonitorPageDiffResult> {+ const { teamId, monitorId, checkId, url, scrapeId, doc, previous, formats } =+ params;++ const wantsJson = formatsRequestJsonExtraction(formats);+ const wantsGitDiff = formatsRequestGitDiff(formats);++ if (!previous?.last_scrape_id || previous.is_removed) {+ return {+ status: "new",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];++ if (wantsJson) {+ const previousJson = isPlainObject(previousDoc?.json)+ ? (previousDoc!.json as Record<string, unknown>)+ : undefined;+ const currentJson = isPlainObject(doc?.json)+ ? (doc.json as Record<string, unknown>)+ : undefined;++ // If the current scrape didn't produce a JSON document we can't+ // compute a JSON diff; treat as `changed` to be safe (matches the+ // markdown-missing branch's behavior).+ if (!currentJson) {+ return {+ status: "changed",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const result = diffMonitorJson(previousJson, currentJson);++ // Mixed-mode: also compute the markdown diff so a monitor that asked+ // for both ["json","git-diff"] modes reports `changed` whenever either+ // surface changed, with both diffs persisted side by side.+ let markdownSidecar: { text: string; json: unknown } | undefined;+ if (wantsGitDiff) {+ const previousMarkdown = previousDoc?.markdown;+ const currentMarkdown = doc?.markdown;+ if (previousMarkdown && currentMarkdown) {+ const md = diffMonitorMarkdown(previousMarkdown, currentMarkdown);+ if (md.status === "changed") {+ markdownSidecar = { text: md.text, json: md.json };+ }+ }+ }++ if (result.status === "same" && !markdownSidecar) {+ return {+ status: "same",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const diffGcsKey = monitorDiffGcsKey({+ teamId,+ monitorId,+ checkId,+ pageId: uuidv7(),+ });+ const artifact: MonitorDiffArtifact = {+ kind: "json",+ url,+ previousScrapeId: previous.last_scrape_id,+ currentScrapeId: scrapeId,+ generatedAt: new Date().toISOString(),+ // When only markdown changed, the per-field JSON diff is empty.+ json: result.status === "changed" ? result.json : {},+ snapshot: currentJson,+ ...(markdownSidecar ? { markdown: markdownSidecar } : {}),+ };+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, artifact);+ return {+ status: "changed",+ diffGcsKey,+ diffTextBytes: sizes.textBytes,+ diffJsonBytes: sizes.jsonBytes,+ };+ }++ // Markdown path (existing behavior).+ const previousMarkdown = previousDoc?.markdown;+ const currentMarkdown = doc?.markdown;++ // Only treat genuinely missing markdown as the fallback "changed" case.+ // An empty string is a valid scrape result (e.g. a page that renders no+ // textual content) and should still flow through diffMonitorMarkdown so+ // we can correctly report "same" when both runs returned "".+ if (previousMarkdown == null || currentMarkdown == null) {+ return {+ status: "changed",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);+ if (diff.status === "same") {+ return {+ status: "same",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const diffGcsKey = monitorDiffGcsKey({+ teamId,+ monitorId,+ checkId,+ pageId: uuidv7(),+ });+ const artifact: MonitorDiffArtifact = {+ kind: "markdown",+ url,+ previousScrapeId: previous.last_scrape_id,+ currentScrapeId: scrapeId,+ generatedAt: new Date().toISOString(),+ text: diff.text,+ json: diff.json,+ };+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, artifact);+ return {+ status: "changed",+ diffGcsKey,+ diffTextBytes: sizes.textBytes,+ diffJsonBytes: sizes.jsonBytes,+ };+}diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.tsindex 3197367d3..b4a719eab 100644--- a/apps/api/src/services/monitoring/diff.ts+++ b/apps/api/src/services/monitoring/diff.ts@@ -1,13 +1,15 @@import gitDiff from "git-diff";import parseDiff from "parse-diff";-type MonitoringDiffResult =+type MonitorMarkdownDiffResult =| {+ kind: "markdown";status: "same";text?: undefined;json?: undefined;}| {+ kind: "markdown";status: "changed";text: string;json: {@@ -31,6 +33,16 @@ type MonitoringDiffResult =};};+type MonitorJsonDiffResult =+ | { kind: "json"; status: "same"; json?: undefined }+ | {+ kind: "json";+ status: "changed";+ json: Record<string, { previous: unknown; current: unknown }>;+ };++type MonitoringDiffResult = MonitorMarkdownDiffResult;+function normalizeMarkdownForChangeTracking(markdown: string): string {return [...markdown.replace(/\s+/g, "").replace(/\[iframe\]\(.+?\)/g, "")].sort()@@ -45,7 +57,7 @@ export function diffMonitorMarkdown(normalizeMarkdownForChangeTracking(previousMarkdown) ===normalizeMarkdownForChangeTracking(currentMarkdown)) {- return { status: "same" };+ return { kind: "markdown", status: "same" };}const text = gitDiff(previousMarkdown, currentMarkdown, {@@ -55,6 +67,7 @@ export function diffMonitorMarkdown(const structured = parseDiff(text);return {+ kind: "markdown",status: "changed",text,json: {@@ -103,3 +116,155 @@ export function diffMonitorMarkdown(},};}++function normalizeJsonValue(value: unknown): unknown {+ if (typeof value === "string") return value.normalize("NFC");+ return value;+}++function jsonValuesEqual(a: unknown, b: unknown): boolean {+ const na = normalizeJsonValue(a);+ const nb = normalizeJsonValue(b);+ if (na === nb) return true;+ if (typeof na !== typeof nb) return false;+ if (na && nb && typeof na === "object") {+ // Deep, key-order-independent equality. Using JSON.stringify here would+ // report a spurious diff whenever the LLM returned the same fields in a+ // different order between runs.+ if (Array.isArray(na) || Array.isArray(nb)) {+ if (!Array.isArray(na) || !Array.isArray(nb)) return false;+ if (na.length !== nb.length) return false;+ for (let i = 0; i < na.length; i++) {+ if (!jsonValuesEqual(na[i], nb[i])) return false;+ }+ return true;+ }+ const aKeys = Object.keys(na as Record<string, unknown>);+ const bKeys = Object.keys(nb as Record<string, unknown>);+ if (aKeys.length !== bKeys.length) return false;+ for (const k of aKeys) {+ if (!Object.prototype.hasOwnProperty.call(nb, k)) return false;+ if (+ !jsonValuesEqual(+ (na as Record<string, unknown>)[k],+ (nb as Record<string, unknown>)[k],+ )+ ) {+ return false;+ }+ }+ return true;+ }+ return false;+}++/**+ * Diff two JSON snapshots (current vs previous scrape `doc.json`) for a+ * monitor. Returns `same` if every field is unchanged after NFC+ * normalization, otherwise `changed` with a per-field {previous, current}+ * map containing only the fields that differ.+ */+export function diffMonitorJson(+ previous: Record<string, unknown> | undefined,+ current: Record<string, unknown> | undefined,+): MonitorJsonDiffResult {+ const prev = previous ?? {};+ const curr = current ?? {};+ const keys = new Set([...Object.keys(prev), ...Object.keys(curr)]);+ const diff: Record<string, { previous: unknown; current: unknown }> = {};++ for (const key of keys) {+ if (!jsonValuesEqual(prev[key], curr[key])) {+ diff[key] = { previous: prev[key], current: curr[key] };+ }+ }++ if (Object.keys(diff).length === 0) {+ return { kind: "json", status: "same" };+ }+ return { kind: "json", status: "changed", json: diff };+}++/**+ * True iff the monitor's scrapeOptions.formats requests a git-diff style+ * markdown diff via a `{type:"changeTracking", modes:[...,"git-diff",...]}`+ * format. Plain `"markdown"` does not count — that's the default scrape+ * output and the markdown-diff path runs whenever JSON extraction wasn't+ * requested.+ */+export function formatsRequestGitDiff(formats: unknown): boolean {+ if (!Array.isArray(formats)) return false;+ for (const entry of formats) {+ if (entry && typeof entry === "object") {+ const obj = entry as { type?: unknown; modes?: unknown };+ if (obj.type === "changeTracking") {+ const modes = Array.isArray(obj.modes) ? obj.modes : [];+ if (modes.includes("git-diff")) return true;+ }+ }+ }+ return false;+}++/**+ * True iff the monitor's scrapeOptions.formats requests JSON extraction —+ * either as a plain `{type:"json"}` format or as a change-tracking format+ * with `"json"` in its modes.+ */+export function formatsRequestJsonExtraction(formats: unknown): boolean {+ if (!Array.isArray(formats)) return false;+ for (const entry of formats) {+ if (typeof entry === "string" && entry === "json") return true;+ if (entry && typeof entry === "object") {+ const obj = entry as { type?: unknown; modes?: unknown };+ if (obj.type === "json") return true;+ if (obj.type === "changeTracking") {+ const modes = Array.isArray(obj.modes) ? obj.modes : [];+ if (modes.includes("json")) return true;+ }+ }+ }+ return false;+}++/**+ * Collapse monitor-level format shorthand into what the scrape engine+ * understands. Today the runner owns history; we always want the scrape to+ * return current values via the plain `{type:"json"}` format. A+ * `{type:"changeTracking", modes:["json"], schema, prompt}` entry becomes+ * `{type:"json", schema, prompt}`; if the user asked for `git-diff` mode in+ * addition, that mode is dropped (markdown is already requested elsewhere).+ */+export function normalizeMonitorFormats(formats: unknown): unknown[] {+ if (!Array.isArray(formats)) return [];+ const out: unknown[] = [];+ for (const entry of formats) {+ if (+ entry &&+ typeof entry === "object" &&+ (entry as { type?: unknown }).type === "changeTracking"+ ) {+ const obj = entry as {+ type: "changeTracking";+ modes?: unknown;+ schema?: unknown;+ prompt?: unknown;+ tag?: unknown;+ };+ const modes = Array.isArray(obj.modes) ? obj.modes : [];+ if (modes.includes("json")) {+ out.push({+ type: "json",+ ...(typeof obj.prompt === "string" ? { prompt: obj.prompt } : {}),+ ...(obj.schema && typeof obj.schema === "object"+ ? { schema: obj.schema }+ : {}),+ });+ }+ // git-diff mode is implicit via the markdown format already added.+ continue;+ }+ out.push(entry);+ }+ return out;+}diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsindex 2c06706f1..fbd82dc0f 100644--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@ -1,14 +1,8 @@-import { v7 as uuidv7 } from "uuid";import { NuQJob } from "../worker/nuq";import { ScrapeJobData } from "../../types";-import { getJobFromGCS } from "../../lib/gcs-jobs";-import {- monitorDiffGcsKey,- saveMonitorDiffArtifact,-} from "../../lib/gcs-monitoring";import { logger as _logger } from "../../lib/logger";import { createWebhookSender, WebhookEvent } from "../webhook";-import { diffMonitorMarkdown } from "./diff";+import { computeAndPersistPageDiff } from "./diff-orchestrator";import {getMonitorForUpdate,getMonitorPage,@@ -88,42 +82,32 @@ export async function recordMonitorScrapeSuccess(url,});- let status: "same" | "new" | "changed" = "new";- let diffGcsKey: string | null = null;- let diffTextBytes: number | null = null;- let diffJsonBytes: number | null = null;-- if (previous?.last_scrape_id && !previous.is_removed) {- const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];- const previousMarkdown = previousDoc?.markdown;- const currentMarkdown = doc?.markdown;-- if (previousMarkdown && currentMarkdown) {- const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);- status = diff.status;-- if (diff.status === "changed") {- diffGcsKey = monitorDiffGcsKey({- teamId: job.data.team_id,- monitorId: monitoring.monitorId,- checkId: monitoring.checkId,- pageId: uuidv7(),- });- const sizes = await saveMonitorDiffArtifact(diffGcsKey, {- url,- previousScrapeId: previous.last_scrape_id,- currentScrapeId: job.id,- text: diff.text,- json: diff.json,- generatedAt: new Date().toISOString(),- });- diffTextBytes = sizes.textBytes;- diffJsonBytes = sizes.jsonBytes;- }- } else {- status = "changed";- }- }+ // The monitor row's target carries the canonical formats; fetch it to+ // decide whether this run is a JSON-extraction monitor.+ const monitorForRun = await getMonitorForUpdate(+ job.data.team_id,+ monitoring.monitorId,+ );+ const targetFormats = monitorForRun?.targets?.find(+ (t: any) => t.id === monitoring.targetId,+ )?.scrapeOptions?.formats;++ const { status, diffGcsKey, diffTextBytes, diffJsonBytes } =+ await computeAndPersistPageDiff({+ teamId: job.data.team_id,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ url,+ scrapeId: job.id,+ doc,+ previous: previous+ ? {+ last_scrape_id: previous.last_scrape_id,+ is_removed: previous.is_removed,+ }+ : null,+ formats: targetFormats,+ });await upsertMonitorPage({monitorId: monitoring.monitorId,diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex ed110dd18..45ed76835 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -6,11 +6,8 @@ import { processJobInternal } from "../worker/scrape-worker";import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";import { ScrapeJobData } from "../../types";import { getJobFromGCS } from "../../lib/gcs-jobs";-import {- monitorDiffGcsKey,- saveMonitorDiffArtifact,-} from "../../lib/gcs-monitoring";-import { diffMonitorMarkdown } from "./diff";+import { computeAndPersistPageDiff } from "./diff-orchestrator";+import { normalizeMonitorFormats } from "./diff";import { autumnService } from "../autumn/autumn.service";import { getBillingQueue } from "../queue-service";import {@@ -115,9 +112,15 @@ function recoverScrapeTargetRunsFromMonitor(function withMonitorScrapeDefaults(options: Record<string, unknown>,): ScrapeOptions {+ // Monitor owns history: rewrite changeTracking-json to plain json before+ // handing the formats array to the scrape engine, so we always get back+ // current values to diff against the previous run.+ const formats = Array.isArray(options.formats)+ ? normalizeMonitorFormats(options.formats)+ : options.formats;return {maxAge: 0,- ...withMarkdownFormat(options),+ ...withMarkdownFormat({ ...options, formats }),};}@@ -217,42 +220,22 @@ async function diffAndPersistPage(params: {url: params.url,});- let status: PageResult["status"] = "new";- let diffGcsKey: string | null = null;- let diffTextBytes: number | null = null;- let diffJsonBytes: number | null = null;-- if (previous?.last_scrape_id && !previous.is_removed) {- const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];- const previousMarkdown = previousDoc?.markdown;- const currentMarkdown = params.doc?.markdown;-- if (previousMarkdown && currentMarkdown) {- const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);- status = diff.status;-- if (diff.status === "changed") {- diffGcsKey = monitorDiffGcsKey({- teamId: params.monitor.team_id,- monitorId: params.monitor.id,- checkId: params.check.id,- pageId: uuidv7(),- });- const sizes = await saveMonitorDiffArtifact(diffGcsKey, {- url: params.url,- previousScrapeId: previous.last_scrape_id,- currentScrapeId: params.scrapeId,- text: diff.text,- json: diff.json,- generatedAt: new Date().toISOString(),- });- diffTextBytes = sizes.textBytes;- diffJsonBytes = sizes.jsonBytes;- }- } else {- status = "changed";- }- }+ const { status, diffGcsKey, diffTextBytes, diffJsonBytes } =+ await computeAndPersistPageDiff({+ teamId: params.monitor.team_id,+ monitorId: params.monitor.id,+ checkId: params.check.id,+ url: params.url,+ scrapeId: params.scrapeId,+ doc: params.doc,+ previous: previous+ ? {+ last_scrape_id: previous.last_scrape_id,+ is_removed: previous.is_removed,+ }+ : null,+ formats: params.target.scrapeOptions?.formats,+ });await upsertMonitorPage({monitorId: params.monitor.id,diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex da297be8f..d9279e2a1 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.4.0</Version>+ <Version>1.5.0</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs b/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.csindex ed5c7ef83..bb71069f5 100644--- a/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs@@ -179,6 +179,61 @@ public class MonitorCheckpublic string? UpdatedAt { get; set; }}+/// <summary>+/// Per-field diff entry returned for monitors that requested JSON+/// extraction. The keys on <see cref="MonitorPageDiff.Json"/> (when used+/// in JSON or mixed mode) are field paths in the extracted JSON; the+/// values describe what changed between the previous and current run.+/// </summary>+public class MonitorJsonFieldDiff+{+ [JsonPropertyName("previous")]+ public object? Previous { get; set; }++ [JsonPropertyName("current")]+ public object? Current { get; set; }+}++/// <summary>+/// Diff payload returned alongside a monitor page when its scrape+/// produced a change. The shape depends on what the monitor's formats+/// asked for:+/// <list type="bullet">+/// <item>Markdown-only monitors: <see cref="Text"/> holds the unified+/// diff and <see cref="Json"/> holds the parseDiff AST+/// (<c>{ "files": [...] }</c>).</item>+/// <item>JSON-extraction monitors: <see cref="Json"/> holds the+/// per-field <see cref="MonitorJsonFieldDiff"/> map and+/// <see cref="Text"/> is null.</item>+/// <item>Mixed (JSON + git-diff) monitors: both fields are populated:+/// <see cref="Json"/> is the per-field diff and <see cref="Text"/>+/// is the markdown sidecar.</item>+/// </list>+/// <see cref="Json"/> is exposed as <see cref="object"/> because its+/// concrete shape depends on the monitor mode; callers should+/// re-deserialize with <c>System.Text.Json</c> into either a+/// <c>Dictionary<string, MonitorJsonFieldDiff></c> (JSON / mixed+/// mode) or a wrapper containing the <c>files</c> array (markdown mode).+/// </summary>+public class MonitorPageDiff+{+ [JsonPropertyName("text")]+ public string? Text { get; set; }++ [JsonPropertyName("json")]+ public object? Json { get; set; }+}++/// <summary>+/// Snapshot of the current JSON extraction at this run. Present on JSON+/// and mixed-mode monitors; absent for markdown-only monitors.+/// </summary>+public class MonitorPageSnapshot+{+ [JsonPropertyName("json")]+ public Dictionary<string, object>? Json { get; set; }+}+public class MonitorCheckPage{[JsonPropertyName("id")]@@ -209,7 +264,10 @@ public class MonitorCheckPagepublic object? Metadata { get; set; }[JsonPropertyName("diff")]- public object? Diff { get; set; }+ public MonitorPageDiff? Diff { get; set; }++ [JsonPropertyName("snapshot")]+ public MonitorPageSnapshot? Snapshot { get; set; }[JsonPropertyName("createdAt")]public string? CreatedAt { get; set; }diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.goindex 33b1df63e..5d435c4bd 100644--- a/apps/go-sdk/models.go+++ b/apps/go-sdk/models.go@@ -180,19 +180,54 @@ type MonitorCheck struct {UpdatedAt string `json:"updatedAt,omitempty"`}+// MonitorJsonFieldDiff is a single field-level diff returned for monitors+// that requested JSON extraction. Keys are field paths in the extracted+// JSON; values describe what changed between the previous and current run.+type MonitorJsonFieldDiff struct {+ Previous interface{} `json:"previous"`+ Current interface{} `json:"current"`+}++// MonitorPageDiff is the diff payload returned alongside a monitor page+// when its scrape produced a change. The shape depends on what the+// monitor's formats asked for:+//+// - markdown-only monitors → Text holds the unified diff and JSON+// holds the parseDiff AST (a {"files": [...]} object).+// - JSON-extraction monitors → JSON holds the per-field+// map[string]MonitorJsonFieldDiff and Text is empty.+// - mixed (JSON + git-diff) monitors → both fields are populated:+// JSON is the per-field diff and Text is the markdown sidecar.+//+// JSON is left as interface{} so callers can decode into either of the+// two possible shapes; use json.Unmarshal with a concrete target when+// the monitor's mode is known.+type MonitorPageDiff struct {+ Text string `json:"text,omitempty"`+ JSON interface{} `json:"json,omitempty"`+}++// MonitorPageSnapshot is the snapshot of the current JSON extraction at+// this run. It is present on JSON and mixed-mode monitors and absent+// for markdown-only monitors.+type MonitorPageSnapshot struct {+ JSON map[string]interface{} `json:"json,omitempty"`+}+// MonitorCheckPage is a single page result in a monitor check.type MonitorCheckPage struct {- ID string `json:"id"`- TargetID string `json:"targetId"`- URL string `json:"url"`- Status string `json:"status"`- PreviousScrapeID string `json:"previousScrapeId,omitempty"`- CurrentScrapeID string `json:"currentScrapeId,omitempty"`- StatusCode *int `json:"statusCode,omitempty"`- Error string `json:"error,omitempty"`- Metadata interface{} `json:"metadata,omitempty"`- Diff interface{} `json:"diff,omitempty"`- CreatedAt string `json:"createdAt,omitempty"`+ ID string `json:"id"`+ TargetID string `json:"targetId"`+ URL string `json:"url"`+ Status string `json:"status"`+ PreviousScrapeID string `json:"previousScrapeId,omitempty"`+ CurrentScrapeID string `json:"currentScrapeId,omitempty"`+ StatusCode *int `json:"statusCode,omitempty"`+ Error string `json:"error,omitempty"`+ Metadata interface{} `json:"metadata,omitempty"`+ Diff *MonitorPageDiff `json:"diff,omitempty"`+ Snapshot *MonitorPageSnapshot `json:"snapshot,omitempty"`+ CreatedAt string `json:"createdAt,omitempty"`}// MonitorCheckDetail includes paginated page results and inline diffs.diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex a6ab7a966..82c6fbcf5 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.3.0"+const Version = "1.4.0"diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex b0a7ffab0..c179499e1 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.6.0"+version = "1.7.0"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.javaindex 231d3788e..ed547021f 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java@@ -13,7 +13,8 @@ public class MonitorCheckPage {private Integer statusCode;private String error;private Object metadata;- private Object diff;+ private MonitorPageDiff diff;+ private MonitorPageSnapshot snapshot;private String createdAt;public String getId() { return id; }@@ -25,6 +26,7 @@ public class MonitorCheckPage {public Integer getStatusCode() { return statusCode; }public String getError() { return error; }public Object getMetadata() { return metadata; }- public Object getDiff() { return diff; }+ public MonitorPageDiff getDiff() { return diff; }+ public MonitorPageSnapshot getSnapshot() { return snapshot; }public String getCreatedAt() { return createdAt; }}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorJsonFieldDiff.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorJsonFieldDiff.javanew file mode 100644index 000000000..fa5c001f9--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorJsonFieldDiff.java@@ -0,0 +1,19 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++/**+ * Per-field diff entry returned for monitors that requested JSON+ * extraction. The keys on {@link MonitorPageDiff#getJson()} (when used in+ * JSON or mixed mode) are field paths in the extracted JSON; the values+ * are instances of this class describing what changed between the+ * previous and current run.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class MonitorJsonFieldDiff {+ private Object previous;+ private Object current;++ public Object getPrevious() { return previous; }+ public Object getCurrent() { return current; }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageDiff.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageDiff.javanew file mode 100644index 000000000..cf4ef3a70--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageDiff.java@@ -0,0 +1,35 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++/**+ * Diff payload returned alongside a monitor page when its scrape+ * produced a change. The shape depends on what the monitor's formats+ * asked for:+ *+ * <ul>+ * <li>Markdown-only monitors: {@code text} holds the unified diff+ * and {@code json} holds the {@code parseDiff} AST (a+ * {@code {"files": [...]}} object).</li>+ * <li>JSON-extraction monitors: {@code json} holds the per-field+ * map of {@link MonitorJsonFieldDiff} entries and {@code text} is+ * absent.</li>+ * <li>Mixed (JSON + git-diff) monitors: both fields are populated:+ * {@code json} is the per-field diff map and {@code text} is the+ * markdown sidecar.</li>+ * </ul>+ *+ * <p>{@code json} is exposed as {@link Object} because its concrete+ * shape depends on the monitor mode; callers should cast or re-deserialize+ * with Jackson into either a {@code Map<String, MonitorJsonFieldDiff>}+ * (JSON / mixed mode) or a {@code Map<String, Object>} containing the+ * {@code files} key (markdown mode).+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class MonitorPageDiff {+ private String text;+ private Object json;++ public String getText() { return text; }+ public Object getJson() { return json; }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageSnapshot.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageSnapshot.javanew file mode 100644index 000000000..72081b671--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorPageSnapshot.java@@ -0,0 +1,15 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import java.util.Map;++/**+ * Snapshot of the current JSON extraction at this run. Present on JSON+ * and mixed-mode monitors; absent for markdown-only monitors.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class MonitorPageSnapshot {+ private Map<String, Object> json;++ public Map<String, Object> getJson() { return json; }+}diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 3a6f436c6..dc8c03cf1 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.23.0",+ "version": "4.24.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 2ceae804d..baa69a431 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -1,20 +1,20 @@-import type { ZodTypeAny } from 'zod';+import type { ZodTypeAny } from "zod";// Public types for Firecrawl JS/TS SDK v2 (camelCase only)export type FormatString =- | 'markdown'- | 'html'- | 'rawHtml'- | 'links'- | 'images'- | 'screenshot'- | 'summary'- | 'changeTracking'- | 'json'- | 'attributes'- | 'branding'- | 'audio'- | 'video';+ | "markdown"+ | "html"+ | "rawHtml"+ | "links"+ | "images"+ | "screenshot"+ | "summary"+ | "changeTracking"+ | "json"+ | "attributes"+ | "branding"+ | "audio"+ | "video";export interface Viewport {width: number;@@ -26,27 +26,27 @@ export interface Format {}export interface JsonFormat extends Format {- type: 'json';+ type: "json";prompt?: string;schema?: Record<string, unknown> | ZodTypeAny;}export interface ScreenshotFormat {- type: 'screenshot';+ type: "screenshot";fullPage?: boolean;quality?: number;viewport?: Viewport | { width: number; height: number };}export interface ChangeTrackingFormat extends Format {- type: 'changeTracking';- modes: ('git-diff' | 'json')[];+ type: "changeTracking";+ modes: ("git-diff" | "json")[];schema?: Record<string, unknown>;prompt?: string;tag?: string;}export interface AttributesFormat extends Format {- type: 'attributes';+ type: "attributes";selectors: Array<{selector: string;attribute: string;@@ -54,20 +54,20 @@ export interface AttributesFormat extends Format {}export interface QuestionFormat {- type: 'question';+ type: "question";question: string;}export interface HighlightsFormat {- type: 'highlights';+ type: "highlights";query: string;}/** @deprecated Use QuestionFormat or HighlightsFormat instead. */export interface QueryFormat {- type: 'query';+ type: "query";prompt: string;- mode?: 'freeform' | 'directQuote';+ mode?: "freeform" | "directQuote";}export type FormatOption =@@ -83,7 +83,7 @@ export type FormatOption =export type ParseFormatString = Exclude<FormatString,- 'screenshot' | 'changeTracking' | 'branding' | 'audio' | 'video'+ "screenshot" | "changeTracking" | "branding" | "audio" | "video">;export interface ParseFormat {@@ -105,62 +105,62 @@ export interface LocationConfig {}export interface WaitAction {- type: 'wait';+ type: "wait";milliseconds?: number;selector?: string;}export interface ScreenshotAction {- type: 'screenshot';+ type: "screenshot";fullPage?: boolean;quality?: number;viewport?: Viewport | { width: number; height: number };}export interface ClickAction {- type: 'click';+ type: "click";selector: string;}export interface WriteAction {- type: 'write';+ type: "write";text: string;}export interface PressAction {- type: 'press';+ type: "press";key: string;}export interface ScrollAction {- type: 'scroll';- direction: 'up' | 'down';+ type: "scroll";+ direction: "up" | "down";selector?: string;}export interface ScrapeAction {- type: 'scrape';+ type: "scrape";}export interface ExecuteJavascriptAction {- type: 'executeJavascript';+ type: "executeJavascript";script: string;}export interface PDFAction {- type: 'pdf';+ type: "pdf";format?:- | 'A0'- | 'A1'- | 'A2'- | 'A3'- | 'A4'- | 'A5'- | 'A6'- | 'Letter'- | 'Legal'- | 'Tabloid'- | 'Ledger';+ | "A0"+ | "A1"+ | "A2"+ | "A3"+ | "A4"+ | "A5"+ | "A6"+ | "Letter"+ | "Legal"+ | "Tabloid"+ | "Ledger";landscape?: boolean;scale?: number;}@@ -185,7 +185,9 @@ export interface ScrapeOptions {timeout?: number;waitFor?: number;mobile?: boolean;- parsers?: Array<string | { type: 'pdf'; mode?: 'fast' | 'auto' | 'ocr'; maxPages?: number }>;+ parsers?: Array<+ string | { type: "pdf"; mode?: "fast" | "auto" | "ocr"; maxPages?: number }+ >;actions?: ActionOption[];location?: LocationConfig;skipTlsVerification?: boolean;@@ -193,7 +195,7 @@ export interface ScrapeOptions {fastMode?: boolean;useMock?: string;blockAds?: boolean;- proxy?: 'basic' | 'stealth' | 'enhanced' | 'auto' | string;+ proxy?: "basic" | "stealth" | "enhanced" | "auto" | string;maxAge?: number;minAge?: number;storeInCache?: boolean;@@ -222,30 +224,35 @@ export interface ParseFile {export type ParseOptions = Omit<ScrapeOptions,- | 'formats'- | 'waitFor'- | 'mobile'- | 'actions'- | 'location'- | 'maxAge'- | 'minAge'- | 'storeInCache'- | 'lockdown'- | 'proxy'+ | "formats"+ | "waitFor"+ | "mobile"+ | "actions"+ | "location"+ | "maxAge"+ | "minAge"+ | "storeInCache"+ | "lockdown"+ | "proxy"> & {formats?: ParseFormatOption[];- proxy?: 'basic' | 'auto';+ proxy?: "basic" | "auto";};export interface WebhookConfig {url: string;headers?: Record<string, string>;metadata?: Record<string, string>;- events?: Array<'completed' | 'failed' | 'page' | 'started'>;+ events?: Array<"completed" | "failed" | "page" | "started">;}// Agent webhook events differ from crawl: has 'action' and 'cancelled', no 'page'-export type AgentWebhookEvent = 'started' | 'action' | 'completed' | 'failed' | 'cancelled';+export type AgentWebhookEvent =+ | "started"+ | "action"+ | "completed"+ | "failed"+ | "cancelled";export interface AgentWebhookConfig {url: string;@@ -255,7 +262,7 @@ export interface AgentWebhookConfig {}export interface BrandingProfile {- colorScheme?: 'light' | 'dark';+ colorScheme?: "light" | "dark";logo?: string | null;fonts?: Array<{family: string;@@ -377,13 +384,13 @@ export interface BrandingProfile {};personality?: {tone:- | 'professional'- | 'playful'- | 'modern'- | 'traditional'- | 'minimalist'- | 'bold';- energy: 'low' | 'medium' | 'high';+ | "professional"+ | "playful"+ | "modern"+ | "traditional"+ | "minimalist"+ | "bold";+ energy: "low" | "medium" | "high";targetAudience: string;};[key: string]: unknown;@@ -435,8 +442,8 @@ export interface DocumentMetadata {numPages?: number;contentType?: string;timezone?: string;- proxyUsed?: 'basic' | 'stealth';- cacheState?: 'hit' | 'miss';+ proxyUsed?: "basic" | "stealth";+ cacheState?: "hit" | "miss";cachedAt?: string;creditsUsed?: number;concurrencyLimited?: boolean;@@ -518,15 +525,15 @@ export interface SearchData {}export interface CategoryOption {- type: 'github' | 'research' | 'pdf';+ type: "github" | "research" | "pdf";}export interface SearchRequest {query: string;sources?: Array<- 'web' | 'news' | 'images' | { type: 'web' | 'news' | 'images' }+ "web" | "news" | "images" | { type: "web" | "news" | "images" }>;- categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;+ categories?: Array<"github" | "research" | "pdf" | CategoryOption>;includeDomains?: string[];excludeDomains?: string[];limit?: number;@@ -544,7 +551,7 @@ export interface CrawlOptions {excludePaths?: string[] | null;includePaths?: string[] | null;maxDiscoveryDepth?: number | null;- sitemap?: 'skip' | 'include' | 'only';+ sitemap?: "skip" | "include" | "only";ignoreQueryParameters?: boolean;deduplicateSimilarURLs?: boolean;limit?: number | null;@@ -570,7 +577,7 @@ export interface CrawlResponse {export interface CrawlJob {id: string;- status: 'scraping' | 'completed' | 'failed' | 'cancelled';+ status: "scraping" | "completed" | "failed" | "cancelled";total: number;completed: number;creditsUsed?: number;@@ -599,7 +606,7 @@ export interface BatchScrapeResponse {export interface BatchScrapeJob {id: string;- status: 'scraping' | 'completed' | 'failed' | 'cancelled';+ status: "scraping" | "completed" | "failed" | "cancelled";completed: number;total: number;creditsUsed?: number;@@ -614,7 +621,7 @@ export interface MapData {export interface MapOptions {search?: string;- sitemap?: 'only' | 'include' | 'skip';+ sitemap?: "only" | "include" | "skip";includeSubdomains?: boolean;ignoreQueryParameters?: boolean;limit?: number;@@ -648,14 +655,14 @@ export interface MonitorWebhookConfig {export interface MonitorScrapeTarget {id?: string;- type: 'scrape';+ type: "scrape";urls: string[];scrapeOptions?: ScrapeOptions;}export interface MonitorCrawlTarget {id?: string;- type: 'crawl';+ type: "crawl";url: string;crawlOptions?: CrawlOptions;scrapeOptions?: ScrapeOptions;@@ -674,7 +681,7 @@ export interface CreateMonitorRequest {export interface UpdateMonitorRequest {name?: string;- status?: 'active' | 'paused';+ status?: "active" | "paused";schedule?: MonitorSchedule;webhook?: MonitorWebhookConfig | null;notification?: MonitorNotification | null;@@ -694,7 +701,7 @@ export interface MonitorSummary {export interface Monitor {id: string;name: string;- status: 'active' | 'paused' | 'deleted';+ status: "active" | "paused" | "deleted";schedule: MonitorSchedule;nextRunAt?: string | null;lastRunAt?: string | null;@@ -713,13 +720,13 @@ export interface MonitorCheck {id: string;monitorId: string;status:- | 'queued'- | 'running'- | 'completed'- | 'failed'- | 'partial'- | 'skipped_overlap';- trigger: 'scheduled' | 'manual';+ | "queued"+ | "running"+ | "completed"+ | "failed"+ | "partial"+ | "skipped_overlap";+ trigger: "scheduled" | "manual";scheduledFor?: string | null;startedAt?: string | null;finishedAt?: string | null;@@ -727,11 +734,11 @@ export interface MonitorCheck {reservedCredits?: number | null;actualCredits?: number | null;billingStatus:- | 'not_applicable'- | 'reserved'- | 'confirmed'- | 'released'- | 'failed';+ | "not_applicable"+ | "reserved"+ | "confirmed"+ | "released"+ | "failed";summary: MonitorSummary;targetResults?: unknown;notificationStatus?: unknown;@@ -740,17 +747,48 @@ export interface MonitorCheck {updatedAt: string;}+/** Per-field diff for monitors that requested JSON extraction. */+export interface MonitorJsonFieldDiff {+ [field: string]: { previous: unknown; current: unknown };+}++/**+ * Diff payload returned alongside a monitor page when its scrape produced+ * a change. The shape depends on what the monitor's formats asked for:+ *+ * - markdown-only monitors → `{ text, json }` where `json` is the+ * `parseDiff` AST (a `{ files: [...] }` object).+ * - JSON-extraction monitors → `{ json }` where `json` is the per-field+ * `{ previous, current }` map.+ * - Mixed (JSON + git-diff) monitors → both `text` (markdown sidecar)+ * and `json` (field-level diff) are present.+ */+export interface MonitorPageDiff {+ text?: string;+ /** Markdown variants: parseDiff AST. JSON variants: per-field diff. */+ json?: MonitorJsonFieldDiff | { files: unknown[] };+}++/**+ * Snapshot of the current JSON extraction at this run. Present on JSON+ * and mixed-mode monitors; absent for markdown-only.+ */+export interface MonitorPageSnapshot {+ json?: Record<string, unknown>;+}+export interface MonitorCheckPage {id: string;targetId: string;url: string;- status: 'same' | 'new' | 'changed' | 'removed' | 'error';+ status: "same" | "new" | "changed" | "removed" | "error";previousScrapeId?: string | null;currentScrapeId?: string | null;statusCode?: number | null;error?: string | null;metadata?: unknown;- diff?: unknown;+ diff?: MonitorPageDiff | null;+ snapshot?: MonitorPageSnapshot | null;createdAt: string;}@@ -775,7 +813,7 @@ export type GetMonitorCheckOptions = PaginationConfig & {export interface ExtractResponse {success?: boolean;id?: string;- status?: 'processing' | 'completed' | 'failed' | 'cancelled';+ status?: "processing" | "completed" | "failed" | "cancelled";data?: unknown;error?: string;warning?: string;@@ -794,16 +832,16 @@ export interface AgentResponse {export interface AgentStatusResponse {success: boolean;- status: 'processing' | 'completed' | 'failed';+ status: "processing" | "completed" | "failed";error?: string;data?: unknown;- model?: 'spark-1-pro' | 'spark-1-mini';+ model?: "spark-1-pro" | "spark-1-mini";expiresAt: string;creditsUsed?: number;}export interface AgentOptions {- model: 'FIRE-1' | 'v3-beta';+ model: "FIRE-1" | "v3-beta";}export interface ConcurrencyCheck {@@ -889,10 +927,10 @@ export class SdkError extends Error {status?: number,code?: string,details?: unknown,- jobId?: string+ jobId?: string,) {super(message);- this.name = 'FirecrawlSdkError';+ this.name = "FirecrawlSdkError";this.status = status;this.code = code;this.details = details;@@ -902,16 +940,20 @@ export class SdkError extends Error {export class JobTimeoutError extends SdkError {timeoutSeconds: number;- constructor(jobId: string, timeoutSeconds: number, jobType: 'batch' | 'crawl' = 'batch') {- const jobTypeLabel = jobType === 'batch' ? 'batch scrape' : 'crawl';+ constructor(+ jobId: string,+ timeoutSeconds: number,+ jobType: "batch" | "crawl" = "batch",+ ) {+ const jobTypeLabel = jobType === "batch" ? "batch scrape" : "crawl";super(`${jobTypeLabel.charAt(0).toUpperCase() + jobTypeLabel.slice(1)} job ${jobId} did not complete within ${timeoutSeconds} seconds`,undefined,- 'JOB_TIMEOUT',+ "JOB_TIMEOUT",undefined,- jobId+ jobId,);- this.name = 'JobTimeoutError';+ this.name = "JobTimeoutError";this.timeoutSeconds = timeoutSeconds;}}diff --git a/apps/php-sdk/src/Models/MonitorCheckDetail.php b/apps/php-sdk/src/Models/MonitorCheckDetail.phpindex f6709c81e..8f75538aa 100644--- a/apps/php-sdk/src/Models/MonitorCheckDetail.php+++ b/apps/php-sdk/src/Models/MonitorCheckDetail.php@@ -7,7 +7,15 @@ namespace Firecrawl\Models;final class MonitorCheckDetail extends MonitorCheck{/**- * @param list<array<string, mixed>> $pages+ * @param list<array<string, mixed>> $pages Each page array contains+ * keys: id, targetId, url, status, previousScrapeId,+ * currentScrapeId, statusCode, error, metadata, diff, snapshot,+ * createdAt. The `diff` value (when present) is an array with+ * `text` (markdown unified diff) and/or `json` (parseDiff AST+ * for markdown monitors, or a per-field+ * `{previous, current}` map for JSON-extraction monitors). The+ * `snapshot` value (JSON / mixed mode only) is an array with a+ * `json` key holding the current JSON extraction at this run.*/public function __construct(?string $id = null,@@ -80,7 +88,12 @@ final class MonitorCheckDetail extends MonitorCheckreturn $check;}- /** @return list<array<string, mixed>> */+ /**+ * @return list<array<string, mixed>> Each page array contains the+ * standard monitor page fields plus the `diff` and `snapshot`+ * payloads when present. See the constructor PHPDoc for the+ * shape of those nested values.+ */public function getPages(): array { return $this->pages; }public function getNext(): ?string { return $this->next; }}diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.phpindex 1ed1826b6..ac8491217 100644--- a/apps/php-sdk/src/Version.php+++ b/apps/php-sdk/src/Version.php@@ -6,5 +6,5 @@ namespace Firecrawl;final class Version{- public const SDK_VERSION = '1.3.0';+ public const SDK_VERSION = '1.4.0';}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex a5e0b906f..804a56270 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.26.1"+__version__ = "4.27.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 0ef08b9e2..f5e2f46ab 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -931,6 +931,28 @@ class MonitorCheck(BaseModel):updated_at: str = Field(alias="updatedAt")+class MonitorPageDiff(BaseModel):+ """Diff payload returned alongside a monitor page.++ Markdown-only monitors populate both `text` (unified diff) and `json`+ (the parseDiff AST). JSON-extraction monitors populate `json` only,+ where `json` is the per-field `{previous, current}` map. Mixed-mode+ monitors (JSON + git-diff) populate both `json` (field diff) and+ `text` (markdown sidecar).+ """+ model_config = {"populate_by_name": True, "extra": "allow"}++ text: Optional[str] = None+ json: Optional[Any] = None # markdown→parseDiff AST | json→field diff+++class MonitorPageSnapshot(BaseModel):+ """Current JSON extraction at this run. JSON / mixed mode only."""+ model_config = {"populate_by_name": True, "extra": "allow"}++ json: Optional[Dict[str, Any]] = None++class MonitorCheckPage(BaseModel):model_config = {"populate_by_name": True, "extra": "allow"}@@ -943,7 +965,8 @@ class MonitorCheckPage(BaseModel):status_code: Optional[int] = Field(default=None, alias="statusCode")error: Optional[str] = Nonemetadata: Optional[Any] = None- diff: Optional[Any] = None+ diff: Optional[MonitorPageDiff] = None+ snapshot: Optional[MonitorPageSnapshot] = Nonecreated_at: str = Field(alias="createdAt")diff --git a/apps/ruby-sdk/lib/firecrawl/models/monitor.rb b/apps/ruby-sdk/lib/firecrawl/models/monitor.rbindex 8948baf23..5245aeeb0 100644--- a/apps/ruby-sdk/lib/firecrawl/models/monitor.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/monitor.rb@@ -55,6 +55,19 @@ module Firecrawlendend+ # A monitor check with paginated page results inlined.+ #+ # Each entry in {#pages} is a {Hash} with the standard monitor page+ # fields (id, targetId, url, status, previousScrapeId,+ # currentScrapeId, statusCode, error, metadata, createdAt) plus:+ #+ # * +"diff"+ – when the page changed. A hash with +"text"+ (markdown+ # unified diff) and/or +"json"+ (parseDiff AST for markdown+ # monitors, or a per-field +{ "previous", "current" }+ map for+ # JSON-extraction monitors).+ # * +"snapshot"+ – present on JSON / mixed-mode monitors. A hash+ # with a +"json"+ key holding the current JSON extraction at this+ # run.class MonitorCheckDetail < MonitorCheckattr_accessor :pages, :next_urldiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex 7bb66e07a..7cffb0bd0 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.5.0"+ VERSION = "1.6.0"enddiff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex 2c3cb7cc3..1eaf8660e 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.5.0"+version = "2.6.0"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 2c48e2edd..da9d860d1 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.5.0"+version = "2.6.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/monitor.rs b/apps/rust-sdk/src/monitor.rsindex 2b7e1b493..1273c4420 100644--- a/apps/rust-sdk/src/monitor.rs+++ b/apps/rust-sdk/src/monitor.rs@@ -91,6 +91,42 @@ pub struct MonitorCheck {pub updated_at: String,}+/// Per-field diff entry returned for monitors that requested JSON extraction.+#[derive(Deserialize, Serialize, Debug, Clone)]+pub struct MonitorJsonFieldDiff {+ pub previous: Value,+ pub current: Value,+}++/// Diff payload returned alongside a monitor page when its scrape produced+/// a change. The shape depends on what the monitor's formats asked for:+///+/// - markdown-only monitors → `text` is the unified diff and `json` is+/// the `parseDiff` AST (a `{ "files": [...] }` object).+/// - JSON-extraction monitors → `json` is the per-field+/// `{ previous, current }` map and `text` is absent.+/// - mixed (JSON + git-diff) monitors → both `text` (markdown sidecar)+/// and `json` (field-level diff) are present.+///+/// `json` is kept as a raw [`serde_json::Value`] so callers can decode it+/// into either shape (`HashMap<String, MonitorJsonFieldDiff>` for the+/// field-diff case, or a `{ files: [...] }` struct for the parseDiff AST).+#[derive(Deserialize, Serialize, Debug, Clone)]+pub struct MonitorPageDiff {+ #[serde(skip_serializing_if = "Option::is_none")]+ pub text: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub json: Option<Value>,+}++/// Snapshot of the current JSON extraction at this run. Present on JSON+/// and mixed-mode monitors; absent for markdown-only.+#[derive(Deserialize, Serialize, Debug, Clone)]+pub struct MonitorPageSnapshot {+ #[serde(skip_serializing_if = "Option::is_none")]+ pub json: Option<Value>,+}+#[derive(Deserialize, Serialize, Debug, Clone)]#[serde(rename_all = "camelCase")]pub struct MonitorCheckPage {@@ -103,7 +139,8 @@ pub struct MonitorCheckPage {pub status_code: Option<u16>,pub error: Option<String>,pub metadata: Option<Value>,- pub diff: Option<Value>,+ pub diff: Option<MonitorPageDiff>,+ pub snapshot: Option<MonitorPageSnapshot>,pub created_at: String,}