Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(monitoring): JSON-mode diff path with snapshot and markdown sidecar
Developers Digeste57d5f439c72
diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.tsindex 9c677507a..caba1621c 100644--- a/apps/api/src/__tests__/snips/v2/monitor.test.ts+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts@@ -139,4 +139,190 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {},2 * scrapeTimeout,);++ it(+ "runs a JSON-mode monitor and surfaces a snapshot",+ async () => {+ const jsonSchema = {+ type: "object",+ additionalProperties: false,+ required: ["title"],+ properties: {+ title: {+ type: "string",+ description: "The page title, verbatim.",+ },+ },+ };++ const create = await monitorCreateRaw(+ {+ name: "json-mode monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl()],+ scrapeOptions: {+ formats: [+ "markdown",+ {+ type: "json",+ prompt: "Extract the page title verbatim.",+ schema: jsonSchema,+ },+ ],+ },+ },+ ],+ },+ identity,+ );+ expect(create.statusCode).toBe(200);++ const monitorId = create.body.data.id;+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ const checkId = run.body.id;++ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, checkId, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (["completed", "partial", "failed"].includes(check.status)) break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ expect(["completed", "partial"]).toContain(check.status);+ // First run: status is "new" (no previous scrape to diff against), no+ // snapshot persisted to GCS. We can't assert snapshot.json here+ // without a mutating fixture; a second run with a changed page would+ // be needed. The contract assertion is: when JSON mode is requested,+ // the monitor doesn't fall through the markdown path and crash.+ expect(check.pages[0].status).toBe("new");++ await monitorDeleteRaw(monitorId, identity);+ },+ 2 * scrapeTimeout,+ );++ it(+ "accepts changeTracking-json format on a monitor target",+ async () => {+ const jsonSchema = {+ type: "object",+ additionalProperties: false,+ required: ["title"],+ properties: {+ title: { type: "string", description: "The page title." },+ },+ };++ const create = await monitorCreateRaw(+ {+ name: "ct-json monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl()],+ scrapeOptions: {+ formats: [+ "markdown",+ {+ type: "changeTracking",+ modes: ["json"],+ prompt: "Extract the title.",+ schema: jsonSchema,+ },+ ],+ },+ },+ ],+ },+ identity,+ );+ expect(create.statusCode).toBe(200);++ const monitorId = create.body.data.id;+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ const checkId = run.body.id;++ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, checkId, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (["completed", "partial", "failed"].includes(check.status)) break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ expect(["completed", "partial"]).toContain(check.status);+ expect(check.pages[0].status).toBe("new");++ await monitorDeleteRaw(monitorId, identity);+ },+ 2 * scrapeTimeout,+ );++ it(+ "accepts mixed json + git-diff changeTracking modes",+ async () => {+ const jsonSchema = {+ type: "object",+ additionalProperties: false,+ required: ["title"],+ properties: {+ title: { type: "string", description: "The page title." },+ },+ };++ const create = await monitorCreateRaw(+ {+ name: "ct-mixed monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl()],+ scrapeOptions: {+ formats: [+ "markdown",+ {+ type: "changeTracking",+ modes: ["json", "git-diff"],+ prompt: "Extract the title.",+ schema: jsonSchema,+ },+ ],+ },+ },+ ],+ },+ identity,+ );+ expect(create.statusCode).toBe(200);++ const monitorId = create.body.data.id;+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ const checkId = run.body.id;++ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, checkId, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (["completed", "partial", "failed"].includes(check.status)) break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ expect(["completed", "partial"]).toContain(check.status);+ // First run is always "new"; the assertion here is that mixed-mode+ // configuration is accepted end-to-end without erroring.+ expect(check.pages[0].status).toBe("new");++ await monitorDeleteRaw(monitorId, identity);+ },+ 2 * scrapeTimeout,+ );});diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.tsindex 914b43c85..7ea2dbcec 100644--- a/apps/api/src/controllers/v2/monitor.ts+++ b/apps/api/src/controllers/v2/monitor.ts@@ -314,19 +314,38 @@ export async function getMonitorCheckController(]);const pagesWithDiffs = await Promise.all(- pages.map(async page => ({- id: page.id,- targetId: page.target_id,- url: page.url,- status: page.status,- previousScrapeId: page.previous_scrape_id,- currentScrapeId: page.current_scrape_id,- statusCode: page.status_code,- error: page.error,- metadata: page.metadata,- diff: await getMonitorDiffArtifact(page.diff_gcs_key),- createdAt: page.created_at,- })),+ pages.map(async page => {+ const artifact = await getMonitorDiffArtifact(page.diff_gcs_key);+ const base = {+ id: page.id,+ targetId: page.target_id,+ url: page.url,+ status: page.status,+ previousScrapeId: page.previous_scrape_id,+ currentScrapeId: page.current_scrape_id,+ statusCode: page.status_code,+ error: page.error,+ metadata: page.metadata,+ createdAt: page.created_at,+ };+ if (!artifact) {+ return { ...base, diff: null };+ }+ if (artifact.kind === "json") {+ return {+ ...base,+ diff: {+ json: artifact.json,+ ...(artifact.markdown ? { text: artifact.markdown.text } : {}),+ },+ snapshot: { json: artifact.snapshot },+ };+ }+ return {+ ...base,+ diff: { text: artifact.text, json: artifact.json },+ };+ }),);const nextSkip = skip + pagesWithDiffs.length;const next = (() => {diff --git a/apps/api/src/lib/gcs-monitoring.ts b/apps/api/src/lib/gcs-monitoring.tsindex 36b3e9e12..7f7cd6394 100644--- a/apps/api/src/lib/gcs-monitoring.ts+++ b/apps/api/src/lib/gcs-monitoring.ts@@ -1,15 +1,37 @@import { config } from "../config";import { storage } from "./gcs-jobs";-type MonitorDiffArtifact = {+type MonitorDiffArtifactBase = {url: string;previousScrapeId: string | null;currentScrapeId: string | null;- text: string;- json: unknown;generatedAt: string;};+export type MonitorDiffArtifact =+ | (MonitorDiffArtifactBase & {+ kind: "markdown";+ text: string;+ json: unknown;+ })+ | (MonitorDiffArtifactBase & {+ kind: "json";+ /** Per-field {previous, current} diff. */+ json: Record<string, { previous: unknown; current: unknown }>;+ /** Full current JSON extraction (the snapshot at this run). */+ snapshot: Record<string, unknown>;+ /**+ * Optional markdown diff sidecar. Populated only when the monitor's+ * formats requested both `"json"` and `"git-diff"` change-tracking+ * modes β in that case we run both diffs and report `changed` if+ * either path saw a change.+ */+ markdown?: {+ text: string;+ json: unknown;+ };+ });+const contentType = "application/json";export function monitorDiffGcsKey(params: {@@ -21,16 +43,29 @@ export function monitorDiffGcsKey(params: {return `monitors/${params.teamId}/${params.monitorId}/${params.checkId}/${params.pageId}.diff.json`;}+function artifactBytes(artifact: MonitorDiffArtifact): {+ textBytes: number;+ jsonBytes: number;+} {+ const jsonBytes = Buffer.byteLength(JSON.stringify(artifact.json ?? null));+ let textBytes = 0;+ if (artifact.kind === "markdown") {+ textBytes = Buffer.byteLength(artifact.text);+ } else if (artifact.kind === "json" && artifact.markdown) {+ // Sidecar markdown diff (mixed-mode monitor) β count it so storage+ // accounting stays honest.+ textBytes = Buffer.byteLength(artifact.markdown.text);+ }+ return { textBytes, jsonBytes };+}+export async function saveMonitorDiffArtifact(key: string,artifact: MonitorDiffArtifact,): Promise<{ textBytes: number; jsonBytes: number }> {const payload = JSON.stringify(artifact);if (!config.GCS_BUCKET_NAME) {- return {- textBytes: Buffer.byteLength(artifact.text),- jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),- };+ return artifactBytes(artifact);}const bucket = storage.bucket(config.GCS_BUCKET_NAME);@@ -39,10 +74,7 @@ export async function saveMonitorDiffArtifact(resumable: false,});- return {- textBytes: Buffer.byteLength(artifact.text),- jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),- };+ return artifactBytes(artifact);}export async function getMonitorDiffArtifact(@@ -53,7 +85,15 @@ export async function getMonitorDiffArtifact(const bucket = storage.bucket(config.GCS_BUCKET_NAME);try {const [contents] = await bucket.file(key).download();- return JSON.parse(contents.toString()) as MonitorDiffArtifact;+ const parsed = JSON.parse(+ contents.toString(),+ ) as Partial<MonitorDiffArtifact>;+ // Backwards compat: historical artifacts predate the `kind` field and+ // are always markdown.+ if (!parsed.kind) {+ return { ...(parsed as any), kind: "markdown" } as MonitorDiffArtifact;+ }+ return parsed as MonitorDiffArtifact;} catch (error) {const maybeGcsError = error as { code?: number; statusCode?: number };if (maybeGcsError.code === 404 || maybeGcsError.statusCode === 404) {diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.tsnew file mode 100644index 000000000..d568f055e--- /dev/null+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts@@ -0,0 +1,186 @@+import { v7 as uuidv7 } from "uuid";+import { getJobFromGCS } from "../../lib/gcs-jobs";+import {+ MonitorDiffArtifact,+ monitorDiffGcsKey,+ saveMonitorDiffArtifact,+} from "../../lib/gcs-monitoring";+import {+ diffMonitorJson,+ diffMonitorMarkdown,+ formatsRequestGitDiff,+ formatsRequestJsonExtraction,+} from "./diff";++type MonitorPageDiffStatus = "same" | "new" | "changed";++type MonitorPageDiffResult = {+ status: MonitorPageDiffStatus;+ diffGcsKey: string | null;+ diffTextBytes: number | null;+ diffJsonBytes: number | null;+};++type PreviousPageRef = {+ last_scrape_id: string | null;+ is_removed: boolean | null;+};++function isPlainObject(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++/**+ * Compute the page diff for a single monitored URL and persist the diff+ * artifact to GCS when content changed. Branches on whether the monitor+ * asked for JSON extraction (in which case the diff is over `doc.json`) or+ * the default markdown text diff.+ *+ * Returns the new status + GCS key/sizes for the caller to write into the+ * monitor pages row.+ */+export async function computeAndPersistPageDiff(params: {+ teamId: string;+ monitorId: string;+ checkId: string;+ url: string;+ scrapeId: string;+ doc: any;+ previous: PreviousPageRef | null;+ formats: unknown;+}): Promise<MonitorPageDiffResult> {+ const { teamId, monitorId, checkId, url, scrapeId, doc, previous, formats } =+ params;++ const wantsJson = formatsRequestJsonExtraction(formats);+ const wantsGitDiff = formatsRequestGitDiff(formats);++ if (!previous?.last_scrape_id || previous.is_removed) {+ return {+ status: "new",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];++ if (wantsJson) {+ const previousJson = isPlainObject(previousDoc?.json)+ ? (previousDoc!.json as Record<string, unknown>)+ : undefined;+ const currentJson = isPlainObject(doc?.json)+ ? (doc.json as Record<string, unknown>)+ : undefined;++ // If the current scrape didn't produce a JSON document we can't+ // compute a JSON diff; treat as `changed` to be safe (matches the+ // markdown-missing branch's behavior).+ if (!currentJson) {+ return {+ status: "changed",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const result = diffMonitorJson(previousJson, currentJson);++ // Mixed-mode: also compute the markdown diff so a monitor that asked+ // for both ["json","git-diff"] modes reports `changed` whenever either+ // surface changed, with both diffs persisted side by side.+ let markdownSidecar: { text: string; json: unknown } | undefined;+ if (wantsGitDiff) {+ const previousMarkdown = previousDoc?.markdown;+ const currentMarkdown = doc?.markdown;+ if (previousMarkdown && currentMarkdown) {+ const md = diffMonitorMarkdown(previousMarkdown, currentMarkdown);+ if (md.status === "changed") {+ markdownSidecar = { text: md.text, json: md.json };+ }+ }+ }++ if (result.status === "same" && !markdownSidecar) {+ return {+ status: "same",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const diffGcsKey = monitorDiffGcsKey({+ teamId,+ monitorId,+ checkId,+ pageId: uuidv7(),+ });+ const artifact: MonitorDiffArtifact = {+ kind: "json",+ url,+ previousScrapeId: previous.last_scrape_id,+ currentScrapeId: scrapeId,+ generatedAt: new Date().toISOString(),+ // When only markdown changed, the per-field JSON diff is empty.+ json: result.status === "changed" ? result.json : {},+ snapshot: currentJson,+ ...(markdownSidecar ? { markdown: markdownSidecar } : {}),+ };+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, artifact);+ return {+ status: "changed",+ diffGcsKey,+ diffTextBytes: sizes.textBytes,+ diffJsonBytes: sizes.jsonBytes,+ };+ }++ // Markdown path (existing behavior).+ const previousMarkdown = previousDoc?.markdown;+ const currentMarkdown = doc?.markdown;++ if (!previousMarkdown || !currentMarkdown) {+ return {+ status: "changed",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);+ if (diff.status === "same") {+ return {+ status: "same",+ diffGcsKey: null,+ diffTextBytes: null,+ diffJsonBytes: null,+ };+ }++ const diffGcsKey = monitorDiffGcsKey({+ teamId,+ monitorId,+ checkId,+ pageId: uuidv7(),+ });+ const artifact: MonitorDiffArtifact = {+ kind: "markdown",+ url,+ previousScrapeId: previous.last_scrape_id,+ currentScrapeId: scrapeId,+ generatedAt: new Date().toISOString(),+ text: diff.text,+ json: diff.json,+ };+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, artifact);+ return {+ status: "changed",+ diffGcsKey,+ diffTextBytes: sizes.textBytes,+ diffJsonBytes: sizes.jsonBytes,+ };+}diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.tsindex 3197367d3..1d535aa55 100644--- a/apps/api/src/services/monitoring/diff.ts+++ b/apps/api/src/services/monitoring/diff.ts@@ -1,13 +1,15 @@import gitDiff from "git-diff";import parseDiff from "parse-diff";-type MonitoringDiffResult =+type MonitorMarkdownDiffResult =| {+ kind: "markdown";status: "same";text?: undefined;json?: undefined;}| {+ kind: "markdown";status: "changed";text: string;json: {@@ -31,6 +33,16 @@ type MonitoringDiffResult =};};+type MonitorJsonDiffResult =+ | { kind: "json"; status: "same"; json?: undefined }+ | {+ kind: "json";+ status: "changed";+ json: Record<string, { previous: unknown; current: unknown }>;+ };++type MonitoringDiffResult = MonitorMarkdownDiffResult;+function normalizeMarkdownForChangeTracking(markdown: string): string {return [...markdown.replace(/\s+/g, "").replace(/\[iframe\]\(.+?\)/g, "")].sort()@@ -45,7 +57,7 @@ export function diffMonitorMarkdown(normalizeMarkdownForChangeTracking(previousMarkdown) ===normalizeMarkdownForChangeTracking(currentMarkdown)) {- return { status: "same" };+ return { kind: "markdown", status: "same" };}const text = gitDiff(previousMarkdown, currentMarkdown, {@@ -55,6 +67,7 @@ export function diffMonitorMarkdown(const structured = parseDiff(text);return {+ kind: "markdown",status: "changed",text,json: {@@ -103,3 +116,130 @@ export function diffMonitorMarkdown(},};}++function normalizeJsonValue(value: unknown): unknown {+ if (typeof value === "string") return value.normalize("NFC");+ return value;+}++function jsonValuesEqual(a: unknown, b: unknown): boolean {+ const na = normalizeJsonValue(a);+ const nb = normalizeJsonValue(b);+ if (na === nb) return true;+ if (typeof na !== typeof nb) return false;+ if (na && nb && typeof na === "object") {+ return JSON.stringify(na) === JSON.stringify(nb);+ }+ return false;+}++/**+ * Diff two JSON snapshots (current vs previous scrape `doc.json`) for a+ * monitor. Returns `same` if every field is unchanged after NFC+ * normalization, otherwise `changed` with a per-field {previous, current}+ * map containing only the fields that differ.+ */+export function diffMonitorJson(+ previous: Record<string, unknown> | undefined,+ current: Record<string, unknown> | undefined,+): MonitorJsonDiffResult {+ const prev = previous ?? {};+ const curr = current ?? {};+ const keys = new Set([...Object.keys(prev), ...Object.keys(curr)]);+ const diff: Record<string, { previous: unknown; current: unknown }> = {};++ for (const key of keys) {+ if (!jsonValuesEqual(prev[key], curr[key])) {+ diff[key] = { previous: prev[key], current: curr[key] };+ }+ }++ if (Object.keys(diff).length === 0) {+ return { kind: "json", status: "same" };+ }+ return { kind: "json", status: "changed", json: diff };+}++/**+ * True iff the monitor's scrapeOptions.formats requests a git-diff style+ * markdown diff via a `{type:"changeTracking", modes:[...,"git-diff",...]}`+ * format. Plain `"markdown"` does not count β that's the default scrape+ * output and the markdown-diff path runs whenever JSON extraction wasn't+ * requested.+ */+export function formatsRequestGitDiff(formats: unknown): boolean {+ if (!Array.isArray(formats)) return false;+ for (const entry of formats) {+ if (entry && typeof entry === "object") {+ const obj = entry as { type?: unknown; modes?: unknown };+ if (obj.type === "changeTracking") {+ const modes = Array.isArray(obj.modes) ? obj.modes : [];+ if (modes.includes("git-diff")) return true;+ }+ }+ }+ return false;+}++/**+ * True iff the monitor's scrapeOptions.formats requests JSON extraction β+ * either as a plain `{type:"json"}` format or as a change-tracking format+ * with `"json"` in its modes.+ */+export function formatsRequestJsonExtraction(formats: unknown): boolean {+ if (!Array.isArray(formats)) return false;+ for (const entry of formats) {+ if (typeof entry === "string" && entry === "json") return true;+ if (entry && typeof entry === "object") {+ const obj = entry as { type?: unknown; modes?: unknown };+ if (obj.type === "json") return true;+ if (obj.type === "changeTracking") {+ const modes = Array.isArray(obj.modes) ? obj.modes : [];+ if (modes.includes("json")) return true;+ }+ }+ }+ return false;+}++/**+ * Collapse monitor-level format shorthand into what the scrape engine+ * understands. Today the runner owns history; we always want the scrape to+ * return current values via the plain `{type:"json"}` format. A+ * `{type:"changeTracking", modes:["json"], schema, prompt}` entry becomes+ * `{type:"json", schema, prompt}`; if the user asked for `git-diff` mode in+ * addition, that mode is dropped (markdown is already requested elsewhere).+ */+export function normalizeMonitorFormats(formats: unknown): unknown[] {+ if (!Array.isArray(formats)) return [];+ const out: unknown[] = [];+ for (const entry of formats) {+ if (+ entry &&+ typeof entry === "object" &&+ (entry as { type?: unknown }).type === "changeTracking"+ ) {+ const obj = entry as {+ type: "changeTracking";+ modes?: unknown;+ schema?: unknown;+ prompt?: unknown;+ tag?: unknown;+ };+ const modes = Array.isArray(obj.modes) ? obj.modes : [];+ if (modes.includes("json")) {+ out.push({+ type: "json",+ ...(typeof obj.prompt === "string" ? { prompt: obj.prompt } : {}),+ ...(obj.schema && typeof obj.schema === "object"+ ? { schema: obj.schema }+ : {}),+ });+ }+ // git-diff mode is implicit via the markdown format already added.+ continue;+ }+ out.push(entry);+ }+ return out;+}diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsindex 2c06706f1..fbd82dc0f 100644--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@ -1,14 +1,8 @@-import { v7 as uuidv7 } from "uuid";import { NuQJob } from "../worker/nuq";import { ScrapeJobData } from "../../types";-import { getJobFromGCS } from "../../lib/gcs-jobs";-import {- monitorDiffGcsKey,- saveMonitorDiffArtifact,-} from "../../lib/gcs-monitoring";import { logger as _logger } from "../../lib/logger";import { createWebhookSender, WebhookEvent } from "../webhook";-import { diffMonitorMarkdown } from "./diff";+import { computeAndPersistPageDiff } from "./diff-orchestrator";import {getMonitorForUpdate,getMonitorPage,@@ -88,42 +82,32 @@ export async function recordMonitorScrapeSuccess(url,});- let status: "same" | "new" | "changed" = "new";- let diffGcsKey: string | null = null;- let diffTextBytes: number | null = null;- let diffJsonBytes: number | null = null;-- if (previous?.last_scrape_id && !previous.is_removed) {- const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];- const previousMarkdown = previousDoc?.markdown;- const currentMarkdown = doc?.markdown;-- if (previousMarkdown && currentMarkdown) {- const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);- status = diff.status;-- if (diff.status === "changed") {- diffGcsKey = monitorDiffGcsKey({- teamId: job.data.team_id,- monitorId: monitoring.monitorId,- checkId: monitoring.checkId,- pageId: uuidv7(),- });- const sizes = await saveMonitorDiffArtifact(diffGcsKey, {- url,- previousScrapeId: previous.last_scrape_id,- currentScrapeId: job.id,- text: diff.text,- json: diff.json,- generatedAt: new Date().toISOString(),- });- diffTextBytes = sizes.textBytes;- diffJsonBytes = sizes.jsonBytes;- }- } else {- status = "changed";- }- }+ // The monitor row's target carries the canonical formats; fetch it to+ // decide whether this run is a JSON-extraction monitor.+ const monitorForRun = await getMonitorForUpdate(+ job.data.team_id,+ monitoring.monitorId,+ );+ const targetFormats = monitorForRun?.targets?.find(+ (t: any) => t.id === monitoring.targetId,+ )?.scrapeOptions?.formats;++ const { status, diffGcsKey, diffTextBytes, diffJsonBytes } =+ await computeAndPersistPageDiff({+ teamId: job.data.team_id,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ url,+ scrapeId: job.id,+ doc,+ previous: previous+ ? {+ last_scrape_id: previous.last_scrape_id,+ is_removed: previous.is_removed,+ }+ : null,+ formats: targetFormats,+ });await upsertMonitorPage({monitorId: monitoring.monitorId,diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 42a74a479..835a9582a 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -6,11 +6,8 @@ import { processJobInternal } from "../worker/scrape-worker";import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";import { ScrapeJobData } from "../../types";import { getJobFromGCS } from "../../lib/gcs-jobs";-import {- monitorDiffGcsKey,- saveMonitorDiffArtifact,-} from "../../lib/gcs-monitoring";-import { diffMonitorMarkdown } from "./diff";+import { computeAndPersistPageDiff } from "./diff-orchestrator";+import { normalizeMonitorFormats } from "./diff";import { autumnService } from "../autumn/autumn.service";import { getBillingQueue } from "../queue-service";import {@@ -114,9 +111,15 @@ function recoverScrapeTargetRunsFromMonitor(function withMonitorScrapeDefaults(options: Record<string, unknown>,): ScrapeOptions {+ // Monitor owns history: rewrite changeTracking-json to plain json before+ // handing the formats array to the scrape engine, so we always get back+ // current values to diff against the previous run.+ const formats = Array.isArray(options.formats)+ ? normalizeMonitorFormats(options.formats)+ : options.formats;return {maxAge: 0,- ...withMarkdownFormat(options),+ ...withMarkdownFormat({ ...options, formats }),};}@@ -216,42 +219,22 @@ async function diffAndPersistPage(params: {url: params.url,});- let status: PageResult["status"] = "new";- let diffGcsKey: string | null = null;- let diffTextBytes: number | null = null;- let diffJsonBytes: number | null = null;-- if (previous?.last_scrape_id && !previous.is_removed) {- const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];- const previousMarkdown = previousDoc?.markdown;- const currentMarkdown = params.doc?.markdown;-- if (previousMarkdown && currentMarkdown) {- const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);- status = diff.status;-- if (diff.status === "changed") {- diffGcsKey = monitorDiffGcsKey({- teamId: params.monitor.team_id,- monitorId: params.monitor.id,- checkId: params.check.id,- pageId: uuidv7(),- });- const sizes = await saveMonitorDiffArtifact(diffGcsKey, {- url: params.url,- previousScrapeId: previous.last_scrape_id,- currentScrapeId: params.scrapeId,- text: diff.text,- json: diff.json,- generatedAt: new Date().toISOString(),- });- diffTextBytes = sizes.textBytes;- diffJsonBytes = sizes.jsonBytes;- }- } else {- status = "changed";- }- }+ const { status, diffGcsKey, diffTextBytes, diffJsonBytes } =+ await computeAndPersistPageDiff({+ teamId: params.monitor.team_id,+ monitorId: params.monitor.id,+ checkId: params.check.id,+ url: params.url,+ scrapeId: params.scrapeId,+ doc: params.doc,+ previous: previous+ ? {+ last_scrape_id: previous.last_scrape_id,+ is_removed: previous.is_removed,+ }+ : null,+ formats: params.target.scrapeOptions?.formats,+ });await upsertMonitorPage({monitorId: params.monitor.id,