Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(monitoring): JSON-mode diff path with snapshot and markdown sidecar

Developers Digeste57d5f439c72
diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.ts
index 9c677507a..caba1621c 100644
--- a/apps/api/src/__tests__/snips/v2/monitor.test.ts
+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts
@@ -139,4 +139,190 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
},
2 * scrapeTimeout,
);
+
+ it(
+ "runs a JSON-mode monitor and surfaces a snapshot",
+ async () => {
+ const jsonSchema = {
+ type: "object",
+ additionalProperties: false,
+ required: ["title"],
+ properties: {
+ title: {
+ type: "string",
+ description: "The page title, verbatim.",
+ },
+ },
+ };
+
+ const create = await monitorCreateRaw(
+ {
+ name: "json-mode monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl()],
+ scrapeOptions: {
+ formats: [
+ "markdown",
+ {
+ type: "json",
+ prompt: "Extract the page title verbatim.",
+ schema: jsonSchema,
+ },
+ ],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+
+ const monitorId = create.body.data.id;
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ const checkId = run.body.id;
+
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, checkId, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (["completed", "partial", "failed"].includes(check.status)) break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ expect(["completed", "partial"]).toContain(check.status);
+ // First run: status is "new" (no previous scrape to diff against), no
+ // snapshot persisted to GCS. We can't assert snapshot.json here
+ // without a mutating fixture; a second run with a changed page would
+ // be needed. The contract assertion is: when JSON mode is requested,
+ // the monitor doesn't fall through the markdown path and crash.
+ expect(check.pages[0].status).toBe("new");
+
+ await monitorDeleteRaw(monitorId, identity);
+ },
+ 2 * scrapeTimeout,
+ );
+
+ it(
+ "accepts changeTracking-json format on a monitor target",
+ async () => {
+ const jsonSchema = {
+ type: "object",
+ additionalProperties: false,
+ required: ["title"],
+ properties: {
+ title: { type: "string", description: "The page title." },
+ },
+ };
+
+ const create = await monitorCreateRaw(
+ {
+ name: "ct-json monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl()],
+ scrapeOptions: {
+ formats: [
+ "markdown",
+ {
+ type: "changeTracking",
+ modes: ["json"],
+ prompt: "Extract the title.",
+ schema: jsonSchema,
+ },
+ ],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+
+ const monitorId = create.body.data.id;
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ const checkId = run.body.id;
+
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, checkId, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (["completed", "partial", "failed"].includes(check.status)) break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ expect(["completed", "partial"]).toContain(check.status);
+ expect(check.pages[0].status).toBe("new");
+
+ await monitorDeleteRaw(monitorId, identity);
+ },
+ 2 * scrapeTimeout,
+ );
+
+ it(
+ "accepts mixed json + git-diff changeTracking modes",
+ async () => {
+ const jsonSchema = {
+ type: "object",
+ additionalProperties: false,
+ required: ["title"],
+ properties: {
+ title: { type: "string", description: "The page title." },
+ },
+ };
+
+ const create = await monitorCreateRaw(
+ {
+ name: "ct-mixed monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl()],
+ scrapeOptions: {
+ formats: [
+ "markdown",
+ {
+ type: "changeTracking",
+ modes: ["json", "git-diff"],
+ prompt: "Extract the title.",
+ schema: jsonSchema,
+ },
+ ],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+
+ const monitorId = create.body.data.id;
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ const checkId = run.body.id;
+
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, checkId, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (["completed", "partial", "failed"].includes(check.status)) break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ expect(["completed", "partial"]).toContain(check.status);
+ // First run is always "new"; the assertion here is that mixed-mode
+ // configuration is accepted end-to-end without erroring.
+ expect(check.pages[0].status).toBe("new");
+
+ await monitorDeleteRaw(monitorId, identity);
+ },
+ 2 * scrapeTimeout,
+ );
});
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
index 914b43c85..7ea2dbcec 100644
--- a/apps/api/src/controllers/v2/monitor.ts
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -314,19 +314,38 @@ export async function getMonitorCheckController(
]);
const pagesWithDiffs = await Promise.all(
- pages.map(async page => ({
- id: page.id,
- targetId: page.target_id,
- url: page.url,
- status: page.status,
- previousScrapeId: page.previous_scrape_id,
- currentScrapeId: page.current_scrape_id,
- statusCode: page.status_code,
- error: page.error,
- metadata: page.metadata,
- diff: await getMonitorDiffArtifact(page.diff_gcs_key),
- createdAt: page.created_at,
- })),
+ pages.map(async page => {
+ const artifact = await getMonitorDiffArtifact(page.diff_gcs_key);
+ const base = {
+ id: page.id,
+ targetId: page.target_id,
+ url: page.url,
+ status: page.status,
+ previousScrapeId: page.previous_scrape_id,
+ currentScrapeId: page.current_scrape_id,
+ statusCode: page.status_code,
+ error: page.error,
+ metadata: page.metadata,
+ createdAt: page.created_at,
+ };
+ if (!artifact) {
+ return { ...base, diff: null };
+ }
+ if (artifact.kind === "json") {
+ return {
+ ...base,
+ diff: {
+ json: artifact.json,
+ ...(artifact.markdown ? { text: artifact.markdown.text } : {}),
+ },
+ snapshot: { json: artifact.snapshot },
+ };
+ }
+ return {
+ ...base,
+ diff: { text: artifact.text, json: artifact.json },
+ };
+ }),
);
const nextSkip = skip + pagesWithDiffs.length;
const next = (() => {
diff --git a/apps/api/src/lib/gcs-monitoring.ts b/apps/api/src/lib/gcs-monitoring.ts
index 36b3e9e12..7f7cd6394 100644
--- a/apps/api/src/lib/gcs-monitoring.ts
+++ b/apps/api/src/lib/gcs-monitoring.ts
@@ -1,15 +1,37 @@
import { config } from "../config";
import { storage } from "./gcs-jobs";
-type MonitorDiffArtifact = {
+type MonitorDiffArtifactBase = {
url: string;
previousScrapeId: string | null;
currentScrapeId: string | null;
- text: string;
- json: unknown;
generatedAt: string;
};
+export type MonitorDiffArtifact =
+ | (MonitorDiffArtifactBase & {
+ kind: "markdown";
+ text: string;
+ json: unknown;
+ })
+ | (MonitorDiffArtifactBase & {
+ kind: "json";
+ /** Per-field {previous, current} diff. */
+ json: Record<string, { previous: unknown; current: unknown }>;
+ /** Full current JSON extraction (the snapshot at this run). */
+ snapshot: Record<string, unknown>;
+ /**
+ * Optional markdown diff sidecar. Populated only when the monitor's
+ * formats requested both `"json"` and `"git-diff"` change-tracking
+ * modes β€” in that case we run both diffs and report `changed` if
+ * either path saw a change.
+ */
+ markdown?: {
+ text: string;
+ json: unknown;
+ };
+ });
+
const contentType = "application/json";
export function monitorDiffGcsKey(params: {
@@ -21,16 +43,29 @@ export function monitorDiffGcsKey(params: {
return `monitors/${params.teamId}/${params.monitorId}/${params.checkId}/${params.pageId}.diff.json`;
}
+function artifactBytes(artifact: MonitorDiffArtifact): {
+ textBytes: number;
+ jsonBytes: number;
+} {
+ const jsonBytes = Buffer.byteLength(JSON.stringify(artifact.json ?? null));
+ let textBytes = 0;
+ if (artifact.kind === "markdown") {
+ textBytes = Buffer.byteLength(artifact.text);
+ } else if (artifact.kind === "json" && artifact.markdown) {
+ // Sidecar markdown diff (mixed-mode monitor) β€” count it so storage
+ // accounting stays honest.
+ textBytes = Buffer.byteLength(artifact.markdown.text);
+ }
+ return { textBytes, jsonBytes };
+}
+
export async function saveMonitorDiffArtifact(
key: string,
artifact: MonitorDiffArtifact,
): Promise<{ textBytes: number; jsonBytes: number }> {
const payload = JSON.stringify(artifact);
if (!config.GCS_BUCKET_NAME) {
- return {
- textBytes: Buffer.byteLength(artifact.text),
- jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),
- };
+ return artifactBytes(artifact);
}
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
@@ -39,10 +74,7 @@ export async function saveMonitorDiffArtifact(
resumable: false,
});
- return {
- textBytes: Buffer.byteLength(artifact.text),
- jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),
- };
+ return artifactBytes(artifact);
}
export async function getMonitorDiffArtifact(
@@ -53,7 +85,15 @@ export async function getMonitorDiffArtifact(
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
try {
const [contents] = await bucket.file(key).download();
- return JSON.parse(contents.toString()) as MonitorDiffArtifact;
+ const parsed = JSON.parse(
+ contents.toString(),
+ ) as Partial<MonitorDiffArtifact>;
+ // Backwards compat: historical artifacts predate the `kind` field and
+ // are always markdown.
+ if (!parsed.kind) {
+ return { ...(parsed as any), kind: "markdown" } as MonitorDiffArtifact;
+ }
+ return parsed as MonitorDiffArtifact;
} catch (error) {
const maybeGcsError = error as { code?: number; statusCode?: number };
if (maybeGcsError.code === 404 || maybeGcsError.statusCode === 404) {
diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.ts
new file mode 100644
index 000000000..d568f055e
--- /dev/null
+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts
@@ -0,0 +1,186 @@
+import { v7 as uuidv7 } from "uuid";
+import { getJobFromGCS } from "../../lib/gcs-jobs";
+import {
+ MonitorDiffArtifact,
+ monitorDiffGcsKey,
+ saveMonitorDiffArtifact,
+} from "../../lib/gcs-monitoring";
+import {
+ diffMonitorJson,
+ diffMonitorMarkdown,
+ formatsRequestGitDiff,
+ formatsRequestJsonExtraction,
+} from "./diff";
+
+type MonitorPageDiffStatus = "same" | "new" | "changed";
+
+type MonitorPageDiffResult = {
+ status: MonitorPageDiffStatus;
+ diffGcsKey: string | null;
+ diffTextBytes: number | null;
+ diffJsonBytes: number | null;
+};
+
+type PreviousPageRef = {
+ last_scrape_id: string | null;
+ is_removed: boolean | null;
+};
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Compute the page diff for a single monitored URL and persist the diff
+ * artifact to GCS when content changed. Branches on whether the monitor
+ * asked for JSON extraction (in which case the diff is over `doc.json`) or
+ * the default markdown text diff.
+ *
+ * Returns the new status + GCS key/sizes for the caller to write into the
+ * monitor pages row.
+ */
+export async function computeAndPersistPageDiff(params: {
+ teamId: string;
+ monitorId: string;
+ checkId: string;
+ url: string;
+ scrapeId: string;
+ doc: any;
+ previous: PreviousPageRef | null;
+ formats: unknown;
+}): Promise<MonitorPageDiffResult> {
+ const { teamId, monitorId, checkId, url, scrapeId, doc, previous, formats } =
+ params;
+
+ const wantsJson = formatsRequestJsonExtraction(formats);
+ const wantsGitDiff = formatsRequestGitDiff(formats);
+
+ if (!previous?.last_scrape_id || previous.is_removed) {
+ return {
+ status: "new",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];
+
+ if (wantsJson) {
+ const previousJson = isPlainObject(previousDoc?.json)
+ ? (previousDoc!.json as Record<string, unknown>)
+ : undefined;
+ const currentJson = isPlainObject(doc?.json)
+ ? (doc.json as Record<string, unknown>)
+ : undefined;
+
+ // If the current scrape didn't produce a JSON document we can't
+ // compute a JSON diff; treat as `changed` to be safe (matches the
+ // markdown-missing branch's behavior).
+ if (!currentJson) {
+ return {
+ status: "changed",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const result = diffMonitorJson(previousJson, currentJson);
+
+ // Mixed-mode: also compute the markdown diff so a monitor that asked
+ // for both ["json","git-diff"] modes reports `changed` whenever either
+ // surface changed, with both diffs persisted side by side.
+ let markdownSidecar: { text: string; json: unknown } | undefined;
+ if (wantsGitDiff) {
+ const previousMarkdown = previousDoc?.markdown;
+ const currentMarkdown = doc?.markdown;
+ if (previousMarkdown && currentMarkdown) {
+ const md = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
+ if (md.status === "changed") {
+ markdownSidecar = { text: md.text, json: md.json };
+ }
+ }
+ }
+
+ if (result.status === "same" && !markdownSidecar) {
+ return {
+ status: "same",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const diffGcsKey = monitorDiffGcsKey({
+ teamId,
+ monitorId,
+ checkId,
+ pageId: uuidv7(),
+ });
+ const artifact: MonitorDiffArtifact = {
+ kind: "json",
+ url,
+ previousScrapeId: previous.last_scrape_id,
+ currentScrapeId: scrapeId,
+ generatedAt: new Date().toISOString(),
+ // When only markdown changed, the per-field JSON diff is empty.
+ json: result.status === "changed" ? result.json : {},
+ snapshot: currentJson,
+ ...(markdownSidecar ? { markdown: markdownSidecar } : {}),
+ };
+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, artifact);
+ return {
+ status: "changed",
+ diffGcsKey,
+ diffTextBytes: sizes.textBytes,
+ diffJsonBytes: sizes.jsonBytes,
+ };
+ }
+
+ // Markdown path (existing behavior).
+ const previousMarkdown = previousDoc?.markdown;
+ const currentMarkdown = doc?.markdown;
+
+ if (!previousMarkdown || !currentMarkdown) {
+ return {
+ status: "changed",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
+ if (diff.status === "same") {
+ return {
+ status: "same",
+ diffGcsKey: null,
+ diffTextBytes: null,
+ diffJsonBytes: null,
+ };
+ }
+
+ const diffGcsKey = monitorDiffGcsKey({
+ teamId,
+ monitorId,
+ checkId,
+ pageId: uuidv7(),
+ });
+ const artifact: MonitorDiffArtifact = {
+ kind: "markdown",
+ url,
+ previousScrapeId: previous.last_scrape_id,
+ currentScrapeId: scrapeId,
+ generatedAt: new Date().toISOString(),
+ text: diff.text,
+ json: diff.json,
+ };
+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, artifact);
+ return {
+ status: "changed",
+ diffGcsKey,
+ diffTextBytes: sizes.textBytes,
+ diffJsonBytes: sizes.jsonBytes,
+ };
+}
diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.ts
index 3197367d3..1d535aa55 100644
--- a/apps/api/src/services/monitoring/diff.ts
+++ b/apps/api/src/services/monitoring/diff.ts
@@ -1,13 +1,15 @@
import gitDiff from "git-diff";
import parseDiff from "parse-diff";
-type MonitoringDiffResult =
+type MonitorMarkdownDiffResult =
| {
+ kind: "markdown";
status: "same";
text?: undefined;
json?: undefined;
}
| {
+ kind: "markdown";
status: "changed";
text: string;
json: {
@@ -31,6 +33,16 @@ type MonitoringDiffResult =
};
};
+type MonitorJsonDiffResult =
+ | { kind: "json"; status: "same"; json?: undefined }
+ | {
+ kind: "json";
+ status: "changed";
+ json: Record<string, { previous: unknown; current: unknown }>;
+ };
+
+type MonitoringDiffResult = MonitorMarkdownDiffResult;
+
function normalizeMarkdownForChangeTracking(markdown: string): string {
return [...markdown.replace(/\s+/g, "").replace(/\[iframe\]\(.+?\)/g, "")]
.sort()
@@ -45,7 +57,7 @@ export function diffMonitorMarkdown(
normalizeMarkdownForChangeTracking(previousMarkdown) ===
normalizeMarkdownForChangeTracking(currentMarkdown)
) {
- return { status: "same" };
+ return { kind: "markdown", status: "same" };
}
const text = gitDiff(previousMarkdown, currentMarkdown, {
@@ -55,6 +67,7 @@ export function diffMonitorMarkdown(
const structured = parseDiff(text);
return {
+ kind: "markdown",
status: "changed",
text,
json: {
@@ -103,3 +116,130 @@ export function diffMonitorMarkdown(
},
};
}
+
+function normalizeJsonValue(value: unknown): unknown {
+ if (typeof value === "string") return value.normalize("NFC");
+ return value;
+}
+
+function jsonValuesEqual(a: unknown, b: unknown): boolean {
+ const na = normalizeJsonValue(a);
+ const nb = normalizeJsonValue(b);
+ if (na === nb) return true;
+ if (typeof na !== typeof nb) return false;
+ if (na && nb && typeof na === "object") {
+ return JSON.stringify(na) === JSON.stringify(nb);
+ }
+ return false;
+}
+
+/**
+ * Diff two JSON snapshots (current vs previous scrape `doc.json`) for a
+ * monitor. Returns `same` if every field is unchanged after NFC
+ * normalization, otherwise `changed` with a per-field {previous, current}
+ * map containing only the fields that differ.
+ */
+export function diffMonitorJson(
+ previous: Record<string, unknown> | undefined,
+ current: Record<string, unknown> | undefined,
+): MonitorJsonDiffResult {
+ const prev = previous ?? {};
+ const curr = current ?? {};
+ const keys = new Set([...Object.keys(prev), ...Object.keys(curr)]);
+ const diff: Record<string, { previous: unknown; current: unknown }> = {};
+
+ for (const key of keys) {
+ if (!jsonValuesEqual(prev[key], curr[key])) {
+ diff[key] = { previous: prev[key], current: curr[key] };
+ }
+ }
+
+ if (Object.keys(diff).length === 0) {
+ return { kind: "json", status: "same" };
+ }
+ return { kind: "json", status: "changed", json: diff };
+}
+
+/**
+ * True iff the monitor's scrapeOptions.formats requests a git-diff style
+ * markdown diff via a `{type:"changeTracking", modes:[...,"git-diff",...]}`
+ * format. Plain `"markdown"` does not count β€” that's the default scrape
+ * output and the markdown-diff path runs whenever JSON extraction wasn't
+ * requested.
+ */
+export function formatsRequestGitDiff(formats: unknown): boolean {
+ if (!Array.isArray(formats)) return false;
+ for (const entry of formats) {
+ if (entry && typeof entry === "object") {
+ const obj = entry as { type?: unknown; modes?: unknown };
+ if (obj.type === "changeTracking") {
+ const modes = Array.isArray(obj.modes) ? obj.modes : [];
+ if (modes.includes("git-diff")) return true;
+ }
+ }
+ }
+ return false;
+}
+
+/**
+ * True iff the monitor's scrapeOptions.formats requests JSON extraction β€”
+ * either as a plain `{type:"json"}` format or as a change-tracking format
+ * with `"json"` in its modes.
+ */
+export function formatsRequestJsonExtraction(formats: unknown): boolean {
+ if (!Array.isArray(formats)) return false;
+ for (const entry of formats) {
+ if (typeof entry === "string" && entry === "json") return true;
+ if (entry && typeof entry === "object") {
+ const obj = entry as { type?: unknown; modes?: unknown };
+ if (obj.type === "json") return true;
+ if (obj.type === "changeTracking") {
+ const modes = Array.isArray(obj.modes) ? obj.modes : [];
+ if (modes.includes("json")) return true;
+ }
+ }
+ }
+ return false;
+}
+
+/**
+ * Collapse monitor-level format shorthand into what the scrape engine
+ * understands. Today the runner owns history; we always want the scrape to
+ * return current values via the plain `{type:"json"}` format. A
+ * `{type:"changeTracking", modes:["json"], schema, prompt}` entry becomes
+ * `{type:"json", schema, prompt}`; if the user asked for `git-diff` mode in
+ * addition, that mode is dropped (markdown is already requested elsewhere).
+ */
+export function normalizeMonitorFormats(formats: unknown): unknown[] {
+ if (!Array.isArray(formats)) return [];
+ const out: unknown[] = [];
+ for (const entry of formats) {
+ if (
+ entry &&
+ typeof entry === "object" &&
+ (entry as { type?: unknown }).type === "changeTracking"
+ ) {
+ const obj = entry as {
+ type: "changeTracking";
+ modes?: unknown;
+ schema?: unknown;
+ prompt?: unknown;
+ tag?: unknown;
+ };
+ const modes = Array.isArray(obj.modes) ? obj.modes : [];
+ if (modes.includes("json")) {
+ out.push({
+ type: "json",
+ ...(typeof obj.prompt === "string" ? { prompt: obj.prompt } : {}),
+ ...(obj.schema && typeof obj.schema === "object"
+ ? { schema: obj.schema }
+ : {}),
+ });
+ }
+ // git-diff mode is implicit via the markdown format already added.
+ continue;
+ }
+ out.push(entry);
+ }
+ return out;
+}
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.ts
index 2c06706f1..fbd82dc0f 100644
--- a/apps/api/src/services/monitoring/results.ts
+++ b/apps/api/src/services/monitoring/results.ts
@@ -1,14 +1,8 @@
-import { v7 as uuidv7 } from "uuid";
import { NuQJob } from "../worker/nuq";
import { ScrapeJobData } from "../../types";
-import { getJobFromGCS } from "../../lib/gcs-jobs";
-import {
- monitorDiffGcsKey,
- saveMonitorDiffArtifact,
-} from "../../lib/gcs-monitoring";
import { logger as _logger } from "../../lib/logger";
import { createWebhookSender, WebhookEvent } from "../webhook";
-import { diffMonitorMarkdown } from "./diff";
+import { computeAndPersistPageDiff } from "./diff-orchestrator";
import {
getMonitorForUpdate,
getMonitorPage,
@@ -88,42 +82,32 @@ export async function recordMonitorScrapeSuccess(
url,
});
- let status: "same" | "new" | "changed" = "new";
- let diffGcsKey: string | null = null;
- let diffTextBytes: number | null = null;
- let diffJsonBytes: number | null = null;
-
- if (previous?.last_scrape_id && !previous.is_removed) {
- const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];
- const previousMarkdown = previousDoc?.markdown;
- const currentMarkdown = doc?.markdown;
-
- if (previousMarkdown && currentMarkdown) {
- const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
- status = diff.status;
-
- if (diff.status === "changed") {
- diffGcsKey = monitorDiffGcsKey({
- teamId: job.data.team_id,
- monitorId: monitoring.monitorId,
- checkId: monitoring.checkId,
- pageId: uuidv7(),
- });
- const sizes = await saveMonitorDiffArtifact(diffGcsKey, {
- url,
- previousScrapeId: previous.last_scrape_id,
- currentScrapeId: job.id,
- text: diff.text,
- json: diff.json,
- generatedAt: new Date().toISOString(),
- });
- diffTextBytes = sizes.textBytes;
- diffJsonBytes = sizes.jsonBytes;
- }
- } else {
- status = "changed";
- }
- }
+ // The monitor row's target carries the canonical formats; fetch it to
+ // decide whether this run is a JSON-extraction monitor.
+ const monitorForRun = await getMonitorForUpdate(
+ job.data.team_id,
+ monitoring.monitorId,
+ );
+ const targetFormats = monitorForRun?.targets?.find(
+ (t: any) => t.id === monitoring.targetId,
+ )?.scrapeOptions?.formats;
+
+ const { status, diffGcsKey, diffTextBytes, diffJsonBytes } =
+ await computeAndPersistPageDiff({
+ teamId: job.data.team_id,
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ url,
+ scrapeId: job.id,
+ doc,
+ previous: previous
+ ? {
+ last_scrape_id: previous.last_scrape_id,
+ is_removed: previous.is_removed,
+ }
+ : null,
+ formats: targetFormats,
+ });
await upsertMonitorPage({
monitorId: monitoring.monitorId,
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 42a74a479..835a9582a 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -6,11 +6,8 @@ import { processJobInternal } from "../worker/scrape-worker";
import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";
import { ScrapeJobData } from "../../types";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-import {
- monitorDiffGcsKey,
- saveMonitorDiffArtifact,
-} from "../../lib/gcs-monitoring";
-import { diffMonitorMarkdown } from "./diff";
+import { computeAndPersistPageDiff } from "./diff-orchestrator";
+import { normalizeMonitorFormats } from "./diff";
import { autumnService } from "../autumn/autumn.service";
import { getBillingQueue } from "../queue-service";
import {
@@ -114,9 +111,15 @@ function recoverScrapeTargetRunsFromMonitor(
function withMonitorScrapeDefaults(
options: Record<string, unknown>,
): ScrapeOptions {
+ // Monitor owns history: rewrite changeTracking-json to plain json before
+ // handing the formats array to the scrape engine, so we always get back
+ // current values to diff against the previous run.
+ const formats = Array.isArray(options.formats)
+ ? normalizeMonitorFormats(options.formats)
+ : options.formats;
return {
maxAge: 0,
- ...withMarkdownFormat(options),
+ ...withMarkdownFormat({ ...options, formats }),
};
}
@@ -216,42 +219,22 @@ async function diffAndPersistPage(params: {
url: params.url,
});
- let status: PageResult["status"] = "new";
- let diffGcsKey: string | null = null;
- let diffTextBytes: number | null = null;
- let diffJsonBytes: number | null = null;
-
- if (previous?.last_scrape_id && !previous.is_removed) {
- const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];
- const previousMarkdown = previousDoc?.markdown;
- const currentMarkdown = params.doc?.markdown;
-
- if (previousMarkdown && currentMarkdown) {
- const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
- status = diff.status;
-
- if (diff.status === "changed") {
- diffGcsKey = monitorDiffGcsKey({
- teamId: params.monitor.team_id,
- monitorId: params.monitor.id,
- checkId: params.check.id,
- pageId: uuidv7(),
- });
- const sizes = await saveMonitorDiffArtifact(diffGcsKey, {
- url: params.url,
- previousScrapeId: previous.last_scrape_id,
- currentScrapeId: params.scrapeId,
- text: diff.text,
- json: diff.json,
- generatedAt: new Date().toISOString(),
- });
- diffTextBytes = sizes.textBytes;
- diffJsonBytes = sizes.jsonBytes;
- }
- } else {
- status = "changed";
- }
- }
+ const { status, diffGcsKey, diffTextBytes, diffJsonBytes } =
+ await computeAndPersistPageDiff({
+ teamId: params.monitor.team_id,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ url: params.url,
+ scrapeId: params.scrapeId,
+ doc: params.doc,
+ previous: previous
+ ? {
+ last_scrape_id: previous.last_scrape_id,
+ is_removed: previous.is_removed,
+ }
+ : null,
+ formats: params.target.scrapeOptions?.formats,
+ });
await upsertMonitorPage({
monitorId: params.monitor.id,