Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(monitoring): address PR review feedback on diff path
Developers Digestb49b8fab79fc
diff --git a/apps/api/src/lib/gcs-monitoring.ts b/apps/api/src/lib/gcs-monitoring.tsindex 7f7cd6394..a40373316 100644--- a/apps/api/src/lib/gcs-monitoring.ts+++ b/apps/api/src/lib/gcs-monitoring.ts@@ -85,15 +85,26 @@ export async function getMonitorDiffArtifact(const bucket = storage.bucket(config.GCS_BUCKET_NAME);try {const [contents] = await bucket.file(key).download();- const parsed = JSON.parse(- contents.toString(),- ) as Partial<MonitorDiffArtifact>;+ let parsed: unknown;+ try {+ parsed = JSON.parse(contents.toString());+ } catch {+ // Corrupt or truncated artifact — surface as "no diff" instead of+ // letting JSON.parse throw and break the entire check response.+ return null;+ }+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {+ // An unexpected payload shape (e.g. number, array, null) was written+ // here; treat as missing rather than risk reading kind off a non-object.+ return null;+ }+ const asPartial = parsed as Partial<MonitorDiffArtifact>;// Backwards compat: historical artifacts predate the `kind` field and// are always markdown.- if (!parsed.kind) {- return { ...(parsed as any), kind: "markdown" } as MonitorDiffArtifact;+ if (!asPartial.kind) {+ return { ...(asPartial as any), kind: "markdown" } as MonitorDiffArtifact;}- return parsed as MonitorDiffArtifact;+ return asPartial as MonitorDiffArtifact;} catch (error) {const maybeGcsError = error as { code?: number; statusCode?: number };if (maybeGcsError.code === 404 || maybeGcsError.statusCode === 404) {diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.tsindex d568f055e..e9a4fd6e5 100644--- a/apps/api/src/services/monitoring/diff-orchestrator.ts+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts@@ -142,7 +142,11 @@ export async function computeAndPersistPageDiff(params: {const previousMarkdown = previousDoc?.markdown;const currentMarkdown = doc?.markdown;- if (!previousMarkdown || !currentMarkdown) {+ // Only treat genuinely missing markdown as the fallback "changed" case.+ // An empty string is a valid scrape result (e.g. a page that renders no+ // textual content) and should still flow through diffMonitorMarkdown so+ // we can correctly report "same" when both runs returned "".+ if (previousMarkdown == null || currentMarkdown == null) {return {status: "changed",diffGcsKey: null,diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.tsindex 1d535aa55..b4a719eab 100644--- a/apps/api/src/services/monitoring/diff.ts+++ b/apps/api/src/services/monitoring/diff.ts@@ -128,7 +128,32 @@ function jsonValuesEqual(a: unknown, b: unknown): boolean {if (na === nb) return true;if (typeof na !== typeof nb) return false;if (na && nb && typeof na === "object") {- return JSON.stringify(na) === JSON.stringify(nb);+ // Deep, key-order-independent equality. Using JSON.stringify here would+ // report a spurious diff whenever the LLM returned the same fields in a+ // different order between runs.+ if (Array.isArray(na) || Array.isArray(nb)) {+ if (!Array.isArray(na) || !Array.isArray(nb)) return false;+ if (na.length !== nb.length) return false;+ for (let i = 0; i < na.length; i++) {+ if (!jsonValuesEqual(na[i], nb[i])) return false;+ }+ return true;+ }+ const aKeys = Object.keys(na as Record<string, unknown>);+ const bKeys = Object.keys(nb as Record<string, unknown>);+ if (aKeys.length !== bKeys.length) return false;+ for (const k of aKeys) {+ if (!Object.prototype.hasOwnProperty.call(nb, k)) return false;+ if (+ !jsonValuesEqual(+ (na as Record<string, unknown>)[k],+ (nb as Record<string, unknown>)[k],+ )+ ) {+ return false;+ }+ }+ return true;}return false;}