Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

monitor search: dedupe by canonical URL, not snippet text

Developers Digest8b80cacac7a1
diff --git a/apps/api/src/services/monitoring/search/dedupe.ts b/apps/api/src/services/monitoring/search/dedupe.ts
index e6eeed314..ed2e55885 100644
--- a/apps/api/src/services/monitoring/search/dedupe.ts
+++ b/apps/api/src/services/monitoring/search/dedupe.ts
@@ -22,12 +22,12 @@ export function stableSerpFingerprint(src: {
snippet?: string;
description?: string;
}): string {
- const text = [src.title, src.snippet ?? src.description]
- .filter(Boolean)
- .join("\n")
- .trim();
+ // A search result is identified by its canonical URL. Snippets and titles drift
+ // between searches (engines reword/truncate them), so hashing that text would
+ // re-flag the same URL as "changed" every check — causing the same result to be
+ // re-judged and re-billed. Keying on the URL keeps already-seen results free.
return createHash("sha256")
- .update(text || canonicalizeUrl(src.url ?? ""))
+ .update(canonicalizeUrl(src.url ?? ""))
.digest("hex")
.slice(0, 24);
}