Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Retry search results that were skipped by a transient scrape failure
Developers Digest6dbbe9c946d5
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex e5bea732d..080ca6d77 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -46,8 +46,15 @@ function evaluateSerpCandidate(knownCurrent?.lastCheckedAt &&Date.now() - Date.parse(knownCurrent.lastCheckedAt) > recheckMs,);+ // A prior transient scrape failure (timeout / anti-bot / budget-exceeded) persists+ // the URL as "skipped". Without this, reuse would re-emit "skipped" forever and the+ // result is never re-scraped or re-judged — a real alert silently lost. Retry it.+ const wasSkipped = knownCurrent?.lastStatus === "skipped";const isNewOrChanged =- !knownCurrent || knownCurrent.fingerprint !== fingerprint || dueForRecheck;+ !knownCurrent ||+ knownCurrent.fingerprint !== fingerprint ||+ dueForRecheck ||+ wasSkipped;return { canonical, fingerprint, knownCurrent, isNewOrChanged };}