Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(monitor): gate search results through injected blocklist predicate
Developers Digest185b2ed5562d
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 398a6ddd0..2ed57dff5 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -90,6 +90,7 @@ function run(knownEvents?: { key: string; label: string }[];goalVersion?: string;alertMode?: "first_match" | "every_new_result" | "material_dev";+ isBlocked?: (url: string) => boolean;} = {},) {return runSearchTarget({@@ -101,6 +102,7 @@ function run(},monitorCheckId: "check-1",scrapePage: (...a: unknown[]) => scrapePageMock(...a),+ isBlocked: over.isBlocked,goalVersion: over.goalVersion ?? "gv1",knownPages: over.knownPages ?? new Map(),knownEvents: over.knownEvents ?? [],@@ -538,4 +540,26 @@ describe("domain scoping", () => {expect(out.sources.map(s => s.url)).toEqual(["https://news.com/openai"]);expect(scrapePageMock).toHaveBeenCalledTimes(1);});++ it("blocklisted URLs are dropped before scrape/judge/billing", async () => {+ setSearchResults([+ { url: "https://blocked.com/x", title: "blocked", description: "x" },+ {+ url: "https://news.com/openai",+ title: "OpenAI files S-1",+ description: "x",+ },+ ]);+ setVerdictsByUrl({ "https://news.com/openai": verdict() });+ const out = await run({+ isBlocked: url => url.includes("blocked.com"),+ });+ expect(out.resultCount).toBe(1);+ expect(out.sources.map(s => s.url)).toEqual(["https://news.com/openai"]);+ expect(out.pageUpserts.map(p => p.url)).toEqual([+ "https://news.com/openai",+ ]);+ expect(scrapePageMock).toHaveBeenCalledTimes(1);+ expect(out.resultsJudged).toBe(1);+ });});diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex f5d7f1e35..acae8183c 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -259,6 +259,9 @@ export async function runSearchTarget(params: {target: SearchTargetInput;monitorCheckId: string;scrapePage: ScrapeSearchPage;+ // Injected blocklist predicate (keeps this module free of worker/auth deps).+ // Blocked URLs are dropped before scrape/judge/billing. Defaults to allow-all.+ isBlocked?: (url: string) => boolean;goalVersion: string;knownPages: Map<string, KnownPage>;knownEvents: KnownEvent[];@@ -266,6 +269,7 @@ export async function runSearchTarget(params: {logger: Logger;}): Promise<SearchTargetRunResult> {const { target, knownPages, goalVersion, logger } = params;+ const isBlocked = params.isBlocked ?? (() => false);const judgeEnabled = params.monitor.judgeEnabled;const runStart = Date.now();@@ -285,6 +289,7 @@ export async function runSearchTarget(params: {let skipped = 0;let matches = 0;let searchResultsBilled = 0;+ let blocked = 0;let resultsJudged = 0;let judgeScrapeFailures = 0;@@ -315,6 +320,10 @@ export async function runSearchTarget(params: {for (const r of results) {if (!r.url) continue;if (isExcludedDomain(r.url, target.excludeDomains)) continue;+ if (isBlocked(r.url)) {+ blocked += 1;+ continue;+ }const canonical = canonicalizeUrl(r.url);const existingIdx = seenThisRun.get(canonical);if (existingIdx !== undefined) {@@ -336,6 +345,13 @@ export async function runSearchTarget(params: {}resultCount = candidates.length;+ if (blocked > 0) {+ logger.info("search monitor: dropped blocklisted results before judging", {+ blocked,+ kept: resultCount,+ });+ }+// An empty retrieval is indistinguishable from "nothing matched"; log to diagnose.if (resultCount === 0) {logger.warn(