Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(monitor): gate search results through injected blocklist predicate

Developers Digest185b2ed5562d
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.ts
index 398a6ddd0..2ed57dff5 100644
--- a/apps/api/src/services/monitoring/search/run.test.ts
+++ b/apps/api/src/services/monitoring/search/run.test.ts
@@ -90,6 +90,7 @@ function run(
knownEvents?: { key: string; label: string }[];
goalVersion?: string;
alertMode?: "first_match" | "every_new_result" | "material_dev";
+ isBlocked?: (url: string) => boolean;
} = {},
) {
return runSearchTarget({
@@ -101,6 +102,7 @@ function run(
},
monitorCheckId: "check-1",
scrapePage: (...a: unknown[]) => scrapePageMock(...a),
+ isBlocked: over.isBlocked,
goalVersion: over.goalVersion ?? "gv1",
knownPages: over.knownPages ?? new Map(),
knownEvents: over.knownEvents ?? [],
@@ -538,4 +540,26 @@ describe("domain scoping", () => {
expect(out.sources.map(s => s.url)).toEqual(["https://news.com/openai"]);
expect(scrapePageMock).toHaveBeenCalledTimes(1);
});
+
+ it("blocklisted URLs are dropped before scrape/judge/billing", async () => {
+ setSearchResults([
+ { url: "https://blocked.com/x", title: "blocked", description: "x" },
+ {
+ url: "https://news.com/openai",
+ title: "OpenAI files S-1",
+ description: "x",
+ },
+ ]);
+ setVerdictsByUrl({ "https://news.com/openai": verdict() });
+ const out = await run({
+ isBlocked: url => url.includes("blocked.com"),
+ });
+ expect(out.resultCount).toBe(1);
+ expect(out.sources.map(s => s.url)).toEqual(["https://news.com/openai"]);
+ expect(out.pageUpserts.map(p => p.url)).toEqual([
+ "https://news.com/openai",
+ ]);
+ expect(scrapePageMock).toHaveBeenCalledTimes(1);
+ expect(out.resultsJudged).toBe(1);
+ });
});
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index f5d7f1e35..acae8183c 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -259,6 +259,9 @@ export async function runSearchTarget(params: {
target: SearchTargetInput;
monitorCheckId: string;
scrapePage: ScrapeSearchPage;
+ // Injected blocklist predicate (keeps this module free of worker/auth deps).
+ // Blocked URLs are dropped before scrape/judge/billing. Defaults to allow-all.
+ isBlocked?: (url: string) => boolean;
goalVersion: string;
knownPages: Map<string, KnownPage>;
knownEvents: KnownEvent[];
@@ -266,6 +269,7 @@ export async function runSearchTarget(params: {
logger: Logger;
}): Promise<SearchTargetRunResult> {
const { target, knownPages, goalVersion, logger } = params;
+ const isBlocked = params.isBlocked ?? (() => false);
const judgeEnabled = params.monitor.judgeEnabled;
const runStart = Date.now();
@@ -285,6 +289,7 @@ export async function runSearchTarget(params: {
let skipped = 0;
let matches = 0;
let searchResultsBilled = 0;
+ let blocked = 0;
let resultsJudged = 0;
let judgeScrapeFailures = 0;
@@ -315,6 +320,10 @@ export async function runSearchTarget(params: {
for (const r of results) {
if (!r.url) continue;
if (isExcludedDomain(r.url, target.excludeDomains)) continue;
+ if (isBlocked(r.url)) {
+ blocked += 1;
+ continue;
+ }
const canonical = canonicalizeUrl(r.url);
const existingIdx = seenThisRun.get(canonical);
if (existingIdx !== undefined) {
@@ -336,6 +345,13 @@ export async function runSearchTarget(params: {
}
resultCount = candidates.length;
+ if (blocked > 0) {
+ logger.info("search monitor: dropped blocklisted results before judging", {
+ blocked,
+ kept: resultCount,
+ });
+ }
+
// An empty retrieval is indistinguishable from "nothing matched"; log to diagnose.
if (resultCount === 0) {
logger.warn(