Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Add domain scoping to monitor signature and tests

Developers Digest9b428ca2cf1f
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index d92201c56..c65684a82 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1879,7 +1879,7 @@ const pdfCategoryOptions = z.strictObject({
type: z.literal("pdf"),
});
-export const searchDomainSchema = z
+const searchDomainSchema = z
.string()
.trim()
.toLowerCase()
diff --git a/apps/api/src/lib/tracking.ts b/apps/api/src/lib/tracking.ts
index a8d687612..a58466c1e 100644
--- a/apps/api/src/lib/tracking.ts
+++ b/apps/api/src/lib/tracking.ts
@@ -116,9 +116,6 @@ function estimateMonitorTargetCredits(target: MonitorTarget): number {
if (target.type === "scrape") {
return target.urls.length;
}
- if (target.type === "search") {
- return target.maxResults;
- }
return positiveIntegerOrNull(target.crawlOptions?.limit) ?? 10000;
}
@@ -130,20 +127,6 @@ function monitorTargetSignature(target: MonitorTarget): string {
urls: target.urls.map(normalizeTrackingUrl).sort(),
});
}
- if (target.type === "search") {
- return hashSignature({
- type: "search",
- queries: [...target.queries].sort(),
- searchWindow: target.searchWindow,
- alertMode: target.alertMode,
- includeDomains: target.includeDomains
- ? [...target.includeDomains].sort()
- : [],
- excludeDomains: target.excludeDomains
- ? [...target.excludeDomains].sort()
- : [],
- });
- }
return hashSignature({
type: "crawl",
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
index ce35d98ba..bfb913c8d 100644
--- a/apps/api/src/services/monitoring/types.ts
+++ b/apps/api/src/services/monitoring/types.ts
@@ -1,7 +1,6 @@
import { z } from "zod";
import {
crawlerOptions,
- searchDomainSchema,
URL as urlSchema,
type ScrapeOptions,
} from "../../controllers/v2/types";
@@ -37,33 +36,33 @@ const crawlTargetSchema = z.strictObject({
scrapeOptions: scrapeOptionsSchema,
});
-// Search monitor target. Instead of watching fixed URLs (scrape/crawl), this runs a set
-// of web-search queries on the monitor's schedule, judges each result against the goal,
-// dedupes by URL and by real-world event, and alerts on genuinely-new meaningful matches.
-// Search-specific state rides existing jsonb columns (config here; URL dedup memory in
-// monitor_pages.metadata; event index in monitors.last_check_summary jsonb for v1) β€” no
-// new tables/columns required to ship.
+const monitorDomainSchema = z
+ .string()
+ .trim()
+ .toLowerCase()
+ .refine(
+ value =>
+ /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/.test(
+ value,
+ ),
+ "Domain must be a valid hostname without protocol or path",
+ );
+
const searchTargetSchema = z
.strictObject({
id: z.string().uuid().optional(),
type: z.literal("search"),
- // The web-search queries. (v1: caller/LLM-supplied; see search/queries.ts.)
queries: z.array(z.string().min(1).max(256)).min(1).max(10),
- // Unified schedule/window step β†’ mapped to a Firecrawl `tbs` window in search/run.ts.
searchWindow: z
.enum(["5m", "15m", "1h", "6h", "24h", "7d"])
.optional()
.default("24h"),
- // first_match = alert once per real-world event; every_new_result = alert on each new URL.
alertMode: z
.enum(["first_match", "every_new_result"])
.optional()
.default("first_match"),
- // Domain scoping β†’ site:/-site: operators (search/run.ts). Mirrors the v2 search API:
- // reuse searchDomainSchema and keep include/exclude mutually exclusive.
- includeDomains: z.array(searchDomainSchema).max(50).optional(),
- excludeDomains: z.array(searchDomainSchema).max(50).optional(),
- // Max results judged per run (caps Firecrawl scrape + LLM cost).
+ includeDomains: z.array(monitorDomainSchema).max(50).optional(),
+ excludeDomains: z.array(monitorDomainSchema).max(50).optional(),
maxResults: z.number().int().min(1).max(50).optional().default(10),
scrapeOptions: scrapeOptionsSchema,
})