Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
test: cover deterministicjson monitor pricing (estimate + reconciliation)
Developers Digest3816997d662e
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex c65684a82..d92201c56 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1879,7 +1879,7 @@ const pdfCategoryOptions = z.strictObject({type: z.literal("pdf"),});-const searchDomainSchema = z+export const searchDomainSchema = z.string().trim().toLowerCase()diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 94f031300..f6f8659ae 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -1,6 +1,7 @@import { v7 as uuidv7 } from "uuid";import type { Logger } from "winston";import { search } from "../../../search";+import { buildSearchQuery } from "../../../lib/search-query-builder";import { scrapeURL } from "../../../scraper/scrapeURL";import { scrapeOptions } from "../../../controllers/v2/types";import { CostTracking } from "../../../lib/cost-tracking";@@ -37,6 +38,7 @@ type SearchTargetInput = {searchWindow: string;alertMode: "first_match" | "every_new_result";includeDomains?: string[];+ excludeDomains?: string[];maxResults: number;};@@ -115,8 +117,13 @@ export async function runSearchTarget(params: {const candidates: Array<{ url: string; title: string; description: string }> =[];for (const query of target.queries) {+ // Scope to domains the same way the v2 search API does (site:/-site: operators).+ const { query: scopedQuery } = buildSearchQuery(query, undefined, {+ includeDomains: target.includeDomains,+ excludeDomains: target.excludeDomains,+ });const results = await search({- query,+ query: scopedQuery,logger,num_results: target.maxResults,tbs,diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex 85bf510c5..ce35d98ba 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -1,6 +1,7 @@import { z } from "zod";import {crawlerOptions,+ searchDomainSchema,URL as urlSchema,type ScrapeOptions,} from "../../controllers/v2/types";@@ -42,26 +43,34 @@ const crawlTargetSchema = z.strictObject({// Search-specific state rides existing jsonb columns (config here; URL dedup memory in// monitor_pages.metadata; event index in monitors.last_check_summary jsonb for v1) β no// new tables/columns required to ship.-const searchTargetSchema = z.strictObject({- id: z.string().uuid().optional(),- type: z.literal("search"),- // The web-search queries. (v1: caller/LLM-supplied; see search/queries.ts.)- queries: z.array(z.string().min(1).max(256)).min(1).max(10),- // Unified schedule/window step β mapped to a Firecrawl `tbs` window in search/run.ts.- searchWindow: z- .enum(["5m", "15m", "1h", "6h", "24h", "7d"])- .optional()- .default("24h"),- // first_match = alert once per real-world event; every_new_result = alert on each new URL.- alertMode: z- .enum(["first_match", "every_new_result"])- .optional()- .default("first_match"),- includeDomains: z.array(z.string().min(1)).max(50).optional(),- // Max results judged per run (caps Firecrawl scrape + LLM cost).- maxResults: z.number().int().min(1).max(50).optional().default(10),- scrapeOptions: scrapeOptionsSchema,-});+const searchTargetSchema = z+ .strictObject({+ id: z.string().uuid().optional(),+ type: z.literal("search"),+ // The web-search queries. (v1: caller/LLM-supplied; see search/queries.ts.)+ queries: z.array(z.string().min(1).max(256)).min(1).max(10),+ // Unified schedule/window step β mapped to a Firecrawl `tbs` window in search/run.ts.+ searchWindow: z+ .enum(["5m", "15m", "1h", "6h", "24h", "7d"])+ .optional()+ .default("24h"),+ // first_match = alert once per real-world event; every_new_result = alert on each new URL.+ alertMode: z+ .enum(["first_match", "every_new_result"])+ .optional()+ .default("first_match"),+ // Domain scoping β site:/-site: operators (search/run.ts). Mirrors the v2 search API:+ // reuse searchDomainSchema and keep include/exclude mutually exclusive.+ includeDomains: z.array(searchDomainSchema).max(50).optional(),+ excludeDomains: z.array(searchDomainSchema).max(50).optional(),+ // Max results judged per run (caps Firecrawl scrape + LLM cost).+ maxResults: z.number().int().min(1).max(50).optional().default(10),+ scrapeOptions: scrapeOptionsSchema,+ })+ .refine(+ t => !(t.includeDomains?.length && t.excludeDomains?.length),+ "includeDomains and excludeDomains cannot both be specified",+ );const monitorTargetSchema = z.union([scrapeTargetSchema,@@ -158,7 +167,8 @@ function requireGoalForSearchTargets(const targets = input.targets;if (!Array.isArray(targets)) return;const hasSearchTarget = targets.some(- t => t && typeof t === "object" && (t as { type?: unknown }).type === "search",+ t =>+ t && typeof t === "object" && (t as { type?: unknown }).type === "search",);const goal = input.goal;if (