Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(monitor): judgeEnabled actually gates the LLM judge (off = raw results, no LLM/credits)
Developers Digestd86b2a91b571
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex c8d7cf322..02e78665d 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -992,6 +992,11 @@ async function runMonitorSearchTarget(params: {teamId: monitor.team_id,goal: monitor.goal,subject: monitor.name,+ // Read fresh from the persisted monitor on every check, so toggling the+ // "Only alert on meaningful changes" judge via PATCH/UI takes effect on+ // the NEXT check (no redeploy). When false, runSearchTarget collapses to+ // raw mode: deterministic URLs + dedup, no LLM judge, no LLM credits.+ judgeEnabled: Boolean(monitor.judge_enabled),},target: {id: target.id,diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex 6b1b20409..864aeb901 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -61,13 +61,18 @@ const llmCriteria = {thirdPartyOnly: false,};-function runParams(targetOver: Record<string, unknown> = {}) {+function runParams(+ targetOver: Record<string, unknown> = {},+ monitorOver: Record<string, unknown> = {},+) {return {monitor: {id: "mon_1",teamId: "team_1",goal: "Alert me when Firecrawl launches a new product",subject: "Firecrawl",+ judgeEnabled: true,+ ...monitorOver,},target: {id: "tgt_1",@@ -341,3 +346,74 @@ describe("event state stamps + judgment on alert pages", () => {expect(alertUpsert.metadata.eventSatisfiedAt).toBe("2026-06-01T00:00:00Z");});});++describe("judgeEnabled gates the LLM judge", () => {+ it("judge OFF behaves like raw: no LLM stages, no LLM credits, no concept/rationale", async () => {+ // Monitor still has a goal and depth:"deep" — but judgeEnabled=false must+ // collapse to raw and skip every LLM stage.+ searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);++ const result = await runSearchTarget(+ runParams({ depth: "deep" }, { judgeEnabled: false }),+ );++ // No LLM stage ran.+ expect(routeMock).not.toHaveBeenCalled();+ expect(snippetsMock).not.toHaveBeenCalled();+ expect(scrapeURLMock).not.toHaveBeenCalled();+ expect(reviewAlertMock).not.toHaveBeenCalled();+ expect(resolveEventMock).not.toHaveBeenCalled();+ expect(summarizeRunMock).not.toHaveBeenCalled();+ expect(criteriaLlmMock).not.toHaveBeenCalled();++ // No LLM credits billed; search still ran and was billed.+ expect(result.llmCredits).toBe(0);+ expect(result.resultCount).toBe(2);++ // Raw alerts: deterministic searchStatus from dedup, but no LLM+ // concept/rationale on any upsert.+ for (const upsert of result.pageUpserts) {+ expect(upsert.scraped).toBe(false);+ expect(upsert.metadata.concept).toBeUndefined();+ expect(upsert.metadata.rationale).toBeUndefined();+ expect(upsert.metadata.searchStatus).toBeDefined();+ }+ });++ it("judge OFF with no goal still runs search and returns raw results (no throw)", async () => {+ searchMock.mockResolvedValue([serpRow(1)]);++ const result = await runSearchTarget(+ runParams({ depth: "deep" }, { judgeEnabled: false, goal: null }),+ );++ expect(result.resultCount).toBe(1);+ expect(result.llmCredits).toBe(0);+ expect(scrapeURLMock).not.toHaveBeenCalled();+ });++ it("judge ON with depth:deep still runs the full judge", async () => {+ searchMock.mockResolvedValue([serpRow(1)]);+ routeMock.mockResolvedValue([+ { id: "result_1", decision: "scrape", priority: 1, reason: "" },+ ]);+ scrapeURLMock.mockResolvedValue({+ success: true,+ document: {+ json: verdict(),+ markdown: "Firecrawl announced a new product today in prose.",+ metadata: {},+ },+ });++ const result = await runSearchTarget(+ runParams({ depth: "deep" }, { judgeEnabled: true }),+ );++ expect(routeMock).toHaveBeenCalled();+ expect(scrapeURLMock).toHaveBeenCalled();+ expect(result.matches).toBe(1);+ const alertUpsert = result.pageUpserts.find(u => u.status === "alert")!;+ expect(alertUpsert.metadata.concept).toBeDefined();+ });+});diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 33c9dccb3..a75dc137d 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -74,6 +74,7 @@ const baseMonitor = {teamId: "team1",goal: "Alert when OpenAI files for IPO",subject: "OpenAI",+ judgeEnabled: true,};function run(diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 7d2673089..bd83bc215 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -339,17 +339,23 @@ export async function runSearchTarget(params: {? "standard": "deep";+ // From here on, any depth !== "raw" path requires a goal. The top-of-function+ // guard guarantees judgeEnabled implies a non-empty goal; depth !== "raw"+ // implies judgeEnabled, so this is always satisfied when judging actually+ // runs; assert it for the type checker (and as a defensive invariant).+ const judgeGoal: string = goal ?? "";+let criteria: GoalCriteria = compileGoalCriteria({// goal can be empty in raw/judge-off mode; criteria is only consumed by// the LLM judge stages, which never run when depth === "raw".- goal: goal ?? "",+ goal: judgeGoal,subject,goalVersion,});if (llmStagesAvailable && depth !== "raw") {try {criteria = await compileGoalCriteriaWithLlm({- goal,+ goal: judgeGoal,subject,queries: target.queries,goalVersion,@@ -374,7 +380,7 @@ export async function runSearchTarget(params: {}try {const decisions = await routeSearchResults({- goal,+ goal: judgeGoal,subject,searchWindow: target.searchWindow,maxResults: target.maxResults,@@ -434,7 +440,7 @@ export async function runSearchTarget(params: {if (depth === "standard" && snippetCandidates.length > 0) {try {const verdicts = await judgeSnippets({- goal,+ goal: judgeGoal,subject,searchWindow: target.searchWindow,costTracking,@@ -611,7 +617,7 @@ export async function runSearchTarget(params: {{type: "json",schema: verdictJsonSchema,- prompt: buildJudgePrompt(goal, subject, target.searchWindow),+ prompt: buildJudgePrompt(judgeGoal, subject, target.searchWindow),},],timeout: 20000,@@ -701,7 +707,7 @@ export async function runSearchTarget(params: {if (decision === "notify" && llmStagesAvailable) {try {const skeptic: SkepticVerdict = await reviewAlert({- goal,+ goal: judgeGoal,subject,criteria,costTracking,@@ -774,7 +780,7 @@ export async function runSearchTarget(params: {let resolution;try {resolution = await resolveEvent({- goal,+ goal: judgeGoal,subject,costTracking,result: {@@ -810,7 +816,7 @@ export async function runSearchTarget(params: {} else if (target.alertMode === "material_dev") {try {const dev = await judgeMaterialDevelopment({- goal,+ goal: judgeGoal,subject,eventLabel,costTracking,@@ -900,7 +906,7 @@ export async function runSearchTarget(params: {} else if (matches > 0) {try {const out = await summarizeRun({- goal,+ goal: judgeGoal,subject,costTracking,evidence: sources