Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(monitor): judgeEnabled actually gates the LLM judge (off = raw results, no LLM/credits)

Developers Digestd86b2a91b571
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index c8d7cf322..02e78665d 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -992,6 +992,11 @@ async function runMonitorSearchTarget(params: {
teamId: monitor.team_id,
goal: monitor.goal,
subject: monitor.name,
+ // Read fresh from the persisted monitor on every check, so toggling the
+ // "Only alert on meaningful changes" judge via PATCH/UI takes effect on
+ // the NEXT check (no redeploy). When false, runSearchTarget collapses to
+ // raw mode: deterministic URLs + dedup, no LLM judge, no LLM credits.
+ judgeEnabled: Boolean(monitor.judge_enabled),
},
target: {
id: target.id,
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index 6b1b20409..864aeb901 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -61,13 +61,18 @@ const llmCriteria = {
thirdPartyOnly: false,
};
-function runParams(targetOver: Record<string, unknown> = {}) {
+function runParams(
+ targetOver: Record<string, unknown> = {},
+ monitorOver: Record<string, unknown> = {},
+) {
return {
monitor: {
id: "mon_1",
teamId: "team_1",
goal: "Alert me when Firecrawl launches a new product",
subject: "Firecrawl",
+ judgeEnabled: true,
+ ...monitorOver,
},
target: {
id: "tgt_1",
@@ -341,3 +346,74 @@ describe("event state stamps + judgment on alert pages", () => {
expect(alertUpsert.metadata.eventSatisfiedAt).toBe("2026-06-01T00:00:00Z");
});
});
+
+describe("judgeEnabled gates the LLM judge", () => {
+ it("judge OFF behaves like raw: no LLM stages, no LLM credits, no concept/rationale", async () => {
+ // Monitor still has a goal and depth:"deep" — but judgeEnabled=false must
+ // collapse to raw and skip every LLM stage.
+ searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
+
+ const result = await runSearchTarget(
+ runParams({ depth: "deep" }, { judgeEnabled: false }),
+ );
+
+ // No LLM stage ran.
+ expect(routeMock).not.toHaveBeenCalled();
+ expect(snippetsMock).not.toHaveBeenCalled();
+ expect(scrapeURLMock).not.toHaveBeenCalled();
+ expect(reviewAlertMock).not.toHaveBeenCalled();
+ expect(resolveEventMock).not.toHaveBeenCalled();
+ expect(summarizeRunMock).not.toHaveBeenCalled();
+ expect(criteriaLlmMock).not.toHaveBeenCalled();
+
+ // No LLM credits billed; search still ran and was billed.
+ expect(result.llmCredits).toBe(0);
+ expect(result.resultCount).toBe(2);
+
+ // Raw alerts: deterministic searchStatus from dedup, but no LLM
+ // concept/rationale on any upsert.
+ for (const upsert of result.pageUpserts) {
+ expect(upsert.scraped).toBe(false);
+ expect(upsert.metadata.concept).toBeUndefined();
+ expect(upsert.metadata.rationale).toBeUndefined();
+ expect(upsert.metadata.searchStatus).toBeDefined();
+ }
+ });
+
+ it("judge OFF with no goal still runs search and returns raw results (no throw)", async () => {
+ searchMock.mockResolvedValue([serpRow(1)]);
+
+ const result = await runSearchTarget(
+ runParams({ depth: "deep" }, { judgeEnabled: false, goal: null }),
+ );
+
+ expect(result.resultCount).toBe(1);
+ expect(result.llmCredits).toBe(0);
+ expect(scrapeURLMock).not.toHaveBeenCalled();
+ });
+
+ it("judge ON with depth:deep still runs the full judge", async () => {
+ searchMock.mockResolvedValue([serpRow(1)]);
+ routeMock.mockResolvedValue([
+ { id: "result_1", decision: "scrape", priority: 1, reason: "" },
+ ]);
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict(),
+ markdown: "Firecrawl announced a new product today in prose.",
+ metadata: {},
+ },
+ });
+
+ const result = await runSearchTarget(
+ runParams({ depth: "deep" }, { judgeEnabled: true }),
+ );
+
+ expect(routeMock).toHaveBeenCalled();
+ expect(scrapeURLMock).toHaveBeenCalled();
+ expect(result.matches).toBe(1);
+ const alertUpsert = result.pageUpserts.find(u => u.status === "alert")!;
+ expect(alertUpsert.metadata.concept).toBeDefined();
+ });
+});
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.ts
index 33c9dccb3..a75dc137d 100644
--- a/apps/api/src/services/monitoring/search/run.test.ts
+++ b/apps/api/src/services/monitoring/search/run.test.ts
@@ -74,6 +74,7 @@ const baseMonitor = {
teamId: "team1",
goal: "Alert when OpenAI files for IPO",
subject: "OpenAI",
+ judgeEnabled: true,
};
function run(
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index 7d2673089..bd83bc215 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -339,17 +339,23 @@ export async function runSearchTarget(params: {
? "standard"
: "deep";
+ // From here on, any depth !== "raw" path requires a goal. The top-of-function
+ // guard guarantees judgeEnabled implies a non-empty goal; depth !== "raw"
+ // implies judgeEnabled, so this is always satisfied when judging actually
+ // runs; assert it for the type checker (and as a defensive invariant).
+ const judgeGoal: string = goal ?? "";
+
let criteria: GoalCriteria = compileGoalCriteria({
// goal can be empty in raw/judge-off mode; criteria is only consumed by
// the LLM judge stages, which never run when depth === "raw".
- goal: goal ?? "",
+ goal: judgeGoal,
subject,
goalVersion,
});
if (llmStagesAvailable && depth !== "raw") {
try {
criteria = await compileGoalCriteriaWithLlm({
- goal,
+ goal: judgeGoal,
subject,
queries: target.queries,
goalVersion,
@@ -374,7 +380,7 @@ export async function runSearchTarget(params: {
}
try {
const decisions = await routeSearchResults({
- goal,
+ goal: judgeGoal,
subject,
searchWindow: target.searchWindow,
maxResults: target.maxResults,
@@ -434,7 +440,7 @@ export async function runSearchTarget(params: {
if (depth === "standard" && snippetCandidates.length > 0) {
try {
const verdicts = await judgeSnippets({
- goal,
+ goal: judgeGoal,
subject,
searchWindow: target.searchWindow,
costTracking,
@@ -611,7 +617,7 @@ export async function runSearchTarget(params: {
{
type: "json",
schema: verdictJsonSchema,
- prompt: buildJudgePrompt(goal, subject, target.searchWindow),
+ prompt: buildJudgePrompt(judgeGoal, subject, target.searchWindow),
},
],
timeout: 20000,
@@ -701,7 +707,7 @@ export async function runSearchTarget(params: {
if (decision === "notify" && llmStagesAvailable) {
try {
const skeptic: SkepticVerdict = await reviewAlert({
- goal,
+ goal: judgeGoal,
subject,
criteria,
costTracking,
@@ -774,7 +780,7 @@ export async function runSearchTarget(params: {
let resolution;
try {
resolution = await resolveEvent({
- goal,
+ goal: judgeGoal,
subject,
costTracking,
result: {
@@ -810,7 +816,7 @@ export async function runSearchTarget(params: {
} else if (target.alertMode === "material_dev") {
try {
const dev = await judgeMaterialDevelopment({
- goal,
+ goal: judgeGoal,
subject,
eventLabel,
costTracking,
@@ -900,7 +906,7 @@ export async function runSearchTarget(params: {
} else if (matches > 0) {
try {
const out = await summarizeRun({
- goal,
+ goal: judgeGoal,
subject,
costTracking,
evidence: sources