Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Refactor search run tests and clarify alerts

Developers Digest978ceb5fe550
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index db7a1d36a..d3899f48f 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -1,5 +1,6 @@
import type { Logger } from "winston";
import type { SearchVerdict } from "./judge";
+import { canonicalizeUrl } from "./dedupe";
// vi.mock is hoisted above declarations, so the mocks its factories reference
// are created in vi.hoisted() (also hoisted) to avoid any TDZ surprises.
@@ -151,39 +152,12 @@ describe("standard depth (snippet judging, no scrapes)", () => {
});
});
-describe("alert boundary", () => {
+describe("alert boundary (the per-page JSON verdict is the gate)", () => {
beforeEach(() => {
searchMock.mockResolvedValue([serpRow(1)]);
- routeMock.mockResolvedValue([
- { id: "result_1", decision: "scrape", priority: 1, reason: "" },
- ]);
- });
-
- it("verifier downgrades a competitor-dominated story to watching", async () => {
- searchMock.mockResolvedValue([
- {
- url: "https://news.example.com/parallel",
- title: "Parallel launches Monitor API",
- description: "Parallel launched a product; Firecrawl is a competitor.",
- },
- ]);
- scrapeURLMock.mockResolvedValue({
- success: true,
- document: {
- json: verdict({ concept: "Parallel Monitor API launch" }),
- markdown:
- "Parallel launched its Monitor API today. Firecrawl is named as a competitor.",
- metadata: {},
- },
- });
-
- const result = await runSearchTarget(runParams());
- expect(result.matches).toBe(0);
- expect(result.sources[0].status).toBe("watching");
- expect(reviewAlertMock).not.toHaveBeenCalled();
});
- it("skeptic refutation downgrades to watching", async () => {
+ it("a clean alert verdict on a fresh result alerts", async () => {
scrapeURLMock.mockResolvedValue({
success: true,
document: {
@@ -192,70 +166,46 @@ describe("alert boundary", () => {
metadata: {},
},
});
- reviewAlertMock.mockResolvedValue({
- refuted: true,
- failureMode: "adjacent_event",
- reason: "funding, not a launch",
- });
-
- const result = await runSearchTarget(runParams());
- expect(result.matches).toBe(0);
- expect(result.sources[0].status).toBe("watching");
- });
-
- it("skeptic outage fails open (alert proceeds)", async () => {
- scrapeURLMock.mockResolvedValue({
- success: true,
- document: {
- json: verdict(),
- markdown: "Firecrawl announced a new product today in prose.",
- metadata: {},
- },
- });
- reviewAlertMock.mockRejectedValue(new Error("skeptic down"));
const result = await runSearchTarget(runParams());
expect(result.matches).toBe(1);
+ expect(result.sources[0].status).toBe("alert");
});
- it("resolver outage fails open to a new event (alert proceeds)", async () => {
+ it("a watch verdict does not alert", async () => {
scrapeURLMock.mockResolvedValue({
success: true,
document: {
- json: verdict(),
+ json: verdict({ alertAction: "watch" }),
markdown: "Firecrawl announced a new product today in prose.",
metadata: {},
},
});
- resolveEventMock.mockRejectedValue(new Error("resolver down"));
const result = await runSearchTarget(runParams());
- expect(result.matches).toBe(1);
- expect(result.sources[0].status).toBe("alert");
+ expect(result.matches).toBe(0);
+ expect(result.sources[0].status).toBe("watching");
});
- it("criteria LLM failure keeps the deterministic compile (run completes)", async () => {
- criteriaLlmMock.mockRejectedValue(new Error("criteria down"));
+ it("an ignore verdict does not alert", async () => {
scrapeURLMock.mockResolvedValue({
success: true,
document: {
- json: verdict(),
- markdown: "Firecrawl announced a new product today in prose.",
+ json: verdict({ alertAction: "ignore" }),
+ markdown: "Off-topic prose.",
metadata: {},
},
});
const result = await runSearchTarget(runParams());
- expect(result.matches).toBe(1);
+ expect(result.matches).toBe(0);
+ expect(result.sources[0].status).toBe("ignored");
});
});
describe("event state stamps + judgment on alert pages", () => {
it("stamps satisfiedAt/alertCount/lastAlertAt and a judgment on the alerting page", async () => {
searchMock.mockResolvedValue([serpRow(1)]);
- routeMock.mockResolvedValue([
- { id: "result_1", decision: "scrape", priority: 1, reason: "" },
- ]);
scrapeURLMock.mockResolvedValue({
success: true,
document: {
@@ -277,11 +227,8 @@ describe("event state stamps + judgment on alert pages", () => {
});
});
- it("a later alert on a known satisfied event increments alertCount and keeps satisfiedAt", async () => {
+ it("a later alert on a known event (every_new_result) increments alertCount and keeps satisfiedAt", async () => {
searchMock.mockResolvedValue([serpRow(1)]);
- routeMock.mockResolvedValue([
- { id: "result_1", decision: "scrape", priority: 1, reason: "" },
- ]);
scrapeURLMock.mockResolvedValue({
success: true,
document: {
@@ -290,19 +237,12 @@ describe("event state stamps + judgment on alert pages", () => {
metadata: {},
},
});
- resolveEventMock.mockResolvedValue({
- matchedKey: "evt-known",
- isNew: false,
- label: "Firecrawl product launch",
- reason: "",
- });
- materialDevMock.mockResolvedValue({ material: true, reason: "new stage" });
const result = await runSearchTarget({
- ...runParams({ alertMode: "material_dev" }),
+ ...runParams({ alertMode: "every_new_result" }),
knownEvents: [
{
- key: "evt-known",
+ key: canonicalizeUrl(serpRow(1).url),
label: "Firecrawl product launch",
satisfiedAt: "2026-06-01T00:00:00Z",
alertCount: 2,
@@ -325,14 +265,9 @@ describe("judgeEnabled gates the LLM judge", () => {
runParams({ depth: "deep" }, { judgeEnabled: false }),
);
- // No LLM stage ran.
- expect(routeMock).not.toHaveBeenCalled();
+ // No judge stage ran: raw mode neither snippet-judges nor scrapes.
expect(snippetsMock).not.toHaveBeenCalled();
expect(scrapeURLMock).not.toHaveBeenCalled();
- expect(reviewAlertMock).not.toHaveBeenCalled();
- expect(resolveEventMock).not.toHaveBeenCalled();
- expect(summarizeRunMock).not.toHaveBeenCalled();
- expect(criteriaLlmMock).not.toHaveBeenCalled();
// No judge credits billed (raw → nothing judged); search still ran and was billed.
expect(result.judgeCredits).toBe(0);
@@ -362,11 +297,8 @@ describe("judgeEnabled gates the LLM judge", () => {
expect(scrapeURLMock).not.toHaveBeenCalled();
});
- it("judge ON with depth:deep still runs the full judge", async () => {
+ it("judge ON with depth:deep scrapes and judges per page", async () => {
searchMock.mockResolvedValue([serpRow(1)]);
- routeMock.mockResolvedValue([
- { id: "result_1", decision: "scrape", priority: 1, reason: "" },
- ]);
scrapeURLMock.mockResolvedValue({
success: true,
document: {
@@ -380,7 +312,6 @@ describe("judgeEnabled gates the LLM judge", () => {
runParams({ depth: "deep" }, { judgeEnabled: true }),
);
- expect(routeMock).toHaveBeenCalled();
expect(scrapeURLMock).toHaveBeenCalled();
expect(result.matches).toBe(1);
const alertUpsert = result.pageUpserts.find(u => u.status === "alert")!;
@@ -390,12 +321,8 @@ describe("judgeEnabled gates the LLM judge", () => {
describe("deep-path scrape failures mark the check degraded (no silent empty)", () => {
beforeEach(() => {
- // Two candidates routed to scrape; scrape outcome varies per test.
+ // Two candidates scraped; scrape outcome varies per test.
searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
- routeMock.mockResolvedValue([
- { id: "result_1", decision: "scrape", priority: 2, reason: "" },
- { id: "result_2", decision: "scrape", priority: 1, reason: "" },
- ]);
});
it("degraded=true when EVERY deep-path scrape fails (judging expected, nothing judged)", async () => {
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.ts
index 9adffa0d0..904bb269c 100644
--- a/apps/api/src/services/monitoring/search/run.test.ts
+++ b/apps/api/src/services/monitoring/search/run.test.ts
@@ -405,43 +405,24 @@ describe("scrape payload validity (real validator, no mocks)", () => {
});
});
-describe("material_dev alert mode", () => {
- const serp = [
- {
- url: "https://reuters.com/openai",
- title: "OpenAI prices IPO",
- description: "priced",
- },
- ];
+describe("material_dev alert mode (dedups a known URL like first_match)", () => {
+ const url = "https://reuters.com/openai";
+ const serp = [{ url, title: "OpenAI prices IPO", description: "priced" }];
beforeEach(() => {
setSearchResults(serp);
- setVerdictsByUrl({ "https://reuters.com/openai": verdict() });
- resolveEventMock.mockResolvedValue({
- matchedKey: "evt-1",
- isNew: false,
- label: "OpenAI IPO",
- reason: "same",
- });
+ setVerdictsByUrl({ [url]: verdict() });
});
- it("re-alerts on a material development of a known event", async () => {
- materialDevMock.mockResolvedValue({ material: true, reason: "now priced" });
- const out = await run({
- alertMode: "material_dev",
- knownEvents: [{ key: "evt-1", label: "OpenAI IPO" }],
- });
+ it("alerts on a URL not yet seen", async () => {
+ const out = await run({ alertMode: "material_dev" });
expect(out.matches).toBe(1);
expect(out.sources[0].status).toBe("alert");
});
- it("suppresses a non-material retelling of a known event", async () => {
- materialDevMock.mockResolvedValue({
- material: false,
- reason: "just a retelling",
- });
+ it("suppresses a URL that is already a known event", async () => {
const out = await run({
alertMode: "material_dev",
- knownEvents: [{ key: "evt-1", label: "OpenAI IPO" }],
+ knownEvents: [{ key: canonicalizeUrl(url), label: "OpenAI IPO" }],
});
expect(out.matches).toBe(0);
expect(out.sources[0].status).toBe("already_seen");