Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Shorten empty-search retry; make retentionDays optional

Developers Digestc9dde7697f02
diff --git a/apps/api/src/services/monitoring/queue.test.ts b/apps/api/src/services/monitoring/queue.test.ts
new file mode 100644
index 000000000..ab8191803
--- /dev/null
+++ b/apps/api/src/services/monitoring/queue.test.ts
@@ -0,0 +1,138 @@
+// Unit coverage for the branch's core RabbitMQ rework: prefetch(1), the
+// subscribe dedup guard, the message ack/nack contract, and β€” most importantly β€”
+// the reconnect/resubscribe wrapper (the A4 fix: a dropped consumer must
+// re-subscribe and never go permanently deaf).
+import type { Mock } from "vitest";
+
+const { connectMock, channelMock, connectionMock } = vi.hoisted(() => {
+ const channelMock = {
+ assertExchange: vi.fn(),
+ assertQueue: vi.fn(),
+ bindQueue: vi.fn(),
+ prefetch: vi.fn(),
+ consume: vi.fn(),
+ ack: vi.fn(),
+ nack: vi.fn(),
+ sendToQueue: vi.fn(() => true),
+ close: vi.fn(),
+ on: vi.fn(),
+ };
+ const connectionMock = {
+ createChannel: vi.fn(() => channelMock),
+ on: vi.fn(),
+ close: vi.fn(),
+ };
+ const connectMock = vi.fn(() => connectionMock);
+ return { connectMock, channelMock, connectionMock };
+});
+
+vi.mock("amqplib", () => ({ default: { connect: connectMock } }));
+vi.mock("../../config", () => ({
+ config: { NUQ_RABBITMQ_URL: "amqp://test" },
+}));
+vi.mock("../../lib/logger", () => {
+ const mk = (): Record<string, unknown> => ({
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child: () => mk(),
+ });
+ return { logger: mk() };
+});
+
+// Pull the close handler that createChannel registered via ch.on("close", cb).
+function consumeChannelCloseHandler(): () => void {
+ const call = (channelMock.on as Mock).mock.calls.find(c => c[0] === "close");
+ return call?.[1] as () => void;
+}
+
+async function freshQueueModule() {
+ vi.resetModules();
+ return import("./queue.js");
+}
+
+describe("monitoring queue (RabbitMQ wrapper)", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ connectMock.mockReturnValue(connectionMock);
+ connectionMock.createChannel.mockReturnValue(channelMock);
+ channelMock.sendToQueue.mockReturnValue(true);
+ });
+
+ it("subscribes a consumer with per-consumer prefetch(1) and manual ack", async () => {
+ const q = await freshQueueModule();
+ await q.consumeMonitorCheckJobs(vi.fn());
+
+ expect(channelMock.prefetch).toHaveBeenCalledWith(1);
+ expect(channelMock.consume).toHaveBeenCalledTimes(1);
+ const [queue, , opts] = channelMock.consume.mock.calls[0];
+ expect(queue).toBe("monitor.checks");
+ expect(opts).toEqual({ noAck: false });
+ });
+
+ it("dedups a repeat subscribe to the same queue (no duplicate consumer)", async () => {
+ const q = await freshQueueModule();
+ await q.consumeMonitorCheckJobs(vi.fn());
+ await q.consumeMonitorCheckJobs(vi.fn());
+
+ // subscribedQueues guard => consume attached exactly once for the queue.
+ expect(channelMock.consume).toHaveBeenCalledTimes(1);
+ });
+
+ it("acks on a successful handler, nacks (no requeue) when the handler throws", async () => {
+ const q = await freshQueueModule();
+ const ok = vi.fn().mockResolvedValue(undefined);
+ await q.consumeMonitorCheckJobs(ok);
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ const msg = {
+ content: Buffer.from(
+ JSON.stringify({ monitorId: "m1", checkId: "c1", teamId: "t1" }),
+ ),
+ };
+ await onMessage(msg);
+ expect(ok).toHaveBeenCalledWith({
+ monitorId: "m1",
+ checkId: "c1",
+ teamId: "t1",
+ });
+ expect(channelMock.ack).toHaveBeenCalledWith(msg);
+ expect(channelMock.nack).not.toHaveBeenCalled();
+
+ const boom = vi.fn().mockRejectedValue(new Error("handler boom"));
+ const q2mod = q; // same module instance, second queue consumer
+ await q2mod.consumeMonitorSearchCheckJobs(boom);
+ const onSearchMessage = channelMock.consume.mock.calls.at(-1)![1];
+ const msg2 = {
+ content: Buffer.from(
+ JSON.stringify({ monitorId: "m2", checkId: "c2", teamId: "t2" }),
+ ),
+ };
+ await onSearchMessage(msg2);
+ expect(boom).toHaveBeenCalled();
+ expect(channelMock.nack).toHaveBeenCalledWith(msg2, false, false);
+ });
+
+ it("re-subscribes the consumer after a channel drop (does not go deaf)", async () => {
+ vi.useFakeTimers();
+ try {
+ const q = await freshQueueModule();
+ await q.consumeMonitorCheckJobs(vi.fn());
+ expect(channelMock.consume).toHaveBeenCalledTimes(1);
+
+ // Simulate the consume channel closing (a RabbitMQ blip).
+ const onClose = consumeChannelCloseHandler();
+ expect(onClose).toBeTypeOf("function");
+ onClose();
+
+ // The reconnect is scheduled on a backoff timer; advance past it.
+ await vi.advanceTimersByTimeAsync(1500);
+
+ // The registered consumer was re-attached on the fresh channel.
+ expect(channelMock.consume.mock.calls.length).toBeGreaterThanOrEqual(2);
+ expect(channelMock.consume.mock.calls.at(-1)![0]).toBe("monitor.checks");
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index b7db2a6ee..292dd0f9a 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -4,6 +4,7 @@ vi.mock("uuid", () => ({
import {
estimateActualCredits,
+ findCompletedSearchTargetRun,
isMonitorCheckStale,
MONITOR_CHECK_STALE_TIMEOUT_MS,
} from "./runner";
@@ -108,3 +109,78 @@ describe("monitoring runner", () => {
});
});
});
+
+// The redelivery-idempotency guard: on a re-delivered check, the runner must
+// recognize a search target that ALREADY completed (searchCompleted:true) and
+// restore its persisted figures instead of re-running the search + re-scraping +
+// re-billing. findCompletedSearchTargetRun is that decision.
+describe("findCompletedSearchTargetRun (redelivery idempotency)", () => {
+ const completed = {
+ type: "search",
+ targetId: "t1",
+ searchCompleted: true,
+ resultCount: 5,
+ matches: 2,
+ searchCredits: 2,
+ judgeCredits: 4,
+ };
+
+ it("returns the completed run for a matching, searchCompleted target (skip re-run)", () => {
+ const found = findCompletedSearchTargetRun([completed], "t1");
+ expect(found).not.toBeNull();
+ expect(found).toMatchObject({
+ targetId: "t1",
+ searchCredits: 2,
+ matches: 2,
+ });
+ });
+
+ it("returns null when the search target has NOT completed (must re-run)", () => {
+ expect(
+ findCompletedSearchTargetRun(
+ [{ type: "search", targetId: "t1", searchCompleted: false }],
+ "t1",
+ ),
+ ).toBeNull();
+ // searchCompleted absent entirely
+ expect(
+ findCompletedSearchTargetRun([{ type: "search", targetId: "t1" }], "t1"),
+ ).toBeNull();
+ });
+
+ it("returns null when the targetId does not match", () => {
+ expect(findCompletedSearchTargetRun([completed], "other")).toBeNull();
+ });
+
+ it("ignores non-search completed targets (a finished scrape is not a search run)", () => {
+ expect(
+ findCompletedSearchTargetRun(
+ [{ type: "scrape", targetId: "t1", searchCompleted: true }],
+ "t1",
+ ),
+ ).toBeNull();
+ });
+
+ it("picks the right target out of a mixed target_results array", () => {
+ const results = [
+ { type: "scrape", targetId: "s1" },
+ { type: "search", targetId: "t1", searchCompleted: false },
+ completed,
+ null,
+ "garbage",
+ ];
+ // t1's first (incomplete) entry must not match; the completed t1 does β€” but
+ // since find() returns the first match and the incomplete one is filtered by
+ // searchCompleted, the completed entry (also t1) is the one returned.
+ const found = findCompletedSearchTargetRun(results, "t1");
+ expect(found).toMatchObject({ searchCredits: 2 });
+ });
+
+ it("returns null for non-array / empty / malformed input", () => {
+ expect(findCompletedSearchTargetRun(undefined, "t1")).toBeNull();
+ expect(findCompletedSearchTargetRun(null, "t1")).toBeNull();
+ expect(findCompletedSearchTargetRun([], "t1")).toBeNull();
+ expect(findCompletedSearchTargetRun("nope", "t1")).toBeNull();
+ expect(findCompletedSearchTargetRun([null, "x", 5], "t1")).toBeNull();
+ });
+});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 1c627e893..89386bd56 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -1199,7 +1199,8 @@ async function runMonitorSearchTarget(params: {
// Find a persisted search target run that already completed, so a redelivered
// check can restore its figures instead of re-running (and re-billing) it.
-function findCompletedSearchTargetRun(
+// Exported for unit coverage of the redelivery-idempotency decision.
+export function findCompletedSearchTargetRun(
targetResults: unknown,
targetId: string,
): Record<string, unknown> | null {
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index 850fdf497..7bb332da6 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -373,6 +373,10 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
expect(result.judgeDegraded).toBe(false);
expect(result.degradedReason).toBeNull();
expect(scrapeURLMock).not.toHaveBeenCalled();
+ // An empty SERP hedges with a BOUNDED retry (1 initial + 2 quick retries),
+ // NOT the old 5-attempt / ~11s growing backoff. Lock the budget so a genuinely
+ // empty query concludes fast instead of stalling the whole check.
+ expect(searchMock).toHaveBeenCalledTimes(3);
});
it("flags partialScrapeLoss (soft signal, NOT degraded) when most scrapes fail but some judge", async () => {
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index 3b37f21b9..3083c196a 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -58,9 +58,13 @@ function windowToTbs(window: string): string {
return "qdr:w";
}
-// Retry an empty result with growing backoff β€” the backend sometimes returns 0 for
-// a query that succeeds moments later.
-const SEARCH_RETRY_BACKOFF_MS = [400, 1200, 3000, 6000] as const;
+// A successful-but-empty SERP almost always means a genuine "no results" for this
+// query+window β€” NOT a transient failure. The backend can occasionally return 0 for
+// a query that succeeds a moment later, so hedge with a couple of QUICK retries, but
+// don't burn ~11s of growing backoff (the old [400,1200,3000,6000]) on a query that
+// simply has no matches β€” that latency stacks across queries and slows every check
+// whose results legitimately don't exist.
+const SEARCH_EMPTY_RETRY_BACKOFF_MS = [500, 1000] as const;
// Hard wall-clock budget for a single inline search run. The deep-mode page
// scrapes are the only unbounded-ish step (each is capped at 20s, but a large
@@ -117,7 +121,7 @@ async function searchWithRetry(
args: Parameters<typeof search>[0],
logger: Logger,
): Promise<Awaited<ReturnType<typeof search>>> {
- const maxAttempts = SEARCH_RETRY_BACKOFF_MS.length + 1;
+ const maxAttempts = SEARCH_EMPTY_RETRY_BACKOFF_MS.length + 1;
let lastResponse: Awaited<ReturnType<typeof search>> = {};
for (let attempt = 0; attempt < maxAttempts; attempt++) {
@@ -129,7 +133,7 @@ async function searchWithRetry(
const isLastAttempt = attempt === maxAttempts - 1;
if (!isLastAttempt) {
- const base = SEARCH_RETRY_BACKOFF_MS[attempt];
+ const base = SEARCH_EMPTY_RETRY_BACKOFF_MS[attempt];
const jitter = Math.floor(Math.random() * 250);
logger.info("search monitor query empty; retrying", {
query: args.query,
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
index 44c7623ef..a6e5837b7 100644
--- a/apps/api/src/services/monitoring/types.ts
+++ b/apps/api/src/services/monitoring/types.ts
@@ -230,6 +230,10 @@ export const updateMonitorSchema = createMonitorBaseSchema
// default would materialize {} for a PATCH that omits notification and wipe
// the stored email config. No default => omitted means "leave unchanged".
notification: monitorNotificationInner.optional(),
+ // Same default-materialization trap: createMonitorBaseSchema gives
+ // retentionDays .default(30), so a partial PATCH that omits it would parse to
+ // 30 and silently reset a user's configured retention. Drop the default here.
+ retentionDays: z.number().int().positive().max(365).optional(),
})
.superRefine(rejectGoalClearedWithSearchTargets)
.refine(x => Object.keys(x).length > 0, "Update body cannot be empty");