Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Shorten empty-search retry; make retentionDays optional
Developers Digestc9dde7697f02
diff --git a/apps/api/src/services/monitoring/queue.test.ts b/apps/api/src/services/monitoring/queue.test.tsnew file mode 100644index 000000000..ab8191803--- /dev/null+++ b/apps/api/src/services/monitoring/queue.test.ts@@ -0,0 +1,138 @@+// Unit coverage for the branch's core RabbitMQ rework: prefetch(1), the+// subscribe dedup guard, the message ack/nack contract, and β most importantly β+// the reconnect/resubscribe wrapper (the A4 fix: a dropped consumer must+// re-subscribe and never go permanently deaf).+import type { Mock } from "vitest";++const { connectMock, channelMock, connectionMock } = vi.hoisted(() => {+ const channelMock = {+ assertExchange: vi.fn(),+ assertQueue: vi.fn(),+ bindQueue: vi.fn(),+ prefetch: vi.fn(),+ consume: vi.fn(),+ ack: vi.fn(),+ nack: vi.fn(),+ sendToQueue: vi.fn(() => true),+ close: vi.fn(),+ on: vi.fn(),+ };+ const connectionMock = {+ createChannel: vi.fn(() => channelMock),+ on: vi.fn(),+ close: vi.fn(),+ };+ const connectMock = vi.fn(() => connectionMock);+ return { connectMock, channelMock, connectionMock };+});++vi.mock("amqplib", () => ({ default: { connect: connectMock } }));+vi.mock("../../config", () => ({+ config: { NUQ_RABBITMQ_URL: "amqp://test" },+}));+vi.mock("../../lib/logger", () => {+ const mk = (): Record<string, unknown> => ({+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child: () => mk(),+ });+ return { logger: mk() };+});++// Pull the close handler that createChannel registered via ch.on("close", cb).+function consumeChannelCloseHandler(): () => void {+ const call = (channelMock.on as Mock).mock.calls.find(c => c[0] === "close");+ return call?.[1] as () => void;+}++async function freshQueueModule() {+ vi.resetModules();+ return import("./queue.js");+}++describe("monitoring queue (RabbitMQ wrapper)", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ connectMock.mockReturnValue(connectionMock);+ connectionMock.createChannel.mockReturnValue(channelMock);+ channelMock.sendToQueue.mockReturnValue(true);+ });++ it("subscribes a consumer with per-consumer prefetch(1) and manual ack", async () => {+ const q = await freshQueueModule();+ await q.consumeMonitorCheckJobs(vi.fn());++ expect(channelMock.prefetch).toHaveBeenCalledWith(1);+ expect(channelMock.consume).toHaveBeenCalledTimes(1);+ const [queue, , opts] = channelMock.consume.mock.calls[0];+ expect(queue).toBe("monitor.checks");+ expect(opts).toEqual({ noAck: false });+ });++ it("dedups a repeat subscribe to the same queue (no duplicate consumer)", async () => {+ const q = await freshQueueModule();+ await q.consumeMonitorCheckJobs(vi.fn());+ await q.consumeMonitorCheckJobs(vi.fn());++ // subscribedQueues guard => consume attached exactly once for the queue.+ expect(channelMock.consume).toHaveBeenCalledTimes(1);+ });++ it("acks on a successful handler, nacks (no requeue) when the handler throws", async () => {+ const q = await freshQueueModule();+ const ok = vi.fn().mockResolvedValue(undefined);+ await q.consumeMonitorCheckJobs(ok);+ const onMessage = channelMock.consume.mock.calls[0][1];++ const msg = {+ content: Buffer.from(+ JSON.stringify({ monitorId: "m1", checkId: "c1", teamId: "t1" }),+ ),+ };+ await onMessage(msg);+ expect(ok).toHaveBeenCalledWith({+ monitorId: "m1",+ checkId: "c1",+ teamId: "t1",+ });+ expect(channelMock.ack).toHaveBeenCalledWith(msg);+ expect(channelMock.nack).not.toHaveBeenCalled();++ const boom = vi.fn().mockRejectedValue(new Error("handler boom"));+ const q2mod = q; // same module instance, second queue consumer+ await q2mod.consumeMonitorSearchCheckJobs(boom);+ const onSearchMessage = channelMock.consume.mock.calls.at(-1)![1];+ const msg2 = {+ content: Buffer.from(+ JSON.stringify({ monitorId: "m2", checkId: "c2", teamId: "t2" }),+ ),+ };+ await onSearchMessage(msg2);+ expect(boom).toHaveBeenCalled();+ expect(channelMock.nack).toHaveBeenCalledWith(msg2, false, false);+ });++ it("re-subscribes the consumer after a channel drop (does not go deaf)", async () => {+ vi.useFakeTimers();+ try {+ const q = await freshQueueModule();+ await q.consumeMonitorCheckJobs(vi.fn());+ expect(channelMock.consume).toHaveBeenCalledTimes(1);++ // Simulate the consume channel closing (a RabbitMQ blip).+ const onClose = consumeChannelCloseHandler();+ expect(onClose).toBeTypeOf("function");+ onClose();++ // The reconnect is scheduled on a backoff timer; advance past it.+ await vi.advanceTimersByTimeAsync(1500);++ // The registered consumer was re-attached on the fresh channel.+ expect(channelMock.consume.mock.calls.length).toBeGreaterThanOrEqual(2);+ expect(channelMock.consume.mock.calls.at(-1)![0]).toBe("monitor.checks");+ } finally {+ vi.useRealTimers();+ }+ });+});diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex b7db2a6ee..292dd0f9a 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -4,6 +4,7 @@ vi.mock("uuid", () => ({import {estimateActualCredits,+ findCompletedSearchTargetRun,isMonitorCheckStale,MONITOR_CHECK_STALE_TIMEOUT_MS,} from "./runner";@@ -108,3 +109,78 @@ describe("monitoring runner", () => {});});});++// The redelivery-idempotency guard: on a re-delivered check, the runner must+// recognize a search target that ALREADY completed (searchCompleted:true) and+// restore its persisted figures instead of re-running the search + re-scraping ++// re-billing. findCompletedSearchTargetRun is that decision.+describe("findCompletedSearchTargetRun (redelivery idempotency)", () => {+ const completed = {+ type: "search",+ targetId: "t1",+ searchCompleted: true,+ resultCount: 5,+ matches: 2,+ searchCredits: 2,+ judgeCredits: 4,+ };++ it("returns the completed run for a matching, searchCompleted target (skip re-run)", () => {+ const found = findCompletedSearchTargetRun([completed], "t1");+ expect(found).not.toBeNull();+ expect(found).toMatchObject({+ targetId: "t1",+ searchCredits: 2,+ matches: 2,+ });+ });++ it("returns null when the search target has NOT completed (must re-run)", () => {+ expect(+ findCompletedSearchTargetRun(+ [{ type: "search", targetId: "t1", searchCompleted: false }],+ "t1",+ ),+ ).toBeNull();+ // searchCompleted absent entirely+ expect(+ findCompletedSearchTargetRun([{ type: "search", targetId: "t1" }], "t1"),+ ).toBeNull();+ });++ it("returns null when the targetId does not match", () => {+ expect(findCompletedSearchTargetRun([completed], "other")).toBeNull();+ });++ it("ignores non-search completed targets (a finished scrape is not a search run)", () => {+ expect(+ findCompletedSearchTargetRun(+ [{ type: "scrape", targetId: "t1", searchCompleted: true }],+ "t1",+ ),+ ).toBeNull();+ });++ it("picks the right target out of a mixed target_results array", () => {+ const results = [+ { type: "scrape", targetId: "s1" },+ { type: "search", targetId: "t1", searchCompleted: false },+ completed,+ null,+ "garbage",+ ];+ // t1's first (incomplete) entry must not match; the completed t1 does β but+ // since find() returns the first match and the incomplete one is filtered by+ // searchCompleted, the completed entry (also t1) is the one returned.+ const found = findCompletedSearchTargetRun(results, "t1");+ expect(found).toMatchObject({ searchCredits: 2 });+ });++ it("returns null for non-array / empty / malformed input", () => {+ expect(findCompletedSearchTargetRun(undefined, "t1")).toBeNull();+ expect(findCompletedSearchTargetRun(null, "t1")).toBeNull();+ expect(findCompletedSearchTargetRun([], "t1")).toBeNull();+ expect(findCompletedSearchTargetRun("nope", "t1")).toBeNull();+ expect(findCompletedSearchTargetRun([null, "x", 5], "t1")).toBeNull();+ });+});diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 1c627e893..89386bd56 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -1199,7 +1199,8 @@ async function runMonitorSearchTarget(params: {// Find a persisted search target run that already completed, so a redelivered// check can restore its figures instead of re-running (and re-billing) it.-function findCompletedSearchTargetRun(+// Exported for unit coverage of the redelivery-idempotency decision.+export function findCompletedSearchTargetRun(targetResults: unknown,targetId: string,): Record<string, unknown> | null {diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex 850fdf497..7bb332da6 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -373,6 +373,10 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",expect(result.judgeDegraded).toBe(false);expect(result.degradedReason).toBeNull();expect(scrapeURLMock).not.toHaveBeenCalled();+ // An empty SERP hedges with a BOUNDED retry (1 initial + 2 quick retries),+ // NOT the old 5-attempt / ~11s growing backoff. Lock the budget so a genuinely+ // empty query concludes fast instead of stalling the whole check.+ expect(searchMock).toHaveBeenCalledTimes(3);});it("flags partialScrapeLoss (soft signal, NOT degraded) when most scrapes fail but some judge", async () => {diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 3b37f21b9..3083c196a 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -58,9 +58,13 @@ function windowToTbs(window: string): string {return "qdr:w";}-// Retry an empty result with growing backoff β the backend sometimes returns 0 for-// a query that succeeds moments later.-const SEARCH_RETRY_BACKOFF_MS = [400, 1200, 3000, 6000] as const;+// A successful-but-empty SERP almost always means a genuine "no results" for this+// query+window β NOT a transient failure. The backend can occasionally return 0 for+// a query that succeeds a moment later, so hedge with a couple of QUICK retries, but+// don't burn ~11s of growing backoff (the old [400,1200,3000,6000]) on a query that+// simply has no matches β that latency stacks across queries and slows every check+// whose results legitimately don't exist.+const SEARCH_EMPTY_RETRY_BACKOFF_MS = [500, 1000] as const;// Hard wall-clock budget for a single inline search run. The deep-mode page// scrapes are the only unbounded-ish step (each is capped at 20s, but a large@@ -117,7 +121,7 @@ async function searchWithRetry(args: Parameters<typeof search>[0],logger: Logger,): Promise<Awaited<ReturnType<typeof search>>> {- const maxAttempts = SEARCH_RETRY_BACKOFF_MS.length + 1;+ const maxAttempts = SEARCH_EMPTY_RETRY_BACKOFF_MS.length + 1;let lastResponse: Awaited<ReturnType<typeof search>> = {};for (let attempt = 0; attempt < maxAttempts; attempt++) {@@ -129,7 +133,7 @@ async function searchWithRetry(const isLastAttempt = attempt === maxAttempts - 1;if (!isLastAttempt) {- const base = SEARCH_RETRY_BACKOFF_MS[attempt];+ const base = SEARCH_EMPTY_RETRY_BACKOFF_MS[attempt];const jitter = Math.floor(Math.random() * 250);logger.info("search monitor query empty; retrying", {query: args.query,diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex 44c7623ef..a6e5837b7 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -230,6 +230,10 @@ export const updateMonitorSchema = createMonitorBaseSchema// default would materialize {} for a PATCH that omits notification and wipe// the stored email config. No default => omitted means "leave unchanged".notification: monitorNotificationInner.optional(),+ // Same default-materialization trap: createMonitorBaseSchema gives+ // retentionDays .default(30), so a partial PATCH that omits it would parse to+ // 30 and silently reset a user's configured retention. Drop the default here.+ retentionDays: z.number().int().positive().max(365).optional(),}).superRefine(rejectGoalClearedWithSearchTargets).refine(x => Object.keys(x).length > 0, "Update body cannot be empty");