Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Bound inline search run with a wall-clock budget

Developers Digest801bffffa372
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index d3899f48f..65547194f 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -422,3 +422,28 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
expect(result.degradedReason).toMatch(/judged|incomplete/i);
});
});
+
+describe("run time budget (a single check can't wedge the consumer)", () => {
+ it("stops waiting on a hung pre-scrape and returns skipped+degraded", async () => {
+ vi.useFakeTimers();
+ try {
+ searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
+ // Scrapes never resolve — the pre-scrape can't finish on its own.
+ scrapeURLMock.mockReturnValue(new Promise(() => {}));
+
+ const pending = runSearchTarget(runParams({ depth: "deep" }));
+ // Advance past the 4-minute run budget so the deadline fires.
+ await vi.advanceTimersByTimeAsync(4 * 60 * 1000 + 1000);
+ const result = await pending;
+
+ expect(result.matches).toBe(0);
+ expect(result.resultsJudged).toBe(0);
+ expect(result.judgeDegraded).toBe(true);
+ const skipped = result.pageUpserts.filter(u => u.status === "skipped");
+ expect(skipped).toHaveLength(2);
+ expect(skipped[0].metadata.searchStatus).toBe("skipped");
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index d3e3a8ff2..ad526bbe4 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -70,6 +70,32 @@ function windowToTbs(window: string): string {
// a query that succeeds moments later.
const SEARCH_RETRY_BACKOFF_MS = [400, 1200, 3000, 6000] as const;
+// Hard wall-clock budget for a single inline search run. The deep-mode page
+// scrapes are the only unbounded-ish step (each is capped at 20s, but a large
+// candidate set could still add up); if the concurrent pre-scrape blows past
+// this, we stop waiting and let the remaining candidates fall through as
+// skipped/degraded rather than holding the consumer. Kept well under the
+// 10-minute search stale timeout so a run always returns before the reaper.
+const SEARCH_RUN_BUDGET_MS = 4 * 60 * 1000;
+
+// Race a promise against a wall-clock deadline. Resolves false if the work
+// finishes first, true if the deadline hits. Never rejects; the underlying work
+// keeps running (bounded by its own per-scrape timeout) and populates its cache.
+async function raceDeadline(
+ work: Promise<unknown>,
+ budgetMs: number,
+): Promise<boolean> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const timedOut = await Promise.race([
+ work.then(() => false),
+ new Promise<boolean>(resolve => {
+ timer = setTimeout(() => resolve(true), Math.max(0, budgetMs));
+ }),
+ ]);
+ if (timer) clearTimeout(timer);
+ return timedOut;
+}
+
async function searchWithRetry(
args: Parameters<typeof search>[0],
logger: Logger,
@@ -217,6 +243,7 @@ export async function runSearchTarget(params: {
}): Promise<SearchTargetRunResult> {
const { target, knownPages, goalVersion, logger } = params;
const judgeEnabled = params.monitor.judgeEnabled;
+ const runStart = Date.now();
const goal = params.monitor.goal?.trim();
if (judgeEnabled && !goal) {
@@ -402,13 +429,17 @@ export async function runSearchTarget(params: {
string,
{ doc: ScrapeSearchResult | null; error?: unknown }
>();
+ // Set if the concurrent pre-scrape exceeds the run budget: any candidate not
+ // yet cached is then treated as a scrape failure instead of being inline-scraped,
+ // so the loop returns promptly rather than holding the consumer.
+ let budgetExceeded = false;
if (depth === "deep" && llmStagesAvailable) {
const judgePrompt = buildJudgePrompt(
judgeGoal,
subject,
target.searchWindow,
);
- await Promise.all(
+ const preScrape = Promise.all(
selected
.filter(
c =>
@@ -426,6 +457,19 @@ export async function runSearchTarget(params: {
}
}),
);
+ const remainingBudget = SEARCH_RUN_BUDGET_MS - (Date.now() - runStart);
+ budgetExceeded = await raceDeadline(preScrape, remainingBudget);
+ if (budgetExceeded) {
+ logger.warn(
+ "search monitor run exceeded its time budget; finishing with partial scrapes",
+ {
+ monitorId: params.monitor.id,
+ targetId: target.id,
+ budgetMs: SEARCH_RUN_BUDGET_MS,
+ scraped: deepDocs.size,
+ },
+ );
+ }
}
for (const c of selected) {
@@ -574,6 +618,14 @@ export async function runSearchTarget(params: {
);
continue;
}
+ // The pre-scrape ran out of time before caching this page. Treat it as a
+ // scrape failure rather than inline-scraping (which would re-incur the time
+ // we just budgeted out of), so the run returns promptly.
+ if (!cached && budgetExceeded) {
+ judgeScrapeFailures += 1;
+ pushUnevaluatedPage("run budget exceeded before scrape");
+ continue;
+ }
try {
doc = cached
? cached.doc