Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): let fdb wait use caller timeout
mogery1d58410c6bd3
diff --git a/apps/api/src/__tests__/nuq-fdb/router.test.ts b/apps/api/src/__tests__/nuq-fdb/router.test.tsindex 5c5a9fdb7..2d4741b78 100644--- a/apps/api/src/__tests__/nuq-fdb/router.test.ts+++ b/apps/api/src/__tests__/nuq-fdb/router.test.ts@@ -110,6 +110,45 @@ describeIf("NuQ router (forced FDB mode)", () => {expect(jobsRead.length).toBe(1);});+ test("optional FDB waitForJob uses caller timeout, not the quick optional-op timeout", async () => {+ const forcedBackend = config.NUQ_BACKEND;+ config.NUQ_BACKEND = "pg";+ try {+ const teamId = randomUUID();+ const jobId = randomUUID();+ const { jobs, backloggedCount } = await fdbEnqueueScrapeJobs(+ [+ {+ jobId,+ data: {+ mode: "single_urls",+ url: "https://example.com",+ team_id: teamId,+ } as any,+ priority: 0,+ listenable: true,+ backlogTimeoutMs: 60_000,+ },+ ],+ teamId,+ );+ expect(jobs[0].backend).toBe("fdb");+ expect(backloggedCount).toBe(0);++ const wait = scrapeQueue.waitForJob(jobId, 15_000);++ await new Promise(resolve => setTimeout(resolve, 800));++ const taken = await scrapeQueueFdb.getJobToProcess();+ expect(taken?.id).toBe(jobId);+ await scrapeQueueFdb.jobFinish(jobId, taken!.lock!, { ok: true });++ await expect(wait).resolves.toBeDefined();+ } finally {+ config.NUQ_BACKEND = forcedBackend;+ }+ });+test("routed group lifecycle incl. crawl_finished consumption and cancel", async () => {const teamId = randomUUID();const gid = randomUUID();diff --git a/apps/api/src/services/worker/nuq-router.ts b/apps/api/src/services/worker/nuq-router.tsindex fb0e14265..8748faf14 100644--- a/apps/api/src/services/worker/nuq-router.ts+++ b/apps/api/src/services/worker/nuq-router.ts@@ -518,7 +518,10 @@ class RoutedScrapeQueue {logger: Logger = _logger,): Promise<T> {if ((await getJobQueueBackend(id)) === "fdb") {- return optionalFdb(() => scrapeQueueFdb.waitForJob(id, timeout, logger));+ // Waiting is intentionally long-lived; callers pass the real scrape+ // timeout. optionalFdb is only for quick FDB reads/writes and applies a+ // 500ms guard, which would incorrectly fail synchronous FDB-backed jobs.+ return scrapeQueueFdb.waitForJob(id, timeout, logger);}return scrapeQueuePg.waitForJob(id, timeout, logger) as Promise<T>;}