Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): noq concurrency check integration
delong319ab92b404ed
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex aca00a019..9aba82ead 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -102,7 +102,7 @@ export async function scrapeController(req.acuc?.concurrency || 1,aborter.signal,timeout ?? 60_000,- async () => {+ async limited => {const jobPriority = await getJobPriority({team_id: req.auth.team_id,basePriority: 10,@@ -142,6 +142,7 @@ export async function scrapeController(startTime: controllerStartTime,zeroDataRetention: zeroDataRetention ?? false,apiKeyId: req.acuc?.api_key_id ?? null,+ concurrencyLimited: limited,},};diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex bbc80c1ca..e75b32f77 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -136,7 +136,7 @@ export async function scrapeController(req.acuc?.concurrency || 1,aborter.signal,timeout ?? 60_000,- async () => {+ async limited => {const jobPriority = await getJobPriority({team_id: req.auth.team_id,basePriority: 10,@@ -195,6 +195,7 @@ export async function scrapeController(startTime: controllerStartTime,zeroDataRetention,apiKeyId: req.acuc?.api_key_id ?? null,+ concurrencyLimited: limited,},};diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 56792fe13..301fc53ff 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -54,7 +54,10 @@ export async function pushConcurrencyLimitActiveJob(await getRedisConnection().zadd(constructKey(team_id), now + timeout, id);}-async function removeConcurrencyLimitActiveJob(team_id: string, id: string) {+export async function removeConcurrencyLimitActiveJob(+ team_id: string,+ id: string,+) {await getRedisConnection().zrem(constructKey(team_id), id);}diff --git a/apps/api/src/services/worker/team-semaphore.ts b/apps/api/src/services/worker/team-semaphore.tsindex 169c11470..923bd9f81 100644--- a/apps/api/src/services/worker/team-semaphore.ts+++ b/apps/api/src/services/worker/team-semaphore.ts@@ -1,3 +1,7 @@+import {+ pushConcurrencyLimitActiveJob,+ removeConcurrencyLimitActiveJob,+} from "../../lib/concurrency-limit";import { isSelfHosted } from "../../lib/deployment";import { ScrapeJobTimeoutError, TransportableError } from "../../lib/error";import { logger as _logger } from "../../lib/logger";@@ -12,10 +16,17 @@ const activeSemaphores = new Gauge({const semaphoreAcquireDuration = new Histogram({name: "noq_semaphore_acquire_duration_seconds",help: "Semaphore acquire time",- // Buckets in seconds: 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10sbuckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],});+const semaphoreHoldDuration = new Histogram({+ name: "noq_semaphore_hold_duration_seconds",+ help: "Semaphore hold time",+ buckets: [+ 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 180, 240, 300, 600,+ ],+});+const { scripts, runScript, ensure } = nuqRedis;const SEMAPHORE_TTL = 30 * 1000;@@ -61,7 +72,6 @@ async function acquireBlocking(let totalRemoved = 0;let failedOnce = false;- // Start the timer for the histogramconst endTimer = semaphoreAcquireDuration.startTimer();do {@@ -84,7 +94,6 @@ async function acquireBlocking(totalRemoved++;if (granted === 1) {- // Stop the timer and record the durationendTimer();return { limited: failedOnce, removed: totalRemoved };}@@ -175,15 +184,26 @@ async function withSemaphore<T>(signal,});+ const endTimer = semaphoreHoldDuration.startTimer();const hb = startHeartbeat(teamId, holderId, SEMAPHORE_TTL / 2);activeSemaphores.inc();try {+ await pushConcurrencyLimitActiveJob(teamId, holderId, 60 * 1000);+const result = await Promise.race([func(limited), hb.promise]);return result;} finally {+ await removeConcurrencyLimitActiveJob(teamId, holderId).catch(() => {+ _logger.warn("Failed to remove concurrency limit active job", {+ teamId,+ jobId: holderId,+ });+ });+activeSemaphores.dec();hb.stop();+ endTimer();await release(teamId, holderId).catch(() => {});}