Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): noq concurrency check integration

delong319ab92b404ed
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index aca00a019..9aba82ead 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -102,7 +102,7 @@ export async function scrapeController(
req.acuc?.concurrency || 1,
aborter.signal,
timeout ?? 60_000,
- async () => {
+ async limited => {
const jobPriority = await getJobPriority({
team_id: req.auth.team_id,
basePriority: 10,
@@ -142,6 +142,7 @@ export async function scrapeController(
startTime: controllerStartTime,
zeroDataRetention: zeroDataRetention ?? false,
apiKeyId: req.acuc?.api_key_id ?? null,
+ concurrencyLimited: limited,
},
};
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index bbc80c1ca..e75b32f77 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -136,7 +136,7 @@ export async function scrapeController(
req.acuc?.concurrency || 1,
aborter.signal,
timeout ?? 60_000,
- async () => {
+ async limited => {
const jobPriority = await getJobPriority({
team_id: req.auth.team_id,
basePriority: 10,
@@ -195,6 +195,7 @@ export async function scrapeController(
startTime: controllerStartTime,
zeroDataRetention,
apiKeyId: req.acuc?.api_key_id ?? null,
+ concurrencyLimited: limited,
},
};
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 56792fe13..301fc53ff 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -54,7 +54,10 @@ export async function pushConcurrencyLimitActiveJob(
await getRedisConnection().zadd(constructKey(team_id), now + timeout, id);
}
-async function removeConcurrencyLimitActiveJob(team_id: string, id: string) {
+export async function removeConcurrencyLimitActiveJob(
+ team_id: string,
+ id: string,
+) {
await getRedisConnection().zrem(constructKey(team_id), id);
}
diff --git a/apps/api/src/services/worker/team-semaphore.ts b/apps/api/src/services/worker/team-semaphore.ts
index 169c11470..923bd9f81 100644
--- a/apps/api/src/services/worker/team-semaphore.ts
+++ b/apps/api/src/services/worker/team-semaphore.ts
@@ -1,3 +1,7 @@
+import {
+ pushConcurrencyLimitActiveJob,
+ removeConcurrencyLimitActiveJob,
+} from "../../lib/concurrency-limit";
import { isSelfHosted } from "../../lib/deployment";
import { ScrapeJobTimeoutError, TransportableError } from "../../lib/error";
import { logger as _logger } from "../../lib/logger";
@@ -12,10 +16,17 @@ const activeSemaphores = new Gauge({
const semaphoreAcquireDuration = new Histogram({
name: "noq_semaphore_acquire_duration_seconds",
help: "Semaphore acquire time",
- // Buckets in seconds: 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
});
+const semaphoreHoldDuration = new Histogram({
+ name: "noq_semaphore_hold_duration_seconds",
+ help: "Semaphore hold time",
+ buckets: [
+ 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 180, 240, 300, 600,
+ ],
+});
+
const { scripts, runScript, ensure } = nuqRedis;
const SEMAPHORE_TTL = 30 * 1000;
@@ -61,7 +72,6 @@ async function acquireBlocking(
let totalRemoved = 0;
let failedOnce = false;
- // Start the timer for the histogram
const endTimer = semaphoreAcquireDuration.startTimer();
do {
@@ -84,7 +94,6 @@ async function acquireBlocking(
totalRemoved++;
if (granted === 1) {
- // Stop the timer and record the duration
endTimer();
return { limited: failedOnce, removed: totalRemoved };
}
@@ -175,15 +184,26 @@ async function withSemaphore<T>(
signal,
});
+ const endTimer = semaphoreHoldDuration.startTimer();
const hb = startHeartbeat(teamId, holderId, SEMAPHORE_TTL / 2);
activeSemaphores.inc();
try {
+ await pushConcurrencyLimitActiveJob(teamId, holderId, 60 * 1000);
+
const result = await Promise.race([func(limited), hb.promise]);
return result;
} finally {
+ await removeConcurrencyLimitActiveJob(teamId, holderId).catch(() => {
+ _logger.warn("Failed to remove concurrency limit active job", {
+ teamId,
+ jobId: holderId,
+ });
+ });
+
activeSemaphores.dec();
hb.stop();
+ endTimer();
await release(teamId, holderId).catch(() => {});
}