Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): concurrency limit info in scrape metadata (#2497)

delong3f40d19d8cf38
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index d78c4af80..47e850017 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -97,8 +97,12 @@ export async function scrapeController(
0,
);
+ let lockTime: number | null = null;
+ let concurrencyLimited: boolean = false;
+
let timeoutHandle: NodeJS.Timeout | null = null;
let doc: Document | null = null;
+
try {
const lockStart = Date.now();
const aborter = new AbortController();
@@ -121,11 +125,13 @@ export async function scrapeController(
basePriority: 10,
});
- const lockTime = Date.now() - lockStart;
+ lockTime = Date.now() - lockStart;
+ concurrencyLimited = limited;
logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {
teamId: req.auth.team_id,
lockTime,
+ limited,
});
const job: NuQJob<ScrapeJobData> = {
@@ -241,11 +247,22 @@ export async function scrapeController(
totalWait,
usedLlm,
formats: req.body.formats,
+ concurrencyLimited,
+ concurrencyQueueDurationMs: lockTime || undefined,
});
return res.status(200).json({
success: true,
- data: doc!,
+ data: {
+ ...doc!,
+ metadata: {
+ ...doc!.metadata,
+ concurrencyLimited,
+ concurrencyQueueDurationMs: concurrencyLimited
+ ? lockTime || 0
+ : undefined,
+ },
+ },
scrape_id: origin?.includes("website") ? jobId : undefined,
});
}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index c5f70f5e2..1768632d3 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1054,6 +1054,8 @@ export type Document = {
creditsUsed?: number;
postprocessorsUsed?: string[];
indexId?: string; // ID used to store the document in the index (GCS)
+ concurrencyLimited?: boolean;
+ concurrencyQueueDurationMs?: number;
// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
};
serpResults?: {
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index af2d5556d..78c005902 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -130,8 +130,12 @@ export async function scrapeController(
0,
);
- let doc: Document | null = null;
+ let lockTime: number | null = null;
+ let concurrencyLimited: boolean = false;
+
let timeoutHandle: NodeJS.Timeout | null = null;
+ let doc: Document | null = null;
+
try {
const lockStart = Date.now();
const aborter = new AbortController();
@@ -155,12 +159,13 @@ export async function scrapeController(
basePriority: 10,
});
- // TODO: send 429 on abort
- const lockTime = Date.now() - lockStart;
+ lockTime = Date.now() - lockStart;
+ concurrencyLimited = limited;
logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {
teamId: req.auth.team_id,
lockTime,
+ limited,
});
// Wait for job completion span
@@ -339,11 +344,22 @@ export async function scrapeController(
totalWait,
usedLlm,
formats,
+ concurrencyLimited,
+ concurrencyQueueDurationMs: lockTime || undefined,
});
return res.status(200).json({
success: true,
- data: doc!,
+ data: {
+ ...doc!,
+ metadata: {
+ ...doc!.metadata,
+ concurrencyLimited,
+ concurrencyQueueDurationMs: concurrencyLimited
+ ? lockTime || 0
+ : undefined,
+ },
+ },
scrape_id: origin?.includes("website") ? jobId : undefined,
});
},
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index f03d4d146..c3ff9869c 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -991,6 +991,8 @@ export type Document = {
creditsUsed?: number;
postprocessorsUsed?: string[];
indexId?: string; // ID used to store the document in the index (GCS)
+ concurrencyLimited?: boolean;
+ concurrencyQueueDurationMs?: number;
// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
};
serpResults?: {