Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): concurrency limit info in scrape metadata (#2497)
delong3f40d19d8cf38
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex d78c4af80..47e850017 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -97,8 +97,12 @@ export async function scrapeController(0,);+ let lockTime: number | null = null;+ let concurrencyLimited: boolean = false;+let timeoutHandle: NodeJS.Timeout | null = null;let doc: Document | null = null;+try {const lockStart = Date.now();const aborter = new AbortController();@@ -121,11 +125,13 @@ export async function scrapeController(basePriority: 10,});- const lockTime = Date.now() - lockStart;+ lockTime = Date.now() - lockStart;+ concurrencyLimited = limited;logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {teamId: req.auth.team_id,lockTime,+ limited,});const job: NuQJob<ScrapeJobData> = {@@ -241,11 +247,22 @@ export async function scrapeController(totalWait,usedLlm,formats: req.body.formats,+ concurrencyLimited,+ concurrencyQueueDurationMs: lockTime || undefined,});return res.status(200).json({success: true,- data: doc!,+ data: {+ ...doc!,+ metadata: {+ ...doc!.metadata,+ concurrencyLimited,+ concurrencyQueueDurationMs: concurrencyLimited+ ? lockTime || 0+ : undefined,+ },+ },scrape_id: origin?.includes("website") ? jobId : undefined,});}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex c5f70f5e2..1768632d3 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1054,6 +1054,8 @@ export type Document = {creditsUsed?: number;postprocessorsUsed?: string[];indexId?: string; // ID used to store the document in the index (GCS)+ concurrencyLimited?: boolean;+ concurrencyQueueDurationMs?: number;// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;};serpResults?: {diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex af2d5556d..78c005902 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -130,8 +130,12 @@ export async function scrapeController(0,);- let doc: Document | null = null;+ let lockTime: number | null = null;+ let concurrencyLimited: boolean = false;+let timeoutHandle: NodeJS.Timeout | null = null;+ let doc: Document | null = null;+try {const lockStart = Date.now();const aborter = new AbortController();@@ -155,12 +159,13 @@ export async function scrapeController(basePriority: 10,});- // TODO: send 429 on abort- const lockTime = Date.now() - lockStart;+ lockTime = Date.now() - lockStart;+ concurrencyLimited = limited;logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {teamId: req.auth.team_id,lockTime,+ limited,});// Wait for job completion span@@ -339,11 +344,22 @@ export async function scrapeController(totalWait,usedLlm,formats,+ concurrencyLimited,+ concurrencyQueueDurationMs: lockTime || undefined,});return res.status(200).json({success: true,- data: doc!,+ data: {+ ...doc!,+ metadata: {+ ...doc!.metadata,+ concurrencyLimited,+ concurrencyQueueDurationMs: concurrencyLimited+ ? lockTime || 0+ : undefined,+ },+ },scrape_id: origin?.includes("website") ? jobId : undefined,});},diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex f03d4d146..c3ff9869c 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -991,6 +991,8 @@ export type Document = {creditsUsed?: number;postprocessorsUsed?: string[];indexId?: string; // ID used to store the document in the index (GCS)+ concurrencyLimited?: boolean;+ concurrencyQueueDurationMs?: number;// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;};serpResults?: {