Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrape): add warning to document if it was concurrency limited (#1348)
Gergő Móricz200de9e7e703
diff --git a/apps/api/src/__tests__/snips/billing.test.ts b/apps/api/src/__tests__/snips/billing.test.tsindex 3a4898ef7..8afa5a441 100644--- a/apps/api/src/__tests__/snips/billing.test.ts+++ b/apps/api/src/__tests__/snips/billing.test.ts@@ -97,7 +97,7 @@ describe("Billing tests", () => {const rc2 = (await creditUsage()).remaining_credits;expect(rc1 - rc2).toBe(12);- }, 300000);+ }, 600000);it("bills crawl correctly", async () => {const rc1 = (await creditUsage()).remaining_credits;diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex bbb17d6b1..1ce2211cb 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -84,6 +84,8 @@ async function addScrapeJobRaw(// logger.error("Error sending notification (concurrency limit reached): ", error);// });}++ webScraperOptions.concurrencyLimited = true;await _addScrapeJobToConcurrencyQueue(webScraperOptions,diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 96de8d720..52f1f55ae 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -923,6 +923,10 @@ async function processJob(job: Job & { id: string }, token: string) {delete doc.rawHtml;}+ if (job.data.concurrencyLimited) {+ doc.warning = "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." + (doc.warning ? " " + doc.warning : "");+ }+const data = {success: true,result: {