Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrape): add warning to document if it was concurrency limited (#1348)

Gergő Móricz200de9e7e703
diff --git a/apps/api/src/__tests__/snips/billing.test.ts b/apps/api/src/__tests__/snips/billing.test.ts
index 3a4898ef7..8afa5a441 100644
--- a/apps/api/src/__tests__/snips/billing.test.ts
+++ b/apps/api/src/__tests__/snips/billing.test.ts
@@ -97,7 +97,7 @@ describe("Billing tests", () => {
const rc2 = (await creditUsage()).remaining_credits;
expect(rc1 - rc2).toBe(12);
- }, 300000);
+ }, 600000);
it("bills crawl correctly", async () => {
const rc1 = (await creditUsage()).remaining_credits;
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index bbb17d6b1..1ce2211cb 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -84,6 +84,8 @@ async function addScrapeJobRaw(
// logger.error("Error sending notification (concurrency limit reached): ", error);
// });
}
+
+ webScraperOptions.concurrencyLimited = true;
await _addScrapeJobToConcurrencyQueue(
webScraperOptions,
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 96de8d720..52f1f55ae 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -923,6 +923,10 @@ async function processJob(job: Job & { id: string }, token: string) {
delete doc.rawHtml;
}
+ if (job.data.concurrencyLimited) {
+ doc.warning = "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." + (doc.warning ? " " + doc.warning : "");
+ }
+
const data = {
success: true,
result: {