Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(concurrency-limit): set limit based on plan
Gergő Móricz3621e191bd7e
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex a95c85a68..3ebee9768 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -171,7 +171,8 @@ export async function crawlController(req: Request, res: Response) {url,mode: "single_urls",crawlerOptions: crawlerOptions,- team_id: team_id,+ team_id,+ plan,pageOptions: pageOptions,origin: req.body.origin ?? defaultOrigin,crawl_id: id,@@ -211,7 +212,8 @@ export async function crawlController(req: Request, res: Response) {url,mode: "single_urls",crawlerOptions: crawlerOptions,- team_id: team_id,+ team_id,+ plan,pageOptions: pageOptions,origin: req.body.origin ?? defaultOrigin,crawl_id: id,diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.tsindex f8706867a..bceb1df94 100644--- a/apps/api/src/controllers/v0/crawlPreview.ts+++ b/apps/api/src/controllers/v0/crawlPreview.ts@@ -107,7 +107,8 @@ export async function crawlPreviewController(req: Request, res: Response) {url,mode: "single_urls",crawlerOptions: crawlerOptions,- team_id: team_id,+ team_id,+ plan,pageOptions: pageOptions,origin: "website-preview",crawl_id: id,@@ -121,7 +122,8 @@ export async function crawlPreviewController(req: Request, res: Response) {url,mode: "single_urls",crawlerOptions: crawlerOptions,- team_id: team_id,+ team_id,+ plan,pageOptions: pageOptions,origin: "website-preview",crawl_id: id,diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex db304fed1..cfb1366c9 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -60,6 +60,7 @@ export async function scrapeHelper(mode: "single_urls",crawlerOptions,team_id,+ plan,pageOptions,extractorOptions,origin: req.body.origin ?? defaultOrigin,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex e0883fa8e..4efe279ae 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -106,6 +106,7 @@ export async function crawlController(url,mode: "single_urls",team_id: req.auth.team_id,+ plan: req.auth.plan,crawlerOptions,pageOptions,origin: "api",@@ -138,6 +139,7 @@ export async function crawlController(mode: "single_urls",crawlerOptions: crawlerOptions,team_id: req.auth.team_id,+ plan: req.auth.plan,pageOptions: pageOptions,origin: "api",crawl_id: id,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 419749176..753848769 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -44,6 +44,7 @@ export async function scrapeController(mode: "single_urls",crawlerOptions: {},team_id: req.auth.team_id,+ plan: req.auth.plan,pageOptions,extractorOptions,origin: req.body.origin,diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 7468a0500..050d672de 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -34,9 +34,10 @@ import {deleteJobPriority,getJobPriority,} from "../../src/lib/job-priority";-import { PlanType } from "../types";+import { PlanType, RateLimiterMode } from "../types";import { getJobs } from "../../src/controllers/v1/crawl-status";import { configDotenv } from "dotenv";+import { getRateLimiterPoints } from "./rate-limiter";configDotenv();if (process.env.ENV === "production") {@@ -131,9 +132,9 @@ const workerFun = async (if (job) {const concurrencyLimiterKey = "concurrency-limiter:" + job.data?.team_id;- if (job.data && job.data.team_id) {+ if (job.data && job.data.team_id && job.data.plan) {const concurrencyLimiterThrottledKey = "concurrency-limiter:" + job.data.team_id + ":throttled";- const concurrencyLimit = 10; // TODO: determine based on price id+ const concurrencyLimit = getRateLimiterPoints(RateLimiterMode.Scrape, undefined, job.data.plan);const now = Date.now();const stalledJobTimeoutMs = 2 * 60 * 1000;const throttledJobTimeoutMs = 10 * 60 * 1000;@@ -382,6 +383,7 @@ async function processJob(job: Job, token: string) {mode: "single_urls",crawlerOptions: sc.crawlerOptions,team_id: sc.team_id,+ plan: job.data.plan,pageOptions: sc.pageOptions,origin: job.data.origin,crawl_id: job.data.crawl_id,diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex 51a0ecfaa..e0fc5646b 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -123,14 +123,18 @@ const testSuiteTokens = ["a01ccae", "6254cf9", "0f96e673", "23befa1b", "69141c4"const manual = ["69be9e74-7624-4990-b20d-08e0acc70cf6"];-export function getRateLimiter(+function makePlanKey(plan?: string) {+ return plan ? plan.replace("-", "") : "default"; // "default"+}++export function getRateLimiterPoints(mode: RateLimiterMode,- token: string,+ token?: string,plan?: string,teamId?: string) {- if (testSuiteTokens.some(testToken => token.includes(testToken))) {+ if (token && testSuiteTokens.some(testToken => token.includes(testToken))) {return testSuiteRateLimiter;}@@ -146,9 +150,17 @@ export function getRateLimiter(if (!rateLimitConfig) return serverRateLimiter;- const planKey = plan ? plan.replace("-", "") : "default"; // "default"const points =- rateLimitConfig[planKey] || rateLimitConfig.default || rateLimitConfig; // 5+ rateLimitConfig[makePlanKey(plan)] || rateLimitConfig.default || rateLimitConfig; // 5++ return points;+}- return createRateLimiter(`${mode}-${planKey}`, points);+export function getRateLimiter(+ mode: RateLimiterMode,+ token?: string,+ plan?: string,+ teamId?: string+) {+ return createRateLimiter(`${mode}-${makePlanKey(plan)}`, getRateLimiterPoints(mode, token, plan, teamId));}diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 3795ce1e0..a03176dab 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -28,6 +28,7 @@ export interface WebScraperOptions {pageOptions: any;extractorOptions?: any;team_id: string;+ plan: string;origin?: string;crawl_id?: string;sitemapped?: boolean;