Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixed more stuff
Nicolas18f9cd09e175
diff --git a/apps/api/src/controllers/v1/concurrency-check.ts b/apps/api/src/controllers/v1/concurrency-check.tsnew file mode 100644index 000000000..6ed4fa551--- /dev/null+++ b/apps/api/src/controllers/v1/concurrency-check.ts@@ -0,0 +1,25 @@+import { authenticateUser } from "../auth";+import {+ ConcurrencyCheckParams,+ ConcurrencyCheckResponse,+ RequestWithAuth,+} from "./types";+import { RateLimiterMode } from "../../types";+import { Response } from "express";+import { redisConnection } from "../../services/queue-service";+// Basically just middleware and error wrapping+export async function concurrencyCheckController(+ req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>,+ res: Response<ConcurrencyCheckResponse>+) {+ const concurrencyLimiterKey = "concurrency-limiter:" + req.params.teamId;+ const now = Date.now();+ const activeJobsOfTeam = await redisConnection.zrangebyscore(+ concurrencyLimiterKey,+ now,+ Infinity+ );+ return res+ .status(200)+ .json({ success: true, concurrency: activeJobsOfTeam.length });+}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 45db51b5b..3781eb78a 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -294,6 +294,17 @@ export type CrawlStatusParams = {jobId: string;};+export type ConcurrencyCheckParams = {+ teamId: string;+};++export type ConcurrencyCheckResponse =+ | ErrorResponse+ | {+ success: true;+ concurrency: number;+ };+export type CrawlStatusResponse =| ErrorResponse| {diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 49a41ce78..b0ceceb40 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -16,6 +16,7 @@ import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";import { crawlCancelController } from "../controllers/v1/crawl-cancel";import { Logger } from "../lib/logger";import { scrapeStatusController } from "../controllers/v1/scrape-status";+import { concurrencyCheckController } from "../controllers/v1/concurrency-check";// import { crawlPreviewController } from "../../src/controllers/v1/crawlPreview";// import { crawlJobStatusPreviewController } from "../../src/controllers/v1/status";// import { searchController } from "../../src/controllers/v1/search";@@ -140,11 +141,19 @@ v1Router.get(wrap(scrapeStatusController));+v1Router.get(+ "/concurrency-check",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(concurrencyCheckController)+);+v1Router.ws("/crawl/:jobId",crawlStatusWSController);++// v1Router.post("/crawlWebsitePreview", crawlPreviewController);diff --git a/apps/api/src/services/rate-limiter.test.ts b/apps/api/src/services/rate-limiter.test.tsindex 3e2523016..ba4a0a733 100644--- a/apps/api/src/services/rate-limiter.test.ts+++ b/apps/api/src/services/rate-limiter.test.ts@@ -49,7 +49,7 @@ describe("Rate Limiter Service", () => {"nonexistent" as RateLimiterMode,"test-prefix:someToken");- expect(limiter).toBe(serverRateLimiter);+ expect(limiter.points).toBe(serverRateLimiter.points);});it("should return the correct rate limiter based on mode and plan", () => {@@ -210,7 +210,7 @@ describe("Rate Limiter Service", () => {"test-prefix:someToken","starter");- expect(limiter2.points).toBe(3);+ expect(limiter2.points).toBe(10);const limiter3 = getRateLimiter("crawl" as RateLimiterMode,@@ -233,7 +233,7 @@ describe("Rate Limiter Service", () => {"test-prefix:someToken","starter");- expect(limiter2.points).toBe(20);+ expect(limiter2.points).toBe(100);const limiter3 = getRateLimiter("scrape" as RateLimiterMode,@@ -263,14 +263,14 @@ describe("Rate Limiter Service", () => {"test-prefix:someToken","starter");- expect(limiter2.points).toBe(20);+ expect(limiter2.points).toBe(50);const limiter3 = getRateLimiter("search" as RateLimiterMode,"test-prefix:someToken","standard");- expect(limiter3.points).toBe(40);+ expect(limiter3.points).toBe(50);});it("should return the correct rate limiter for 'preview' mode", () => {diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex e0fc5646b..21e059487 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -132,8 +132,22 @@ export function getRateLimiterPoints(token?: string,plan?: string,teamId?: string-) {+) : number {+ const rateLimitConfig = RATE_LIMITS[mode]; // {default : 5}++ if (!rateLimitConfig) return RATE_LIMITS.account.default;+ const points : number =+ rateLimitConfig[makePlanKey(plan)] || rateLimitConfig.default; // 5+ return points;+}++export function getRateLimiter(+ mode: RateLimiterMode,+ token?: string,+ plan?: string,+ teamId?: string+ ) : RateLimiterRedis {if (token && testSuiteTokens.some(testToken => token.includes(testToken))) {return testSuiteRateLimiter;}@@ -145,22 +159,6 @@ export function getRateLimiterPoints(if(teamId && manual.includes(teamId)) {return manualRateLimiter;}-- const rateLimitConfig = RATE_LIMITS[mode]; // {default : 5}-- if (!rateLimitConfig) return serverRateLimiter;-- const points =- rateLimitConfig[makePlanKey(plan)] || rateLimitConfig.default || rateLimitConfig; // 5-- return points;-}--export function getRateLimiter(- mode: RateLimiterMode,- token?: string,- plan?: string,- teamId?: string-) {+return createRateLimiter(`${mode}-${makePlanKey(plan)}`, getRateLimiterPoints(mode, token, plan, teamId));}