Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v0): better usage tracking
Gergő Móricz5cd20c828fb7
diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.tsindex a454f700c..ff8b88040 100644--- a/apps/api/src/controllers/v0/crawl-cancel.ts+++ b/apps/api/src/controllers/v0/crawl-cancel.ts@@ -26,6 +26,9 @@ export async function crawlCancelController(req: Request, res: Response) {redisEvictConnection.sadd("teams_using_v0", team_id).catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));++ redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + req.params.jobId + ":cancel")+ .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));const sc = await getCrawl(req.params.jobId);if (!sc) {diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 9f6e5b7bb..78c2ae0b9 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -87,6 +87,9 @@ export async function crawlStatusController(req: Request, res: Response) {redisEvictConnection.sadd("teams_using_v0", team_id).catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));++ redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + req.params.jobId + ":status")+ .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));const sc = await getCrawl(req.params.jobId);if (!sc) {diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex abd858a91..63b90e809 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -45,8 +45,13 @@ export async function crawlController(req: Request, res: Response) {return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." });}+ const id = uuidv4();+redisEvictConnection.sadd("teams_using_v0", team_id).catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));++ redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + id)+ .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));if (req.headers["x-idempotency-key"]) {const isIdempotencyValid = await validateIdempotencyKey(req);@@ -154,8 +159,6 @@ export async function crawlController(req: Request, res: Response) {// }// }- const id = uuidv4();-await logCrawl(id, team_id);const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(diff --git a/apps/api/src/controllers/v0/keyAuth.ts b/apps/api/src/controllers/v0/keyAuth.tsindex febac4115..87179e19f 100644--- a/apps/api/src/controllers/v0/keyAuth.ts+++ b/apps/api/src/controllers/v0/keyAuth.ts@@ -19,6 +19,9 @@ export const keyAuthController = async (req: Request, res: Response) => {redisEvictConnection.sadd("teams_using_v0", auth.team_id).catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id: auth.team_id }));++ redisEvictConnection.sadd("teams_using_v0:" + auth.team_id, "keyAuth")+ .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id: auth.team_id }));// if success, return success: truereturn res.status(200).json({ success: true });diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex d4bd73bbc..3e02e4e3d 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -181,9 +181,14 @@ export async function scrapeController(req: Request, res: Response) {return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." });}+ const jobId = uuidv4();+redisEvictConnection.sadd("teams_using_v0", team_id).catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));+ redisEvictConnection.sadd("teams_using_v0:" + team_id, "scrape:" + jobId)+ .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));+const crawlerOptions = req.body.crawlerOptions ?? {};const pageOptions = { ...defaultPageOptions, ...req.body.pageOptions };const extractorOptions = {@@ -228,8 +233,6 @@ export async function scrapeController(req: Request, res: Response) {});}- const jobId = uuidv4();-const startTime = new Date().getTime();const result = await scrapeHelper(jobId,diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex aaff178c2..2341bac45 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -174,9 +174,14 @@ export async function searchController(req: Request, res: Response) {return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." });}+ const jobId = uuidv4();+redisEvictConnection.sadd("teams_using_v0", team_id).catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));+ redisEvictConnection.sadd("teams_using_v0:" + team_id, "search:" + jobId)+ .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));+const crawlerOptions = req.body.crawlerOptions ?? {};const pageOptions = req.body.pageOptions ?? {includeHtml: req.body.pageOptions?.includeHtml ?? false,@@ -189,8 +194,6 @@ export async function searchController(req: Request, res: Response) {const searchOptions = req.body.searchOptions ?? { limit: 5 };- const jobId = uuidv4();-try {const { success: creditsCheckSuccess, message: creditsCheckMessage } =await checkTeamCredits(chunk, team_id, 1);