Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): tally system rework (#2430)
Gergő Móriczc9ab3ad696d4
diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.tsindex fbaeb6ae3..69d2217f0 100644--- a/apps/api/src/controllers/v0/admin/metrics.ts+++ b/apps/api/src/controllers/v0/admin/metrics.ts@@ -38,6 +38,11 @@ ${Object.entries(metrics)`concurrency_limit_queue_job_count{team_id="${key}"} ${value}`,).join("\n")}++# HELP billed_teams_count The number of teams that have been billed but not yet tallied+# TYPE billed_teams_count gauge+billed_teams_count ${await getRedisConnection().scard("billed_teams")}+${nuqGetLocalMetrics()}${semaphoreMetrics}`);}diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex f227f676c..b73d2b67a 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -319,6 +319,8 @@ async function supaBillTeam(return { success: false, error };}+ await getRedisConnection().sadd("billed_teams", team_id);+// Update cached ACUC to reflect the new credit usage(async () => {for (const apiKey of (data ?? []).map(x => x.api_key)) {diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex b78be69f6..980923624 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -39,6 +39,7 @@ import { BullMQOtel } from "bullmq-otel";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import { crawlGroup } from "../worker/nuq";import { getACUCTeam } from "../../controllers/auth";+import { supabase_service } from "../supabase";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;const workerStalledCheckInterval =@@ -646,6 +647,38 @@ const workerFun = async (process.exit(0);};+async function tallyBilling() {+ const logger = _logger.child({+ module: "index-worker",+ method: "tallyBilling",+ });+ // get up to 100 teams and remove them from set+ const billedTeams = await getRedisConnection().srandmember(+ "billed_teams",+ 100,+ );+ await getRedisConnection().srem("billed_teams", billedTeams);+ logger.info("Starting to update tallies", {+ billedTeams: billedTeams.length,+ });++ for (const teamId of billedTeams) {+ logger.info("Updating tally for team", { teamId });++ const { error } = await supabase_service.rpc("update_tally_5_team", {+ i_team_id: teamId,+ });++ if (error) {+ logger.warn("Failed to update tally for team", { teamId, error });+ } else {+ logger.info("Updated tally for team", { teamId });+ }+ }++ logger.info("Finished updating tallies");+}+const INDEX_INSERT_INTERVAL = 3000;const WEBHOOK_INSERT_INTERVAL = 15000;const OMCE_INSERT_INTERVAL = 5000;@@ -734,6 +767,16 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;);}, DOMAIN_FREQUENCY_INTERVAL);+ const billingTallyInterval = setInterval(+ async () => {+ if (isShuttingDown) {+ return;+ }+ await tallyBilling();+ },+ 5 * 60 * 1000,+ );+// Search indexing is now handled by separate search service// The search service has its own worker that processes the queue// This worker no longer needs to process search index jobs@@ -763,6 +806,7 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;clearInterval(indexRFInserterInterval);clearInterval(omceInserterInterval);clearInterval(domainFrequencyInterval);+ clearInterval(billingTallyInterval);logger.info("All workers shut down, exiting process");})();