Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(auto-recharge): enterprise (#2127)
Gergő Móricz87dc3660e99b
diff --git a/apps/api/package.json b/apps/api/package.jsonindex f944d61d9..7c987c830 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -148,13 +148,20 @@},"pnpm": {"onlyBuiltDependencies": [+ "@mendable/firecrawl-rs","@sentry-internal/node-cpu-profiler",+ "@sentry/cli","bigint-buffer","bufferutil",+ "esbuild","keccak",+ "koffi","libpq",+ "msgpackr-extract",+ "protobufjs",+ "supabase","utf-8-validate",- "supabase"+ "wordpos"],"overrides": {"brace-expansion@>=2.0.0 <=2.0.1": "2.0.2"diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex c5c116fd6..598a85682 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -94,6 +94,7 @@ const mockPreviewACUC: (},price_credits: 99999999,price_should_be_graceful: false,+ price_associated_auto_recharge_price_id: null,credits_used: 0,coupon_credits: 99999999,adjusted_credits_used: 0,@@ -133,6 +134,7 @@ const mockACUC: () => AuthCreditUsageChunk = () => ({},price_credits: 99999999,price_should_be_graceful: false,+ price_associated_auto_recharge_price_id: null,credits_used: 0,coupon_credits: 99999999,adjusted_credits_used: 0,@@ -188,7 +190,7 @@ export async function getACUC(const client =Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;({ data, error } = await client.rpc(- "auth_credit_usage_chunk_34",+ "auth_credit_usage_chunk_35",{input_key: api_key,i_is_extract: isExtract,@@ -315,7 +317,7 @@ export async function getACUCTeam(const client =Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;({ data, error } = await client.rpc(- "auth_credit_usage_chunk_34_from_team",+ "auth_credit_usage_chunk_35_from_team",{input_team: team_id,i_is_extract: isExtract,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 2a1e5c7c7..dc3ec0e54 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1065,6 +1065,7 @@ export type AuthCreditUsageChunk = {price_id: string | null;price_credits: number; // credit limit with assoicated price, or free_credits (500) if free planprice_should_be_graceful: boolean;+ price_associated_auto_recharge_price_id: string | null;credits_used: number;coupon_credits: number; // do not rely on this number to be up to date after calling a billTeamadjusted_credits_used: number; // credits this period minus coupons useddiff --git a/apps/api/src/controllers/v2/token-usage.ts b/apps/api/src/controllers/v2/token-usage.tsindex 4711580fc..ec5034cd8 100644--- a/apps/api/src/controllers/v2/token-usage.ts+++ b/apps/api/src/controllers/v2/token-usage.ts@@ -1,7 +1,6 @@import { Response } from "express";import { ErrorResponse, RequestWithAuth } from "./types";-import { getACUC, getACUCTeam } from "../auth";-import { logger } from "../../lib/logger";+import { getACUCTeam } from "../auth";import { RateLimiterMode } from "../../types";interface TokenUsageResponse {diff --git a/apps/api/src/lib/job-priority.ts b/apps/api/src/lib/job-priority.tsindex baa51f86e..9c1d22b8a 100644--- a/apps/api/src/lib/job-priority.ts+++ b/apps/api/src/lib/job-priority.ts@@ -1,5 +1,5 @@import { RateLimiterMode } from "../types";-import { getACUC, getACUCTeam } from "../controllers/auth";+import { getACUCTeam } from "../controllers/auth";import { redisEvictConnection } from "../services/redis";import { logger } from "./logger";diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.tsindex d82227308..c3a9ca9d2 100644--- a/apps/api/src/services/billing/auto_charge.ts+++ b/apps/api/src/services/billing/auto_charge.ts@@ -3,7 +3,11 @@ import { AuthCreditUsageChunk } from "../../controllers/v1/types";import { clearACUC, clearACUCTeam, getACUC } from "../../controllers/auth";import { redlock } from "../redlock";import { supabase_rr_service, supabase_service } from "../supabase";-import { createPaymentIntent } from "./stripe";+import {+ createPaymentIntent,+ createSubscription,+ customerToUserId,+} from "./stripe";import { issueCredits } from "./issue_credits";import {sendNotification,@@ -34,10 +38,307 @@ export async function autoCharge(message: string;remainingCredits: number;chunk: AuthCreditUsageChunk;+}> {+ if (chunk.price_associated_auto_recharge_price_id !== null) {+ return _autoChargeScale(+ chunk as AuthCreditUsageChunk & {+ price_associated_auto_recharge_price_id: string;+ },+ autoRechargeThreshold,+ );+ } else {+ return _autoChargeSelfServe(chunk, autoRechargeThreshold);+ }+}++async function _autoChargeScale(+ chunk: AuthCreditUsageChunk & {+ price_associated_auto_recharge_price_id: string;+ },+ autoRechargeThreshold: number,+): Promise<{+ success: boolean;+ message: string;+ remainingCredits: number;+ chunk: AuthCreditUsageChunk;+}> {+ const logger = _logger.child({+ module: "auto_charge",+ method: "_autoChargeScale",+ team_id: chunk.team_id,+ teamId: chunk.team_id,+ });++ const resource = `auto-recharge:${chunk.team_id}`;++ try {+ return await redlock.using([resource], 5000, async signal => {+ const updatedChunk = await getACUC(chunk.api_key, false, false);++ if (+ updatedChunk &&+ updatedChunk.remaining_credits < autoRechargeThreshold+ ) {+ // Check for recharges this month++ const currentMonth = new Date();+ currentMonth.setUTCDate(1);+ currentMonth.setUTCHours(0, 0, 0, 0);++ const { data: rechargesThisMonth, error: rechargesThisMonthError } =+ await supabase_service+ .from("subscriptions")+ .select("*")+ .eq("team_id", chunk.team_id)+ .eq("metadata->>auto_recharge", "true")+ .gte("current_period_start", currentMonth.toISOString());++ if (rechargesThisMonthError || !rechargesThisMonth) {+ logger.error("Error fetching recharges this month", {+ error: rechargesThisMonthError,+ });+ return {+ success: false,+ message: "Error fetching recharges this month",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ } else if (rechargesThisMonth.length >= 4) {+ logger.warn("Auto-recharge failed: too many recharges this month");+ return {+ success: false,+ message: "Auto-recharge failed: too many recharges this month",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ } else {+ // Actually re-charge++ const { data: price, error: priceError } = await supabase_service+ .from("prices")+ .select("*")+ .eq("id", chunk.price_associated_auto_recharge_price_id)+ .single();+ if (priceError || !price) {+ logger.error("Error fetching price", { error: priceError });+ return {+ success: false,+ message: "Error fetching price",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }++ if (!chunk.sub_user_id) {+ logger.error("No sub_user_id found in chunk");+ return {+ success: false,+ message: "No sub_user_id found in chunk",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }++ const { data: customer, error: customersError } =+ await supabase_rr_service+ .from("customers")+ .select("id, stripe_customer_id")+ .eq("id", chunk.sub_user_id)+ .single();++ if (customersError || !customer) {+ logger.error("Error fetching customer data", {+ error: customersError,+ });+ return {+ success: false,+ message: "Error fetching customer data",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }++ if (!customer.stripe_customer_id) {+ logger.error("No stripe_customer_id found in customer");+ return {+ success: false,+ message: "No stripe_customer_id found in customer",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }++ if (!chunk.sub_id) {+ logger.error("No sub_id found in chunk");+ return {+ success: false,+ message: "No sub_id found in chunk",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }++ const subscription = await createSubscription(+ chunk.team_id,+ customer.stripe_customer_id,+ chunk.price_associated_auto_recharge_price_id,+ chunk.sub_id,+ );+ if (!subscription) {+ logger.error("Failed to create subscription");+ return {+ success: false,+ message: "Failed to create subscription",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }++ const userId = await customerToUserId(customer.stripe_customer_id);+ if (!userId) {+ logger.error("Failed to get user id from customer");+ return {+ success: false,+ message: "Failed to get user id from customer",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }++ // Try to insert it into subscriptions ourselves in case webhook is slow+ const { error: subscriptionError } = await supabase_service+ .from("subscriptions")+ .insert({+ id: subscription.id,+ user_id: userId,+ metadata: subscription.metadata,+ status: subscription.status,+ price_id: chunk.price_associated_auto_recharge_price_id,+ quantity: 1,+ cancel_at_period_end: false,+ cancel_at: null,+ canceled_at: null,+ current_period_start: subscription.current_period_start+ ? new Date(+ subscription.current_period_start * 1000,+ ).toISOString()+ : null,+ current_period_end: subscription.current_period_end+ ? new Date(subscription.current_period_end * 1000).toISOString()+ : null,+ created: subscription.created+ ? new Date(subscription.created * 1000).toISOString()+ : null,+ ended_at: null,+ trial_start: null,+ trial_end: null,+ team_id: chunk.team_id,+ is_extract: false,+ });++ if (subscriptionError) {+ logger.warn(+ "Failed to add subscription to supabase -- maybe we got sniped by the webhook?",+ { error: subscriptionError },+ );+ }++ // Reset ACUC cache to reflect the new credit balance+ await clearACUC(chunk.api_key);+ await clearACUCTeam(chunk.team_id);++ try {+ // Check for frequent auto-recharges in the past week+ const weeklyAutoRechargeKey = `auto-recharge-weekly:${chunk.team_id}`;+ const weeklyRecharges = await redisRateLimitClient.incr(+ weeklyAutoRechargeKey,+ );+ // Set expiry for 7 days if not already set+ await redisRateLimitClient.expire(+ weeklyAutoRechargeKey,+ 7 * 24 * 60 * 60,+ );++ // If this is the second auto-recharge in a week, send notification+ if (weeklyRecharges >= 2) {+ await sendNotificationWithCustomDays(+ chunk.team_id,+ NotificationType.AUTO_RECHARGE_FREQUENT,+ 7, // Send at most once per week+ false,+ );+ }+ } catch (error) {+ logger.error(`Error sending frequent auto-recharge notification`, {+ error,+ });+ }++ await sendNotification(+ chunk.team_id,+ NotificationType.AUTO_RECHARGE_SUCCESS,+ chunk.sub_current_period_start,+ chunk.sub_current_period_end,+ chunk,+ true,+ );++ return {+ success: true,+ message: "Auto-recharge successful",+ remainingCredits:+ (updatedChunk?.remaining_credits ?? chunk.remaining_credits) ++ price.credits,+ chunk: {+ ...(updatedChunk ?? chunk),+ remaining_credits:+ (updatedChunk?.remaining_credits ?? chunk.remaining_credits) ++ price.credits,+ },+ };+ }+ } else {+ return {+ success: false,+ message: "No need to auto-recharge",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }+ });+ } catch (error) {+ logger.error("Auto-recharge failed", { error });+ return {+ success: false,+ message: "Failed to acquire lock for auto-recharge",+ remainingCredits: chunk.remaining_credits,+ chunk,+ };+ }+}++async function _autoChargeSelfServe(+ chunk: AuthCreditUsageChunk,+ autoRechargeThreshold: number,+): Promise<{+ success: boolean;+ message: string;+ remainingCredits: number;+ chunk: AuthCreditUsageChunk;}> {const logger = _logger.child({module: "auto_charge",- method: "autoCharge",+ method: "_autoChargeSelfServe",team_id: chunk.team_id,teamId: chunk.team_id,});diff --git a/apps/api/src/services/billing/stripe.ts b/apps/api/src/services/billing/stripe.tsindex 0d0b17cfe..8a43e53ca 100644--- a/apps/api/src/services/billing/stripe.ts+++ b/apps/api/src/services/billing/stripe.ts@@ -21,6 +21,7 @@ export async function createPaymentIntent(if (!defaultPaymentMethod) {logger.error(`No default payment method found for customer: ${customer_id}`,+ { team_id },);return { return_status: "failed", charge_id: "" };}@@ -33,6 +34,10 @@ export async function createPaymentIntent(payment_method: defaultPaymentMethod?.id,off_session: true,confirm: true,+ metadata: {+ team_id,+ auto_recharge: "true",+ },});if (paymentIntent.status === "succeeded") {@@ -57,3 +62,119 @@ export async function createPaymentIntent(return { return_status: "failed", charge_id: "" };}}++export async function createSubscription(+ team_id: string,+ customer_id: string,+ price_id: string,+ main_subscription_id: string,+) {+ const defaultPaymentMethod =+ await getCustomerDefaultPaymentMethod(customer_id);+ if (!defaultPaymentMethod) {+ logger.error("No default payment method found for customer", {+ team_id,+ customer_id,+ });+ return null;+ }++ let useCoupons: string[] = [];++ const mainSub = await stripe.subscriptions.retrieve(main_subscription_id, {+ expand: ["discounts"],+ });+ if (mainSub) {+ // Clone coupons from main subscription to auto recharge expansion+ useCoupons = (+ await Promise.all(+ ((mainSub.discounts ?? []) as Stripe.Discount[]).map(async discount => {+ if (discount.coupon.duration === "once") {+ return null;+ }++ if (+ discount.coupon.duration === "repeating" &&+ new Date(discount.end! * 1000) < new Date()+ ) {+ return null;+ }++ if (!discount.coupon.valid) {+ const duration_in_months =+ discount.coupon.duration === "repeating"+ ? Math.round(+ (new Date(discount.end! * 1000).valueOf() -+ new Date(discount.start! * 1000).valueOf()) /+ (1000 * 60 * 60 * 24 * 30),+ )+ : 0;+ const coupon = await stripe.coupons.create({+ name: discount.coupon.name ?? undefined,+ duration: duration_in_months > 0 ? "repeating" : "once",+ ...(duration_in_months > 0 ? { duration_in_months } : {}),+ max_redemptions: 1,+ ...(discount.coupon.amount_off+ ? { amount_off: discount.coupon.amount_off }+ : {}),+ ...(discount.coupon.currency+ ? { currency: discount.coupon.currency }+ : {}),+ ...(discount.coupon.currency_options+ ? { currency_options: discount.coupon.currency_options }+ : {}),+ metadata: {+ ...(discount.coupon.metadata ?? {}),+ team_id,+ auto_recharge: "true",+ },+ ...(discount.coupon.percent_off+ ? { percent_off: discount.coupon.percent_off }+ : {}),+ ...(discount.coupon.applies_to+ ? { applies_to: discount.coupon.applies_to }+ : {}),+ });+ return coupon.id;+ } else {+ return discount.coupon.id;+ }+ }),+ )+ ).filter(x => x !== null);+ } else {+ logger.warn("No main subscription found for customer", {+ team_id,+ customer_id,+ });+ }++ const subscription = await stripe.subscriptions.create({+ customer: customer_id,+ items: [{ price: price_id }],+ off_session: true,+ payment_settings: {+ payment_method_types: [+ (defaultPaymentMethod?.type as Stripe.SubscriptionCreateParams.PaymentSettings.PaymentMethodType) ??+ "card",+ ],+ },+ default_payment_method: defaultPaymentMethod.id,+ collection_method: "charge_automatically",+ metadata: {+ team_id,+ auto_recharge: "true",+ },+ ...(useCoupons.length > 0+ ? { discounts: useCoupons.map(coupon => ({ coupon })) }+ : {}),+ });++ return subscription;+}++export async function customerToUserId(customerId: string) {+ const customer = await stripe.customers.retrieve(customerId);+ if (!customer || customer.deleted) return null;+ return customer.metadata.supabaseUUID ?? null;+}