πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Fetch subs (#1022)
Goldflag0b5c23ec1419
diff --git a/server/src/api/stripe/createPortalSession.ts b/server/src/api/stripe/createPortalSession.tsindex 183d62053..f327d32aa 100644--- a/server/src/api/stripe/createPortalSession.ts+++ b/server/src/api/stripe/createPortalSession.ts@@ -114,14 +114,20 @@ export async function createPortalSession(request: FastifyRequest<{ Body: Portalreturn reply.status(404).send({ error: "No active subscription found" });}- const subscriptionId = subscriptions.data[0].id;-- sessionConfig.flow_data = {- type: "subscription_cancel",- subscription_cancel: {- subscription: subscriptionId,- },- };+ const subscription = subscriptions.data[0];++ // Stripe rejects a subscription_cancel flow when the subscription is already scheduled+ // to cancel at period end ("already set to be canceled at period end"). In that case,+ // fall back to the plain billing portal so the user can review or resume instead of+ // hitting a 400.+ if (!subscription.cancel_at_period_end) {+ sessionConfig.flow_data = {+ type: "subscription_cancel",+ subscription_cancel: {+ subscription: subscription.id,+ },+ };+ }} else if (flowType === "payment_method_update") {sessionConfig.flow_data = {type: "payment_method_update",diff --git a/server/src/api/stripe/updateSubscription.ts b/server/src/api/stripe/updateSubscription.tsindex ad7d7b97a..292cdcae4 100644--- a/server/src/api/stripe/updateSubscription.ts+++ b/server/src/api/stripe/updateSubscription.ts@@ -93,7 +93,8 @@ export async function updateSubscription(...(isTrialing && subscription.trial_end && { trial_end: subscription.trial_end }),});- // The plan changed, so drop the cached subscription for this customer.+ // The plan changed, so drop the cached subscription for this customer (also clears the+ // account-wide snapshot used by the admin endpoints and usage cron).invalidateStripeSubscriptionCache(org.stripeCustomerId);// Get the updated subscription detailsdiff --git a/server/src/api/stripe/webhook.ts b/server/src/api/stripe/webhook.tsindex d8e1b204c..1f7f9dc3f 100644--- a/server/src/api/stripe/webhook.ts+++ b/server/src/api/stripe/webhook.ts@@ -44,7 +44,8 @@ export async function handleWebhook(request: FastifyRequest, reply: FastifyReplyconst stripeCustomerId = session.customer as string;const organizationId = session.metadata?.organizationId; // Retrieve organizationId from metadata- // A new subscription was created β refresh any cached lookup for this customer.+ // A new subscription was created β refresh any cached lookup for this customer+ // (this also clears the account-wide snapshot used by the admin endpoints and cron).invalidateStripeSubscriptionCache(stripeCustomerId);if (stripeCustomerId && organizationId) {diff --git a/server/src/lib/subscriptionUtils.ts b/server/src/lib/subscriptionUtils.tsindex 8a52d10fe..d339f045a 100644--- a/server/src/lib/subscriptionUtils.ts+++ b/server/src/lib/subscriptionUtils.ts@@ -191,6 +191,106 @@ export function invalidateStripeSubscriptionCache(stripeCustomerId: string | nulif (stripeCustomerId) {stripeSubscriptionCache.delete(stripeCustomerId);}+ // Also drop the account-wide snapshot so admin/cron reads pick up the change promptly.+ allStripeSubscriptionsCache = null;+}++/**+ * Account-wide snapshot of the best subscription per customer.+ *+ * The admin endpoints and the usage cron both need subscriptions for many/all customers at+ * once. Instead of one Stripe request per customer, we pull every subscription on the account+ * in a handful of paginated requests (100/page) and group them β so call volume scales with the+ * number of subscriptions, not the number of customers or requests. Cached per process.+ */+const STRIPE_ALL_SUBSCRIPTIONS_TTL_MS = 5 * 60_000;+let allStripeSubscriptionsCache: { value: Map<string, Stripe.Subscription>; expiresAt: number } | null = null;+let allStripeSubscriptionsInflight: Promise<Map<string, Stripe.Subscription>> | null = null;++/** Clears the account-wide subscription snapshot. */+export function invalidateAllStripeSubscriptionsCache(): void {+ allStripeSubscriptionsCache = null;+}++/**+ * Ranks a subscription so the "best" one per customer wins when grouping: prefer+ * active/trialing, then the most recently created.+ */+function rankSubscription(subscription: Stripe.Subscription): number {+ const isActiveOrTrialing = subscription.status === "active" || subscription.status === "trialing";+ // `created` is a unix timestamp (~1.7e9); the status weight (Γ1e13) dominates so an+ // active/trialing sub always outranks a canceled one regardless of creation time.+ return (isActiveOrTrialing ? 1 : 0) * 1e13 + subscription.created;+}++/**+ * Returns the best (active/trialing preferred, else most recent) raw subscription per customer+ * across the whole Stripe account, pulled in bulk and cached. Throws if the bulk fetch fails so+ * callers can decide whether to preserve previous state rather than treat everyone as free.+ */+export async function getAllStripeSubscriptionsByCustomer(): Promise<Map<string, Stripe.Subscription>> {+ if (!stripe) {+ return new Map();+ }++ const cached = allStripeSubscriptionsCache;+ if (cached && cached.expiresAt > Date.now()) {+ return cached.value;+ }++ // De-duplicate concurrent refreshes (e.g. several admin requests landing at once) so only+ // one bulk pagination runs at a time.+ if (allStripeSubscriptionsInflight) {+ return allStripeSubscriptionsInflight;+ }++ allStripeSubscriptionsInflight = (async () => {+ try {+ const byCustomer = new Map<string, Stripe.Subscription>();++ // The SDK auto-paginates this async iterator (100 subscriptions per underlying request).+ for await (const subscription of (stripe as Stripe).subscriptions.list({+ status: "all",+ limit: 100,+ expand: ["data.plan.product"],+ })) {+ const customerId =+ typeof subscription.customer === "string" ? subscription.customer : subscription.customer.id;+ const existing = byCustomer.get(customerId);+ if (!existing || rankSubscription(subscription) > rankSubscription(existing)) {+ byCustomer.set(customerId, subscription);+ }+ }++ allStripeSubscriptionsCache = {+ value: byCustomer,+ expiresAt: Date.now() + STRIPE_ALL_SUBSCRIPTIONS_TTL_MS,+ };+ return byCustomer;+ } finally {+ allStripeSubscriptionsInflight = null;+ }+ })();++ return allStripeSubscriptionsInflight;+}++/**+ * Resolves a customer's StripeSubscriptionInfo from a bulk snapshot, applying the same+ * active/trialing-only semantics as getStripeSubscription (canceled/past_due β no sub β free).+ */+export function stripeSubscriptionInfoFromSnapshot(+ snapshot: Map<string, Stripe.Subscription>,+ stripeCustomerId: string | null+): StripeSubscriptionInfo | null {+ if (!stripeCustomerId) {+ return null;+ }+ const subscription = snapshot.get(stripeCustomerId);+ if (!subscription || (subscription.status !== "active" && subscription.status !== "trialing")) {+ return null;+ }+ return buildStripeSubscriptionInfo(subscription);}/**@@ -258,53 +358,37 @@ async function fetchStripeSubscription(stripeCustomerId: string): Promise<Stripe// Pick the most recently created subscription across both active and trialingconst subscription = allSubs.sort((a, b) => b.created - a.created)[0];- const isTrial = subscription.status === "trialing";-- const subscriptionItem = subscription.items.data[0];- const priceId = subscriptionItem.price.id;+ return buildStripeSubscriptionInfo(subscription);+}- if (!priceId) {- console.error("Subscription item price ID not found");- return null;- }+/**+ * Builds our StripeSubscriptionInfo from a raw Stripe subscription. Pure β does not call Stripe β+ * so it is reused for both per-customer lookups and bulk account snapshots.+ */+function buildStripeSubscriptionInfo(subscription: Stripe.Subscription): StripeSubscriptionInfo | null {+ const isTrial = subscription.status === "trialing";- // Find corresponding plan details from constants- const planDetails = getStripePrices().find((plan: StripePlan) => plan.priceId === priceId);+ const subscriptionItem = subscription.items.data[0];+ const priceId = subscriptionItem?.price.id;- if (!planDetails) {- console.error("Plan details not found for price ID:", priceId);- // Return basic info even without plan details- return {- source: "stripe",- subscriptionId: subscription.id,- priceId,- planName: "Unknown Plan",- eventLimit: 0,- periodStart: getStartOfMonth(),- currentPeriodEnd: new Date(subscriptionItem.current_period_end * 1000),- status: subscription.status,- interval: subscriptionItem.price.recurring?.interval ?? "unknown",- cancelAtPeriodEnd: subscription.cancel_at_period_end,- createdAt: new Date(subscription.created * 1000),- ...(isTrial && subscription.trial_end ? { trialEnd: new Date(subscription.trial_end * 1000) } : {}),- };- }-- // Determine period start- const currentMonthStart = DateTime.now().startOf("month");- const subscriptionStartDate = DateTime.fromSeconds(subscriptionItem.current_period_start);+ if (!priceId) {+ console.error("Subscription item price ID not found");+ return null;+ }- // If subscription started within current month, use that date; otherwise use month start- const periodStart =- subscriptionStartDate >= currentMonthStart ? (subscriptionStartDate.toISODate() as string) : getStartOfMonth();+ // Find corresponding plan details from constants+ const planDetails = getStripePrices().find((plan: StripePlan) => plan.priceId === priceId);+ if (!planDetails) {+ console.error("Plan details not found for price ID:", priceId);+ // Return basic info even without plan detailsreturn {source: "stripe",subscriptionId: subscription.id,priceId,- planName: planDetails.name,- eventLimit: planDetails.limits.events,- periodStart,+ planName: "Unknown Plan",+ eventLimit: 0,+ periodStart: getStartOfMonth(),currentPeriodEnd: new Date(subscriptionItem.current_period_end * 1000),status: subscription.status,interval: subscriptionItem.price.recurring?.interval ?? "unknown",@@ -312,6 +396,30 @@ async function fetchStripeSubscription(stripeCustomerId: string): Promise<StripecreatedAt: new Date(subscription.created * 1000),...(isTrial && subscription.trial_end ? { trialEnd: new Date(subscription.trial_end * 1000) } : {}),};+ }++ // Determine period start+ const currentMonthStart = DateTime.now().startOf("month");+ const subscriptionStartDate = DateTime.fromSeconds(subscriptionItem.current_period_start);++ // If subscription started within current month, use that date; otherwise use month start+ const periodStart =+ subscriptionStartDate >= currentMonthStart ? (subscriptionStartDate.toISODate() as string) : getStartOfMonth();++ return {+ source: "stripe",+ subscriptionId: subscription.id,+ priceId,+ planName: planDetails.name,+ eventLimit: planDetails.limits.events,+ periodStart,+ currentPeriodEnd: new Date(subscriptionItem.current_period_end * 1000),+ status: subscription.status,+ interval: subscriptionItem.price.recurring?.interval ?? "unknown",+ cancelAtPeriodEnd: subscription.cancel_at_period_end,+ createdAt: new Date(subscription.created * 1000),+ ...(isTrial && subscription.trial_end ? { trialEnd: new Date(subscription.trial_end * 1000) } : {}),+ };}/**@@ -378,20 +486,50 @@ export async function getBestSubscription(getStripeSubscription(stripeCustomerId, { throwOnError: throwOnStripeError }),]);+ if (stripeSub) {+ return stripeSub;+ }++ if (appsumoSub) {+ return appsumoSub;+ }++ return freeSubscription();+}++/**+ * Like getBestSubscription, but uses an already-resolved Stripe subscription (e.g. from a bulk+ * account snapshot) instead of making a per-customer Stripe call. Used by callers that resolve+ * many orgs at once (admin endpoints, the usage cron).+ * Priority: CustomPlan > Override > Stripe > AppSumo > Free.+ */+export async function getBestSubscriptionFromStripeSub(+ organizationId: string,+ stripeSub: StripeSubscriptionInfo | null+): Promise<SubscriptionInfo> {+ const customSub = await getCustomPlanSubscription(organizationId);+ if (customSub) {+ return customSub;+ }++ const overrideSub = await getOverrideSubscription(organizationId);+ if (overrideSub) {+ return overrideSub;+ }if (stripeSub) {return stripeSub;}+ const appsumoSub = await getAppSumoSubscription(organizationId);if (appsumoSub) {return appsumoSub;}- // Return whichever one exists- if (appsumoSub) return appsumoSub;- if (stripeSub) return stripeSub;+ return freeSubscription();+}- // Default to free tier+function freeSubscription(): FreeSubscriptionInfo {return {source: "free",eventLimit: DEFAULT_EVENT_LIMIT,diff --git a/server/src/services/admin/subscriptionService.ts b/server/src/services/admin/subscriptionService.tsindex 701579230..8166ef73d 100644--- a/server/src/services/admin/subscriptionService.ts+++ b/server/src/services/admin/subscriptionService.ts@@ -1,8 +1,10 @@import { sql } from "drizzle-orm";import { DateTime } from "luxon";+import Stripe from "stripe";import { db } from "../../db/postgres/postgres.js";import { APPSUMO_TIER_LIMITS, DEFAULT_EVENT_LIMIT, getStripePrices } from "../../lib/const.js";import { stripe } from "../../lib/stripe.js";+import { getAllStripeSubscriptionsByCustomer } from "../../lib/subscriptionUtils.js";export interface SubscriptionData {id: string;@@ -16,7 +18,45 @@ export interface SubscriptionData {}/**- * Fetches subscription data for multiple Stripe customer IDs+ * Projects a raw Stripe subscription into the admin SubscriptionData shape. Unlike the app-facing+ * lookup, the admin view keeps non-active subscriptions (canceled/past_due) so it can display them.+ */+function buildAdminSubscriptionData(+ subscription: Stripe.Subscription,+ includeFullDetails: boolean+): SubscriptionData | null {+ const subscriptionItem = subscription.items.data[0];+ const priceId = subscriptionItem?.price.id;++ if (!priceId) {+ return null;+ }++ const planDetails = getStripePrices().find(plan => plan.priceId === priceId);++ const data: SubscriptionData = {+ id: subscription.id,+ planName: planDetails?.name || "Unknown Plan",+ status: subscription.status,+ };++ if (includeFullDetails) {+ data.currentPeriodStart = new Date(subscriptionItem.current_period_start * 1000);+ data.currentPeriodEnd = new Date(subscriptionItem.current_period_end * 1000);+ data.cancelAtPeriodEnd = subscription.cancel_at_period_end;+ data.eventLimit = planDetails?.limits.events || 0;+ data.interval = subscriptionItem.price.recurring?.interval ?? "unknown";+ }++ return data;+}++/**+ * Fetches subscription data for multiple Stripe customer IDs.+ *+ * Rather than one Stripe request per customer, this reads from a single cached account-wide+ * snapshot (a handful of paginated requests for the whole account) and picks out the customers+ * we care about β so admin loads don't scale Stripe calls with the customer count or refetch rate.* @param stripeCustomerIds Set of Stripe customer IDs to fetch subscriptions for* @param includeFullDetails Whether to include full subscription details (periods, limits, etc.)* @returns Map of customer ID to subscription data@@ -31,63 +71,25 @@ async function fetchSubscriptionsForCustomers(return subscriptionMap;}- const stripeClient = stripe;-- // Fetch subscriptions per-customer with bounded concurrency so cost scales with- // the number of orgs we care about rather than the entire Stripe account.- const customerIds = Array.from(stripeCustomerIds);- const CONCURRENCY = 10;-- const fetchForCustomer = async (customerId: string) => {- try {- const subscriptions = await stripeClient.subscriptions.list({- customer: customerId,- status: "all",- limit: 100,- expand: ["data.plan.product"],- });-- // Prefer an active/trialing subscription, matching the previous behavior.- const subscription =- subscriptions.data.find(sub => sub.status === "active" || sub.status === "trialing") ??- subscriptions.data[0];-- if (!subscription) {- return;- }-- const subscriptionItem = subscription.items.data[0];- const priceId = subscriptionItem?.price.id;-- if (!priceId) {- return;- }-- const planDetails = getStripePrices().find(plan => plan.priceId === priceId);-- const subscriptionData: SubscriptionData = {- id: subscription.id,- planName: planDetails?.name || "Unknown Plan",- status: subscription.status,- };-- if (includeFullDetails) {- subscriptionData.currentPeriodStart = new Date(subscriptionItem.current_period_start * 1000);- subscriptionData.currentPeriodEnd = new Date(subscriptionItem.current_period_end * 1000);- subscriptionData.cancelAtPeriodEnd = subscription.cancel_at_period_end;- subscriptionData.eventLimit = planDetails?.limits.events || 0;- subscriptionData.interval = subscriptionItem.price.recurring?.interval ?? "unknown";- }+ let snapshot: Map<string, Stripe.Subscription>;+ try {+ snapshot = await getAllStripeSubscriptionsByCustomer();+ } catch (error) {+ // Bulk fetch failed (e.g. rate limit) β render orgs as free for this load rather than+ // failing the whole admin page. The next load retries once the snapshot can refresh.+ console.error("Error fetching Stripe subscriptions in bulk:", error);+ return subscriptionMap;+ }- subscriptionMap.set(customerId, subscriptionData);- } catch (error) {- console.error(`Error fetching Stripe subscription for customer ${customerId}:`, error);+ for (const customerId of stripeCustomerIds) {+ const subscription = snapshot.get(customerId);+ if (!subscription) {+ continue;+ }+ const value = buildAdminSubscriptionData(subscription, includeFullDetails);+ if (value) {+ subscriptionMap.set(customerId, value);}- };-- for (let i = 0; i < customerIds.length; i += CONCURRENCY) {- const batch = customerIds.slice(i, i + CONCURRENCY);- await Promise.all(batch.map(fetchForCustomer));}return subscriptionMap;diff --git a/server/src/services/usageService.ts b/server/src/services/usageService.tsindex 2f49aa605..218ef9e14 100644--- a/server/src/services/usageService.ts+++ b/server/src/services/usageService.ts@@ -1,6 +1,7 @@import { and, eq } from "drizzle-orm";import { DateTime } from "luxon";import * as cron from "node-cron";+import Stripe from "stripe";import { processResults } from "../api/analytics/utils/utils.js";import { clickhouse } from "../db/clickhouse/clickhouse.js";import { db } from "../db/postgres/postgres.js";@@ -8,7 +9,11 @@ import { member, organization, sites, user } from "../db/postgres/schema.js";import { IS_CLOUD } from "../lib/const.js";import { sendApproachingLimitEmail, sendLimitExceededEmail } from "../lib/email/email.js";import { createServiceLogger } from "../lib/logger/logger.js";-import { getBestSubscription } from "../lib/subscriptionUtils.js";+import {+ getAllStripeSubscriptionsByCustomer,+ getBestSubscriptionFromStripeSub,+ stripeSubscriptionInfoFromSnapshot,+} from "../lib/subscriptionUtils.js";type UsageUpdateCallback = () => void;@@ -123,24 +128,24 @@ class UsageService {* Checks both AppSumo and Stripe subscriptions and uses the one with the higher event limit.* @returns [eventLimit, periodStartDate]*/- private async getOrganizationSubscriptionInfo(orgData: {- id: string;- stripeCustomerId: string | null;- createdAt: string;- name: string;- }): Promise<[number, string | null]> {+ private async getOrganizationSubscriptionInfo(+ orgData: {+ id: string;+ stripeCustomerId: string | null;+ createdAt: string;+ name: string;+ },+ stripeSubscriptions: Map<string, Stripe.Subscription>+ ): Promise<[number, string | null]> {// Special case for specific organizationsif (orgData.name === "rybbit" || orgData.name === "Zam") {return [Infinity, this.getStartOfMonth()];}- // Get the best subscription (highest event limit from AppSumo or Stripe).- // throwOnStripeError ensures a transient Stripe failure surfaces as an error here- // instead of silently resolving to the free tier (which would wrongly flag a paying- // org as over-limit). The caller skips the org when this throws.- const subscription = await getBestSubscription(orgData.id, orgData.stripeCustomerId, {- throwOnStripeError: true,- });+ // Resolve this org's Stripe subscription from the bulk snapshot (no per-org Stripe call),+ // then layer in custom plan / override / AppSumo via the same priority rules as elsewhere.+ const stripeSub = stripeSubscriptionInfoFromSnapshot(stripeSubscriptions, orgData.stripeCustomerId);+ const subscription = await getBestSubscriptionFromStripeSub(orgData.id, stripeSub);// Log subscription detailsif (subscription.source === "appsumo") {@@ -203,6 +208,18 @@ class UsageService {this.logger.info("Starting check of monthly event usage for organizations...");try {+ // Step 0: Pull every customer's subscription from Stripe in one bulk pass (a handful of+ // paginated calls) instead of one call per org. If this fails (e.g. rate limit/outage),+ // skip the whole run rather than treating every paying org as free β which would wrongly+ // flag them over-limit, block ingestion, and email their owners.+ let stripeSubscriptions: Map<string, Stripe.Subscription>;+ try {+ stripeSubscriptions = await getAllStripeSubscriptionsByCustomer();+ } catch (error) {+ this.logger.error(error as Error, "Skipping usage check: failed to fetch Stripe subscriptions in bulk");+ return;+ }+// Step 1: Get all sites with their organization IDsconst allSites = await this.getAllSites();@@ -246,21 +263,7 @@ class UsageService {const wasOverLimit = orgData.overMonthlyLimit ?? false;const alreadyNotifiedApproaching = orgData.approachingLimitNotifiedPeriodStart === monthStart;- let eventLimit: number;- try {- [eventLimit] = await this.getOrganizationSubscriptionInfo(orgData);- } catch (error) {- // We couldn't determine this org's subscription (e.g. a Stripe rate limit).- // Treating that as the free tier would wrongly mark a paying org as over-limit,- // blocking its event ingestion and emailing its owner. Skip the org and leave- // its existing limit state untouched until a later run can fetch the real plan.- this.logger.warn(- `Skipping organization ${orgData.name}: could not determine subscription (${- (error as Error).message- }). Preserving previous limit state.`- );- continue;- }+ const [eventLimit] = await this.getOrganizationSubscriptionInfo(orgData, stripeSubscriptions);const isOverLimit = eventCount > eventLimit;let sendApproaching = false;