🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Make import limits match monthly quota
Edward Cui8c9a7e278908
diff --git a/server/src/lib/subscriptionUtils.ts b/server/src/lib/subscriptionUtils.tsnew file mode 100644index 000000000..3629d3905--- /dev/null+++ b/server/src/lib/subscriptionUtils.ts@@ -0,0 +1,75 @@+import { IS_CLOUD } from "./const.js";+import { db } from "../db/postgres/postgres.js";+import { organization } from "../db/postgres/schema.js";+import { eq } from "drizzle-orm";+import { getSubscriptionInner } from "../api/stripe/getSubscription.js";++export interface TierInfo {+ tier: "Free" | "Standard" | "Pro";+ monthsAllowed: number;+}++export interface SubscriptionInfo {+ planName: string;+ eventLimit: number;+ tierInfo: TierInfo;+}++export function getTierInfo(planName: string): TierInfo {+ if (planName === "free") {+ return {+ tier: "Free",+ monthsAllowed: 6,+ };+ }++ if (planName.startsWith("standard")) {+ return {+ tier: "Standard",+ monthsAllowed: 24,+ };+ }++ if (planName.startsWith("pro")) {+ return {+ tier: "Pro",+ monthsAllowed: 60,+ };+ }++ // Default to free tier for unknown plans+ return {+ tier: "Free",+ monthsAllowed: 6,+ };+}++export async function getOrganizationSubscriptionInfo(organizationId: string): Promise<SubscriptionInfo | null> {+ if (!IS_CLOUD) {+ return null;+ }++ const [org] = await db+ .select({ name: organization.name })+ .from(organization)+ .where(eq(organization.id, organizationId))+ .limit(1);++ if (!org) {+ return null;+ }++ const subscription = await getSubscriptionInner(organizationId);++ if (!subscription) {+ return null;+ }++ const tierInfo = getTierInfo(subscription.planName);++ return {+ planName: subscription.planName,+ eventLimit: subscription.eventLimit,+ tierInfo,+ };+}diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.tsindex 49e09c09c..55da0ffe3 100644--- a/server/src/services/import/importLimiter.ts+++ b/server/src/services/import/importLimiter.ts@@ -1,10 +1,10 @@import { IS_CLOUD } from "../../lib/const.js";import { db } from "../../db/postgres/postgres.js";-import { and, count, eq, inArray, sum } from "drizzle-orm";+import { and, count, eq, inArray } from "drizzle-orm";import { importStatus, sites } from "../../db/postgres/schema.js";+import { ImportQuotaTracker } from "./importQuotaChecker.js";export class ImportLimiter {- private static readonly IMPORTED_EVENTS_LIMIT = 1_000_000;private static readonly CONCURRENT_IMPORT_LIMIT = 1;static async checkConcurrentImportLimit(siteId: number): Promise<@@ -51,16 +51,7 @@ export class ImportLimiter {return { allowed: true, organizationId: siteResult.organizationId };}- static async countImportableEvents(organizationId: string): Promise<number> {- if (!IS_CLOUD) {- return Infinity;- }-- const [importedEvents] = await db- .select({ total: sum(importStatus.importedEvents) })- .from(importStatus)- .where(eq(importStatus.organizationId, organizationId));-- return this.IMPORTED_EVENTS_LIMIT - Number(importedEvents.total ?? 0);+ static async createQuotaTracker(organizationId: string): Promise<ImportQuotaTracker> {+ return ImportQuotaTracker.create(organizationId);}}diff --git a/server/src/services/import/importQuotaChecker.ts b/server/src/services/import/importQuotaChecker.tsnew file mode 100644index 000000000..5355fff05--- /dev/null+++ b/server/src/services/import/importQuotaChecker.ts@@ -0,0 +1,211 @@+import { DateTime } from "luxon";+import { clickhouse } from "../../db/clickhouse/clickhouse.js";+import { db } from "../../db/postgres/postgres.js";+import { sites } from "../../db/postgres/schema.js";+import { eq } from "drizzle-orm";+import { processResults } from "../../api/analytics/utils.js";+import { getOrganizationSubscriptionInfo } from "../../lib/subscriptionUtils.js";+import { IS_CLOUD } from "../../lib/const.js";++export interface MonthlyQuotaInfo {+ month: string; // "202501" format+ used: number;+ limit: number;+ remaining: number;+}++export class ImportQuotaTracker {+ private monthlyUsage: Map<string, number>;+ private readonly monthlyLimit: number;+ private readonly historicalWindowMonths: number;+ private readonly oldestAllowedMonth: string;++ private constructor(+ monthlyUsage: Map<string, number>,+ monthlyLimit: number,+ historicalWindowMonths: number,+ oldestAllowedMonth: string+ ) {+ this.monthlyUsage = monthlyUsage;+ this.monthlyLimit = monthlyLimit;+ this.historicalWindowMonths = historicalWindowMonths;+ this.oldestAllowedMonth = oldestAllowedMonth;+ }++ static async create(organizationId: string): Promise<ImportQuotaTracker> {+ if (!IS_CLOUD) {+ return new ImportQuotaTracker(new Map(), Infinity, Infinity, "190001");+ }++ const subscriptionInfo = await getOrganizationSubscriptionInfo(organizationId);+ if (!subscriptionInfo) {+ throw new Error(`No subscription found for organization ${organizationId}`);+ }++ const monthlyLimit = subscriptionInfo.eventLimit;+ const historicalWindowMonths = subscriptionInfo.tierInfo.monthsAllowed;++ const oldestAllowedDate = DateTime.now().minus({ months: historicalWindowMonths }).startOf("month");+ const oldestAllowedMonth = oldestAllowedDate.toFormat("yyyyMM");++ const siteRecords = await db+ .select({ siteId: sites.siteId })+ .from(sites)+ .where(eq(sites.organizationId, organizationId));++ const siteIds = siteRecords.map(s => s.siteId);++ if (siteIds.length === 0) {+ return new ImportQuotaTracker(new Map(), monthlyLimit, historicalWindowMonths, oldestAllowedMonth);+ }++ const monthlyUsage = await ImportQuotaTracker.queryMonthlyUsage(siteIds, oldestAllowedDate.toFormat("yyyy-MM-dd"));++ return new ImportQuotaTracker(monthlyUsage, monthlyLimit, historicalWindowMonths, oldestAllowedMonth);+ }++ private static async queryMonthlyUsage(siteIds: number[], startDate: string): Promise<Map<string, number>> {+ const monthlyUsage = new Map<string, number>();++ if (siteIds.length === 0) {+ return monthlyUsage;+ }++ const grandfatheredSites = siteIds.filter(id => id < 2000);+ const newSites = siteIds.filter(id => id >= 2000);++ try {+ // Query grandfathered sites (pageviews only)+ if (grandfatheredSites.length > 0) {+ const grandfatheredResult = await clickhouse.query({+ query: `+ SELECT+ toYYYYMM(timestamp) as month,+ COUNT(*) as count+ FROM events+ WHERE site_id IN (${grandfatheredSites.join(",")})+ AND type = 'pageview'+ AND timestamp >= toDate('${startDate}')+ GROUP BY month+ ORDER BY month+ `,+ format: "JSONEachRow",+ });++ const rows = await processResults<{ month: string; count: string }>(grandfatheredResult);+ for (const row of rows) {+ const existing = monthlyUsage.get(row.month) || 0;+ monthlyUsage.set(row.month, existing + parseInt(row.count, 10));+ }+ }++ // Query new sites (all event types)+ if (newSites.length > 0) {+ const newSitesResult = await clickhouse.query({+ query: `+ SELECT+ toYYYYMM(timestamp) as month,+ COUNT(*) as count+ FROM events+ WHERE site_id IN (${newSites.join(",")})+ AND type IN ('pageview', 'custom_event', 'performance')+ AND timestamp >= toDate('${startDate}')+ GROUP BY month+ ORDER BY month+ `,+ format: "JSONEachRow",+ });++ const rows = await processResults<{ month: string; count: string }>(newSitesResult);+ for (const row of rows) {+ const existing = monthlyUsage.get(row.month) || 0;+ monthlyUsage.set(row.month, existing + parseInt(row.count, 10));+ }+ }++ return monthlyUsage;+ } catch (error) {+ console.error(`Error querying ClickHouse for monthly usage:`, error);+ return new Map();+ }+ }++ canImportEvent(timestamp: string): boolean {+ if (this.monthlyLimit === Infinity) {+ return true;+ }++ const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });+ if (!dt.isValid) {+ console.warn(`Invalid timestamp format: ${timestamp}`);+ return false;+ }++ const month = dt.toFormat("yyyyMM");++ if (month < this.oldestAllowedMonth) {+ return false;+ }++ const used = this.monthlyUsage.get(month) || 0;+ if (used >= this.monthlyLimit) {+ return false;+ }++ this.monthlyUsage.set(month, used + 1);+ return true;+ }++ getMonthQuota(month: string): MonthlyQuotaInfo {+ const used = this.monthlyUsage.get(month) || 0;+ return {+ month,+ used,+ limit: this.monthlyLimit,+ remaining: Math.max(0, this.monthlyLimit - used),+ };+ }++ getAllMonthQuotas(): MonthlyQuotaInfo[] {+ const quotas: MonthlyQuotaInfo[] = [];++ if (this.monthlyLimit === Infinity) {+ return quotas;+ }++ const now = DateTime.now();+ for (let i = 0; i < this.historicalWindowMonths; i++) {+ const monthDate = now.minus({ months: i }).startOf("month");+ const month = monthDate.toFormat("yyyyMM");+ quotas.push(this.getMonthQuota(month));+ }++ return quotas.reverse();+ }++ getSummary(): {+ totalMonthsInWindow: number;+ monthsAtCapacity: number;+ monthsWithSpace: number;+ oldestAllowedMonth: string;+ } {+ if (this.monthlyLimit === Infinity) {+ return {+ totalMonthsInWindow: this.historicalWindowMonths,+ monthsAtCapacity: 0,+ monthsWithSpace: this.historicalWindowMonths,+ oldestAllowedMonth: this.oldestAllowedMonth,+ };+ }++ const quotas = this.getAllMonthQuotas();+ const monthsAtCapacity = quotas.filter(q => q.remaining === 0).length;++ return {+ totalMonthsInWindow: this.historicalWindowMonths,+ monthsAtCapacity,+ monthsWithSpace: quotas.length - monthsAtCapacity,+ oldestAllowedMonth: this.oldestAllowedMonth,+ };+ }+}diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 89c2dc35a..1ab97e079 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -89,20 +89,14 @@ export async function registerCsvParseWorker() {const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;try {- const importableEvents = await ImportLimiter.countImportableEvents(organization);- if (importableEvents <= 0) {- await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");- const deleteResult = await deleteImportFile(storageLocation, isR2Storage);- if (!deleteResult.success) {- console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);- }- return;- }+ const quotaTracker = await ImportLimiter.createQuotaTracker(organization);const chunkSize = 5000;let chunk: UmamiEvent[] = [];- let rowsProcessed = 0;- let chunksSent = 0; // Track total chunks sent+ let totalAccepted = 0;+ let totalSkippedQuota = 0;+ let totalSkippedDate = 0;+ let chunksSent = 0;const stream = isR2Storage? await createR2FileStream(storageLocation, source)@@ -110,20 +104,29 @@ export async function registerCsvParseWorker() {await ImportStatusManager.updateStatus(importId, "processing");- // Create date filter once for performance (not per-row)const isDateInRange = createDateRangeFilter(startDate, endDate);for await (const data of stream) {- if (!data.created_at || !isDateInRange(data.created_at)) {+ // Skip rows with missing or invalid dates+ if (!data.created_at) {continue;}- if (rowsProcessed >= importableEvents) {- break;+ // Apply user-specified date range filter+ if (!isDateInRange(data.created_at)) {+ totalSkippedDate++;+ continue;+ }++ // Check per-month quota (includes historical window check)+ if (!quotaTracker.canImportEvent(data.created_at)) {+ totalSkippedQuota++;+ continue;}+ // Event passed all filters - add to chunkchunk.push(data);- rowsProcessed++;+ totalAccepted++;if (chunk.length >= chunkSize) {await jobQueue.send(DATA_INSERT_QUEUE, {@@ -139,6 +142,26 @@ export async function registerCsvParseWorker() {}}+ console.info(+ `[Import ${importId}] Processed CSV: ${totalAccepted} events accepted, ` ++ `${totalSkippedQuota} skipped (quota/window), ${totalSkippedDate} skipped (date filter)`+ );++ // Check if no events could be imported due to quotas+ if (totalAccepted === 0 && totalSkippedQuota > 0) {+ const quotaSummary = quotaTracker.getSummary();+ const errorMessage =+ `No events could be imported. All ${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` ++ `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` ++ `Try importing newer data or upgrade your plan for higher monthly quotas.`;+ await ImportStatusManager.updateStatus(importId, "failed", errorMessage);+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);+ if (!deleteResult.success) {+ console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);+ }+ return;+ }+// Send final chunk if any data remainsif (chunk.length > 0) {await jobQueue.send(DATA_INSERT_QUEUE, {