Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Make import limits match monthly quota

Edward Cui8c9a7e278908
diff --git a/server/src/lib/subscriptionUtils.ts b/server/src/lib/subscriptionUtils.ts
new file mode 100644
index 000000000..3629d3905
--- /dev/null
+++ b/server/src/lib/subscriptionUtils.ts
@@ -0,0 +1,75 @@
+import { IS_CLOUD } from "./const.js";
+import { db } from "../db/postgres/postgres.js";
+import { organization } from "../db/postgres/schema.js";
+import { eq } from "drizzle-orm";
+import { getSubscriptionInner } from "../api/stripe/getSubscription.js";
+
+export interface TierInfo {
+ tier: "Free" | "Standard" | "Pro";
+ monthsAllowed: number;
+}
+
+export interface SubscriptionInfo {
+ planName: string;
+ eventLimit: number;
+ tierInfo: TierInfo;
+}
+
+export function getTierInfo(planName: string): TierInfo {
+ if (planName === "free") {
+ return {
+ tier: "Free",
+ monthsAllowed: 6,
+ };
+ }
+
+ if (planName.startsWith("standard")) {
+ return {
+ tier: "Standard",
+ monthsAllowed: 24,
+ };
+ }
+
+ if (planName.startsWith("pro")) {
+ return {
+ tier: "Pro",
+ monthsAllowed: 60,
+ };
+ }
+
+ // Default to free tier for unknown plans
+ return {
+ tier: "Free",
+ monthsAllowed: 6,
+ };
+}
+
+export async function getOrganizationSubscriptionInfo(organizationId: string): Promise<SubscriptionInfo | null> {
+ if (!IS_CLOUD) {
+ return null;
+ }
+
+ const [org] = await db
+ .select({ name: organization.name })
+ .from(organization)
+ .where(eq(organization.id, organizationId))
+ .limit(1);
+
+ if (!org) {
+ return null;
+ }
+
+ const subscription = await getSubscriptionInner(organizationId);
+
+ if (!subscription) {
+ return null;
+ }
+
+ const tierInfo = getTierInfo(subscription.planName);
+
+ return {
+ planName: subscription.planName,
+ eventLimit: subscription.eventLimit,
+ tierInfo,
+ };
+}
diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.ts
index 49e09c09c..55da0ffe3 100644
--- a/server/src/services/import/importLimiter.ts
+++ b/server/src/services/import/importLimiter.ts
@@ -1,10 +1,10 @@
import { IS_CLOUD } from "../../lib/const.js";
import { db } from "../../db/postgres/postgres.js";
-import { and, count, eq, inArray, sum } from "drizzle-orm";
+import { and, count, eq, inArray } from "drizzle-orm";
import { importStatus, sites } from "../../db/postgres/schema.js";
+import { ImportQuotaTracker } from "./importQuotaChecker.js";
export class ImportLimiter {
- private static readonly IMPORTED_EVENTS_LIMIT = 1_000_000;
private static readonly CONCURRENT_IMPORT_LIMIT = 1;
static async checkConcurrentImportLimit(siteId: number): Promise<
@@ -51,16 +51,7 @@ export class ImportLimiter {
return { allowed: true, organizationId: siteResult.organizationId };
}
- static async countImportableEvents(organizationId: string): Promise<number> {
- if (!IS_CLOUD) {
- return Infinity;
- }
-
- const [importedEvents] = await db
- .select({ total: sum(importStatus.importedEvents) })
- .from(importStatus)
- .where(eq(importStatus.organizationId, organizationId));
-
- return this.IMPORTED_EVENTS_LIMIT - Number(importedEvents.total ?? 0);
+ static async createQuotaTracker(organizationId: string): Promise<ImportQuotaTracker> {
+ return ImportQuotaTracker.create(organizationId);
}
}
diff --git a/server/src/services/import/importQuotaChecker.ts b/server/src/services/import/importQuotaChecker.ts
new file mode 100644
index 000000000..5355fff05
--- /dev/null
+++ b/server/src/services/import/importQuotaChecker.ts
@@ -0,0 +1,211 @@
+import { DateTime } from "luxon";
+import { clickhouse } from "../../db/clickhouse/clickhouse.js";
+import { db } from "../../db/postgres/postgres.js";
+import { sites } from "../../db/postgres/schema.js";
+import { eq } from "drizzle-orm";
+import { processResults } from "../../api/analytics/utils.js";
+import { getOrganizationSubscriptionInfo } from "../../lib/subscriptionUtils.js";
+import { IS_CLOUD } from "../../lib/const.js";
+
+export interface MonthlyQuotaInfo {
+ month: string; // "202501" format
+ used: number;
+ limit: number;
+ remaining: number;
+}
+
+export class ImportQuotaTracker {
+ private monthlyUsage: Map<string, number>;
+ private readonly monthlyLimit: number;
+ private readonly historicalWindowMonths: number;
+ private readonly oldestAllowedMonth: string;
+
+ private constructor(
+ monthlyUsage: Map<string, number>,
+ monthlyLimit: number,
+ historicalWindowMonths: number,
+ oldestAllowedMonth: string
+ ) {
+ this.monthlyUsage = monthlyUsage;
+ this.monthlyLimit = monthlyLimit;
+ this.historicalWindowMonths = historicalWindowMonths;
+ this.oldestAllowedMonth = oldestAllowedMonth;
+ }
+
+ static async create(organizationId: string): Promise<ImportQuotaTracker> {
+ if (!IS_CLOUD) {
+ return new ImportQuotaTracker(new Map(), Infinity, Infinity, "190001");
+ }
+
+ const subscriptionInfo = await getOrganizationSubscriptionInfo(organizationId);
+ if (!subscriptionInfo) {
+ throw new Error(`No subscription found for organization ${organizationId}`);
+ }
+
+ const monthlyLimit = subscriptionInfo.eventLimit;
+ const historicalWindowMonths = subscriptionInfo.tierInfo.monthsAllowed;
+
+ const oldestAllowedDate = DateTime.now().minus({ months: historicalWindowMonths }).startOf("month");
+ const oldestAllowedMonth = oldestAllowedDate.toFormat("yyyyMM");
+
+ const siteRecords = await db
+ .select({ siteId: sites.siteId })
+ .from(sites)
+ .where(eq(sites.organizationId, organizationId));
+
+ const siteIds = siteRecords.map(s => s.siteId);
+
+ if (siteIds.length === 0) {
+ return new ImportQuotaTracker(new Map(), monthlyLimit, historicalWindowMonths, oldestAllowedMonth);
+ }
+
+ const monthlyUsage = await ImportQuotaTracker.queryMonthlyUsage(siteIds, oldestAllowedDate.toFormat("yyyy-MM-dd"));
+
+ return new ImportQuotaTracker(monthlyUsage, monthlyLimit, historicalWindowMonths, oldestAllowedMonth);
+ }
+
+ private static async queryMonthlyUsage(siteIds: number[], startDate: string): Promise<Map<string, number>> {
+ const monthlyUsage = new Map<string, number>();
+
+ if (siteIds.length === 0) {
+ return monthlyUsage;
+ }
+
+ const grandfatheredSites = siteIds.filter(id => id < 2000);
+ const newSites = siteIds.filter(id => id >= 2000);
+
+ try {
+ // Query grandfathered sites (pageviews only)
+ if (grandfatheredSites.length > 0) {
+ const grandfatheredResult = await clickhouse.query({
+ query: `
+ SELECT
+ toYYYYMM(timestamp) as month,
+ COUNT(*) as count
+ FROM events
+ WHERE site_id IN (${grandfatheredSites.join(",")})
+ AND type = 'pageview'
+ AND timestamp >= toDate('${startDate}')
+ GROUP BY month
+ ORDER BY month
+ `,
+ format: "JSONEachRow",
+ });
+
+ const rows = await processResults<{ month: string; count: string }>(grandfatheredResult);
+ for (const row of rows) {
+ const existing = monthlyUsage.get(row.month) || 0;
+ monthlyUsage.set(row.month, existing + parseInt(row.count, 10));
+ }
+ }
+
+ // Query new sites (all event types)
+ if (newSites.length > 0) {
+ const newSitesResult = await clickhouse.query({
+ query: `
+ SELECT
+ toYYYYMM(timestamp) as month,
+ COUNT(*) as count
+ FROM events
+ WHERE site_id IN (${newSites.join(",")})
+ AND type IN ('pageview', 'custom_event', 'performance')
+ AND timestamp >= toDate('${startDate}')
+ GROUP BY month
+ ORDER BY month
+ `,
+ format: "JSONEachRow",
+ });
+
+ const rows = await processResults<{ month: string; count: string }>(newSitesResult);
+ for (const row of rows) {
+ const existing = monthlyUsage.get(row.month) || 0;
+ monthlyUsage.set(row.month, existing + parseInt(row.count, 10));
+ }
+ }
+
+ return monthlyUsage;
+ } catch (error) {
+ console.error(`Error querying ClickHouse for monthly usage:`, error);
+ return new Map();
+ }
+ }
+
+ canImportEvent(timestamp: string): boolean {
+ if (this.monthlyLimit === Infinity) {
+ return true;
+ }
+
+ const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
+ if (!dt.isValid) {
+ console.warn(`Invalid timestamp format: ${timestamp}`);
+ return false;
+ }
+
+ const month = dt.toFormat("yyyyMM");
+
+ if (month < this.oldestAllowedMonth) {
+ return false;
+ }
+
+ const used = this.monthlyUsage.get(month) || 0;
+ if (used >= this.monthlyLimit) {
+ return false;
+ }
+
+ this.monthlyUsage.set(month, used + 1);
+ return true;
+ }
+
+ getMonthQuota(month: string): MonthlyQuotaInfo {
+ const used = this.monthlyUsage.get(month) || 0;
+ return {
+ month,
+ used,
+ limit: this.monthlyLimit,
+ remaining: Math.max(0, this.monthlyLimit - used),
+ };
+ }
+
+ getAllMonthQuotas(): MonthlyQuotaInfo[] {
+ const quotas: MonthlyQuotaInfo[] = [];
+
+ if (this.monthlyLimit === Infinity) {
+ return quotas;
+ }
+
+ const now = DateTime.now();
+ for (let i = 0; i < this.historicalWindowMonths; i++) {
+ const monthDate = now.minus({ months: i }).startOf("month");
+ const month = monthDate.toFormat("yyyyMM");
+ quotas.push(this.getMonthQuota(month));
+ }
+
+ return quotas.reverse();
+ }
+
+ getSummary(): {
+ totalMonthsInWindow: number;
+ monthsAtCapacity: number;
+ monthsWithSpace: number;
+ oldestAllowedMonth: string;
+ } {
+ if (this.monthlyLimit === Infinity) {
+ return {
+ totalMonthsInWindow: this.historicalWindowMonths,
+ monthsAtCapacity: 0,
+ monthsWithSpace: this.historicalWindowMonths,
+ oldestAllowedMonth: this.oldestAllowedMonth,
+ };
+ }
+
+ const quotas = this.getAllMonthQuotas();
+ const monthsAtCapacity = quotas.filter(q => q.remaining === 0).length;
+
+ return {
+ totalMonthsInWindow: this.historicalWindowMonths,
+ monthsAtCapacity,
+ monthsWithSpace: quotas.length - monthsAtCapacity,
+ oldestAllowedMonth: this.oldestAllowedMonth,
+ };
+ }
+}
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 89c2dc35a..1ab97e079 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -89,20 +89,14 @@ export async function registerCsvParseWorker() {
const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
try {
- const importableEvents = await ImportLimiter.countImportableEvents(organization);
- if (importableEvents <= 0) {
- await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
- const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
- if (!deleteResult.success) {
- console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);
- }
- return;
- }
+ const quotaTracker = await ImportLimiter.createQuotaTracker(organization);
const chunkSize = 5000;
let chunk: UmamiEvent[] = [];
- let rowsProcessed = 0;
- let chunksSent = 0; // Track total chunks sent
+ let totalAccepted = 0;
+ let totalSkippedQuota = 0;
+ let totalSkippedDate = 0;
+ let chunksSent = 0;
const stream = isR2Storage
? await createR2FileStream(storageLocation, source)
@@ -110,20 +104,29 @@ export async function registerCsvParseWorker() {
await ImportStatusManager.updateStatus(importId, "processing");
- // Create date filter once for performance (not per-row)
const isDateInRange = createDateRangeFilter(startDate, endDate);
for await (const data of stream) {
- if (!data.created_at || !isDateInRange(data.created_at)) {
+ // Skip rows with missing or invalid dates
+ if (!data.created_at) {
continue;
}
- if (rowsProcessed >= importableEvents) {
- break;
+ // Apply user-specified date range filter
+ if (!isDateInRange(data.created_at)) {
+ totalSkippedDate++;
+ continue;
+ }
+
+ // Check per-month quota (includes historical window check)
+ if (!quotaTracker.canImportEvent(data.created_at)) {
+ totalSkippedQuota++;
+ continue;
}
+ // Event passed all filters - add to chunk
chunk.push(data);
- rowsProcessed++;
+ totalAccepted++;
if (chunk.length >= chunkSize) {
await jobQueue.send(DATA_INSERT_QUEUE, {
@@ -139,6 +142,26 @@ export async function registerCsvParseWorker() {
}
}
+ console.info(
+ `[Import ${importId}] Processed CSV: ${totalAccepted} events accepted, ` +
+ `${totalSkippedQuota} skipped (quota/window), ${totalSkippedDate} skipped (date filter)`
+ );
+
+ // Check if no events could be imported due to quotas
+ if (totalAccepted === 0 && totalSkippedQuota > 0) {
+ const quotaSummary = quotaTracker.getSummary();
+ const errorMessage =
+ `No events could be imported. All ${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
+ `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +
+ `Try importing newer data or upgrade your plan for higher monthly quotas.`;
+ await ImportStatusManager.updateStatus(importId, "failed", errorMessage);
+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
+ if (!deleteResult.success) {
+ console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);
+ }
+ return;
+ }
+
// Send final chunk if any data remains
if (chunk.length > 0) {
await jobQueue.send(DATA_INSERT_QUEUE, {