🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Fix race conditions
Edward Cui056dc64465ee
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 7eb6be59d..3fcc484b5 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -72,16 +72,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequeconst transformedEvents = UmamiImportMapper.transform(events, site, importId);const invalidEventCount = events.length - transformedEvents.length;- const eventsWithinQuota = [];- let skippedDueToQuota = 0;-- for (const event of transformedEvents) {- if (quotaTracker.canImportEvent(event.timestamp)) {- eventsWithinQuota.push(event);- } else {- skippedDueToQuota++;- }- }+ const timestamps = transformedEvents.map(e => e.timestamp);+ const allowedIndices = quotaTracker.canImportBatch(timestamps);++ const eventsWithinQuota = allowedIndices.map(i => transformedEvents[i]);+ const skippedDueToQuota = transformedEvents.length - eventsWithinQuota.length;if (eventsWithinQuota.length > 0) {await clickhouse.insert({diff --git a/server/src/services/import/importQuotaManager.ts b/server/src/services/import/importQuotaManager.tsindex 98d819822..8e445cd9c 100644--- a/server/src/services/import/importQuotaManager.ts+++ b/server/src/services/import/importQuotaManager.ts@@ -9,12 +9,13 @@ interface CachedTracker {class ImportQuotaManager {private trackers = new Map<string, CachedTracker>();private activeImports = new Map<string, number>(); // stores startedAt timestamp+ private trackerCreationLocks = new Map<string, Promise<ImportQuotaTracker>>(); // prevents duplicate tracker creationprivate readonly CONCURRENT_IMPORT_LIMIT = 1;private readonly TRACKER_TTL_MS = 30 * 60 * 1000; // 30 minutesprivate readonly IMPORT_TIMEOUT_MS = 2 * 60 * 60 * 1000; // 2 hours- async getTracker(organizationId: string) {+ async getTracker(organizationId: string): Promise<ImportQuotaTracker> {const now = Date.now();const cached = this.trackers.get(organizationId);@@ -23,10 +24,25 @@ class ImportQuotaManager {return cached.tracker;}- const tracker = await ImportQuotaTracker.create(organizationId);- this.trackers.set(organizationId, { tracker, lastAccessed: now });+ // Check if tracker is currently being created+ const existingCreation = this.trackerCreationLocks.get(organizationId);+ if (existingCreation) {+ return await existingCreation;+ }++ // Create lock for this organization's tracker creation+ const creationPromise = (async () => {+ try {+ const tracker = await ImportQuotaTracker.create(organizationId);+ this.trackers.set(organizationId, { tracker, lastAccessed: now });+ return tracker;+ } finally {+ this.trackerCreationLocks.delete(organizationId);+ }+ })();- return tracker;+ this.trackerCreationLocks.set(organizationId, creationPromise);+ return await creationPromise;}startImport(organizationId: string): boolean {diff --git a/server/src/services/import/importQuotaTracker.ts b/server/src/services/import/importQuotaTracker.tsindex 7b474d33c..a21a8fd5f 100644--- a/server/src/services/import/importQuotaTracker.ts+++ b/server/src/services/import/importQuotaTracker.ts@@ -155,33 +155,53 @@ export class ImportQuotaTracker {}}- canImportEvent(timestamp: string): boolean {+ /**+ * Atomically check and reserve quota for a batch of events.+ * This method prevents race conditions by checking all events in a batch+ * and atomically updating usage counts for all months at once.+ *+ * @param timestamps - Array of event timestamps to check+ * @returns Array of indices indicating which events can be imported+ */+ canImportBatch(timestamps: string[]): number[] {if (this.monthlyLimit === Infinity) {- return true;- }-- const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });- if (!dt.isValid) {- return false;+ return timestamps.map((_, i) => i);}+ const allowedIndices: number[] = [];+ const monthlyIncrements = new Map<string, number>();const now = DateTime.utc();- if (dt > now) {- return false;- }- const month = dt.toFormat("yyyyMM");- if (month < this.oldestAllowedMonth) {- return false;+ for (let i = 0; i < timestamps.length; i++) {+ const timestamp = timestamps[i];+ const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });++ if (!dt.isValid || dt > now) {+ continue;+ }++ const month = dt.toFormat("yyyyMM");+ if (month < this.oldestAllowedMonth) {+ continue;+ }++ const currentUsage = this.monthlyUsage.get(month) || 0;+ const incrementInBatch = monthlyIncrements.get(month) || 0;+ const totalUsage = currentUsage + incrementInBatch;++ if (totalUsage < this.monthlyLimit) {+ allowedIndices.push(i);+ monthlyIncrements.set(month, incrementInBatch + 1);+ }}- const used = this.monthlyUsage.get(month) || 0;- if (used >= this.monthlyLimit) {- return false;+ // Atomically apply all increments at once+ for (const [month, increment] of monthlyIncrements) {+ const current = this.monthlyUsage.get(month) || 0;+ this.monthlyUsage.set(month, current + increment);}- this.monthlyUsage.set(month, used + 1);- return true;+ return allowedIndices;}getOldestAllowedMonth(): string {