Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Fix race conditions

Edward Cui056dc64465ee
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 7eb6be59d..3fcc484b5 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -72,16 +72,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
const transformedEvents = UmamiImportMapper.transform(events, site, importId);
const invalidEventCount = events.length - transformedEvents.length;
- const eventsWithinQuota = [];
- let skippedDueToQuota = 0;
-
- for (const event of transformedEvents) {
- if (quotaTracker.canImportEvent(event.timestamp)) {
- eventsWithinQuota.push(event);
- } else {
- skippedDueToQuota++;
- }
- }
+ const timestamps = transformedEvents.map(e => e.timestamp);
+ const allowedIndices = quotaTracker.canImportBatch(timestamps);
+
+ const eventsWithinQuota = allowedIndices.map(i => transformedEvents[i]);
+ const skippedDueToQuota = transformedEvents.length - eventsWithinQuota.length;
if (eventsWithinQuota.length > 0) {
await clickhouse.insert({
diff --git a/server/src/services/import/importQuotaManager.ts b/server/src/services/import/importQuotaManager.ts
index 98d819822..8e445cd9c 100644
--- a/server/src/services/import/importQuotaManager.ts
+++ b/server/src/services/import/importQuotaManager.ts
@@ -9,12 +9,13 @@ interface CachedTracker {
class ImportQuotaManager {
private trackers = new Map<string, CachedTracker>();
private activeImports = new Map<string, number>(); // stores startedAt timestamp
+ private trackerCreationLocks = new Map<string, Promise<ImportQuotaTracker>>(); // prevents duplicate tracker creation
private readonly CONCURRENT_IMPORT_LIMIT = 1;
private readonly TRACKER_TTL_MS = 30 * 60 * 1000; // 30 minutes
private readonly IMPORT_TIMEOUT_MS = 2 * 60 * 60 * 1000; // 2 hours
- async getTracker(organizationId: string) {
+ async getTracker(organizationId: string): Promise<ImportQuotaTracker> {
const now = Date.now();
const cached = this.trackers.get(organizationId);
@@ -23,10 +24,25 @@ class ImportQuotaManager {
return cached.tracker;
}
- const tracker = await ImportQuotaTracker.create(organizationId);
- this.trackers.set(organizationId, { tracker, lastAccessed: now });
+ // Check if tracker is currently being created
+ const existingCreation = this.trackerCreationLocks.get(organizationId);
+ if (existingCreation) {
+ return await existingCreation;
+ }
+
+ // Create lock for this organization's tracker creation
+ const creationPromise = (async () => {
+ try {
+ const tracker = await ImportQuotaTracker.create(organizationId);
+ this.trackers.set(organizationId, { tracker, lastAccessed: now });
+ return tracker;
+ } finally {
+ this.trackerCreationLocks.delete(organizationId);
+ }
+ })();
- return tracker;
+ this.trackerCreationLocks.set(organizationId, creationPromise);
+ return await creationPromise;
}
startImport(organizationId: string): boolean {
diff --git a/server/src/services/import/importQuotaTracker.ts b/server/src/services/import/importQuotaTracker.ts
index 7b474d33c..a21a8fd5f 100644
--- a/server/src/services/import/importQuotaTracker.ts
+++ b/server/src/services/import/importQuotaTracker.ts
@@ -155,33 +155,53 @@ export class ImportQuotaTracker {
}
}
- canImportEvent(timestamp: string): boolean {
+ /**
+ * Atomically check and reserve quota for a batch of events.
+ * This method prevents race conditions by checking all events in a batch
+ * and atomically updating usage counts for all months at once.
+ *
+ * @param timestamps - Array of event timestamps to check
+ * @returns Array of indices indicating which events can be imported
+ */
+ canImportBatch(timestamps: string[]): number[] {
if (this.monthlyLimit === Infinity) {
- return true;
- }
-
- const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
- if (!dt.isValid) {
- return false;
+ return timestamps.map((_, i) => i);
}
+ const allowedIndices: number[] = [];
+ const monthlyIncrements = new Map<string, number>();
const now = DateTime.utc();
- if (dt > now) {
- return false;
- }
- const month = dt.toFormat("yyyyMM");
- if (month < this.oldestAllowedMonth) {
- return false;
+ for (let i = 0; i < timestamps.length; i++) {
+ const timestamp = timestamps[i];
+ const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
+
+ if (!dt.isValid || dt > now) {
+ continue;
+ }
+
+ const month = dt.toFormat("yyyyMM");
+ if (month < this.oldestAllowedMonth) {
+ continue;
+ }
+
+ const currentUsage = this.monthlyUsage.get(month) || 0;
+ const incrementInBatch = monthlyIncrements.get(month) || 0;
+ const totalUsage = currentUsage + incrementInBatch;
+
+ if (totalUsage < this.monthlyLimit) {
+ allowedIndices.push(i);
+ monthlyIncrements.set(month, incrementInBatch + 1);
+ }
}
- const used = this.monthlyUsage.get(month) || 0;
- if (used >= this.monthlyLimit) {
- return false;
+ // Atomically apply all increments at once
+ for (const [month, increment] of monthlyIncrements) {
+ const current = this.monthlyUsage.get(month) || 0;
+ this.monthlyUsage.set(month, current + increment);
}
- this.monthlyUsage.set(month, used + 1);
- return true;
+ return allowedIndices;
}
getOldestAllowedMonth(): string {