Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Remove status

Edward Cui89d02a61f31d
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.ts
index 838092f6c..6ee076647 100644
--- a/client/src/api/admin/import.ts
+++ b/client/src/api/admin/import.ts
@@ -5,10 +5,11 @@ import { APIResponse } from "@/api/types";
interface GetSiteImportsResponse {
importId: string;
platform: "umami" | null;
- status: "pending" | "processing" | "completed" | "failed";
importedEvents: number;
- errorMessage: string | null;
+ skippedEvents: number;
+ invalidEvents: number;
startedAt: string;
+ completedAt: string | null;
}
interface CreateSiteImportResponse {
@@ -28,9 +29,8 @@ export function useGetSiteImports(site: number) {
queryKey: ["get-site-imports", site],
queryFn: async () => await authedFetch<APIResponse<GetSiteImportsResponse[]>>(`/get-site-imports/${site}`),
refetchInterval: data => {
- const hasActiveImports = data.state.data?.data.some(
- imp => imp.status === "processing" || imp.status === "pending"
- );
+ // Check if there are any imports that haven't completed yet (completedAt is null)
+ const hasActiveImports = data.state.data?.data.some(imp => imp.completedAt === null);
return hasActiveImports ? 5000 : false;
},
placeholderData: { data: [] },
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsx
index 2170b9246..8a6423e4d 100644
--- a/client/src/components/SiteSettings/ImportManager.tsx
+++ b/client/src/components/SiteSettings/ImportManager.tsx
@@ -147,38 +147,19 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
}
};
- const getStatusInfo = (status: string) => {
- switch (status.toLowerCase()) {
- case "completed":
- return {
- color: "bg-green-100 text-green-800 border-green-200",
- icon: CheckCircle2,
- label: "Completed",
- };
- case "failed":
- return {
- color: "bg-red-100 text-red-800 border-red-200",
- icon: AlertCircle,
- label: "Failed",
- };
- case "processing":
- return {
- color: "bg-blue-100 text-blue-800 border-blue-200",
- icon: Loader2,
- label: "Processing",
- };
- case "pending":
- return {
- color: "bg-yellow-100 text-yellow-800 border-yellow-200",
- icon: Clock,
- label: "Pending",
- };
- default:
- return {
- color: "bg-gray-100 text-gray-800 border-gray-200",
- icon: Clock,
- label: status,
- };
+ const getStatusInfo = (completedAt: string | null) => {
+ if (completedAt === null) {
+ return {
+ color: "bg-blue-100 text-blue-800 border-blue-200",
+ icon: Loader2,
+ label: "In Progress",
+ };
+ } else {
+ return {
+ color: "bg-green-100 text-green-800 border-green-200",
+ icon: CheckCircle2,
+ label: "Completed",
+ };
}
};
@@ -195,8 +176,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
}, [data?.data]);
// Check if there's an active import (cloud only)
- const hasActiveImport =
- IS_CLOUD && sortedImports.some(imp => imp.status === "pending" || imp.status === "processing");
+ const hasActiveImport = IS_CLOUD && sortedImports.some(imp => imp.completedAt === null);
const isImportDisabled =
!selectedFile || !!fileError || createImportMutation.isPending || disabled || hasActiveImport;
@@ -340,13 +320,15 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<TableHead>Started At</TableHead>
<TableHead>Platform</TableHead>
<TableHead>Status</TableHead>
- <TableHead className="text-right">Events</TableHead>
+ <TableHead className="text-right">Imported</TableHead>
+ <TableHead className="text-right">Skipped</TableHead>
+ <TableHead className="text-right">Invalid</TableHead>
<TableHead className="text-right">Actions</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{sortedImports.map(imp => {
- const statusInfo = getStatusInfo(imp.status);
+ const statusInfo = getStatusInfo(imp.completedAt);
const StatusIcon = statusInfo.icon;
const startedAt = DateTime.fromISO(imp.startedAt).toFormat("MMM dd, yyyy HH:mm");
@@ -359,30 +341,48 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
</Badge>
</TableCell>
<TableCell>
- <div className="flex items-center gap-2">
- <Badge className={`${statusInfo.color} flex items-center gap-1`}>
- <StatusIcon className={`h-3 w-3 ${imp.status === "processing" ? "animate-spin" : ""}`} />
- {statusInfo.label}
- </Badge>
- {imp.errorMessage && (
- <TooltipProvider>
- <Tooltip>
- <TooltipTrigger asChild>
- <button type="button" className="text-red-600 hover:text-red-700">
- <Info className="h-4 w-4" />
- </button>
- </TooltipTrigger>
- <TooltipContent side="right" className="max-w-sm">
- <p className="text-sm">{imp.errorMessage}</p>
- </TooltipContent>
- </Tooltip>
- </TooltipProvider>
- )}
- </div>
+ <Badge className={`${statusInfo.color} flex items-center gap-1`}>
+ <StatusIcon className={`h-3 w-3 ${imp.completedAt === null ? "animate-spin" : ""}`} />
+ {statusInfo.label}
+ </Badge>
</TableCell>
<TableCell className="text-right">{imp.importedEvents.toLocaleString()}</TableCell>
<TableCell className="text-right">
- {(imp.status === "completed" || imp.status === "failed") && (
+ {imp.skippedEvents > 0 ? (
+ <TooltipProvider>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="text-yellow-600 cursor-help">
+ {imp.skippedEvents.toLocaleString()}
+ </span>
+ </TooltipTrigger>
+ <TooltipContent>
+ <p className="text-sm">Events exceeded quota or date range limits</p>
+ </TooltipContent>
+ </Tooltip>
+ </TooltipProvider>
+ ) : (
+ <span className="text-muted-foreground">0</span>
+ )}
+ </TableCell>
+ <TableCell className="text-right">
+ {imp.invalidEvents > 0 ? (
+ <TooltipProvider>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="text-red-600 cursor-help">{imp.invalidEvents.toLocaleString()}</span>
+ </TooltipTrigger>
+ <TooltipContent>
+ <p className="text-sm">Events failed validation</p>
+ </TooltipContent>
+ </Tooltip>
+ </TooltipProvider>
+ ) : (
+ <span className="text-muted-foreground">0</span>
+ )}
+ </TableCell>
+ <TableCell className="text-right">
+ {imp.completedAt !== null && (
<Button
variant="outline"
size="sm"
diff --git a/client/src/lib/import/csvParser.ts b/client/src/lib/import/csvParser.ts
index 074515e96..548c8a122 100644
--- a/client/src/lib/import/csvParser.ts
+++ b/client/src/lib/import/csvParser.ts
@@ -26,9 +26,14 @@ interface UmamiEvent {
}
+export interface ImportProgress {
+ importedEvents: number;
+ skippedEvents: number;
+ invalidEvents: number;
+}
+
export class CsvParser {
private aborted = false;
- private quotaExceeded = false;
private siteId: number = 0;
private importId: string = "";
@@ -39,15 +44,25 @@ export class CsvParser {
private isProcessing = false;
private parsingComplete = false;
+ // Track cumulative counts
+ private totalImported = 0;
+ private totalSkipped = 0;
+ private totalInvalid = 0;
+
+ // Callback for progress updates
+ private onProgress?: (progress: ImportProgress) => void;
+
startImport(
file: File,
siteId: number,
importId: string,
earliestAllowedDate: string,
- latestAllowedDate: string
+ latestAllowedDate: string,
+ onProgress?: (progress: ImportProgress) => void
): void {
this.siteId = siteId;
this.importId = importId;
+ this.onProgress = onProgress;
this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");
this.latestAllowedDate = DateTime.fromFormat(latestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).endOf("day");
@@ -68,7 +83,7 @@ export class CsvParser {
worker: true,
chunkSize: 9 * 1024 * 1024,
chunk: (results, parser) => {
- if (this.aborted || this.quotaExceeded) {
+ if (this.aborted) {
parser.abort();
return;
}
@@ -102,7 +117,7 @@ export class CsvParser {
this.isProcessing = true;
while (this.uploadQueue.length > 0) {
- if (this.aborted || this.quotaExceeded) {
+ if (this.aborted) {
this.uploadQueue = [];
break;
}
@@ -169,10 +184,6 @@ export class CsvParser {
}
private async uploadChunk(events: UmamiEvent[], isLastBatch: boolean): Promise<void> {
- if (this.quotaExceeded) {
- return;
- }
-
// Skip empty chunks unless it's the last one (needed for finalization)
if (events.length === 0 && !isLastBatch) {
return;
@@ -180,8 +191,9 @@ export class CsvParser {
try {
const data = await authedFetch<{
- quotaExceeded?: boolean;
- message?: string;
+ imported: number;
+ skipped: number;
+ invalid: number;
}>(`/api/batch-import-events/${this.siteId}/${this.importId}`, undefined, {
method: "POST",
data: {
@@ -190,12 +202,21 @@ export class CsvParser {
},
});
- if (data.quotaExceeded) {
- this.quotaExceeded = true;
- this.aborted = true;
- return;
+ // Update cumulative counts
+ this.totalImported += data.imported;
+ this.totalSkipped += data.skipped;
+ this.totalInvalid += data.invalid;
+
+ // Notify progress callback if provided
+ if (this.onProgress) {
+ this.onProgress({
+ importedEvents: this.totalImported,
+ skippedEvents: this.totalSkipped,
+ invalidEvents: this.totalInvalid,
+ });
}
} catch (error) {
+ console.error("Error uploading chunk:", error);
this.aborted = true;
return;
}
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 4d7689652..df410a61d 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -2,9 +2,9 @@ import { FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
-import { updateImportProgress, updateImportStatus, getImportById } from "../../services/import/importStatusManager.js";
+import { updateImportProgress, completeImport, getImportById } from "../../services/import/importStatusManager.js";
import { UmamiImportMapper } from "../../services/import/mappings/umami.js";
-import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";
+import { importQuotaManager } from "../../services/import/importQuotaManager.js";
import { db } from "../../db/postgres/postgres.js";
import { sites, importStatus } from "../../db/postgres/schema.js";
import { eq } from "drizzle-orm";
@@ -47,7 +47,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
return reply.status(403).send({ error: "Forbidden" });
}
- // Verify import exists and is in valid state
+ // Verify import exists
const importRecord = await getImportById(importId);
if (!importRecord) {
return reply.status(404).send({ error: "Import not found" });
@@ -57,18 +57,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
return reply.status(400).send({ error: "Import does not belong to this site" });
}
- if (importRecord.status === "completed") {
+ // Check if import is already completed
+ if (importRecord.completedAt) {
return reply.status(400).send({ error: "Import already completed" });
}
- if (importRecord.status === "failed") {
- return reply.status(400).send({ error: "Import has failed" });
- }
-
- if (importRecord.status === "pending") {
- await updateImportStatus(importId, "processing");
- }
-
// Auto-detect platform if not set (first batch)
let detectedPlatform = importRecord.platform;
if (!detectedPlatform) {
@@ -94,11 +87,14 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
}
try {
- const quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);
+ // Get cached quota tracker from singleton (much faster!)
+ const quotaTracker = await importQuotaManager.getTracker(siteRecord.organizationId);
+ // Transform events (validate and convert to internal format)
const transformedEvents = UmamiImportMapper.transform(events, site, importId);
const invalidEventCount = events.length - transformedEvents.length;
+ // Filter events through quota checker
const eventsWithinQuota = [];
let skippedDueToQuota = 0;
@@ -110,57 +106,33 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
}
}
- if (eventsWithinQuota.length === 0) {
- const quotaSummary = quotaTracker.getSummary();
- let quotaMessage =
- `All ${events.length} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
- `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity.`;
-
- if (invalidEventCount > 0) {
- quotaMessage += ` ${invalidEventCount} events were invalid.`;
- }
-
- if (isLastBatch) {
- await updateImportStatus(importId, "completed", quotaMessage);
- }
-
- return reply.send({
- quotaExceeded: true,
- message: quotaMessage,
+ // Insert events to ClickHouse (even if 0 events, we still need to track progress)
+ if (eventsWithinQuota.length > 0) {
+ await clickhouse.insert({
+ table: "events",
+ values: eventsWithinQuota,
+ format: "JSONEachRow",
});
}
- await clickhouse.insert({
- table: "events",
- values: eventsWithinQuota,
- format: "JSONEachRow",
- });
-
- await updateImportProgress(importId, eventsWithinQuota.length);
+ // Update import progress
+ await updateImportProgress(importId, eventsWithinQuota.length, skippedDueToQuota, invalidEventCount);
+ // If this is the last batch, mark import as completed
if (isLastBatch) {
- let finalMessage: string | undefined = undefined;
- const messages: string[] = [];
-
- if (skippedDueToQuota > 0) {
- messages.push(`${skippedDueToQuota} events were skipped due to quota limits`);
- }
-
- if (invalidEventCount > 0) {
- messages.push(`${invalidEventCount} events were invalid`);
- }
-
- if (messages.length > 0) {
- finalMessage = `Import completed. ${messages.join("; ")}.`;
- }
-
- await updateImportStatus(importId, "completed", finalMessage);
+ await completeImport(importId);
+ importQuotaManager.completeImport(siteRecord.organizationId, importId);
}
- return reply.send({});
+ // Return counts to client
+ return reply.send({
+ imported: eventsWithinQuota.length,
+ skipped: skippedDueToQuota,
+ invalid: invalidEventCount,
+ });
} catch (insertError) {
const errorMessage = insertError instanceof Error ? insertError.message : "Unknown error";
- await updateImportStatus(importId, "failed", `Failed to insert events: ${errorMessage}`);
+ console.error("Failed to insert events:", errorMessage);
return reply.status(500).send({
error: `Failed to insert events: ${errorMessage}`,
diff --git a/server/src/api/sites/createSiteImport.ts b/server/src/api/sites/createSiteImport.ts
index bb97ba890..e14c1d8e9 100644
--- a/server/src/api/sites/createSiteImport.ts
+++ b/server/src/api/sites/createSiteImport.ts
@@ -1,9 +1,11 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
-import { ImportLimiter } from "../../services/import/importLimiter.js";
-import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";
+import { importQuotaManager } from "../../services/import/importQuotaManager.js";
import { DateTime } from "luxon";
import { z } from "zod";
+import { db } from "../../db/postgres/postgres.js";
+import { sites, importStatus } from "../../db/postgres/schema.js";
+import { eq } from "drizzle-orm";
const createSiteImportRequestSchema = z
.object({
@@ -35,26 +37,26 @@ export async function createSiteImport(request: FastifyRequest<CreateSiteImportR
return reply.status(403).send({ error: "Forbidden" });
}
- // Check organization and get initial limit check
- const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(siteId);
- if (!concurrentImportLimitResult.allowed) {
- return reply.status(429).send({ error: concurrentImportLimitResult.reason });
- }
+ // Get site's organization
+ const [siteRecord] = await db
+ .select({ organizationId: sites.organizationId })
+ .from(sites)
+ .where(eq(sites.siteId, siteId))
+ .limit(1);
- const organization = concurrentImportLimitResult.organizationId;
+ if (!siteRecord) {
+ return reply.status(404).send({ error: "Site not found" });
+ }
- // Atomically create import status with concurrency check to prevent race conditions
- const importResult = await ImportLimiter.createImportWithConcurrencyCheck({
- siteId,
- organizationId: organization,
- });
+ const organizationId = siteRecord.organizationId;
- if (!importResult.success) {
- return reply.status(429).send({ error: importResult.reason });
+ // Check concurrent import limit using singleton
+ if (!importQuotaManager.canStartImport(organizationId)) {
+ return reply.status(429).send({ error: "Only 1 concurrent import allowed per organization" });
}
- // Get quota information to determine allowed date ranges
- const quotaTracker = await ImportQuotaTracker.create(organization);
+ // Get quota information from cached tracker
+ const quotaTracker = await importQuotaManager.getTracker(organizationId);
const summary = quotaTracker.getSummary();
// Calculate the earliest and latest allowed dates
@@ -62,9 +64,21 @@ export async function createSiteImport(request: FastifyRequest<CreateSiteImportR
const earliestAllowedDate = oldestAllowedDate.toFormat("yyyy-MM-dd");
const latestAllowedDate = DateTime.utc().toFormat("yyyy-MM-dd");
+ // Create import record
+ const [importRecord] = await db
+ .insert(importStatus)
+ .values({
+ siteId,
+ organizationId,
+ })
+ .returning({ importId: importStatus.importId });
+
+ // Register import with quota manager
+ importQuotaManager.registerImport(organizationId, importRecord.importId);
+
return reply.send({
data: {
- importId: importResult.importId,
+ importId: importRecord.importId,
allowedDateRange: {
earliestAllowedDate,
latestAllowedDate,
diff --git a/server/src/api/sites/deleteSiteImport.ts b/server/src/api/sites/deleteSiteImport.ts
index a340d9bf3..cecd30943 100644
--- a/server/src/api/sites/deleteSiteImport.ts
+++ b/server/src/api/sites/deleteSiteImport.ts
@@ -3,6 +3,10 @@ import { z } from "zod";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { getImportById, deleteImport } from "../../services/import/importStatusManager.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
+import { importQuotaManager } from "../../services/import/importQuotaManager.js";
+import { db } from "../../db/postgres/postgres.js";
+import { sites } from "../../db/postgres/schema.js";
+import { eq } from "drizzle-orm";
const deleteImportRequestSchema = z
.object({
@@ -43,12 +47,20 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportReque
return reply.status(403).send({ error: "Import does not belong to this site" });
}
- if (importRecord.status === "pending" || importRecord.status === "processing") {
+ // Cannot delete imports that are still in progress
+ if (importRecord.completedAt === null) {
return reply.status(400).send({ error: "Cannot delete active import" });
}
const siteId = Number(site);
+ // Get organization ID for quota manager notification
+ const [siteRecord] = await db
+ .select({ organizationId: sites.organizationId })
+ .from(sites)
+ .where(eq(sites.siteId, siteId))
+ .limit(1);
+
try {
await clickhouse.command({
query: `DELETE FROM events WHERE import_id = {importId:String} AND site_id = {siteId:UInt16}`,
@@ -74,6 +86,11 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportReque
});
}
+ // Notify quota manager that import is no longer active
+ if (siteRecord) {
+ importQuotaManager.completeImport(siteRecord.organizationId, importId);
+ }
+
return reply.send({
data: {
message: "Import deleted successfully",
diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.ts
index dc426b6b0..d5a7613b9 100644
--- a/server/src/api/sites/getSiteImports.ts
+++ b/server/src/api/sites/getSiteImports.ts
@@ -36,14 +36,17 @@ export async function getSiteImports(request: FastifyRequest<GetSiteImportsReque
const imports = await getImportsForSite(siteId);
return reply.send({
- data: imports.map(({ importId, platform, status, importedEvents, errorMessage, startedAt }) => ({
- importId,
- platform,
- status,
- importedEvents,
- errorMessage,
- startedAt,
- })),
+ data: imports.map(
+ ({ importId, platform, importedEvents, skippedEvents, invalidEvents, startedAt, completedAt }) => ({
+ importId,
+ platform,
+ importedEvents,
+ skippedEvents,
+ invalidEvents,
+ startedAt,
+ completedAt,
+ })
+ ),
});
} catch (error) {
console.error("Error fetching imports:", error);
diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.ts
index 2e657a664..12b217aa7 100644
--- a/server/src/db/postgres/schema.ts
+++ b/server/src/db/postgres/schema.ts
@@ -624,8 +624,6 @@ export const gscConnections = pgTable("gsc_connections", {
export const importPlatformEnum = pgEnum("import_platform_enum", ["umami"]);
-export const importStatusEnum = pgEnum("import_status_enum", ["pending", "processing", "completed", "failed"]);
-
export const importStatus = pgTable(
"import_status",
{
@@ -633,9 +631,9 @@ export const importStatus = pgTable(
siteId: integer("site_id").notNull(),
organizationId: text("organization_id").notNull(),
platform: importPlatformEnum("platform"),
- status: importStatusEnum("status").notNull().default("pending"),
importedEvents: integer("imported_events").notNull().default(0),
- errorMessage: text("error_message"),
+ skippedEvents: integer("skipped_events").notNull().default(0),
+ invalidEvents: integer("invalid_events").notNull().default(0),
startedAt: timestamp("started_at", { mode: "string" }).notNull().defaultNow(),
completedAt: timestamp("completed_at", { mode: "string" }),
},
diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.ts
deleted file mode 100644
index efb6ee441..000000000
--- a/server/src/services/import/importLimiter.ts
+++ /dev/null
@@ -1,99 +0,0 @@
-import { IS_CLOUD } from "../../lib/const.js";
-import { db } from "../../db/postgres/postgres.js";
-import { and, count, eq, inArray } from "drizzle-orm";
-import { importStatus, sites } from "../../db/postgres/schema.js";
-import type { InsertImportStatus } from "./importStatusManager.js";
-
-export class ImportLimiter {
- private static readonly CONCURRENT_IMPORT_LIMIT = 1;
-
- /**
- * Check concurrent import limit for a site.
- * Returns organization ID if allowed.
- */
- static async checkConcurrentImportLimit(siteId: number): Promise<
- | {
- allowed: false;
- reason: string;
- }
- | {
- allowed: true;
- organizationId: string;
- }
- > {
- const [siteResult] = await db
- .select({ organizationId: sites.organizationId })
- .from(sites)
- .where(eq(sites.siteId, siteId))
- .limit(1);
-
- if (!siteResult) {
- return { allowed: false, reason: "Site not found." };
- }
-
- // For self-hosted, no limit
- if (!IS_CLOUD) {
- return { allowed: true, organizationId: siteResult.organizationId };
- }
-
- // Check current concurrent imports
- // Note: This check alone has a race condition, but the actual import creation
- // is atomic (see createImportWithConcurrencyCheck)
- const [concurrentImportResult] = await db
- .select({ count: count() })
- .from(importStatus)
- .where(
- and(
- eq(importStatus.organizationId, siteResult.organizationId),
- inArray(importStatus.status, ["pending", "processing"])
- )
- );
-
- if (concurrentImportResult.count >= this.CONCURRENT_IMPORT_LIMIT) {
- return {
- allowed: false,
- reason: `Only ${this.CONCURRENT_IMPORT_LIMIT} concurrent import allowed per organization.`,
- };
- }
-
- return { allowed: true, organizationId: siteResult.organizationId };
- }
-
- /**
- * Create an import status record with atomic concurrent import check.
- * This prevents race conditions by checking and inserting in a single transaction.
- */
- static async createImportWithConcurrencyCheck(
- data: InsertImportStatus
- ): Promise<{ success: true; importId: string } | { success: false; reason: string }> {
- if (!IS_CLOUD) {
- const [result] = await db.insert(importStatus).values(data).returning({ importId: importStatus.importId });
- return { success: true, importId: result.importId };
- }
-
- // Use a transaction to atomically check and insert
- return await db.transaction(async tx => {
- // Lock the rows to prevent concurrent inserts
- const [concurrentImportResult] = await tx
- .select({ count: count() })
- .from(importStatus)
- .where(
- and(
- eq(importStatus.organizationId, data.organizationId),
- inArray(importStatus.status, ["pending", "processing"])
- )
- )
- .for("update");
-
- if (concurrentImportResult.count >= this.CONCURRENT_IMPORT_LIMIT) {
- return {
- success: false,
- reason: `Only ${this.CONCURRENT_IMPORT_LIMIT} concurrent import allowed per organization.`,
- };
- }
-
- const [result] = await tx.insert(importStatus).values(data).returning({ importId: importStatus.importId });
- return { success: true, importId: result.importId };
- });
- }
-}
diff --git a/server/src/services/import/importQuotaManager.ts b/server/src/services/import/importQuotaManager.ts
new file mode 100644
index 000000000..ea56bcd9b
--- /dev/null
+++ b/server/src/services/import/importQuotaManager.ts
@@ -0,0 +1,195 @@
+import { ImportQuotaTracker } from "./importQuotaChecker.js";
+import { IS_CLOUD } from "../../lib/const.js";
+
+interface CachedTracker {
+ tracker: ImportQuotaTracker;
+ lastAccessed: number;
+}
+
+interface ActiveImport {
+ importId: string;
+ startedAt: number;
+}
+
+/**
+ * Singleton service that manages quota trackers and rate limiting for imports
+ * Caches quota trackers per organization to avoid expensive ClickHouse queries
+ */
+class ImportQuotaManager {
+ // Cache of quota trackers by organizationId
+ private trackers: Map<string, CachedTracker> = new Map();
+
+ // Active imports tracking for rate limiting
+ private activeImports: Map<string, Set<ActiveImport>> = new Map();
+
+ // Configuration
+ private readonly CONCURRENT_IMPORT_LIMIT = 1; // Max concurrent imports per organization
+ private readonly TRACKER_TTL_MS = 30 * 60 * 1000; // 30 minutes
+ private readonly IMPORT_TIMEOUT_MS = 2 * 60 * 60 * 1000; // 2 hours
+
+ /**
+ * Get a quota tracker for the given organization
+ * Returns cached tracker if available, otherwise creates a new one
+ */
+ async getTracker(organizationId: string): Promise<ImportQuotaTracker> {
+ const now = Date.now();
+ const cached = this.trackers.get(organizationId);
+
+ // Return cached tracker if it's still valid
+ if (cached && now - cached.lastAccessed < this.TRACKER_TTL_MS) {
+ cached.lastAccessed = now;
+ return cached.tracker;
+ }
+
+ // Create new tracker
+ const tracker = await ImportQuotaTracker.create(organizationId);
+ this.trackers.set(organizationId, {
+ tracker,
+ lastAccessed: now,
+ });
+
+ return tracker;
+ }
+
+ /**
+ * Check if an organization can start a new import
+ * Based on concurrent import limit
+ */
+ canStartImport(organizationId: string): boolean {
+ if (!IS_CLOUD) {
+ return true; // No limits for self-hosted
+ }
+
+ const activeSet = this.activeImports.get(organizationId);
+ if (!activeSet) {
+ return true; // No active imports
+ }
+
+ // Clean up abandoned imports before checking
+ this.cleanupAbandonedImports(organizationId);
+
+ return activeSet.size < this.CONCURRENT_IMPORT_LIMIT;
+ }
+
+ /**
+ * Register a new import as active
+ */
+ registerImport(organizationId: string, importId: string): void {
+ if (!IS_CLOUD) {
+ return; // No tracking for self-hosted
+ }
+
+ let activeSet = this.activeImports.get(organizationId);
+ if (!activeSet) {
+ activeSet = new Set();
+ this.activeImports.set(organizationId, activeSet);
+ }
+
+ activeSet.add({
+ importId,
+ startedAt: Date.now(),
+ });
+ }
+
+ /**
+ * Mark an import as completed (remove from active tracking)
+ */
+ completeImport(organizationId: string, importId: string): void {
+ const activeSet = this.activeImports.get(organizationId);
+ if (!activeSet) {
+ return;
+ }
+
+ // Find and remove the import
+ for (const activeImport of activeSet) {
+ if (activeImport.importId === importId) {
+ activeSet.delete(activeImport);
+ break;
+ }
+ }
+
+ // Clean up empty sets
+ if (activeSet.size === 0) {
+ this.activeImports.delete(organizationId);
+ }
+ }
+
+ /**
+ * Remove abandoned imports (no activity for > IMPORT_TIMEOUT_MS)
+ */
+ private cleanupAbandonedImports(organizationId: string): void {
+ const activeSet = this.activeImports.get(organizationId);
+ if (!activeSet) {
+ return;
+ }
+
+ const now = Date.now();
+ const toRemove: ActiveImport[] = [];
+
+ for (const activeImport of activeSet) {
+ if (now - activeImport.startedAt > this.IMPORT_TIMEOUT_MS) {
+ toRemove.push(activeImport);
+ }
+ }
+
+ for (const importToRemove of toRemove) {
+ activeSet.delete(importToRemove);
+ }
+
+ // Clean up empty sets
+ if (activeSet.size === 0) {
+ this.activeImports.delete(organizationId);
+ }
+ }
+
+ /**
+ * Periodic cleanup of stale trackers and abandoned imports
+ * Called automatically by setInterval
+ */
+ cleanup(): void {
+ const now = Date.now();
+
+ // Clean up stale trackers
+ for (const [orgId, cached] of this.trackers.entries()) {
+ if (now - cached.lastAccessed > this.TRACKER_TTL_MS) {
+ this.trackers.delete(orgId);
+ }
+ }
+
+ // Clean up abandoned imports across all organizations
+ for (const orgId of this.activeImports.keys()) {
+ this.cleanupAbandonedImports(orgId);
+ }
+ }
+
+ /**
+ * Get debug information about current state
+ */
+ getDebugInfo(): {
+ cachedTrackers: number;
+ activeImports: Record<string, number>;
+ } {
+ const activeImports: Record<string, number> = {};
+ for (const [orgId, activeSet] of this.activeImports.entries()) {
+ activeImports[orgId] = activeSet.size;
+ }
+
+ return {
+ cachedTrackers: this.trackers.size,
+ activeImports,
+ };
+ }
+}
+
+// Export singleton instance
+export const importQuotaManager = new ImportQuotaManager();
+
+// Set up automatic cleanup every 15 minutes
+if (IS_CLOUD) {
+ setInterval(
+ () => {
+ importQuotaManager.cleanup();
+ },
+ 15 * 60 * 1000
+ );
+}
diff --git a/server/src/services/import/importStatusManager.ts b/server/src/services/import/importStatusManager.ts
index 0588663fa..aaf8a36f8 100644
--- a/server/src/services/import/importStatusManager.ts
+++ b/server/src/services/import/importStatusManager.ts
@@ -6,21 +6,27 @@ import { DateTime } from "luxon";
export type SelectImportStatus = typeof importStatus.$inferSelect;
export type InsertImportStatus = typeof importStatus.$inferInsert;
-export async function updateImportStatus(
+export async function updateImportProgress(
importId: string,
- status: InsertImportStatus["status"],
- errorMessage?: string
+ importedEvents: number,
+ skippedEvents: number,
+ invalidEvents: number
): Promise<void> {
- const completedAt = status === "completed" || status === "failed" ? DateTime.utc().toISO() : null;
-
- await db.update(importStatus).set({ status, errorMessage, completedAt }).where(eq(importStatus.importId, importId));
+ await db
+ .update(importStatus)
+ .set({
+ importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`,
+ skippedEvents: sql`${importStatus.skippedEvents} + ${skippedEvents}`,
+ invalidEvents: sql`${importStatus.invalidEvents} + ${invalidEvents}`,
+ })
+ .where(eq(importStatus.importId, importId));
}
-export async function updateImportProgress(importId: string, importedEvents: number): Promise<void> {
+export async function completeImport(importId: string): Promise<void> {
await db
.update(importStatus)
.set({
- importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`,
+ completedAt: DateTime.utc().toISO(),
})
.where(eq(importStatus.importId, importId));
}