🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Remove status
Edward Cui89d02a61f31d
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.tsindex 838092f6c..6ee076647 100644--- a/client/src/api/admin/import.ts+++ b/client/src/api/admin/import.ts@@ -5,10 +5,11 @@ import { APIResponse } from "@/api/types";interface GetSiteImportsResponse {importId: string;platform: "umami" | null;- status: "pending" | "processing" | "completed" | "failed";importedEvents: number;- errorMessage: string | null;+ skippedEvents: number;+ invalidEvents: number;startedAt: string;+ completedAt: string | null;}interface CreateSiteImportResponse {@@ -28,9 +29,8 @@ export function useGetSiteImports(site: number) {queryKey: ["get-site-imports", site],queryFn: async () => await authedFetch<APIResponse<GetSiteImportsResponse[]>>(`/get-site-imports/${site}`),refetchInterval: data => {- const hasActiveImports = data.state.data?.data.some(- imp => imp.status === "processing" || imp.status === "pending"- );+ // Check if there are any imports that haven't completed yet (completedAt is null)+ const hasActiveImports = data.state.data?.data.some(imp => imp.completedAt === null);return hasActiveImports ? 5000 : false;},placeholderData: { data: [] },diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex 2170b9246..8a6423e4d 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -147,38 +147,19 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {}};- const getStatusInfo = (status: string) => {- switch (status.toLowerCase()) {- case "completed":- return {- color: "bg-green-100 text-green-800 border-green-200",- icon: CheckCircle2,- label: "Completed",- };- case "failed":- return {- color: "bg-red-100 text-red-800 border-red-200",- icon: AlertCircle,- label: "Failed",- };- case "processing":- return {- color: "bg-blue-100 text-blue-800 border-blue-200",- icon: Loader2,- label: "Processing",- };- case "pending":- return {- color: "bg-yellow-100 text-yellow-800 border-yellow-200",- icon: Clock,- label: "Pending",- };- default:- return {- color: "bg-gray-100 text-gray-800 border-gray-200",- icon: Clock,- label: status,- };+ const getStatusInfo = (completedAt: string | null) => {+ if (completedAt === null) {+ return {+ color: "bg-blue-100 text-blue-800 border-blue-200",+ icon: Loader2,+ label: "In Progress",+ };+ } else {+ return {+ color: "bg-green-100 text-green-800 border-green-200",+ icon: CheckCircle2,+ label: "Completed",+ };}};@@ -195,8 +176,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {}, [data?.data]);// Check if there's an active import (cloud only)- const hasActiveImport =- IS_CLOUD && sortedImports.some(imp => imp.status === "pending" || imp.status === "processing");+ const hasActiveImport = IS_CLOUD && sortedImports.some(imp => imp.completedAt === null);const isImportDisabled =!selectedFile || !!fileError || createImportMutation.isPending || disabled || hasActiveImport;@@ -340,13 +320,15 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<TableHead>Started At</TableHead><TableHead>Platform</TableHead><TableHead>Status</TableHead>- <TableHead className="text-right">Events</TableHead>+ <TableHead className="text-right">Imported</TableHead>+ <TableHead className="text-right">Skipped</TableHead>+ <TableHead className="text-right">Invalid</TableHead><TableHead className="text-right">Actions</TableHead></TableRow></TableHeader><TableBody>{sortedImports.map(imp => {- const statusInfo = getStatusInfo(imp.status);+ const statusInfo = getStatusInfo(imp.completedAt);const StatusIcon = statusInfo.icon;const startedAt = DateTime.fromISO(imp.startedAt).toFormat("MMM dd, yyyy HH:mm");@@ -359,30 +341,48 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</Badge></TableCell><TableCell>- <div className="flex items-center gap-2">- <Badge className={`${statusInfo.color} flex items-center gap-1`}>- <StatusIcon className={`h-3 w-3 ${imp.status === "processing" ? "animate-spin" : ""}`} />- {statusInfo.label}- </Badge>- {imp.errorMessage && (- <TooltipProvider>- <Tooltip>- <TooltipTrigger asChild>- <button type="button" className="text-red-600 hover:text-red-700">- <Info className="h-4 w-4" />- </button>- </TooltipTrigger>- <TooltipContent side="right" className="max-w-sm">- <p className="text-sm">{imp.errorMessage}</p>- </TooltipContent>- </Tooltip>- </TooltipProvider>- )}- </div>+ <Badge className={`${statusInfo.color} flex items-center gap-1`}>+ <StatusIcon className={`h-3 w-3 ${imp.completedAt === null ? "animate-spin" : ""}`} />+ {statusInfo.label}+ </Badge></TableCell><TableCell className="text-right">{imp.importedEvents.toLocaleString()}</TableCell><TableCell className="text-right">- {(imp.status === "completed" || imp.status === "failed") && (+ {imp.skippedEvents > 0 ? (+ <TooltipProvider>+ <Tooltip>+ <TooltipTrigger asChild>+ <span className="text-yellow-600 cursor-help">+ {imp.skippedEvents.toLocaleString()}+ </span>+ </TooltipTrigger>+ <TooltipContent>+ <p className="text-sm">Events exceeded quota or date range limits</p>+ </TooltipContent>+ </Tooltip>+ </TooltipProvider>+ ) : (+ <span className="text-muted-foreground">0</span>+ )}+ </TableCell>+ <TableCell className="text-right">+ {imp.invalidEvents > 0 ? (+ <TooltipProvider>+ <Tooltip>+ <TooltipTrigger asChild>+ <span className="text-red-600 cursor-help">{imp.invalidEvents.toLocaleString()}</span>+ </TooltipTrigger>+ <TooltipContent>+ <p className="text-sm">Events failed validation</p>+ </TooltipContent>+ </Tooltip>+ </TooltipProvider>+ ) : (+ <span className="text-muted-foreground">0</span>+ )}+ </TableCell>+ <TableCell className="text-right">+ {imp.completedAt !== null && (<Buttonvariant="outline"size="sm"diff --git a/client/src/lib/import/csvParser.ts b/client/src/lib/import/csvParser.tsindex 074515e96..548c8a122 100644--- a/client/src/lib/import/csvParser.ts+++ b/client/src/lib/import/csvParser.ts@@ -26,9 +26,14 @@ interface UmamiEvent {}+export interface ImportProgress {+ importedEvents: number;+ skippedEvents: number;+ invalidEvents: number;+}+export class CsvParser {private aborted = false;- private quotaExceeded = false;private siteId: number = 0;private importId: string = "";@@ -39,15 +44,25 @@ export class CsvParser {private isProcessing = false;private parsingComplete = false;+ // Track cumulative counts+ private totalImported = 0;+ private totalSkipped = 0;+ private totalInvalid = 0;++ // Callback for progress updates+ private onProgress?: (progress: ImportProgress) => void;+startImport(file: File,siteId: number,importId: string,earliestAllowedDate: string,- latestAllowedDate: string+ latestAllowedDate: string,+ onProgress?: (progress: ImportProgress) => void): void {this.siteId = siteId;this.importId = importId;+ this.onProgress = onProgress;this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");this.latestAllowedDate = DateTime.fromFormat(latestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).endOf("day");@@ -68,7 +83,7 @@ export class CsvParser {worker: true,chunkSize: 9 * 1024 * 1024,chunk: (results, parser) => {- if (this.aborted || this.quotaExceeded) {+ if (this.aborted) {parser.abort();return;}@@ -102,7 +117,7 @@ export class CsvParser {this.isProcessing = true;while (this.uploadQueue.length > 0) {- if (this.aborted || this.quotaExceeded) {+ if (this.aborted) {this.uploadQueue = [];break;}@@ -169,10 +184,6 @@ export class CsvParser {}private async uploadChunk(events: UmamiEvent[], isLastBatch: boolean): Promise<void> {- if (this.quotaExceeded) {- return;- }-// Skip empty chunks unless it's the last one (needed for finalization)if (events.length === 0 && !isLastBatch) {return;@@ -180,8 +191,9 @@ export class CsvParser {try {const data = await authedFetch<{- quotaExceeded?: boolean;- message?: string;+ imported: number;+ skipped: number;+ invalid: number;}>(`/api/batch-import-events/${this.siteId}/${this.importId}`, undefined, {method: "POST",data: {@@ -190,12 +202,21 @@ export class CsvParser {},});- if (data.quotaExceeded) {- this.quotaExceeded = true;- this.aborted = true;- return;+ // Update cumulative counts+ this.totalImported += data.imported;+ this.totalSkipped += data.skipped;+ this.totalInvalid += data.invalid;++ // Notify progress callback if provided+ if (this.onProgress) {+ this.onProgress({+ importedEvents: this.totalImported,+ skippedEvents: this.totalSkipped,+ invalidEvents: this.totalInvalid,+ });}} catch (error) {+ console.error("Error uploading chunk:", error);this.aborted = true;return;}diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 4d7689652..df410a61d 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -2,9 +2,9 @@ import { FastifyReply, FastifyRequest } from "fastify";import { z } from "zod";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";-import { updateImportProgress, updateImportStatus, getImportById } from "../../services/import/importStatusManager.js";+import { updateImportProgress, completeImport, getImportById } from "../../services/import/importStatusManager.js";import { UmamiImportMapper } from "../../services/import/mappings/umami.js";-import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";+import { importQuotaManager } from "../../services/import/importQuotaManager.js";import { db } from "../../db/postgres/postgres.js";import { sites, importStatus } from "../../db/postgres/schema.js";import { eq } from "drizzle-orm";@@ -47,7 +47,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequereturn reply.status(403).send({ error: "Forbidden" });}- // Verify import exists and is in valid state+ // Verify import existsconst importRecord = await getImportById(importId);if (!importRecord) {return reply.status(404).send({ error: "Import not found" });@@ -57,18 +57,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequereturn reply.status(400).send({ error: "Import does not belong to this site" });}- if (importRecord.status === "completed") {+ // Check if import is already completed+ if (importRecord.completedAt) {return reply.status(400).send({ error: "Import already completed" });}- if (importRecord.status === "failed") {- return reply.status(400).send({ error: "Import has failed" });- }-- if (importRecord.status === "pending") {- await updateImportStatus(importId, "processing");- }-// Auto-detect platform if not set (first batch)let detectedPlatform = importRecord.platform;if (!detectedPlatform) {@@ -94,11 +87,14 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}try {- const quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);+ // Get cached quota tracker from singleton (much faster!)+ const quotaTracker = await importQuotaManager.getTracker(siteRecord.organizationId);+ // Transform events (validate and convert to internal format)const transformedEvents = UmamiImportMapper.transform(events, site, importId);const invalidEventCount = events.length - transformedEvents.length;+ // Filter events through quota checkerconst eventsWithinQuota = [];let skippedDueToQuota = 0;@@ -110,57 +106,33 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}}- if (eventsWithinQuota.length === 0) {- const quotaSummary = quotaTracker.getSummary();- let quotaMessage =- `All ${events.length} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +- `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity.`;-- if (invalidEventCount > 0) {- quotaMessage += ` ${invalidEventCount} events were invalid.`;- }-- if (isLastBatch) {- await updateImportStatus(importId, "completed", quotaMessage);- }-- return reply.send({- quotaExceeded: true,- message: quotaMessage,+ // Insert events to ClickHouse (even if 0 events, we still need to track progress)+ if (eventsWithinQuota.length > 0) {+ await clickhouse.insert({+ table: "events",+ values: eventsWithinQuota,+ format: "JSONEachRow",});}- await clickhouse.insert({- table: "events",- values: eventsWithinQuota,- format: "JSONEachRow",- });-- await updateImportProgress(importId, eventsWithinQuota.length);+ // Update import progress+ await updateImportProgress(importId, eventsWithinQuota.length, skippedDueToQuota, invalidEventCount);+ // If this is the last batch, mark import as completedif (isLastBatch) {- let finalMessage: string | undefined = undefined;- const messages: string[] = [];-- if (skippedDueToQuota > 0) {- messages.push(`${skippedDueToQuota} events were skipped due to quota limits`);- }-- if (invalidEventCount > 0) {- messages.push(`${invalidEventCount} events were invalid`);- }-- if (messages.length > 0) {- finalMessage = `Import completed. ${messages.join("; ")}.`;- }-- await updateImportStatus(importId, "completed", finalMessage);+ await completeImport(importId);+ importQuotaManager.completeImport(siteRecord.organizationId, importId);}- return reply.send({});+ // Return counts to client+ return reply.send({+ imported: eventsWithinQuota.length,+ skipped: skippedDueToQuota,+ invalid: invalidEventCount,+ });} catch (insertError) {const errorMessage = insertError instanceof Error ? insertError.message : "Unknown error";- await updateImportStatus(importId, "failed", `Failed to insert events: ${errorMessage}`);+ console.error("Failed to insert events:", errorMessage);return reply.status(500).send({error: `Failed to insert events: ${errorMessage}`,diff --git a/server/src/api/sites/createSiteImport.ts b/server/src/api/sites/createSiteImport.tsindex bb97ba890..e14c1d8e9 100644--- a/server/src/api/sites/createSiteImport.ts+++ b/server/src/api/sites/createSiteImport.ts@@ -1,9 +1,11 @@import { FastifyReply, FastifyRequest } from "fastify";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";-import { ImportLimiter } from "../../services/import/importLimiter.js";-import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";+import { importQuotaManager } from "../../services/import/importQuotaManager.js";import { DateTime } from "luxon";import { z } from "zod";+import { db } from "../../db/postgres/postgres.js";+import { sites, importStatus } from "../../db/postgres/schema.js";+import { eq } from "drizzle-orm";const createSiteImportRequestSchema = z.object({@@ -35,26 +37,26 @@ export async function createSiteImport(request: FastifyRequest<CreateSiteImportRreturn reply.status(403).send({ error: "Forbidden" });}- // Check organization and get initial limit check- const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(siteId);- if (!concurrentImportLimitResult.allowed) {- return reply.status(429).send({ error: concurrentImportLimitResult.reason });- }+ // Get site's organization+ const [siteRecord] = await db+ .select({ organizationId: sites.organizationId })+ .from(sites)+ .where(eq(sites.siteId, siteId))+ .limit(1);- const organization = concurrentImportLimitResult.organizationId;+ if (!siteRecord) {+ return reply.status(404).send({ error: "Site not found" });+ }- // Atomically create import status with concurrency check to prevent race conditions- const importResult = await ImportLimiter.createImportWithConcurrencyCheck({- siteId,- organizationId: organization,- });+ const organizationId = siteRecord.organizationId;- if (!importResult.success) {- return reply.status(429).send({ error: importResult.reason });+ // Check concurrent import limit using singleton+ if (!importQuotaManager.canStartImport(organizationId)) {+ return reply.status(429).send({ error: "Only 1 concurrent import allowed per organization" });}- // Get quota information to determine allowed date ranges- const quotaTracker = await ImportQuotaTracker.create(organization);+ // Get quota information from cached tracker+ const quotaTracker = await importQuotaManager.getTracker(organizationId);const summary = quotaTracker.getSummary();// Calculate the earliest and latest allowed dates@@ -62,9 +64,21 @@ export async function createSiteImport(request: FastifyRequest<CreateSiteImportRconst earliestAllowedDate = oldestAllowedDate.toFormat("yyyy-MM-dd");const latestAllowedDate = DateTime.utc().toFormat("yyyy-MM-dd");+ // Create import record+ const [importRecord] = await db+ .insert(importStatus)+ .values({+ siteId,+ organizationId,+ })+ .returning({ importId: importStatus.importId });++ // Register import with quota manager+ importQuotaManager.registerImport(organizationId, importRecord.importId);+return reply.send({data: {- importId: importResult.importId,+ importId: importRecord.importId,allowedDateRange: {earliestAllowedDate,latestAllowedDate,diff --git a/server/src/api/sites/deleteSiteImport.ts b/server/src/api/sites/deleteSiteImport.tsindex a340d9bf3..cecd30943 100644--- a/server/src/api/sites/deleteSiteImport.ts+++ b/server/src/api/sites/deleteSiteImport.ts@@ -3,6 +3,10 @@ import { z } from "zod";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { getImportById, deleteImport } from "../../services/import/importStatusManager.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";+import { importQuotaManager } from "../../services/import/importQuotaManager.js";+import { db } from "../../db/postgres/postgres.js";+import { sites } from "../../db/postgres/schema.js";+import { eq } from "drizzle-orm";const deleteImportRequestSchema = z.object({@@ -43,12 +47,20 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportRequereturn reply.status(403).send({ error: "Import does not belong to this site" });}- if (importRecord.status === "pending" || importRecord.status === "processing") {+ // Cannot delete imports that are still in progress+ if (importRecord.completedAt === null) {return reply.status(400).send({ error: "Cannot delete active import" });}const siteId = Number(site);+ // Get organization ID for quota manager notification+ const [siteRecord] = await db+ .select({ organizationId: sites.organizationId })+ .from(sites)+ .where(eq(sites.siteId, siteId))+ .limit(1);+try {await clickhouse.command({query: `DELETE FROM events WHERE import_id = {importId:String} AND site_id = {siteId:UInt16}`,@@ -74,6 +86,11 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportReque});}+ // Notify quota manager that import is no longer active+ if (siteRecord) {+ importQuotaManager.completeImport(siteRecord.organizationId, importId);+ }+return reply.send({data: {message: "Import deleted successfully",diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.tsindex dc426b6b0..d5a7613b9 100644--- a/server/src/api/sites/getSiteImports.ts+++ b/server/src/api/sites/getSiteImports.ts@@ -36,14 +36,17 @@ export async function getSiteImports(request: FastifyRequest<GetSiteImportsRequeconst imports = await getImportsForSite(siteId);return reply.send({- data: imports.map(({ importId, platform, status, importedEvents, errorMessage, startedAt }) => ({- importId,- platform,- status,- importedEvents,- errorMessage,- startedAt,- })),+ data: imports.map(+ ({ importId, platform, importedEvents, skippedEvents, invalidEvents, startedAt, completedAt }) => ({+ importId,+ platform,+ importedEvents,+ skippedEvents,+ invalidEvents,+ startedAt,+ completedAt,+ })+ ),});} catch (error) {console.error("Error fetching imports:", error);diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.tsindex 2e657a664..12b217aa7 100644--- a/server/src/db/postgres/schema.ts+++ b/server/src/db/postgres/schema.ts@@ -624,8 +624,6 @@ export const gscConnections = pgTable("gsc_connections", {export const importPlatformEnum = pgEnum("import_platform_enum", ["umami"]);-export const importStatusEnum = pgEnum("import_status_enum", ["pending", "processing", "completed", "failed"]);-export const importStatus = pgTable("import_status",{@@ -633,9 +631,9 @@ export const importStatus = pgTable(siteId: integer("site_id").notNull(),organizationId: text("organization_id").notNull(),platform: importPlatformEnum("platform"),- status: importStatusEnum("status").notNull().default("pending"),importedEvents: integer("imported_events").notNull().default(0),- errorMessage: text("error_message"),+ skippedEvents: integer("skipped_events").notNull().default(0),+ invalidEvents: integer("invalid_events").notNull().default(0),startedAt: timestamp("started_at", { mode: "string" }).notNull().defaultNow(),completedAt: timestamp("completed_at", { mode: "string" }),},diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.tsdeleted file mode 100644index efb6ee441..000000000--- a/server/src/services/import/importLimiter.ts+++ /dev/null@@ -1,99 +0,0 @@-import { IS_CLOUD } from "../../lib/const.js";-import { db } from "../../db/postgres/postgres.js";-import { and, count, eq, inArray } from "drizzle-orm";-import { importStatus, sites } from "../../db/postgres/schema.js";-import type { InsertImportStatus } from "./importStatusManager.js";--export class ImportLimiter {- private static readonly CONCURRENT_IMPORT_LIMIT = 1;-- /**- * Check concurrent import limit for a site.- * Returns organization ID if allowed.- */- static async checkConcurrentImportLimit(siteId: number): Promise<- | {- allowed: false;- reason: string;- }- | {- allowed: true;- organizationId: string;- }- > {- const [siteResult] = await db- .select({ organizationId: sites.organizationId })- .from(sites)- .where(eq(sites.siteId, siteId))- .limit(1);-- if (!siteResult) {- return { allowed: false, reason: "Site not found." };- }-- // For self-hosted, no limit- if (!IS_CLOUD) {- return { allowed: true, organizationId: siteResult.organizationId };- }-- // Check current concurrent imports- // Note: This check alone has a race condition, but the actual import creation- // is atomic (see createImportWithConcurrencyCheck)- const [concurrentImportResult] = await db- .select({ count: count() })- .from(importStatus)- .where(- and(- eq(importStatus.organizationId, siteResult.organizationId),- inArray(importStatus.status, ["pending", "processing"])- )- );-- if (concurrentImportResult.count >= this.CONCURRENT_IMPORT_LIMIT) {- return {- allowed: false,- reason: `Only ${this.CONCURRENT_IMPORT_LIMIT} concurrent import allowed per organization.`,- };- }-- return { allowed: true, organizationId: siteResult.organizationId };- }-- /**- * Create an import status record with atomic concurrent import check.- * This prevents race conditions by checking and inserting in a single transaction.- */- static async createImportWithConcurrencyCheck(- data: InsertImportStatus- ): Promise<{ success: true; importId: string } | { success: false; reason: string }> {- if (!IS_CLOUD) {- const [result] = await db.insert(importStatus).values(data).returning({ importId: importStatus.importId });- return { success: true, importId: result.importId };- }-- // Use a transaction to atomically check and insert- return await db.transaction(async tx => {- // Lock the rows to prevent concurrent inserts- const [concurrentImportResult] = await tx- .select({ count: count() })- .from(importStatus)- .where(- and(- eq(importStatus.organizationId, data.organizationId),- inArray(importStatus.status, ["pending", "processing"])- )- )- .for("update");-- if (concurrentImportResult.count >= this.CONCURRENT_IMPORT_LIMIT) {- return {- success: false,- reason: `Only ${this.CONCURRENT_IMPORT_LIMIT} concurrent import allowed per organization.`,- };- }-- const [result] = await tx.insert(importStatus).values(data).returning({ importId: importStatus.importId });- return { success: true, importId: result.importId };- });- }-}diff --git a/server/src/services/import/importQuotaManager.ts b/server/src/services/import/importQuotaManager.tsnew file mode 100644index 000000000..ea56bcd9b--- /dev/null+++ b/server/src/services/import/importQuotaManager.ts@@ -0,0 +1,195 @@+import { ImportQuotaTracker } from "./importQuotaChecker.js";+import { IS_CLOUD } from "../../lib/const.js";++interface CachedTracker {+ tracker: ImportQuotaTracker;+ lastAccessed: number;+}++interface ActiveImport {+ importId: string;+ startedAt: number;+}++/**+ * Singleton service that manages quota trackers and rate limiting for imports+ * Caches quota trackers per organization to avoid expensive ClickHouse queries+ */+class ImportQuotaManager {+ // Cache of quota trackers by organizationId+ private trackers: Map<string, CachedTracker> = new Map();++ // Active imports tracking for rate limiting+ private activeImports: Map<string, Set<ActiveImport>> = new Map();++ // Configuration+ private readonly CONCURRENT_IMPORT_LIMIT = 1; // Max concurrent imports per organization+ private readonly TRACKER_TTL_MS = 30 * 60 * 1000; // 30 minutes+ private readonly IMPORT_TIMEOUT_MS = 2 * 60 * 60 * 1000; // 2 hours++ /**+ * Get a quota tracker for the given organization+ * Returns cached tracker if available, otherwise creates a new one+ */+ async getTracker(organizationId: string): Promise<ImportQuotaTracker> {+ const now = Date.now();+ const cached = this.trackers.get(organizationId);++ // Return cached tracker if it's still valid+ if (cached && now - cached.lastAccessed < this.TRACKER_TTL_MS) {+ cached.lastAccessed = now;+ return cached.tracker;+ }++ // Create new tracker+ const tracker = await ImportQuotaTracker.create(organizationId);+ this.trackers.set(organizationId, {+ tracker,+ lastAccessed: now,+ });++ return tracker;+ }++ /**+ * Check if an organization can start a new import+ * Based on concurrent import limit+ */+ canStartImport(organizationId: string): boolean {+ if (!IS_CLOUD) {+ return true; // No limits for self-hosted+ }++ const activeSet = this.activeImports.get(organizationId);+ if (!activeSet) {+ return true; // No active imports+ }++ // Clean up abandoned imports before checking+ this.cleanupAbandonedImports(organizationId);++ return activeSet.size < this.CONCURRENT_IMPORT_LIMIT;+ }++ /**+ * Register a new import as active+ */+ registerImport(organizationId: string, importId: string): void {+ if (!IS_CLOUD) {+ return; // No tracking for self-hosted+ }++ let activeSet = this.activeImports.get(organizationId);+ if (!activeSet) {+ activeSet = new Set();+ this.activeImports.set(organizationId, activeSet);+ }++ activeSet.add({+ importId,+ startedAt: Date.now(),+ });+ }++ /**+ * Mark an import as completed (remove from active tracking)+ */+ completeImport(organizationId: string, importId: string): void {+ const activeSet = this.activeImports.get(organizationId);+ if (!activeSet) {+ return;+ }++ // Find and remove the import+ for (const activeImport of activeSet) {+ if (activeImport.importId === importId) {+ activeSet.delete(activeImport);+ break;+ }+ }++ // Clean up empty sets+ if (activeSet.size === 0) {+ this.activeImports.delete(organizationId);+ }+ }++ /**+ * Remove abandoned imports (no activity for > IMPORT_TIMEOUT_MS)+ */+ private cleanupAbandonedImports(organizationId: string): void {+ const activeSet = this.activeImports.get(organizationId);+ if (!activeSet) {+ return;+ }++ const now = Date.now();+ const toRemove: ActiveImport[] = [];++ for (const activeImport of activeSet) {+ if (now - activeImport.startedAt > this.IMPORT_TIMEOUT_MS) {+ toRemove.push(activeImport);+ }+ }++ for (const importToRemove of toRemove) {+ activeSet.delete(importToRemove);+ }++ // Clean up empty sets+ if (activeSet.size === 0) {+ this.activeImports.delete(organizationId);+ }+ }++ /**+ * Periodic cleanup of stale trackers and abandoned imports+ * Called automatically by setInterval+ */+ cleanup(): void {+ const now = Date.now();++ // Clean up stale trackers+ for (const [orgId, cached] of this.trackers.entries()) {+ if (now - cached.lastAccessed > this.TRACKER_TTL_MS) {+ this.trackers.delete(orgId);+ }+ }++ // Clean up abandoned imports across all organizations+ for (const orgId of this.activeImports.keys()) {+ this.cleanupAbandonedImports(orgId);+ }+ }++ /**+ * Get debug information about current state+ */+ getDebugInfo(): {+ cachedTrackers: number;+ activeImports: Record<string, number>;+ } {+ const activeImports: Record<string, number> = {};+ for (const [orgId, activeSet] of this.activeImports.entries()) {+ activeImports[orgId] = activeSet.size;+ }++ return {+ cachedTrackers: this.trackers.size,+ activeImports,+ };+ }+}++// Export singleton instance+export const importQuotaManager = new ImportQuotaManager();++// Set up automatic cleanup every 15 minutes+if (IS_CLOUD) {+ setInterval(+ () => {+ importQuotaManager.cleanup();+ },+ 15 * 60 * 1000+ );+}diff --git a/server/src/services/import/importStatusManager.ts b/server/src/services/import/importStatusManager.tsindex 0588663fa..aaf8a36f8 100644--- a/server/src/services/import/importStatusManager.ts+++ b/server/src/services/import/importStatusManager.ts@@ -6,21 +6,27 @@ import { DateTime } from "luxon";export type SelectImportStatus = typeof importStatus.$inferSelect;export type InsertImportStatus = typeof importStatus.$inferInsert;-export async function updateImportStatus(+export async function updateImportProgress(importId: string,- status: InsertImportStatus["status"],- errorMessage?: string+ importedEvents: number,+ skippedEvents: number,+ invalidEvents: number): Promise<void> {- const completedAt = status === "completed" || status === "failed" ? DateTime.utc().toISO() : null;-- await db.update(importStatus).set({ status, errorMessage, completedAt }).where(eq(importStatus.importId, importId));+ await db+ .update(importStatus)+ .set({+ importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`,+ skippedEvents: sql`${importStatus.skippedEvents} + ${skippedEvents}`,+ invalidEvents: sql`${importStatus.invalidEvents} + ${invalidEvents}`,+ })+ .where(eq(importStatus.importId, importId));}-export async function updateImportProgress(importId: string, importedEvents: number): Promise<void> {+export async function completeImport(importId: string): Promise<void> {await db.update(importStatus).set({- importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`,+ completedAt: DateTime.utc().toISO(),}).where(eq(importStatus.importId, importId));}