Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

A bunch of improvements

Edward Cuife9407536fa6
diff --git a/server/src/api/sites/deleteSiteImport.ts b/server/src/api/sites/deleteSiteImport.ts
index 53a1df234..eb6bf0c74 100644
--- a/server/src/api/sites/deleteSiteImport.ts
+++ b/server/src/api/sites/deleteSiteImport.ts
@@ -1,10 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
-import { ImportStatusManager } from "../../services/import/importStatusManager.js";
-import { deleteImportFile } from "../../services/import/utils.js";
-import { IS_CLOUD } from "../../lib/const.js";
-import { r2Storage } from "../../services/storage/r2StorageService.js";
+import { getImportById, deleteImport } from "../../services/import/importStatusManager.js";
+import { deleteImportFile, getImportStorageLocation } from "../../services/import/utils.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
const deleteImportRequestSchema = z
@@ -37,7 +35,7 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportReque
return reply.status(403).send({ error: "Forbidden" });
}
- const importRecord = await ImportStatusManager.getImportById(importId);
+ const importRecord = await getImportById(importId);
if (!importRecord) {
return reply.status(404).send({ error: "Import not found" });
}
@@ -50,19 +48,17 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportReque
return reply.status(400).send({ error: "Cannot delete active import" });
}
- // Delete the import file if it exists
- try {
- let storageLocation: string;
- if (IS_CLOUD && r2Storage.isEnabled()) {
- storageLocation = `imports/${importId}/${importRecord.fileName}`;
- } else {
- storageLocation = `/tmp/imports/${importId}.csv`;
- }
+ const siteId = Number(site);
- await deleteImportFile(storageLocation, IS_CLOUD && r2Storage.isEnabled());
- } catch (fileError) {
- // Log the error but don't fail the deletion - the file might not exist
- console.warn(`Failed to delete import file for ${importId}:`, fileError);
+ // Delete the import record from the database FIRST (fail fast)
+ // If this fails, we haven't deleted any data yet
+ try {
+ await deleteImport(importId);
+ } catch (dbError) {
+ console.error(`Failed to delete import record ${importId}:`, dbError);
+ return reply.status(500).send({
+ error: "Failed to delete import record",
+ });
}
// Delete events from ClickHouse that were imported with this importId
@@ -72,7 +68,7 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportReque
query: `DELETE FROM events WHERE import_id = {importId:String} AND site_id = {siteId:UInt16}`,
query_params: {
importId: importId,
- siteId: Number(site),
+ siteId: siteId,
},
});
console.log(`Deleted events for import ${importId} from ClickHouse`);
@@ -83,14 +79,12 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportReque
});
}
- // Delete the import record from the database
- try {
- await ImportStatusManager.deleteImport(importId);
- } catch (dbError) {
- console.error(`Failed to delete import record ${importId}:`, dbError);
- return reply.status(500).send({
- error: "Import events deleted but failed to remove import record",
- });
+ // Delete the import file if it exists
+ // This is best-effort - we don't fail the entire operation if the file is already gone
+ const storage = getImportStorageLocation(importId, importRecord.fileName);
+ const deleteResult = await deleteImportFile(storage.location, storage.isR2);
+ if (!deleteResult.success) {
+ console.warn(`Failed to delete import file for ${importId}: ${deleteResult.error}`);
}
return reply.send({
diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.ts
index 4c3319582..082244861 100644
--- a/server/src/api/sites/getSiteImports.ts
+++ b/server/src/api/sites/getSiteImports.ts
@@ -1,6 +1,6 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
-import { ImportStatusManager } from "../../services/import/importStatusManager.js";
+import { getImportsForSite } from "../../services/import/importStatusManager.js";
import { z } from "zod";
const getSiteImportsRequestSchema = z
@@ -26,23 +26,24 @@ export async function getSiteImports(request: FastifyRequest<GetSiteImportsReque
}
const { site } = parsed.data.params;
+ const siteId = Number(site);
const userHasAccess = await getUserHasAdminAccessToSite(request, site);
if (!userHasAccess) {
return reply.status(403).send({ error: "Forbidden" });
}
- const imports = await ImportStatusManager.getImportsForSite(Number(site));
+ const imports = await getImportsForSite(siteId);
return reply.send({
- data: imports.map(imp => ({
- importId: imp.importId,
- source: imp.source,
- status: imp.status,
- importedEvents: imp.importedEvents,
- errorMessage: imp.errorMessage,
- startedAt: imp.startedAt,
- fileName: imp.fileName,
+ data: imports.map(({ importId, source, status, importedEvents, errorMessage, startedAt, fileName }) => ({
+ importId,
+ source,
+ status,
+ importedEvents,
+ errorMessage,
+ startedAt,
+ fileName,
})),
});
} catch (error) {
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
index 07ca13e5f..f9c2e5df3 100644
--- a/server/src/api/sites/importSiteData.ts
+++ b/server/src/api/sites/importSiteData.ts
@@ -1,17 +1,15 @@
import { createWriteStream } from "node:fs";
import { mkdir } from "node:fs/promises";
import { randomUUID } from "node:crypto";
-import path from "node:path";
import { pipeline } from "node:stream/promises";
import { FastifyRequest, FastifyReply } from "fastify";
import { DateTime } from "luxon";
import { z } from "zod";
-import { IS_CLOUD } from "../../lib/const.js";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { getJobQueue } from "../../queues/jobQueueFactory.js";
import { ImportLimiter } from "../../services/import/importLimiter.js";
-import { ImportStatusManager } from "../../services/import/importStatusManager.js";
-import { deleteImportFile } from "../../services/import/utils.js";
+import { updateImportStatus } from "../../services/import/importStatusManager.js";
+import { deleteImportFile, getImportStorageLocation } from "../../services/import/utils.js";
import { CSV_PARSE_QUEUE } from "../../services/import/workers/jobs.js";
import { r2Storage } from "../../services/storage/r2StorageService.js";
@@ -27,25 +25,24 @@ const importDataFieldsSchema = z
fields: z
.object({
source: z.enum(["umami"]),
- startDate: z.string().refine(isValidDate).optional(),
- endDate: z.string().refine(isValidDate).optional(),
+ startDate: z.string().refine(isValidDate, { message: "Invalid start date format" }).optional(),
+ endDate: z.string().refine(isValidDate, { message: "Invalid end date format" }).optional(),
})
- .refine(fields => {
- if (fields.startDate && fields.endDate) {
- const start = parseDate(fields.startDate);
- const end = parseDate(fields.endDate);
- return start <= end;
- }
- return true;
- })
- .refine(fields => {
- if (fields.startDate) {
+ .refine(
+ data => {
+ if (!data.startDate || !data.endDate) return true;
+ return parseDate(data.startDate) <= parseDate(data.endDate);
+ },
+ { message: "Start date must be before or equal to end date" }
+ )
+ .refine(
+ data => {
+ if (!data.startDate) return true;
const today = DateTime.utc().startOf("day");
- const start = parseDate(fields.startDate);
- return start <= today;
- }
- return true;
- }),
+ return parseDate(data.startDate) <= today;
+ },
+ { message: "Start date cannot be in the future" }
+ ),
})
.strict();
@@ -78,11 +75,6 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,
return reply.status(403).send({ error: "Forbidden" });
}
- const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(Number(site));
- if (!concurrentImportLimitResult.allowed) {
- return reply.status(429).send({ error: concurrentImportLimitResult.reason });
- }
-
const data = await request.file();
if (!data) {
return reply.status(400).send({ error: "No file uploaded." });
@@ -105,41 +97,45 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,
}
const { source, startDate, endDate } = parsedFields.data.fields;
- const organization = concurrentImportLimitResult.organizationId;
+ const siteId = Number(site);
const importId = randomUUID();
- await ImportStatusManager.createImportStatus({
+ // Check organization and get initial limit check
+ const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(siteId);
+ if (!concurrentImportLimitResult.allowed) {
+ return reply.status(429).send({ error: concurrentImportLimitResult.reason });
+ }
+
+ const organization = concurrentImportLimitResult.organizationId;
+
+ // Atomically create import status with concurrency check to prevent race conditions
+ const createResult = await ImportLimiter.createImportWithConcurrencyCheck({
importId,
- siteId: Number(site),
+ siteId,
organizationId: organization,
source,
status: "pending",
fileName: data.filename,
});
- let storageLocation: string;
-
- try {
- if (IS_CLOUD && r2Storage.isEnabled()) {
- const r2Key = `imports/${importId}/${data.filename}`;
+ if (!createResult.success) {
+ return reply.status(429).send({ error: createResult.reason });
+ }
- await r2Storage.storeImportFile(r2Key, data.file);
- storageLocation = r2Key;
+ const storage = getImportStorageLocation(importId, data.filename);
- console.log(`[Import] File streamed to R2: ${r2Key}`);
+ try {
+ if (storage.isR2) {
+ await r2Storage.storeImportFile(storage.location, data.file);
+ console.log(`[Import] File streamed to R2: ${storage.location}`);
} else {
const importDir = "/tmp/imports";
- const savedFileName = `${importId}.csv`;
- const tempFilePath = path.join(importDir, savedFileName);
-
await mkdir(importDir, { recursive: true });
- await pipeline(data.file, createWriteStream(tempFilePath));
- storageLocation = tempFilePath;
-
- console.log(`[Import] File stored locally: ${tempFilePath}`);
+ await pipeline(data.file, createWriteStream(storage.location));
+ console.log(`[Import] File stored locally: ${storage.location}`);
}
} catch (fileError) {
- await ImportStatusManager.updateStatus(importId, "failed", "Failed to save uploaded file");
+ await updateImportStatus(importId, "failed", "Failed to save uploaded file");
console.error("Failed to save uploaded file:", fileError);
return reply.status(500).send({ error: "Could not process file upload." });
}
@@ -150,15 +146,15 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,
site,
importId,
source,
- storageLocation,
- isR2Storage: IS_CLOUD && r2Storage.isEnabled(),
+ storageLocation: storage.location,
+ isR2Storage: storage.isR2,
organization,
startDate,
endDate,
});
} catch (queueError) {
- await ImportStatusManager.updateStatus(importId, "failed", "Failed to queue import job");
- await deleteImportFile(storageLocation, IS_CLOUD && r2Storage.isEnabled());
+ await updateImportStatus(importId, "failed", "Failed to queue import job");
+ await deleteImportFile(storage.location, storage.isR2);
console.error("Failed to enqueue import job:", queueError);
return reply.status(500).send({ error: "Failed to initiate import process." });
}
diff --git a/server/src/index.ts b/server/src/index.ts
index cdee3952e..9716e1e75 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -75,6 +75,7 @@ import { trackEvent } from "./services/tracker/trackEvent.js";
// need to import telemetry service here to start it
import { telemetryService } from "./services/telemetryService.js";
import { weeklyReportService } from "./services/weekyReports/weeklyReportService.js";
+import { importCleanupService } from "./services/import/importCleanupService.js";
import { extractSiteId } from "./utils.js";
import { getTrackingConfig } from "./api/sites/getTrackingConfig.js";
import { getSiteImports } from "./api/sites/getSiteImports.js";
@@ -441,6 +442,9 @@ const start = async () => {
await registerCsvParseWorker();
await registerDataInsertWorker();
+ // Initialize import cleanup service (runs daily to clean up old import files)
+ importCleanupService.initializeCleanupCron();
+
// Start the server first
await server.listen({ port: 3001, host: "0.0.0.0" });
server.log.info("Server is listening on http://0.0.0.0:3001");
diff --git a/server/src/lib/subscriptionUtils.ts b/server/src/lib/subscriptionUtils.ts
index 3629d3905..b0efce97b 100644
--- a/server/src/lib/subscriptionUtils.ts
+++ b/server/src/lib/subscriptionUtils.ts
@@ -38,6 +38,7 @@ export function getTierInfo(planName: string): TierInfo {
}
// Default to free tier for unknown plans
+ console.warn(`Unknown plan name: "${planName}", defaulting to Free tier`);
return {
tier: "Free",
monthsAllowed: 6,
diff --git a/server/src/queues/adapters/bullMQAdapter.ts b/server/src/queues/adapters/bullMQAdapter.ts
index 6210b6855..0862e4450 100644
--- a/server/src/queues/adapters/bullMQAdapter.ts
+++ b/server/src/queues/adapters/bullMQAdapter.ts
@@ -175,8 +175,11 @@ export class BullMQAdapter implements IJobQueue {
data: job.data,
};
- // BullMQ processes one job at a time per worker
- // We wrap it in an array to match the interface
+ // NOTE: Interface Impedance Mismatch
+ // BullMQ processes one job at a time per worker, while pg-boss can batch jobs.
+ // The IJobQueue interface expects handlers to accept arrays to support pg-boss batching.
+ // Here we wrap single BullMQ jobs in an array to maintain interface compatibility.
+ // This is an acceptable tradeoff: pg-boss gets true batching, BullMQ gets array of 1 item.
await handler([normalizedJob]);
},
{
diff --git a/server/src/services/import/importCleanupService.ts b/server/src/services/import/importCleanupService.ts
new file mode 100644
index 000000000..e8cb47315
--- /dev/null
+++ b/server/src/services/import/importCleanupService.ts
@@ -0,0 +1,104 @@
+import * as cron from "node-cron";
+import { DateTime } from "luxon";
+import { db } from "../../db/postgres/postgres.js";
+import { importStatus } from "../../db/postgres/schema.js";
+import { sql } from "drizzle-orm";
+import { deleteImportFile, getImportStorageLocation } from "./utils.js";
+
+class ImportCleanupService {
+ private cleanupTask: cron.ScheduledTask | null = null;
+
+ constructor() {}
+
+ /**
+ * Initialize the cleanup cron job.
+ * Runs daily at 2 AM UTC to clean up orphaned import files.
+ */
+ initializeCleanupCron() {
+ console.info("[ImportCleanup] Initializing cleanup cron");
+
+ // Schedule cleanup to run daily at 2 AM UTC
+ this.cleanupTask = cron.schedule(
+ "0 2 * * *",
+ async () => {
+ try {
+ await this.cleanupOrphanedFiles();
+ } catch (error) {
+ console.error("[ImportCleanup] Error during cleanup:", error);
+ }
+ },
+ { timezone: "UTC" }
+ );
+
+ console.info("[ImportCleanup] Cleanup initialized (runs daily at 2 AM UTC)");
+ }
+
+ /**
+ * Clean up orphaned import files that are more than 7 days old
+ * and belong to completed or failed imports.
+ */
+ private async cleanupOrphanedFiles() {
+ console.info("[ImportCleanup] Starting cleanup of orphaned import files");
+
+ const sevenDaysAgo = DateTime.utc().minus({ days: 7 }).toISO();
+
+ try {
+ // Find completed/failed imports older than 7 days that still have files
+ const oldImports = await db
+ .select({
+ importId: importStatus.importId,
+ fileName: importStatus.fileName,
+ })
+ .from(importStatus)
+ .where(
+ sql`${importStatus.status} IN ('completed', 'failed')
+ AND ${importStatus.startedAt} < ${sevenDaysAgo}`
+ );
+
+ console.info(`[ImportCleanup] Found ${oldImports.length} old imports to check`);
+
+ let deletedCount = 0;
+ let failedCount = 0;
+
+ for (const importRecord of oldImports) {
+ const storage = getImportStorageLocation(importRecord.importId, importRecord.fileName);
+
+ // Attempt to delete the file
+ const result = await deleteImportFile(storage.location, storage.isR2);
+
+ if (result.success) {
+ deletedCount++;
+ } else {
+ // File might already be deleted or doesn't exist - this is not an error
+ if (result.error?.includes("ENOENT") || result.error?.includes("NoSuchKey")) {
+ // File doesn't exist, which is fine
+ continue;
+ }
+ failedCount++;
+ console.warn(`[ImportCleanup] Failed to delete file for import ${importRecord.importId}: ${result.error}`);
+ }
+ }
+
+ console.info(
+ `[ImportCleanup] Cleanup complete: ${deletedCount} files deleted, ${failedCount} failures, ${oldImports.length - deletedCount - failedCount} already gone`
+ );
+ } catch (error) {
+ console.error("[ImportCleanup] Error during cleanup:", error);
+ }
+ }
+
+ async triggerManualCleanup() {
+ console.info("[ImportCleanup] Manual cleanup triggered");
+ await this.cleanupOrphanedFiles();
+ }
+
+ stopCleanupCron() {
+ if (this.cleanupTask) {
+ this.cleanupTask.stop();
+ console.info("[ImportCleanup] Cleanup cron stopped");
+ }
+ }
+}
+
+// Export singleton instance
+export const importCleanupService = new ImportCleanupService();
diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.ts
index 55da0ffe3..095267742 100644
--- a/server/src/services/import/importLimiter.ts
+++ b/server/src/services/import/importLimiter.ts
@@ -3,10 +3,15 @@ import { db } from "../../db/postgres/postgres.js";
import { and, count, eq, inArray } from "drizzle-orm";
import { importStatus, sites } from "../../db/postgres/schema.js";
import { ImportQuotaTracker } from "./importQuotaChecker.js";
+import type { InsertImportStatus } from "./importStatusManager.js";
export class ImportLimiter {
private static readonly CONCURRENT_IMPORT_LIMIT = 1;
+ /**
+ * Check concurrent import limit for a site.
+ * Returns organization ID if allowed.
+ */
static async checkConcurrentImportLimit(siteId: number): Promise<
| {
allowed: false;
@@ -27,10 +32,14 @@ export class ImportLimiter {
return { allowed: false, reason: "Site not found." };
}
+ // For self-hosted, no limit
if (!IS_CLOUD) {
return { allowed: true, organizationId: siteResult.organizationId };
}
+ // Check current concurrent imports
+ // Note: This check alone has a race condition, but the actual import creation
+ // is atomic (see createImportWithConcurrencyCheck)
const [concurrentImportResult] = await db
.select({ count: count() })
.from(importStatus)
@@ -51,6 +60,44 @@ export class ImportLimiter {
return { allowed: true, organizationId: siteResult.organizationId };
}
+ /**
+ * Create an import status record with atomic concurrent import check.
+ * This prevents race conditions by checking and inserting in a single transaction.
+ */
+ static async createImportWithConcurrencyCheck(
+ data: InsertImportStatus
+ ): Promise<{ success: true } | { success: false; reason: string }> {
+ if (!IS_CLOUD) {
+ await db.insert(importStatus).values(data);
+ return { success: true };
+ }
+
+ // Use a transaction to atomically check and insert
+ return await db.transaction(async tx => {
+ // Lock the rows to prevent concurrent inserts
+ const [concurrentImportResult] = await tx
+ .select({ count: count() })
+ .from(importStatus)
+ .where(
+ and(
+ eq(importStatus.organizationId, data.organizationId),
+ inArray(importStatus.status, ["pending", "processing"])
+ )
+ )
+ .for("update");
+
+ if (concurrentImportResult.count >= this.CONCURRENT_IMPORT_LIMIT) {
+ return {
+ success: false,
+ reason: `Only ${this.CONCURRENT_IMPORT_LIMIT} concurrent import allowed per organization.`,
+ };
+ }
+
+ await tx.insert(importStatus).values(data);
+ return { success: true };
+ });
+ }
+
static async createQuotaTracker(organizationId: string): Promise<ImportQuotaTracker> {
return ImportQuotaTracker.create(organizationId);
}
diff --git a/server/src/services/import/importQuotaChecker.ts b/server/src/services/import/importQuotaChecker.ts
index 25ed26933..75344ce5d 100644
--- a/server/src/services/import/importQuotaChecker.ts
+++ b/server/src/services/import/importQuotaChecker.ts
@@ -7,13 +7,6 @@ import { processResults } from "../../api/analytics/utils.js";
import { getOrganizationSubscriptionInfo } from "../../lib/subscriptionUtils.js";
import { IS_CLOUD } from "../../lib/const.js";
-export interface MonthlyQuotaInfo {
- month: string; // "202501" format
- used: number;
- limit: number;
- remaining: number;
-}
-
export class ImportQuotaTracker {
private monthlyUsage: Map<string, number>;
private readonly monthlyLimit: number;
@@ -75,66 +68,65 @@ export class ImportQuotaTracker {
const newSites = siteIds.filter(id => id >= 2000);
try {
- // Query grandfathered sites (pageviews only)
- if (grandfatheredSites.length > 0) {
- const grandfatheredResult = await clickhouse.query({
- query: `
- SELECT
- toYYYYMM(timestamp) as month,
- COUNT(*) as count
- FROM events
- WHERE site_id IN {siteIds:Array(Int32)}
- AND type = 'pageview'
- AND timestamp >= toDate({startDate:String})
- GROUP BY month
- ORDER BY month
- `,
- query_params: {
- siteIds: grandfatheredSites,
- startDate: startDate,
- },
- format: "JSONEachRow",
- });
-
- const rows = await processResults<{ month: string; count: number }>(grandfatheredResult);
- for (const row of rows) {
- const existing = monthlyUsage.get(row.month) || 0;
- monthlyUsage.set(row.month, existing + row.count);
- }
- }
-
- // Query new sites (all event types)
- if (newSites.length > 0) {
- const newSitesResult = await clickhouse.query({
- query: `
- SELECT
- toYYYYMM(timestamp) as month,
- COUNT(*) as count
- FROM events
- WHERE site_id IN {siteIds:Array(Int32)}
- AND type IN ('pageview', 'custom_event', 'performance')
- AND timestamp >= toDate({startDate:String})
- GROUP BY month
- ORDER BY month
- `,
- query_params: {
- siteIds: newSites,
- startDate: startDate,
- },
- format: "JSONEachRow",
- });
-
- const rows = await processResults<{ month: string; count: number }>(newSitesResult);
- for (const row of rows) {
- const existing = monthlyUsage.get(row.month) || 0;
- monthlyUsage.set(row.month, existing + row.count);
- }
+ // Combine queries into a single UNION query for better performance
+ // Grandfathered sites (< 2000): count pageviews only
+ // New sites (>= 2000): count pageviews, custom_events, and performance events
+ const result = await clickhouse.query({
+ query: `
+ SELECT
+ toYYYYMM(timestamp) as month,
+ SUM(count) as count
+ FROM (
+ ${
+ grandfatheredSites.length > 0
+ ? `
+ SELECT
+ timestamp,
+ 1 as count
+ FROM events
+ WHERE site_id IN {grandfatheredSites:Array(Int32)}
+ AND type = 'pageview'
+ AND timestamp >= toDate({startDate:String})
+ `
+ : ""
+ }
+ ${grandfatheredSites.length > 0 && newSites.length > 0 ? "UNION ALL" : ""}
+ ${
+ newSites.length > 0
+ ? `
+ SELECT
+ timestamp,
+ 1 as count
+ FROM events
+ WHERE site_id IN {newSites:Array(Int32)}
+ AND type IN ('pageview', 'custom_event', 'performance')
+ AND timestamp >= toDate({startDate:String})
+ `
+ : ""
+ }
+ )
+ GROUP BY month
+ ORDER BY month
+ `,
+ query_params: {
+ ...(grandfatheredSites.length > 0 && { grandfatheredSites }),
+ ...(newSites.length > 0 && { newSites }),
+ startDate: startDate,
+ },
+ format: "JSONEachRow",
+ });
+
+ const rows = await processResults<{ month: string; count: number }>(result);
+ for (const row of rows) {
+ monthlyUsage.set(row.month, row.count);
}
return monthlyUsage;
} catch (error) {
console.error(`Error querying ClickHouse for monthly usage:`, error);
- return new Map();
+ throw new Error(
+ `Failed to query monthly usage for quota check: ${error instanceof Error ? error.message : "Unknown error"}`
+ );
}
}
@@ -164,33 +156,6 @@ export class ImportQuotaTracker {
return true;
}
- getMonthQuota(month: string): MonthlyQuotaInfo {
- const used = this.monthlyUsage.get(month) || 0;
- return {
- month,
- used,
- limit: this.monthlyLimit,
- remaining: Math.max(0, this.monthlyLimit - used),
- };
- }
-
- getAllMonthQuotas(): MonthlyQuotaInfo[] {
- const quotas: MonthlyQuotaInfo[] = [];
-
- if (this.monthlyLimit === Infinity) {
- return quotas;
- }
-
- const now = DateTime.utc();
- for (let i = 0; i < this.historicalWindowMonths; i++) {
- const monthDate = now.minus({ months: i }).startOf("month");
- const month = monthDate.toFormat("yyyyMM");
- quotas.push(this.getMonthQuota(month));
- }
-
- return quotas.reverse();
- }
-
getSummary(): {
totalMonthsInWindow: number;
monthsAtCapacity: number;
@@ -206,13 +171,18 @@ export class ImportQuotaTracker {
};
}
- const quotas = this.getAllMonthQuotas();
- const monthsAtCapacity = quotas.filter(q => q.remaining === 0).length;
+ // Calculate directly from stored data instead of regenerating all monthly quotas
+ let monthsAtCapacity = 0;
+ for (const usage of this.monthlyUsage.values()) {
+ if (usage >= this.monthlyLimit) {
+ monthsAtCapacity++;
+ }
+ }
return {
totalMonthsInWindow: this.historicalWindowMonths,
monthsAtCapacity,
- monthsWithSpace: quotas.length - monthsAtCapacity,
+ monthsWithSpace: this.historicalWindowMonths - monthsAtCapacity,
oldestAllowedMonth: this.oldestAllowedMonth,
};
}
diff --git a/server/src/services/import/importStatusManager.ts b/server/src/services/import/importStatusManager.ts
index 959efdf08..91d5e4e8b 100644
--- a/server/src/services/import/importStatusManager.ts
+++ b/server/src/services/import/importStatusManager.ts
@@ -3,81 +3,46 @@ import { eq, desc, sql } from "drizzle-orm";
import { importStatus } from "../../db/postgres/schema.js";
import { DateTime } from "luxon";
-type SelectImportStatus = typeof importStatus.$inferSelect;
-type InsertImportStatus = typeof importStatus.$inferInsert;
+export type SelectImportStatus = typeof importStatus.$inferSelect;
+export type InsertImportStatus = typeof importStatus.$inferInsert;
-export class ImportStatusManager {
- static async createImportStatus(data: InsertImportStatus): Promise<void> {
- try {
- await db.insert(importStatus).values(data);
- } catch (error) {
- console.error("Failed to create import status:", error);
- throw new Error(`Failed to create import status: ${error instanceof Error ? error.message : "Unknown error"}`);
- }
- }
+export async function createImportStatus(data: InsertImportStatus): Promise<void> {
+ await db.insert(importStatus).values(data);
+}
- static async updateStatus(
- importId: string,
- status: InsertImportStatus["status"],
- errorMessage?: string
- ): Promise<void> {
- const completedAt = status === "completed" || status === "failed" ? DateTime.utc().toISO() : null;
+export async function updateImportStatus(
+ importId: string,
+ status: InsertImportStatus["status"],
+ errorMessage?: string
+): Promise<void> {
+ const completedAt = status === "completed" || status === "failed" ? DateTime.utc().toISO() : null;
- try {
- await db
- .update(importStatus)
- .set({ status, errorMessage, completedAt })
- .where(eq(importStatus.importId, importId));
- } catch (error) {
- console.error(`Failed to update import status for ${importId}:`, error);
- throw new Error(`Failed to update import status: ${error instanceof Error ? error.message : "Unknown error"}`);
- }
- }
+ await db.update(importStatus).set({ status, errorMessage, completedAt }).where(eq(importStatus.importId, importId));
+}
- static async updateProgress(importId: string, importedEvents: number): Promise<void> {
- try {
- await db
- .update(importStatus)
- .set({
- importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`,
- })
- .where(eq(importStatus.importId, importId));
- } catch (error) {
- console.error(`Failed to update import progress for ${importId}:`, error);
- throw new Error(`Failed to update import progress: ${error instanceof Error ? error.message : "Unknown error"}`);
- }
- }
+export async function updateImportProgress(importId: string, importedEvents: number): Promise<void> {
+ await db
+ .update(importStatus)
+ .set({
+ importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`,
+ })
+ .where(eq(importStatus.importId, importId));
+}
- static async getImportsForSite(siteId: number, limit = 10): Promise<SelectImportStatus[]> {
- try {
- return await db.query.importStatus.findMany({
- where: eq(importStatus.siteId, siteId),
- orderBy: [desc(importStatus.startedAt)],
- limit,
- });
- } catch (error) {
- console.error(`Failed to get imports for site ${siteId}:`, error);
- throw new Error(`Failed to get imports for site: ${error instanceof Error ? error.message : "Unknown error"}`);
- }
- }
+export async function getImportsForSite(siteId: number, limit = 10): Promise<SelectImportStatus[]> {
+ return await db.query.importStatus.findMany({
+ where: eq(importStatus.siteId, siteId),
+ orderBy: [desc(importStatus.startedAt)],
+ limit,
+ });
+}
- static async deleteImport(importId: string): Promise<void> {
- try {
- await db.delete(importStatus).where(eq(importStatus.importId, importId));
- } catch (error) {
- console.error(`Failed to delete import ${importId}:`, error);
- throw new Error(`Failed to delete import: ${error instanceof Error ? error.message : "Unknown error"}`);
- }
- }
+export async function deleteImport(importId: string): Promise<void> {
+ await db.delete(importStatus).where(eq(importStatus.importId, importId));
+}
- static async getImportById(importId: string): Promise<SelectImportStatus | undefined> {
- try {
- return await db.query.importStatus.findFirst({
- where: eq(importStatus.importId, importId),
- });
- } catch (error) {
- console.error(`Failed to get import ${importId}:`, error);
- throw new Error(`Failed to get import: ${error instanceof Error ? error.message : "Unknown error"}`);
- }
- }
+export async function getImportById(importId: string): Promise<SelectImportStatus | undefined> {
+ return await db.query.importStatus.findFirst({
+ where: eq(importStatus.importId, importId),
+ });
}
diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.ts
index 311957874..bfe6e1333 100644
--- a/server/src/services/import/utils.ts
+++ b/server/src/services/import/utils.ts
@@ -1,4 +1,5 @@
import { unlink } from "node:fs/promises";
+import { IS_CLOUD } from "../../lib/const.js";
import { r2Storage } from "../storage/r2StorageService.js";
export interface DeleteFileResult {
@@ -6,6 +7,25 @@ export interface DeleteFileResult {
error?: string;
}
+export interface StorageLocationInfo {
+ location: string;
+ isR2: boolean;
+}
+
+export function getImportStorageLocation(importId: string, filename: string): StorageLocationInfo {
+ const isR2 = IS_CLOUD && r2Storage.isEnabled();
+ if (isR2) {
+ return {
+ location: `imports/${importId}/${filename}`,
+ isR2: true,
+ };
+ }
+ return {
+ location: `/tmp/imports/${importId}.csv`,
+ isR2: false,
+ };
+}
+
/**
* Delete an import file from storage.
* Returns result instead of throwing to prevent worker crashes.
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 2c3d21c70..d38d34cb1 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -6,7 +6,7 @@ import { getJobQueue } from "../../../queues/jobQueueFactory.js";
import { r2Storage } from "../../storage/r2StorageService.js";
import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
-import { ImportStatusManager } from "../importStatusManager.js";
+import { updateImportStatus } from "../importStatusManager.js";
import { ImportLimiter } from "../importLimiter.js";
import { deleteImportFile } from "../utils.js";
@@ -109,7 +109,13 @@ export async function registerCsvParseWorker() {
? await createR2FileStream(storageLocation, source)
: await createLocalFileStream(storageLocation, source);
- await ImportStatusManager.updateStatus(importId, "processing");
+ // Add explicit error handler before starting to consume the stream
+ stream.on("error", error => {
+ console.error(`[Import ${importId}] Stream error:`, error);
+ // Error will be caught by the outer try/catch
+ });
+
+ await updateImportStatus(importId, "processing");
// Set timeout to prevent indefinite processing
processingTimeout = setTimeout(() => {
@@ -175,7 +181,7 @@ export async function registerCsvParseWorker() {
`No events could be imported. All ${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
`${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +
`Try importing newer data or upgrade your plan for higher monthly quotas.`;
- await ImportStatusManager.updateStatus(importId, "failed", errorMessage);
+ await updateImportStatus(importId, "failed", errorMessage);
const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
if (!deleteResult.success) {
console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);
@@ -214,7 +220,7 @@ export async function registerCsvParseWorker() {
? error.message.replace(/\/[^\s]+/g, "[path]").substring(0, 500)
: "An unexpected error occurred during import processing";
- await ImportStatusManager.updateStatus(importId, "failed", safeMessage);
+ await updateImportStatus(importId, "failed", safeMessage);
// Don't re-throw - worker should continue processing other jobs
console.error(`[Import ${importId}] Import failed, worker continuing`);
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 794c6e02c..162e36f25 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -2,7 +2,7 @@ import { getJobQueue } from "../../../queues/jobQueueFactory.js";
import { UmamiImportMapper } from "../mappings/umami.js";
import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { ImportStatusManager } from "../importStatusManager.js";
+import { updateImportStatus, updateImportProgress } from "../importStatusManager.js";
const getImportDataMapping = (source: string) => {
switch (source) {
@@ -22,14 +22,14 @@ export async function registerDataInsertWorker() {
// Handle finalization signal
if (allChunksSent) {
try {
- await ImportStatusManager.updateStatus(importId, "completed");
+ await updateImportStatus(importId, "completed");
console.log(`[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`);
return;
} catch (error) {
console.error(`[Import ${importId}] Failed to mark as completed:`, error);
// Try to update to failed status, but don't crash worker
try {
- await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");
+ await updateImportStatus(importId, "failed", "Failed to complete import");
} catch (updateError) {
console.error(`[Import ${importId}] Could not update status to failed:`, updateError);
}
@@ -52,7 +52,7 @@ export async function registerDataInsertWorker() {
// Update progress (non-critical - log if fails but don't crash)
try {
- await ImportStatusManager.updateProgress(importId, transformedRecords.length);
+ await updateImportProgress(importId, transformedRecords.length);
} catch (progressError) {
console.warn(
`[Import ${importId}] Progress update failed (data inserted successfully):`,
@@ -72,7 +72,7 @@ export async function registerDataInsertWorker() {
: "Data insertion failed due to unknown error";
try {
- await ImportStatusManager.updateStatus(importId, "failed", safeMessage);
+ await updateImportStatus(importId, "failed", safeMessage);
} catch (updateError) {
console.error(`[Import ${importId}] Could not update status to failed:`, updateError);
}