🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
A bunch of improvements
Edward Cuife9407536fa6
diff --git a/server/src/api/sites/deleteSiteImport.ts b/server/src/api/sites/deleteSiteImport.tsindex 53a1df234..eb6bf0c74 100644--- a/server/src/api/sites/deleteSiteImport.ts+++ b/server/src/api/sites/deleteSiteImport.ts@@ -1,10 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";import { z } from "zod";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";-import { ImportStatusManager } from "../../services/import/importStatusManager.js";-import { deleteImportFile } from "../../services/import/utils.js";-import { IS_CLOUD } from "../../lib/const.js";-import { r2Storage } from "../../services/storage/r2StorageService.js";+import { getImportById, deleteImport } from "../../services/import/importStatusManager.js";+import { deleteImportFile, getImportStorageLocation } from "../../services/import/utils.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";const deleteImportRequestSchema = z@@ -37,7 +35,7 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportRequereturn reply.status(403).send({ error: "Forbidden" });}- const importRecord = await ImportStatusManager.getImportById(importId);+ const importRecord = await getImportById(importId);if (!importRecord) {return reply.status(404).send({ error: "Import not found" });}@@ -50,19 +48,17 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportRequereturn reply.status(400).send({ error: "Cannot delete active import" });}- // Delete the import file if it exists- try {- let storageLocation: string;- if (IS_CLOUD && r2Storage.isEnabled()) {- storageLocation = `imports/${importId}/${importRecord.fileName}`;- } else {- storageLocation = `/tmp/imports/${importId}.csv`;- }+ const siteId = Number(site);- await deleteImportFile(storageLocation, IS_CLOUD && r2Storage.isEnabled());- } catch (fileError) {- // Log the error but don't fail the deletion - the file might not exist- console.warn(`Failed to delete import file for ${importId}:`, fileError);+ // Delete the import record from the database FIRST (fail fast)+ // If this fails, we haven't deleted any data yet+ try {+ await deleteImport(importId);+ } catch (dbError) {+ console.error(`Failed to delete import record ${importId}:`, dbError);+ return reply.status(500).send({+ error: "Failed to delete import record",+ });}// Delete events from ClickHouse that were imported with this importId@@ -72,7 +68,7 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportRequequery: `DELETE FROM events WHERE import_id = {importId:String} AND site_id = {siteId:UInt16}`,query_params: {importId: importId,- siteId: Number(site),+ siteId: siteId,},});console.log(`Deleted events for import ${importId} from ClickHouse`);@@ -83,14 +79,12 @@ export async function deleteSiteImport(request: FastifyRequest<DeleteImportReque});}- // Delete the import record from the database- try {- await ImportStatusManager.deleteImport(importId);- } catch (dbError) {- console.error(`Failed to delete import record ${importId}:`, dbError);- return reply.status(500).send({- error: "Import events deleted but failed to remove import record",- });+ // Delete the import file if it exists+ // This is best-effort - we don't fail the entire operation if the file is already gone+ const storage = getImportStorageLocation(importId, importRecord.fileName);+ const deleteResult = await deleteImportFile(storage.location, storage.isR2);+ if (!deleteResult.success) {+ console.warn(`Failed to delete import file for ${importId}: ${deleteResult.error}`);}return reply.send({diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.tsindex 4c3319582..082244861 100644--- a/server/src/api/sites/getSiteImports.ts+++ b/server/src/api/sites/getSiteImports.ts@@ -1,6 +1,6 @@import { FastifyReply, FastifyRequest } from "fastify";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";-import { ImportStatusManager } from "../../services/import/importStatusManager.js";+import { getImportsForSite } from "../../services/import/importStatusManager.js";import { z } from "zod";const getSiteImportsRequestSchema = z@@ -26,23 +26,24 @@ export async function getSiteImports(request: FastifyRequest<GetSiteImportsReque}const { site } = parsed.data.params;+ const siteId = Number(site);const userHasAccess = await getUserHasAdminAccessToSite(request, site);if (!userHasAccess) {return reply.status(403).send({ error: "Forbidden" });}- const imports = await ImportStatusManager.getImportsForSite(Number(site));+ const imports = await getImportsForSite(siteId);return reply.send({- data: imports.map(imp => ({- importId: imp.importId,- source: imp.source,- status: imp.status,- importedEvents: imp.importedEvents,- errorMessage: imp.errorMessage,- startedAt: imp.startedAt,- fileName: imp.fileName,+ data: imports.map(({ importId, source, status, importedEvents, errorMessage, startedAt, fileName }) => ({+ importId,+ source,+ status,+ importedEvents,+ errorMessage,+ startedAt,+ fileName,})),});} catch (error) {diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsindex 07ca13e5f..f9c2e5df3 100644--- a/server/src/api/sites/importSiteData.ts+++ b/server/src/api/sites/importSiteData.ts@@ -1,17 +1,15 @@import { createWriteStream } from "node:fs";import { mkdir } from "node:fs/promises";import { randomUUID } from "node:crypto";-import path from "node:path";import { pipeline } from "node:stream/promises";import { FastifyRequest, FastifyReply } from "fastify";import { DateTime } from "luxon";import { z } from "zod";-import { IS_CLOUD } from "../../lib/const.js";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { getJobQueue } from "../../queues/jobQueueFactory.js";import { ImportLimiter } from "../../services/import/importLimiter.js";-import { ImportStatusManager } from "../../services/import/importStatusManager.js";-import { deleteImportFile } from "../../services/import/utils.js";+import { updateImportStatus } from "../../services/import/importStatusManager.js";+import { deleteImportFile, getImportStorageLocation } from "../../services/import/utils.js";import { CSV_PARSE_QUEUE } from "../../services/import/workers/jobs.js";import { r2Storage } from "../../services/storage/r2StorageService.js";@@ -27,25 +25,24 @@ const importDataFieldsSchema = zfields: z.object({source: z.enum(["umami"]),- startDate: z.string().refine(isValidDate).optional(),- endDate: z.string().refine(isValidDate).optional(),+ startDate: z.string().refine(isValidDate, { message: "Invalid start date format" }).optional(),+ endDate: z.string().refine(isValidDate, { message: "Invalid end date format" }).optional(),})- .refine(fields => {- if (fields.startDate && fields.endDate) {- const start = parseDate(fields.startDate);- const end = parseDate(fields.endDate);- return start <= end;- }- return true;- })- .refine(fields => {- if (fields.startDate) {+ .refine(+ data => {+ if (!data.startDate || !data.endDate) return true;+ return parseDate(data.startDate) <= parseDate(data.endDate);+ },+ { message: "Start date must be before or equal to end date" }+ )+ .refine(+ data => {+ if (!data.startDate) return true;const today = DateTime.utc().startOf("day");- const start = parseDate(fields.startDate);- return start <= today;- }- return true;- }),+ return parseDate(data.startDate) <= today;+ },+ { message: "Start date cannot be in the future" }+ ),}).strict();@@ -78,11 +75,6 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,return reply.status(403).send({ error: "Forbidden" });}- const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(Number(site));- if (!concurrentImportLimitResult.allowed) {- return reply.status(429).send({ error: concurrentImportLimitResult.reason });- }-const data = await request.file();if (!data) {return reply.status(400).send({ error: "No file uploaded." });@@ -105,41 +97,45 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,}const { source, startDate, endDate } = parsedFields.data.fields;- const organization = concurrentImportLimitResult.organizationId;+ const siteId = Number(site);const importId = randomUUID();- await ImportStatusManager.createImportStatus({+ // Check organization and get initial limit check+ const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(siteId);+ if (!concurrentImportLimitResult.allowed) {+ return reply.status(429).send({ error: concurrentImportLimitResult.reason });+ }++ const organization = concurrentImportLimitResult.organizationId;++ // Atomically create import status with concurrency check to prevent race conditions+ const createResult = await ImportLimiter.createImportWithConcurrencyCheck({importId,- siteId: Number(site),+ siteId,organizationId: organization,source,status: "pending",fileName: data.filename,});- let storageLocation: string;-- try {- if (IS_CLOUD && r2Storage.isEnabled()) {- const r2Key = `imports/${importId}/${data.filename}`;+ if (!createResult.success) {+ return reply.status(429).send({ error: createResult.reason });+ }- await r2Storage.storeImportFile(r2Key, data.file);- storageLocation = r2Key;+ const storage = getImportStorageLocation(importId, data.filename);- console.log(`[Import] File streamed to R2: ${r2Key}`);+ try {+ if (storage.isR2) {+ await r2Storage.storeImportFile(storage.location, data.file);+ console.log(`[Import] File streamed to R2: ${storage.location}`);} else {const importDir = "/tmp/imports";- const savedFileName = `${importId}.csv`;- const tempFilePath = path.join(importDir, savedFileName);-await mkdir(importDir, { recursive: true });- await pipeline(data.file, createWriteStream(tempFilePath));- storageLocation = tempFilePath;-- console.log(`[Import] File stored locally: ${tempFilePath}`);+ await pipeline(data.file, createWriteStream(storage.location));+ console.log(`[Import] File stored locally: ${storage.location}`);}} catch (fileError) {- await ImportStatusManager.updateStatus(importId, "failed", "Failed to save uploaded file");+ await updateImportStatus(importId, "failed", "Failed to save uploaded file");console.error("Failed to save uploaded file:", fileError);return reply.status(500).send({ error: "Could not process file upload." });}@@ -150,15 +146,15 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,site,importId,source,- storageLocation,- isR2Storage: IS_CLOUD && r2Storage.isEnabled(),+ storageLocation: storage.location,+ isR2Storage: storage.isR2,organization,startDate,endDate,});} catch (queueError) {- await ImportStatusManager.updateStatus(importId, "failed", "Failed to queue import job");- await deleteImportFile(storageLocation, IS_CLOUD && r2Storage.isEnabled());+ await updateImportStatus(importId, "failed", "Failed to queue import job");+ await deleteImportFile(storage.location, storage.isR2);console.error("Failed to enqueue import job:", queueError);return reply.status(500).send({ error: "Failed to initiate import process." });}diff --git a/server/src/index.ts b/server/src/index.tsindex cdee3952e..9716e1e75 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -75,6 +75,7 @@ import { trackEvent } from "./services/tracker/trackEvent.js";// need to import telemetry service here to start itimport { telemetryService } from "./services/telemetryService.js";import { weeklyReportService } from "./services/weekyReports/weeklyReportService.js";+import { importCleanupService } from "./services/import/importCleanupService.js";import { extractSiteId } from "./utils.js";import { getTrackingConfig } from "./api/sites/getTrackingConfig.js";import { getSiteImports } from "./api/sites/getSiteImports.js";@@ -441,6 +442,9 @@ const start = async () => {await registerCsvParseWorker();await registerDataInsertWorker();+ // Initialize import cleanup service (runs daily to clean up old import files)+ importCleanupService.initializeCleanupCron();+// Start the server firstawait server.listen({ port: 3001, host: "0.0.0.0" });server.log.info("Server is listening on http://0.0.0.0:3001");diff --git a/server/src/lib/subscriptionUtils.ts b/server/src/lib/subscriptionUtils.tsindex 3629d3905..b0efce97b 100644--- a/server/src/lib/subscriptionUtils.ts+++ b/server/src/lib/subscriptionUtils.ts@@ -38,6 +38,7 @@ export function getTierInfo(planName: string): TierInfo {}// Default to free tier for unknown plans+ console.warn(`Unknown plan name: "${planName}", defaulting to Free tier`);return {tier: "Free",monthsAllowed: 6,diff --git a/server/src/queues/adapters/bullMQAdapter.ts b/server/src/queues/adapters/bullMQAdapter.tsindex 6210b6855..0862e4450 100644--- a/server/src/queues/adapters/bullMQAdapter.ts+++ b/server/src/queues/adapters/bullMQAdapter.ts@@ -175,8 +175,11 @@ export class BullMQAdapter implements IJobQueue {data: job.data,};- // BullMQ processes one job at a time per worker- // We wrap it in an array to match the interface+ // NOTE: Interface Impedance Mismatch+ // BullMQ processes one job at a time per worker, while pg-boss can batch jobs.+ // The IJobQueue interface expects handlers to accept arrays to support pg-boss batching.+ // Here we wrap single BullMQ jobs in an array to maintain interface compatibility.+ // This is an acceptable tradeoff: pg-boss gets true batching, BullMQ gets array of 1 item.await handler([normalizedJob]);},{diff --git a/server/src/services/import/importCleanupService.ts b/server/src/services/import/importCleanupService.tsnew file mode 100644index 000000000..e8cb47315--- /dev/null+++ b/server/src/services/import/importCleanupService.ts@@ -0,0 +1,104 @@+import * as cron from "node-cron";+import { DateTime } from "luxon";+import { db } from "../../db/postgres/postgres.js";+import { importStatus } from "../../db/postgres/schema.js";+import { sql } from "drizzle-orm";+import { deleteImportFile, getImportStorageLocation } from "./utils.js";++class ImportCleanupService {+ private cleanupTask: cron.ScheduledTask | null = null;++ constructor() {}++ /**+ * Initialize the cleanup cron job.+ * Runs daily at 2 AM UTC to clean up orphaned import files.+ */+ initializeCleanupCron() {+ console.info("[ImportCleanup] Initializing cleanup cron");++ // Schedule cleanup to run daily at 2 AM UTC+ this.cleanupTask = cron.schedule(+ "0 2 * * *",+ async () => {+ try {+ await this.cleanupOrphanedFiles();+ } catch (error) {+ console.error("[ImportCleanup] Error during cleanup:", error);+ }+ },+ { timezone: "UTC" }+ );++ console.info("[ImportCleanup] Cleanup initialized (runs daily at 2 AM UTC)");+ }++ /**+ * Clean up orphaned import files that are more than 7 days old+ * and belong to completed or failed imports.+ */+ private async cleanupOrphanedFiles() {+ console.info("[ImportCleanup] Starting cleanup of orphaned import files");++ const sevenDaysAgo = DateTime.utc().minus({ days: 7 }).toISO();++ try {+ // Find completed/failed imports older than 7 days that still have files+ const oldImports = await db+ .select({+ importId: importStatus.importId,+ fileName: importStatus.fileName,+ })+ .from(importStatus)+ .where(+ sql`${importStatus.status} IN ('completed', 'failed')+ AND ${importStatus.startedAt} < ${sevenDaysAgo}`+ );++ console.info(`[ImportCleanup] Found ${oldImports.length} old imports to check`);++ let deletedCount = 0;+ let failedCount = 0;++ for (const importRecord of oldImports) {+ const storage = getImportStorageLocation(importRecord.importId, importRecord.fileName);++ // Attempt to delete the file+ const result = await deleteImportFile(storage.location, storage.isR2);++ if (result.success) {+ deletedCount++;+ } else {+ // File might already be deleted or doesn't exist - this is not an error+ if (result.error?.includes("ENOENT") || result.error?.includes("NoSuchKey")) {+ // File doesn't exist, which is fine+ continue;+ }+ failedCount++;+ console.warn(`[ImportCleanup] Failed to delete file for import ${importRecord.importId}: ${result.error}`);+ }+ }++ console.info(+ `[ImportCleanup] Cleanup complete: ${deletedCount} files deleted, ${failedCount} failures, ${oldImports.length - deletedCount - failedCount} already gone`+ );+ } catch (error) {+ console.error("[ImportCleanup] Error during cleanup:", error);+ }+ }++ async triggerManualCleanup() {+ console.info("[ImportCleanup] Manual cleanup triggered");+ await this.cleanupOrphanedFiles();+ }++ stopCleanupCron() {+ if (this.cleanupTask) {+ this.cleanupTask.stop();+ console.info("[ImportCleanup] Cleanup cron stopped");+ }+ }+}++// Export singleton instance+export const importCleanupService = new ImportCleanupService();diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.tsindex 55da0ffe3..095267742 100644--- a/server/src/services/import/importLimiter.ts+++ b/server/src/services/import/importLimiter.ts@@ -3,10 +3,15 @@ import { db } from "../../db/postgres/postgres.js";import { and, count, eq, inArray } from "drizzle-orm";import { importStatus, sites } from "../../db/postgres/schema.js";import { ImportQuotaTracker } from "./importQuotaChecker.js";+import type { InsertImportStatus } from "./importStatusManager.js";export class ImportLimiter {private static readonly CONCURRENT_IMPORT_LIMIT = 1;+ /**+ * Check concurrent import limit for a site.+ * Returns organization ID if allowed.+ */static async checkConcurrentImportLimit(siteId: number): Promise<| {allowed: false;@@ -27,10 +32,14 @@ export class ImportLimiter {return { allowed: false, reason: "Site not found." };}+ // For self-hosted, no limitif (!IS_CLOUD) {return { allowed: true, organizationId: siteResult.organizationId };}+ // Check current concurrent imports+ // Note: This check alone has a race condition, but the actual import creation+ // is atomic (see createImportWithConcurrencyCheck)const [concurrentImportResult] = await db.select({ count: count() }).from(importStatus)@@ -51,6 +60,44 @@ export class ImportLimiter {return { allowed: true, organizationId: siteResult.organizationId };}+ /**+ * Create an import status record with atomic concurrent import check.+ * This prevents race conditions by checking and inserting in a single transaction.+ */+ static async createImportWithConcurrencyCheck(+ data: InsertImportStatus+ ): Promise<{ success: true } | { success: false; reason: string }> {+ if (!IS_CLOUD) {+ await db.insert(importStatus).values(data);+ return { success: true };+ }++ // Use a transaction to atomically check and insert+ return await db.transaction(async tx => {+ // Lock the rows to prevent concurrent inserts+ const [concurrentImportResult] = await tx+ .select({ count: count() })+ .from(importStatus)+ .where(+ and(+ eq(importStatus.organizationId, data.organizationId),+ inArray(importStatus.status, ["pending", "processing"])+ )+ )+ .for("update");++ if (concurrentImportResult.count >= this.CONCURRENT_IMPORT_LIMIT) {+ return {+ success: false,+ reason: `Only ${this.CONCURRENT_IMPORT_LIMIT} concurrent import allowed per organization.`,+ };+ }++ await tx.insert(importStatus).values(data);+ return { success: true };+ });+ }+static async createQuotaTracker(organizationId: string): Promise<ImportQuotaTracker> {return ImportQuotaTracker.create(organizationId);}diff --git a/server/src/services/import/importQuotaChecker.ts b/server/src/services/import/importQuotaChecker.tsindex 25ed26933..75344ce5d 100644--- a/server/src/services/import/importQuotaChecker.ts+++ b/server/src/services/import/importQuotaChecker.ts@@ -7,13 +7,6 @@ import { processResults } from "../../api/analytics/utils.js";import { getOrganizationSubscriptionInfo } from "../../lib/subscriptionUtils.js";import { IS_CLOUD } from "../../lib/const.js";-export interface MonthlyQuotaInfo {- month: string; // "202501" format- used: number;- limit: number;- remaining: number;-}-export class ImportQuotaTracker {private monthlyUsage: Map<string, number>;private readonly monthlyLimit: number;@@ -75,66 +68,65 @@ export class ImportQuotaTracker {const newSites = siteIds.filter(id => id >= 2000);try {- // Query grandfathered sites (pageviews only)- if (grandfatheredSites.length > 0) {- const grandfatheredResult = await clickhouse.query({- query: `- SELECT- toYYYYMM(timestamp) as month,- COUNT(*) as count- FROM events- WHERE site_id IN {siteIds:Array(Int32)}- AND type = 'pageview'- AND timestamp >= toDate({startDate:String})- GROUP BY month- ORDER BY month- `,- query_params: {- siteIds: grandfatheredSites,- startDate: startDate,- },- format: "JSONEachRow",- });-- const rows = await processResults<{ month: string; count: number }>(grandfatheredResult);- for (const row of rows) {- const existing = monthlyUsage.get(row.month) || 0;- monthlyUsage.set(row.month, existing + row.count);- }- }-- // Query new sites (all event types)- if (newSites.length > 0) {- const newSitesResult = await clickhouse.query({- query: `- SELECT- toYYYYMM(timestamp) as month,- COUNT(*) as count- FROM events- WHERE site_id IN {siteIds:Array(Int32)}- AND type IN ('pageview', 'custom_event', 'performance')- AND timestamp >= toDate({startDate:String})- GROUP BY month- ORDER BY month- `,- query_params: {- siteIds: newSites,- startDate: startDate,- },- format: "JSONEachRow",- });-- const rows = await processResults<{ month: string; count: number }>(newSitesResult);- for (const row of rows) {- const existing = monthlyUsage.get(row.month) || 0;- monthlyUsage.set(row.month, existing + row.count);- }+ // Combine queries into a single UNION query for better performance+ // Grandfathered sites (< 2000): count pageviews only+ // New sites (>= 2000): count pageviews, custom_events, and performance events+ const result = await clickhouse.query({+ query: `+ SELECT+ toYYYYMM(timestamp) as month,+ SUM(count) as count+ FROM (+ ${+ grandfatheredSites.length > 0+ ? `+ SELECT+ timestamp,+ 1 as count+ FROM events+ WHERE site_id IN {grandfatheredSites:Array(Int32)}+ AND type = 'pageview'+ AND timestamp >= toDate({startDate:String})+ `+ : ""+ }+ ${grandfatheredSites.length > 0 && newSites.length > 0 ? "UNION ALL" : ""}+ ${+ newSites.length > 0+ ? `+ SELECT+ timestamp,+ 1 as count+ FROM events+ WHERE site_id IN {newSites:Array(Int32)}+ AND type IN ('pageview', 'custom_event', 'performance')+ AND timestamp >= toDate({startDate:String})+ `+ : ""+ }+ )+ GROUP BY month+ ORDER BY month+ `,+ query_params: {+ ...(grandfatheredSites.length > 0 && { grandfatheredSites }),+ ...(newSites.length > 0 && { newSites }),+ startDate: startDate,+ },+ format: "JSONEachRow",+ });++ const rows = await processResults<{ month: string; count: number }>(result);+ for (const row of rows) {+ monthlyUsage.set(row.month, row.count);}return monthlyUsage;} catch (error) {console.error(`Error querying ClickHouse for monthly usage:`, error);- return new Map();+ throw new Error(+ `Failed to query monthly usage for quota check: ${error instanceof Error ? error.message : "Unknown error"}`+ );}}@@ -164,33 +156,6 @@ export class ImportQuotaTracker {return true;}- getMonthQuota(month: string): MonthlyQuotaInfo {- const used = this.monthlyUsage.get(month) || 0;- return {- month,- used,- limit: this.monthlyLimit,- remaining: Math.max(0, this.monthlyLimit - used),- };- }-- getAllMonthQuotas(): MonthlyQuotaInfo[] {- const quotas: MonthlyQuotaInfo[] = [];-- if (this.monthlyLimit === Infinity) {- return quotas;- }-- const now = DateTime.utc();- for (let i = 0; i < this.historicalWindowMonths; i++) {- const monthDate = now.minus({ months: i }).startOf("month");- const month = monthDate.toFormat("yyyyMM");- quotas.push(this.getMonthQuota(month));- }-- return quotas.reverse();- }-getSummary(): {totalMonthsInWindow: number;monthsAtCapacity: number;@@ -206,13 +171,18 @@ export class ImportQuotaTracker {};}- const quotas = this.getAllMonthQuotas();- const monthsAtCapacity = quotas.filter(q => q.remaining === 0).length;+ // Calculate directly from stored data instead of regenerating all monthly quotas+ let monthsAtCapacity = 0;+ for (const usage of this.monthlyUsage.values()) {+ if (usage >= this.monthlyLimit) {+ monthsAtCapacity++;+ }+ }return {totalMonthsInWindow: this.historicalWindowMonths,monthsAtCapacity,- monthsWithSpace: quotas.length - monthsAtCapacity,+ monthsWithSpace: this.historicalWindowMonths - monthsAtCapacity,oldestAllowedMonth: this.oldestAllowedMonth,};}diff --git a/server/src/services/import/importStatusManager.ts b/server/src/services/import/importStatusManager.tsindex 959efdf08..91d5e4e8b 100644--- a/server/src/services/import/importStatusManager.ts+++ b/server/src/services/import/importStatusManager.ts@@ -3,81 +3,46 @@ import { eq, desc, sql } from "drizzle-orm";import { importStatus } from "../../db/postgres/schema.js";import { DateTime } from "luxon";-type SelectImportStatus = typeof importStatus.$inferSelect;-type InsertImportStatus = typeof importStatus.$inferInsert;+export type SelectImportStatus = typeof importStatus.$inferSelect;+export type InsertImportStatus = typeof importStatus.$inferInsert;-export class ImportStatusManager {- static async createImportStatus(data: InsertImportStatus): Promise<void> {- try {- await db.insert(importStatus).values(data);- } catch (error) {- console.error("Failed to create import status:", error);- throw new Error(`Failed to create import status: ${error instanceof Error ? error.message : "Unknown error"}`);- }- }+export async function createImportStatus(data: InsertImportStatus): Promise<void> {+ await db.insert(importStatus).values(data);+}- static async updateStatus(- importId: string,- status: InsertImportStatus["status"],- errorMessage?: string- ): Promise<void> {- const completedAt = status === "completed" || status === "failed" ? DateTime.utc().toISO() : null;+export async function updateImportStatus(+ importId: string,+ status: InsertImportStatus["status"],+ errorMessage?: string+): Promise<void> {+ const completedAt = status === "completed" || status === "failed" ? DateTime.utc().toISO() : null;- try {- await db- .update(importStatus)- .set({ status, errorMessage, completedAt })- .where(eq(importStatus.importId, importId));- } catch (error) {- console.error(`Failed to update import status for ${importId}:`, error);- throw new Error(`Failed to update import status: ${error instanceof Error ? error.message : "Unknown error"}`);- }- }+ await db.update(importStatus).set({ status, errorMessage, completedAt }).where(eq(importStatus.importId, importId));+}- static async updateProgress(importId: string, importedEvents: number): Promise<void> {- try {- await db- .update(importStatus)- .set({- importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`,- })- .where(eq(importStatus.importId, importId));- } catch (error) {- console.error(`Failed to update import progress for ${importId}:`, error);- throw new Error(`Failed to update import progress: ${error instanceof Error ? error.message : "Unknown error"}`);- }- }+export async function updateImportProgress(importId: string, importedEvents: number): Promise<void> {+ await db+ .update(importStatus)+ .set({+ importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`,+ })+ .where(eq(importStatus.importId, importId));+}- static async getImportsForSite(siteId: number, limit = 10): Promise<SelectImportStatus[]> {- try {- return await db.query.importStatus.findMany({- where: eq(importStatus.siteId, siteId),- orderBy: [desc(importStatus.startedAt)],- limit,- });- } catch (error) {- console.error(`Failed to get imports for site ${siteId}:`, error);- throw new Error(`Failed to get imports for site: ${error instanceof Error ? error.message : "Unknown error"}`);- }- }+export async function getImportsForSite(siteId: number, limit = 10): Promise<SelectImportStatus[]> {+ return await db.query.importStatus.findMany({+ where: eq(importStatus.siteId, siteId),+ orderBy: [desc(importStatus.startedAt)],+ limit,+ });+}- static async deleteImport(importId: string): Promise<void> {- try {- await db.delete(importStatus).where(eq(importStatus.importId, importId));- } catch (error) {- console.error(`Failed to delete import ${importId}:`, error);- throw new Error(`Failed to delete import: ${error instanceof Error ? error.message : "Unknown error"}`);- }- }+export async function deleteImport(importId: string): Promise<void> {+ await db.delete(importStatus).where(eq(importStatus.importId, importId));+}- static async getImportById(importId: string): Promise<SelectImportStatus | undefined> {- try {- return await db.query.importStatus.findFirst({- where: eq(importStatus.importId, importId),- });- } catch (error) {- console.error(`Failed to get import ${importId}:`, error);- throw new Error(`Failed to get import: ${error instanceof Error ? error.message : "Unknown error"}`);- }- }+export async function getImportById(importId: string): Promise<SelectImportStatus | undefined> {+ return await db.query.importStatus.findFirst({+ where: eq(importStatus.importId, importId),+ });}diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.tsindex 311957874..bfe6e1333 100644--- a/server/src/services/import/utils.ts+++ b/server/src/services/import/utils.ts@@ -1,4 +1,5 @@import { unlink } from "node:fs/promises";+import { IS_CLOUD } from "../../lib/const.js";import { r2Storage } from "../storage/r2StorageService.js";export interface DeleteFileResult {@@ -6,6 +7,25 @@ export interface DeleteFileResult {error?: string;}+export interface StorageLocationInfo {+ location: string;+ isR2: boolean;+}++export function getImportStorageLocation(importId: string, filename: string): StorageLocationInfo {+ const isR2 = IS_CLOUD && r2Storage.isEnabled();+ if (isR2) {+ return {+ location: `imports/${importId}/${filename}`,+ isR2: true,+ };+ }+ return {+ location: `/tmp/imports/${importId}.csv`,+ isR2: false,+ };+}+/*** Delete an import file from storage.* Returns result instead of throwing to prevent worker crashes.diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 2c3d21c70..d38d34cb1 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -6,7 +6,7 @@ import { getJobQueue } from "../../../queues/jobQueueFactory.js";import { r2Storage } from "../../storage/r2StorageService.js";import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";-import { ImportStatusManager } from "../importStatusManager.js";+import { updateImportStatus } from "../importStatusManager.js";import { ImportLimiter } from "../importLimiter.js";import { deleteImportFile } from "../utils.js";@@ -109,7 +109,13 @@ export async function registerCsvParseWorker() {? await createR2FileStream(storageLocation, source): await createLocalFileStream(storageLocation, source);- await ImportStatusManager.updateStatus(importId, "processing");+ // Add explicit error handler before starting to consume the stream+ stream.on("error", error => {+ console.error(`[Import ${importId}] Stream error:`, error);+ // Error will be caught by the outer try/catch+ });++ await updateImportStatus(importId, "processing");// Set timeout to prevent indefinite processingprocessingTimeout = setTimeout(() => {@@ -175,7 +181,7 @@ export async function registerCsvParseWorker() {`No events could be imported. All ${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +`${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +`Try importing newer data or upgrade your plan for higher monthly quotas.`;- await ImportStatusManager.updateStatus(importId, "failed", errorMessage);+ await updateImportStatus(importId, "failed", errorMessage);const deleteResult = await deleteImportFile(storageLocation, isR2Storage);if (!deleteResult.success) {console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);@@ -214,7 +220,7 @@ export async function registerCsvParseWorker() {? error.message.replace(/\/[^\s]+/g, "[path]").substring(0, 500): "An unexpected error occurred during import processing";- await ImportStatusManager.updateStatus(importId, "failed", safeMessage);+ await updateImportStatus(importId, "failed", safeMessage);// Don't re-throw - worker should continue processing other jobsconsole.error(`[Import ${importId}] Import failed, worker continuing`);diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex 794c6e02c..162e36f25 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -2,7 +2,7 @@ import { getJobQueue } from "../../../queues/jobQueueFactory.js";import { UmamiImportMapper } from "../mappings/umami.js";import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { ImportStatusManager } from "../importStatusManager.js";+import { updateImportStatus, updateImportProgress } from "../importStatusManager.js";const getImportDataMapping = (source: string) => {switch (source) {@@ -22,14 +22,14 @@ export async function registerDataInsertWorker() {// Handle finalization signalif (allChunksSent) {try {- await ImportStatusManager.updateStatus(importId, "completed");+ await updateImportStatus(importId, "completed");console.log(`[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`);return;} catch (error) {console.error(`[Import ${importId}] Failed to mark as completed:`, error);// Try to update to failed status, but don't crash workertry {- await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");+ await updateImportStatus(importId, "failed", "Failed to complete import");} catch (updateError) {console.error(`[Import ${importId}] Could not update status to failed:`, updateError);}@@ -52,7 +52,7 @@ export async function registerDataInsertWorker() {// Update progress (non-critical - log if fails but don't crash)try {- await ImportStatusManager.updateProgress(importId, transformedRecords.length);+ await updateImportProgress(importId, transformedRecords.length);} catch (progressError) {console.warn(`[Import ${importId}] Progress update failed (data inserted successfully):`,@@ -72,7 +72,7 @@ export async function registerDataInsertWorker() {: "Data insertion failed due to unknown error";try {- await ImportStatusManager.updateStatus(importId, "failed", safeMessage);+ await updateImportStatus(importId, "failed", safeMessage);} catch (updateError) {console.error(`[Import ${importId}] Could not update status to failed:`, updateError);}