🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
batchImportEvents
Edward Cui557a0dcb8368
diff --git a/client/src/lib/import/csvParser.ts b/client/src/lib/import/csvParser.tsindex d1117b985..706f740a0 100644--- a/client/src/lib/import/csvParser.ts+++ b/client/src/lib/import/csvParser.ts@@ -25,13 +25,6 @@ interface UmamiEvent {created_at: string;}--export interface ImportProgress {- importedEvents: number;- skippedEvents: number;- invalidEvents: number;-}-export class CsvParser {private aborted = false;private siteId: number = 0;@@ -45,14 +38,6 @@ export class CsvParser {private isProcessing = false;private parsingComplete = false;- // Track cumulative counts- private totalImported = 0;- private totalSkipped = 0;- private totalInvalid = 0;-- // Callback for progress updates- private onProgress?: (progress: ImportProgress) => void;-startImport(file: File,siteId: number,@@ -60,12 +45,10 @@ export class CsvParser {platform: "umami",earliestAllowedDate: string,latestAllowedDate: string,- onProgress?: (progress: ImportProgress) => void): void {this.siteId = siteId;this.importId = importId;this.platform = platform;- this.onProgress = onProgress;this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");this.latestAllowedDate = DateTime.fromFormat(latestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).endOf("day");@@ -193,31 +176,13 @@ export class CsvParser {}try {- const data = await authedFetch<{- imported: number;- skipped: number;- invalid: number;- }>(`/api/batch-import-events/${this.siteId}/${this.importId}`, undefined, {+ await authedFetch(`/api/batch-import-events/${this.siteId}/${this.importId}`, undefined, {method: "POST",data: {events,isLastBatch,},});-- // Update cumulative counts- this.totalImported += data.imported;- this.totalSkipped += data.skipped;- this.totalInvalid += data.invalid;-- // Notify progress callback if provided- if (this.onProgress) {- this.onProgress({- importedEvents: this.totalImported,- skippedEvents: this.totalSkipped,- invalidEvents: this.totalInvalid,- });- }} catch (error) {console.error("Error uploading chunk:", error);this.aborted = true;diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex ceae2ac1e..2696ce332 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -16,7 +16,7 @@ const batchImportRequestSchema = zimportId: z.string().uuid(),}),body: z.object({- events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema).min(1).max(10000),+ events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema).min(1),isLastBatch: z.boolean().optional(),}),})@@ -47,7 +47,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequereturn reply.status(403).send({ error: "Forbidden" });}- // Verify import existsconst importRecord = await getImportById(importId);if (!importRecord) {return reply.status(404).send({ error: "Import not found" });@@ -57,14 +56,10 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequereturn reply.status(400).send({ error: "Import does not belong to this site" });}- // Check if import is already completedif (importRecord.completedAt) {return reply.status(400).send({ error: "Import already completed" });}- // Platform is set during import creation- const platform = importRecord.platform;-const [siteRecord] = await db.select({ organizationId: sites.organizationId }).from(sites)@@ -76,14 +71,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}try {- // Get cached quota tracker from singleton (much faster!)const quotaTracker = await importQuotaManager.getTracker(siteRecord.organizationId);- // Transform events (validate and convert to internal format)const transformedEvents = UmamiImportMapper.transform(events, site, importId);const invalidEventCount = events.length - transformedEvents.length;- // Filter events through quota checkerconst eventsWithinQuota = [];let skippedDueToQuota = 0;@@ -95,7 +87,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}}- // Insert events to ClickHouse (even if 0 events, we still need to track progress)if (eventsWithinQuota.length > 0) {await clickhouse.insert({table: "events",@@ -104,21 +95,14 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque});}- // Update import progressawait updateImportProgress(importId, eventsWithinQuota.length, skippedDueToQuota, invalidEventCount);- // If this is the last batch, mark import as completedif (isLastBatch) {await completeImport(importId);importQuotaManager.completeImport(siteRecord.organizationId);}- // Return counts to client- return reply.send({- imported: eventsWithinQuota.length,- skipped: skippedDueToQuota,- invalid: invalidEventCount,- });+ return reply.send();} catch (insertError) {const errorMessage = insertError instanceof Error ? insertError.message : "Unknown error";console.error("Failed to insert events:", errorMessage);