Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

batchImportEvents

Edward Cui557a0dcb8368
diff --git a/client/src/lib/import/csvParser.ts b/client/src/lib/import/csvParser.ts
index d1117b985..706f740a0 100644
--- a/client/src/lib/import/csvParser.ts
+++ b/client/src/lib/import/csvParser.ts
@@ -25,13 +25,6 @@ interface UmamiEvent {
created_at: string;
}
-
-export interface ImportProgress {
- importedEvents: number;
- skippedEvents: number;
- invalidEvents: number;
-}
-
export class CsvParser {
private aborted = false;
private siteId: number = 0;
@@ -45,14 +38,6 @@ export class CsvParser {
private isProcessing = false;
private parsingComplete = false;
- // Track cumulative counts
- private totalImported = 0;
- private totalSkipped = 0;
- private totalInvalid = 0;
-
- // Callback for progress updates
- private onProgress?: (progress: ImportProgress) => void;
-
startImport(
file: File,
siteId: number,
@@ -60,12 +45,10 @@ export class CsvParser {
platform: "umami",
earliestAllowedDate: string,
latestAllowedDate: string,
- onProgress?: (progress: ImportProgress) => void
): void {
this.siteId = siteId;
this.importId = importId;
this.platform = platform;
- this.onProgress = onProgress;
this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");
this.latestAllowedDate = DateTime.fromFormat(latestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).endOf("day");
@@ -193,31 +176,13 @@ export class CsvParser {
}
try {
- const data = await authedFetch<{
- imported: number;
- skipped: number;
- invalid: number;
- }>(`/api/batch-import-events/${this.siteId}/${this.importId}`, undefined, {
+ await authedFetch(`/api/batch-import-events/${this.siteId}/${this.importId}`, undefined, {
method: "POST",
data: {
events,
isLastBatch,
},
});
-
- // Update cumulative counts
- this.totalImported += data.imported;
- this.totalSkipped += data.skipped;
- this.totalInvalid += data.invalid;
-
- // Notify progress callback if provided
- if (this.onProgress) {
- this.onProgress({
- importedEvents: this.totalImported,
- skippedEvents: this.totalSkipped,
- invalidEvents: this.totalInvalid,
- });
- }
} catch (error) {
console.error("Error uploading chunk:", error);
this.aborted = true;
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index ceae2ac1e..2696ce332 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -16,7 +16,7 @@ const batchImportRequestSchema = z
importId: z.string().uuid(),
}),
body: z.object({
- events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema).min(1).max(10000),
+ events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema).min(1),
isLastBatch: z.boolean().optional(),
}),
})
@@ -47,7 +47,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
return reply.status(403).send({ error: "Forbidden" });
}
- // Verify import exists
const importRecord = await getImportById(importId);
if (!importRecord) {
return reply.status(404).send({ error: "Import not found" });
@@ -57,14 +56,10 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
return reply.status(400).send({ error: "Import does not belong to this site" });
}
- // Check if import is already completed
if (importRecord.completedAt) {
return reply.status(400).send({ error: "Import already completed" });
}
- // Platform is set during import creation
- const platform = importRecord.platform;
-
const [siteRecord] = await db
.select({ organizationId: sites.organizationId })
.from(sites)
@@ -76,14 +71,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
}
try {
- // Get cached quota tracker from singleton (much faster!)
const quotaTracker = await importQuotaManager.getTracker(siteRecord.organizationId);
- // Transform events (validate and convert to internal format)
const transformedEvents = UmamiImportMapper.transform(events, site, importId);
const invalidEventCount = events.length - transformedEvents.length;
- // Filter events through quota checker
const eventsWithinQuota = [];
let skippedDueToQuota = 0;
@@ -95,7 +87,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
}
}
- // Insert events to ClickHouse (even if 0 events, we still need to track progress)
if (eventsWithinQuota.length > 0) {
await clickhouse.insert({
table: "events",
@@ -104,21 +95,14 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
});
}
- // Update import progress
await updateImportProgress(importId, eventsWithinQuota.length, skippedDueToQuota, invalidEventCount);
- // If this is the last batch, mark import as completed
if (isLastBatch) {
await completeImport(importId);
importQuotaManager.completeImport(siteRecord.organizationId);
}
- // Return counts to client
- return reply.send({
- imported: eventsWithinQuota.length,
- skipped: skippedDueToQuota,
- invalid: invalidEventCount,
- });
+ return reply.send();
} catch (insertError) {
const errorMessage = insertError instanceof Error ? insertError.message : "Unknown error";
console.error("Failed to insert events:", errorMessage);