Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Simplify batchImportEvents

Edward Cui097f0e14ce58
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.ts
index 5761e22d8..d57778ffd 100644
--- a/client/src/lib/import/csv-worker-manager.ts
+++ b/client/src/lib/import/csv-worker-manager.ts
@@ -143,7 +143,7 @@ export class CSVWorkerManager {
this.uploadInProgress = true;
try {
- const response = await fetch(`/api/batch-import-events/${this.siteId}`, {
+ const response = await fetch(`/api/batch-import-events/${this.siteId}/${this.importId}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
@@ -151,7 +151,6 @@ export class CSVWorkerManager {
credentials: "include",
body: JSON.stringify({
events,
- importId: this.importId,
}),
});
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 7d74164bf..2d45b89dc 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -12,38 +12,14 @@ import { createServiceLogger } from "../../lib/logger/logger.js";
const logger = createServiceLogger("import:batch");
-// Zod schema for Umami event (from CSV)
-const umamiEventSchema = z.object({
- session_id: z.string(),
- hostname: z.string(),
- browser: z.string(),
- os: z.string(),
- device: z.string(),
- screen: z.string(),
- language: z.string(),
- country: z.string(),
- region: z.string(),
- city: z.string(),
- url_path: z.string(),
- url_query: z.string(),
- referrer_path: z.string(),
- referrer_query: z.string(),
- referrer_domain: z.string(),
- page_title: z.string(),
- event_type: z.string(),
- event_name: z.string(),
- distinct_id: z.string(),
- created_at: z.string(),
-});
-
const batchImportRequestSchema = z
.object({
params: z.object({
site: z.string().min(1),
+ importId: z.string().uuid(),
}),
body: z.object({
- events: z.array(umamiEventSchema).min(1).max(10000), // Properly typed Umami events
- importId: z.string().uuid(),
+ events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema).min(1).max(10000),
}),
})
.strict();
@@ -62,11 +38,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
if (!parsed.success) {
logger.error({ error: parsed.error }, "Validation error");
- return reply.status(400).send({ error: "Validation error", details: parsed.error.flatten() });
+ return reply.status(400).send({ error: "Validation error" });
}
- const { site } = parsed.data.params;
- const { events, importId } = parsed.data.body;
+ const { site, importId } = parsed.data.params;
+ const { events } = parsed.data.body;
const siteId = Number(site);
const userHasAccess = await getUserHasAdminAccessToSite(request, site);
@@ -96,7 +72,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
return reply.status(400).send({ error: "Import has failed" });
}
- // Update status to processing if still pending
if (importRecord.status === "pending") {
await updateImportStatus(importId, "processing");
}
@@ -104,20 +79,18 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
// Auto-detect platform if not set (first batch)
let detectedPlatform = importRecord.platform;
if (!detectedPlatform) {
- // Detect platform based on event structure
- // For now, we only support Umami, so if events match Umami schema, it's Umami
- detectedPlatform = "umami";
+ const firstEvent = events[0];
- // Update import record with detected platform
- await db
- .update(importStatus)
- .set({ platform: detectedPlatform })
- .where(eq(importStatus.importId, importId));
+ if (UmamiImportMapper.umamiEventKeyOnlySchema.safeParse(firstEvent).success) {
+ detectedPlatform = "umami";
+ } else {
+ logger.error({ importId, firstEventKeys: Object.keys(firstEvent) }, "Unable to detect platform from event structure");
+ return reply.status(400).send({ error: "Unable to detect platform from event structure" });
+ }
- logger.info({ importId, detectedPlatform }, "Auto-detected platform");
+ await db.update(importStatus).set({ platform: detectedPlatform }).where(eq(importStatus.importId, importId));
}
- // Get organization ID for quota checking
const [siteRecord] = await db
.select({ organizationId: sites.organizationId })
.from(sites)
@@ -130,16 +103,14 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
}
try {
- // Create quota tracker for server-side quota checking
const quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);
- // Filter events based on quota availability
const eventsWithinQuota: UmamiEvent[] = [];
let skippedDueToQuota = 0;
for (const event of events) {
if (!event.created_at) {
- continue; // Skip events without timestamp
+ continue;
}
if (quotaTracker.canImportEvent(event.created_at)) {
@@ -149,8 +120,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
}
}
- // If all events were skipped due to quota, log warning
- if (eventsWithinQuota.length === 0 && events.length > 0) {
+ if (eventsWithinQuota.length === 0) {
const quotaSummary = quotaTracker.getSummary();
const errorMessage =
`All ${events.length} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
@@ -165,7 +135,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
});
}
- // Transform events using server-side mapper (single source of truth)
const transformedEvents = UmamiImportMapper.transform(eventsWithinQuota, site, importId);
if (transformedEvents.length === 0) {
@@ -177,7 +146,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
});
}
- // Insert transformed events into ClickHouse
await clickhouse.insert({
table: "events",
values: transformedEvents,
@@ -193,7 +161,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
"Batch inserted successfully"
);
- // Update progress
await updateImportProgress(importId, transformedEvents.length);
return reply.send({
diff --git a/server/src/index.ts b/server/src/index.ts
index bfa53a558..045107bec 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -351,9 +351,9 @@ server.get("/api/session-replay/list/:site", getSessionReplays);
server.get("/api/session-replay/:sessionId/:site", getSessionReplayEvents);
// Imports
-server.post("/api/create-site-import/:site", createSiteImport);
-server.post("/api/batch-import-events/:site", batchImportEvents);
server.get("/api/get-site-imports/:site", getSiteImports);
+server.post("/api/create-site-import/:site", createSiteImport);
+server.post("/api/batch-import-events/:site/:importId", batchImportEvents);
server.delete("/api/delete-site-import/:site/:importId", deleteSiteImport);
// Administrative
diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.ts
index 5a9258aa2..786d07d1e 100644
--- a/server/src/services/import/mappings/umami.ts
+++ b/server/src/services/import/mappings/umami.ts
@@ -3,6 +3,7 @@ import { getChannel } from "../../tracker/getChannel.js";
import { RybbitEvent } from "./rybbit.js";
import { z } from "zod";
import { createServiceLogger } from "../../../lib/logger/logger.js";
+import { deriveKeyOnlySchema } from "./utils.js";
const logger = createServiceLogger("import:umami-mapper");
@@ -193,6 +194,8 @@ export class UmamiImportMapper {
created_at: z.string().regex(/^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/),
});
+ static readonly umamiEventKeyOnlySchema = deriveKeyOnlySchema(UmamiImportMapper.umamiEventSchema);
+
static transform(events: UmamiEvent[], site: string, importId: string): RybbitEvent[] {
return events.reduce<RybbitEvent[]>((acc, event) => {
const parsed = UmamiImportMapper.umamiEventSchema.safeParse(event);
diff --git a/server/src/services/import/mappings/utils.ts b/server/src/services/import/mappings/utils.ts
new file mode 100644
index 000000000..93f59f8e8
--- /dev/null
+++ b/server/src/services/import/mappings/utils.ts
@@ -0,0 +1,18 @@
+import { z } from "zod";
+
+export function deriveKeyOnlySchema<T extends z.ZodRawShape>(
+ schema: z.ZodObject<T>
+): z.ZodObject<{
+ [K in keyof T]: z.ZodString;
+}> {
+ const shape = schema.shape;
+ const keyOnlyShape = Object.keys(shape).reduce(
+ (acc, key) => {
+ acc[key as keyof T] = z.string();
+ return acc;
+ },
+ {} as { [K in keyof T]: z.ZodString }
+ );
+
+ return z.object(keyOnlyShape);
+}