🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify batchImportEvents
Edward Cui097f0e14ce58
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex 5761e22d8..d57778ffd 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -143,7 +143,7 @@ export class CSVWorkerManager {this.uploadInProgress = true;try {- const response = await fetch(`/api/batch-import-events/${this.siteId}`, {+ const response = await fetch(`/api/batch-import-events/${this.siteId}/${this.importId}`, {method: "POST",headers: {"Content-Type": "application/json",@@ -151,7 +151,6 @@ export class CSVWorkerManager {credentials: "include",body: JSON.stringify({events,- importId: this.importId,}),});diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 7d74164bf..2d45b89dc 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -12,38 +12,14 @@ import { createServiceLogger } from "../../lib/logger/logger.js";const logger = createServiceLogger("import:batch");-// Zod schema for Umami event (from CSV)-const umamiEventSchema = z.object({- session_id: z.string(),- hostname: z.string(),- browser: z.string(),- os: z.string(),- device: z.string(),- screen: z.string(),- language: z.string(),- country: z.string(),- region: z.string(),- city: z.string(),- url_path: z.string(),- url_query: z.string(),- referrer_path: z.string(),- referrer_query: z.string(),- referrer_domain: z.string(),- page_title: z.string(),- event_type: z.string(),- event_name: z.string(),- distinct_id: z.string(),- created_at: z.string(),-});-const batchImportRequestSchema = z.object({params: z.object({site: z.string().min(1),+ importId: z.string().uuid(),}),body: z.object({- events: z.array(umamiEventSchema).min(1).max(10000), // Properly typed Umami events- importId: z.string().uuid(),+ events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema).min(1).max(10000),}),}).strict();@@ -62,11 +38,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequeif (!parsed.success) {logger.error({ error: parsed.error }, "Validation error");- return reply.status(400).send({ error: "Validation error", details: parsed.error.flatten() });+ return reply.status(400).send({ error: "Validation error" });}- const { site } = parsed.data.params;- const { events, importId } = parsed.data.body;+ const { site, importId } = parsed.data.params;+ const { events } = parsed.data.body;const siteId = Number(site);const userHasAccess = await getUserHasAdminAccessToSite(request, site);@@ -96,7 +72,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequereturn reply.status(400).send({ error: "Import has failed" });}- // Update status to processing if still pendingif (importRecord.status === "pending") {await updateImportStatus(importId, "processing");}@@ -104,20 +79,18 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque// Auto-detect platform if not set (first batch)let detectedPlatform = importRecord.platform;if (!detectedPlatform) {- // Detect platform based on event structure- // For now, we only support Umami, so if events match Umami schema, it's Umami- detectedPlatform = "umami";+ const firstEvent = events[0];- // Update import record with detected platform- await db- .update(importStatus)- .set({ platform: detectedPlatform })- .where(eq(importStatus.importId, importId));+ if (UmamiImportMapper.umamiEventKeyOnlySchema.safeParse(firstEvent).success) {+ detectedPlatform = "umami";+ } else {+ logger.error({ importId, firstEventKeys: Object.keys(firstEvent) }, "Unable to detect platform from event structure");+ return reply.status(400).send({ error: "Unable to detect platform from event structure" });+ }- logger.info({ importId, detectedPlatform }, "Auto-detected platform");+ await db.update(importStatus).set({ platform: detectedPlatform }).where(eq(importStatus.importId, importId));}- // Get organization ID for quota checkingconst [siteRecord] = await db.select({ organizationId: sites.organizationId }).from(sites)@@ -130,16 +103,14 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}try {- // Create quota tracker for server-side quota checkingconst quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);- // Filter events based on quota availabilityconst eventsWithinQuota: UmamiEvent[] = [];let skippedDueToQuota = 0;for (const event of events) {if (!event.created_at) {- continue; // Skip events without timestamp+ continue;}if (quotaTracker.canImportEvent(event.created_at)) {@@ -149,8 +120,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}}- // If all events were skipped due to quota, log warning- if (eventsWithinQuota.length === 0 && events.length > 0) {+ if (eventsWithinQuota.length === 0) {const quotaSummary = quotaTracker.getSummary();const errorMessage =`All ${events.length} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +@@ -165,7 +135,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque});}- // Transform events using server-side mapper (single source of truth)const transformedEvents = UmamiImportMapper.transform(eventsWithinQuota, site, importId);if (transformedEvents.length === 0) {@@ -177,7 +146,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque});}- // Insert transformed events into ClickHouseawait clickhouse.insert({table: "events",values: transformedEvents,@@ -193,7 +161,6 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque"Batch inserted successfully");- // Update progressawait updateImportProgress(importId, transformedEvents.length);return reply.send({diff --git a/server/src/index.ts b/server/src/index.tsindex bfa53a558..045107bec 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -351,9 +351,9 @@ server.get("/api/session-replay/list/:site", getSessionReplays);server.get("/api/session-replay/:sessionId/:site", getSessionReplayEvents);// Imports-server.post("/api/create-site-import/:site", createSiteImport);-server.post("/api/batch-import-events/:site", batchImportEvents);server.get("/api/get-site-imports/:site", getSiteImports);+server.post("/api/create-site-import/:site", createSiteImport);+server.post("/api/batch-import-events/:site/:importId", batchImportEvents);server.delete("/api/delete-site-import/:site/:importId", deleteSiteImport);// Administrativediff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.tsindex 5a9258aa2..786d07d1e 100644--- a/server/src/services/import/mappings/umami.ts+++ b/server/src/services/import/mappings/umami.ts@@ -3,6 +3,7 @@ import { getChannel } from "../../tracker/getChannel.js";import { RybbitEvent } from "./rybbit.js";import { z } from "zod";import { createServiceLogger } from "../../../lib/logger/logger.js";+import { deriveKeyOnlySchema } from "./utils.js";const logger = createServiceLogger("import:umami-mapper");@@ -193,6 +194,8 @@ export class UmamiImportMapper {created_at: z.string().regex(/^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/),});+ static readonly umamiEventKeyOnlySchema = deriveKeyOnlySchema(UmamiImportMapper.umamiEventSchema);+static transform(events: UmamiEvent[], site: string, importId: string): RybbitEvent[] {return events.reduce<RybbitEvent[]>((acc, event) => {const parsed = UmamiImportMapper.umamiEventSchema.safeParse(event);diff --git a/server/src/services/import/mappings/utils.ts b/server/src/services/import/mappings/utils.tsnew file mode 100644index 000000000..93f59f8e8--- /dev/null+++ b/server/src/services/import/mappings/utils.ts@@ -0,0 +1,18 @@+import { z } from "zod";++export function deriveKeyOnlySchema<T extends z.ZodRawShape>(+ schema: z.ZodObject<T>+): z.ZodObject<{+ [K in keyof T]: z.ZodString;+}> {+ const shape = schema.shape;+ const keyOnlyShape = Object.keys(shape).reduce(+ (acc, key) => {+ acc[key as keyof T] = z.string();+ return acc;+ },+ {} as { [K in keyof T]: z.ZodString }+ );++ return z.object(keyOnlyShape);+}