🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Rename source to platform
Edward Cui0089bf011102
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.tsindex dd6717e0d..0be9d43a3 100644--- a/client/src/api/admin/import.ts+++ b/client/src/api/admin/import.ts@@ -4,7 +4,7 @@ import { APIResponse } from "@/api/types";interface GetSiteImportsResponse {importId: string;- source: "umami";+ platform: "umami";status: "pending" | "processing" | "completed" | "failed";importedEvents: number;errorMessage: string | null;@@ -14,7 +14,7 @@ interface GetSiteImportsResponse {interface ImportSiteDataParams {file: File;- source: string;+ platform: string;startDate?: string;endDate?: string;}@@ -49,7 +49,7 @@ export function useImportSiteData(site: number) {mutationFn: async (params: ImportSiteDataParams) => {const formData = new FormData();- formData.append("source", params.source);+ formData.append("platform", params.platform);if (params.startDate) {formData.append("startDate", params.startDate);}diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex 30cd999be..a714f3e90 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -38,10 +38,10 @@ interface ImportManagerProps {const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MBconst ALLOWED_FILE_TYPES = ["text/csv"];const ALLOWED_EXTENSIONS = [".csv"];-const DATA_SOURCES = [{ value: "umami", label: "Umami" }] as const;+const PLATFORMS = [{ value: "umami", label: "Umami" }] as const;const importFormSchema = z.object({- source: z.enum(["umami"], { required_error: "Please select a data source" }),+ platform: z.enum(["umami"], { required_error: "Please select a platform" }),file: z.custom<FileList>().refine(files => files.length === 1, "Please select a file")@@ -114,7 +114,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {resolver: zodResolver(importFormSchema),mode: "onChange",defaultValues: {- source: "" as "umami",+ platform: "" as "umami",dateRange: {},},});@@ -144,7 +144,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {mutation.mutate({file,- source: data.source,+ platform: data.platform,startDate,endDate,},@@ -256,14 +256,14 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {)}<form onSubmit={handleSubmit(onSubmit)} className="space-y-4">- {/* Data Source Selection */}+ {/* Platform Selection */}<div className="space-y-2">- <Label htmlFor="source" className="flex items-center gap-2">+ <Label htmlFor="platform" className="flex items-center gap-2"><Database className="h-4 w-4" />- Data Source+ Platform</Label><Controller- name="source"+ name="platform"control={control}render={({ field }) => (<Select@@ -272,19 +272,19 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {disabled={disabled || mutation.isPending || hasActiveImport}><SelectTrigger>- <SelectValue placeholder="Select data source" />+ <SelectValue placeholder="Select platform" /></SelectTrigger><SelectContent>- {DATA_SOURCES.map(dataSource => (- <SelectItem key={dataSource.value} value={dataSource.value}>- {dataSource.label}+ {PLATFORMS.map(platform => (+ <SelectItem key={platform.value} value={platform.value}>+ {platform.label}</SelectItem>))}</SelectContent></Select>)}/>- {errors.source && <p className="text-sm text-red-600">{errors.source.message}</p>}+ {errors.platform && <p className="text-sm text-red-600">{errors.platform.message}</p>}</div>{/* Date Range Picker */}@@ -426,7 +426,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<TableHeader><TableRow><TableHead>File</TableHead>- <TableHead>Source</TableHead>+ <TableHead>Platform</TableHead><TableHead>Status</TableHead><TableHead className="text-right">Events</TableHead><TableHead className="text-right">Actions</TableHead>@@ -446,7 +446,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</TableCell><TableCell><Badge variant="outline" className="capitalize">- {imp.source}+ {imp.platform}</Badge></TableCell><TableCell>diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.tsindex 082244861..2bb41f636 100644--- a/server/src/api/sites/getSiteImports.ts+++ b/server/src/api/sites/getSiteImports.ts@@ -36,9 +36,9 @@ export async function getSiteImports(request: FastifyRequest<GetSiteImportsRequeconst imports = await getImportsForSite(siteId);return reply.send({- data: imports.map(({ importId, source, status, importedEvents, errorMessage, startedAt, fileName }) => ({+ data: imports.map(({ importId, platform, status, importedEvents, errorMessage, startedAt, fileName }) => ({importId,- source,+ platform,status,importedEvents,errorMessage,diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsindex 3d9d0508e..19f1b055d 100644--- a/server/src/api/sites/importSiteData.ts+++ b/server/src/api/sites/importSiteData.ts@@ -11,7 +11,7 @@ import { getJobQueue } from "../../queues/jobQueueFactory.js";import { ImportLimiter } from "../../services/import/importLimiter.js";import { updateImportStatus } from "../../services/import/importStatusManager.js";import { deleteImportFile, getImportStorageLocation } from "../../services/import/utils.js";-import { CSV_PARSE_QUEUE } from "../../services/import/workers/jobs.js";+import { CSV_PARSE_QUEUE, CsvParseJob } from "../../services/import/workers/jobs.js";import { r2Storage } from "../../services/storage/r2StorageService.js";const isValidDate = (date: string) => {@@ -25,7 +25,7 @@ const importDataFieldsSchema = z.object({fields: z.object({- source: z.enum(["umami"]),+ platform: z.enum(["umami"]),startDate: z.string().refine(isValidDate, { message: "Invalid start date format" }).optional(),endDate: z.string().refine(isValidDate, { message: "Invalid end date format" }).optional(),})@@ -87,7 +87,7 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,const parsedFields = importDataFieldsSchema.safeParse({fields: {- source: (data.fields.source as any)?.value,+ platform: (data.fields.platform as any)?.value,startDate: (data.fields.startDate as any)?.value,endDate: (data.fields.endDate as any)?.value,},@@ -97,7 +97,7 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,return reply.status(400).send({ error: "Validation error" });}- const { source, startDate, endDate } = parsedFields.data.fields;+ const { platform, startDate, endDate } = parsedFields.data.fields;const siteId = Number(site);const importId = randomUUID();@@ -114,8 +114,7 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,importId,siteId,organizationId: organization,- source,- status: "pending",+ platform,fileName: data.filename,});@@ -143,10 +142,10 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,try {const jobQueue = getJobQueue();- await jobQueue.send(CSV_PARSE_QUEUE, {+ await jobQueue.send<CsvParseJob>(CSV_PARSE_QUEUE, {site,importId,- source,+ platform,storageLocation: storage.location,isR2Storage: storage.isR2,organization,diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.tsindex 6ed182f13..a1ccf1570 100644--- a/server/src/db/postgres/schema.ts+++ b/server/src/db/postgres/schema.ts@@ -600,7 +600,7 @@ export const notificationChannels = pgTable(]);-export const importSourceEnum = pgEnum("import_source_enum", ["umami"]);+export const importPlatformEnum = pgEnum("import_platform_enum", ["umami"]);export const importStatusEnum = pgEnum("import_status_enum", ["pending", "processing", "completed", "failed"]);@@ -611,7 +611,7 @@ export const importStatus = pgTable(importId: text("import_id").primaryKey().notNull(),siteId: integer("site_id").notNull(),organizationId: text("organization_id").notNull(),- source: importSourceEnum("source").notNull(),+ platform: importPlatformEnum("platform").notNull(),status: importStatusEnum("status").notNull().default("pending"),importedEvents: integer("imported_events").notNull().default(0),errorMessage: text("error_message"),diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 4f182af39..b31c57f34 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -4,39 +4,39 @@ import { parse } from "@fast-csv/parse";import { DateTime } from "luxon";import { getJobQueue } from "../../../queues/jobQueueFactory.js";import { r2Storage } from "../../storage/r2StorageService.js";-import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";+import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE, DataInsertJob } from "./jobs.js";import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";import { updateImportStatus } from "../importStatusManager.js";import { ImportLimiter } from "../importLimiter.js";import { deleteImportFile } from "../utils.js";-const getImportDataHeaders = (source: string) => {- switch (source) {+const getImportDataHeaders = (platform: string) => {+ switch (platform) {case "umami":return umamiHeaders;default:- throw new Error(`Unsupported import source: ${source}`);+ throw new Error(`Unsupported platform: ${platform}`);}};-const createR2FileStream = async (storageLocation: string, source: string) => {+const createR2FileStream = async (storageLocation: string, platform: string) => {console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);const fileStream = await r2Storage.getImportFileStream(storageLocation);return fileStream.pipe(parse({- headers: getImportDataHeaders(source),+ headers: getImportDataHeaders(platform),renameHeaders: true,ignoreEmpty: true,}));};-const createLocalFileStream = async (storageLocation: string, source: string) => {+const createLocalFileStream = async (storageLocation: string, platform: string) => {console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);await access(storageLocation, constants.F_OK | constants.R_OK);return createReadStream(storageLocation).pipe(parse({- headers: getImportDataHeaders(source),+ headers: getImportDataHeaders(platform),renameHeaders: true,ignoreEmpty: true,})@@ -86,7 +86,7 @@ export async function registerCsvParseWorker() {const jobQueue = getJobQueue();await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([job]) => {- const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;+ const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;let stream: ReturnType<typeof parse> | null = null;let processingTimeout: NodeJS.Timeout | null = null;@@ -106,8 +106,8 @@ export async function registerCsvParseWorker() {let chunksSent = 0;stream = isR2Storage- ? await createR2FileStream(storageLocation, source)- : await createLocalFileStream(storageLocation, source);+ ? await createR2FileStream(storageLocation, platform)+ : await createLocalFileStream(storageLocation, platform);// Add explicit error handler before starting to consume the streamstream.on("error", error => {@@ -156,10 +156,10 @@ export async function registerCsvParseWorker() {totalAccepted++;if (chunk.length >= chunkSize) {- await jobQueue.send(DATA_INSERT_QUEUE, {+ await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {site,importId,- source,+ platform,chunk,chunkNumber: chunksSent,allChunksSent: false,@@ -191,10 +191,10 @@ export async function registerCsvParseWorker() {// Send final chunk if any data remainsif (chunk.length > 0) {- await jobQueue.send(DATA_INSERT_QUEUE, {+ await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {site,importId,- source,+ platform,chunk,chunkNumber: chunksSent,allChunksSent: false,@@ -203,10 +203,10 @@ export async function registerCsvParseWorker() {}// Send finalization signal with total chunk count- await jobQueue.send(DATA_INSERT_QUEUE, {+ await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {site,importId,- source,+ platform,chunk: [],totalChunks: chunksSent,allChunksSent: true,diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex 162e36f25..b00427de6 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -4,12 +4,12 @@ import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { updateImportStatus, updateImportProgress } from "../importStatusManager.js";-const getImportDataMapping = (source: string) => {- switch (source) {+const getImportDataMapping = (platform: string) => {+ switch (platform) {case "umami":return UmamiImportMapper;default:- throw new Error(`Unsupported import source: ${source}`);+ throw new Error(`Unsupported platform: ${platform}`);}};@@ -17,7 +17,7 @@ export async function registerDataInsertWorker() {const jobQueue = getJobQueue();await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([job]) => {- const { site, importId, source, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;+ const { site, importId, platform, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;// Handle finalization signalif (allChunksSent) {@@ -40,7 +40,7 @@ export async function registerDataInsertWorker() {// Process data chunktry {- const dataMapper = getImportDataMapping(source);+ const dataMapper = getImportDataMapping(platform);const transformedRecords = dataMapper.transform(chunk, site, importId);// Insert to ClickHouse (critical - must succeed)diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.tsindex 0bade893d..9319348e0 100644--- a/server/src/services/import/workers/jobs.ts+++ b/server/src/services/import/workers/jobs.ts@@ -7,7 +7,7 @@ export const DATA_INSERT_QUEUE = "data-insert";interface ImportJob {site: string;importId: string;- source: "umami";+ platform: "umami";}export interface CsvParseJob extends ImportJob {