Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Rename source to platform

Edward Cui0089bf011102
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.ts
index dd6717e0d..0be9d43a3 100644
--- a/client/src/api/admin/import.ts
+++ b/client/src/api/admin/import.ts
@@ -4,7 +4,7 @@ import { APIResponse } from "@/api/types";
interface GetSiteImportsResponse {
importId: string;
- source: "umami";
+ platform: "umami";
status: "pending" | "processing" | "completed" | "failed";
importedEvents: number;
errorMessage: string | null;
@@ -14,7 +14,7 @@ interface GetSiteImportsResponse {
interface ImportSiteDataParams {
file: File;
- source: string;
+ platform: string;
startDate?: string;
endDate?: string;
}
@@ -49,7 +49,7 @@ export function useImportSiteData(site: number) {
mutationFn: async (params: ImportSiteDataParams) => {
const formData = new FormData();
- formData.append("source", params.source);
+ formData.append("platform", params.platform);
if (params.startDate) {
formData.append("startDate", params.startDate);
}
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsx
index 30cd999be..a714f3e90 100644
--- a/client/src/components/SiteSettings/ImportManager.tsx
+++ b/client/src/components/SiteSettings/ImportManager.tsx
@@ -38,10 +38,10 @@ interface ImportManagerProps {
const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
const ALLOWED_FILE_TYPES = ["text/csv"];
const ALLOWED_EXTENSIONS = [".csv"];
-const DATA_SOURCES = [{ value: "umami", label: "Umami" }] as const;
+const PLATFORMS = [{ value: "umami", label: "Umami" }] as const;
const importFormSchema = z.object({
- source: z.enum(["umami"], { required_error: "Please select a data source" }),
+ platform: z.enum(["umami"], { required_error: "Please select a platform" }),
file: z
.custom<FileList>()
.refine(files => files.length === 1, "Please select a file")
@@ -114,7 +114,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
resolver: zodResolver(importFormSchema),
mode: "onChange",
defaultValues: {
- source: "" as "umami",
+ platform: "" as "umami",
dateRange: {},
},
});
@@ -144,7 +144,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
mutation.mutate(
{
file,
- source: data.source,
+ platform: data.platform,
startDate,
endDate,
},
@@ -256,14 +256,14 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
)}
<form onSubmit={handleSubmit(onSubmit)} className="space-y-4">
- {/* Data Source Selection */}
+ {/* Platform Selection */}
<div className="space-y-2">
- <Label htmlFor="source" className="flex items-center gap-2">
+ <Label htmlFor="platform" className="flex items-center gap-2">
<Database className="h-4 w-4" />
- Data Source
+ Platform
</Label>
<Controller
- name="source"
+ name="platform"
control={control}
render={({ field }) => (
<Select
@@ -272,19 +272,19 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
disabled={disabled || mutation.isPending || hasActiveImport}
>
<SelectTrigger>
- <SelectValue placeholder="Select data source" />
+ <SelectValue placeholder="Select platform" />
</SelectTrigger>
<SelectContent>
- {DATA_SOURCES.map(dataSource => (
- <SelectItem key={dataSource.value} value={dataSource.value}>
- {dataSource.label}
+ {PLATFORMS.map(platform => (
+ <SelectItem key={platform.value} value={platform.value}>
+ {platform.label}
</SelectItem>
))}
</SelectContent>
</Select>
)}
/>
- {errors.source && <p className="text-sm text-red-600">{errors.source.message}</p>}
+ {errors.platform && <p className="text-sm text-red-600">{errors.platform.message}</p>}
</div>
{/* Date Range Picker */}
@@ -426,7 +426,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<TableHeader>
<TableRow>
<TableHead>File</TableHead>
- <TableHead>Source</TableHead>
+ <TableHead>Platform</TableHead>
<TableHead>Status</TableHead>
<TableHead className="text-right">Events</TableHead>
<TableHead className="text-right">Actions</TableHead>
@@ -446,7 +446,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
</TableCell>
<TableCell>
<Badge variant="outline" className="capitalize">
- {imp.source}
+ {imp.platform}
</Badge>
</TableCell>
<TableCell>
diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.ts
index 082244861..2bb41f636 100644
--- a/server/src/api/sites/getSiteImports.ts
+++ b/server/src/api/sites/getSiteImports.ts
@@ -36,9 +36,9 @@ export async function getSiteImports(request: FastifyRequest<GetSiteImportsReque
const imports = await getImportsForSite(siteId);
return reply.send({
- data: imports.map(({ importId, source, status, importedEvents, errorMessage, startedAt, fileName }) => ({
+ data: imports.map(({ importId, platform, status, importedEvents, errorMessage, startedAt, fileName }) => ({
importId,
- source,
+ platform,
status,
importedEvents,
errorMessage,
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
index 3d9d0508e..19f1b055d 100644
--- a/server/src/api/sites/importSiteData.ts
+++ b/server/src/api/sites/importSiteData.ts
@@ -11,7 +11,7 @@ import { getJobQueue } from "../../queues/jobQueueFactory.js";
import { ImportLimiter } from "../../services/import/importLimiter.js";
import { updateImportStatus } from "../../services/import/importStatusManager.js";
import { deleteImportFile, getImportStorageLocation } from "../../services/import/utils.js";
-import { CSV_PARSE_QUEUE } from "../../services/import/workers/jobs.js";
+import { CSV_PARSE_QUEUE, CsvParseJob } from "../../services/import/workers/jobs.js";
import { r2Storage } from "../../services/storage/r2StorageService.js";
const isValidDate = (date: string) => {
@@ -25,7 +25,7 @@ const importDataFieldsSchema = z
.object({
fields: z
.object({
- source: z.enum(["umami"]),
+ platform: z.enum(["umami"]),
startDate: z.string().refine(isValidDate, { message: "Invalid start date format" }).optional(),
endDate: z.string().refine(isValidDate, { message: "Invalid end date format" }).optional(),
})
@@ -87,7 +87,7 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,
const parsedFields = importDataFieldsSchema.safeParse({
fields: {
- source: (data.fields.source as any)?.value,
+ platform: (data.fields.platform as any)?.value,
startDate: (data.fields.startDate as any)?.value,
endDate: (data.fields.endDate as any)?.value,
},
@@ -97,7 +97,7 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,
return reply.status(400).send({ error: "Validation error" });
}
- const { source, startDate, endDate } = parsedFields.data.fields;
+ const { platform, startDate, endDate } = parsedFields.data.fields;
const siteId = Number(site);
const importId = randomUUID();
@@ -114,8 +114,7 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,
importId,
siteId,
organizationId: organization,
- source,
- status: "pending",
+ platform,
fileName: data.filename,
});
@@ -143,10 +142,10 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,
try {
const jobQueue = getJobQueue();
- await jobQueue.send(CSV_PARSE_QUEUE, {
+ await jobQueue.send<CsvParseJob>(CSV_PARSE_QUEUE, {
site,
importId,
- source,
+ platform,
storageLocation: storage.location,
isR2Storage: storage.isR2,
organization,
diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.ts
index 6ed182f13..a1ccf1570 100644
--- a/server/src/db/postgres/schema.ts
+++ b/server/src/db/postgres/schema.ts
@@ -600,7 +600,7 @@ export const notificationChannels = pgTable(
]
);
-export const importSourceEnum = pgEnum("import_source_enum", ["umami"]);
+export const importPlatformEnum = pgEnum("import_platform_enum", ["umami"]);
export const importStatusEnum = pgEnum("import_status_enum", ["pending", "processing", "completed", "failed"]);
@@ -611,7 +611,7 @@ export const importStatus = pgTable(
importId: text("import_id").primaryKey().notNull(),
siteId: integer("site_id").notNull(),
organizationId: text("organization_id").notNull(),
- source: importSourceEnum("source").notNull(),
+ platform: importPlatformEnum("platform").notNull(),
status: importStatusEnum("status").notNull().default("pending"),
importedEvents: integer("imported_events").notNull().default(0),
errorMessage: text("error_message"),
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 4f182af39..b31c57f34 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -4,39 +4,39 @@ import { parse } from "@fast-csv/parse";
import { DateTime } from "luxon";
import { getJobQueue } from "../../../queues/jobQueueFactory.js";
import { r2Storage } from "../../storage/r2StorageService.js";
-import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";
+import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE, DataInsertJob } from "./jobs.js";
import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
import { updateImportStatus } from "../importStatusManager.js";
import { ImportLimiter } from "../importLimiter.js";
import { deleteImportFile } from "../utils.js";
-const getImportDataHeaders = (source: string) => {
- switch (source) {
+const getImportDataHeaders = (platform: string) => {
+ switch (platform) {
case "umami":
return umamiHeaders;
default:
- throw new Error(`Unsupported import source: ${source}`);
+ throw new Error(`Unsupported platform: ${platform}`);
}
};
-const createR2FileStream = async (storageLocation: string, source: string) => {
+const createR2FileStream = async (storageLocation: string, platform: string) => {
console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);
const fileStream = await r2Storage.getImportFileStream(storageLocation);
return fileStream.pipe(
parse({
- headers: getImportDataHeaders(source),
+ headers: getImportDataHeaders(platform),
renameHeaders: true,
ignoreEmpty: true,
})
);
};
-const createLocalFileStream = async (storageLocation: string, source: string) => {
+const createLocalFileStream = async (storageLocation: string, platform: string) => {
console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);
await access(storageLocation, constants.F_OK | constants.R_OK);
return createReadStream(storageLocation).pipe(
parse({
- headers: getImportDataHeaders(source),
+ headers: getImportDataHeaders(platform),
renameHeaders: true,
ignoreEmpty: true,
})
@@ -86,7 +86,7 @@ export async function registerCsvParseWorker() {
const jobQueue = getJobQueue();
await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([job]) => {
- const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
+ const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
let stream: ReturnType<typeof parse> | null = null;
let processingTimeout: NodeJS.Timeout | null = null;
@@ -106,8 +106,8 @@ export async function registerCsvParseWorker() {
let chunksSent = 0;
stream = isR2Storage
- ? await createR2FileStream(storageLocation, source)
- : await createLocalFileStream(storageLocation, source);
+ ? await createR2FileStream(storageLocation, platform)
+ : await createLocalFileStream(storageLocation, platform);
// Add explicit error handler before starting to consume the stream
stream.on("error", error => {
@@ -156,10 +156,10 @@ export async function registerCsvParseWorker() {
totalAccepted++;
if (chunk.length >= chunkSize) {
- await jobQueue.send(DATA_INSERT_QUEUE, {
+ await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
site,
importId,
- source,
+ platform,
chunk,
chunkNumber: chunksSent,
allChunksSent: false,
@@ -191,10 +191,10 @@ export async function registerCsvParseWorker() {
// Send final chunk if any data remains
if (chunk.length > 0) {
- await jobQueue.send(DATA_INSERT_QUEUE, {
+ await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
site,
importId,
- source,
+ platform,
chunk,
chunkNumber: chunksSent,
allChunksSent: false,
@@ -203,10 +203,10 @@ export async function registerCsvParseWorker() {
}
// Send finalization signal with total chunk count
- await jobQueue.send(DATA_INSERT_QUEUE, {
+ await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
site,
importId,
- source,
+ platform,
chunk: [],
totalChunks: chunksSent,
allChunksSent: true,
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 162e36f25..b00427de6 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -4,12 +4,12 @@ import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { updateImportStatus, updateImportProgress } from "../importStatusManager.js";
-const getImportDataMapping = (source: string) => {
- switch (source) {
+const getImportDataMapping = (platform: string) => {
+ switch (platform) {
case "umami":
return UmamiImportMapper;
default:
- throw new Error(`Unsupported import source: ${source}`);
+ throw new Error(`Unsupported platform: ${platform}`);
}
};
@@ -17,7 +17,7 @@ export async function registerDataInsertWorker() {
const jobQueue = getJobQueue();
await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([job]) => {
- const { site, importId, source, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;
+ const { site, importId, platform, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;
// Handle finalization signal
if (allChunksSent) {
@@ -40,7 +40,7 @@ export async function registerDataInsertWorker() {
// Process data chunk
try {
- const dataMapper = getImportDataMapping(source);
+ const dataMapper = getImportDataMapping(platform);
const transformedRecords = dataMapper.transform(chunk, site, importId);
// Insert to ClickHouse (critical - must succeed)
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
index 0bade893d..9319348e0 100644
--- a/server/src/services/import/workers/jobs.ts
+++ b/server/src/services/import/workers/jobs.ts
@@ -7,7 +7,7 @@ export const DATA_INSERT_QUEUE = "data-insert";
interface ImportJob {
site: string;
importId: string;
- source: "umami";
+ platform: "umami";
}
export interface CsvParseJob extends ImportJob {