Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

prettier

Edward Cuia4dc031f2ca9
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.ts
index 00e62818c..dd6717e0d 100644
--- a/client/src/api/admin/import.ts
+++ b/client/src/api/admin/import.ts
@@ -31,9 +31,9 @@ export function useGetSiteImports(site: number) {
return useQuery({
queryKey: ["get-site-imports", site],
queryFn: async () => await authedFetch<APIResponse<GetSiteImportsResponse[]>>(`/get-site-imports/${site}`),
- refetchInterval: (data) => {
- const hasActiveImports = data.state.data?.data.some(imp =>
- imp.status === "processing" || imp.status === "pending"
+ refetchInterval: data => {
+ const hasActiveImports = data.state.data?.data.some(
+ imp => imp.status === "processing" || imp.status === "pending"
);
return hasActiveImports ? 5000 : false;
},
@@ -65,7 +65,7 @@ export function useImportSiteData(site: number) {
},
onSuccess: async () => {
await queryClient.invalidateQueries({
- queryKey: ["get-site-imports", site]
+ queryKey: ["get-site-imports", site],
});
},
retry: false,
@@ -77,13 +77,17 @@ export function useDeleteSiteImport(site: number) {
return useMutation({
mutationFn: async (importId: string) => {
- return await authedFetch<APIResponse<DeleteImportResponse>>(`/delete-site-import/${site}/${importId}`, undefined, {
- method: "DELETE",
- });
+ return await authedFetch<APIResponse<DeleteImportResponse>>(
+ `/delete-site-import/${site}/${importId}`,
+ undefined,
+ {
+ method: "DELETE",
+ }
+ );
},
onSuccess: async () => {
await queryClient.invalidateQueries({
- queryKey: ["get-site-imports", site]
+ queryKey: ["get-site-imports", site],
});
},
retry: false,
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsx
index 854d9fffd..789b276d0 100644
--- a/client/src/components/SiteSettings/ImportManager.tsx
+++ b/client/src/components/SiteSettings/ImportManager.tsx
@@ -4,21 +4,8 @@ import { useState, useMemo, useRef } from "react";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
-import {
- Select,
- SelectContent,
- SelectItem,
- SelectTrigger,
- SelectValue,
-} from "@/components/ui/select";
-import {
- Table,
- TableBody,
- TableCell,
- TableHead,
- TableHeader,
- TableRow,
-} from "@/components/ui/table";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import {
AlertDialog,
AlertDialogAction,
@@ -33,16 +20,7 @@ import { Alert, AlertDescription } from "@/components/ui/alert";
import { Badge } from "@/components/ui/badge";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { Separator } from "@/components/ui/separator";
-import {
- Upload,
- FileText,
- AlertCircle,
- CheckCircle2,
- Clock,
- Loader2,
- Database,
- Trash2,
-} from "lucide-react";
+import { Upload, FileText, AlertCircle, CheckCircle2, Clock, Loader2, Database, Trash2 } from "lucide-react";
import { useGetSiteImports, useImportSiteData, useDeleteSiteImport } from "@/api/admin/import";
import { SplitDateRangePicker, DateRange } from "@/components/SplitDateRangePicker";
@@ -59,9 +37,7 @@ interface FileValidationError {
const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
const ALLOWED_FILE_TYPES = ["text/csv"];
const ALLOWED_EXTENSIONS = [".csv"];
-const DATA_SOURCES = [
- { value: "umami", label: "Umami" },
-];
+const DATA_SOURCES = [{ value: "umami", label: "Umami" }];
export function ImportManager({ siteId, disabled }: ImportManagerProps) {
const [file, setFile] = useState<File | null>(null);
@@ -89,14 +65,14 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
if (file.size > MAX_FILE_SIZE) {
return {
type: "size",
- message: `File size (${(file.size / 1024 / 1024).toFixed(1)}MB) exceeds the maximum limit of 100MB.`
+ message: `File size (${(file.size / 1024 / 1024).toFixed(1)}MB) exceeds the maximum limit of 100MB.`,
};
}
if (!ALLOWED_FILE_TYPES.includes(file.type) && file.type !== "") {
return {
type: "type",
- message: "Invalid file type. Only CSV files are accepted."
+ message: "Invalid file type. Only CSV files are accepted.",
};
}
@@ -104,14 +80,14 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
if (!ALLOWED_EXTENSIONS.includes(extension)) {
return {
type: "type",
- message: "Invalid file extension. Only .csv files are accepted."
+ message: "Invalid file extension. Only .csv files are accepted.",
};
}
if (file.name.length > 255) {
return {
type: "name",
- message: "Filename is too long. Please use a shorter filename."
+ message: "Filename is too long. Please use a shorter filename.",
};
}
@@ -182,31 +158,31 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
return {
color: "bg-green-100 text-green-800 border-green-200",
icon: CheckCircle2,
- label: "Completed"
+ label: "Completed",
};
case "failed":
return {
color: "bg-red-100 text-red-800 border-red-200",
icon: AlertCircle,
- label: "Failed"
+ label: "Failed",
};
case "processing":
return {
color: "bg-blue-100 text-blue-800 border-blue-200",
icon: Loader2,
- label: "Processing"
+ label: "Processing",
};
case "pending":
return {
color: "bg-yellow-100 text-yellow-800 border-yellow-200",
icon: Clock,
- label: "Pending"
+ label: "Pending",
};
default:
return {
color: "bg-gray-100 text-gray-800 border-gray-200",
icon: Clock,
- label: status
+ label: status,
};
}
};
@@ -234,9 +210,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<Upload className="h-5 w-5" />
Import Data
</CardTitle>
- <CardDescription>
- Import data from other analytics platforms. Supports CSV files up to 100MB.
- </CardDescription>
+ <CardDescription>Import data from other analytics platforms. Supports CSV files up to 100MB.</CardDescription>
</CardHeader>
<CardContent className="space-y-4">
{/* Data Source Selection */}
@@ -245,16 +219,12 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<Database className="h-4 w-4" />
Data Source
</Label>
- <Select
- value={source}
- onValueChange={setSource}
- disabled={disabled || mutation.isPending}
- >
+ <Select value={source} onValueChange={setSource} disabled={disabled || mutation.isPending}>
<SelectTrigger>
<SelectValue placeholder="Select data source" />
</SelectTrigger>
<SelectContent>
- {DATA_SOURCES.map((dataSource) => (
+ {DATA_SOURCES.map(dataSource => (
<SelectItem key={dataSource.value} value={dataSource.value}>
{dataSource.label}
</SelectItem>
@@ -296,19 +266,13 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<Alert variant="destructive">
<div className="flex items-center gap-2">
<AlertCircle className="h-4 w-4" />
- <AlertDescription>
- {fileError.message}
- </AlertDescription>
+ <AlertDescription>{fileError.message}</AlertDescription>
</div>
</Alert>
)}
{/* Import Button */}
- <Button
- onClick={handleImportClick}
- disabled={isImportDisabled}
- className="w-full sm:w-auto"
- >
+ <Button onClick={handleImportClick} disabled={isImportDisabled} className="w-full sm:w-auto">
{mutation.isPending ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
@@ -339,9 +303,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<Alert className="border-green-200 bg-green-50">
<div className="flex items-center gap-2">
<CheckCircle2 className="h-4 w-4 text-green-600" />
- <AlertDescription>
- File uploaded successfully and is being processed.
- </AlertDescription>
+ <AlertDescription>File uploaded successfully and is being processed.</AlertDescription>
</div>
</Alert>
)}
@@ -351,9 +313,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<Alert className="border-green-200 bg-green-50">
<div className="flex items-center gap-2">
<CheckCircle2 className="h-4 w-4 text-green-600" />
- <AlertDescription>
- Import deleted successfully.
- </AlertDescription>
+ <AlertDescription>Import deleted successfully.</AlertDescription>
</div>
</Alert>
)}
@@ -376,9 +336,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<Card>
<CardHeader>
<CardTitle>Import History</CardTitle>
- <CardDescription>
- Track the status of your data imports
- </CardDescription>
+ <CardDescription>Track the status of your data imports</CardDescription>
</CardHeader>
<CardContent>
{isLoading && !data ? (
@@ -389,9 +347,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
) : error ? (
<Alert variant="destructive">
<AlertCircle className="h-4 w-4" />
- <AlertDescription>
- Failed to load import history. Please try refreshing the page.
- </AlertDescription>
+ <AlertDescription>Failed to load import history. Please try refreshing the page.</AlertDescription>
</Alert>
) : !data?.data?.length ? (
<div className="text-center py-8 text-muted-foreground">
@@ -412,7 +368,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
</TableRow>
</TableHeader>
<TableBody>
- {sortedImports.map((imp) => {
+ {sortedImports.map(imp => {
const statusInfo = getStatusInfo(imp.status);
const StatusIcon = statusInfo.icon;
@@ -430,14 +386,8 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
</TableCell>
<TableCell>
<div className="space-y-1">
- <Badge
- className={`${statusInfo.color} flex items-center gap-1 w-fit`}
- >
- <StatusIcon
- className={`h-3 w-3 ${
- imp.status === "processing" ? "animate-spin" : ""
- }`}
- />
+ <Badge className={`${statusInfo.color} flex items-center gap-1 w-fit`}>
+ <StatusIcon className={`h-3 w-3 ${imp.status === "processing" ? "animate-spin" : ""}`} />
{statusInfo.label}
</Badge>
{imp.errorMessage && (
@@ -447,9 +397,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
)}
</div>
</TableCell>
- <TableCell className="text-right">
- {imp.importedEvents.toLocaleString()}
- </TableCell>
+ <TableCell className="text-right">{imp.importedEvents.toLocaleString()}</TableCell>
<TableCell className="text-right">
{(imp.status === "completed" || imp.status === "failed") && (
<Button
@@ -483,15 +431,13 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<AlertDialogHeader>
<AlertDialogTitle>Confirm Large File Import</AlertDialogTitle>
<AlertDialogDescription>
- You're about to import a large file.
- This may take several minutes to process. Are you sure you want to continue?
+ You're about to import a large file. This may take several minutes to process. Are you sure you want to
+ continue?
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
- <AlertDialogAction onClick={handleImport}>
- Yes, Import File
- </AlertDialogAction>
+ <AlertDialogAction onClick={handleImport}>Yes, Import File</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
@@ -502,8 +448,8 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<AlertDialogHeader>
<AlertDialogTitle>Delete Import</AlertDialogTitle>
<AlertDialogDescription>
- Are you sure you want to delete this import? This action cannot be undone.
- The import data and associated files will be permanently removed.
+ Are you sure you want to delete this import? This action cannot be undone. The import data and associated
+ files will be permanently removed.
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
diff --git a/client/src/components/SiteSettings/SiteSettings.tsx b/client/src/components/SiteSettings/SiteSettings.tsx
index d2c35620b..511a72c5d 100644
--- a/client/src/components/SiteSettings/SiteSettings.tsx
+++ b/client/src/components/SiteSettings/SiteSettings.tsx
@@ -74,10 +74,7 @@ function SiteSettingsInner({ siteMetadata, trigger }: { siteMetadata: SiteRespon
<ApiKeyManager siteId={siteMetadata.siteId} disabled={disabled} />
</TabsContent>
- <TabsContent
- value="import"
- className="pt-4 space-y-4 max-h-[70vh] overflow-y-auto"
- >
+ <TabsContent value="import" className="pt-4 space-y-4 max-h-[70vh] overflow-y-auto">
<ImportManager siteId={siteMetadata.siteId} disabled={disabled} />
</TabsContent>
diff --git a/client/src/components/SplitDateRangePicker.tsx b/client/src/components/SplitDateRangePicker.tsx
index 8ea8a2cc5..f50537425 100644
--- a/client/src/components/SplitDateRangePicker.tsx
+++ b/client/src/components/SplitDateRangePicker.tsx
@@ -7,11 +7,7 @@ import { cn } from "@/lib/utils";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import { Calendar } from "@/components/ui/calendar";
-import {
- Popover,
- PopoverContent,
- PopoverTrigger,
-} from "@/components/ui/popover";
+import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
export interface DateRange {
startDate?: DateTime;
@@ -49,7 +45,7 @@ export function SplitDateRangePicker({
maxDate = DateTime.utc(),
minDate,
className,
- layout = "horizontal"
+ layout = "horizontal",
}: DateRangePickerProps) {
const [internalValue, setInternalValue] = useState<DateRange>({});
@@ -83,9 +79,7 @@ export function SplitDateRangePicker({
const hasDateSelected = !!(startDate || endDate);
- const gridClass = layout === "horizontal"
- ? "grid-cols-1 sm:grid-cols-2"
- : "grid-cols-1";
+ const gridClass = layout === "horizontal" ? "grid-cols-1 sm:grid-cols-2" : "grid-cols-1";
return (
<div className={cn("space-y-4", className)}>
@@ -103,10 +97,7 @@ export function SplitDateRangePicker({
<PopoverTrigger asChild>
<Button
variant="outline"
- className={cn(
- "w-full justify-start text-left font-normal",
- !startDate && "text-muted-foreground"
- )}
+ className={cn("w-full justify-start text-left font-normal", !startDate && "text-muted-foreground")}
disabled={disabled}
>
<CalendarIcon className="mr-2 h-4 w-4" />
@@ -118,7 +109,7 @@ export function SplitDateRangePicker({
mode="single"
selected={startDate?.toJSDate()}
onSelect={handleStartDateChange}
- disabled={(date) => {
+ disabled={date => {
const d = DateTime.fromJSDate(date, { zone: "utc" }).startOf("day");
if (maxDate && d > maxDate.startOf("day")) return true;
if (minDate && d < minDate.startOf("day")) return true;
@@ -137,10 +128,7 @@ export function SplitDateRangePicker({
<PopoverTrigger asChild>
<Button
variant="outline"
- className={cn(
- "w-full justify-start text-left font-normal",
- !endDate && "text-muted-foreground"
- )}
+ className={cn("w-full justify-start text-left font-normal", !endDate && "text-muted-foreground")}
disabled={disabled}
>
<CalendarIcon className="mr-2 h-4 w-4" />
@@ -152,7 +140,7 @@ export function SplitDateRangePicker({
mode="single"
selected={endDate?.toJSDate()}
onSelect={handleEndDateChange}
- disabled={(date) => {
+ disabled={date => {
const d = DateTime.fromJSDate(date, { zone: "utc" }).startOf("day");
if (maxDate && d > maxDate.startOf("day")) return true;
if (minDate && d < minDate.startOf("day")) return true;
@@ -168,13 +156,7 @@ export function SplitDateRangePicker({
{showClear && hasDateSelected && (
<div className="flex items-center gap-2">
- <Button
- variant="outline"
- size="sm"
- onClick={handleClear}
- className="shrink-0"
- disabled={disabled}
- >
+ <Button variant="outline" size="sm" onClick={handleClear} className="shrink-0" disabled={disabled}>
<X className="h-4 w-4 mr-1" />
{clearButtonText}
</Button>
diff --git a/server/src/api/sites/deleteSiteImport.ts b/server/src/api/sites/deleteSiteImport.ts
index d321aa170..f09a859b2 100644
--- a/server/src/api/sites/deleteSiteImport.ts
+++ b/server/src/api/sites/deleteSiteImport.ts
@@ -7,21 +7,20 @@ import { IS_CLOUD } from "../../lib/const.js";
import { r2Storage } from "../../services/storage/r2StorageService.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
-const deleteImportRequestSchema = z.object({
- params: z.object({
- site: z.string().min(1),
- importId: z.string().min(1),
- }),
-}).strict();
+const deleteImportRequestSchema = z
+ .object({
+ params: z.object({
+ site: z.string().min(1),
+ importId: z.string().min(1),
+ }),
+ })
+ .strict();
type DeleteImportRequest = {
Params: z.infer<typeof deleteImportRequestSchema.shape.params>;
};
-export async function deleteSiteImport(
- request: FastifyRequest<DeleteImportRequest>,
- reply: FastifyReply,
-) {
+export async function deleteSiteImport(request: FastifyRequest<DeleteImportRequest>, reply: FastifyReply) {
try {
const parsed = deleteImportRequestSchema.safeParse({
params: request.params,
@@ -80,7 +79,7 @@ export async function deleteSiteImport(
} catch (chError) {
console.error(`Failed to delete ClickHouse events for ${importId}:`, chError);
return reply.status(500).send({
- error: "Failed to delete imported events from database"
+ error: "Failed to delete imported events from database",
});
}
@@ -89,8 +88,8 @@ export async function deleteSiteImport(
return reply.send({
data: {
- message: "Import deleted successfully"
- }
+ message: "Import deleted successfully",
+ },
});
} catch (error) {
console.error("Error deleting import:", error);
diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.ts
index c2355acea..4c3319582 100644
--- a/server/src/api/sites/getSiteImports.ts
+++ b/server/src/api/sites/getSiteImports.ts
@@ -3,20 +3,19 @@ import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { ImportStatusManager } from "../../services/import/importStatusManager.js";
import { z } from "zod";
-const getSiteImportsRequestSchema = z.object({
- params: z.object({
- site: z.string().min(1),
- }),
-}).strict();
+const getSiteImportsRequestSchema = z
+ .object({
+ params: z.object({
+ site: z.string().min(1),
+ }),
+ })
+ .strict();
type GetSiteImportsRequest = {
Params: z.infer<typeof getSiteImportsRequestSchema.shape.params>;
};
-export async function getSiteImports(
- request: FastifyRequest<GetSiteImportsRequest>,
- reply: FastifyReply,
-) {
+export async function getSiteImports(request: FastifyRequest<GetSiteImportsRequest>, reply: FastifyReply) {
try {
const parsed = getSiteImportsRequestSchema.safeParse({
params: request.params,
@@ -44,7 +43,7 @@ export async function getSiteImports(
errorMessage: imp.errorMessage,
startedAt: imp.startedAt,
fileName: imp.fileName,
- }))
+ })),
});
} catch (error) {
console.error("Error fetching imports:", error);
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
index 1c4e355ae..07ca13e5f 100644
--- a/server/src/api/sites/importSiteData.ts
+++ b/server/src/api/sites/importSiteData.ts
@@ -22,42 +22,46 @@ const isValidDate = (date: string) => {
const parseDate = (date: string) => DateTime.fromFormat(date, "yyyy-MM-dd", { zone: "utc" });
-const importDataFieldsSchema = z.object({
- fields: z.object({
- source: z.enum(["umami"]),
- startDate: z.string().refine(isValidDate).optional(),
- endDate: z.string().refine(isValidDate).optional(),
- }).refine((fields) => {
- if (fields.startDate && fields.endDate) {
- const start = parseDate(fields.startDate);
- const end = parseDate(fields.endDate);
- return start <= end;
- }
- return true;
- }).refine((fields) => {
- if (fields.startDate) {
- const today = DateTime.utc().startOf("day");
- const start = parseDate(fields.startDate);
- return start <= today;
- }
- return true;
- }),
-}).strict();
-
-const importDataRequestSchema = z.object({
- params: z.object({
- site: z.string().min(1),
- }),
-}).strict();
+const importDataFieldsSchema = z
+ .object({
+ fields: z
+ .object({
+ source: z.enum(["umami"]),
+ startDate: z.string().refine(isValidDate).optional(),
+ endDate: z.string().refine(isValidDate).optional(),
+ })
+ .refine(fields => {
+ if (fields.startDate && fields.endDate) {
+ const start = parseDate(fields.startDate);
+ const end = parseDate(fields.endDate);
+ return start <= end;
+ }
+ return true;
+ })
+ .refine(fields => {
+ if (fields.startDate) {
+ const today = DateTime.utc().startOf("day");
+ const start = parseDate(fields.startDate);
+ return start <= today;
+ }
+ return true;
+ }),
+ })
+ .strict();
+
+const importDataRequestSchema = z
+ .object({
+ params: z.object({
+ site: z.string().min(1),
+ }),
+ })
+ .strict();
type ImportDataRequest = {
Params: z.infer<typeof importDataRequestSchema.shape.params>;
};
-export async function importSiteData(
- request: FastifyRequest<ImportDataRequest>,
- reply: FastifyReply,
-) {
+export async function importSiteData(request: FastifyRequest<ImportDataRequest>, reply: FastifyReply) {
try {
const parsedParams = importDataRequestSchema.safeParse({
params: request.params,
@@ -162,7 +166,7 @@ export async function importSiteData(
return reply.status(202).send({
data: {
message: "File upload accepted and is now being processed.",
- }
+ },
});
} catch (error) {
console.error("Unexpected error during import:", error);
diff --git a/server/src/db/postgres/boss.ts b/server/src/db/postgres/boss.ts
index b4d930fcc..df864fe7e 100644
--- a/server/src/db/postgres/boss.ts
+++ b/server/src/db/postgres/boss.ts
@@ -4,14 +4,14 @@ import dotenv from "dotenv";
dotenv.config();
const boss = new PgBoss({
- host: process.env.POSTGRES_HOST || "postgres",
- port: parseInt(process.env.POSTGRES_PORT || "5432", 10),
- database: process.env.POSTGRES_DB,
- user: process.env.POSTGRES_USER,
- password: process.env.POSTGRES_PASSWORD,
- schema: "pgboss",
- application_name: "data-import-system",
- retryLimit: 0,
+ host: process.env.POSTGRES_HOST || "postgres",
+ port: parseInt(process.env.POSTGRES_PORT || "5432", 10),
+ database: process.env.POSTGRES_DB,
+ user: process.env.POSTGRES_USER,
+ password: process.env.POSTGRES_PASSWORD,
+ schema: "pgboss",
+ application_name: "data-import-system",
+ retryLimit: 0,
});
boss.on("error", error => console.error(error));
diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.ts
index 8a5ca94e2..6ed182f13 100644
--- a/server/src/db/postgres/schema.ts
+++ b/server/src/db/postgres/schema.ts
@@ -66,7 +66,9 @@ export const sites = pgTable(
createdBy: text("created_by")
.notNull()
.references(() => user.id),
- organizationId: text("organization_id").notNull().references(() => organization.id),
+ organizationId: text("organization_id")
+ .notNull()
+ .references(() => organization.id),
public: boolean().default(false),
saltUserIds: boolean().default(false),
blockBots: boolean().default(true).notNull(),
@@ -598,16 +600,9 @@ export const notificationChannels = pgTable(
]
);
-export const importSourceEnum = pgEnum("import_source_enum", [
- "umami",
-]);
+export const importSourceEnum = pgEnum("import_source_enum", ["umami"]);
-export const importStatusEnum = pgEnum("import_status_enum", [
- "pending",
- "processing",
- "completed",
- "failed",
-]);
+export const importStatusEnum = pgEnum("import_status_enum", ["pending", "processing", "completed", "failed"]);
// Import status table for tracking data import progress
export const importStatus = pgTable(
@@ -624,7 +619,7 @@ export const importStatus = pgTable(
completedAt: timestamp("completed_at", { mode: "string" }),
fileName: text("file_name").notNull(),
},
- (table) => [
+ table => [
foreignKey({
columns: [table.siteId],
foreignColumns: [sites.siteId],
diff --git a/server/src/index.ts b/server/src/index.ts
index 9bff26913..cdee3952e 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -166,7 +166,7 @@ server.register(multipart, {
limits: {
fileSize: IS_CLOUD ? 100 * 1024 * 1024 : undefined, // 100 MB max file size, infinite for self-hosted
files: 1, // only allow 1 file
- }
+ },
});
// Serve static files
diff --git a/server/src/queues/adapters/pgBossAdapter.ts b/server/src/queues/adapters/pgBossAdapter.ts
index 3b6efdbc8..17c4c3450 100644
--- a/server/src/queues/adapters/pgBossAdapter.ts
+++ b/server/src/queues/adapters/pgBossAdapter.ts
@@ -16,7 +16,7 @@ export class PgBossAdapter implements IJobQueue {
retryLimit: 0,
});
- this.boss.on("error", (error) => {
+ this.boss.on("error", error => {
console.error("[PgBoss] Error:", error);
});
}
@@ -35,11 +35,7 @@ export class PgBossAdapter implements IJobQueue {
await this.boss.createQueue(queueName);
}
- async send<T = any>(
- queueName: string,
- data: T,
- options?: { priority?: number; delay?: number }
- ): Promise<string> {
+ async send<T = any>(queueName: string, data: T, options?: { priority?: number; delay?: number }): Promise<string> {
const jobId = await this.boss.send(queueName, data as object, {
priority: options?.priority,
startAfter: options?.delay ? new Date(Date.now() + options.delay) : undefined,
@@ -64,7 +60,7 @@ export class PgBossAdapter implements IJobQueue {
pollingIntervalSeconds: config.pollingIntervalSeconds ?? 2,
},
async (jobs: Job<T>[]) => {
- const normalizedJobs: JobData<T>[] = jobs.map((job) => ({
+ const normalizedJobs: JobData<T>[] = jobs.map(job => ({
id: job.id,
data: job.data,
}));
diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.ts
index 812a04847..49e09c09c 100644
--- a/server/src/services/import/importLimiter.ts
+++ b/server/src/services/import/importLimiter.ts
@@ -7,13 +7,16 @@ export class ImportLimiter {
private static readonly IMPORTED_EVENTS_LIMIT = 1_000_000;
private static readonly CONCURRENT_IMPORT_LIMIT = 1;
- static async checkConcurrentImportLimit(siteId: number): Promise<{
- allowed: false;
- reason: string;
- } | {
- allowed: true;
- organizationId: string
- }> {
+ static async checkConcurrentImportLimit(siteId: number): Promise<
+ | {
+ allowed: false;
+ reason: string;
+ }
+ | {
+ allowed: true;
+ organizationId: string;
+ }
+ > {
const [siteResult] = await db
.select({ organizationId: sites.organizationId })
.from(sites)
diff --git a/server/src/services/import/importStatusManager.ts b/server/src/services/import/importStatusManager.ts
index cd4063095..959efdf08 100644
--- a/server/src/services/import/importStatusManager.ts
+++ b/server/src/services/import/importStatusManager.ts
@@ -34,10 +34,7 @@ export class ImportStatusManager {
}
}
- static async updateProgress(
- importId: string,
- importedEvents: number,
- ): Promise<void> {
+ static async updateProgress(importId: string, importedEvents: number): Promise<void> {
try {
await db
.update(importStatus)
diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.ts
index 43b25a987..9cc51a232 100644
--- a/server/src/services/import/mappings/umami.ts
+++ b/server/src/services/import/mappings/umami.ts
@@ -104,26 +104,32 @@ export class UmamiImportMapper {
edge: "Edge",
};
- private static readonly browserSchema = z.string().max(30).transform((browser) => {
- const key = browser.toLowerCase();
- return UmamiImportMapper.browserMap[key] ?? browser;
- });
+ private static readonly browserSchema = z
+ .string()
+ .max(30)
+ .transform(browser => {
+ const key = browser.toLowerCase();
+ return UmamiImportMapper.browserMap[key] ?? browser;
+ });
private static readonly osMap: Record<string, string> = {
"windows 10": "Windows",
"windows 7": "Windows",
"windows server 2003": "Windows",
"mac os": "macOS",
- "ios": "iOS",
+ ios: "iOS",
"android os": "Android",
- "linux": "Linux",
+ linux: "Linux",
"chrome os": "Chrome OS",
};
- private static readonly osSchema = z.string().max(25).transform((os) => {
- const key = os.toLowerCase();
- return UmamiImportMapper.osMap[key] ?? os;
- });
+ private static readonly osSchema = z
+ .string()
+ .max(25)
+ .transform(os => {
+ const key = os.toLowerCase();
+ return UmamiImportMapper.osMap[key] ?? os;
+ });
private static readonly deviceMap: Record<string, "Desktop" | "Mobile"> = {
laptop: "Desktop",
@@ -132,10 +138,13 @@ export class UmamiImportMapper {
tablet: "Mobile",
};
- private static readonly deviceSchema = z.string().max(20).transform((device) => {
- const key = device.toLowerCase();
- return UmamiImportMapper.deviceMap[key] ?? device;
- });
+ private static readonly deviceSchema = z
+ .string()
+ .max(20)
+ .transform(device => {
+ const key = device.toLowerCase();
+ return UmamiImportMapper.deviceMap[key] ?? device;
+ });
private static readonly umamiEventSchema = z.object({
session_id: z.string().uuid(),
@@ -144,17 +153,35 @@ export class UmamiImportMapper {
browser: UmamiImportMapper.browserSchema,
os: UmamiImportMapper.osSchema,
device: UmamiImportMapper.deviceSchema,
- screen: z.string().regex(/^\d{1,5}x\d{1,5}$/).or(z.literal("")),
+ screen: z
+ .string()
+ .regex(/^\d{1,5}x\d{1,5}$/)
+ .or(z.literal("")),
language: z.string().max(35),
- country: z.string().regex(/^[A-Z]{2}$/).or(z.literal("")),
- region: z.string().regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/).or(z.literal("")),
+ country: z
+ .string()
+ .regex(/^[A-Z]{2}$/)
+ .or(z.literal("")),
+ region: z
+ .string()
+ .regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/)
+ .or(z.literal("")),
city: z.string().max(60),
url_path: z.string().max(2048),
- url_query: z.string().max(2048).transform((querystring) => querystring ? `?${querystring}` : ""),
+ url_query: z
+ .string()
+ .max(2048)
+ .transform(querystring => (querystring ? `?${querystring}` : "")),
referrer_path: z.string().max(2048),
- referrer_query: z.string().max(2048).transform((querystring) => querystring ? `?${querystring}` : ""),
- referrer_domain: z.string().max(253).transform((url) => url ? `https://${url}` : ""),
+ referrer_query: z
+ .string()
+ .max(2048)
+ .transform(querystring => (querystring ? `?${querystring}` : "")),
+ referrer_domain: z
+ .string()
+ .max(253)
+ .transform(url => (url ? `https://${url}` : "")),
page_title: z.string().max(512),
event_type: z.enum(["1", "2"]),
diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.ts
index adf6af32a..311957874 100644
--- a/server/src/services/import/utils.ts
+++ b/server/src/services/import/utils.ts
@@ -10,10 +10,7 @@ export interface DeleteFileResult {
* Delete an import file from storage.
* Returns result instead of throwing to prevent worker crashes.
*/
-export const deleteImportFile = async (
- storageLocation: string,
- isR2Storage: boolean
-): Promise<DeleteFileResult> => {
+export const deleteImportFile = async (storageLocation: string, isR2Storage: boolean): Promise<DeleteFileResult> => {
try {
if (isR2Storage) {
await r2Storage.deleteImportFile(storageLocation);
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 5819d6aab..89c2dc35a 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -22,21 +22,25 @@ const getImportDataHeaders = (source: string) => {
const createR2FileStream = async (storageLocation: string, source: string) => {
console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);
const fileStream = await r2Storage.getImportFileStream(storageLocation);
- return fileStream.pipe(parse({
- headers: getImportDataHeaders(source),
- renameHeaders: true,
- ignoreEmpty: true,
- }));
+ return fileStream.pipe(
+ parse({
+ headers: getImportDataHeaders(source),
+ renameHeaders: true,
+ ignoreEmpty: true,
+ })
+ );
};
const createLocalFileStream = async (storageLocation: string, source: string) => {
console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);
await access(storageLocation, constants.F_OK | constants.R_OK);
- return createReadStream(storageLocation).pipe(parse({
- headers: getImportDataHeaders(source),
- renameHeaders: true,
- ignoreEmpty: true,
- }));
+ return createReadStream(storageLocation).pipe(
+ parse({
+ headers: getImportDataHeaders(source),
+ renameHeaders: true,
+ ignoreEmpty: true,
+ })
+ );
};
/**
@@ -49,9 +53,7 @@ const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {
? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")
: null;
- const endDate = endDateStr
- ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day")
- : null;
+ const endDate = endDateStr ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day") : null;
// Validate parsed dates
if (startDate && !startDate.isValid) {
@@ -83,11 +85,8 @@ const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {
export async function registerCsvParseWorker() {
const jobQueue = getJobQueue();
- await jobQueue.work<CsvParseJob>(
- CSV_PARSE_QUEUE,
- { batchSize: 1, pollingIntervalSeconds: 10 },
- async ([job]) => {
- const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
+ await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([job]) => {
+ const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
try {
const importableEvents = await ImportLimiter.countImportableEvents(organization);
@@ -162,25 +161,22 @@ export async function registerCsvParseWorker() {
totalChunks: chunksSent,
allChunksSent: true,
});
- } catch (error) {
- console.error("Error in CSV parse worker:", error);
- await ImportStatusManager.updateStatus(
- importId,
- "failed",
- error instanceof Error ? error.message : "Unknown error occurred"
- );
- throw error;
- } finally {
- // Clean up file - don't throw on failure to prevent worker crashes
- const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
- if (!deleteResult.success) {
- console.warn(
- `[Import ${importId}] File cleanup failed, will remain in storage: ${deleteResult.error}`
- );
- // File will be orphaned but import status is already recorded
- // Could implement a cleanup job here to retry later
- }
+ } catch (error) {
+ console.error("Error in CSV parse worker:", error);
+ await ImportStatusManager.updateStatus(
+ importId,
+ "failed",
+ error instanceof Error ? error.message : "Unknown error occurred"
+ );
+ throw error;
+ } finally {
+ // Clean up file - don't throw on failure to prevent worker crashes
+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
+ if (!deleteResult.success) {
+ console.warn(`[Import ${importId}] File cleanup failed, will remain in storage: ${deleteResult.error}`);
+ // File will be orphaned but import status is already recorded
+ // Could implement a cleanup job here to retry later
}
}
- );
+ });
}
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index dcd601c42..3ec14d2ca 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -16,66 +16,54 @@ const getImportDataMapping = (source: string) => {
export async function registerDataInsertWorker() {
const jobQueue = getJobQueue();
- await jobQueue.work<DataInsertJob>(
- DATA_INSERT_QUEUE,
- { batchSize: 1, pollingIntervalSeconds: 2 },
- async ([job]) => {
- const { site, importId, source, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;
+ await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([job]) => {
+ const { site, importId, source, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;
- // Handle finalization signal
- if (allChunksSent) {
- try {
- await ImportStatusManager.updateStatus(importId, "completed");
- console.log(
- `[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`
- );
- return;
- } catch (error) {
- console.error(`[Import ${importId}] Failed to mark as completed:`, error);
- await ImportStatusManager.updateStatus(
- importId,
- "failed",
- "Failed to complete import"
- );
- throw error;
- }
- }
-
- // Process data chunk
+ // Handle finalization signal
+ if (allChunksSent) {
try {
- const dataMapper = getImportDataMapping(source);
- const transformedRecords = dataMapper.transform(chunk, site, importId);
+ await ImportStatusManager.updateStatus(importId, "completed");
+ console.log(`[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`);
+ return;
+ } catch (error) {
+ console.error(`[Import ${importId}] Failed to mark as completed:`, error);
+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");
+ throw error;
+ }
+ }
- // Insert to ClickHouse (critical - must succeed)
- await clickhouse.insert({
- table: "events",
- values: transformedRecords,
- format: "JSONEachRow",
- });
+ // Process data chunk
+ try {
+ const dataMapper = getImportDataMapping(source);
+ const transformedRecords = dataMapper.transform(chunk, site, importId);
- // Update progress (non-critical - log if fails but don't crash)
- try {
- await ImportStatusManager.updateProgress(importId, transformedRecords.length);
- } catch (progressError) {
- console.warn(
- `[Import ${importId}] Progress update failed (data inserted successfully):`,
- progressError instanceof Error ? progressError.message : progressError
- );
- // Don't throw - data is safely in ClickHouse, progress can be off slightly
- }
+ // Insert to ClickHouse (critical - must succeed)
+ await clickhouse.insert({
+ table: "events",
+ values: transformedRecords,
+ format: "JSONEachRow",
+ });
- console.log(
- `[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`
- );
- } catch (error) {
- console.error(`[Import ${importId}] ClickHouse insert failed:`, error);
- await ImportStatusManager.updateStatus(
- importId,
- "failed",
- `Data insertion failed: ${error instanceof Error ? error.message : "Unknown error"}`
+ // Update progress (non-critical - log if fails but don't crash)
+ try {
+ await ImportStatusManager.updateProgress(importId, transformedRecords.length);
+ } catch (progressError) {
+ console.warn(
+ `[Import ${importId}] Progress update failed (data inserted successfully):`,
+ progressError instanceof Error ? progressError.message : progressError
);
- throw error;
+ // Don't throw - data is safely in ClickHouse, progress can be off slightly
}
+
+ console.log(`[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`);
+ } catch (error) {
+ console.error(`[Import ${importId}] ClickHouse insert failed:`, error);
+ await ImportStatusManager.updateStatus(
+ importId,
+ "failed",
+ `Data insertion failed: ${error instanceof Error ? error.message : "Unknown error"}`
+ );
+ throw error;
}
- );
+ });
}
diff --git a/server/src/services/import/workers/queues.ts b/server/src/services/import/workers/queues.ts
index a574e1fd5..b6f60d25f 100644
--- a/server/src/services/import/workers/queues.ts
+++ b/server/src/services/import/workers/queues.ts
@@ -10,4 +10,4 @@ export const createJobQueues = async () => {
} catch (error) {
throw new Error(`Failed to create job queues: ${error instanceof Error ? error.message : "Unknown error"}`);
}
-}
+};
diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.ts
index 798943413..9f268bb4f 100644
--- a/server/src/services/storage/r2StorageService.ts
+++ b/server/src/services/storage/r2StorageService.ts
@@ -1,9 +1,4 @@
-import {
- S3Client,
- PutObjectCommand,
- GetObjectCommand,
- DeleteObjectCommand,
-} from "@aws-sdk/client-s3";
+import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { NodeHttpHandler } from "@smithy/node-http-handler";
import { Readable } from "stream";
@@ -245,7 +240,7 @@ class R2StorageService {
new GetObjectCommand({
Bucket: this.bucketName,
Key: key,
- }),
+ })
);
if (!response.Body) {
@@ -273,7 +268,7 @@ class R2StorageService {
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: key,
- }),
+ })
);
this.logger.info({ key }, "Successfully deleted import file");
} catch (error) {