🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
prettier
Edward Cuia4dc031f2ca9
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.tsindex 00e62818c..dd6717e0d 100644--- a/client/src/api/admin/import.ts+++ b/client/src/api/admin/import.ts@@ -31,9 +31,9 @@ export function useGetSiteImports(site: number) {return useQuery({queryKey: ["get-site-imports", site],queryFn: async () => await authedFetch<APIResponse<GetSiteImportsResponse[]>>(`/get-site-imports/${site}`),- refetchInterval: (data) => {- const hasActiveImports = data.state.data?.data.some(imp =>- imp.status === "processing" || imp.status === "pending"+ refetchInterval: data => {+ const hasActiveImports = data.state.data?.data.some(+ imp => imp.status === "processing" || imp.status === "pending");return hasActiveImports ? 5000 : false;},@@ -65,7 +65,7 @@ export function useImportSiteData(site: number) {},onSuccess: async () => {await queryClient.invalidateQueries({- queryKey: ["get-site-imports", site]+ queryKey: ["get-site-imports", site],});},retry: false,@@ -77,13 +77,17 @@ export function useDeleteSiteImport(site: number) {return useMutation({mutationFn: async (importId: string) => {- return await authedFetch<APIResponse<DeleteImportResponse>>(`/delete-site-import/${site}/${importId}`, undefined, {- method: "DELETE",- });+ return await authedFetch<APIResponse<DeleteImportResponse>>(+ `/delete-site-import/${site}/${importId}`,+ undefined,+ {+ method: "DELETE",+ }+ );},onSuccess: async () => {await queryClient.invalidateQueries({- queryKey: ["get-site-imports", site]+ queryKey: ["get-site-imports", site],});},retry: false,diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex 854d9fffd..789b276d0 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -4,21 +4,8 @@ import { useState, useMemo, useRef } from "react";import { Button } from "@/components/ui/button";import { Input } from "@/components/ui/input";import { Label } from "@/components/ui/label";-import {- Select,- SelectContent,- SelectItem,- SelectTrigger,- SelectValue,-} from "@/components/ui/select";-import {- Table,- TableBody,- TableCell,- TableHead,- TableHeader,- TableRow,-} from "@/components/ui/table";+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";+import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";import {AlertDialog,AlertDialogAction,@@ -33,16 +20,7 @@ import { Alert, AlertDescription } from "@/components/ui/alert";import { Badge } from "@/components/ui/badge";import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";import { Separator } from "@/components/ui/separator";-import {- Upload,- FileText,- AlertCircle,- CheckCircle2,- Clock,- Loader2,- Database,- Trash2,-} from "lucide-react";+import { Upload, FileText, AlertCircle, CheckCircle2, Clock, Loader2, Database, Trash2 } from "lucide-react";import { useGetSiteImports, useImportSiteData, useDeleteSiteImport } from "@/api/admin/import";import { SplitDateRangePicker, DateRange } from "@/components/SplitDateRangePicker";@@ -59,9 +37,7 @@ interface FileValidationError {const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MBconst ALLOWED_FILE_TYPES = ["text/csv"];const ALLOWED_EXTENSIONS = [".csv"];-const DATA_SOURCES = [- { value: "umami", label: "Umami" },-];+const DATA_SOURCES = [{ value: "umami", label: "Umami" }];export function ImportManager({ siteId, disabled }: ImportManagerProps) {const [file, setFile] = useState<File | null>(null);@@ -89,14 +65,14 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {if (file.size > MAX_FILE_SIZE) {return {type: "size",- message: `File size (${(file.size / 1024 / 1024).toFixed(1)}MB) exceeds the maximum limit of 100MB.`+ message: `File size (${(file.size / 1024 / 1024).toFixed(1)}MB) exceeds the maximum limit of 100MB.`,};}if (!ALLOWED_FILE_TYPES.includes(file.type) && file.type !== "") {return {type: "type",- message: "Invalid file type. Only CSV files are accepted."+ message: "Invalid file type. Only CSV files are accepted.",};}@@ -104,14 +80,14 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {if (!ALLOWED_EXTENSIONS.includes(extension)) {return {type: "type",- message: "Invalid file extension. Only .csv files are accepted."+ message: "Invalid file extension. Only .csv files are accepted.",};}if (file.name.length > 255) {return {type: "name",- message: "Filename is too long. Please use a shorter filename."+ message: "Filename is too long. Please use a shorter filename.",};}@@ -182,31 +158,31 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {return {color: "bg-green-100 text-green-800 border-green-200",icon: CheckCircle2,- label: "Completed"+ label: "Completed",};case "failed":return {color: "bg-red-100 text-red-800 border-red-200",icon: AlertCircle,- label: "Failed"+ label: "Failed",};case "processing":return {color: "bg-blue-100 text-blue-800 border-blue-200",icon: Loader2,- label: "Processing"+ label: "Processing",};case "pending":return {color: "bg-yellow-100 text-yellow-800 border-yellow-200",icon: Clock,- label: "Pending"+ label: "Pending",};default:return {color: "bg-gray-100 text-gray-800 border-gray-200",icon: Clock,- label: status+ label: status,};}};@@ -234,9 +210,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Upload className="h-5 w-5" />Import Data</CardTitle>- <CardDescription>- Import data from other analytics platforms. Supports CSV files up to 100MB.- </CardDescription>+ <CardDescription>Import data from other analytics platforms. Supports CSV files up to 100MB.</CardDescription></CardHeader><CardContent className="space-y-4">{/* Data Source Selection */}@@ -245,16 +219,12 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Database className="h-4 w-4" />Data Source</Label>- <Select- value={source}- onValueChange={setSource}- disabled={disabled || mutation.isPending}- >+ <Select value={source} onValueChange={setSource} disabled={disabled || mutation.isPending}><SelectTrigger><SelectValue placeholder="Select data source" /></SelectTrigger><SelectContent>- {DATA_SOURCES.map((dataSource) => (+ {DATA_SOURCES.map(dataSource => (<SelectItem key={dataSource.value} value={dataSource.value}>{dataSource.label}</SelectItem>@@ -296,19 +266,13 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Alert variant="destructive"><div className="flex items-center gap-2"><AlertCircle className="h-4 w-4" />- <AlertDescription>- {fileError.message}- </AlertDescription>+ <AlertDescription>{fileError.message}</AlertDescription></div></Alert>)}{/* Import Button */}- <Button- onClick={handleImportClick}- disabled={isImportDisabled}- className="w-full sm:w-auto"- >+ <Button onClick={handleImportClick} disabled={isImportDisabled} className="w-full sm:w-auto">{mutation.isPending ? (<><Loader2 className="mr-2 h-4 w-4 animate-spin" />@@ -339,9 +303,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Alert className="border-green-200 bg-green-50"><div className="flex items-center gap-2"><CheckCircle2 className="h-4 w-4 text-green-600" />- <AlertDescription>- File uploaded successfully and is being processed.- </AlertDescription>+ <AlertDescription>File uploaded successfully and is being processed.</AlertDescription></div></Alert>)}@@ -351,9 +313,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Alert className="border-green-200 bg-green-50"><div className="flex items-center gap-2"><CheckCircle2 className="h-4 w-4 text-green-600" />- <AlertDescription>- Import deleted successfully.- </AlertDescription>+ <AlertDescription>Import deleted successfully.</AlertDescription></div></Alert>)}@@ -376,9 +336,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Card><CardHeader><CardTitle>Import History</CardTitle>- <CardDescription>- Track the status of your data imports- </CardDescription>+ <CardDescription>Track the status of your data imports</CardDescription></CardHeader><CardContent>{isLoading && !data ? (@@ -389,9 +347,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {) : error ? (<Alert variant="destructive"><AlertCircle className="h-4 w-4" />- <AlertDescription>- Failed to load import history. Please try refreshing the page.- </AlertDescription>+ <AlertDescription>Failed to load import history. Please try refreshing the page.</AlertDescription></Alert>) : !data?.data?.length ? (<div className="text-center py-8 text-muted-foreground">@@ -412,7 +368,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</TableRow></TableHeader><TableBody>- {sortedImports.map((imp) => {+ {sortedImports.map(imp => {const statusInfo = getStatusInfo(imp.status);const StatusIcon = statusInfo.icon;@@ -430,14 +386,8 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</TableCell><TableCell><div className="space-y-1">- <Badge- className={`${statusInfo.color} flex items-center gap-1 w-fit`}- >- <StatusIcon- className={`h-3 w-3 ${- imp.status === "processing" ? "animate-spin" : ""- }`}- />+ <Badge className={`${statusInfo.color} flex items-center gap-1 w-fit`}>+ <StatusIcon className={`h-3 w-3 ${imp.status === "processing" ? "animate-spin" : ""}`} />{statusInfo.label}</Badge>{imp.errorMessage && (@@ -447,9 +397,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {)}</div></TableCell>- <TableCell className="text-right">- {imp.importedEvents.toLocaleString()}- </TableCell>+ <TableCell className="text-right">{imp.importedEvents.toLocaleString()}</TableCell><TableCell className="text-right">{(imp.status === "completed" || imp.status === "failed") && (<Button@@ -483,15 +431,13 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<AlertDialogHeader><AlertDialogTitle>Confirm Large File Import</AlertDialogTitle><AlertDialogDescription>- You're about to import a large file.- This may take several minutes to process. Are you sure you want to continue?+ You're about to import a large file. This may take several minutes to process. Are you sure you want to+ continue?</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Cancel</AlertDialogCancel>- <AlertDialogAction onClick={handleImport}>- Yes, Import File- </AlertDialogAction>+ <AlertDialogAction onClick={handleImport}>Yes, Import File</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>@@ -502,8 +448,8 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<AlertDialogHeader><AlertDialogTitle>Delete Import</AlertDialogTitle><AlertDialogDescription>- Are you sure you want to delete this import? This action cannot be undone.- The import data and associated files will be permanently removed.+ Are you sure you want to delete this import? This action cannot be undone. The import data and associated+ files will be permanently removed.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter>diff --git a/client/src/components/SiteSettings/SiteSettings.tsx b/client/src/components/SiteSettings/SiteSettings.tsxindex d2c35620b..511a72c5d 100644--- a/client/src/components/SiteSettings/SiteSettings.tsx+++ b/client/src/components/SiteSettings/SiteSettings.tsx@@ -74,10 +74,7 @@ function SiteSettingsInner({ siteMetadata, trigger }: { siteMetadata: SiteRespon<ApiKeyManager siteId={siteMetadata.siteId} disabled={disabled} /></TabsContent>- <TabsContent- value="import"- className="pt-4 space-y-4 max-h-[70vh] overflow-y-auto"- >+ <TabsContent value="import" className="pt-4 space-y-4 max-h-[70vh] overflow-y-auto"><ImportManager siteId={siteMetadata.siteId} disabled={disabled} /></TabsContent>diff --git a/client/src/components/SplitDateRangePicker.tsx b/client/src/components/SplitDateRangePicker.tsxindex 8ea8a2cc5..f50537425 100644--- a/client/src/components/SplitDateRangePicker.tsx+++ b/client/src/components/SplitDateRangePicker.tsx@@ -7,11 +7,7 @@ import { cn } from "@/lib/utils";import { Button } from "@/components/ui/button";import { Label } from "@/components/ui/label";import { Calendar } from "@/components/ui/calendar";-import {- Popover,- PopoverContent,- PopoverTrigger,-} from "@/components/ui/popover";+import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";export interface DateRange {startDate?: DateTime;@@ -49,7 +45,7 @@ export function SplitDateRangePicker({maxDate = DateTime.utc(),minDate,className,- layout = "horizontal"+ layout = "horizontal",}: DateRangePickerProps) {const [internalValue, setInternalValue] = useState<DateRange>({});@@ -83,9 +79,7 @@ export function SplitDateRangePicker({const hasDateSelected = !!(startDate || endDate);- const gridClass = layout === "horizontal"- ? "grid-cols-1 sm:grid-cols-2"- : "grid-cols-1";+ const gridClass = layout === "horizontal" ? "grid-cols-1 sm:grid-cols-2" : "grid-cols-1";return (<div className={cn("space-y-4", className)}>@@ -103,10 +97,7 @@ export function SplitDateRangePicker({<PopoverTrigger asChild><Buttonvariant="outline"- className={cn(- "w-full justify-start text-left font-normal",- !startDate && "text-muted-foreground"- )}+ className={cn("w-full justify-start text-left font-normal", !startDate && "text-muted-foreground")}disabled={disabled}><CalendarIcon className="mr-2 h-4 w-4" />@@ -118,7 +109,7 @@ export function SplitDateRangePicker({mode="single"selected={startDate?.toJSDate()}onSelect={handleStartDateChange}- disabled={(date) => {+ disabled={date => {const d = DateTime.fromJSDate(date, { zone: "utc" }).startOf("day");if (maxDate && d > maxDate.startOf("day")) return true;if (minDate && d < minDate.startOf("day")) return true;@@ -137,10 +128,7 @@ export function SplitDateRangePicker({<PopoverTrigger asChild><Buttonvariant="outline"- className={cn(- "w-full justify-start text-left font-normal",- !endDate && "text-muted-foreground"- )}+ className={cn("w-full justify-start text-left font-normal", !endDate && "text-muted-foreground")}disabled={disabled}><CalendarIcon className="mr-2 h-4 w-4" />@@ -152,7 +140,7 @@ export function SplitDateRangePicker({mode="single"selected={endDate?.toJSDate()}onSelect={handleEndDateChange}- disabled={(date) => {+ disabled={date => {const d = DateTime.fromJSDate(date, { zone: "utc" }).startOf("day");if (maxDate && d > maxDate.startOf("day")) return true;if (minDate && d < minDate.startOf("day")) return true;@@ -168,13 +156,7 @@ export function SplitDateRangePicker({{showClear && hasDateSelected && (<div className="flex items-center gap-2">- <Button- variant="outline"- size="sm"- onClick={handleClear}- className="shrink-0"- disabled={disabled}- >+ <Button variant="outline" size="sm" onClick={handleClear} className="shrink-0" disabled={disabled}><X className="h-4 w-4 mr-1" />{clearButtonText}</Button>diff --git a/server/src/api/sites/deleteSiteImport.ts b/server/src/api/sites/deleteSiteImport.tsindex d321aa170..f09a859b2 100644--- a/server/src/api/sites/deleteSiteImport.ts+++ b/server/src/api/sites/deleteSiteImport.ts@@ -7,21 +7,20 @@ import { IS_CLOUD } from "../../lib/const.js";import { r2Storage } from "../../services/storage/r2StorageService.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";-const deleteImportRequestSchema = z.object({- params: z.object({- site: z.string().min(1),- importId: z.string().min(1),- }),-}).strict();+const deleteImportRequestSchema = z+ .object({+ params: z.object({+ site: z.string().min(1),+ importId: z.string().min(1),+ }),+ })+ .strict();type DeleteImportRequest = {Params: z.infer<typeof deleteImportRequestSchema.shape.params>;};-export async function deleteSiteImport(- request: FastifyRequest<DeleteImportRequest>,- reply: FastifyReply,-) {+export async function deleteSiteImport(request: FastifyRequest<DeleteImportRequest>, reply: FastifyReply) {try {const parsed = deleteImportRequestSchema.safeParse({params: request.params,@@ -80,7 +79,7 @@ export async function deleteSiteImport(} catch (chError) {console.error(`Failed to delete ClickHouse events for ${importId}:`, chError);return reply.status(500).send({- error: "Failed to delete imported events from database"+ error: "Failed to delete imported events from database",});}@@ -89,8 +88,8 @@ export async function deleteSiteImport(return reply.send({data: {- message: "Import deleted successfully"- }+ message: "Import deleted successfully",+ },});} catch (error) {console.error("Error deleting import:", error);diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.tsindex c2355acea..4c3319582 100644--- a/server/src/api/sites/getSiteImports.ts+++ b/server/src/api/sites/getSiteImports.ts@@ -3,20 +3,19 @@ import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { ImportStatusManager } from "../../services/import/importStatusManager.js";import { z } from "zod";-const getSiteImportsRequestSchema = z.object({- params: z.object({- site: z.string().min(1),- }),-}).strict();+const getSiteImportsRequestSchema = z+ .object({+ params: z.object({+ site: z.string().min(1),+ }),+ })+ .strict();type GetSiteImportsRequest = {Params: z.infer<typeof getSiteImportsRequestSchema.shape.params>;};-export async function getSiteImports(- request: FastifyRequest<GetSiteImportsRequest>,- reply: FastifyReply,-) {+export async function getSiteImports(request: FastifyRequest<GetSiteImportsRequest>, reply: FastifyReply) {try {const parsed = getSiteImportsRequestSchema.safeParse({params: request.params,@@ -44,7 +43,7 @@ export async function getSiteImports(errorMessage: imp.errorMessage,startedAt: imp.startedAt,fileName: imp.fileName,- }))+ })),});} catch (error) {console.error("Error fetching imports:", error);diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsindex 1c4e355ae..07ca13e5f 100644--- a/server/src/api/sites/importSiteData.ts+++ b/server/src/api/sites/importSiteData.ts@@ -22,42 +22,46 @@ const isValidDate = (date: string) => {const parseDate = (date: string) => DateTime.fromFormat(date, "yyyy-MM-dd", { zone: "utc" });-const importDataFieldsSchema = z.object({- fields: z.object({- source: z.enum(["umami"]),- startDate: z.string().refine(isValidDate).optional(),- endDate: z.string().refine(isValidDate).optional(),- }).refine((fields) => {- if (fields.startDate && fields.endDate) {- const start = parseDate(fields.startDate);- const end = parseDate(fields.endDate);- return start <= end;- }- return true;- }).refine((fields) => {- if (fields.startDate) {- const today = DateTime.utc().startOf("day");- const start = parseDate(fields.startDate);- return start <= today;- }- return true;- }),-}).strict();--const importDataRequestSchema = z.object({- params: z.object({- site: z.string().min(1),- }),-}).strict();+const importDataFieldsSchema = z+ .object({+ fields: z+ .object({+ source: z.enum(["umami"]),+ startDate: z.string().refine(isValidDate).optional(),+ endDate: z.string().refine(isValidDate).optional(),+ })+ .refine(fields => {+ if (fields.startDate && fields.endDate) {+ const start = parseDate(fields.startDate);+ const end = parseDate(fields.endDate);+ return start <= end;+ }+ return true;+ })+ .refine(fields => {+ if (fields.startDate) {+ const today = DateTime.utc().startOf("day");+ const start = parseDate(fields.startDate);+ return start <= today;+ }+ return true;+ }),+ })+ .strict();++const importDataRequestSchema = z+ .object({+ params: z.object({+ site: z.string().min(1),+ }),+ })+ .strict();type ImportDataRequest = {Params: z.infer<typeof importDataRequestSchema.shape.params>;};-export async function importSiteData(- request: FastifyRequest<ImportDataRequest>,- reply: FastifyReply,-) {+export async function importSiteData(request: FastifyRequest<ImportDataRequest>, reply: FastifyReply) {try {const parsedParams = importDataRequestSchema.safeParse({params: request.params,@@ -162,7 +166,7 @@ export async function importSiteData(return reply.status(202).send({data: {message: "File upload accepted and is now being processed.",- }+ },});} catch (error) {console.error("Unexpected error during import:", error);diff --git a/server/src/db/postgres/boss.ts b/server/src/db/postgres/boss.tsindex b4d930fcc..df864fe7e 100644--- a/server/src/db/postgres/boss.ts+++ b/server/src/db/postgres/boss.ts@@ -4,14 +4,14 @@ import dotenv from "dotenv";dotenv.config();const boss = new PgBoss({- host: process.env.POSTGRES_HOST || "postgres",- port: parseInt(process.env.POSTGRES_PORT || "5432", 10),- database: process.env.POSTGRES_DB,- user: process.env.POSTGRES_USER,- password: process.env.POSTGRES_PASSWORD,- schema: "pgboss",- application_name: "data-import-system",- retryLimit: 0,+ host: process.env.POSTGRES_HOST || "postgres",+ port: parseInt(process.env.POSTGRES_PORT || "5432", 10),+ database: process.env.POSTGRES_DB,+ user: process.env.POSTGRES_USER,+ password: process.env.POSTGRES_PASSWORD,+ schema: "pgboss",+ application_name: "data-import-system",+ retryLimit: 0,});boss.on("error", error => console.error(error));diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.tsindex 8a5ca94e2..6ed182f13 100644--- a/server/src/db/postgres/schema.ts+++ b/server/src/db/postgres/schema.ts@@ -66,7 +66,9 @@ export const sites = pgTable(createdBy: text("created_by").notNull().references(() => user.id),- organizationId: text("organization_id").notNull().references(() => organization.id),+ organizationId: text("organization_id")+ .notNull()+ .references(() => organization.id),public: boolean().default(false),saltUserIds: boolean().default(false),blockBots: boolean().default(true).notNull(),@@ -598,16 +600,9 @@ export const notificationChannels = pgTable(]);-export const importSourceEnum = pgEnum("import_source_enum", [- "umami",-]);+export const importSourceEnum = pgEnum("import_source_enum", ["umami"]);-export const importStatusEnum = pgEnum("import_status_enum", [- "pending",- "processing",- "completed",- "failed",-]);+export const importStatusEnum = pgEnum("import_status_enum", ["pending", "processing", "completed", "failed"]);// Import status table for tracking data import progressexport const importStatus = pgTable(@@ -624,7 +619,7 @@ export const importStatus = pgTable(completedAt: timestamp("completed_at", { mode: "string" }),fileName: text("file_name").notNull(),},- (table) => [+ table => [foreignKey({columns: [table.siteId],foreignColumns: [sites.siteId],diff --git a/server/src/index.ts b/server/src/index.tsindex 9bff26913..cdee3952e 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -166,7 +166,7 @@ server.register(multipart, {limits: {fileSize: IS_CLOUD ? 100 * 1024 * 1024 : undefined, // 100 MB max file size, infinite for self-hostedfiles: 1, // only allow 1 file- }+ },});// Serve static filesdiff --git a/server/src/queues/adapters/pgBossAdapter.ts b/server/src/queues/adapters/pgBossAdapter.tsindex 3b6efdbc8..17c4c3450 100644--- a/server/src/queues/adapters/pgBossAdapter.ts+++ b/server/src/queues/adapters/pgBossAdapter.ts@@ -16,7 +16,7 @@ export class PgBossAdapter implements IJobQueue {retryLimit: 0,});- this.boss.on("error", (error) => {+ this.boss.on("error", error => {console.error("[PgBoss] Error:", error);});}@@ -35,11 +35,7 @@ export class PgBossAdapter implements IJobQueue {await this.boss.createQueue(queueName);}- async send<T = any>(- queueName: string,- data: T,- options?: { priority?: number; delay?: number }- ): Promise<string> {+ async send<T = any>(queueName: string, data: T, options?: { priority?: number; delay?: number }): Promise<string> {const jobId = await this.boss.send(queueName, data as object, {priority: options?.priority,startAfter: options?.delay ? new Date(Date.now() + options.delay) : undefined,@@ -64,7 +60,7 @@ export class PgBossAdapter implements IJobQueue {pollingIntervalSeconds: config.pollingIntervalSeconds ?? 2,},async (jobs: Job<T>[]) => {- const normalizedJobs: JobData<T>[] = jobs.map((job) => ({+ const normalizedJobs: JobData<T>[] = jobs.map(job => ({id: job.id,data: job.data,}));diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.tsindex 812a04847..49e09c09c 100644--- a/server/src/services/import/importLimiter.ts+++ b/server/src/services/import/importLimiter.ts@@ -7,13 +7,16 @@ export class ImportLimiter {private static readonly IMPORTED_EVENTS_LIMIT = 1_000_000;private static readonly CONCURRENT_IMPORT_LIMIT = 1;- static async checkConcurrentImportLimit(siteId: number): Promise<{- allowed: false;- reason: string;- } | {- allowed: true;- organizationId: string- }> {+ static async checkConcurrentImportLimit(siteId: number): Promise<+ | {+ allowed: false;+ reason: string;+ }+ | {+ allowed: true;+ organizationId: string;+ }+ > {const [siteResult] = await db.select({ organizationId: sites.organizationId }).from(sites)diff --git a/server/src/services/import/importStatusManager.ts b/server/src/services/import/importStatusManager.tsindex cd4063095..959efdf08 100644--- a/server/src/services/import/importStatusManager.ts+++ b/server/src/services/import/importStatusManager.ts@@ -34,10 +34,7 @@ export class ImportStatusManager {}}- static async updateProgress(- importId: string,- importedEvents: number,- ): Promise<void> {+ static async updateProgress(importId: string, importedEvents: number): Promise<void> {try {await db.update(importStatus)diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.tsindex 43b25a987..9cc51a232 100644--- a/server/src/services/import/mappings/umami.ts+++ b/server/src/services/import/mappings/umami.ts@@ -104,26 +104,32 @@ export class UmamiImportMapper {edge: "Edge",};- private static readonly browserSchema = z.string().max(30).transform((browser) => {- const key = browser.toLowerCase();- return UmamiImportMapper.browserMap[key] ?? browser;- });+ private static readonly browserSchema = z+ .string()+ .max(30)+ .transform(browser => {+ const key = browser.toLowerCase();+ return UmamiImportMapper.browserMap[key] ?? browser;+ });private static readonly osMap: Record<string, string> = {"windows 10": "Windows","windows 7": "Windows","windows server 2003": "Windows","mac os": "macOS",- "ios": "iOS",+ ios: "iOS","android os": "Android",- "linux": "Linux",+ linux: "Linux","chrome os": "Chrome OS",};- private static readonly osSchema = z.string().max(25).transform((os) => {- const key = os.toLowerCase();- return UmamiImportMapper.osMap[key] ?? os;- });+ private static readonly osSchema = z+ .string()+ .max(25)+ .transform(os => {+ const key = os.toLowerCase();+ return UmamiImportMapper.osMap[key] ?? os;+ });private static readonly deviceMap: Record<string, "Desktop" | "Mobile"> = {laptop: "Desktop",@@ -132,10 +138,13 @@ export class UmamiImportMapper {tablet: "Mobile",};- private static readonly deviceSchema = z.string().max(20).transform((device) => {- const key = device.toLowerCase();- return UmamiImportMapper.deviceMap[key] ?? device;- });+ private static readonly deviceSchema = z+ .string()+ .max(20)+ .transform(device => {+ const key = device.toLowerCase();+ return UmamiImportMapper.deviceMap[key] ?? device;+ });private static readonly umamiEventSchema = z.object({session_id: z.string().uuid(),@@ -144,17 +153,35 @@ export class UmamiImportMapper {browser: UmamiImportMapper.browserSchema,os: UmamiImportMapper.osSchema,device: UmamiImportMapper.deviceSchema,- screen: z.string().regex(/^\d{1,5}x\d{1,5}$/).or(z.literal("")),+ screen: z+ .string()+ .regex(/^\d{1,5}x\d{1,5}$/)+ .or(z.literal("")),language: z.string().max(35),- country: z.string().regex(/^[A-Z]{2}$/).or(z.literal("")),- region: z.string().regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/).or(z.literal("")),+ country: z+ .string()+ .regex(/^[A-Z]{2}$/)+ .or(z.literal("")),+ region: z+ .string()+ .regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/)+ .or(z.literal("")),city: z.string().max(60),url_path: z.string().max(2048),- url_query: z.string().max(2048).transform((querystring) => querystring ? `?${querystring}` : ""),+ url_query: z+ .string()+ .max(2048)+ .transform(querystring => (querystring ? `?${querystring}` : "")),referrer_path: z.string().max(2048),- referrer_query: z.string().max(2048).transform((querystring) => querystring ? `?${querystring}` : ""),- referrer_domain: z.string().max(253).transform((url) => url ? `https://${url}` : ""),+ referrer_query: z+ .string()+ .max(2048)+ .transform(querystring => (querystring ? `?${querystring}` : "")),+ referrer_domain: z+ .string()+ .max(253)+ .transform(url => (url ? `https://${url}` : "")),page_title: z.string().max(512),event_type: z.enum(["1", "2"]),diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.tsindex adf6af32a..311957874 100644--- a/server/src/services/import/utils.ts+++ b/server/src/services/import/utils.ts@@ -10,10 +10,7 @@ export interface DeleteFileResult {* Delete an import file from storage.* Returns result instead of throwing to prevent worker crashes.*/-export const deleteImportFile = async (- storageLocation: string,- isR2Storage: boolean-): Promise<DeleteFileResult> => {+export const deleteImportFile = async (storageLocation: string, isR2Storage: boolean): Promise<DeleteFileResult> => {try {if (isR2Storage) {await r2Storage.deleteImportFile(storageLocation);diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 5819d6aab..89c2dc35a 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -22,21 +22,25 @@ const getImportDataHeaders = (source: string) => {const createR2FileStream = async (storageLocation: string, source: string) => {console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);const fileStream = await r2Storage.getImportFileStream(storageLocation);- return fileStream.pipe(parse({- headers: getImportDataHeaders(source),- renameHeaders: true,- ignoreEmpty: true,- }));+ return fileStream.pipe(+ parse({+ headers: getImportDataHeaders(source),+ renameHeaders: true,+ ignoreEmpty: true,+ })+ );};const createLocalFileStream = async (storageLocation: string, source: string) => {console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);await access(storageLocation, constants.F_OK | constants.R_OK);- return createReadStream(storageLocation).pipe(parse({- headers: getImportDataHeaders(source),- renameHeaders: true,- ignoreEmpty: true,- }));+ return createReadStream(storageLocation).pipe(+ parse({+ headers: getImportDataHeaders(source),+ renameHeaders: true,+ ignoreEmpty: true,+ })+ );};/**@@ -49,9 +53,7 @@ const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day"): null;- const endDate = endDateStr- ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day")- : null;+ const endDate = endDateStr ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day") : null;// Validate parsed datesif (startDate && !startDate.isValid) {@@ -83,11 +85,8 @@ const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {export async function registerCsvParseWorker() {const jobQueue = getJobQueue();- await jobQueue.work<CsvParseJob>(- CSV_PARSE_QUEUE,- { batchSize: 1, pollingIntervalSeconds: 10 },- async ([job]) => {- const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;+ await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([job]) => {+ const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;try {const importableEvents = await ImportLimiter.countImportableEvents(organization);@@ -162,25 +161,22 @@ export async function registerCsvParseWorker() {totalChunks: chunksSent,allChunksSent: true,});- } catch (error) {- console.error("Error in CSV parse worker:", error);- await ImportStatusManager.updateStatus(- importId,- "failed",- error instanceof Error ? error.message : "Unknown error occurred"- );- throw error;- } finally {- // Clean up file - don't throw on failure to prevent worker crashes- const deleteResult = await deleteImportFile(storageLocation, isR2Storage);- if (!deleteResult.success) {- console.warn(- `[Import ${importId}] File cleanup failed, will remain in storage: ${deleteResult.error}`- );- // File will be orphaned but import status is already recorded- // Could implement a cleanup job here to retry later- }+ } catch (error) {+ console.error("Error in CSV parse worker:", error);+ await ImportStatusManager.updateStatus(+ importId,+ "failed",+ error instanceof Error ? error.message : "Unknown error occurred"+ );+ throw error;+ } finally {+ // Clean up file - don't throw on failure to prevent worker crashes+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);+ if (!deleteResult.success) {+ console.warn(`[Import ${importId}] File cleanup failed, will remain in storage: ${deleteResult.error}`);+ // File will be orphaned but import status is already recorded+ // Could implement a cleanup job here to retry later}}- );+ });}diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex dcd601c42..3ec14d2ca 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -16,66 +16,54 @@ const getImportDataMapping = (source: string) => {export async function registerDataInsertWorker() {const jobQueue = getJobQueue();- await jobQueue.work<DataInsertJob>(- DATA_INSERT_QUEUE,- { batchSize: 1, pollingIntervalSeconds: 2 },- async ([job]) => {- const { site, importId, source, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;+ await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([job]) => {+ const { site, importId, source, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;- // Handle finalization signal- if (allChunksSent) {- try {- await ImportStatusManager.updateStatus(importId, "completed");- console.log(- `[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`- );- return;- } catch (error) {- console.error(`[Import ${importId}] Failed to mark as completed:`, error);- await ImportStatusManager.updateStatus(- importId,- "failed",- "Failed to complete import"- );- throw error;- }- }-- // Process data chunk+ // Handle finalization signal+ if (allChunksSent) {try {- const dataMapper = getImportDataMapping(source);- const transformedRecords = dataMapper.transform(chunk, site, importId);+ await ImportStatusManager.updateStatus(importId, "completed");+ console.log(`[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`);+ return;+ } catch (error) {+ console.error(`[Import ${importId}] Failed to mark as completed:`, error);+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");+ throw error;+ }+ }- // Insert to ClickHouse (critical - must succeed)- await clickhouse.insert({- table: "events",- values: transformedRecords,- format: "JSONEachRow",- });+ // Process data chunk+ try {+ const dataMapper = getImportDataMapping(source);+ const transformedRecords = dataMapper.transform(chunk, site, importId);- // Update progress (non-critical - log if fails but don't crash)- try {- await ImportStatusManager.updateProgress(importId, transformedRecords.length);- } catch (progressError) {- console.warn(- `[Import ${importId}] Progress update failed (data inserted successfully):`,- progressError instanceof Error ? progressError.message : progressError- );- // Don't throw - data is safely in ClickHouse, progress can be off slightly- }+ // Insert to ClickHouse (critical - must succeed)+ await clickhouse.insert({+ table: "events",+ values: transformedRecords,+ format: "JSONEachRow",+ });- console.log(- `[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`- );- } catch (error) {- console.error(`[Import ${importId}] ClickHouse insert failed:`, error);- await ImportStatusManager.updateStatus(- importId,- "failed",- `Data insertion failed: ${error instanceof Error ? error.message : "Unknown error"}`+ // Update progress (non-critical - log if fails but don't crash)+ try {+ await ImportStatusManager.updateProgress(importId, transformedRecords.length);+ } catch (progressError) {+ console.warn(+ `[Import ${importId}] Progress update failed (data inserted successfully):`,+ progressError instanceof Error ? progressError.message : progressError);- throw error;+ // Don't throw - data is safely in ClickHouse, progress can be off slightly}++ console.log(`[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`);+ } catch (error) {+ console.error(`[Import ${importId}] ClickHouse insert failed:`, error);+ await ImportStatusManager.updateStatus(+ importId,+ "failed",+ `Data insertion failed: ${error instanceof Error ? error.message : "Unknown error"}`+ );+ throw error;}- );+ });}diff --git a/server/src/services/import/workers/queues.ts b/server/src/services/import/workers/queues.tsindex a574e1fd5..b6f60d25f 100644--- a/server/src/services/import/workers/queues.ts+++ b/server/src/services/import/workers/queues.ts@@ -10,4 +10,4 @@ export const createJobQueues = async () => {} catch (error) {throw new Error(`Failed to create job queues: ${error instanceof Error ? error.message : "Unknown error"}`);}-}+};diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.tsindex 798943413..9f268bb4f 100644--- a/server/src/services/storage/r2StorageService.ts+++ b/server/src/services/storage/r2StorageService.ts@@ -1,9 +1,4 @@-import {- S3Client,- PutObjectCommand,- GetObjectCommand,- DeleteObjectCommand,-} from "@aws-sdk/client-s3";+import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from "@aws-sdk/client-s3";import { Upload } from "@aws-sdk/lib-storage";import { NodeHttpHandler } from "@smithy/node-http-handler";import { Readable } from "stream";@@ -245,7 +240,7 @@ class R2StorageService {new GetObjectCommand({Bucket: this.bucketName,Key: key,- }),+ }));if (!response.Body) {@@ -273,7 +268,7 @@ class R2StorageService {new DeleteObjectCommand({Bucket: this.bucketName,Key: key,- }),+ }));this.logger.info({ key }, "Successfully deleted import file");} catch (error) {