OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat: new importer (#214)
Carl-Gerhard Lindesvärd212254d31a75
diff --git a/apps/api/package.json b/apps/api/package.jsonindex e42cc2435..9a14c7d35 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -64,7 +64,7 @@"@types/uuid": "^10.0.0","@types/ws": "^8.5.14","js-yaml": "^4.1.0",- "tsdown": "^0.14.2",+ "tsdown": "0.14.2","typescript": "catalog:"}}diff --git a/apps/start/Dockerfile b/apps/start/Dockerfileindex 369cd0246..4853b7a62 100644--- a/apps/start/Dockerfile+++ b/apps/start/Dockerfile@@ -29,6 +29,7 @@ COPY apps/start/package.json ./apps/start/COPY packages/trpc/package.json packages/trpc/COPY packages/json/package.json packages/json/COPY packages/common/package.json packages/common/+COPY packages/importer/package.json packages/importer/COPY packages/payments/package.json packages/payments/COPY packages/constants/package.json packages/constants/COPY packages/validation/package.json packages/validation/@@ -85,6 +86,7 @@ COPY --from=build /app/packages/trpc/package.json ./packages/trpc/COPY --from=build /app/packages/auth/package.json ./packages/auth/COPY --from=build /app/packages/json/package.json ./packages/json/COPY --from=build /app/packages/common/package.json ./packages/common/+COPY --from=build /app/packages/importer/package.json ./packages/importer/COPY --from=build /app/packages/payments/package.json ./packages/payments/COPY --from=build /app/packages/constants/package.json ./packages/constants/COPY --from=build /app/packages/validation/package.json ./packages/validation/diff --git a/apps/start/package.json b/apps/start/package.jsonindex 4ab99a63a..d5b97664a 100644--- a/apps/start/package.json+++ b/apps/start/package.json@@ -8,7 +8,7 @@"preview": "vite preview","deploy": "npx wrangler deploy","cf-typegen": "wrangler types",- "build": "vite build",+ "build": "pnpm with-env vite build","serve": "vite preview","test": "vitest run","format": "biome format",@@ -30,6 +30,7 @@"@openpanel/common": "workspace:^","@openpanel/constants": "workspace:^","@openpanel/integrations": "workspace:^",+ "@openpanel/importer": "workspace:^","@openpanel/json": "workspace:*","@openpanel/payments": "workspace:*","@openpanel/sdk-info": "workspace:^",diff --git a/apps/start/src/components/integrations/integration-card.tsx b/apps/start/src/components/integrations/integration-card.tsxindex ab80513ce..c0a35edde 100644--- a/apps/start/src/components/integrations/integration-card.tsx+++ b/apps/start/src/components/integrations/integration-card.tsx@@ -50,12 +50,15 @@ export function IntegrationCardHeaderButtons({export function IntegrationCardLogoImage({src,backgroundColor,+ className,}: {src: string;backgroundColor: string;+ className?: string;}) {return (<IntegrationCardLogo+ className={className}style={{backgroundColor,}}diff --git a/apps/start/src/components/project-mapper.tsx b/apps/start/src/components/project-mapper.tsxnew file mode 100644index 000000000..92002a171--- /dev/null+++ b/apps/start/src/components/project-mapper.tsx@@ -0,0 +1,116 @@+import { Button } from '@/components/ui/button';+import { Input } from '@/components/ui/input';+import { Label } from '@/components/ui/label';+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from '@/components/ui/select';+import type { IServiceProject } from '@openpanel/db';+import { PlusIcon, XIcon } from 'lucide-react';+import type {+ Control,+ FieldArrayWithId,+ UseFieldArrayAppend,+ UseFieldArrayRemove,+ UseFormRegister,+ UseFormSetValue,+ UseFormWatch,+} from 'react-hook-form';++interface ProjectMapperProps {+ fields: FieldArrayWithId<any, 'projectMapper', 'id'>[];+ append: UseFieldArrayAppend<any, 'projectMapper'>;+ remove: UseFieldArrayRemove;+ projects: IServiceProject[];+ register: UseFormRegister<any>;+ watch: UseFormWatch<any>;+ setValue: UseFormSetValue<any>;+}++export function ProjectMapper({+ fields,+ append,+ remove,+ projects,+ register,+ watch,+ setValue,+}: ProjectMapperProps) {+ return (+ <div className="space-y-2">+ <div className="flex items-center justify-between">+ <Label className="mb-0">Project Mapper (Optional)</Label>+ <Button+ type="button"+ variant="outline"+ size="sm"+ onClick={() => append({ from: '', to: '' })}+ >+ <PlusIcon className="mr-1 h-4 w-4" />+ Add Mapping+ </Button>+ </div>+ {fields.length === 0 && (+ <p className="text-sm text-muted-foreground leading-normal">+ Map source project IDs to your OpenPanel projects. If you skip mapping+ all data will be imported to your current project.+ </p>+ )}++ {fields.length > 0 && (+ <div className="space-y-2">+ {fields.map((field, index) => (+ <div key={field.id} className="gap-2 rounded-md border p-3 row">+ <div className="grid grid-cols-2 gap-2">+ <div className="flex-1">+ <Label className="text-xs text-muted-foreground">+ From (Source Project ID)+ </Label>+ <Input+ placeholder="e.g., abc123"+ {...register(`projectMapper.${index}.from`)}+ className="mt-1"+ />+ </div>+ <div className="flex-1">+ <Label className="text-xs text-muted-foreground">+ To (OpenPanel Project)+ </Label>+ <Select+ value={watch(`projectMapper.${index}.to`)}+ onValueChange={(value) =>+ setValue(`projectMapper.${index}.to`, value)+ }+ >+ <SelectTrigger className="mt-1 w-full" size="sm">+ <SelectValue placeholder="Select project" />+ </SelectTrigger>+ <SelectContent>+ {projects.slice(0, 10).map((project) => (+ <SelectItem key={project.id} value={project.id}>+ {project.name}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div>+ </div>+ <Button+ type="button"+ variant="ghost"+ size="icon"+ onClick={() => remove(index)}+ className="mt-5"+ >+ <XIcon className="h-4 w-4" />+ </Button>+ </div>+ ))}+ </div>+ )}+ </div>+ );+}diff --git a/apps/start/src/components/ui/badge.tsx b/apps/start/src/components/ui/badge.tsxindex e63ad31fb..b8b3da2a8 100644--- a/apps/start/src/components/ui/badge.tsx+++ b/apps/start/src/components/ui/badge.tsx@@ -9,6 +9,11 @@ const badgeVariants = cva({variants: {variant: {+ success:+ 'border-transparent bg-emerald-700 text-white [a&]:hover:bg-emerald-700/90',+ warning:+ 'border-transparent bg-yellow-500 text-white [a&]:hover:bg-yellow-500/90',+ info: 'border-transparent bg-blue-500 text-white [a&]:hover:bg-blue-500/90',default:'border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90',secondary:diff --git a/apps/start/src/modals/add-import.tsx b/apps/start/src/modals/add-import.tsxnew file mode 100644index 000000000..03a77e354--- /dev/null+++ b/apps/start/src/modals/add-import.tsx@@ -0,0 +1,282 @@+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';+import { toast } from 'sonner';++import { InputWithLabel, WithLabel } from '@/components/forms/input-with-label';+import { ProjectMapper } from '@/components/project-mapper';+import { Button } from '@/components/ui/button';+import { useAppParams } from '@/hooks/use-app-params';+import { useTRPC } from '@/integrations/trpc/react';+import { cn } from '@/lib/utils';+import { zodResolver } from '@hookform/resolvers/zod';+import type {+ IImportConfig,+ IMixpanelImportConfig,+ IUmamiImportConfig,+} from '@openpanel/validation';+import {+ zMixpanelImportConfig,+ zUmamiImportConfig,+} from '@openpanel/validation';+import { format } from 'date-fns';+import { CalendarIcon } from 'lucide-react';+import { useFieldArray, useForm } from 'react-hook-form';+import type { z } from 'zod';+import { popModal, pushModal } from '.';+import { ModalContent, ModalHeader } from './Modal/Container';++type Provider = 'umami' | 'plausible' | 'mixpanel';++interface AddImportProps {+ provider: Provider;+ name: string;+ types: ('file' | 'api')[];+}++type UmamiFormData = z.infer<typeof zUmamiImportConfig>;+type MixpanelFormData = z.infer<typeof zMixpanelImportConfig>;++interface UmamiImportProps {+ onSubmit: (config: IUmamiImportConfig) => void;+ isPending: boolean;+ organizationId: string;+}++function UmamiImport({+ onSubmit,+ isPending,+ organizationId,+}: UmamiImportProps) {+ const trpc = useTRPC();+ const { data: projects = [] } = useQuery(+ trpc.project.list.queryOptions({+ organizationId,+ }),+ );++ const form = useForm<UmamiFormData>({+ resolver: zodResolver(zUmamiImportConfig),+ defaultValues: {+ provider: 'umami',+ type: 'file',+ fileUrl: '',+ projectMapper: [],+ },+ });++ const { fields, append, remove } = useFieldArray({+ control: form.control,+ name: 'projectMapper',+ });++ const handleSubmit = form.handleSubmit((data) => {+ onSubmit(data);+ });++ return (+ <form onSubmit={handleSubmit} className="space-y-4">+ <div className="space-y-4 py-4">+ <InputWithLabel+ label="File URL"+ placeholder="https://example.com/export.csv"+ error={form.formState.errors.fileUrl?.message}+ info="Provide a publicly accessible URL to your exported CSV file."+ {...form.register('fileUrl')}+ />++ <ProjectMapper+ fields={fields}+ append={append}+ remove={remove}+ projects={projects}+ register={form.register}+ watch={form.watch}+ setValue={form.setValue}+ />+ </div>++ <div className="flex justify-between">+ <Button type="button" variant="outline" onClick={() => popModal()}>+ Cancel+ </Button>+ <Button type="submit" disabled={isPending}>+ {isPending ? 'Starting...' : 'Start Import'}+ </Button>+ </div>+ </form>+ );+}++interface MixpanelImportProps {+ onSubmit: (config: IMixpanelImportConfig) => void;+ isPending: boolean;+ organizationId: string;+}++function MixpanelImport({+ onSubmit,+ isPending,+ organizationId,+}: MixpanelImportProps) {+ const trpc = useTRPC();+ const form = useForm<MixpanelFormData>({+ resolver: zodResolver(zMixpanelImportConfig),+ defaultValues: {+ provider: 'mixpanel',+ type: 'api',+ serviceAccount: '',+ serviceSecret: '',+ projectId: '',+ from: '',+ to: '',+ },+ });++ const handleDateRangeSelect = () => {+ pushModal('DateRangerPicker', {+ startDate: form.getValues('from')+ ? new Date(form.getValues('from'))+ : undefined,+ endDate: form.getValues('to')+ ? new Date(form.getValues('to'))+ : undefined,+ onChange: ({ startDate, endDate }) => {+ form.setValue('from', format(startDate, 'yyyy-MM-dd'));+ form.setValue('to', format(endDate, 'yyyy-MM-dd'));+ form.trigger('from');+ form.trigger('to');+ },+ });+ };++ const handleSubmit = form.handleSubmit((data) => {+ onSubmit(data);+ });++ return (+ <form onSubmit={handleSubmit} className="space-y-4">+ <div className="space-y-4 py-4">+ <InputWithLabel+ label="Service Account"+ placeholder="Eg. xxx.xxx.mp-service-account"+ error={form.formState.errors.serviceAccount?.message}+ {...form.register('serviceAccount')}+ />++ <InputWithLabel+ label="Service Secret"+ type="password"+ placeholder="Your Mixpanel service secret"+ error={form.formState.errors.serviceSecret?.message}+ {...form.register('serviceSecret')}+ />++ <InputWithLabel+ label="Project ID"+ placeholder="Your Mixpanel project ID"+ error={form.formState.errors.projectId?.message}+ {...form.register('projectId')}+ />++ <WithLabel+ label="Date Range"+ info={+ !form.getValues('from') || !form.getValues('to')+ ? 'Select the date range for importing data'+ : undefined+ }+ >+ <Button+ type="button"+ variant="outline"+ className={cn(+ 'w-full justify-start text-left font-normal',+ (!form.getValues('from') || !form.getValues('to')) &&+ 'text-muted-foreground',+ )}+ onClick={handleDateRangeSelect}+ >+ <CalendarIcon className="mr-2 h-4 w-4" />+ {form.getValues('from') && form.getValues('to') ? (+ <>+ {format(new Date(form.getValues('from')), 'LLL dd, y')} -{' '}+ {format(new Date(form.getValues('to')), 'LLL dd, y')}+ </>+ ) : (+ <span>Pick a date range</span>+ )}+ </Button>+ </WithLabel>++ <InputWithLabel+ label="Screen View Property"+ placeholder="Enter the name of the property that contains the screen name"+ info="Leave empty if not applicable"+ error={form.formState.errors.mapScreenViewProperty?.message}+ {...form.register('mapScreenViewProperty')}+ />+ </div>++ <div className="flex justify-between">+ <Button type="button" variant="outline" onClick={() => popModal()}>+ Cancel+ </Button>+ <Button type="submit" disabled={isPending}>+ {isPending ? 'Starting...' : 'Start Import'}+ </Button>+ </div>+ </form>+ );+}++export default function AddImport({ provider, name }: AddImportProps) {+ const { projectId, organizationId } = useAppParams();+ const trpc = useTRPC();+ const queryClient = useQueryClient();++ const createImport = useMutation(+ trpc.import.create.mutationOptions({+ onSuccess() {+ toast.success('Import started', {+ description: 'Your data import has been queued for processing.',+ });+ popModal();+ queryClient.invalidateQueries(trpc.import.list.pathFilter());+ },+ onError: (error) => {+ toast.error('Import failed', {+ description: error.message,+ });+ },+ }),+ );++ const handleImportSubmit = (config: IImportConfig) => {+ createImport.mutate({+ projectId,+ provider: config.provider,+ config,+ });+ };++ return (+ <ModalContent>+ <ModalHeader title={`Import from ${name}`} />++ {provider === 'umami' && (+ <UmamiImport+ onSubmit={handleImportSubmit}+ isPending={createImport.isPending}+ organizationId={organizationId}+ />+ )}++ {provider === 'mixpanel' && (+ <MixpanelImport+ onSubmit={handleImportSubmit}+ isPending={createImport.isPending}+ organizationId={organizationId}+ />+ )}+ </ModalContent>+ );+}diff --git a/apps/start/src/modals/date-ranger-picker.tsx b/apps/start/src/modals/date-ranger-picker.tsxindex 7c1142291..277fa163c 100644--- a/apps/start/src/modals/date-ranger-picker.tsx+++ b/apps/start/src/modals/date-ranger-picker.tsx@@ -27,6 +27,7 @@ export default function DateRangerPicker({return (<ModalContent className="p-4 md:p-8 min-w-fit"><Calendar+ captionLayout="dropdown"initialFocusmode="range"defaultMonth={subMonths(diff --git a/apps/start/src/modals/index.tsx b/apps/start/src/modals/index.tsxindex 47ac6c5c1..709951d8f 100644--- a/apps/start/src/modals/index.tsx+++ b/apps/start/src/modals/index.tsx@@ -5,6 +5,7 @@ import OverviewTopPagesModal from '@/components/overview/overview-top-pages-modaimport Instructions from './Instructions';import AddClient from './add-client';import AddDashboard from './add-dashboard';+import AddImport from './add-import';import AddIntegration from './add-integration';import AddNotificationRule from './add-notification-rule';import AddProject from './add-project';@@ -38,6 +39,7 @@ const modals = {EditClient: EditClient,AddProject: AddProject,AddClient: AddClient,+ AddImport: AddImport,Confirm: Confirm,SaveReport: SaveReport,AddDashboard: AddDashboard,diff --git a/apps/start/src/routeTree.gen.ts b/apps/start/src/routeTree.gen.tsindex 9840b4d69..e221cf0ea 100644--- a/apps/start/src/routeTree.gen.ts+++ b/apps/start/src/routeTree.gen.ts@@ -56,6 +56,7 @@ import { Route as AppOrganizationIdProjectIdSettingsTabsIndexRouteImport } fromimport { Route as AppOrganizationIdProjectIdProfilesTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.profiles._tabs.index'import { Route as AppOrganizationIdProjectIdNotificationsTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.notifications._tabs.index'import { Route as AppOrganizationIdProjectIdEventsTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.events._tabs.index'+import { Route as AppOrganizationIdProjectIdSettingsTabsImportsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.imports'import { Route as AppOrganizationIdProjectIdSettingsTabsEventsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.events'import { Route as AppOrganizationIdProjectIdSettingsTabsDetailsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.details'import { Route as AppOrganizationIdProjectIdSettingsTabsClientsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.clients'@@ -381,6 +382,12 @@ const AppOrganizationIdProjectIdEventsTabsIndexRoute =path: '/',getParentRoute: () => AppOrganizationIdProjectIdEventsTabsRoute,} as any)+const AppOrganizationIdProjectIdSettingsTabsImportsRoute =+ AppOrganizationIdProjectIdSettingsTabsImportsRouteImport.update({+ id: '/imports',+ path: '/imports',+ getParentRoute: () => AppOrganizationIdProjectIdSettingsTabsRoute,+ } as any)const AppOrganizationIdProjectIdSettingsTabsEventsRoute =AppOrganizationIdProjectIdSettingsTabsEventsRouteImport.update({id: '/events',@@ -515,6 +522,7 @@ export interface FileRoutesByFullPath {'/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute'/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute'/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute+ '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute'/$organizationId/$projectId/events/': typeof AppOrganizationIdProjectIdEventsTabsIndexRoute'/$organizationId/$projectId/notifications/': typeof AppOrganizationIdProjectIdNotificationsTabsIndexRoute'/$organizationId/$projectId/profiles/': typeof AppOrganizationIdProjectIdProfilesTabsIndexRoute@@ -569,6 +577,7 @@ export interface FileRoutesByTo {'/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute'/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute'/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute+ '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute'/$organizationId/$projectId/profiles/$profileId/events': typeof AppOrganizationIdProjectIdProfilesProfileIdTabsEventsRoute}export interface FileRoutesById {@@ -633,6 +642,7 @@ export interface FileRoutesById {'/_app/$organizationId/$projectId_/settings/_tabs/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute'/_app/$organizationId/$projectId_/settings/_tabs/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute'/_app/$organizationId/$projectId_/settings/_tabs/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute+ '/_app/$organizationId/$projectId_/settings/_tabs/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute'/_app/$organizationId/$projectId_/events/_tabs/': typeof AppOrganizationIdProjectIdEventsTabsIndexRoute'/_app/$organizationId/$projectId_/notifications/_tabs/': typeof AppOrganizationIdProjectIdNotificationsTabsIndexRoute'/_app/$organizationId/$projectId_/profiles/_tabs/': typeof AppOrganizationIdProjectIdProfilesTabsIndexRoute@@ -692,6 +702,7 @@ export interface FileRouteTypes {| '/$organizationId/$projectId/settings/clients'| '/$organizationId/$projectId/settings/details'| '/$organizationId/$projectId/settings/events'+ | '/$organizationId/$projectId/settings/imports'| '/$organizationId/$projectId/events/'| '/$organizationId/$projectId/notifications/'| '/$organizationId/$projectId/profiles/'@@ -746,6 +757,7 @@ export interface FileRouteTypes {| '/$organizationId/$projectId/settings/clients'| '/$organizationId/$projectId/settings/details'| '/$organizationId/$projectId/settings/events'+ | '/$organizationId/$projectId/settings/imports'| '/$organizationId/$projectId/profiles/$profileId/events'id:| '__root__'@@ -809,6 +821,7 @@ export interface FileRouteTypes {| '/_app/$organizationId/$projectId_/settings/_tabs/clients'| '/_app/$organizationId/$projectId_/settings/_tabs/details'| '/_app/$organizationId/$projectId_/settings/_tabs/events'+ | '/_app/$organizationId/$projectId_/settings/_tabs/imports'| '/_app/$organizationId/$projectId_/events/_tabs/'| '/_app/$organizationId/$projectId_/notifications/_tabs/'| '/_app/$organizationId/$projectId_/profiles/_tabs/'@@ -1194,6 +1207,13 @@ declare module '@tanstack/react-router' {preLoaderRoute: typeof AppOrganizationIdProjectIdEventsTabsIndexRouteImportparentRoute: typeof AppOrganizationIdProjectIdEventsTabsRoute}+ '/_app/$organizationId/$projectId_/settings/_tabs/imports': {+ id: '/_app/$organizationId/$projectId_/settings/_tabs/imports'+ path: '/imports'+ fullPath: '/$organizationId/$projectId/settings/imports'+ preLoaderRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRouteImport+ parentRoute: typeof AppOrganizationIdProjectIdSettingsTabsRoute+ }'/_app/$organizationId/$projectId_/settings/_tabs/events': {id: '/_app/$organizationId/$projectId_/settings/_tabs/events'path: '/events'@@ -1521,6 +1541,7 @@ interface AppOrganizationIdProjectIdSettingsTabsRouteChildren {AppOrganizationIdProjectIdSettingsTabsClientsRoute: typeof AppOrganizationIdProjectIdSettingsTabsClientsRouteAppOrganizationIdProjectIdSettingsTabsDetailsRoute: typeof AppOrganizationIdProjectIdSettingsTabsDetailsRouteAppOrganizationIdProjectIdSettingsTabsEventsRoute: typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute+ AppOrganizationIdProjectIdSettingsTabsImportsRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRouteAppOrganizationIdProjectIdSettingsTabsIndexRoute: typeof AppOrganizationIdProjectIdSettingsTabsIndexRoute}@@ -1532,6 +1553,8 @@ const AppOrganizationIdProjectIdSettingsTabsRouteChildren: AppOrganizationIdProjAppOrganizationIdProjectIdSettingsTabsDetailsRoute,AppOrganizationIdProjectIdSettingsTabsEventsRoute:AppOrganizationIdProjectIdSettingsTabsEventsRoute,+ AppOrganizationIdProjectIdSettingsTabsImportsRoute:+ AppOrganizationIdProjectIdSettingsTabsImportsRoute,AppOrganizationIdProjectIdSettingsTabsIndexRoute:AppOrganizationIdProjectIdSettingsTabsIndexRoute,}diff --git a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsxnew file mode 100644index 000000000..123e78757--- /dev/null+++ b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx@@ -0,0 +1,303 @@+import { FullPageEmptyState } from '@/components/full-page-empty-state';+import {+ IntegrationCard,+ IntegrationCardFooter,+ IntegrationCardLogoImage,+} from '@/components/integrations/integration-card';+import { Skeleton } from '@/components/skeleton';+import { Badge } from '@/components/ui/badge';+import { Button } from '@/components/ui/button';+import {+ Table,+ TableBody,+ TableCell,+ TableHead,+ TableHeader,+ TableRow,+} from '@/components/ui/table';+import { Tooltiper } from '@/components/ui/tooltip';+import { useAppParams } from '@/hooks/use-app-params';+import { useTRPC } from '@/integrations/trpc/react';+import { pushModal } from '@/modals';+import { IMPORT_PROVIDERS } from '@openpanel/importer/providers';+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';+import { createFileRoute } from '@tanstack/react-router';+import { formatDistanceToNow } from 'date-fns';+import {+ CheckCircleIcon,+ Download,+ InfoIcon,+ Loader2Icon,+ XCircleIcon,+} from 'lucide-react';+import { toast } from 'sonner';++export const Route = createFileRoute(+ '/_app/$organizationId/$projectId_/settings/_tabs/imports',+)({+ component: ImportsSettings,+});++function ImportsSettings() {+ const { projectId } = useAppParams();+ const trpc = useTRPC();+ const queryClient = useQueryClient();++ const importsQuery = useQuery(+ trpc.import.list.queryOptions(+ { projectId },+ {+ refetchInterval: 5000,+ },+ ),+ );+ const imports = importsQuery.data ?? [];++ const deleteImport = useMutation(+ trpc.import.delete.mutationOptions({+ onSuccess: () => {+ toast.success('Import deleted', {+ description: 'The import has been successfully deleted.',+ });+ queryClient.invalidateQueries(trpc.import.list.pathFilter());+ },+ }),+ );++ const retryImport = useMutation(+ trpc.import.retry.mutationOptions({+ onSuccess: () => {+ toast.success('Import retried', {+ description: 'The import has been queued for processing again.',+ });+ queryClient.invalidateQueries(trpc.import.list.pathFilter());+ },+ }),+ );++ const handleProviderSelect = (+ provider: (typeof IMPORT_PROVIDERS)[number],+ ) => {+ pushModal('AddImport', {+ provider: provider.id,+ name: provider.name,+ types: provider.types,+ });+ };++ const getStatusBadge = (status: string, errorMessage: string | null) => {+ const variants: Record<string, any> = {+ pending: 'secondary',+ processing: 'default',+ completed: 'success',+ failed: 'destructive',+ };+ const icons: Record<string, React.ReactNode> = {+ pending: <Loader2Icon className="w-4 h-4 animate-spin" />,+ processing: <Loader2Icon className="w-4 h-4 animate-spin" />,+ completed: <CheckCircleIcon className="w-4 h-4" />,+ failed: <XCircleIcon className="w-4 h-4" />,+ };++ if (status === 'failed') {+ return (+ <Tooltiper+ content={errorMessage}+ tooltipClassName="max-w-xs break-words"+ >+ <Badge variant={variants[status] || 'default'} className="capitalize">+ {icons[status] || null}+ {status}+ </Badge>+ </Tooltiper>+ );+ }++ return (+ <Badge variant={variants[status] || 'default'} className="capitalize">+ {icons[status] || null}+ {status}+ </Badge>+ );+ };++ return (+ <div className="space-y-8">+ <div>+ <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">+ {IMPORT_PROVIDERS.map((provider) => (+ <IntegrationCard+ key={provider.id}+ icon={+ <IntegrationCardLogoImage+ src={provider.logo}+ backgroundColor={provider.backgroundColor}+ className="p-4"+ />+ }+ name={provider.name}+ description={provider.description}+ >+ <IntegrationCardFooter className="row justify-end">+ <Button+ variant="ghost"+ onClick={() => handleProviderSelect(provider)}+ >+ <Download className="w-4 h-4 mr-2" />+ Import Data+ </Button>+ </IntegrationCardFooter>+ </IntegrationCard>+ ))}+ </div>+ </div>++ <div>+ <h3 className="text-lg font-medium mb-4">Import History</h3>++ <div className="border rounded-lg">+ <Table>+ <TableHeader>+ <TableRow>+ <TableHead>Provider</TableHead>+ <TableHead>Created</TableHead>+ <TableHead>Status</TableHead>+ <TableHead>Progress</TableHead>+ <TableHead>Config</TableHead>+ <TableHead className="text-right">Actions</TableHead>+ </TableRow>+ </TableHeader>+ <TableBody>+ {!importsQuery.isLoading && imports.length === 0 && (+ <TableRow>+ <TableCell colSpan={6}>+ <FullPageEmptyState+ title="No imports yet"+ description="Your import history will appear here."+ />+ </TableCell>+ </TableRow>+ )}+ {importsQuery.isLoading &&+ [1, 2, 3, 4].map((index) => (+ <TableRow key={index}>+ <TableCell>+ <Skeleton className="h-4 w-3/5" />+ </TableCell>+ <TableCell>+ <Skeleton className="h-4 w-3/5" />+ </TableCell>+ <TableCell>+ <Skeleton className="h-4 w-3/5" />+ </TableCell>+ <TableCell>+ <Skeleton className="h-4 w-3/5" />+ </TableCell>+ <TableCell>+ <Skeleton className="h-4 w-3/5" />+ </TableCell>+ <TableCell className="text-right justify-end row">+ <Skeleton className="h-4 w-3/5" />+ </TableCell>+ </TableRow>+ ))}+ {imports.map((imp) => (+ <TableRow key={imp.id}>+ <TableCell className="font-medium capitalize">+ <div className="row gap-2 items-center">+ <div>{imp.config.provider}</div>+ <Badge variant="outline" className="uppercase">+ {imp.config.type}+ </Badge>+ </div>+ </TableCell>+ <TableCell>+ {formatDistanceToNow(new Date(imp.createdAt), {+ addSuffix: true,+ })}+ </TableCell>+ <TableCell>+ <div className="space-y-1">+ {getStatusBadge(imp.status, imp.errorMessage)}+ {imp.statusMessage && (+ <div className="text-xs text-muted-foreground truncate">+ {imp.statusMessage}+ </div>+ )}+ </div>+ </TableCell>+ <TableCell>+ {imp.totalEvents > 0 ? (+ <div className="space-y-1 font-mono">+ <div className="text-sm">+ {imp.processedEvents.toLocaleString()}+ {' / '}+ <Tooltiper+ content="Estimated number of events. Can be inaccurate depending on the provider."+ tooltipClassName="max-w-xs"+ >+ {imp.totalEvents.toLocaleString()}{' '}+ <InfoIcon className="w-4 h-4 inline-block relative -top-px" />+ </Tooltiper>+ </div>+ {imp.status === 'processing' && (+ <div className="w-full bg-secondary rounded-full h-1.5">+ <div+ className="bg-primary h-1.5 rounded-full transition-all"+ style={{+ width: `${Math.min(Math.round((imp.processedEvents / imp.totalEvents) * 100), 100)}%`,+ }}+ />+ </div>+ )}+ </div>+ ) : imp.totalEvents === -1 ? (+ <div className="font-mono text-sm">+ {imp.processedEvents.toLocaleString()}+ {' / '}+ N/A+ </div>+ ) : (+ '-'+ )}+ </TableCell>++ <TableCell>+ <Tooltiper+ content={+ <pre className="font-mono text-sm leading-normal whitespace-pre-wrap break-words">+ {JSON.stringify(imp.config, null, 2)}+ </pre>+ }+ tooltipClassName="max-w-xs"+ >+ <Badge>Config</Badge>+ </Tooltiper>+ </TableCell>+ <TableCell className="text-right space-x-2">+ {imp.status === 'failed' && (+ <Button+ variant="outline"+ size="sm"+ onClick={() => retryImport.mutate({ id: imp.id })}+ >+ Retry+ </Button>+ )}+ <Button+ variant="ghost"+ size="sm"+ onClick={() => deleteImport.mutate({ id: imp.id })}+ >+ Delete+ </Button>+ </TableCell>+ </TableRow>+ ))}+ </TableBody>+ </Table>+ </div>+ </div>+ </div>+ );+}diff --git a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsxindex 39dc0cb31..0c7f217a8 100644--- a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx+++ b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx@@ -42,6 +42,7 @@ function ProjectDashboard() {{ id: 'details', label: 'Details' },{ id: 'events', label: 'Events' },{ id: 'clients', label: 'Clients' },+ { id: 'imports', label: 'Imports' },];const handleTabChange = (tabId: string) => {diff --git a/apps/start/src/utils/math.ts b/apps/start/src/utils/math.tsindex e75c1964d..e41d4e1f5 100644--- a/apps/start/src/utils/math.ts+++ b/apps/start/src/utils/math.ts@@ -1 +1 @@-export * from '@openpanel/common/src/math';+export * from '@openpanel/common';diff --git a/apps/start/src/utils/op.ts b/apps/start/src/utils/op.tsindex 3e9cfee1e..0ba0fa951 100644--- a/apps/start/src/utils/op.ts+++ b/apps/start/src/utils/op.ts@@ -1,8 +1,20 @@import { OpenPanel } from '@openpanel/web';-export const op = new OpenPanel({- clientId: import.meta.env.VITE_OP_CLIENT_ID,- trackScreenViews: true,- trackOutgoingLinks: true,- trackAttributes: true,-});+const clientId = import.meta.env.VITE_OP_CLIENT_ID;++const createOpInstance = () => {+ if (!clientId || clientId === 'undefined') {+ return new Proxy({} as OpenPanel, {+ get: () => () => {},+ });+ }++ return new OpenPanel({+ clientId,+ trackScreenViews: true,+ trackOutgoingLinks: true,+ trackAttributes: true,+ });+};++export const op = createOpInstance();diff --git a/apps/start/src/utils/slug.ts b/apps/start/src/utils/slug.tsindex ebdad3b9f..e41d4e1f5 100644--- a/apps/start/src/utils/slug.ts+++ b/apps/start/src/utils/slug.ts@@ -1 +1 @@-export * from '@openpanel/common/src/slug';+export * from '@openpanel/common';diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfileindex 7f953f59a..b816beef7 100644--- a/apps/worker/Dockerfile+++ b/apps/worker/Dockerfile@@ -35,6 +35,7 @@ COPY packages/redis/package.json ./packages/redis/COPY packages/queue/package.json ./packages/queue/COPY packages/logger/package.json ./packages/logger/COPY packages/common/package.json ./packages/common/+COPY packages/importer/package.json ./packages/importer/COPY packages/constants/package.json ./packages/constants/COPY packages/validation/package.json ./packages/validation/COPY packages/integrations/package.json packages/integrations/@@ -80,9 +81,10 @@ COPY --from=build /app/packages/geo ./packages/geoCOPY --from=build /app/packages/json ./packages/jsonCOPY --from=build /app/packages/email ./packages/emailCOPY --from=build /app/packages/redis ./packages/redis-COPY --from=build /app/packages/logger ./packages/loggerCOPY --from=build /app/packages/queue ./packages/queue+COPY --from=build /app/packages/logger ./packages/loggerCOPY --from=build /app/packages/common ./packages/common+COPY --from=build /app/packages/importer ./packages/importerCOPY --from=build /app/packages/validation ./packages/validationCOPY --from=build /app/packages/integrations ./packages/integrationsCOPY --from=build /app/tooling/typescript ./tooling/typescriptdiff --git a/apps/worker/package.json b/apps/worker/package.jsonindex c9b548785..fc1b2d2e6 100644--- a/apps/worker/package.json+++ b/apps/worker/package.json@@ -8,8 +8,7 @@"testing": "WORKER_PORT=9999 pnpm dev","start": "node dist/index.js","build": "rm -rf dist && tsdown",- "typecheck": "tsc --noEmit",- "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./src/referrers/index.ts"+ "typecheck": "tsc --noEmit"},"dependencies": {"@bull-board/api": "6.13.1",@@ -20,6 +19,7 @@"@openpanel/integrations": "workspace:^","@openpanel/json": "workspace:*","@openpanel/logger": "workspace:*",+ "@openpanel/importer": "workspace:*","@openpanel/queue": "workspace:*","@openpanel/redis": "workspace:*","bullmq": "^5.8.7",@@ -38,7 +38,7 @@"@types/source-map-support": "^0.5.10","@types/sqlstring": "^2.3.2","@types/uuid": "^9.0.8",- "tsdown": "^0.14.2",+ "tsdown": "0.14.2","typescript": "catalog:"}}diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.tsindex d99e14b3f..d64bcf106 100644--- a/apps/worker/src/boot-workers.ts+++ b/apps/worker/src/boot-workers.ts@@ -5,6 +5,7 @@ import {type EventsQueuePayloadIncomingEvent,cronQueue,eventsGroupQueue,+ importQueue,miscQueue,notificationQueue,queueLogger,@@ -19,6 +20,7 @@ import { Worker as GroupWorker } from 'groupmq';import { cronJob } from './jobs/cron';import { eventsJob } from './jobs/events';import { incomingEventPure } from './jobs/events.incoming-event';+import { importJob } from './jobs/import';import { miscJob } from './jobs/misc';import { notificationJob } from './jobs/notification';import { sessionsJob } from './jobs/sessions';@@ -56,13 +58,18 @@ export async function bootWorkers() {workerOptions,);const miscWorker = new Worker(miscQueue.name, miscJob, workerOptions);+ const importWorker = new Worker(importQueue.name, importJob, {+ ...workerOptions,+ concurrency: Number.parseInt(process.env.IMPORT_JOB_CONCURRENCY || '1', 10),+ });const workers = [sessionsWorker,cronWorker,notificationWorker,miscWorker,- eventsGroupWorker,+ importWorker,+ // eventsGroupWorker,];workers.forEach((worker) => {@@ -148,7 +155,15 @@ export async function bootWorkers() {['uncaughtException', 'unhandledRejection', 'SIGTERM', 'SIGINT'].forEach((evt) => {process.on(evt, (code) => {- exitHandler(evt, code);+ if (process.env.NODE_ENV === 'production') {+ exitHandler(evt, code);+ } else {+ logger.info('Shutting down for development', {+ event: evt,+ code,+ });+ process.exit(0);+ }});},);diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.tsindex bb7793d7d..74bb3cb62 100644--- a/apps/worker/src/index.ts+++ b/apps/worker/src/index.ts@@ -5,6 +5,7 @@ import { createInitialSalts } from '@openpanel/db';import {cronQueue,eventsGroupQueue,+ importQueue,miscQueue,notificationQueue,sessionsQueue,@@ -31,13 +32,14 @@ async function start() {if (process.env.DISABLE_BULLBOARD === undefined) {const serverAdapter = new ExpressAdapter();serverAdapter.setBasePath('/');- ({+ createBullBoard({queues: [new BullBoardGroupMQAdapter(eventsGroupQueue) as any,new BullMQAdapter(sessionsQueue),new BullMQAdapter(cronQueue),new BullMQAdapter(notificationQueue),new BullMQAdapter(miscQueue),+ new BullMQAdapter(importQueue),],serverAdapter: serverAdapter,});diff --git a/apps/worker/src/jobs/cron.delete-projects.ts b/apps/worker/src/jobs/cron.delete-projects.tsindex abb8f6848..eab4f84c5 100644--- a/apps/worker/src/jobs/cron.delete-projects.ts+++ b/apps/worker/src/jobs/cron.delete-projects.ts@@ -54,7 +54,7 @@ export async function deleteProjects(job: Job<CronQueuePayload>) {await ch.command({query,clickhouse_settings: {- lightweight_deletes_sync: 0,+ lightweight_deletes_sync: '0',},});}diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex ecd28a550..bb3f3743e 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -1,12 +1,15 @@import { logger as baseLogger } from '@/utils/logger';-import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';import {createSessionEndJob,createSessionStart,getSessionEnd,} from '@/utils/session-handler';import { isSameDomain, parsePath } from '@openpanel/common';-import { parseUserAgent } from '@openpanel/common/server';+import {+ getReferrerWithQuery,+ parseReferrer,+ parseUserAgent,+} from '@openpanel/common/server';import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';import {checkNotificationRulesForEvent,@@ -15,10 +18,9 @@ import {} from '@openpanel/db';import type { ILogger } from '@openpanel/logger';import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';-import { getLock } from '@openpanel/redis';-import { DelayedError, type Job } from 'bullmq';-import { omit } from 'ramda';+import type { Job } from 'bullmq';import * as R from 'ramda';+import { omit } from 'ramda';import { v4 as uuid } from 'uuid';const GLOBAL_PROPERTIES = ['__path', '__referrer'];@@ -115,9 +117,9 @@ export async function incomingEventPure(latitude: geo.latitude,path,origin,- referrer: utmReferrer?.url || referrer?.url || '',+ referrer: referrer?.url || '',referrerName: utmReferrer?.name || referrer?.name || '',- referrerType: utmReferrer?.type || referrer?.type || '',+ referrerType: referrer?.type || utmReferrer?.type || '',os: uaInfo.os,osVersion: uaInfo.osVersion,browser: uaInfo.browser,diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.tsindex 292453d4c..d5d9dd0c5 100644--- a/apps/worker/src/jobs/events.incoming-events.test.ts+++ b/apps/worker/src/jobs/events.incoming-events.test.ts@@ -99,7 +99,7 @@ describe('incomingEvent', () => {origin: 'https://example.com',referrer: '',referrerName: '',- referrerType: 'unknown',+ referrerType: '',sdkName: jobData.payload.headers['openpanel-sdk-name'],sdkVersion: jobData.payload.headers['openpanel-sdk-version'],};@@ -207,7 +207,7 @@ describe('incomingEvent', () => {origin: 'https://example.com',referrer: '',referrerName: '',- referrerType: 'unknown',+ referrerType: '',sdkName: jobData.payload.headers['openpanel-sdk-name'],sdkVersion: jobData.payload.headers['openpanel-sdk-version'],};diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.tsnew file mode 100644index 000000000..0346d5832--- /dev/null+++ b/apps/worker/src/jobs/import.ts@@ -0,0 +1,332 @@+import {+ type IClickhouseEvent,+ type ImportSteps,+ type Prisma,+ backfillSessionsToProduction,+ createSessionsStartEndEvents,+ db,+ formatClickhouseDate,+ generateSessionIds,+ getImportDateBounds,+ getImportProgress,+ insertImportBatch,+ markImportComplete,+ moveImportsToProduction,+ updateImportStatus,+} from '@openpanel/db';+import { MixpanelProvider, UmamiProvider } from '@openpanel/importer';+import type { ILogger } from '@openpanel/logger';+import type { ImportQueuePayload } from '@openpanel/queue';+import type { Job } from 'bullmq';+import { logger } from '../utils/logger';++const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10);++/**+ * Yields control back to the event loop to prevent stalled jobs+ */+async function yieldToEventLoop(): Promise<void> {+ return new Promise((resolve) => {+ setTimeout(resolve, 100);+ });+}++export async function importJob(job: Job<ImportQueuePayload>) {+ const { importId } = job.data.payload;++ const record = await db.import.findUniqueOrThrow({+ where: { id: importId },+ include: {+ project: true,+ },+ });++ const jobLogger = logger.child({+ importId,+ config: record.config,+ });++ type ValidStep = Exclude<ImportSteps, 'failed' | 'completed'>;+ const steps: Record<ValidStep, number> = {+ loading: 0,+ generating_session_ids: 1,+ creating_sessions: 2,+ moving: 3,+ backfilling_sessions: 4,+ };++ jobLogger.info('Starting import job');+ const providerInstance = createProvider(record, jobLogger);++ try {+ // Check if this is a resume operation+ const isNewImport = record.currentStep === null;++ if (isNewImport) {+ await updateImportStatus(jobLogger, job, importId, {+ step: 'loading',+ });+ } else {+ jobLogger.info('Resuming import from previous state', {+ currentStep: record.currentStep,+ currentBatch: record.currentBatch,+ });+ }++ // Try to get a precomputed total for better progress reporting+ const totalEvents = await providerInstance+ .getTotalEventsCount()+ .catch(() => -1);+ let processedEvents = record.processedEvents;++ const resumeLoadingFrom =+ (record.currentStep === 'loading' && record.currentBatch) || undefined;++ const resumeGeneratingSessionIdsFrom =+ (record.currentStep === 'generating_session_ids' &&+ record.currentBatch) ||+ undefined;++ const resumeCreatingSessionsFrom =+ (record.currentStep === 'creating_sessions' && record.currentBatch) ||+ undefined;++ const resumeMovingFrom =+ (record.currentStep === 'moving' && record.currentBatch) || undefined;++ const resumeBackfillingSessionsFrom =+ (record.currentStep === 'backfilling_sessions' && record.currentBatch) ||+ undefined;++ // Example:+ // shouldRunStep(0) // currStep = 2 (should not run)+ // shouldRunStep(1) // currStep = 2 (should not run)+ // shouldRunStep(2) // currStep = 2 (should run)+ // shouldRunStep(3) // currStep = 2 (should run)+ const shouldRunStep = (step: ValidStep) => {+ if (isNewImport) {+ return true;+ }++ const stepToRunIndex = steps[step];+ const currentStepIndex = steps[record.currentStep as ValidStep];+ return stepToRunIndex >= currentStepIndex;+ };++ async function whileBounds(+ from: string | undefined,+ callback: (from: string, to: string) => Promise<void>,+ ) {+ const bounds = await getImportDateBounds(importId, from);+ if (bounds.min && bounds.max) {+ const start = new Date(bounds.min);+ const end = new Date(bounds.max);+ let cursor = new Date(start);+ while (cursor < end) {+ const next = new Date(cursor);+ next.setDate(next.getDate() + 1);+ await callback(+ formatClickhouseDate(cursor, true),+ formatClickhouseDate(next, true),+ );+ cursor = next;++ // Yield control back to event loop after processing each day+ await yieldToEventLoop();+ }+ }+ }++ // Phase 1: Fetch & Transform - Process events in batches+ if (shouldRunStep('loading')) {+ const eventBatch: any = [];+ for await (const rawEvent of providerInstance.parseSource(+ resumeLoadingFrom,+ )) {+ // Validate event+ if (+ !providerInstance.validate(+ // @ts-expect-error+ rawEvent,+ )+ ) {+ jobLogger.warn('Skipping invalid event', { rawEvent });+ continue;+ }++ eventBatch.push(rawEvent);++ // Process batch when it reaches the batch size+ if (eventBatch.length >= BATCH_SIZE) {+ jobLogger.info('Processing batch', { batchSize: eventBatch.length });++ const transformedEvents: IClickhouseEvent[] = eventBatch.map(+ (+ // @ts-expect-error+ event,+ ) => providerInstance!.transformEvent(event),+ );++ await insertImportBatch(transformedEvents, importId);++ processedEvents += eventBatch.length;+ eventBatch.length = 0;++ const createdAt = new Date(transformedEvents[0]?.created_at || '')+ .toISOString()+ .split('T')[0];++ await updateImportStatus(jobLogger, job, importId, {+ step: 'loading',+ batch: createdAt,+ totalEvents,+ processedEvents,+ });++ // Yield control back to event loop after processing each batch+ await yieldToEventLoop();+ }+ }++ // Process remaining events in the last batch+ if (eventBatch.length > 0) {+ const transformedEvents = eventBatch.map(+ (+ // @ts-expect-error+ event,+ ) => providerInstance!.transformEvent(event),+ );++ await insertImportBatch(transformedEvents, importId);++ processedEvents += eventBatch.length;+ eventBatch.length = 0;++ const createdAt = new Date(transformedEvents[0]?.created_at || '')+ .toISOString()+ .split('T')[0];++ await updateImportStatus(jobLogger, job, importId, {+ step: 'loading',+ batch: createdAt,+ });++ // Yield control back to event loop after processing final batch+ await yieldToEventLoop();+ }+ }++ // Phase 2: Generate session IDs if provider requires it+ if (+ shouldRunStep('generating_session_ids') &&+ providerInstance.shouldGenerateSessionIds()+ ) {+ await whileBounds(resumeGeneratingSessionIdsFrom, async (from) => {+ console.log('Generating session IDs', { from });+ await generateSessionIds(importId, from);+ await updateImportStatus(jobLogger, job, importId, {+ step: 'generating_session_ids',+ batch: from,+ });++ // Yield control back to event loop after processing each day+ await yieldToEventLoop();+ });++ jobLogger.info('Session ID generation complete');+ }++ // Phase 3-5: Process in daily batches for robustness++ if (shouldRunStep('creating_sessions')) {+ await whileBounds(resumeCreatingSessionsFrom, async (from) => {+ await createSessionsStartEndEvents(importId, from);+ await updateImportStatus(jobLogger, job, importId, {+ step: 'creating_sessions',+ batch: from,+ });++ // Yield control back to event loop after processing each day+ await yieldToEventLoop();+ });+ }++ if (shouldRunStep('moving')) {+ await whileBounds(resumeMovingFrom, async (from) => {+ await moveImportsToProduction(importId, from);+ await updateImportStatus(jobLogger, job, importId, {+ step: 'moving',+ batch: from,+ });++ // Yield control back to event loop after processing each day+ await yieldToEventLoop();+ });+ }++ if (shouldRunStep('backfilling_sessions')) {+ await whileBounds(resumeBackfillingSessionsFrom, async (from) => {+ await backfillSessionsToProduction(importId, from);+ await updateImportStatus(jobLogger, job, importId, {+ step: 'backfilling_sessions',+ batch: from,+ });++ // Yield control back to event loop after processing each day+ await yieldToEventLoop();+ });+ }++ await markImportComplete(importId);+ await updateImportStatus(jobLogger, job, importId, {+ step: 'completed',+ });+ jobLogger.info('Import marked as complete');++ // Get final progress+ const finalProgress = await getImportProgress(importId);++ jobLogger.info('Import job completed successfully', {+ totalEvents: finalProgress.totalEvents,+ insertedEvents: finalProgress.insertedEvents,+ status: finalProgress.status,+ });++ return {+ success: true,+ totalEvents: finalProgress.totalEvents,+ processedEvents: finalProgress.insertedEvents,+ };+ } catch (error) {+ jobLogger.error('Import job failed', { error });++ // Mark import as failed+ try {+ const errorMsg = error instanceof Error ? error.message : 'Unknown error';+ await updateImportStatus(jobLogger, job, importId, {+ step: 'failed',+ errorMessage: errorMsg,+ });+ jobLogger.warn('Import marked as failed', { error: errorMsg });+ } catch (markError) {+ jobLogger.error('Failed to mark import as failed', { error, markError });+ }++ throw error;+ }+}++function createProvider(+ record: Prisma.ImportGetPayload<{ include: { project: true } }>,+ jobLogger: ILogger,+) {+ const config = record.config;+ switch (config.provider) {+ case 'umami':+ return new UmamiProvider(record.projectId, config, jobLogger);+ case 'mixpanel':+ return new MixpanelProvider(record.projectId, config, jobLogger);+ default:+ throw new Error(`Unknown provider: ${config.provider}`);+ }+}diff --git a/package.json b/package.jsonindex 0860945fd..c29855854 100644--- a/package.json+++ b/package.json@@ -9,7 +9,7 @@"scripts": {"test": "vitest run","gen:bots": "pnpm -r --filter api gen:bots",- "gen:referrers": "pnpm -r --filter worker gen:referrers",+ "gen:referrers": "pnpm -r --filter common gen:referrers","dock:up": "docker compose up -d","dock:down": "docker compose down","dock:ch": "docker compose exec -it op-ch clickhouse-client -d openpanel",diff --git a/packages/cli/package.json b/packages/cli/package.jsondeleted file mode 100644index d352ce721..000000000--- a/packages/cli/package.json+++ /dev/null@@ -1,33 +0,0 @@-{- "name": "@openpanel/cli",- "version": "0.0.1-beta",- "type": "module",- "module": "index.ts",- "bin": {- "openpanel": "dist/bin/cli.js"- },- "scripts": {- "build": "rm -rf dist && tsup",- "typecheck": "tsc --noEmit"- },- "dependencies": {- "@openpanel/common": "workspace:*",- "arg": "^5.0.2",- "glob": "^10.4.3",- "inquirer": "^9.3.5",- "p-limit": "^6.1.0",- "progress": "^2.0.3",- "ramda": "^0.29.1",- "zod": "catalog:"- },- "devDependencies": {- "@openpanel/db": "workspace:^",- "@openpanel/sdk": "workspace:*",- "@openpanel/tsconfig": "workspace:*",- "@types/node": "catalog:",- "@types/progress": "^2.0.7",- "@types/ramda": "^0.30.1",- "tsup": "^7.2.0",- "typescript": "catalog:"- }-}diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.tsdeleted file mode 100644index 872aa7311..000000000--- a/packages/cli/src/cli.ts+++ /dev/null@@ -1,24 +0,0 @@-import arg from 'arg';--import importer from './importer';--function cli() {- const args = arg(- {- '--help': Boolean,- },- {- permissive: true,- },- );-- const [command] = args._;-- switch (command) {- case 'import': {- return importer();- }- }-}--cli();diff --git a/packages/cli/src/importer/importer.ts b/packages/cli/src/importer/importer.tsdeleted file mode 100644index 27be17af6..000000000--- a/packages/cli/src/importer/importer.ts+++ /dev/null@@ -1,467 +0,0 @@-import { randomUUID } from 'node:crypto';-import fs from 'node:fs';-import os from 'node:os';-import path from 'node:path';-import readline from 'node:readline';-import zlib from 'node:zlib';-import Progress from 'progress';-import { assocPath, prop, uniqBy } from 'ramda';--import { isSameDomain, parsePath } from '@openpanel/common';-import type { IImportedEvent } from '@openpanel/db';--const BATCH_SIZE = 30_000;-const SLEEP_TIME = 20;-const MAX_CONCURRENT_REQUESTS = 8;--type IMixpanelEvent = {- event: string;- properties: {- [key: string]: unknown;- time: number;- $current_url?: string;- distinct_id?: string;- $device_id?: string;- country_code?: string;- $region?: string;- $city?: string;- $os?: string;- $browser?: string;- $browser_version?: string;- $initial_referrer?: string;- $search_engine?: string;- };-};--function stripMixpanelProperties(obj: Record<string, unknown>) {- return Object.fromEntries(- Object.entries(obj).filter(- ([key]) =>- !key.match(/^(\$|mp_)/) && !['time', 'distinct_id'].includes(key),- ),- );-}--async function* parseJsonStream(- fileStream: fs.ReadStream,-): AsyncGenerator<any, void, unknown> {- const rl = readline.createInterface({- input: fileStream,- crlfDelay: Number.POSITIVE_INFINITY,- });-- let buffer = '';- let bracketCount = 0;-- for await (const line of rl) {- buffer += line;- bracketCount +=- (line.match(/{/g) || []).length - (line.match(/}/g) || []).length;-- if (bracketCount === 0 && buffer.trim()) {- try {- const json = JSON.parse(buffer);- yield json;- } catch (error) {- console.log('Warning: Failed to parse JSON');- console.log('Buffer:', buffer);- }- buffer = '';- }- }-- if (buffer.trim()) {- try {- const json = JSON.parse(buffer);- yield json;- } catch (error) {- console.log('Warning: Failed to parse remaining JSON');- console.log('Buffer:', buffer);- }- }-}--interface Session {- start: number;- end: number;- profileId?: string;- deviceId?: string;- sessionId: string;- firstEvent?: IImportedEvent;- lastEvent?: IImportedEvent;- events: IImportedEvent[];-}--function generateSessionEvents(events: IImportedEvent[]): Session[] {- let sessionList: Session[] = [];- const lastSessionByDevice: Record<string, Session> = {};- const lastSessionByProfile: Record<string, Session> = {};- const thirtyMinutes = 30 * 60 * 1000;-- events.sort(- (a, b) =>- new Date(a.created_at).getTime() - new Date(b.created_at).getTime(),- );-- for (const event of events) {- const eventTime = new Date(event.created_at).getTime();- let deviceSession = event.device_id- ? lastSessionByDevice[event.device_id]- : undefined;- let profileSession = event.profile_id- ? lastSessionByProfile[event.profile_id]- : undefined;-- if (- event.device_id &&- event.device_id !== event.profile_id &&- (!deviceSession || eventTime > deviceSession.end + thirtyMinutes)- ) {- deviceSession = {- start: eventTime,- end: eventTime,- deviceId: event.device_id,- sessionId: randomUUID(),- firstEvent: event,- events: [event],- };- lastSessionByDevice[event.device_id] = deviceSession;- sessionList.push(deviceSession);- } else if (deviceSession) {- deviceSession.end = eventTime;- deviceSession.lastEvent = event;- deviceSession.events.push(event);- }-- if (- event.profile_id &&- event.device_id !== event.profile_id &&- (!profileSession || eventTime > profileSession.end + thirtyMinutes)- ) {- profileSession = {- start: eventTime,- end: eventTime,- profileId: event.profile_id,- sessionId: randomUUID(),- firstEvent: event,- events: [event],- };- lastSessionByProfile[event.profile_id] = profileSession;- sessionList.push(profileSession);- } else if (profileSession) {- profileSession.end = eventTime;- profileSession.lastEvent = event;- profileSession.events.push(event);- }-- if (- deviceSession &&- profileSession &&- deviceSession.sessionId !== profileSession.sessionId- ) {- const unifiedSession = {- ...deviceSession,- ...profileSession,- events: [...deviceSession.events, ...profileSession.events],- start: Math.min(deviceSession.start, profileSession.start),- end: Math.max(deviceSession.end, profileSession.end),- sessionId: deviceSession.sessionId,- };- lastSessionByDevice[event.device_id] = unifiedSession;- lastSessionByProfile[event.profile_id] = unifiedSession;- sessionList = sessionList.filter(- (session) =>- session.sessionId !== deviceSession?.sessionId &&- session.sessionId !== profileSession?.sessionId,- );- sessionList.push(unifiedSession);- }- }-- return sessionList;-}--function createEventObject(event: IMixpanelEvent): IImportedEvent {- const getReferrer = (referrer: string | undefined) => {- if (!referrer) {- return '';- }-- if (referrer === '$direct') {- return '';- }-- if (isSameDomain(referrer, event.properties.$current_url)) {- return '';- }-- return referrer;- };- const url = parsePath(event.properties.$current_url);- return {- profile_id: event.properties.distinct_id- ? String(event.properties.distinct_id).replace(/^\$device:/, '')- : (event.properties.$device_id ?? ''),- name: event.event,- created_at: new Date(event.properties.time * 1000).toISOString(),- properties: {- ...stripMixpanelProperties(event.properties),- ...(event.properties.$current_url- ? {- __query: url.query,- __hash: url.hash,- }- : {}),- },- country: event.properties.country_code ?? '',- region: event.properties.$region ?? '',- city: event.properties.$city ?? '',- os: event.properties.$os ?? '',- browser: event.properties.$browser ?? '',- browser_version: event.properties.$browser_version- ? String(event.properties.$browser_version)- : '',- referrer: getReferrer(event.properties.$initial_referrer),- referrer_type: event.properties.$search_engine ? 'search' : '',- referrer_name: event.properties.$search_engine ?? '',- device_id: event.properties.$device_id ?? '',- session_id: '',- project_id: '',- path: url.path,- origin: url.origin,- os_version: '',- model: '',- longitude: null,- latitude: null,- id: randomUUID(),- duration: 0,- device: event.properties.$current_url ? '' : 'server',- brand: '',- sdk_name: '',- sdk_version: '',- };-}--function isMixpanelEvent(event: any): event is IMixpanelEvent {- return (- typeof event === 'object' &&- event !== null &&- typeof event?.event === 'string' &&- typeof event?.properties === 'object' &&- event?.properties !== null &&- typeof event?.properties.time === 'number'- );-}--async function processFile(file: string): Promise<IImportedEvent[]> {- const fileStream = fs.createReadStream(file);- const events: IImportedEvent[] = [];- for await (const event of parseJsonStream(fileStream)) {- if (Array.isArray(event)) {- for (const item of event) {- if (isMixpanelEvent(item)) {- events.push(createEventObject(item));- } else {- console.log('Not a Mixpanel event', item);- }- }- } else {- if (isMixpanelEvent(event)) {- events.push(createEventObject(event));- } else {- console.log('Not a Mixpanel event', event);- }- }- }- return events;-}--function processEvents(events: IImportedEvent[]): IImportedEvent[] {- const sessions = generateSessionEvents(events);- const processedEvents = sessions.flatMap((session) =>- [- session.firstEvent && {- ...session.firstEvent,- id: randomUUID(),- created_at: new Date(- new Date(session.firstEvent.created_at).getTime() - 1000,- ).toISOString(),- session_id: session.sessionId,- name: 'session_start',- },- ...uniqBy(- prop('id'),- session.events.map((event) =>- assocPath(['session_id'], session.sessionId, event),- ),- ),- session.lastEvent && {- ...session.lastEvent,- id: randomUUID(),- created_at: new Date(- new Date(session.lastEvent.created_at).getTime() + 1000,- ).toISOString(),- session_id: session.sessionId,- name: 'session_end',- },- ].filter((item): item is IImportedEvent => !!item),- );-- return [- ...processedEvents,- ...events.filter((event) => {- return !event.profile_id && !event.device_id;- }),- ];-}--async function sendBatchToAPI(- batch: IImportedEvent[],- {- apiUrl,- clientId,- clientSecret,- }: {- apiUrl: string;- clientId: string;- clientSecret: string;- },-) {- async function request() {- const res = await fetch(`${apiUrl}/import/events`, {- method: 'POST',- headers: {- 'Content-Encoding': 'gzip',- 'Content-Type': 'application/json',- 'openpanel-client-id': clientId,- 'openpanel-client-secret': clientSecret,- },- body: Buffer.from(zlib.gzipSync(JSON.stringify(batch))),- });- if (!res.ok) {- throw new Error(`Failed to send batch: ${await res.text()}`);- }- await new Promise((resolve) => setTimeout(resolve, SLEEP_TIME));- }-- try {- await request();- } catch (e) {- console.log('Error sending batch, retrying...');- await new Promise((resolve) => setTimeout(resolve, 1000));- try {- await request();- } catch (e) {- console.log('Error sending batch, skipping...');- fs.writeFileSync(- path.join(- os.tmpdir(),- `openpanel/failed-import-batch-${batch[0]?.created_at ? new Date(batch[0]?.created_at).toISOString() : Date.now()}.json`,- ),- JSON.stringify(batch, null, 2),- );- }- }-}--async function processFiles({- files,- apiUrl,- clientId,- clientSecret,-}: {- files: string[];- apiUrl: string;- clientId: string;- clientSecret: string;-}) {- const progress = new Progress(- 'Processing (:current/:total) :file [:bar] :percent | :savedEvents saved events | :status',- {- total: files.length,- width: 20,- },- );- let savedEvents = 0;- let currentBatch: IImportedEvent[] = [];- let apiBatching = [];-- for (const file of files) {- progress.tick({- file,- savedEvents,- status: 'reading file',- });- const events = await processFile(file);- progress.render({- file,- savedEvents,- status: 'processing events',- });- const processedEvents = processEvents(events);- for (const event of processedEvents) {- currentBatch.push(event);- if (currentBatch.length >= BATCH_SIZE) {- apiBatching.push(currentBatch);- savedEvents += currentBatch.length;- progress.render({ file, savedEvents, status: 'saving events' });- currentBatch = [];- }-- if (apiBatching.length >= MAX_CONCURRENT_REQUESTS) {- await Promise.all(- apiBatching.map((batch) =>- sendBatchToAPI(batch, {- apiUrl,- clientId,- clientSecret,- }),- ),- );- apiBatching = [];- }- }- }-- if (currentBatch.length > 0) {- await sendBatchToAPI(currentBatch, {- apiUrl,- clientId,- clientSecret,- });- savedEvents += currentBatch.length;- progress.render({ file: 'Complete', savedEvents, status: 'Complete' });- }-}--export async function importFiles({- files,- apiUrl,- clientId,- clientSecret,-}: {- files: string[];- apiUrl: string;- clientId: string;- clientSecret: string;-}) {- if (files.length === 0) {- console.log('No files found');- return;- }-- console.log(`Found ${files.length} files to process`);-- const startTime = Date.now();- await processFiles({- files,- apiUrl,- clientId,- clientSecret,- });- const endTime = Date.now();-- console.log(- `\nProcessing completed in ${(endTime - startTime) / 1000} seconds`,- );-}diff --git a/packages/cli/src/importer/index.ts b/packages/cli/src/importer/index.tsdeleted file mode 100644index fcd1f196b..000000000--- a/packages/cli/src/importer/index.ts+++ /dev/null@@ -1,59 +0,0 @@-import path from 'node:path';-import arg from 'arg';-import { glob } from 'glob';--import { importFiles } from './importer';--export default async function importer() {- const args = arg(- {- '--glob': String,- '--api-url': String,- '--client-id': String,- '--client-secret': String,- '--dry-run': Boolean,- '--from': Number,- '--to': Number,- },- {- permissive: true,- },- );-- if (!args['--glob']) {- throw new Error('Missing --glob argument');- }-- if (!args['--client-id']) {- throw new Error('Missing --client-id argument');- }-- if (!args['--client-secret']) {- throw new Error('Missing --client-secret argument');- }-- const cwd = process.cwd();-- const fileMatcher = path.resolve(cwd, args['--glob']);- const allFiles = await glob([fileMatcher], { root: '/' });- allFiles.sort((a, b) => a.localeCompare(b));-- const files = allFiles.slice(- args['--from'] ?? 0,- args['--to'] ?? Number.MAX_SAFE_INTEGER,- );-- if (args['--dry-run']) {- files.forEach((file, index) => {- console.log(`Would import (index: ${index}): ${file}`);- });- return;- }-- return importFiles({- files,- clientId: args['--client-id'],- clientSecret: args['--client-secret'],- apiUrl: args['--api-url'] ?? 'https://api.openpanel.dev',- });-}diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.jsondeleted file mode 100644index fa4341f12..000000000--- a/packages/cli/tsconfig.json+++ /dev/null@@ -1,8 +0,0 @@-{- "extends": "@openpanel/tsconfig/base.json",- "compilerOptions": {- "incremental": false,- "outDir": "dist"- },- "exclude": ["dist"]-}diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.tsdeleted file mode 100644index 628edb0ae..000000000--- a/packages/cli/tsup.config.ts+++ /dev/null@@ -1,11 +0,0 @@-import { defineConfig } from 'tsup';--export default defineConfig({- entry: ['src/cli.ts'],- format: ['cjs', 'esm'],- dts: true,- splitting: false,- sourcemap: false,- clean: true,- minify: true,-});diff --git a/packages/common/package.json b/packages/common/package.jsonindex a8c914f7c..5eaab8d25 100644--- a/packages/common/package.json+++ b/packages/common/package.json@@ -3,9 +3,14 @@"version": "0.0.1","type": "module","main": "index.ts",+ "exports": {+ ".": "./index.ts",+ "./server": "./server/index.ts"+ },"scripts": {"test": "vitest",- "typecheck": "tsc --noEmit"+ "typecheck": "tsc --noEmit",+ "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./server/referrers/index.ts"},"dependencies": {"@openpanel/constants": "workspace:*",diff --git a/packages/common/scripts/get-referrers.ts b/packages/common/scripts/get-referrers.tsnew file mode 100644index 000000000..27302eb79--- /dev/null+++ b/packages/common/scripts/get-referrers.ts@@ -0,0 +1,96 @@+import fs from 'node:fs';+import path from 'node:path';+import { dirname } from 'node:path';+import { fileURLToPath } from 'node:url';++const __filename = fileURLToPath(import.meta.url);+const __dirname = dirname(__filename);++// extras+const extraReferrers = {+ 'zoom.us': { type: 'social', name: 'Zoom' },+ 'apple.com': { type: 'tech', name: 'Apple' },+ 'adobe.com': { type: 'tech', name: 'Adobe' },+ 'figma.com': { type: 'tech', name: 'Figma' },+ 'wix.com': { type: 'commerce', name: 'Wix' },+ 'gmail.com': { type: 'email', name: 'Gmail' },+ 'notion.so': { type: 'tech', name: 'Notion' },+ 'ebay.com': { type: 'commerce', name: 'eBay' },+ 'github.com': { type: 'tech', name: 'GitHub' },+ 'gitlab.com': { type: 'tech', name: 'GitLab' },+ 'slack.com': { type: 'social', name: 'Slack' },+ 'etsy.com': { type: 'commerce', name: 'Etsy' },+ 'bsky.app': { type: 'social', name: 'Bluesky' },+ 'twitch.tv': { type: 'content', name: 'Twitch' },+ 'dropbox.com': { type: 'tech', name: 'Dropbox' },+ 'outlook.com': { type: 'email', name: 'Outlook' },+ 'medium.com': { type: 'content', name: 'Medium' },+ 'paypal.com': { type: 'commerce', name: 'PayPal' },+ 'discord.com': { type: 'social', name: 'Discord' },+ 'stripe.com': { type: 'commerce', name: 'Stripe' },+ 'spotify.com': { type: 'content', name: 'Spotify' },+ 'netflix.com': { type: 'content', name: 'Netflix' },+ 'whatsapp.com': { type: 'social', name: 'WhatsApp' },+ 'shopify.com': { type: 'commerce', name: 'Shopify' },+ 'microsoft.com': { type: 'tech', name: 'Microsoft' },+ 'alibaba.com': { type: 'commerce', name: 'Alibaba' },+ 'telegram.org': { type: 'social', name: 'Telegram' },+ 'substack.com': { type: 'content', name: 'Substack' },+ 'salesforce.com': { type: 'tech', name: 'Salesforce' },+ 'instagram.com': { type: 'social', name: 'Instagram' },+ 'wikipedia.org': { type: 'content', name: 'Wikipedia' },+ 'mastodon.social': { type: 'social', name: 'Mastodon' },+ 'office.com': { type: 'tech', name: 'Microsoft Office' },+ 'squarespace.com': { type: 'commerce', name: 'Squarespace' },+ 'stackoverflow.com': { type: 'tech', name: 'Stack Overflow' },+ 'teams.microsoft.com': { type: 'social', name: 'Microsoft Teams' },+};++function transform(data: any) {+ const obj: Record<string, unknown> = {};+ for (const type in data) {+ for (const name in data[type]) {+ const domains = data[type][name].domains ?? [];+ for (const domain of domains) {+ obj[domain] = {+ type,+ name,+ };+ }+ }+ }++ return obj;+}++async function main() {+ // Get document, or throw exception on error+ try {+ const data = await fetch(+ 'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json',+ ).then((res) => res.json());++ fs.writeFileSync(+ path.resolve(__dirname, '../../worker/src/referrers/index.ts'),+ [+ '// This file is generated by the script get-referrers.ts',+ '',+ '// The data is fetch from snowplow-referer-parser https://github.com/snowplow-referer-parser/referer-parser',+ `// The orginal referers.yml is based on Piwik's SearchEngines.php and Socials.php, copyright 2012 Matthieu Aubry and available under the GNU General Public License v3.`,+ '',+ `const referrers: Record<string, { type: string, name: string }> = ${JSON.stringify(+ {+ ...transform(data),+ ...extraReferrers,+ },+ )} as const;`,+ 'export default referrers;',+ ].join('\n'),+ 'utf-8',+ );+ } catch (e) {+ console.log(e);+ }+}++main();diff --git a/packages/common/server/index.ts b/packages/common/server/index.tsindex 3bb5bb376..cb5409062 100644--- a/packages/common/server/index.ts+++ b/packages/common/server/index.ts@@ -1,3 +1,5 @@export * from './crypto';export * from './profileId';export * from './parser-user-agent';+export * from './parse-referrer';+export * from './id';diff --git a/apps/worker/src/utils/parse-referrer.test.ts b/packages/common/server/parse-referrer.test.tssimilarity index 93%rename from apps/worker/src/utils/parse-referrer.test.tsrename to packages/common/server/parse-referrer.test.tsindex 66d17c54b..009118b63 100644--- a/apps/worker/src/utils/parse-referrer.test.ts+++ b/packages/common/server/parse-referrer.test.ts@@ -5,13 +5,13 @@ describe('parseReferrer', () => {it('should handle undefined or empty URLs', () => {expect(parseReferrer(undefined)).toEqual({name: '',- type: 'unknown',+ type: '',url: '',});expect(parseReferrer('')).toEqual({name: '',- type: 'unknown',+ type: '',url: '',});});@@ -41,7 +41,7 @@ describe('parseReferrer', () => {it('should handle unknown referrers', () => {expect(parseReferrer('https://unknown-site.com')).toEqual({name: '',- type: 'unknown',+ type: '',url: 'https://unknown-site.com',});});@@ -49,7 +49,7 @@ describe('parseReferrer', () => {it('should handle invalid URLs', () => {expect(parseReferrer('not-a-url')).toEqual({name: '',- type: 'unknown',+ type: '',url: 'not-a-url',});});@@ -64,7 +64,7 @@ describe('getReferrerWithQuery', () => {it('should parse utm_source parameter', () => {expect(getReferrerWithQuery({ utm_source: 'google' })).toEqual({name: 'Google',- type: 'unknown',+ type: 'search',url: '',});});@@ -88,7 +88,7 @@ describe('getReferrerWithQuery', () => {it('should handle case-insensitive matching', () => {expect(getReferrerWithQuery({ utm_source: 'GoOgLe' })).toEqual({name: 'Google',- type: 'unknown',+ type: 'search',url: '',});});@@ -96,7 +96,7 @@ describe('getReferrerWithQuery', () => {it('should handle unknown sources', () => {expect(getReferrerWithQuery({ utm_source: 'unknown-source' })).toEqual({name: 'unknown-source',- type: 'unknown',+ type: '',url: '',});});@@ -110,7 +110,7 @@ describe('getReferrerWithQuery', () => {}),).toEqual({name: 'Google',- type: 'unknown',+ type: 'search',url: '',});});diff --git a/apps/worker/src/utils/parse-referrer.ts b/packages/common/server/parse-referrer.tssimilarity index 68%rename from apps/worker/src/utils/parse-referrer.tsrename to packages/common/server/parse-referrer.tsindex 6e4d86af1..3fe35c5ee 100644--- a/apps/worker/src/utils/parse-referrer.ts+++ b/packages/common/server/parse-referrer.ts@@ -1,6 +1,6 @@-import { stripTrailingSlash } from '@openpanel/common';+import { stripTrailingSlash } from '../src/string';-import referrers from '../referrers';+import referrers from './referrers';function getHostname(url: string | undefined) {if (!url) {@@ -20,7 +20,7 @@ export function parseReferrer(url: string | undefined) {return {name: match?.name ?? '',- type: match?.type ?? 'unknown',+ type: match?.type ?? '',url: stripTrailingSlash(url ?? ''),};}@@ -32,16 +32,23 @@ export function getReferrerWithQuery(return null;}- const source = query.utm_source ?? query.ref ?? query.utm_referrer ?? '';+ const source = (+ query.utm_source ??+ query.ref ??+ query.utm_referrer ??+ ''+ ).toLowerCase();if (source === '') {return null;}const match =+ referrers[source] ||+ referrers[`${source}.com`] ||Object.values(referrers).find(- (referrer) => referrer.name.toLowerCase() === source.toLowerCase(),- ) || referrers[source];+ (referrer) => referrer.name.toLowerCase() === source,+ );if (match) {return {@@ -53,7 +60,7 @@ export function getReferrerWithQuery(return {name: source,- type: 'unknown',+ type: '',url: '',};}diff --git a/packages/common/server/parser-user-agent.ts b/packages/common/server/parser-user-agent.tsindex b84b9768c..9238de3b8 100644--- a/packages/common/server/parser-user-agent.ts+++ b/packages/common/server/parser-user-agent.ts@@ -68,6 +68,7 @@ const parse = (ua: string): UAParser.IResult => {return res;};+export type UserAgentInfo = ReturnType<typeof parseUserAgent>;export function parseUserAgent(ua?: string | null,overrides?: Record<string, unknown>,@@ -80,13 +81,35 @@ export function parseUserAgent(}return {- os: overrides?.__os || res.os.name,- osVersion: overrides?.__osVersion || res.os.version,- browser: overrides?.__browser || res.browser.name,- browserVersion: overrides?.__browserVersion || res.browser.version,- device: overrides?.__device || res.device.type || getDevice(ua),- brand: overrides?.__brand || res.device.vendor,- model: overrides?.__model || res.device.model,+ os:+ typeof overrides?.__os === 'string' && overrides?.__os+ ? overrides?.__os+ : res.os.name,+ osVersion:+ typeof overrides?.__osVersion === 'string' && overrides?.__osVersion+ ? overrides?.__osVersion+ : res.os.version,+ browser:+ typeof overrides?.__browser === 'string' && overrides?.__browser+ ? overrides?.__browser+ : res.browser.name,+ browserVersion:+ typeof overrides?.__browserVersion === 'string' &&+ overrides?.__browserVersion+ ? overrides?.__browserVersion+ : res.browser.version,+ device:+ typeof overrides?.__device === 'string' && overrides?.__device+ ? overrides?.__device+ : res.device.type || getDevice(ua),+ brand:+ typeof overrides?.__brand === 'string' && overrides?.__brand+ ? overrides?.__brand+ : res.device.vendor,+ model:+ typeof overrides?.__model === 'string' && overrides?.__model+ ? overrides?.__model+ : res.device.model,isServer: false,} as const;}diff --git a/apps/worker/src/referrers/index.ts b/packages/common/server/referrers/index.tssimilarity index 100%rename from apps/worker/src/referrers/index.tsrename to packages/common/server/referrers/index.tsdiff --git a/apps/worker/src/referrers/referrers.readme.md b/packages/common/server/referrers/referrers.readme.mdsimilarity index 100%rename from apps/worker/src/referrers/referrers.readme.mdrename to packages/common/server/referrers/referrers.readme.mddiff --git a/packages/common/src/object.test.ts b/packages/common/src/object.test.tsnew file mode 100644index 000000000..d33dcb789--- /dev/null+++ b/packages/common/src/object.test.ts@@ -0,0 +1,48 @@+import { describe, expect, it } from 'vitest';+import { toDots } from './object';++describe('toDots', () => {+ it('should convert an object to a dot object', () => {+ const obj = {+ a: 1,+ b: 2,+ array: ['1', '2', '3'],+ arrayWithObjects: [{ a: 1 }, { b: 2 }, { c: 3 }],+ objectWithArrays: { a: [1, 2, 3] },+ null: null,+ undefined: undefined,+ empty: '',+ jsonString: '{"a": 1, "b": 2}',+ };+ expect(toDots(obj)).toEqual({+ a: '1',+ b: '2',+ 'array.0': '1',+ 'array.1': '2',+ 'array.2': '3',+ 'arrayWithObjects.0.a': '1',+ 'arrayWithObjects.1.b': '2',+ 'arrayWithObjects.2.c': '3',+ 'objectWithArrays.a.0': '1',+ 'objectWithArrays.a.1': '2',+ 'objectWithArrays.a.2': '3',+ 'jsonString.a': '1',+ 'jsonString.b': '2',+ });+ });++ it('should handle malformed JSON strings gracefully', () => {+ const obj = {+ validJson: '{"key":"value"}',+ malformedJson: '{"key":"unterminated string',+ startsWithBrace: '{not json at all',+ startsWithBracket: '[also not json',+ regularString: 'normal string',+ };++ expect(toDots(obj)).toEqual({+ 'validJson.key': 'value',+ regularString: 'normal string',+ });+ });+});diff --git a/packages/common/src/object.ts b/packages/common/src/object.tsindex dd3a237e0..ebcb0917c 100644--- a/packages/common/src/object.ts+++ b/packages/common/src/object.ts@@ -1,5 +1,18 @@import { anyPass, assocPath, isEmpty, isNil, reject } from 'ramda';+function isValidJsonString(value: string): boolean {+ return (+ (value.startsWith('{') && value.endsWith('}')) ||+ (value.startsWith('[') && value.endsWith(']'))+ );+}+function isMalformedJsonString(value: string): boolean {+ return (+ (value.startsWith('{') && !value.endsWith('}')) ||+ (value.startsWith('[') && !value.endsWith(']'))+ );+}+export function toDots(obj: Record<string, unknown>,path = '',@@ -19,10 +32,28 @@ export function toDots(};}- if (value === undefined || value === null) {+ if (value === undefined || value === null || value === '') {+ return acc;+ }++ if (typeof value === 'string' && isMalformedJsonString(value)) {+ // Skip itreturn acc;}+ // Fix nested json strings - but catch parse errors for malformed JSON+ if (typeof value === 'string' && isValidJsonString(value)) {+ try {+ return {+ ...acc,+ ...toDots(JSON.parse(value), `${path}${key}.`),+ };+ } catch {+ // Skip it+ return acc;+ }+ }+const cleanedValue =typeof value === 'string'? removeInvalidSurrogates(value).trim()diff --git a/packages/db/code-migrations/5-add-imports-table.sql b/packages/db/code-migrations/5-add-imports-table.sqlnew file mode 100644index 000000000..b9144b41f--- /dev/null+++ b/packages/db/code-migrations/5-add-imports-table.sql@@ -0,0 +1,47 @@+CREATE TABLE IF NOT EXISTS events_imports_replicated ON CLUSTER '{cluster}' (+ `id` UUID DEFAULT generateUUIDv4(),+ `name` LowCardinality(String),+ `sdk_name` LowCardinality(String),+ `sdk_version` LowCardinality(String),+ `device_id` String CODEC(ZSTD(3)),+ `profile_id` String CODEC(ZSTD(3)),+ `project_id` String CODEC(ZSTD(3)),+ `session_id` String CODEC(LZ4),+ `path` String CODEC(ZSTD(3)),+ `origin` String CODEC(ZSTD(3)),+ `referrer` String CODEC(ZSTD(3)),+ `referrer_name` String CODEC(ZSTD(3)),+ `referrer_type` LowCardinality(String),+ `duration` UInt64 CODEC(Delta(4), LZ4),+ `properties` Map(String, String) CODEC(ZSTD(3)),+ `created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)),+ `country` LowCardinality(FixedString(2)),+ `city` String,+ `region` LowCardinality(String),+ `longitude` Nullable(Float32) CODEC(Gorilla, LZ4),+ `latitude` Nullable(Float32) CODEC(Gorilla, LZ4),+ `os` LowCardinality(String),+ `os_version` LowCardinality(String),+ `browser` LowCardinality(String),+ `browser_version` LowCardinality(String),+ `device` LowCardinality(String),+ `brand` LowCardinality(String),+ `model` LowCardinality(String),+ `imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4),+ `import_id` String CODEC(ZSTD(3)),+ `import_status` LowCardinality(String) DEFAULT 'pending',+ `imported_at_meta` DateTime DEFAULT now()+)+ENGINE = ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/openpanel/v1/{table}', '{replica}')+PARTITION BY toYYYYMM(imported_at_meta)+ORDER BY (import_id, created_at)+SETTINGS index_granularity = 8192;++---++CREATE TABLE IF NOT EXISTS events_imports ON CLUSTER '{cluster}' AS events_imports_replicated+ENGINE = Distributed('{cluster}', currentDatabase(), events_imports_replicated, cityHash64(import_id));++---++ALTER TABLE events_imports_replicated ON CLUSTER '{cluster}' MODIFY TTL imported_at_meta + INTERVAL 7 DAY;\ No newline at end of filediff --git a/packages/db/code-migrations/5-add-imports-table.ts b/packages/db/code-migrations/5-add-imports-table.tsnew file mode 100644index 000000000..e9062c74b--- /dev/null+++ b/packages/db/code-migrations/5-add-imports-table.ts@@ -0,0 +1,90 @@+import fs from 'node:fs';+import path from 'node:path';+import { TABLE_NAMES } from '../src/clickhouse/client';+import {+ createTable,+ modifyTTL,+ runClickhouseMigrationCommands,+} from '../src/clickhouse/migration';+import { getIsCluster } from './helpers';++export async function up() {+ const isClustered = getIsCluster();++ const sqls: string[] = [+ ...createTable({+ name: 'events_imports',+ columns: [+ // Same columns as events table+ '`id` UUID DEFAULT generateUUIDv4()',+ '`name` LowCardinality(String)',+ '`sdk_name` LowCardinality(String)',+ '`sdk_version` LowCardinality(String)',+ '`device_id` String CODEC(ZSTD(3))',+ '`profile_id` String CODEC(ZSTD(3))',+ '`project_id` String CODEC(ZSTD(3))',+ '`session_id` String CODEC(LZ4)',+ '`path` String CODEC(ZSTD(3))',+ '`origin` String CODEC(ZSTD(3))',+ '`referrer` String CODEC(ZSTD(3))',+ '`referrer_name` String CODEC(ZSTD(3))',+ '`referrer_type` LowCardinality(String)',+ '`duration` UInt64 CODEC(Delta(4), LZ4)',+ '`properties` Map(String, String) CODEC(ZSTD(3))',+ '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',+ '`country` LowCardinality(FixedString(2))',+ '`city` String',+ '`region` LowCardinality(String)',+ '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)',+ '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)',+ '`os` LowCardinality(String)',+ '`os_version` LowCardinality(String)',+ '`browser` LowCardinality(String)',+ '`browser_version` LowCardinality(String)',+ '`device` LowCardinality(String)',+ '`brand` LowCardinality(String)',+ '`model` LowCardinality(String)',+ '`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4)',++ // Additional metadata columns for import tracking+ '`import_id` String CODEC(ZSTD(3))',+ "`import_status` LowCardinality(String) DEFAULT 'pending'",+ '`imported_at_meta` DateTime DEFAULT now()',+ ],+ orderBy: ['import_id', 'created_at'],+ partitionBy: 'toYYYYMM(imported_at_meta)',+ settings: {+ index_granularity: 8192,+ },+ distributionHash: 'cityHash64(import_id)',+ replicatedVersion: '1',+ isClustered,+ }),+ ];++ // Add TTL policy for auto-cleanup after 7 days+ sqls.push(+ modifyTTL({+ tableName: 'events_imports',+ isClustered,+ ttl: 'imported_at_meta + INTERVAL 7 DAY',+ }),+ );++ fs.writeFileSync(+ path.join(__filename.replace('.ts', '.sql')),+ sqls+ .map((sql) =>+ sql+ .trim()+ .replace(/;$/, '')+ .replace(/\n{2,}/g, '\n')+ .concat(';'),+ )+ .join('\n\n---\n\n'),+ );++ if (!process.argv.includes('--dry')) {+ await runClickhouseMigrationCommands(sqls);+ }+}diff --git a/packages/db/index.ts b/packages/db/index.tsindex b0aef4e3d..5a725e8c1 100644--- a/packages/db/index.ts+++ b/packages/db/index.ts@@ -1,5 +1,6 @@export * from './src/prisma-client';export * from './src/clickhouse/client';+export * from './src/clickhouse/csv';export * from './src/sql-builder';export * from './src/services/chart.service';export * from './src/services/clients.service';@@ -23,5 +24,6 @@ export * from './src/services/access.service';export * from './src/buffers';export * from './src/types';export * from './src/clickhouse/query-builder';+export * from './src/services/import.service';export * from './src/services/overview.service';export * from './src/session-context';diff --git a/packages/db/package.json b/packages/db/package.jsonindex 56826516f..284b6d963 100644--- a/packages/db/package.json+++ b/packages/db/package.json@@ -13,7 +13,7 @@"with-env": "dotenv -e ../../.env -c --"},"dependencies": {- "@clickhouse/client": "^1.2.0",+ "@clickhouse/client": "^1.12.1","@openpanel/common": "workspace:*","@openpanel/constants": "workspace:*","@openpanel/json": "workspace:*",diff --git a/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sql b/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sqlnew file mode 100644index 000000000..46fe97732--- /dev/null+++ b/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sql@@ -0,0 +1,22 @@+-- CreateTable+CREATE TABLE "public"."imports" (+ "id" UUID NOT NULL DEFAULT gen_random_uuid(),+ "projectId" TEXT NOT NULL,+ "provider" TEXT NOT NULL,+ "sourceType" TEXT NOT NULL,+ "sourceLocation" TEXT NOT NULL,+ "jobId" TEXT,+ "status" TEXT NOT NULL,+ "config" JSONB NOT NULL DEFAULT '{}',+ "totalEvents" INTEGER NOT NULL DEFAULT 0,+ "processedEvents" INTEGER NOT NULL DEFAULT 0,+ "errorMessage" TEXT,+ "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "completedAt" TIMESTAMP(3),+ "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,++ CONSTRAINT "imports_pkey" PRIMARY KEY ("id")+);++-- AddForeignKey+ALTER TABLE "public"."imports" ADD CONSTRAINT "imports_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;diff --git a/packages/db/prisma/migrations/20251018214030_fix/migration.sql b/packages/db/prisma/migrations/20251018214030_fix/migration.sqlnew file mode 100644index 000000000..1b8aebd8c--- /dev/null+++ b/packages/db/prisma/migrations/20251018214030_fix/migration.sql@@ -0,0 +1,13 @@+/*+ Warnings:++ - You are about to drop the column `provider` on the `imports` table. All the data in the column will be lost.+ - You are about to drop the column `sourceLocation` on the `imports` table. All the data in the column will be lost.+ - You are about to drop the column `sourceType` on the `imports` table. All the data in the column will be lost.++*/+-- AlterTable+ALTER TABLE "public"."imports" DROP COLUMN "provider",+DROP COLUMN "sourceLocation",+DROP COLUMN "sourceType",+ALTER COLUMN "config" DROP DEFAULT;diff --git a/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sql b/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sqlnew file mode 100644index 000000000..ece57d171--- /dev/null+++ b/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sql@@ -0,0 +1,2 @@+-- AlterTable+ALTER TABLE "public"."imports" ADD COLUMN "statusMessage" TEXT;diff --git a/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sql b/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sqlnew file mode 100644index 000000000..b579e0f4b--- /dev/null+++ b/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sql@@ -0,0 +1,3 @@+-- AlterTable+ALTER TABLE "public"."imports" ADD COLUMN "currentBatch" INTEGER NOT NULL DEFAULT 0,+ADD COLUMN "currentStep" TEXT;diff --git a/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sql b/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sqlnew file mode 100644index 000000000..c05db2aec--- /dev/null+++ b/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sql@@ -0,0 +1,14 @@+/*+ Warnings:++ - Changed the type of `status` on the `imports` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required.+ - Made the column `currentStep` on table `imports` required. This step will fail if there are existing NULL values in that column.++*/+-- CreateEnum+CREATE TYPE "public"."ImportStatus" AS ENUM ('pending', 'processing', 'completed', 'failed');++-- AlterTable+ALTER TABLE "public"."imports" DROP COLUMN "status",+ADD COLUMN "status" "public"."ImportStatus" NOT NULL,+ALTER COLUMN "currentStep" SET NOT NULL;diff --git a/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sql b/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sqlnew file mode 100644index 000000000..6afc57871--- /dev/null+++ b/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sql@@ -0,0 +1,2 @@+-- AlterTable+ALTER TABLE "public"."imports" ALTER COLUMN "currentStep" DROP NOT NULL;diff --git a/packages/db/prisma/migrations/20251028152531_imports_again/migration.sql b/packages/db/prisma/migrations/20251028152531_imports_again/migration.sqlnew file mode 100644index 000000000..3abc525e9--- /dev/null+++ b/packages/db/prisma/migrations/20251028152531_imports_again/migration.sql@@ -0,0 +1,4 @@+-- AlterTable+ALTER TABLE "public"."imports" ALTER COLUMN "currentBatch" DROP NOT NULL,+ALTER COLUMN "currentBatch" DROP DEFAULT,+ALTER COLUMN "currentBatch" SET DATA TYPE TEXT;diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prismaindex def5169e5..e6aee8aba 100644--- a/packages/db/prisma/schema.prisma+++ b/packages/db/prisma/schema.prisma@@ -194,6 +194,7 @@ model Project {notificationRules NotificationRule[]notifications Notification[]+ imports Import[]// When deleteAt > now(), the project will be deleteddeleteAt DateTime?@@ -467,3 +468,31 @@ model ResetPassword {@@map("reset_password")}++enum ImportStatus {+ pending+ processing+ completed+ failed+}++model Import {+ id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid+ projectId String+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)+ jobId String? // BullMQ job ID+ status ImportStatus+ statusMessage String? // Human-readable current step like "Importing events (Feb 2025)", "Generating session IDs"+ errorMessage String?+ /// [IPrismaImportConfig]+ config Json+ totalEvents Int @default(0)+ processedEvents Int @default(0)+ currentStep String?+ currentBatch String? // String date 2020-01-01+ createdAt DateTime @default(now())+ completedAt DateTime?+ updatedAt DateTime @default(now()) @updatedAt++ @@map("imports")+}diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.tsindex f87cc9037..d1be225c4 100644--- a/packages/db/src/buffers/base-buffer.ts+++ b/packages/db/src/buffers/base-buffer.ts@@ -1,4 +1,4 @@-import { generateSecureId } from '@openpanel/common/server/id';+import { generateSecureId } from '@openpanel/common/server';import { type ILogger, createLogger } from '@openpanel/logger';import { getRedisCache, runEvery } from '@openpanel/redis';diff --git a/packages/db/src/buffers/event-buffer.test.ts b/packages/db/src/buffers/event-buffer.test.tsindex 0df5b4aea..1babc51bf 100644--- a/packages/db/src/buffers/event-buffer.test.ts+++ b/packages/db/src/buffers/event-buffer.test.ts@@ -10,6 +10,18 @@ import {} from 'vitest';import { ch } from '../clickhouse/client';+const clickhouseSettings = {+ async_insert: 1,+ http_headers_progress_interval_ms: '50000',+ input_format_parallel_parsing: 1,+ max_execution_time: 300,+ max_http_get_redirects: '0',+ max_insert_block_size: '500000',+ send_progress_in_http_headers: 1,+ wait_end_of_query: 1,+ wait_for_async_insert: 1,+};+// Mock transformEvent to avoid circular dependency with buffers -> services -> buffersvi.mock('../services/event.service', () => ({transformEvent: (event: any) => ({@@ -127,6 +139,7 @@ describe('EventBuffer with real Redis', () => {duration: 1000,},],+ clickhouse_settings: clickhouseSettings,});const sessionKey = `event_buffer:session:${first.session_id}`;@@ -171,6 +184,7 @@ describe('EventBuffer with real Redis', () => {format: 'JSONEachRow',table: 'events',values: [first, end],+ clickhouse_settings: clickhouseSettings,});const sessionKey = `event_buffer:session:${first.session_id}`;const storedEvents = await redis.lrange(sessionKey, 0, -1);@@ -502,6 +516,7 @@ describe('EventBuffer with real Redis', () => {format: 'JSONEachRow',table: 'events',values: [end],+ clickhouse_settings: clickhouseSettings,});const sessionKey = `event_buffer:session:${s}`;@@ -552,6 +567,7 @@ describe('EventBuffer with real Redis', () => {format: 'JSONEachRow',table: 'events',values: [view1, view2, view3, end],+ clickhouse_settings: clickhouseSettings,});// Session should be completely empty and removed@@ -596,6 +612,7 @@ describe('EventBuffer with real Redis', () => {format: 'JSONEachRow',table: 'events',values: [{ ...view1, duration: 1000 }],+ clickhouse_settings: clickhouseSettings,});// Session should be REMOVED from ready_sessions (only 1 event left)@@ -620,6 +637,7 @@ describe('EventBuffer with real Redis', () => {format: 'JSONEachRow',table: 'events',values: [{ ...view2, duration: 1000 }],+ clickhouse_settings: clickhouseSettings,});// Session should be REMOVED again (only 1 event left)@@ -667,6 +685,7 @@ describe('EventBuffer with real Redis', () => {format: 'JSONEachRow',table: 'events',values: [view, end],+ clickhouse_settings: clickhouseSettings,});// NOW it should be removed from ready_sessions (because it's empty)diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex b363cb44e..be55866a8 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -1,3 +1,4 @@+import { Readable } from 'node:stream';import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client';import { ClickHouseLogLevel, createClient } from '@clickhouse/client';import sqlstring from 'sqlstring';@@ -23,13 +24,10 @@ type WarnLogParams = LogParams & { err?: Error };class CustomLogger implements Logger {trace({ message, args }: LogParams) {- logger.debug(message, args);+ logger.info(message, args);}debug({ message, args }: LogParams) {- if (message.includes('Query:') && args?.response_status === 200) {- return;- }- logger.debug(message, args);+ logger.info(message, args);}info({ message, args }: LogParams) {logger.info(message, args);@@ -56,14 +54,15 @@ export const TABLE_NAMES = {event_property_values_mv: 'event_property_values_mv',cohort_events_mv: 'cohort_events_mv',sessions: 'sessions',+ events_imports: 'events_imports',};export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {max_open_connections: 30,- request_timeout: 60000,+ request_timeout: 300000,keep_alive: {enabled: true,- idle_socket_ttl: 8000,+ idle_socket_ttl: 60000,},compression: {request: true,@@ -87,7 +86,7 @@ const cleanQuery = (query?: string) =>? query.replace(/\n/g, '').replace(/\s+/g, ' ').trim(): undefined;-async function withRetry<T>(+export async function withRetry<T>(operation: () => Promise<T>,maxRetries = 3,baseDelay = 500,@@ -132,7 +131,34 @@ export const ch = new Proxy(originalCh, {const value = Reflect.get(target, property, receiver);if (property === 'insert') {- return (...args: any[]) => withRetry(() => value.apply(target, args));+ return (...args: any[]) =>+ withRetry(() => {+ args[0].clickhouse_settings = {+ // Allow bigger HTTP payloads/time to stream rows+ async_insert: 1,+ wait_for_async_insert: 1,+ // Increase insert timeouts and buffer sizes for large batches+ max_execution_time: 300,+ max_insert_block_size: '500000',+ max_http_get_redirects: '0',+ // Ensure JSONEachRow stays efficient+ input_format_parallel_parsing: 1,+ // Keep long-running inserts/queries from idling out at proxies by sending progress headers+ send_progress_in_http_headers: 1,+ http_headers_progress_interval_ms: '50000',+ // Ensure server holds the connection until the query is finished+ wait_end_of_query: 1,+ ...args[0].clickhouse_settings,+ };+ return value.apply(target, args);+ });+ }++ if (property === 'command') {+ return (...args: any[]) =>+ withRetry(() => {+ return value.apply(target, args);+ });}return value;@@ -177,6 +203,34 @@ export async function chQueryWithMeta<T extends Record<string, any>>(return response;}+export async function chInsertCSV(tableName: string, rows: string[]) {+ try {+ const now = performance.now();+ // Create a readable stream in binary mode for CSV (similar to EventBuffer)+ const csvStream = Readable.from(rows.join('\n'), {+ objectMode: false,+ });++ await ch.insert({+ table: tableName,+ values: csvStream,+ format: 'CSV',+ clickhouse_settings: {+ format_csv_allow_double_quotes: 1,+ format_csv_allow_single_quotes: 0,+ },+ });++ logger.info('CSV Insert successful', {+ elapsed: performance.now() - now,+ rows: rows.length,+ });+ } catch (error) {+ logger.error('CSV Insert failed:', error);+ throw error;+ }+}+export async function chQuery<T extends Record<string, any>>(query: string,clickhouseSettings?: ClickHouseSettings,diff --git a/packages/db/src/clickhouse/csv.ts b/packages/db/src/clickhouse/csv.tsnew file mode 100644index 000000000..21802f04d--- /dev/null+++ b/packages/db/src/clickhouse/csv.ts@@ -0,0 +1,53 @@+// ClickHouse Map(String, String) format in CSV uses single quotes, not JSON double quotes+// Format: '{'key1':'value1','key2':'value2'}'+// Single quotes inside values must be escaped with backslash: \'+// We also need to escape newlines and control characters to prevent CSV parsing issues+const escapeMapValue = (str: string) => {+ return str+ .replace(/\\/g, '\\\\') // Escape backslashes first+ .replace(/'/g, "\\'") // Escape single quotes+ .replace(/\n/g, '\\n') // Escape newlines+ .replace(/\r/g, '\\r') // Escape carriage returns+ .replace(/\t/g, '\\t') // Escape tabs+ .replace(/\0/g, '\\0'); // Escape null bytes+};++export const csvEscapeJson = (+ value: Record<string, unknown> | null | undefined,+): string => {+ if (value == null) return '';++ // Normalize to strings if your column is Map(String,String)+ const normalized: Record<string, string> = Object.fromEntries(+ Object.entries(value).map(([k, v]) => [+ String(k),+ v == null ? '' : String(v),+ ]),+ );++ // Empty object should return empty Map (without quotes, csvEscapeField will handle if needed)+ if (Object.keys(normalized).length === 0) return '{}';++ const pairs = Object.entries(normalized)+ .map(([k, v]) => `'${escapeMapValue(k)}':'${escapeMapValue(v)}'`)+ .join(',');++ // Return Map format without outer quotes - csvEscapeField will handle CSV escaping+ // This allows csvEscapeField to properly wrap/escape the entire field if it contains newlines/quotes+ return csvEscapeField(`{${pairs}}`);+};++// Escape a CSV field - wrap in double quotes if it contains commas, quotes, or newlines+// Double quotes inside must be doubled (""), per CSV standard+export const csvEscapeField = (value: string | number): string => {+ const str = String(value);++ // If field contains commas, quotes, or newlines, it must be quoted+ if (/[,"\n\r]/.test(str)) {+ // Escape double quotes by doubling them+ const escaped = str.replace(/"/g, '""');+ return `"${escaped}"`;+ }++ return str;+};diff --git a/packages/db/src/clickhouse/migration.ts b/packages/db/src/clickhouse/migration.tsindex 2b65d5610..57f27d34f 100644--- a/packages/db/src/clickhouse/migration.ts+++ b/packages/db/src/clickhouse/migration.ts@@ -115,6 +115,22 @@ ENGINE = Distributed('{cluster}', currentDatabase(), ${replicated(tableName)}, $];}+export const modifyTTL = ({+ tableName,+ isClustered,+ ttl,+}: {+ tableName: string;+ isClustered: boolean;+ ttl: string;+}) => {+ if (isClustered) {+ return `ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' MODIFY TTL ${ttl}`;+ }++ return `ALTER TABLE ${tableName} MODIFY TTL ${ttl}`;+};+/*** Generates ALTER TABLE statements for adding columns*/diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex a182b69b3..6cd68a80a 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -141,6 +141,10 @@ export type IServiceCreateEventPayload = Omit<IServiceEvent,'id' | 'importedAt' | 'profile' | 'meta'>;+export type IServiceImportedEventPayload = Omit<+ IServiceEvent,+ 'profile' | 'meta'+>;export interface IServiceEvent {id: string;diff --git a/packages/db/src/services/import.service.ts b/packages/db/src/services/import.service.tsnew file mode 100644index 000000000..6d0b8f554--- /dev/null+++ b/packages/db/src/services/import.service.ts@@ -0,0 +1,784 @@+import type { ILogger } from '@openpanel/logger';+import sqlstring from 'sqlstring';+import {+ TABLE_NAMES,+ ch,+ chInsertCSV,+ convertClickhouseDateToJs,+ formatClickhouseDate,+} from '../clickhouse/client';+import { csvEscapeField, csvEscapeJson } from '../clickhouse/csv';+import { type Prisma, db } from '../prisma-client';+import type { IClickhouseEvent } from './event.service';++export interface ImportStageResult {+ importId: string;+ totalEvents: number;+ insertedEvents: number;+}++export interface ImportProgress {+ importId: string;+ totalEvents: number;+ insertedEvents: number;+ status: 'pending' | 'processing' | 'processed' | 'failed';+}++/**+ * Insert a batch of events into the imports staging table+ */+export async function insertImportBatch(+ events: IClickhouseEvent[],+ importId: string,+): Promise<ImportStageResult> {+ if (events.length === 0) {+ return { importId, totalEvents: 0, insertedEvents: 0 };+ }++ // Important to have same order as events_imports table+ // CSV format: properly quotes fields that need it+ const csvRows = events.map((event) => {+ // Properties need to be converted to JSON for Map(String, String)+ // All fields must be CSV-escaped when joining with commas+ const fields = [+ csvEscapeField(event.id || ''),+ csvEscapeField(event.name),+ csvEscapeField(event.sdk_name || ''),+ csvEscapeField(event.sdk_version || ''),+ csvEscapeField(event.device_id || ''),+ csvEscapeField(event.profile_id || ''),+ csvEscapeField(event.project_id || ''),+ csvEscapeField(event.session_id || ''),+ csvEscapeField(event.path),+ csvEscapeField(event.origin || ''),+ csvEscapeField(event.referrer || ''),+ csvEscapeField(event.referrer_name || ''),+ csvEscapeField(event.referrer_type || ''),+ csvEscapeField(event.duration ?? 0),+ csvEscapeJson(event.properties),+ csvEscapeField(event.created_at),+ csvEscapeField(event.country || ''),+ csvEscapeField(event.city || ''),+ csvEscapeField(event.region || ''),+ csvEscapeField(event.longitude != null ? event.longitude : '\\N'),+ csvEscapeField(event.latitude != null ? event.latitude : '\\N'),+ csvEscapeField(event.os || ''),+ csvEscapeField(event.os_version || ''),+ csvEscapeField(event.browser || ''),+ csvEscapeField(event.browser_version || ''),+ csvEscapeField(event.device || ''),+ csvEscapeField(event.brand || ''),+ csvEscapeField(event.model || ''),+ csvEscapeField('\\N'), // imported_at (Nullable)+ csvEscapeField(importId),+ csvEscapeField('pending'), // import_status+ csvEscapeField(formatClickhouseDate(new Date())), // imported_at_meta (DateTime, not DateTime64, so no milliseconds)+ ];+ return fields.join(',');+ });++ await chInsertCSV(TABLE_NAMES.events_imports, csvRows);++ return {+ importId,+ totalEvents: events.length,+ insertedEvents: events.length,+ };+}++/**+ * Generate deterministic session IDs for events that don't have them+ * Uses 30-minute time windows to create consistent session IDs across imports+ * Only processes events where device != 'server' and session_id = ''+ */+export async function generateSessionIds(+ importId: string,+ from: string,+): Promise<void> {+ const rangeWhere = [+ 'import_id = {importId:String}',+ "import_status = 'pending'",+ "device != 'server'",+ "session_id = ''",+ from ? 'toDate(created_at) = {from:String}' : '',+ ]+ .filter(Boolean)+ .join(' AND ');++ // Use SQL to generate deterministic session IDs based on device_id + 30-min time windows+ // This ensures same events always get same session IDs regardless of import order+ const updateQuery = `+ ALTER TABLE ${TABLE_NAMES.events_imports}+ UPDATE session_id = lower(hex(MD5(concat(+ device_id,+ '-',+ toString(toInt64(toUnixTimestamp(created_at) / 1800))+ ))))+ WHERE ${rangeWhere}+ `;++ await ch.command({+ query: updateQuery,+ query_params: { importId, from },+ clickhouse_settings: {+ wait_end_of_query: 1,+ mutations_sync: '2', // Wait for mutation to complete on all replicas (critical!)+ send_progress_in_http_headers: 1,+ http_headers_progress_interval_ms: '50000',+ },+ });+}++/**+ * Reconstruct sessions using SQL-based logic+ * This identifies session boundaries and creates session_start/session_end events+ * session_start inherits all properties from the first event in the session+ * session_end inherits all properties from the last event in the session and calculates duration+ */+export async function createSessionsStartEndEvents(+ importId: string,+ from: string,+): Promise<void> {+ // First, let's identify session boundaries and get first/last events for each session+ const rangeWhere = [+ 'import_id = {importId:String}',+ "import_status = 'pending'",+ "session_id != ''", // Only process events that have session IDs+ 'toDate(created_at) = {from:String}',+ ]+ .filter(Boolean)+ .join(' AND ');++ // Use window functions to efficiently get first event (all fields) and last event (only changing fields)+ // session_end only needs: properties, path, origin, created_at - the rest can be inherited from session_start+ const sessionEventsQuery = `+ SELECT+ device_id,+ session_id,+ project_id,+ profile_id,+ argMin((path, origin, referrer, referrer_name, referrer_type, properties, created_at, country, city, region, longitude, latitude, os, os_version, browser, browser_version, device, brand, model), created_at) AS first_event,+ argMax((path, origin, properties, created_at), created_at) AS last_event_fields,+ min(created_at) AS first_timestamp,+ max(created_at) AS last_timestamp+ FROM ${TABLE_NAMES.events_imports}+ WHERE ${rangeWhere}+ AND name NOT IN ('session_start', 'session_end')+ GROUP BY session_id, device_id, project_id, profile_id+ `;++ const sessionEventsResult = await ch.query({+ query: sessionEventsQuery,+ query_params: { importId, from },+ format: 'JSONEachRow',+ });++ const sessionData = (await sessionEventsResult.json()) as Array<{+ device_id: string;+ session_id: string;+ project_id: string;+ profile_id: string;+ first_event: [+ // string, // id+ // string, // name+ string, // path+ string, // origin+ string, // referrer+ string, // referrer_name+ string, // referrer_type+ // number, // duration+ Record<string, unknown>, // properties+ string, // created_at+ string, // country+ string, // city+ string, // region+ number | null, // longitude+ number | null, // latitude+ string, // os+ string, // os_version+ string, // browser+ string, // browser_version+ string, // device+ string, // brand+ string, // model+ // string, // sdk_name+ // string, // sdk_version+ // string, // imported_at+ ];+ last_event_fields: [+ string, // path+ string, // origin+ Record<string, unknown>, // properties+ string, // created_at+ ];+ first_timestamp: string;+ last_timestamp: string;+ }>;++ // Create session_start and session_end events+ const sessionEvents: IClickhouseEvent[] = [];++ for (const session of sessionData) {+ // Destructure first event tuple (all fields)+ const [+ // firstId,+ // firstName,+ firstPath,+ firstOrigin,+ firstReferrer,+ firstReferrerName,+ firstReferrerType,+ // firstDuration,+ firstProperties,+ firstCreatedAt,+ firstCountry,+ firstCity,+ firstRegion,+ firstLongitude,+ firstLatitude,+ firstOs,+ firstOsVersion,+ firstBrowser,+ firstBrowserVersion,+ firstDevice,+ firstBrand,+ firstModel,+ // firstSdkName,+ // firstSdkVersion,+ // firstImportedAt,+ ] = session.first_event;++ // Destructure last event fields (only the changing ones)+ const [lastPath, lastOrigin, lastProperties, lastCreatedAt] =+ session.last_event_fields;++ // Calculate duration in milliseconds+ // Parse timestamps as Date objects to calculate duration+ const firstTime = new Date(session.first_timestamp).getTime();+ const lastTime = new Date(session.last_timestamp).getTime();+ const durationMs = lastTime - firstTime;++ // Helper function to adjust timestamp by milliseconds without timezone conversion+ const adjustTimestamp = (timestamp: string, offsetMs: number): string => {+ // Parse the timestamp, adjust it, and format back to ClickHouse format+ const date = convertClickhouseDateToJs(timestamp);+ date.setTime(date.getTime() + offsetMs);+ return formatClickhouseDate(date);+ };++ // Create session_start event - inherit everything from first event but change name+ // Set created_at to 1 second before the first event+ sessionEvents.push({+ id: crypto.randomUUID(),+ name: 'session_start',+ device_id: session.device_id,+ profile_id: session.profile_id,+ project_id: session.project_id,+ session_id: session.session_id,+ path: firstPath,+ origin: firstOrigin,+ referrer: firstReferrer,+ referrer_name: firstReferrerName,+ referrer_type: firstReferrerType,+ duration: 0, // session_start always has 0 duration+ properties: firstProperties as Record<+ string,+ string | number | boolean | null | undefined+ >,+ created_at: adjustTimestamp(session.first_timestamp, -1000), // 1 second before first event+ country: firstCountry,+ city: firstCity,+ region: firstRegion,+ longitude: firstLongitude,+ latitude: firstLatitude,+ os: firstOs,+ os_version: firstOsVersion,+ browser: firstBrowser,+ browser_version: firstBrowserVersion,+ device: firstDevice,+ brand: firstBrand,+ model: firstModel,+ imported_at: new Date().toISOString(),+ sdk_name: 'import-session-reconstruction',+ sdk_version: '1.0.0',+ });++ // Create session_end event - inherit most from session_start, but use last event's path, origin, properties+ // Set created_at to 1 second after the last event+ sessionEvents.push({+ id: crypto.randomUUID(),+ name: 'session_end',+ device_id: session.device_id,+ profile_id: session.profile_id,+ project_id: session.project_id,+ session_id: session.session_id,+ path: lastPath, // From last event+ origin: lastOrigin, // From last event+ referrer: firstReferrer, // Same as session_start+ referrer_name: firstReferrerName, // Same as session_start+ referrer_type: firstReferrerType, // Same as session_start+ duration: durationMs,+ properties: lastProperties as Record<+ string,+ string | number | boolean | null | undefined+ >, // From last event+ created_at: adjustTimestamp(session.last_timestamp, 500), // 1 second after last event+ country: firstCountry, // Same as session_start+ city: firstCity, // Same as session_start+ region: firstRegion, // Same as session_start+ longitude: firstLongitude, // Same as session_start+ latitude: firstLatitude, // Same as session_start+ os: firstOs, // Same as session_start+ os_version: firstOsVersion, // Same as session_start+ browser: firstBrowser, // Same as session_start+ browser_version: firstBrowserVersion, // Same as session_start+ device: firstDevice, // Same as session_start+ brand: firstBrand, // Same as session_start+ model: firstModel, // Same as session_start+ imported_at: new Date().toISOString(),+ sdk_name: 'import-session-reconstruction',+ sdk_version: '1.0.0',+ });+ }++ // Insert session events into imports table+ if (sessionEvents.length > 0) {+ await insertImportBatch(sessionEvents, importId);+ }+}++/**+ * Migrate all events from imports table to production events table+ * This includes both original events and generated session events+ */+export async function moveImportsToProduction(+ importId: string,+ from: string,+): Promise<void> {+ // Build the WHERE clause for migration+ // For session events (session_start/session_end), we don't filter by their created_at+ // because they're created with adjusted timestamps (±1 second) that might fall outside+ // the date range. Instead, we include them if their session_id has events in this range.+ let whereClause = 'import_id = {importId:String}';++ if (from) {+ whereClause += ` AND (+ (toDate(created_at) = {from:String}) OR+ (+ name IN ('session_start', 'session_end') AND+ session_id IN (+ SELECT DISTINCT session_id+ FROM ${TABLE_NAMES.events_imports}+ WHERE import_id = {importId:String}+ AND toDate(created_at) = {from:String}+ AND name NOT IN ('session_start', 'session_end')+ )+ )+ )`;+ }++ const migrationQuery = `+ INSERT INTO ${TABLE_NAMES.events} (+ id,+ name,+ sdk_name,+ sdk_version,+ device_id,+ profile_id,+ project_id,+ session_id,+ path,+ origin,+ referrer,+ referrer_name,+ referrer_type,+ duration,+ properties,+ created_at,+ country,+ city,+ region,+ longitude,+ latitude,+ os,+ os_version,+ browser,+ browser_version,+ device,+ brand,+ model,+ imported_at+ )+ SELECT+ id,+ name,+ sdk_name,+ sdk_version,+ device_id,+ profile_id,+ project_id,+ session_id,+ path,+ origin,+ referrer,+ referrer_name,+ referrer_type,+ duration,+ properties,+ created_at,+ country,+ city,+ region,+ longitude,+ latitude,+ os,+ os_version,+ browser,+ browser_version,+ device,+ brand,+ model,+ imported_at+ FROM ${TABLE_NAMES.events_imports}+ WHERE ${whereClause}+ ORDER BY created_at ASC+ `;++ await ch.command({+ query: migrationQuery,+ query_params: { importId, from },+ clickhouse_settings: {+ wait_end_of_query: 1,+ // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.+ send_progress_in_http_headers: 1,+ // The interval of sending these progress headers. Here it is less than 60s,+ http_headers_progress_interval_ms: '50000',+ },+ });+}++export async function backfillSessionsToProduction(+ importId: string,+ from: string,+): Promise<void> {+ // After migrating events, populate the sessions table based on the migrated sessions+ // We detect all session_ids involved in this import from the imports table,+ // then aggregate over the production events to construct session rows.+ const sessionsInsertQuery = `+ INSERT INTO ${TABLE_NAMES.sessions} (+ id,+ project_id,+ profile_id,+ device_id,+ created_at,+ ended_at,+ is_bounce,+ entry_origin,+ entry_path,+ exit_origin,+ exit_path,+ screen_view_count,+ revenue,+ event_count,+ duration,+ country,+ region,+ city,+ longitude,+ latitude,+ device,+ brand,+ model,+ browser,+ browser_version,+ os,+ os_version,+ sign,+ version,+ properties,+ utm_medium,+ utm_source,+ utm_campaign,+ utm_content,+ utm_term,+ referrer,+ referrer_name,+ referrer_type+ )+ SELECT+ any(e.session_id) as id,+ any(e.project_id) as project_id,+ if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,+ any(e.device_id) as device_id,+ argMin(e.created_at, e.created_at) as created_at,+ argMax(e.created_at, e.created_at) as ended_at,+ if(+ argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = '',+ if(countIf(e.name = 'screen_view') > 1, true, false),+ argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = 'true'+ ) as is_bounce,+ argMinIf(e.origin, e.created_at, e.name = 'session_start') as entry_origin,+ argMinIf(e.path, e.created_at, e.name = 'session_start') as entry_path,+ argMaxIf(e.origin, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_origin,+ argMaxIf(e.path, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_path,+ countIf(e.name = 'screen_view') as screen_view_count,+ 0 as revenue,+ countIf(e.name != 'screen_view' AND e.name != 'session_start' AND e.name != 'session_end') as event_count,+ sumIf(e.duration, name = 'session_end') AS duration,+ argMinIf(e.country, e.created_at, e.name = 'session_start') as country,+ argMinIf(e.region, e.created_at, e.name = 'session_start') as region,+ argMinIf(e.city, e.created_at, e.name = 'session_start') as city,+ argMinIf(e.longitude, e.created_at, e.name = 'session_start') as longitude,+ argMinIf(e.latitude, e.created_at, e.name = 'session_start') as latitude,+ argMinIf(e.device, e.created_at, e.name = 'session_start') as device,+ argMinIf(e.brand, e.created_at, e.name = 'session_start') as brand,+ argMinIf(e.model, e.created_at, e.name = 'session_start') as model,+ argMinIf(e.browser, e.created_at, e.name = 'session_start') as browser,+ argMinIf(e.browser_version, e.created_at, e.name = 'session_start') as browser_version,+ argMinIf(e.os, e.created_at, e.name = 'session_start') as os,+ argMinIf(e.os_version, e.created_at, e.name = 'session_start') as os_version,+ 1 as sign,+ 1 as version,+ argMinIf(e.properties, e.created_at, e.name = 'session_start') as properties,+ argMinIf(e.properties['__query.utm_medium'], e.created_at, e.name = 'session_start') as utm_medium,+ argMinIf(e.properties['__query.utm_source'], e.created_at, e.name = 'session_start') as utm_source,+ argMinIf(e.properties['__query.utm_campaign'], e.created_at, e.name = 'session_start') as utm_campaign,+ argMinIf(e.properties['__query.utm_content'], e.created_at, e.name = 'session_start') as utm_content,+ argMinIf(e.properties['__query.utm_term'], e.created_at, e.name = 'session_start') as utm_term,+ argMinIf(e.referrer, e.created_at, e.name = 'session_start') as referrer,+ argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name,+ argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_type+ FROM ${TABLE_NAMES.events_imports} e+ WHERE+ e.import_id = ${sqlstring.escape(importId)}+ AND toDate(e.created_at) = ${sqlstring.escape(from)}+ AND e.session_id != ''+ GROUP BY e.session_id+ `;++ await ch.command({+ query: sessionsInsertQuery,+ clickhouse_settings: {+ wait_end_of_query: 1,+ // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.+ send_progress_in_http_headers: 1,+ // The interval of sending these progress headers. Here it is less than 60s,+ http_headers_progress_interval_ms: '50000',+ },+ });+}++/**+ * Mark import as complete by updating status+ */+export async function markImportComplete(importId: string): Promise<void> {+ const updateQuery = `+ ALTER TABLE ${TABLE_NAMES.events_imports}+ UPDATE import_status = 'processed'+ WHERE import_id = {importId:String}+ `;++ await ch.command({+ query: updateQuery,+ query_params: { importId },+ clickhouse_settings: {+ wait_end_of_query: 1,+ mutations_sync: '2', // Wait for mutation to complete+ // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.+ send_progress_in_http_headers: 1,+ // The interval of sending these progress headers. Here it is less than 60s,+ http_headers_progress_interval_ms: '50000',+ },+ });+}++/**+ * Get import progress and status+ */+export async function getImportProgress(+ importId: string,+): Promise<ImportProgress> {+ const progressQuery = `+ SELECT+ import_id,+ COUNT(*) as total_events,+ COUNTIf(import_status = 'pending') as pending_events,+ COUNTIf(import_status = 'processed') as processed_events,+ any(import_status) as status+ FROM ${TABLE_NAMES.events_imports}+ WHERE import_id = {importId:String}+ AND name NOT IN ('session_start', 'session_end')+ GROUP BY import_id+ `;++ const result = await ch.query({+ query: progressQuery,+ query_params: { importId },+ format: 'JSONEachRow',+ });++ const data = (await result.json()) as Array<{+ import_id: string;+ total_events: number;+ pending_events: number;+ processed_events: number;+ status: string;+ }>;++ if (data.length === 0) {+ return {+ importId,+ totalEvents: 0,+ insertedEvents: 0,+ status: 'pending',+ };+ }++ const row = data[0];+ if (!row) {+ return {+ importId,+ totalEvents: 0,+ insertedEvents: 0,+ status: 'pending',+ };+ }++ return {+ importId,+ totalEvents: row.total_events,+ insertedEvents: row.processed_events,+ status: row.status as 'pending' | 'processing' | 'processed' | 'failed',+ };+}++/**+ * Utility: get min/max created_at for an import+ */+export async function getImportDateBounds(+ importId: string,+ fromCreatedAt?: string,+): Promise<{ min: string | null; max: string | null }> {+ const res = await ch.query({+ query: `+ SELECT min(created_at) AS min, max(created_at) AS max+ FROM ${TABLE_NAMES.events_imports}+ WHERE import_id = {importId:String}+ ${fromCreatedAt ? 'AND created_at >= {fromCreatedAt:String}' : ''}+ `,+ query_params: { importId, fromCreatedAt },+ format: 'JSONEachRow',+ });+ const rows = (await res.json()) as Array<{+ min: string | null;+ max: string | null;+ }>;+ return rows.length > 0+ ? {+ min: fromCreatedAt ?? rows[0]?.min ?? null,+ max: rows[0]?.max ?? null,+ }+ : { min: null, max: null };+}++/**+ * Unified method to update all import status information+ * Combines step, batch, progress, and status message updates+ */+export type UpdateImportStatusOptions =+ | {+ step: 'loading';+ batch?: string;+ totalEvents?: number;+ processedEvents?: number;+ }+ | {+ step: 'generating_session_ids';+ batch?: string;+ }+ | {+ step: 'creating_sessions';+ batch?: string;+ }+ | {+ step: 'moving';+ batch?: string;+ }+ | {+ step: 'backfilling_sessions';+ batch?: string;+ }+ | {+ step: 'completed';+ }+ | {+ step: 'failed';+ errorMessage?: string;+ };++export type ImportSteps = UpdateImportStatusOptions['step'];++export async function updateImportStatus(+ jobLogger: ILogger,+ job: {+ updateProgress: (progress: Record<string, any>) => void;+ },+ importId: string,+ options: UpdateImportStatusOptions,+): Promise<void> {+ const data: Prisma.ImportUpdateInput = {};+ switch (options.step) {+ case 'loading':+ data.status = 'processing';+ data.currentStep = 'loading';+ data.currentBatch = options.batch;+ data.statusMessage = options.batch+ ? `Importing events from ${options.batch}`+ : 'Initializing...';+ data.totalEvents = options.totalEvents;+ data.processedEvents = options.processedEvents;+ break;+ case 'generating_session_ids':+ data.currentStep = 'generating_session_ids';+ data.currentBatch = options.batch;+ data.statusMessage = options.batch+ ? `Generating session IDs for ${options.batch}`+ : 'Generating session IDs...';+ break;+ case 'creating_sessions':+ data.currentStep = 'creating_sessions';+ data.currentBatch = options.batch;+ data.statusMessage = `Creating sessions for ${options.batch}`;+ break;+ case 'moving':+ data.currentStep = 'moving';+ data.currentBatch = options.batch;+ data.statusMessage = `Moving imports to production for ${options.batch}`;+ break;+ case 'backfilling_sessions':+ data.currentStep = 'backfilling_sessions';+ data.currentBatch = options.batch;+ data.statusMessage = `Aggregating sessions for ${options.batch}`;+ break;+ case 'completed':+ data.status = 'completed';+ data.currentStep = 'completed';+ data.statusMessage = 'Import completed';+ data.completedAt = new Date();+ break;+ case 'failed':+ data.status = 'failed';+ data.statusMessage = 'Import failed';+ data.errorMessage = options.errorMessage;+ break;+ }++ jobLogger.info('Import status update', data);++ await job.updateProgress(data);++ await db.import.update({+ where: { id: importId },+ data,+ });+}diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.tsindex 016e478e3..aa9fe2fbe 100644--- a/packages/db/src/services/session.service.ts+++ b/packages/db/src/services/session.service.ts@@ -196,7 +196,7 @@ export async function getSessionList({organization?.subscriptionPeriodEventsLimit &&organization?.subscriptionPeriodEventsLimit > 1_000_000? 1- : 7;+ : 360;if (cursor) {const cAt = sqlstring.escape(cursor.createdAt);diff --git a/packages/db/src/types.ts b/packages/db/src/types.tsindex 8d6db5860..7e525f25f 100644--- a/packages/db/src/types.ts+++ b/packages/db/src/types.ts@@ -1,4 +1,5 @@import type {+ IImportConfig,IIntegrationConfig,INotificationRuleConfig,IProjectFilters,@@ -12,6 +13,7 @@ import type { IClickhouseProfile } from './services/profile.service';declare global {namespace PrismaJson {+ type IPrismaImportConfig = IImportConfig;type IPrismaNotificationRuleConfig = INotificationRuleConfig;type IPrismaIntegrationConfig = IIntegrationConfig;type IPrismaNotificationPayload = INotificationPayload;diff --git a/packages/importer/package.json b/packages/importer/package.jsonnew file mode 100644index 000000000..ff91d681a--- /dev/null+++ b/packages/importer/package.json@@ -0,0 +1,35 @@+{+ "name": "@openpanel/importer",+ "version": "0.0.0",+ "type": "module",+ "main": "src/index.ts",+ "scripts": {+ "build": "tsc",+ "dev": "tsc --watch",+ "test": "vitest",+ "test:run": "vitest run"+ },+ "exports": {+ ".": "./src/index.ts",+ "./providers": "./src/providers/metadata.ts"+ },+ "dependencies": {+ "@openpanel/common": "workspace:*",+ "@openpanel/db": "workspace:*",+ "@openpanel/queue": "workspace:*",+ "@openpanel/validation": "workspace:*",+ "csv-parse": "^6.1.0",+ "ramda": "^0.29.1",+ "uuid": "^9.0.1",+ "zod": "catalog:"+ },+ "devDependencies": {+ "@openpanel/logger": "workspace:*",+ "@types/node": "^20.0.0",+ "@types/ramda": "^0.31.1",+ "@types/uuid": "^9.0.7",+ "bullmq": "^5.8.7",+ "typescript": "^5.0.0",+ "vitest": "^1.0.0"+ }+}diff --git a/packages/importer/src/base-provider.ts b/packages/importer/src/base-provider.tsnew file mode 100644index 000000000..e7c6189fe--- /dev/null+++ b/packages/importer/src/base-provider.ts@@ -0,0 +1,121 @@+import type { IClickhouseEvent } from '@openpanel/db';+import type { BaseRawEvent, ErrorContext, ImportJobMetadata } from './types';++export abstract class BaseImportProvider<+ TRawEvent extends BaseRawEvent = BaseRawEvent,+> {+ abstract provider: string;+ abstract version: string;++ /**+ * Stream-read and parse source (file/API) → yields raw events+ * This should be implemented as an async generator to handle large files efficiently+ */+ abstract parseSource(+ overrideFrom?: string,+ ): AsyncGenerator<TRawEvent, void, unknown>;++ /**+ * Convert provider format → IClickhouseEvent+ */+ abstract transformEvent(rawEvent: TRawEvent): IClickhouseEvent;++ /**+ * Validate raw event structure+ */+ abstract validate(rawEvent: TRawEvent): boolean;++ /**+ * Returns how many events will be imported+ */+ abstract getTotalEventsCount(): Promise<number>;++ /**+ * Optional hook: Pre-process batch+ */+ async beforeBatch?(events: TRawEvent[]): Promise<TRawEvent[]> {+ return events;+ }++ /**+ * Optional hook: Get import metadata for tracking+ */+ getImportMetadata?(): ImportJobMetadata;++ /**+ * Optional hook: Custom error handling+ */+ async onError?(error: Error, context?: ErrorContext): Promise<void> {+ // Default: re-throw+ throw error;+ }++ /**+ * Get estimated total events (optional, for progress tracking)+ */+ async getEstimatedTotal?(): Promise<number> {+ return 0;+ }++ /**+ * Indicates whether session IDs should be generated in SQL after import+ * If true, the import job will generate deterministic session IDs based on+ * device_id and timestamp using SQL window functions+ * If false, assumes the provider already generates session IDs during streaming+ */+ shouldGenerateSessionIds(): boolean {+ return false; // Default: assume provider handles it+ }++ /**+ * Utility: Split a date range into chunks to avoid timeout issues with large imports+ * Returns array of [from, to] date pairs in YYYY-MM-DD format+ *+ * @param from - Start date in YYYY-MM-DD format+ * @param to - End date in YYYY-MM-DD format+ * @param chunkSizeDays - Number of days per chunk (default: 1)+ */+ public getDateChunks(+ from: string,+ to: string,+ options?: {+ chunkSizeDays?: number;+ },+ ): Array<[string, string]> {+ const chunks: Array<[string, string]> = [];++ const startDate = new Date(from);+ const endDate = new Date(to);+ const chunkSizeDays = options?.chunkSizeDays ?? 1;++ // Handle case where from and to are the same date+ if (startDate.getTime() === endDate.getTime()) {+ return [[from, to]];+ }++ const cursor = new Date(startDate);++ while (cursor <= endDate) {+ const chunkStart = cursor.toISOString().split('T')[0]!;++ // Calculate chunk end: move forward by (chunkSizeDays - 1) to get the last day of the chunk+ const chunkEndDate = new Date(cursor);+ chunkEndDate.setDate(chunkEndDate.getDate() + (chunkSizeDays - 1));++ // Don't go past the end date+ const chunkEnd =+ chunkEndDate > endDate+ ? endDate.toISOString().split('T')[0]!+ : chunkEndDate.toISOString().split('T')[0]!;++ chunks.push([chunkStart, chunkEnd]);++ // Move cursor to the next chunk start (after the current chunk)+ cursor.setDate(cursor.getDate() + chunkSizeDays);++ if (cursor > endDate) break;+ }++ return chunks;+ }+}diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.tsnew file mode 100644index 000000000..52f4e80df--- /dev/null+++ b/packages/importer/src/index.ts@@ -0,0 +1,13 @@+export { UmamiProvider } from './providers/umami';+export { MixpanelProvider } from './providers/mixpanel';+export type {+ ImportConfig,+ ImportProgress,+ ImportResult,+ BatchResult,+ BaseRawEvent,+ ErrorContext,+ EventProperties,+ ImportJobMetadata,+ ImportStageResult,+} from './types';diff --git a/packages/importer/src/providers/metadata.ts b/packages/importer/src/providers/metadata.tsnew file mode 100644index 000000000..639394b72--- /dev/null+++ b/packages/importer/src/providers/metadata.ts@@ -0,0 +1,30 @@+export type ImportProviderId = 'umami' | 'mixpanel';+export type ImportProviderType = 'file' | 'api';++export interface ImportProviderMeta {+ id: ImportProviderId;+ name: string;+ description: string;+ logo: string;+ backgroundColor: string;+ types: ImportProviderType[];+}++export const IMPORT_PROVIDERS: ImportProviderMeta[] = [+ {+ id: 'umami',+ name: 'Umami',+ description: 'Import your analytics data from Umami',+ logo: 'https://cdn.brandfetch.io/id_3VEohOm/w/180/h/180/theme/dark/logo.png?c=1dxbfHSJFAPEGdCLU4o5B',+ backgroundColor: '#fff',+ types: ['file'],+ },+ {+ id: 'mixpanel',+ name: 'Mixpanel',+ description: 'Import your analytics data from Mixpanel API',+ logo: 'https://cdn.brandfetch.io/idr_rhI2FS/theme/dark/idMJ8uODLv.svg?c=1dxbfHSJFAPEGdCLU4o5B',+ backgroundColor: '#fff',+ types: ['api'],+ },+];diff --git a/packages/importer/src/providers/mixpanel.test.ts b/packages/importer/src/providers/mixpanel.test.tsnew file mode 100644index 000000000..c3f1d0524--- /dev/null+++ b/packages/importer/src/providers/mixpanel.test.ts@@ -0,0 +1,319 @@+import { omit } from 'ramda';+import { describe, expect, it } from 'vitest';+import { MixpanelProvider } from './mixpanel';++describe('mixpanel', () => {+ it('should chunk date range into day chunks', async () => {+ const provider = new MixpanelProvider('pid', {+ from: '2025-01-01',+ to: '2025-01-04',+ serviceAccount: 'sa',+ serviceSecret: 'ss',+ projectId: '123',+ provider: 'mixpanel',+ type: 'api',+ mapScreenViewProperty: undefined,+ });++ const chunks = provider.getDateChunks('2025-01-01', '2025-01-04');+ expect(chunks).toEqual([+ ['2025-01-01', '2025-01-01'],+ ['2025-01-02', '2025-01-02'],+ ['2025-01-03', '2025-01-03'],+ ['2025-01-04', '2025-01-04'],+ ]);+ });++ it('should transform event', async () => {+ const provider = new MixpanelProvider('pid', {+ from: '2025-01-01',+ to: '2025-01-02',+ serviceAccount: 'sa',+ serviceSecret: 'ss',+ projectId: '123',+ provider: 'mixpanel',+ type: 'api',+ mapScreenViewProperty: undefined,+ });++ const rawEvent = {+ event: '$mp_web_page_view',+ properties: {+ time: 1746097970,+ distinct_id: '$device:123',+ $browser: 'Chrome',+ $browser_version: 135,+ $city: 'Mumbai',+ $current_url:+ 'https://domain.com/state/maharashtra?utm_source=google&utm_medium=cpc&utm_campaignid=890&utm_adgroupid=&utm_adid=&utm_term=&utm_device=m&utm_network=x&utm_location=123&gclid=oqneoqow&gad_sour',+ $device: 'Android',+ $device_id: '123',+ $initial_referrer: 'https://referrer.com/',+ $initial_referring_domain: 'referrer.com',+ $insert_id: 'source_id',+ $lib_version: '2.60.0',+ $mp_api_endpoint: 'api-js.mixpanel.com',+ $mp_api_timestamp_ms: 1746078175363,+ $mp_autocapture: true,+ $os: 'Android',+ $referrer: 'https://google.com/',+ $referring_domain: 'referrer.com',+ $region: 'Maharashtra',+ $screen_height: 854,+ $screen_width: 384,+ current_domain: 'domain.com',+ current_page_title:+ 'Landeed: Satbara Utara, 7/12 Extract, Property Card & Index 2',+ current_url_path: '/state/maharashtra',+ current_url_protocol: 'https:',+ current_url_search:+ '?utm_source=google&utm_medium=cpc&utm_campaignid=890&utm_adgroupid=&utm_adid=&utm_term=&utm_device=m&utm_network=x&utm_location=123&gclid=oqneoqow&gad_source=5&gclid=EAIaIQobChMI6MnvhciBjQMVlS-DAx',+ gclid: 'oqneoqow',+ mp_country_code: 'IN',+ mp_lib: 'web',+ mp_processing_time_ms: 1746078175546,+ mp_sent_by_lib_version: '2.60.0',+ utm_medium: 'cpc',+ utm_source: 'google',+ },+ };++ const res = provider.transformEvent(rawEvent);++ expect(res).toMatchObject({+ id: expect.any(String),+ name: 'screen_view',+ device_id: '123',+ profile_id: '123',+ project_id: 'pid',+ session_id: '',+ properties: {+ __source_insert_id: 'source_id',+ __screen: '384x854',+ __lib_version: '2.60.0',+ '__query.utm_source': 'google',+ '__query.utm_medium': 'cpc',+ '__query.utm_campaignid': '890',+ '__query.utm_device': 'm',+ '__query.utm_network': 'x',+ '__query.utm_location': '123',+ '__query.gclid': 'oqneoqow',+ __title:+ 'Landeed: Satbara Utara, 7/12 Extract, Property Card & Index 2',+ },+ created_at: '2025-05-01T11:12:50.000Z',+ country: 'IN',+ city: 'Mumbai',+ region: 'Maharashtra',+ longitude: null,+ latitude: null,+ os: 'Android',+ os_version: undefined,+ browser: 'Chrome',+ browser_version: '',+ device: 'mobile',+ brand: '',+ model: '',+ duration: 0,+ path: '/state/maharashtra',+ origin: 'https://domain.com',+ referrer: 'https://referrer.com',+ referrer_name: 'Google',+ referrer_type: 'search',+ imported_at: expect.any(String),+ sdk_name: 'mixpanel (web)',+ sdk_version: '1.0.0',+ });+ });++ it('should parse stringified JSON in properties and flatten them', async () => {+ const provider = new MixpanelProvider('pid', {+ from: '2025-01-01',+ to: '2025-01-02',+ serviceAccount: 'sa',+ serviceSecret: 'ss',+ projectId: '123',+ provider: 'mixpanel',+ type: 'api',+ mapScreenViewProperty: undefined,+ });++ const rawEvent = {+ event: 'custom_event',+ properties: {+ time: 1746097970,+ distinct_id: '$device:123',+ $device_id: '123',+ $user_id: 'user123',+ mp_lib: 'web',+ // Stringified JSON object - should be parsed and flattened+ area: '{"displayText":"Malab, Nuh, Mewat","id":1189005}',+ // Stringified JSON array - should be parsed and flattened+ tags: '["tag1","tag2","tag3"]',+ // Regular string - should remain as is+ regularString: 'just a string',+ // Number - should be converted to string+ count: 42,+ // Object - should be flattened+ nested: { level1: { level2: 'value' } },+ },+ };++ const res = provider.transformEvent(rawEvent);++ expect(res.properties).toMatchObject({+ // Parsed JSON object should be flattened with dot notation+ 'area.displayText': 'Malab, Nuh, Mewat',+ 'area.id': '1189005',+ // Parsed JSON array should be flattened with numeric indices+ 'tags.0': 'tag1',+ 'tags.1': 'tag2',+ 'tags.2': 'tag3',+ // Regular values+ regularString: 'just a string',+ count: '42',+ // Nested object flattened+ 'nested.level1.level2': 'value',+ });+ });++ it('should handle react-native referrer', async () => {+ const provider = new MixpanelProvider('pid', {+ from: '2025-01-01',+ to: '2025-01-02',+ serviceAccount: 'sa',+ serviceSecret: 'ss',+ projectId: '123',+ provider: 'mixpanel',+ type: 'api',+ mapScreenViewProperty: undefined,+ });++ const rawEvent = {+ event: 'ec_search_error',+ properties: {+ time: 1759947367,+ distinct_id: '3385916',+ $browser: 'Mobile Safari',+ $browser_version: null,+ $city: 'Bengaluru',+ $current_url:+ 'https://web.landeed.com/karnataka/ec-encumbrance-certificate',+ $device: 'iPhone',+ $device_id:+ '199b498af1036c-0e943279a1292e-5c0f4368-51bf4-199b498af1036c',+ $initial_referrer: 'https://www.google.com/',+ $initial_referring_domain: 'www.google.com',+ $insert_id: 'bclkaepeqcfuzt4v',+ $lib_version: '2.60.0',+ $mp_api_endpoint: 'api-js.mixpanel.com',+ $mp_api_timestamp_ms: 1759927570699,+ $os: 'iOS',+ $region: 'Karnataka',+ $screen_height: 852,+ $screen_width: 393,+ $search_engine: 'google',+ $user_id: '3385916',+ binaryReadableVersion: 'NA',+ binaryVersion: 'NA',+ component: '/karnataka/ec-encumbrance-certificate',+ errMsg: 'Request failed with status code 500',+ errType: 'SERVER_ERROR',+ isSilentSearch: false,+ isTimeout: false,+ jsVersion: '0.42.0',+ language: 'english',+ mp_country_code: 'IN',+ mp_lib: 'web',+ mp_processing_time_ms: 1759927592421,+ mp_sent_by_lib_version: '2.60.0',+ os: 'web',+ osVersion:+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1',+ phoneBrand: 'NA',+ phoneManufacturer: 'NA',+ phoneModel: 'NA',+ searchUuid: '68e65d08-fd81-4ded-37d3-2b08d2bc70c3',+ serverVersion: 'web2.0',+ state: 17,+ stateStr: '17',+ statusCode: 500,+ type: 'result_event',+ utm_medium: 'cpc',+ utm_source:+ 'google%26utm_medium=cpc%26utm_campaignid=21380769590%26utm_adgroupid=%26utm_adid=%26utm_term=%26utm_device=m%26utm_network=%26utm_location=9062055%26gclid=%26gad_campaignid=21374496705%26gbraid=0AAAAAoV7mTM9mWFripzQ2Od0xXAfrW6p3%26wbraid=CmAKCQjwi4PHBhCUA',+ },+ };++ const res = provider.transformEvent(rawEvent);++ expect(res.id.length).toBeGreaterThan(30);+ expect(res.imported_at).toMatch(+ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/,+ );+ expect(omit(['id', 'imported_at'], res)).toEqual({+ brand: 'Apple',+ browser: 'GSA',+ browser_version: 'null',+ city: 'Bengaluru',+ country: 'IN',+ created_at: '2025-10-08T18:16:07.000Z',+ device: 'mobile',+ device_id: '199b498af1036c-0e943279a1292e-5c0f4368-51bf4-199b498af1036c',+ duration: 0,+ latitude: null,+ longitude: null,+ model: 'iPhone',+ name: 'ec_search_error',+ origin: 'https://web.landeed.com',+ os: 'iOS',+ os_version: '18.7.0',+ path: '/karnataka/ec-encumbrance-certificate',+ profile_id: '3385916',+ project_id: 'pid',+ properties: {+ __lib_version: '2.60.0',+ '__query.gad_campaignid': '21374496705',+ '__query.gbraid': '0AAAAAoV7mTM9mWFripzQ2Od0xXAfrW6p3',+ '__query.utm_campaignid': '21380769590',+ '__query.utm_device': 'm',+ '__query.utm_location': '9062055',+ '__query.utm_medium': 'cpc',+ '__query.utm_source': 'google',+ '__query.wbraid': 'CmAKCQjwi4PHBhCUA',+ __screen: '393x852',+ __source_insert_id: 'bclkaepeqcfuzt4v',+ __userAgent:+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1',+ binaryReadableVersion: 'NA',+ binaryVersion: 'NA',+ component: '/karnataka/ec-encumbrance-certificate',+ errMsg: 'Request failed with status code 500',+ errType: 'SERVER_ERROR',+ isSilentSearch: 'false',+ isTimeout: 'false',+ jsVersion: '0.42.0',+ language: 'english',+ os: 'web',+ osVersion:+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1',+ phoneBrand: 'NA',+ phoneManufacturer: 'NA',+ phoneModel: 'NA',+ searchUuid: '68e65d08-fd81-4ded-37d3-2b08d2bc70c3',+ serverVersion: 'web2.0',+ state: '17',+ stateStr: '17',+ statusCode: '500',+ type: 'result_event',+ },+ referrer: 'https://www.google.com',+ referrer_name: 'Google',+ referrer_type: 'search',+ region: 'Karnataka',+ sdk_name: 'mixpanel (web)',+ sdk_version: '1.0.0',+ session_id: '',+ });+ });+});diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/importer/src/providers/mixpanel.tsnew file mode 100644index 000000000..1d29bb331--- /dev/null+++ b/packages/importer/src/providers/mixpanel.ts@@ -0,0 +1,452 @@+import { randomUUID } from 'node:crypto';+import { isSameDomain, parsePath, toDots } from '@openpanel/common';+import { type UserAgentInfo, parseUserAgent } from '@openpanel/common/server';+import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';+import type { IClickhouseEvent } from '@openpanel/db';+import type { ILogger } from '@openpanel/logger';+import type { IMixpanelImportConfig } from '@openpanel/validation';+import { z } from 'zod';+import { BaseImportProvider } from '../base-provider';++export const zMixpanelRawEvent = z.object({+ event: z.string(),+ properties: z.record(z.unknown()),+});++export type MixpanelRawEvent = z.infer<typeof zMixpanelRawEvent>;++export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {+ provider = 'mixpanel';+ version = '1.0.0';++ constructor(+ private readonly projectId: string,+ private readonly config: IMixpanelImportConfig,+ private readonly logger?: ILogger,+ ) {+ super();+ }++ async getTotalEventsCount(): Promise<number> {+ // Mixpanel sucks and dont provide a good way to extract total event count within a period+ // jql would work but not accurate and will be deprecated end of 2025+ return -1;+ }++ /**+ * Mixpanel doesn't provide session IDs, so we need to generate them in SQL+ * after all events are imported to ensure deterministic results+ */+ shouldGenerateSessionIds(): boolean {+ return true;+ }++ async *parseSource(+ overrideFrom?: string,+ ): AsyncGenerator<MixpanelRawEvent, void, unknown> {+ yield* this.fetchEventsFromMixpanel(overrideFrom);+ }++ private async *fetchEventsFromMixpanel(+ overrideFrom?: string,+ ): AsyncGenerator<MixpanelRawEvent, void, unknown> {+ const { serviceAccount, serviceSecret, projectId, from, to } = this.config;++ // Split the date range into monthly chunks for reliability+ // Uses base class utility to avoid timeout issues with large date ranges+ const dateChunks = this.getDateChunks(overrideFrom ?? from, to); // 1 month per chunk++ for (const [chunkFrom, chunkTo] of dateChunks) {+ yield* this.fetchEventsForDateRange(+ serviceAccount,+ serviceSecret,+ projectId,+ chunkFrom,+ chunkTo,+ );+ }+ }++ private async *fetchEventsForDateRange(+ serviceAccount: string,+ serviceSecret: string,+ projectId: string,+ from: string,+ to: string,+ ): AsyncGenerator<MixpanelRawEvent, void, unknown> {+ const url = 'https://data.mixpanel.com/api/2.0/export';++ const params = new URLSearchParams({+ from_date: from,+ to_date: to,+ project_id: projectId,+ });++ this.logger?.info('Fetching events from Mixpanel', {+ url: `${url}?${params}`,+ from,+ to,+ projectId,+ serviceAccount,+ });++ const response = await fetch(`${url}?${params}`, {+ method: 'GET',+ headers: {+ Authorization: `Basic ${Buffer.from(`${serviceAccount}:${serviceSecret}`).toString('base64')}`,+ Accept: 'application/json',+ },+ });++ if (!response.ok) {+ throw new Error(+ `Failed to fetch events from Mixpanel: ${response.status} ${response.statusText}`,+ );+ }++ if (!response.body) {+ throw new Error('No response body from Mixpanel API');+ }++ // Stream the response line by line+ const reader = response.body.getReader();+ const decoder = new TextDecoder();+ let buffer = '';++ try {+ while (true) {+ const { done, value } = await reader.read();++ if (done) break;++ buffer += decoder.decode(value, { stream: true });++ // Process complete lines+ const lines = buffer.split('\n');+ buffer = lines.pop() || ''; // Keep the last incomplete line in buffer++ for (const line of lines) {+ if (line.trim()) {+ try {+ const event = JSON.parse(line);+ yield event;+ } catch (error) {+ console.warn('Failed to parse Mixpanel event:', line);+ }+ }+ }+ }++ // Process any remaining line in buffer+ if (buffer.trim()) {+ try {+ const event = JSON.parse(buffer);+ yield event;+ } catch (error) {+ console.warn('Failed to parse final Mixpanel event:', buffer);+ }+ }+ } finally {+ reader.releaseLock();+ }+ }++ validate(rawEvent: MixpanelRawEvent): boolean {+ const res = zMixpanelRawEvent.safeParse(rawEvent);+ return res.success;+ }++ transformEvent(_rawEvent: MixpanelRawEvent): IClickhouseEvent {+ const projectId = this.projectId;+ const rawEvent = zMixpanelRawEvent.parse(_rawEvent);+ const props = rawEvent.properties as Record<string, any>;+ const deviceId = props.$device_id;+ const profileId = String(props.$user_id || props.distinct_id).replace(+ /^\$device:/,+ '',+ );++ // Build full URL from current_url and current_url_search (web only)+ const fullUrl = props.$current_url;+ let path = '';+ let origin = '';+ let hash = '';+ let query: Record<string, string> = {};++ if (fullUrl) {+ const parsed = parsePath(fullUrl);+ path = parsed.path || '';+ origin = parsed.origin || '';+ hash = parsed.hash || '';+ query = parsed.query || {};+ } else if (this.config.mapScreenViewProperty) {+ path = props[this.config.mapScreenViewProperty] || '';+ }++ // Extract referrer information (web only)+ const referrerUrl = props.$initial_referrer || props.$referrer || '';+ const referrer =+ referrerUrl && !isSameDomain(referrerUrl, fullUrl)+ ? parseReferrer(referrerUrl)+ : null;++ // Check for UTM referrer in query params (web only)+ const utmReferrer = getReferrerWithQuery(query);++ // Extract location data+ const country = props.$country || props.mp_country_code || '';+ const city = props.$city || '';+ const region = props.$region || '';++ // For web events, use the standard user agent parsing+ const userAgent = props.osVersion || '';+ const uaInfo = this.isWebEvent(props.mp_lib)+ ? parseUserAgent(userAgent, props)+ : this.parseServerDeviceInfo(props);++ // Map event name - $mp_web_page_view should be screen_view+ let eventName = rawEvent.event;+ if (eventName === '$mp_web_page_view') {+ eventName = 'screen_view';+ }++ // Build properties object - strip Mixpanel-specific properties+ const properties = this.stripMixpanelProperties(props, query);++ if (props.$insert_id) {+ properties.__source_insert_id = String(props.$insert_id);+ }+ // Add useful properties+ if (props.$screen_width && props.$screen_height) {+ properties.__screen = `${props.$screen_width}x${props.$screen_height}`;+ }+ if (props.$screen_dpi) {+ properties.__dpi = props.$screen_dpi;+ }+ if (props.$language) {+ properties.__language = props.$language;+ }+ if (props.$timezone) {+ properties.__timezone = props.$timezone;+ }+ if (props.$app_version) {+ properties.__version = props.$app_version;+ }+ if (props.$app_build_number) {+ properties.__buildNumber = props.$app_build_number;+ }+ if (props.$lib_version) {+ properties.__lib_version = props.$lib_version;+ }++ if (hash) {+ properties.__hash = hash;+ }++ if (Object.keys(query).length > 0) {+ properties.__query = query;+ }++ if (props.current_page_title) {+ properties.__title = props.current_page_title;+ }++ if (userAgent) {+ properties.__userAgent = userAgent;+ }++ // Always use UUID for id to match ClickHouse UUID column+ const event = {+ id: randomUUID(),+ name: eventName,+ device_id: deviceId,+ profile_id: profileId,+ project_id: projectId,+ session_id: '', // Will be generated in SQL after import+ properties: toDots(properties), // Flatten nested objects/arrays to Map(String, String)+ created_at: new Date(props.time * 1000).toISOString(),+ country,+ city,+ region,+ longitude: null,+ latitude: null,+ os: uaInfo.os || props.$os,+ os_version: uaInfo.osVersion || props.$osVersion,+ browser: uaInfo.browser || props.$browser,+ browser_version:+ uaInfo.browserVersion || props.$browserVersion+ ? String(props.$browser_version)+ : '',+ device: this.getDeviceType(props.mp_lib, uaInfo, props),+ brand: uaInfo.brand || '',+ model: uaInfo.model || '',+ duration: 0,+ path,+ origin,+ referrer: referrer?.url || '',+ referrer_name: utmReferrer?.name || referrer?.name || '',+ referrer_type: referrer?.type || utmReferrer?.type || '',+ imported_at: new Date().toISOString(),+ sdk_name: props.mp_lib+ ? `${this.provider} (${props.mp_lib})`+ : this.provider,+ sdk_version: this.version,+ };++ // TODO: Remove this+ // Temporary fix for a client+ const isMightBeScreenView = this.getMightBeScreenView(rawEvent);+ if (isMightBeScreenView && event.name === 'Loaded a Screen') {+ event.name = 'screen_view';+ event.path = isMightBeScreenView;+ }++ // TODO: Remove this+ // This is a hack to get utm tags (not sure if this is just the testing project or all mixpanel projects)+ if (props.utm_source && !properties.__query?.utm_source) {+ const split = decodeURIComponent(props.utm_source).split('&');+ const query = Object.fromEntries(split.map((item) => item.split('=')));+ for (const [key, value] of Object.entries(query)) {+ if (key && value) {+ event.properties[`__query.${key}`] = String(value);+ } else if (+ value === undefined &&+ key &&+ props.utm_source.startsWith(key)+ ) {+ event.properties['__query.utm_source'] = String(key);+ }+ }+ }++ return event;+ }++ private getDeviceType(+ mp_lib: string,+ uaInfo: UserAgentInfo,+ props: Record<string, any>,+ ) {+ // Normalize lib/os/browser data+ const lib = (mp_lib || '').toLowerCase();+ const os = String(props.$os || uaInfo.os || '').toLowerCase();+ const browser = String(+ props.$browser || uaInfo.browser || '',+ ).toLowerCase();++ const isTabletOs = os === 'ipados' || os === 'ipad os' || os === 'ipad';++ // Strong hint from SDK library+ if (['android', 'iphone', 'react-native', 'swift', 'unity'].includes(lib)) {+ return isTabletOs ? 'tablet' : 'mobile';+ }++ // Web or unknown SDKs: infer from OS/Browser+ const isMobileSignal =+ os === 'ios' ||+ os === 'android' ||+ browser.includes('mobile safari') ||+ browser.includes('chrome ios') ||+ browser.includes('android mobile') ||+ browser.includes('samsung internet') ||+ browser.includes('mobile');++ if (isMobileSignal) {+ return 'mobile';+ }++ const isTabletSignal =+ isTabletOs ||+ browser.includes('tablet') ||+ // iPad often reports as Mac OS X with Mobile Safari+ (browser.includes('mobile safari') &&+ (os === 'mac os x' || os === 'macos'));++ if (isTabletSignal) {+ return 'tablet';+ }++ // Default to desktop+ return this.isServerEvent(mp_lib) ? 'server' : 'desktop';+ }++ private isWebEvent(mp_lib: string) {+ return [+ 'web',+ 'android',+ 'iphone',+ 'swift',+ 'unity',+ 'react-native',+ ].includes(mp_lib);+ }++ private isServerEvent(mp_lib: string) {+ return !this.isWebEvent(mp_lib);+ }++ private getMightBeScreenView(rawEvent: MixpanelRawEvent) {+ const props = rawEvent.properties as Record<string, any>;+ return Object.keys(props).find((key) => key.match(/^[A-Z1-9_]+$/));+ }++ private parseServerDeviceInfo(props: Record<string, any>): UserAgentInfo {+ // For mobile events, extract device information from Mixpanel properties+ const os = props.$os || props.os || '';+ const osVersion = props.$os_version || props.osVersion || '';+ const brand = props.$brand || props.phoneBrand || '';+ const model = props.$model || props.phoneModel || '';+ const device = os.toLowerCase();++ return {+ isServer: true,+ os: os,+ osVersion: osVersion,+ browser: '',+ browserVersion: '',+ device: device,+ brand: brand,+ model: model,+ };+ }++ private stripMixpanelProperties(+ properties: Record<string, any>,+ searchParams: Record<string, string>,+ ): Record<string, any> {+ const strip = [+ 'time',+ 'distinct_id',+ 'current_page_title',+ 'current_url_path',+ 'current_url_protocol',+ 'current_url_search',+ 'current_domain',+ ...Object.keys(searchParams),+ ];+ const filtered = Object.fromEntries(+ Object.entries(properties).filter(+ ([key]) => !key.match(/^(\$|mp_|utm_)/) && !strip.includes(key),+ ),+ );++ // Parse JSON strings back to objects/arrays so toDots() can flatten them+ const parsed: Record<string, any> = {};+ for (const [key, value] of Object.entries(filtered)) {+ if (+ typeof value === 'string' &&+ (value.startsWith('{') || value.startsWith('['))+ ) {+ try {+ parsed[key] = JSON.parse(value);+ } catch {+ parsed[key] = value; // Keep as string if parsing fails+ }+ } else {+ parsed[key] = value;+ }+ }++ return parsed;+ }+}diff --git a/packages/importer/src/providers/umami.ts b/packages/importer/src/providers/umami.tsnew file mode 100644index 000000000..f232e1b8e--- /dev/null+++ b/packages/importer/src/providers/umami.ts@@ -0,0 +1,382 @@+import { randomUUID } from 'node:crypto';+import { Readable } from 'node:stream';+import { pipeline } from 'node:stream/promises';+import { createBrotliDecompress, createGunzip } from 'node:zlib';+import { isSameDomain, parsePath } from '@openpanel/common';+import { generateDeviceId } from '@openpanel/common/server';+import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';+import type { IClickhouseEvent } from '@openpanel/db';+import type { ILogger } from '@openpanel/logger';+import type { IUmamiImportConfig } from '@openpanel/validation';+import { parse } from 'csv-parse';+import { assocPath } from 'ramda';+import { z } from 'zod';+import { BaseImportProvider } from '../base-provider';++export const zUmamiRawEvent = z.object({+ // Required fields+ event_type: z.coerce.number(),+ event_name: z.string(),+ created_at: z.coerce.date(),+ event_id: z.string().min(1),+ session_id: z.string().min(1),+ website_id: z.string().min(1),++ // Optional fields that might be empty+ visit_id: z.string().optional(),+ distinct_id: z.string().optional(),+ url_path: z.string().optional(),+ hostname: z.string().optional(),+ referrer_domain: z.string().optional(),+ referrer_path: z.string().optional(),+ referrer_query: z.string().optional(),+ referrer_name: z.string().optional(),+ referrer_type: z.string().optional(),+ country: z.string().optional(),+ city: z.string().optional(),+ region: z.string().optional(),+ browser: z.string().optional(),+ os: z.string().optional(),+ device: z.string().optional(),+ screen: z.string().optional(),+ language: z.string().optional(),+ utm_source: z.string().optional(),+ utm_medium: z.string().optional(),+ utm_campaign: z.string().optional(),+ utm_content: z.string().optional(),+ utm_term: z.string().optional(),+ page_title: z.string().optional(),+ gclid: z.string().optional(),+ fbclid: z.string().optional(),+ msclkid: z.string().optional(),+ ttclid: z.string().optional(),+ li_fat_id: z.string().optional(),+ twclid: z.string().optional(),+ url_query: z.string().optional(),+});+export type UmamiRawEvent = z.infer<typeof zUmamiRawEvent>;++export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {+ provider = 'umami';+ version = '1.0.0';++ constructor(+ private readonly projectId: string,+ private readonly config: IUmamiImportConfig,+ private readonly logger?: ILogger,+ ) {+ super();+ }++ async getTotalEventsCount(): Promise<number> {+ return -1;+ }++ async *parseSource(): AsyncGenerator<UmamiRawEvent, void, unknown> {+ yield* this.parseRemoteFile(this.config.fileUrl);+ }++ private async *parseRemoteFile(+ url: string,+ opts: {+ signal?: AbortSignal;+ maxBytes?: number;+ maxRows?: number;+ } = {},+ ): AsyncGenerator<UmamiRawEvent, void, unknown> {+ const { signal, maxBytes, maxRows } = opts;+ const controller = new AbortController();++ // Link to caller's signal for cancellation+ if (signal) {+ signal.addEventListener('abort', () => controller.abort(), {+ once: true,+ });+ }++ const res = await fetch(url, { signal: controller.signal });+ if (!res.ok || !res.body) {+ throw new Error(+ `Failed to fetch remote file: ${res.status} ${res.statusText}`,+ );+ }++ const contentType = res.headers.get('content-type') || '';+ const contentEnc = res.headers.get('content-encoding') || '';+ const contentLen = Number(res.headers.get('content-length') ?? 0);++ if (+ contentType &&+ !/text\/csv|text\/plain|application\/gzip|application\/octet-stream/i.test(+ contentType,+ )+ ) {+ console.warn(`Warning: Content-Type is ${contentType}, expected CSV-ish`);+ }++ if (maxBytes && contentLen && contentLen > maxBytes) {+ throw new Error(+ `Remote file exceeds size limit (${contentLen} > ${maxBytes})`,+ );+ }++ const looksGzip =+ /\.gz($|\?)/i.test(url) ||+ /gzip/i.test(contentEnc) ||+ /application\/gzip/i.test(contentType);+ const looksBr = /br/i.test(contentEnc) || /\.br($|\?)/i.test(url);++ // WHATWG -> Node stream+ const body = Readable.fromWeb(res.body as any);++ // Optional size guard during stream+ let seenBytes = 0;+ if (maxBytes) {+ body.on('data', (chunk: Buffer) => {+ seenBytes += chunk.length;+ if (seenBytes > maxBytes) {+ controller.abort();+ body.destroy(+ new Error(+ `Stream exceeded size limit (${seenBytes} > ${maxBytes})`,+ ),+ );+ }+ });+ }++ // Build decode chain (gzip/brotli -> CSV parser)+ const decompress = looksGzip+ ? createGunzip()+ : looksBr+ ? createBrotliDecompress()+ : null;++ const parser = parse({+ columns: true, // objects per row+ bom: true, // handle UTF-8 BOM+ relax_column_count: true,+ skip_empty_lines: true,+ });++ // Wire the pipeline for proper backpressure & error propagation+ (async () => {+ try {+ if (decompress) {+ await pipeline(body, decompress, parser, {+ signal: controller.signal,+ });+ } else {+ await pipeline(body, parser, { signal: controller.signal });+ }+ } catch (e) {+ parser.destroy(e as Error);+ }+ })().catch(() => {+ /* handled by iterator */+ });++ let rows = 0;+ try {+ for await (const record of parser) {+ rows++;+ if (maxRows && rows > maxRows) {+ controller.abort();+ throw new Error(`Row limit exceeded (${rows} > ${maxRows})`);+ }+ yield record as UmamiRawEvent;+ }+ } catch (err) {+ throw new Error(+ `Failed to parse remote file from ${url}: ${+ err instanceof Error ? err.message : String(err)+ }`,+ );+ } finally {+ controller.abort(); // ensure fetch stream is torn down+ }+ }++ validate(rawEvent: UmamiRawEvent): boolean {+ const res = zUmamiRawEvent.safeParse(rawEvent);+ return res.success;+ }++ transformEvent(_rawEvent: UmamiRawEvent): IClickhouseEvent {+ const projectId =+ this.config.projectMapper.find(+ (mapper) => mapper.from === _rawEvent.website_id,+ )?.to || this.projectId;++ const rawEvent = zUmamiRawEvent.parse(_rawEvent);+ // Extract device/profile ID - use visit_id as device_id, session_id for session tracking+ const deviceId =+ rawEvent.visit_id ||+ generateDeviceId({+ ip: rawEvent.visit_id!,+ ua: rawEvent.visit_id!,+ origin: projectId,+ salt: 'xxx',+ });+ const profileId = rawEvent.distinct_id || deviceId;++ // Parse URL if available - use same logic as real-time events+ const url = rawEvent.url_path+ ? `https://${[rawEvent.hostname, rawEvent.url_path, rawEvent.url_query]+ .filter(Boolean)+ .join('')}`+ : '';+ const { path, hash, query, origin } = parsePath(url);+ // Extract referrer information - use same logic as real-time events+ const referrerUrl = rawEvent.referrer_domain+ ? `https://${rawEvent.referrer_domain}${rawEvent.referrer_path || ''}`+ : '';++ // Check if referrer is from same domain (like real-time events do)+ const referrer = isSameDomain(referrerUrl, url)+ ? null+ : parseReferrer(referrerUrl);++ // Check for UTM referrer in query params (like real-time events do)+ const utmReferrer = getReferrerWithQuery(query);++ // Extract location data+ const country = rawEvent.country || '';+ const city = rawEvent.city || '';+ const region = rawEvent.region || '';++ // Extract browser/device info+ const browser = rawEvent.browser || '';+ const browserVersion = ''; // Not available in Umami CSV+ const os = rawEvent.os || '';+ const osVersion = ''; // Not available in Umami CSV+ const device = rawEvent.device || '';+ const brand = ''; // Not available in Umami CSV+ const model = ''; // Not available in Umami CSV++ let properties: Record<string, any> = {};++ if (query) {+ properties.__query = query;+ }++ // Add useful properties from Umami data+ if (rawEvent.page_title) properties.__title = rawEvent.page_title;+ if (rawEvent.screen) properties.__screen = rawEvent.screen;+ if (rawEvent.language) properties.__language = rawEvent.language;+ if (rawEvent.utm_source)+ properties = assocPath(+ ['__query', 'utm_source'],+ rawEvent.utm_source,+ properties,+ );+ if (rawEvent.utm_medium)+ properties = assocPath(+ ['__query', 'utm_medium'],+ rawEvent.utm_medium,+ properties,+ );+ if (rawEvent.utm_campaign)+ properties = assocPath(+ ['__query', 'utm_campaign'],+ rawEvent.utm_campaign,+ properties,+ );+ if (rawEvent.utm_content)+ properties = assocPath(+ ['__query', 'utm_content'],+ rawEvent.utm_content,+ properties,+ );+ if (rawEvent.utm_term)+ properties = assocPath(+ ['__query', 'utm_term'],+ rawEvent.utm_term,+ properties,+ );++ return {+ id: rawEvent.event_id || randomUUID(),+ name: rawEvent.event_type === 1 ? 'screen_view' : rawEvent.event_name,+ device_id: deviceId,+ profile_id: profileId,+ project_id: projectId,+ session_id: rawEvent.session_id || '',+ properties,+ created_at: rawEvent.created_at.toISOString(),+ country,+ city,+ region: this.mapRegion(region),+ longitude: null,+ latitude: null,+ os,+ os_version: osVersion,+ browser: this.mapBrowser(browser),+ browser_version: browserVersion,+ device: this.mapDevice(device),+ brand,+ model,+ duration: 0,+ path,+ origin,+ referrer: utmReferrer?.url || referrer?.url || '',+ referrer_name: utmReferrer?.name || referrer?.name || '',+ referrer_type: utmReferrer?.type || referrer?.type || '',+ imported_at: new Date().toISOString(),+ sdk_name: this.provider,+ sdk_version: this.version,+ };+ }++ mapRegion(region: string): string {+ return region.replace(/^[A-Z]{2}\-/, '');+ }++ mapDevice(device: string): string {+ const mapping: Record<string, string> = {+ desktop: 'desktop',+ laptop: 'desktop',+ mobile: 'mobile',+ tablet: 'tablet',+ smarttv: 'smarttv',+ Unknown: 'desktop',+ };++ return mapping[device] || 'desktop';+ }++ mapBrowser(browser: string): string {+ const mapping: Record<string, string> = {+ android: 'Android',+ aol: 'AOL',+ bb10: 'BlackBerry 10',+ beaker: 'Beaker',+ chrome: 'Chrome',+ 'chromium-webview': 'Chrome (webview)',+ crios: 'Chrome (iOS)',+ curl: 'Curl',+ edge: 'Edge',+ 'edge-chromium': 'Edge (Chromium)',+ 'edge-ios': 'Edge (iOS)',+ facebook: 'Facebook',+ firefox: 'Firefox',+ fxios: 'Firefox (iOS)',+ ie: 'IE',+ instagram: 'Instagram',+ ios: 'iOS',+ 'ios-webview': 'iOS (webview)',+ kakaotalk: 'KakaoTalk',+ miui: 'MIUI',+ opera: 'Opera',+ 'opera-mini': 'Opera Mini',+ phantomjs: 'PhantomJS',+ safari: 'Safari',+ samsung: 'Samsung',+ searchbot: 'Searchbot',+ silk: 'Silk',+ yandexbrowser: 'Yandex',+ };++ return mapping[browser] || browser || 'Unknown';+ }+}diff --git a/packages/importer/src/types.ts b/packages/importer/src/types.tsnew file mode 100644index 000000000..ae56bf29e--- /dev/null+++ b/packages/importer/src/types.ts@@ -0,0 +1,80 @@+import type {+ IImportedEvent,+ IServiceCreateEventPayload,+ IServiceImportedEventPayload,+} from '@openpanel/db';++export interface ImportConfig {+ projectId: string;+ provider: string;+ sourceType: 'file' | 'api';+ sourceLocation: string;+}++export interface SessionInfo {+ id: string;+ lastTimestamp: number;+ lastEvent: IServiceImportedEventPayload;+}++export interface ImportProgress {+ totalEvents: number;+ processedEvents: number;+ currentBatch: number;+ totalBatches: number;+}++export interface ImportResult {+ success: boolean;+ totalEvents: number;+ processedEvents: number;+ error?: string;+}++export interface BatchResult {+ events: IServiceImportedEventPayload[];+ sessionEvents: IServiceImportedEventPayload[];+}++// Generic types for raw events from different providers+export interface BaseRawEvent {+ [key: string]: unknown;+}++// Error context for better error handling+export interface ErrorContext {+ batchNumber?: number;+ batchSize?: number;+ eventIndex?: number;+ rawEvent?: BaseRawEvent;+ provider?: string;+}++// Properties type for events - more specific than Record<string, any>+export interface EventProperties {+ [key: string]:+ | string+ | number+ | boolean+ | null+ | undefined+ | Record<string, unknown>;+ __query?: Record<string, unknown>;+ __title?: string;+ __screen?: string;+ __language?: string;+}++// Import job metadata for tracking import progress+export interface ImportJobMetadata {+ importId: string;+ importStatus: 'pending' | 'processing' | 'processed' | 'failed';+ importedAt: Date;+}++// Result of import staging operations+export interface ImportStageResult {+ importId: string;+ totalEvents: number;+ insertedEvents: number;+}diff --git a/packages/importer/tsconfig.json b/packages/importer/tsconfig.jsonnew file mode 100644index 000000000..1d1748e91--- /dev/null+++ b/packages/importer/tsconfig.json@@ -0,0 +1,9 @@+{+ "extends": "../../tooling/typescript/base.json",+ "compilerOptions": {+ "outDir": "./dist",+ "rootDir": "./src"+ },+ "include": ["src/**/*"],+ "exclude": ["node_modules", "dist"]+}diff --git a/packages/importer/vitest.config.ts b/packages/importer/vitest.config.tsnew file mode 100644index 000000000..f87a2039f--- /dev/null+++ b/packages/importer/vitest.config.ts@@ -0,0 +1,3 @@+import { getSharedVitestConfig } from '../../vitest.shared';++export default getSharedVitestConfig({ __dirname });diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex 7511cd09f..9d07372a0 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -110,7 +110,6 @@ export const eventsGroupQueue = new GroupQueue<>({logger: queueLogger,namespace: 'group_events',- // @ts-expect-error - TODO: Fix this in groupmqredis: getRedisGroupQueue(),orderingMethod: 'in-memory',orderingWindowMs,@@ -166,6 +165,21 @@ export const notificationQueue = new Queue<NotificationQueuePayload>(},);+export type ImportQueuePayload = {+ type: 'import';+ payload: {+ importId: string;+ };+};++export const importQueue = new Queue<ImportQueuePayload>('import', {+ connection: getRedisQueue(),+ defaultJobOptions: {+ removeOnComplete: 10,+ removeOnFail: 50,+ },+});+export function addTrialEndingSoonJob(organizationId: string, delay: number) {return miscQueue.add('misc',diff --git a/packages/redis/package.json b/packages/redis/package.jsonindex f958f1d73..742886ab4 100644--- a/packages/redis/package.json+++ b/packages/redis/package.json@@ -8,7 +8,7 @@},"dependencies": {"@openpanel/json": "workspace:*",- "ioredis": "^5.7.0"+ "ioredis": "5.8.2"},"devDependencies": {"@openpanel/db": "workspace:*",diff --git a/packages/trpc/package.json b/packages/trpc/package.jsonindex 630e74d96..6632ce43f 100644--- a/packages/trpc/package.json+++ b/packages/trpc/package.json@@ -16,6 +16,7 @@"@openpanel/payments": "workspace:^","@openpanel/redis": "workspace:*","@openpanel/validation": "workspace:*",+ "@openpanel/queue": "workspace:*","@trpc-limiter/redis": "^0.0.2","@trpc/client": "^11.6.0","@trpc/server": "^11.6.0",diff --git a/packages/trpc/src/root.ts b/packages/trpc/src/root.tsindex 2d7888372..d852c9c1c 100644--- a/packages/trpc/src/root.ts+++ b/packages/trpc/src/root.ts@@ -4,6 +4,7 @@ import { chatRouter } from './routers/chat';import { clientRouter } from './routers/client';import { dashboardRouter } from './routers/dashboard';import { eventRouter } from './routers/event';+import { importRouter } from './routers/import';import { integrationRouter } from './routers/integration';import { notificationRouter } from './routers/notification';import { onboardingRouter } from './routers/onboarding';@@ -40,6 +41,7 @@ export const appRouter = createTRPCRouter({reference: referenceRouter,notification: notificationRouter,integration: integrationRouter,+ import: importRouter,auth: authRouter,subscription: subscriptionRouter,overview: overviewRouter,diff --git a/packages/trpc/src/routers/auth.ts b/packages/trpc/src/routers/auth.tsindex 34a9d19c3..ccfc7f7b3 100644--- a/packages/trpc/src/routers/auth.ts+++ b/packages/trpc/src/routers/auth.ts@@ -12,7 +12,7 @@ import {validateSessionToken,verifyPasswordHash,} from '@openpanel/auth';-import { generateSecureId } from '@openpanel/common/server/id';+import { generateSecureId } from '@openpanel/common/server';import {connectUserToOrganization,db,diff --git a/packages/trpc/src/routers/import.ts b/packages/trpc/src/routers/import.tsnew file mode 100644index 000000000..d77effa13--- /dev/null+++ b/packages/trpc/src/routers/import.ts@@ -0,0 +1,178 @@+import { z } from 'zod';++import { db } from '@openpanel/db';+import { importQueue } from '@openpanel/queue';+import { zCreateImport } from '@openpanel/validation';++import { getProjectAccess } from '../access';+import { TRPCAccessError } from '../errors';+import { createTRPCRouter, protectedProcedure } from '../trpc';++export const importRouter = createTRPCRouter({+ list: protectedProcedure+ .input(z.object({ projectId: z.string() }))+ .query(async ({ input, ctx }) => {+ const access = await getProjectAccess({+ projectId: input.projectId,+ userId: ctx.session.userId,+ });++ if (!access) {+ throw TRPCAccessError('You do not have access to this project');+ }++ return db.import.findMany({+ where: {+ projectId: input.projectId,+ },+ orderBy: {+ createdAt: 'desc',+ },+ });+ }),++ get: protectedProcedure+ .input(z.object({ id: z.string() }))+ .query(async ({ input, ctx }) => {+ const importRecord = await db.import.findUniqueOrThrow({+ where: {+ id: input.id,+ },+ include: {+ project: true,+ },+ });++ const access = await getProjectAccess({+ projectId: importRecord.projectId,+ userId: ctx.session.userId,+ });++ if (!access) {+ throw TRPCAccessError('You do not have access to this import');+ }++ return importRecord;+ }),++ create: protectedProcedure+ .input(zCreateImport)+ .mutation(async ({ input, ctx }) => {+ const access = await getProjectAccess({+ projectId: input.projectId,+ userId: ctx.session.userId,+ });++ if (!access || (typeof access !== 'boolean' && access.level === 'read')) {+ throw TRPCAccessError(+ 'You do not have permission to create imports for this project',+ );+ }++ // Create import record+ const importRecord = await db.import.create({+ data: {+ projectId: input.projectId,+ config: input.config,+ status: 'pending',+ },+ });++ // Add job to queue+ const job = await importQueue.add('import', {+ type: 'import',+ payload: {+ importId: importRecord.id,+ },+ });++ // Update import record with job ID+ await db.import.update({+ where: { id: importRecord.id },+ data: { jobId: job.id },+ });++ return {+ ...importRecord,+ jobId: job.id,+ };+ }),++ delete: protectedProcedure+ .input(z.object({ id: z.string() }))+ .mutation(async ({ input, ctx }) => {+ const importRecord = await db.import.findUniqueOrThrow({+ where: {+ id: input.id,+ },+ });++ const access = await getProjectAccess({+ projectId: importRecord.projectId,+ userId: ctx.session.userId,+ });++ if (!access || (typeof access !== 'boolean' && access.level === 'read')) {+ throw TRPCAccessError(+ 'You do not have permission to delete imports for this project',+ );+ }++ if (importRecord.jobId) {+ const job = await importQueue.getJob(importRecord.jobId);+ if (job) {+ await job.remove();+ }+ }++ return db.import.delete({+ where: {+ id: input.id,+ },+ });+ }),++ retry: protectedProcedure+ .input(z.object({ id: z.string() }))+ .mutation(async ({ input, ctx }) => {+ const importRecord = await db.import.findUniqueOrThrow({+ where: {+ id: input.id,+ },+ });++ const access = await getProjectAccess({+ projectId: importRecord.projectId,+ userId: ctx.session.userId,+ });++ if (!access || (typeof access !== 'boolean' && access.level === 'read')) {+ throw TRPCAccessError(+ 'You do not have permission to retry imports for this project',+ );+ }++ // Only allow retry for failed imports+ if (importRecord.status !== 'failed') {+ throw new Error('Only failed imports can be retried');+ }++ // Add new job to queue+ const job = await importQueue.add('import', {+ type: 'import',+ payload: {+ importId: importRecord.id,+ },+ });++ // Update import record+ return db.import.update({+ where: { id: importRecord.id },+ data: {+ jobId: job.id,+ status: 'pending',+ errorMessage: null,+ },+ });+ }),+});diff --git a/packages/trpc/src/routers/organization.ts b/packages/trpc/src/routers/organization.tsindex 21bf029a9..40ff4ea1d 100644--- a/packages/trpc/src/routers/organization.ts+++ b/packages/trpc/src/routers/organization.ts@@ -11,7 +11,7 @@ import {} from '@openpanel/db';import { zEditOrganization, zInviteUser } from '@openpanel/validation';-import { generateSecureId } from '@openpanel/common/server/id';+import { generateSecureId } from '@openpanel/common/server';import { sendEmail } from '@openpanel/email';import { addDays } from 'date-fns';import { getOrganizationAccess } from '../access';diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.tsindex 44ae45dee..0b1cefe60 100644--- a/packages/validation/src/index.ts+++ b/packages/validation/src/index.ts@@ -435,3 +435,54 @@ export const zEditOrganization = z.object({name: z.string().min(2),timezone: z.string().min(1),});++const zProjectMapper = z.object({+ from: z.string().min(1),+ to: z.string().min(1),+});++const createFileImportConfig = <T extends string>(provider: T) =>+ z.object({+ provider: z.literal(provider),+ type: z.literal('file'),+ fileUrl: z.string().url(),+ });++// Import configs+export const zUmamiImportConfig = createFileImportConfig('umami').extend({+ projectMapper: z.array(zProjectMapper),+});++export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;++export const zPlausibleImportConfig = createFileImportConfig('plausible');+export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;++export const zMixpanelImportConfig = z.object({+ provider: z.literal('mixpanel'),+ type: z.literal('api'),+ serviceAccount: z.string().min(1),+ serviceSecret: z.string().min(1),+ projectId: z.string().min(1),+ from: z.string().min(1),+ to: z.string().min(1),+ mapScreenViewProperty: z.string().optional(),+});+export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;++export type IImportConfig =+ | IUmamiImportConfig+ | IPlausibleImportConfig+ | IMixpanelImportConfig;++export const zCreateImport = z.object({+ projectId: z.string().min(1),+ provider: z.enum(['umami', 'plausible', 'mixpanel']),+ config: z.union([+ zUmamiImportConfig,+ zPlausibleImportConfig,+ zMixpanelImportConfig,+ ]),+});++export type ICreateImport = z.infer<typeof zCreateImport>;diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex f15819bd0..6b6e6377d 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -226,7 +226,7 @@ importers:specifier: ^4.1.0version: 4.1.0tsdown:- specifier: ^0.14.2+ specifier: 0.14.2version: 0.14.2([email protected])typescript:specifier: 'catalog:'@@ -373,6 +373,9 @@ importers:'@openpanel/constants':specifier: workspace:^version: link:../../packages/constants+ '@openpanel/importer':+ specifier: workspace:^+ version: link:../../packages/importer'@openpanel/integrations':specifier: workspace:^version: link:../../packages/integrations@@ -815,6 +818,9 @@ importers:'@openpanel/email':specifier: workspace:*version: link:../../packages/email+ '@openpanel/importer':+ specifier: workspace:*+ version: link:../../packages/importer'@openpanel/integrations':specifier: workspace:^version: link:../../packages/integrations@@ -874,7 +880,7 @@ importers:specifier: ^9.0.8version: 9.0.8tsdown:- specifier: ^0.14.2+ specifier: 0.14.2version: 0.14.2([email protected])typescript:specifier: 'catalog:'@@ -923,58 +929,6 @@ importers:specifier: 'catalog:'version: 5.9.3- packages/cli:- dependencies:- '@openpanel/common':- specifier: workspace:*- version: link:../common- arg:- specifier: ^5.0.2- version: 5.0.2- glob:- specifier: ^10.4.3- version: 10.4.5- inquirer:- specifier: ^9.3.5- version: 9.3.6- p-limit:- specifier: ^6.1.0- version: 6.1.0- progress:- specifier: ^2.0.3- version: 2.0.3- ramda:- specifier: ^0.29.1- version: 0.29.1- zod:- specifier: 'catalog:'- version: 3.24.2- devDependencies:- '@openpanel/db':- specifier: workspace:^- version: link:../db- '@openpanel/sdk':- specifier: workspace:*- version: link:../sdks/sdk- '@openpanel/tsconfig':- specifier: workspace:*- version: link:../../tooling/typescript- '@types/node':- specifier: 'catalog:'- version: 24.7.1- '@types/progress':- specifier: ^2.0.7- version: 2.0.7- '@types/ramda':- specifier: ^0.30.1- version: 0.30.1- tsup:- specifier: ^7.2.0- version: 7.3.0([email protected])([email protected])- typescript:- specifier: 'catalog:'- version: 5.9.3-packages/common:dependencies:'@openpanel/constants':@@ -1048,8 +1002,8 @@ importers:packages/db:dependencies:'@clickhouse/client':- specifier: ^1.2.0- version: 1.2.0+ specifier: ^1.12.1+ version: 1.12.1'@openpanel/common':specifier: workspace:*version: link:../common@@ -1186,6 +1140,55 @@ importers:specifier: 'catalog:'version: 5.9.3+ packages/importer:+ dependencies:+ '@openpanel/common':+ specifier: workspace:*+ version: link:../common+ '@openpanel/db':+ specifier: workspace:*+ version: link:../db+ '@openpanel/queue':+ specifier: workspace:*+ version: link:../queue+ '@openpanel/validation':+ specifier: workspace:*+ version: link:../validation+ csv-parse:+ specifier: ^6.1.0+ version: 6.1.0+ ramda:+ specifier: ^0.29.1+ version: 0.29.1+ uuid:+ specifier: ^9.0.1+ version: 9.0.1+ zod:+ specifier: 'catalog:'+ version: 3.24.2+ devDependencies:+ '@openpanel/logger':+ specifier: workspace:*+ version: link:../logger+ '@types/node':+ specifier: ^20.0.0+ version: 20.14.8+ '@types/ramda':+ specifier: ^0.31.1+ version: 0.31.1+ '@types/uuid':+ specifier: ^9.0.7+ version: 9.0.8+ bullmq:+ specifier: ^5.8.7+ version: 5.8.7+ typescript:+ specifier: ^5.0.0+ version: 5.9.3+ vitest:+ specifier: ^1.0.0+packages/integrations:dependencies:'@slack/bolt':@@ -1320,8 +1323,8 @@ importers:specifier: workspace:*version: link:../jsonioredis:- specifier: ^5.7.0- version: 5.7.0+ specifier: 5.8.2+ version: 5.8.2devDependencies:'@openpanel/db':specifier: workspace:*@@ -1509,6 +1512,9 @@ importers:'@openpanel/payments':specifier: workspace:^version: link:../payments+ '@openpanel/queue':+ specifier: workspace:*+ version: link:../queue'@openpanel/redis':specifier: workspace:*version: link:../redis@@ -2710,9 +2716,16 @@ packages:'@capsizecss/[email protected]':resolution: {integrity: sha512-GrSU71meACqcmIUxPYOJvGKF0yryjN/L1aCuE9DViCTJI7bfkjgYDPD1zbNDcINJwSSP6UaBZY9GAbYDO7re0Q==}+ '@clickhouse/[email protected]':+ resolution: {integrity: sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==}+'@clickhouse/[email protected]':resolution: {integrity: sha512-VfA/C/tVJ2eNe72CaQ7eXmai+yqFEvZjQZiNtvJoOMLP+Vtb6DzqH9nfkgsiHHMhUhhclvt2mFh6+euk1Ea5wA==}+ '@clickhouse/[email protected]':+ resolution: {integrity: sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==}+ engines: {node: '>=16'}+'@clickhouse/[email protected]':resolution: {integrity: sha512-zMp2EhMfp1IrFKr/NjDwNiLsf7nq68nW8lGKszwFe7Iglc6Z5PY9ZA9Hd0XqAk75Q1NmFrkGCP1r3JCM1Nm1Bw==}engines: {node: '>=16'}@@ -2866,6 +2879,12 @@ packages:cpu: [ppc64]os: [aix]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==}+ engines: {node: '>=12'}+ cpu: [ppc64]+ os: [aix]+'@esbuild/[email protected]':resolution: {integrity: sha512-WtKdFM7ls47zkKHFVzMz8opM7LkcsIp9amDUBIAWirg70RM71WRSjdILPsY5Uv1D42ZpUfaPILDlfactHgsRkw==}engines: {node: '>=18'}@@ -2908,6 +2927,12 @@ packages:cpu: [arm64]os: [android]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==}+ engines: {node: '>=12'}+ cpu: [arm64]+ os: [android]+'@esbuild/[email protected]':resolution: {integrity: sha512-Vsm497xFM7tTIPYK9bNTYJyF/lsP590Qc1WxJdlB6ljCbdZKU9SY8i7+Iin4kyhV/KV5J2rOKsBQbB77Ab7L/w==}engines: {node: '>=18'}@@ -2950,6 +2975,12 @@ packages:cpu: [arm]os: [android]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==}+ engines: {node: '>=12'}+ cpu: [arm]+ os: [android]+'@esbuild/[email protected]':resolution: {integrity: sha512-arAtTPo76fJ/ICkXWetLCc9EwEHKaeya4vMrReVlEIUCAUncH7M4bhMQ+M9Vf+FFOZJdTNMXNBrWwW+OXWpSew==}engines: {node: '>=18'}@@ -2992,6 +3023,12 @@ packages:cpu: [x64]os: [android]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==}+ engines: {node: '>=12'}+ cpu: [x64]+ os: [android]+'@esbuild/[email protected]':resolution: {integrity: sha512-t8GrvnFkiIY7pa7mMgJd7p8p8qqYIz1NYiAoKc75Zyv73L3DZW++oYMSHPRarcotTKuSs6m3hTOa5CKHaS02TQ==}engines: {node: '>=18'}@@ -3034,6 +3071,12 @@ packages:cpu: [arm64]os: [darwin]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==}+ engines: {node: '>=12'}+ cpu: [arm64]+ os: [darwin]+'@esbuild/[email protected]':resolution: {integrity: sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw==}engines: {node: '>=18'}@@ -3076,6 +3119,12 @@ packages:cpu: [x64]os: [darwin]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==}+ engines: {node: '>=12'}+ cpu: [x64]+ os: [darwin]+'@esbuild/[email protected]':resolution: {integrity: sha512-rgtz6flkVkh58od4PwTRqxbKH9cOjaXCMZgWD905JOzjFKW+7EiUObfd/Kav+A6Gyud6WZk9w+xu6QLytdi2OA==}engines: {node: '>=18'}@@ -3118,6 +3167,12 @@ packages:cpu: [arm64]os: [freebsd]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==}+ engines: {node: '>=12'}+ cpu: [arm64]+ os: [freebsd]+'@esbuild/[email protected]':resolution: {integrity: sha512-6Mtdq5nHggwfDNLAHkPlyLBpE5L6hwsuXZX8XNmHno9JuL2+bg2BX5tRkwjyfn6sKbxZTq68suOjgWqCicvPXA==}engines: {node: '>=18'}@@ -3160,6 +3215,12 @@ packages:cpu: [x64]os: [freebsd]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==}+ engines: {node: '>=12'}+ cpu: [x64]+ os: [freebsd]+'@esbuild/[email protected]':resolution: {integrity: sha512-D3H+xh3/zphoX8ck4S2RxKR6gHlHDXXzOf6f/9dbFt/NRBDIE33+cVa49Kil4WUjxMGW0ZIYBYtaGCa2+OsQwQ==}engines: {node: '>=18'}@@ -3202,6 +3263,12 @@ packages:cpu: [arm64]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==}+ engines: {node: '>=12'}+ cpu: [arm64]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-TDijPXTOeE3eaMkRYpcy3LarIg13dS9wWHRdwYRnzlwlA370rNdZqbcp0WTyyV/k2zSxfko52+C7jU5F9Tfj1g==}engines: {node: '>=18'}@@ -3244,6 +3311,12 @@ packages:cpu: [arm]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==}+ engines: {node: '>=12'}+ cpu: [arm]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-gJKIi2IjRo5G6Glxb8d3DzYXlxdEj2NlkixPsqePSZMhLudqPhtZ4BUrpIuTjJYXxvF9njql+vRjB2oaC9XpBw==}engines: {node: '>=18'}@@ -3286,6 +3359,12 @@ packages:cpu: [ia32]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==}+ engines: {node: '>=12'}+ cpu: [ia32]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-K40ip1LAcA0byL05TbCQ4yJ4swvnbzHscRmUilrmP9Am7//0UjPreh4lpYzvThT2Quw66MhjG//20mrufm40mA==}engines: {node: '>=18'}@@ -3328,6 +3407,12 @@ packages:cpu: [loong64]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==}+ engines: {node: '>=12'}+ cpu: [loong64]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-0mswrYP/9ai+CU0BzBfPMZ8RVm3RGAN/lmOMgW4aFUSOQBjA31UP8Mr6DDhWSuMwj7jaWOT0p0WoZ6jeHhrD7g==}engines: {node: '>=18'}@@ -3370,6 +3455,12 @@ packages:cpu: [mips64el]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==}+ engines: {node: '>=12'}+ cpu: [mips64el]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-hIKvXm0/3w/5+RDtCJeXqMZGkI2s4oMUGj3/jM0QzhgIASWrGO5/RlzAzm5nNh/awHE0A19h/CvHQe6FaBNrRA==}engines: {node: '>=18'}@@ -3412,6 +3503,12 @@ packages:cpu: [ppc64]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==}+ engines: {node: '>=12'}+ cpu: [ppc64]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-HcZh5BNq0aC52UoocJxaKORfFODWXZxtBaaZNuN3PUX3MoDsChsZqopzi5UupRhPHSEHotoiptqikjN/B77mYQ==}engines: {node: '>=18'}@@ -3454,6 +3551,12 @@ packages:cpu: [riscv64]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==}+ engines: {node: '>=12'}+ cpu: [riscv64]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-bEh7dMn/h3QxeR2KTy1DUszQjUrIHPZKyO6aN1X4BCnhfYhuQqedHaa5MxSQA/06j3GpiIlFGSsy1c7Gf9padw==}engines: {node: '>=18'}@@ -3496,6 +3599,12 @@ packages:cpu: [s390x]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==}+ engines: {node: '>=12'}+ cpu: [s390x]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-ZcQ6+qRkw1UcZGPyrCiHHkmBaj9SiCD8Oqd556HldP+QlpUIe2Wgn3ehQGVoPOvZvtHm8HPx+bH20c9pvbkX3g==}engines: {node: '>=18'}@@ -3538,6 +3647,12 @@ packages:cpu: [x64]os: [linux]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==}+ engines: {node: '>=12'}+ cpu: [x64]+ os: [linux]+'@esbuild/[email protected]':resolution: {integrity: sha512-vbutsFqQ+foy3wSSbmjBXXIJ6PL3scghJoM8zCL142cGaZKAdCZHyf+Bpu/MmX9zT9Q0zFBVKb36Ma5Fzfa8xA==}engines: {node: '>=18'}@@ -3604,6 +3719,12 @@ packages:cpu: [x64]os: [netbsd]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==}+ engines: {node: '>=12'}+ cpu: [x64]+ os: [netbsd]+'@esbuild/[email protected]':resolution: {integrity: sha512-hjQ0R/ulkO8fCYFsG0FZoH+pWgTTDreqpqY7UnQntnaKv95uP5iW3+dChxnx7C3trQQU40S+OgWhUVwCjVFLvg==}engines: {node: '>=18'}@@ -3676,6 +3797,12 @@ packages:cpu: [x64]os: [openbsd]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==}+ engines: {node: '>=12'}+ cpu: [x64]+ os: [openbsd]+'@esbuild/[email protected]':resolution: {integrity: sha512-4ir0aY1NGUhIC1hdoCzr1+5b43mw99uNwVzhIq1OY3QcEwPDO3B7WNXBzaKY5Nsf1+N11i1eOfFcq+D/gOS15Q==}engines: {node: '>=18'}@@ -3730,6 +3857,12 @@ packages:cpu: [x64]os: [sunos]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==}+ engines: {node: '>=12'}+ cpu: [x64]+ os: [sunos]+'@esbuild/[email protected]':resolution: {integrity: sha512-jVzdzsbM5xrotH+W5f1s+JtUy1UWgjU0Cf4wMvffTB8m6wP5/kx0KiaLHlbJO+dMgtxKV8RQ/JvtlFcdZ1zCPA==}engines: {node: '>=18'}@@ -3772,6 +3905,12 @@ packages:cpu: [arm64]os: [win32]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==}+ engines: {node: '>=12'}+ cpu: [arm64]+ os: [win32]+'@esbuild/[email protected]':resolution: {integrity: sha512-iKc8GAslzRpBytO2/aN3d2yb2z8XTVfNV0PjGlCxKo5SgWmNXx82I/Q3aG1tFfS+A2igVCY97TJ8tnYwpUWLCA==}engines: {node: '>=18'}@@ -3814,6 +3953,12 @@ packages:cpu: [ia32]os: [win32]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==}+ engines: {node: '>=12'}+ cpu: [ia32]+ os: [win32]+'@esbuild/[email protected]':resolution: {integrity: sha512-vQW36KZolfIudCcTnaTpmLQ24Ha1RjygBo39/aLkM2kmjkWmZGEJ5Gn9l5/7tzXA42QGIoWbICfg6KLLkIw6yw==}engines: {node: '>=18'}@@ -3856,6 +4001,12 @@ packages:cpu: [x64]os: [win32]+ '@esbuild/[email protected]':+ resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==}+ engines: {node: '>=12'}+ cpu: [x64]+ os: [win32]+'@esbuild/[email protected]':resolution: {integrity: sha512-7IAFPrjSQIJrGsK6flwg7NFmwBoSTyF3rl7If0hNUFQU4ilTsEPL6GuMuU9BfIWVVGuRnuIidkSMC+c0Otu8IA==}engines: {node: '>=18'}@@ -4178,9 +4329,6 @@ packages:resolution: {integrity: sha512-R7Gsg6elpuqdn55fBH2y9oYzrU/yKrSmIsDX4ROT51vohrECFzTf2zw9BfUbOW8xjfmM2QbVoVYdTwhrtEKWSQ==}engines: {node: '>=18'}- '@ioredis/[email protected]':- resolution: {integrity: sha512-M/T6Zewn7sDaBQEqIZ8Rb+i9y8qfGmq+5SDFSf9sA2lUZTmdDLVdOiQaeDp+Q4wElZ9HG1GAX5KhDaidp6LQsQ==}-'@ioredis/[email protected]':resolution: {integrity: sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==}@@ -8570,9 +8718,6 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA==}- '@types/[email protected]':- resolution: {integrity: sha512-m5ObIqwsUp6BZzyiy4RdZpzWGub9bqLJMvZDD0QMXhxjqMHMENlj+SqF5QxoUwaQNFe+8kz8XM8ZQhqkQPTgMQ==}-'@types/[email protected]':resolution: {integrity: sha512-CmyhGZanP88uuC5GpWU9q+fI61j2SkhO3UGMUdfYRE6Bcy0ccyzn1Rqj9YAB/ZY4kOXmNf0ocah5GtphmLMP6Q==}@@ -8588,9 +8733,6 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==}- '@types/[email protected]':- resolution: {integrity: sha512-iadjw02vte8qWx7U0YM++EybBha2CQLPGu9iJ97whVgJUT5Zq9MjAPYUnbfRI2Kpehimf1QjFJYxD0t8nqzu5w==}-'@types/[email protected]':resolution: {integrity: sha512-wA0UT0HeT2fGHzIFV9kWpYz5mdoyLxKrTgMdZQM++5h6pYAFH73HXcQhefg24nD1yivUFEn5KU+EF4b+CXJ4Wg==}@@ -8603,15 +8745,15 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-0BzWVKtSEtignlk+XBuK88Il5wzQwbRVfEkzE8iKm02NYHMGQ/9ffB05M+zXhTCqo50DOIAT9pNSJsjFMMG4rQ==}- '@types/[email protected]':- resolution: {integrity: sha512-aoyF/ADPL6N+/NXXfhPWF+Qj6w1Cql59m9wX0Gi15uyF+bpzXeLd63HPdiTDE2bmLXfNcVufsDPKmbfOrOzTBA==}-'@types/[email protected]':resolution: {integrity: sha512-PyzHvjCalm2BRYjAU6nIB3TprYwMNOUY/7P/N8bSzp9W/yM2YrtGtAnnVtaCNSeOZ8DzKyFDvaqQs7LnWwwmBA==}'@types/[email protected]':resolution: {integrity: sha512-1lWWZ/2YiNttGcIUxQwnvMuh55GIEbn/zlpzzEojAsbxquI/TXQZCRaXsfxG1CHjlqGoqxWePkvaM/5qYHNuvQ==}+ '@types/[email protected]':+ resolution: {integrity: sha512-Vt6sFXnuRpzaEj+yeutA0q3bcAsK7wdPuASIzR9LXqL4gJPyFw8im9qchlbp4ltuf3kDEIRmPJTD/Fkg60dn7g==}+'@types/[email protected]':resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==}@@ -8734,6 +8876,9 @@ packages:peerDependencies:vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0+ '@vitest/[email protected]':+ resolution: {integrity: sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog==}+'@vitest/[email protected]':resolution: {integrity: sha512-7FTQQuuLKmN1Ig/h+h/GO+44Q1IlglPlR2es4ab7Yvfx+Uk5xsv+Ykk+MEt/M2Yn/xGmzaLKxGw2lgy2bwuYqg==}@@ -8751,15 +8896,27 @@ packages:'@vitest/[email protected]':resolution: {integrity: sha512-i6FDiBeJUGLDKADw2Gb01UtUNb12yyXAqC/mmRWuYl+m/U9GS7s8us5ONmGkGpUUo7/iAYzI2ePVfOZTYvUifA==}+ '@vitest/[email protected]':+ resolution: {integrity: sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA==}+'@vitest/[email protected]':resolution: {integrity: sha512-Tae+ogtlNfFei5DggOsSUvkIaSuVywujMj6HzR97AHK6XK8i3BuVyIifWAm/sE3a15lF5RH9yQIrbXYuo0IFyA==}+ '@vitest/[email protected]':+ resolution: {integrity: sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ==}+'@vitest/[email protected]':resolution: {integrity: sha512-XVa5OPNTYUsyqG9skuUkFzAeFnEzDp8hQu7kZ0N25B1+6KjGm4hWLtURyBbsIAOekfWQ7Wuz/N/XXzgYO3deWQ==}+ '@vitest/[email protected]':+ resolution: {integrity: sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw==}+'@vitest/[email protected]':resolution: {integrity: sha512-x6w+ctOEmEXdWaa6TO4ilb7l9DxPR5bwEb6hILKuxfU1NqWT2mpJD9NJN7t3OTfxmVlOMrvtoFJGdgyzZ605lQ==}+ '@vitest/[email protected]':+ resolution: {integrity: sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g==}+'@vitest/[email protected]':resolution: {integrity: sha512-2Ltrpht4OmHO9+c/nmHtF09HWiyWdworqnHIwjfvDyWjuwKbdkcS9AnhsDn+8E2RM4x++foD1/tNuLPVvWG1Rg==}@@ -8920,10 +9077,6 @@ packages:resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==}engines: {node: '>=12'}- resolution: {integrity: sha512-BGcItUBWSMRgOCe+SVZJ+S7yTRG0eGt9cXAHev72yuGcY23hnLA7Bky5L/xLyPINoSN95geovfBkqoTlNZYa7w==}- engines: {node: '>=14'}-resolution: {integrity: sha512-HqZ5rWlFjGiV0tDm3UxxgNRqsOTniqoKZu0pIAfh7TZQMGuZK+hH0drySty0si0QXj1ieop4+SkSfPZBPPkHig==}engines: {node: '>=14'}@@ -9009,6 +9162,9 @@ packages:resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==}+ resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==}+resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==}engines: {node: '>=12'}@@ -9399,6 +9555,10 @@ packages:resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==}+ resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==}+ engines: {node: '>=4'}+resolution: {integrity: sha512-mCuXncKXk5iCLhfhwTc0izo0gtEmpz5CtG2y8GiOINBlMVS6v8TMRc5TaLWKS6692m9+dVVfzgeVxR5UxWHTYw==}engines: {node: '>=12'}@@ -9442,6 +9602,9 @@ packages:resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==}+ resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==}+resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==}engines: {node: '>= 16'}@@ -9836,6 +9999,9 @@ packages:resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}+ resolution: {integrity: sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==}+resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==}@@ -10123,6 +10289,10 @@ packages:resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==}+ resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==}+ engines: {node: '>=6'}+resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==}engines: {node: '>=6'}@@ -10245,6 +10415,10 @@ packages:resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==}+ resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==}+ engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}+resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==}engines: {node: '>=0.3.1'}@@ -10525,6 +10699,11 @@ packages:engines: {node: '>=12'}hasBin: true+ resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==}+ engines: {node: '>=12'}+ hasBin: true+resolution: {integrity: sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ==}engines: {node: '>=18'}@@ -11122,6 +11301,9 @@ packages:resolution: {integrity: sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==}engines: {node: '>=18'}+ resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==}+resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}engines: {node: '>= 0.4'}@@ -11594,10 +11776,6 @@ packages:resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==}- resolution: {integrity: sha512-NUcA93i1lukyXU+riqEyPtSEkyFq8tX90uL659J+qpCZ3rEdViB/APC58oAhIh3+bJln2hzdlZbBZsGNrlsR8g==}- engines: {node: '>=12.22.0'}-resolution: {integrity: sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==}engines: {node: '>=12.22.0'}@@ -12269,6 +12447,10 @@ packages:resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==}engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}+ resolution: {integrity: sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ==}+ engines: {node: '>=14'}+resolution: {integrity: sha512-arhlxbFRmoQHl33a0Zkle/YWlmNwoyt6QNZEIJcqNbdrsix5Lvc4HyyI3EnwxTYlZYc32EbYrQ8SzEZ7dqgg9A==}engines: {node: '>=14'}@@ -12378,6 +12560,9 @@ packages:resolution: {integrity: sha512-uvhvYPC8kGPjXT3MyKMrL3JitEAmDMp30lVkuq/590Mw9ok6pWcFCwXJveo0t5uqYw1UREQHofD+jVpdjBv8wg==}+ resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==}+resolution: {integrity: sha512-kkIp7XSkP78ZxJEsSxW3712C6teJVoeHHwgo9zJ380de7IYyJ2ISlxojcH2pC5OFLewESmnRi/+XCDIEEVyoug==}@@ -13329,8 +13514,8 @@ packages:resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==}engines: {node: '>=10'}- resolution: {integrity: sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==}+ resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==}engines: {node: '>=18'}@@ -13498,6 +13683,9 @@ packages:resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==}+ resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==}+resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==}engines: {node: '>= 14.16'}@@ -15107,6 +15295,9 @@ packages:resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==}engines: {node: '>=0.10.0'}+ resolution: {integrity: sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q==}+resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==}@@ -15374,6 +15565,10 @@ packages:resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==}engines: {node: '>=12.0.0'}+ resolution: {integrity: sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==}+ engines: {node: '>=14.0.0'}+resolution: {integrity: sha512-al6n+QEANGFOMf/dmUMsuS5/r9B06uwlyNjZZql/zv8J7ybHCgoihBNORZCY2mzUuAnomQa2JdhyHKzZxPCrFA==}engines: {node: ^18.0.0 || >=20.0.0}@@ -15382,6 +15577,10 @@ packages:resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==}engines: {node: '>=14.0.0'}+ resolution: {integrity: sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==}+ engines: {node: '>=14.0.0'}+resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==}engines: {node: '>=14.0.0'}@@ -15528,6 +15727,10 @@ packages:resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==}engines: {node: '>=4'}+ resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==}+ engines: {node: '>=4'}+resolution: {integrity: sha512-eaBzG6MxNzEn9kiwvtre90cXaNLkmadMWa1zQMs3XORCXNbsH/OewwbxC5ia9dCxIxnTAsSxXJaa/p5y8DlvJg==}engines: {node: '>=10'}@@ -15636,9 +15839,6 @@ packages:resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}- resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}-resolution: {integrity: sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==}@@ -16024,6 +16224,11 @@ packages:resolution: {integrity: sha512-+pZIP+U3pEJdDCeFmsXwHzV7vNHQC/eIbHklfe2ZCZqayYRH7lQbHcVgsJ0XOOv27hWs4jH4MONgXxHMObTMSA==}+ resolution: {integrity: sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA==}+ engines: {node: ^18.0.0 || >=20.0.0}+ hasBin: true+resolution: {integrity: sha512-uHV4plJ2IxCl4u1up1FQRrqclylKAogbtBfOTwcuJ28xFi+89PZ57BRh+naIRvH70HPwxy5QHYzg1OrEaC7AbA==}engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}@@ -16037,27 +16242,22 @@ packages:vite:optional: true- resolution: {integrity: sha512-5nXH+QsELbFKhsEfWLkHrvgRpTdGJzqOZ+utSdmPTvwHmvU6ITTm3xx+mRusihkcI8GeC7lCDyn3kDtiki9scw==}- engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}+ resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==}+ engines: {node: ^18.0.0 || >=20.0.0}hasBin: truepeerDependencies:- '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0- jiti: '>=1.21.0'+ '@types/node': ^18.0.0 || >=20.0.0less: '*'lightningcss: ^1.21.0sass: '*'sass-embedded: '*'stylus: '*'sugarss: '*'- terser: ^5.16.0- tsx: ^4.8.1- yaml: ^2.4.2+ terser: ^5.4.0peerDependenciesMeta:'@types/node':optional: true- jiti:- optional: trueless:optional: truelightningcss:@@ -16072,13 +16272,9 @@ packages:optional: trueterser:optional: true- tsx:- optional: true- yaml:- optional: true- resolution: {integrity: sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==}+ resolution: {integrity: sha512-5nXH+QsELbFKhsEfWLkHrvgRpTdGJzqOZ+utSdmPTvwHmvU6ITTm3xx+mRusihkcI8GeC7lCDyn3kDtiki9scw==}engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}hasBin: truepeerDependencies:@@ -16117,19 +16313,59 @@ packages:yaml:optional: true- resolution: {integrity: sha512-4nVGliEpxmhCL8DslSAUdxlB6+SMrhB0a1v5ijlh1xB1nEPuy1mxaHxysVucLHuWryAxLWg6a5ei+U4TLn/rFg==}- engines: {node: ^20.19.0 || >=22.12.0}+ resolution: {integrity: sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==}+ engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}hasBin: truepeerDependencies:- '@types/node': ^20.19.0 || >=22.12.0+ '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0jiti: '>=1.21.0'- less: ^4.0.0+ less: '*'lightningcss: ^1.21.0- sass: ^1.70.0- sass-embedded: ^1.70.0- stylus: '>=0.54.8'- sugarss: ^5.0.0+ sass: '*'+ sass-embedded: '*'+ stylus: '*'+ sugarss: '*'+ terser: ^5.16.0+ tsx: ^4.8.1+ yaml: ^2.4.2+ peerDependenciesMeta:+ '@types/node':+ optional: true+ jiti:+ optional: true+ less:+ optional: true+ lightningcss:+ optional: true+ sass:+ optional: true+ sass-embedded:+ optional: true+ stylus:+ optional: true+ sugarss:+ optional: true+ terser:+ optional: true+ tsx:+ optional: true+ yaml:+ optional: true++ resolution: {integrity: sha512-4nVGliEpxmhCL8DslSAUdxlB6+SMrhB0a1v5ijlh1xB1nEPuy1mxaHxysVucLHuWryAxLWg6a5ei+U4TLn/rFg==}+ engines: {node: ^20.19.0 || >=22.12.0}+ hasBin: true+ peerDependencies:+ '@types/node': ^20.19.0 || >=22.12.0+ jiti: '>=1.21.0'+ less: ^4.0.0+ lightningcss: ^1.21.0+ sass: ^1.70.0+ sass-embedded: ^1.70.0+ stylus: '>=0.54.8'+ sugarss: ^5.0.0terser: ^5.16.0tsx: ^4.8.1yaml: ^2.4.2@@ -16173,6 +16409,31 @@ packages:vite:optional: true+ resolution: {integrity: sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag==}+ engines: {node: ^18.0.0 || >=20.0.0}+ hasBin: true+ peerDependencies:+ '@edge-runtime/vm': '*'+ '@types/node': ^18.0.0 || >=20.0.0+ '@vitest/browser': 1.6.1+ '@vitest/ui': 1.6.1+ happy-dom: '*'+ jsdom: '*'+ peerDependenciesMeta:+ '@edge-runtime/vm':+ optional: true+ '@types/node':+ optional: true+ '@vitest/browser':+ optional: true+ '@vitest/ui':+ optional: true+ happy-dom:+ optional: true+ jsdom:+ optional: true+resolution: {integrity: sha512-188iM4hAHQ0km23TN/adso1q5hhwKqUpv+Sd6p5sOuh6FhQnRNW3IsiIpvxqahtBabsJ2SLZgmGSpcYK4wQYJw==}engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}@@ -17931,8 +18192,14 @@ snapshots:transitivePeerDependencies:- encoding+ '@clickhouse/[email protected]': {}+'@clickhouse/[email protected]': {}+ '@clickhouse/[email protected]':+ dependencies:+ '@clickhouse/client-common': 1.12.1+'@clickhouse/[email protected]':dependencies:'@clickhouse/client-common': 1.2.0@@ -18084,6 +18351,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18105,6 +18375,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18126,6 +18399,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18147,6 +18423,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18168,6 +18447,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18189,6 +18471,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18210,6 +18495,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18231,6 +18519,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18252,6 +18543,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18273,6 +18567,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18294,6 +18591,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18315,6 +18615,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18336,6 +18639,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18357,6 +18663,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18378,6 +18687,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18399,6 +18711,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18420,6 +18735,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18453,6 +18771,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18489,6 +18810,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18516,6 +18840,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18537,6 +18864,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18558,6 +18888,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -18579,6 +18912,9 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@esbuild/[email protected]':+ optional: true+'@esbuild/[email protected]':optional: true@@ -19150,8 +19486,6 @@ snapshots:'@inquirer/[email protected]': {}- '@ioredis/[email protected]': {}-'@ioredis/[email protected]': {}'@isaacs/[email protected]':@@ -19177,14 +19511,14 @@ snapshots:dependencies:'@jest/fake-timers': 29.7.0'@jest/types': 29.6.3- '@types/node': 24.7.1+ '@types/node': 20.14.8jest-mock: 29.7.0'@jest/[email protected]':dependencies:'@jest/types': 29.6.3'@sinonjs/fake-timers': 10.3.0- '@types/node': 24.7.1+ '@types/node': 20.14.8jest-message-util: 29.7.0jest-mock: 29.7.0jest-util: 29.7.0@@ -19197,7 +19531,7 @@ snapshots:dependencies:'@types/istanbul-lib-coverage': 2.0.6'@types/istanbul-reports': 3.0.4- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/yargs': 15.0.19chalk: 4.1.2@@ -19206,13 +19540,13 @@ snapshots:'@jest/schemas': 29.6.3'@types/istanbul-lib-coverage': 2.0.6'@types/istanbul-reports': 3.0.4- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/yargs': 17.0.32chalk: 4.1.2'@jridgewell/[email protected]':dependencies:- '@jridgewell/sourcemap-codec': 1.5.0+ '@jridgewell/sourcemap-codec': 1.5.5'@jridgewell/trace-mapping': 0.3.30'@jridgewell/[email protected]':@@ -19224,7 +19558,7 @@ snapshots:'@jridgewell/[email protected]':dependencies:'@jridgewell/set-array': 1.2.1- '@jridgewell/sourcemap-codec': 1.5.0+ '@jridgewell/sourcemap-codec': 1.5.5'@jridgewell/trace-mapping': 0.3.30'@jridgewell/[email protected]':@@ -19252,22 +19586,22 @@ snapshots:'@jridgewell/[email protected]':dependencies:'@jridgewell/resolve-uri': 3.1.2- '@jridgewell/sourcemap-codec': 1.5.0+ '@jridgewell/sourcemap-codec': 1.5.5'@jridgewell/[email protected]':dependencies:'@jridgewell/resolve-uri': 3.1.2- '@jridgewell/sourcemap-codec': 1.5.0+ '@jridgewell/sourcemap-codec': 1.5.5'@jridgewell/[email protected]':dependencies:'@jridgewell/resolve-uri': 3.1.2- '@jridgewell/sourcemap-codec': 1.5.0+ '@jridgewell/sourcemap-codec': 1.5.5'@jridgewell/[email protected]':dependencies:'@jridgewell/resolve-uri': 3.1.2- '@jridgewell/sourcemap-codec': 1.5.0+ '@jridgewell/sourcemap-codec': 1.5.5'@js-sdsl/[email protected]': {}@@ -23051,18 +23385,18 @@ snapshots:'@slack/[email protected]':dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8'@slack/[email protected]':dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8'@slack/[email protected]':dependencies:'@slack/logger': 3.0.0'@slack/web-api': 6.12.1'@types/jsonwebtoken': 8.5.9- '@types/node': 24.7.1+ '@types/node': 20.14.8jsonwebtoken: 9.0.2lodash.isstring: 4.0.1transitivePeerDependencies:@@ -23073,7 +23407,7 @@ snapshots:'@slack/logger': 4.0.0'@slack/web-api': 7.5.0'@types/jsonwebtoken': 9.0.9- '@types/node': 24.7.1+ '@types/node': 20.14.8jsonwebtoken: 9.0.2lodash.isstring: 4.0.1transitivePeerDependencies:@@ -23083,7 +23417,7 @@ snapshots:dependencies:'@slack/logger': 3.0.0'@slack/web-api': 6.12.1- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/ws': 7.4.7eventemitter3: 5.0.1finity: 0.5.4@@ -23100,7 +23434,7 @@ snapshots:'@slack/logger': 3.0.0'@slack/types': 2.14.0'@types/is-stream': 1.1.0- '@types/node': 24.7.1+ '@types/node': 20.14.8axios: 1.7.7eventemitter3: 3.1.2form-data: 2.5.1@@ -23115,7 +23449,7 @@ snapshots:dependencies:'@slack/logger': 4.0.0'@slack/types': 2.14.0- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/retry': 0.12.0axios: 1.7.7eventemitter3: 5.0.1@@ -23741,7 +24075,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:@@ -23775,19 +24109,19 @@ snapshots:'@types/[email protected]':dependencies:'@types/connect': 3.4.38- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]': {}@@ -23798,11 +24132,11 @@ snapshots:'@types/connect': 3.4.38'@types/express': 4.17.21'@types/keygrip': 1.0.6- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/[email protected]': {}@@ -23956,14 +24290,14 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/qs': 6.9.11'@types/range-parser': 1.2.7'@types/send': 0.17.4'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/qs': 6.9.11'@types/range-parser': 1.2.7'@types/send': 0.17.4@@ -24011,7 +24345,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/[email protected]': {}@@ -24027,7 +24361,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/[email protected]':dependencies:@@ -24051,7 +24385,7 @@ snapshots:'@types/http-errors': 2.0.4'@types/keygrip': 1.0.6'@types/koa-compose': 3.2.8- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:@@ -24085,7 +24419,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]': {}@@ -24097,11 +24431,11 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:@@ -24111,10 +24445,6 @@ snapshots:dependencies:undici-types: 5.26.5- '@types/[email protected]':- dependencies:- undici-types: 6.21.0-'@types/[email protected]':dependencies:undici-types: 7.14.0@@ -24131,14 +24461,10 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8pg-protocol: 1.6.1pg-types: 2.2.0- '@types/[email protected]':- dependencies:- '@types/node': 24.7.1-'@types/[email protected]': {}'@types/[email protected]': {}@@ -24149,10 +24475,6 @@ snapshots:dependencies:types-ramda: 0.29.7- '@types/[email protected]':- dependencies:- types-ramda: 0.30.1-'@types/[email protected]':dependencies:types-ramda: 0.30.1@@ -24161,6 +24483,10 @@ snapshots:dependencies:types-ramda: 0.31.0+ '@types/[email protected]':+ dependencies:+ types-ramda: 0.31.0+'@types/[email protected]': {}'@types/[email protected](@types/[email protected])':@@ -24204,13 +24530,13 @@ snapshots:'@types/[email protected]':dependencies:'@types/mime': 1.3.5- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:'@types/http-errors': 2.0.4'@types/mime': 3.0.4- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]': {}@@ -24224,11 +24550,11 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 22.18.0+ '@types/node': 20.14.8'@types/[email protected]':dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/[email protected]': {}@@ -24248,7 +24574,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8'@types/[email protected]':dependencies:@@ -24309,6 +24635,12 @@ snapshots:transitivePeerDependencies:- supports-color+ '@vitest/[email protected]':+ dependencies:+ '@vitest/spy': 1.6.1+ '@vitest/utils': 1.6.1+ chai: 4.5.0+'@vitest/[email protected]':dependencies:'@vitest/spy': 3.1.3@@ -24328,21 +24660,44 @@ snapshots:dependencies:tinyrainbow: 2.0.0+ '@vitest/[email protected]':+ dependencies:+ '@vitest/utils': 1.6.1+ p-limit: 5.0.0+ pathe: 1.1.2+'@vitest/[email protected]':dependencies:'@vitest/utils': 3.1.3pathe: 2.0.3+ '@vitest/[email protected]':+ dependencies:+ magic-string: 0.30.19+ pathe: 1.1.2+ pretty-format: 29.7.0+'@vitest/[email protected]':dependencies:'@vitest/pretty-format': 3.1.3magic-string: 0.30.17pathe: 2.0.3+ '@vitest/[email protected]':+ dependencies:+ tinyspy: 2.2.1+'@vitest/[email protected]':dependencies:tinyspy: 3.0.2+ '@vitest/[email protected]':+ dependencies:+ diff-sequences: 29.6.3+ estree-walker: 3.0.3+ loupe: 2.3.7+ pretty-format: 29.7.0+'@vitest/[email protected]':dependencies:'@vitest/pretty-format': 3.1.3@@ -24513,8 +24868,6 @@ snapshots:- [email protected]: {}-@@ -24616,6 +24969,8 @@ snapshots:+ [email protected]: {}+@@ -24693,9 +25048,9 @@ snapshots:xxhash-wasm: 1.1.0yargs-parser: 21.1.1yocto-spinner: 0.2.2- zod: 3.24.2- zod-to-json-schema: 3.24.5([email protected])- zod-to-ts: 1.2.0([email protected])([email protected])+ zod: 3.25.76+ zod-to-json-schema: 3.24.5([email protected])+ zod-to-ts: 1.2.0([email protected])([email protected])optionalDependencies:sharp: 0.33.5transitivePeerDependencies:@@ -25078,11 +25433,11 @@ snapshots:dependencies:cron-parser: 4.9.0- ioredis: 5.7.0+ ioredis: 5.8.2msgpackr: 1.10.1node-abort-controller: 3.1.1- semver: 7.6.0- tslib: 2.6.2+ semver: 7.7.2+ tslib: 2.7.0uuid: 9.0.1transitivePeerDependencies:- supports-color@@ -25205,6 +25560,16 @@ snapshots:+ dependencies:+ assertion-error: 1.1.0+ check-error: 1.0.3+ deep-eql: 4.1.4+ get-func-name: 2.0.2+ loupe: 2.3.7+ pathval: 1.1.1+ type-detect: 4.1.0+dependencies:assertion-error: 2.0.1@@ -25244,6 +25609,10 @@ snapshots:+ dependencies:+ get-func-name: 2.0.2+@@ -25305,7 +25674,7 @@ snapshots:dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8escape-string-regexp: 4.0.0is-wsl: 2.2.0lighthouse-logger: 1.4.2@@ -25314,7 +25683,7 @@ snapshots:dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8escape-string-regexp: 4.0.0is-wsl: 2.2.0lighthouse-logger: 1.4.2@@ -25660,6 +26029,8 @@ snapshots:+ [email protected]: {}+dependencies:internmap: 1.0.1@@ -25929,6 +26300,10 @@ snapshots:dependencies:character-entities: 2.0.2+ dependencies:+ type-detect: 4.1.0+@@ -26053,6 +26428,8 @@ snapshots:+ [email protected]: {}+@@ -26217,7 +26594,7 @@ snapshots:dependencies:'@types/cookie': 0.4.1'@types/cors': 2.8.17- '@types/node': 24.7.1+ '@types/node': 20.14.8accepts: 1.3.8base64id: 2.0.0cookie: 0.7.2@@ -26471,6 +26848,32 @@ snapshots:'@esbuild/win32-ia32': 0.19.12'@esbuild/win32-x64': 0.19.12+ optionalDependencies:+ '@esbuild/aix-ppc64': 0.21.5+ '@esbuild/android-arm': 0.21.5+ '@esbuild/android-arm64': 0.21.5+ '@esbuild/android-x64': 0.21.5+ '@esbuild/darwin-arm64': 0.21.5+ '@esbuild/darwin-x64': 0.21.5+ '@esbuild/freebsd-arm64': 0.21.5+ '@esbuild/freebsd-x64': 0.21.5+ '@esbuild/linux-arm': 0.21.5+ '@esbuild/linux-arm64': 0.21.5+ '@esbuild/linux-ia32': 0.21.5+ '@esbuild/linux-loong64': 0.21.5+ '@esbuild/linux-mips64el': 0.21.5+ '@esbuild/linux-ppc64': 0.21.5+ '@esbuild/linux-riscv64': 0.21.5+ '@esbuild/linux-s390x': 0.21.5+ '@esbuild/linux-x64': 0.21.5+ '@esbuild/netbsd-x64': 0.21.5+ '@esbuild/openbsd-x64': 0.21.5+ '@esbuild/sunos-x64': 0.21.5+ '@esbuild/win32-arm64': 0.21.5+ '@esbuild/win32-ia32': 0.21.5+ '@esbuild/win32-x64': 0.21.5+optionalDependencies:'@esbuild/aix-ppc64': 0.24.0@@ -27441,6 +27844,8 @@ snapshots:+ [email protected]: {}+dependencies:es-errors: 1.3.0@@ -28092,20 +28497,6 @@ snapshots:dependencies:loose-envify: 1.4.0- dependencies:- '@ioredis/commands': 1.3.0- cluster-key-slot: 1.1.2- debug: 4.4.0- denque: 2.1.0- lodash.defaults: 4.2.0- lodash.isarguments: 3.1.0- redis-errors: 1.2.0- redis-parser: 3.0.0- standard-as-callback: 2.1.0- transitivePeerDependencies:- - supports-color-dependencies:'@ioredis/commands': 1.4.0@@ -28388,7 +28779,7 @@ snapshots:'@jest/environment': 29.7.0'@jest/fake-timers': 29.7.0'@jest/types': 29.6.3- '@types/node': 24.7.1+ '@types/node': 20.14.8jest-mock: 29.7.0jest-util: 29.7.0@@ -28409,13 +28800,13 @@ snapshots:dependencies:'@jest/types': 29.6.3- '@types/node': 24.7.1+ '@types/node': 20.14.8jest-util: 29.7.0dependencies:'@jest/types': 29.6.3- '@types/node': 24.7.1+ '@types/node': 20.14.8chalk: 4.1.2ci-info: 3.9.0graceful-fs: 4.2.11@@ -28432,7 +28823,7 @@ snapshots:dependencies:- '@types/node': 24.7.1+ '@types/node': 20.14.8jest-util: 29.7.0merge-stream: 2.0.0supports-color: 8.1.1@@ -28763,6 +29154,11 @@ snapshots:+ dependencies:+ mlly: 1.8.0+ pkg-types: 1.3.1+dependencies:mlly: 1.8.0@@ -28860,6 +29256,10 @@ snapshots:+ dependencies:+ get-func-name: 2.0.2+@@ -30282,7 +30682,7 @@ snapshots:dependencies:yocto-queue: 0.1.0dependencies:yocto-queue: 1.1.1@@ -30455,6 +30855,8 @@ snapshots:+ [email protected]: {}+@@ -30757,7 +31159,7 @@ snapshots:'@protobufjs/path': 1.1.2'@protobufjs/pool': 1.1.0'@protobufjs/utf8': 1.1.0- '@types/node': 22.18.0+ '@types/node': 20.14.8long: 5.2.3@@ -32462,6 +32864,10 @@ snapshots:+ dependencies:+ js-tokens: 9.0.1+dependencies:js-tokens: 9.0.1@@ -32787,10 +33193,14 @@ snapshots:fdir: 6.5.0([email protected])picomatch: 4.0.3+ [email protected]: {}++ [email protected]: {}+@@ -32853,7 +33263,7 @@ snapshots:dependencies:- ansis: 4.1.0+ ansis: 4.2.0cac: 6.7.14chokidar: 4.0.3debug: 4.4.1@@ -32864,7 +33274,7 @@ snapshots:rolldown-plugin-dts: 0.15.9([email protected])([email protected])semver: 7.7.2tinyexec: 1.0.1- tinyglobby: 0.2.14+ tinyglobby: 0.2.15tree-kill: 1.2.2unconfig: 7.3.3optionalDependencies:@@ -32915,6 +33325,8 @@ snapshots:+ [email protected]: {}+@@ -33047,8 +33459,6 @@ snapshots:- [email protected]: {}-@@ -33387,6 +33797,24 @@ snapshots:d3-time: 3.1.0d3-timer: 3.0.1+ dependencies:+ cac: 6.7.14+ debug: 4.4.1+ pathe: 1.1.2+ picocolors: 1.1.1+ transitivePeerDependencies:+ - '@types/node'+ - less+ - lightningcss+ - sass+ - sass-embedded+ - stylus+ - sugarss+ - supports-color+ - terser+[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:cac: 6.7.14@@ -33419,6 +33847,17 @@ snapshots:- supports-color- typescript+ dependencies:+ esbuild: 0.21.5+ postcss: 8.5.6+ rollup: 4.52.5+ optionalDependencies:+ '@types/node': 20.14.8+ fsevents: 2.3.3+ lightningcss: 1.30.1+ terser: 5.27.1+[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:esbuild: 0.25.9@@ -33475,6 +33914,41 @@ snapshots:optionalDependencies:vite: 6.3.5(@types/[email protected])([email protected])([email protected])([email protected])([email protected])+ [email protected](@types/[email protected])([email protected])([email protected])([email protected]):+ dependencies:+ '@vitest/expect': 1.6.1+ '@vitest/runner': 1.6.1+ '@vitest/snapshot': 1.6.1+ '@vitest/spy': 1.6.1+ '@vitest/utils': 1.6.1+ acorn-walk: 8.3.2+ chai: 4.5.0+ debug: 4.4.1+ execa: 8.0.1+ local-pkg: 0.5.1+ magic-string: 0.30.19+ pathe: 1.1.2+ picocolors: 1.1.1+ std-env: 3.9.0+ strip-literal: 2.1.1+ tinybench: 2.9.0+ tinypool: 0.8.4+ why-is-node-running: 2.3.0+ optionalDependencies:+ '@types/node': 20.14.8+ jsdom: 26.1.0+ transitivePeerDependencies:+ - less+ - lightningcss+ - sass+ - sass-embedded+ - stylus+ - sugarss+ - supports-color+ - terser+[email protected](@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:'@vitest/expect': 3.1.3@@ -33867,10 +34341,14 @@ snapshots:dependencies:zod: 3.24.2+ dependencies:+ zod: 3.25.76+dependencies:typescript: 5.9.3- zod: 3.24.2+ zod: 3.25.76