Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

feat: new importer (#214)

Carl-Gerhard Lindesvärd212254d31a75
diff --git a/apps/api/package.json b/apps/api/package.json
index e42cc2435..9a14c7d35 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -64,7 +64,7 @@
"@types/uuid": "^10.0.0",
"@types/ws": "^8.5.14",
"js-yaml": "^4.1.0",
- "tsdown": "^0.14.2",
+ "tsdown": "0.14.2",
"typescript": "catalog:"
}
}
diff --git a/apps/start/Dockerfile b/apps/start/Dockerfile
index 369cd0246..4853b7a62 100644
--- a/apps/start/Dockerfile
+++ b/apps/start/Dockerfile
@@ -29,6 +29,7 @@ COPY apps/start/package.json ./apps/start/
COPY packages/trpc/package.json packages/trpc/
COPY packages/json/package.json packages/json/
COPY packages/common/package.json packages/common/
+COPY packages/importer/package.json packages/importer/
COPY packages/payments/package.json packages/payments/
COPY packages/constants/package.json packages/constants/
COPY packages/validation/package.json packages/validation/
@@ -85,6 +86,7 @@ COPY --from=build /app/packages/trpc/package.json ./packages/trpc/
COPY --from=build /app/packages/auth/package.json ./packages/auth/
COPY --from=build /app/packages/json/package.json ./packages/json/
COPY --from=build /app/packages/common/package.json ./packages/common/
+COPY --from=build /app/packages/importer/package.json ./packages/importer/
COPY --from=build /app/packages/payments/package.json ./packages/payments/
COPY --from=build /app/packages/constants/package.json ./packages/constants/
COPY --from=build /app/packages/validation/package.json ./packages/validation/
diff --git a/apps/start/package.json b/apps/start/package.json
index 4ab99a63a..d5b97664a 100644
--- a/apps/start/package.json
+++ b/apps/start/package.json
@@ -8,7 +8,7 @@
"preview": "vite preview",
"deploy": "npx wrangler deploy",
"cf-typegen": "wrangler types",
- "build": "vite build",
+ "build": "pnpm with-env vite build",
"serve": "vite preview",
"test": "vitest run",
"format": "biome format",
@@ -30,6 +30,7 @@
"@openpanel/common": "workspace:^",
"@openpanel/constants": "workspace:^",
"@openpanel/integrations": "workspace:^",
+ "@openpanel/importer": "workspace:^",
"@openpanel/json": "workspace:*",
"@openpanel/payments": "workspace:*",
"@openpanel/sdk-info": "workspace:^",
diff --git a/apps/start/src/components/integrations/integration-card.tsx b/apps/start/src/components/integrations/integration-card.tsx
index ab80513ce..c0a35edde 100644
--- a/apps/start/src/components/integrations/integration-card.tsx
+++ b/apps/start/src/components/integrations/integration-card.tsx
@@ -50,12 +50,15 @@ export function IntegrationCardHeaderButtons({
export function IntegrationCardLogoImage({
src,
backgroundColor,
+ className,
}: {
src: string;
backgroundColor: string;
+ className?: string;
}) {
return (
<IntegrationCardLogo
+ className={className}
style={{
backgroundColor,
}}
diff --git a/apps/start/src/components/project-mapper.tsx b/apps/start/src/components/project-mapper.tsx
new file mode 100644
index 000000000..92002a171
--- /dev/null
+++ b/apps/start/src/components/project-mapper.tsx
@@ -0,0 +1,116 @@
+import { Button } from '@/components/ui/button';
+import { Input } from '@/components/ui/input';
+import { Label } from '@/components/ui/label';
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from '@/components/ui/select';
+import type { IServiceProject } from '@openpanel/db';
+import { PlusIcon, XIcon } from 'lucide-react';
+import type {
+ Control,
+ FieldArrayWithId,
+ UseFieldArrayAppend,
+ UseFieldArrayRemove,
+ UseFormRegister,
+ UseFormSetValue,
+ UseFormWatch,
+} from 'react-hook-form';
+
+interface ProjectMapperProps {
+ fields: FieldArrayWithId<any, 'projectMapper', 'id'>[];
+ append: UseFieldArrayAppend<any, 'projectMapper'>;
+ remove: UseFieldArrayRemove;
+ projects: IServiceProject[];
+ register: UseFormRegister<any>;
+ watch: UseFormWatch<any>;
+ setValue: UseFormSetValue<any>;
+}
+
+export function ProjectMapper({
+ fields,
+ append,
+ remove,
+ projects,
+ register,
+ watch,
+ setValue,
+}: ProjectMapperProps) {
+ return (
+ <div className="space-y-2">
+ <div className="flex items-center justify-between">
+ <Label className="mb-0">Project Mapper (Optional)</Label>
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ onClick={() => append({ from: '', to: '' })}
+ >
+ <PlusIcon className="mr-1 h-4 w-4" />
+ Add Mapping
+ </Button>
+ </div>
+ {fields.length === 0 && (
+ <p className="text-sm text-muted-foreground leading-normal">
+ Map source project IDs to your OpenPanel projects. If you skip mapping
+ all data will be imported to your current project.
+ </p>
+ )}
+
+ {fields.length > 0 && (
+ <div className="space-y-2">
+ {fields.map((field, index) => (
+ <div key={field.id} className="gap-2 rounded-md border p-3 row">
+ <div className="grid grid-cols-2 gap-2">
+ <div className="flex-1">
+ <Label className="text-xs text-muted-foreground">
+ From (Source Project ID)
+ </Label>
+ <Input
+ placeholder="e.g., abc123"
+ {...register(`projectMapper.${index}.from`)}
+ className="mt-1"
+ />
+ </div>
+ <div className="flex-1">
+ <Label className="text-xs text-muted-foreground">
+ To (OpenPanel Project)
+ </Label>
+ <Select
+ value={watch(`projectMapper.${index}.to`)}
+ onValueChange={(value) =>
+ setValue(`projectMapper.${index}.to`, value)
+ }
+ >
+ <SelectTrigger className="mt-1 w-full" size="sm">
+ <SelectValue placeholder="Select project" />
+ </SelectTrigger>
+ <SelectContent>
+ {projects.slice(0, 10).map((project) => (
+ <SelectItem key={project.id} value={project.id}>
+ {project.name}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ </div>
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon"
+ onClick={() => remove(index)}
+ className="mt-5"
+ >
+ <XIcon className="h-4 w-4" />
+ </Button>
+ </div>
+ ))}
+ </div>
+ )}
+ </div>
+ );
+}
diff --git a/apps/start/src/components/ui/badge.tsx b/apps/start/src/components/ui/badge.tsx
index e63ad31fb..b8b3da2a8 100644
--- a/apps/start/src/components/ui/badge.tsx
+++ b/apps/start/src/components/ui/badge.tsx
@@ -9,6 +9,11 @@ const badgeVariants = cva(
{
variants: {
variant: {
+ success:
+ 'border-transparent bg-emerald-700 text-white [a&]:hover:bg-emerald-700/90',
+ warning:
+ 'border-transparent bg-yellow-500 text-white [a&]:hover:bg-yellow-500/90',
+ info: 'border-transparent bg-blue-500 text-white [a&]:hover:bg-blue-500/90',
default:
'border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90',
secondary:
diff --git a/apps/start/src/modals/add-import.tsx b/apps/start/src/modals/add-import.tsx
new file mode 100644
index 000000000..03a77e354
--- /dev/null
+++ b/apps/start/src/modals/add-import.tsx
@@ -0,0 +1,282 @@
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
+import { toast } from 'sonner';
+
+import { InputWithLabel, WithLabel } from '@/components/forms/input-with-label';
+import { ProjectMapper } from '@/components/project-mapper';
+import { Button } from '@/components/ui/button';
+import { useAppParams } from '@/hooks/use-app-params';
+import { useTRPC } from '@/integrations/trpc/react';
+import { cn } from '@/lib/utils';
+import { zodResolver } from '@hookform/resolvers/zod';
+import type {
+ IImportConfig,
+ IMixpanelImportConfig,
+ IUmamiImportConfig,
+} from '@openpanel/validation';
+import {
+ zMixpanelImportConfig,
+ zUmamiImportConfig,
+} from '@openpanel/validation';
+import { format } from 'date-fns';
+import { CalendarIcon } from 'lucide-react';
+import { useFieldArray, useForm } from 'react-hook-form';
+import type { z } from 'zod';
+import { popModal, pushModal } from '.';
+import { ModalContent, ModalHeader } from './Modal/Container';
+
+type Provider = 'umami' | 'plausible' | 'mixpanel';
+
+interface AddImportProps {
+ provider: Provider;
+ name: string;
+ types: ('file' | 'api')[];
+}
+
+type UmamiFormData = z.infer<typeof zUmamiImportConfig>;
+type MixpanelFormData = z.infer<typeof zMixpanelImportConfig>;
+
+interface UmamiImportProps {
+ onSubmit: (config: IUmamiImportConfig) => void;
+ isPending: boolean;
+ organizationId: string;
+}
+
+function UmamiImport({
+ onSubmit,
+ isPending,
+ organizationId,
+}: UmamiImportProps) {
+ const trpc = useTRPC();
+ const { data: projects = [] } = useQuery(
+ trpc.project.list.queryOptions({
+ organizationId,
+ }),
+ );
+
+ const form = useForm<UmamiFormData>({
+ resolver: zodResolver(zUmamiImportConfig),
+ defaultValues: {
+ provider: 'umami',
+ type: 'file',
+ fileUrl: '',
+ projectMapper: [],
+ },
+ });
+
+ const { fields, append, remove } = useFieldArray({
+ control: form.control,
+ name: 'projectMapper',
+ });
+
+ const handleSubmit = form.handleSubmit((data) => {
+ onSubmit(data);
+ });
+
+ return (
+ <form onSubmit={handleSubmit} className="space-y-4">
+ <div className="space-y-4 py-4">
+ <InputWithLabel
+ label="File URL"
+ placeholder="https://example.com/export.csv"
+ error={form.formState.errors.fileUrl?.message}
+ info="Provide a publicly accessible URL to your exported CSV file."
+ {...form.register('fileUrl')}
+ />
+
+ <ProjectMapper
+ fields={fields}
+ append={append}
+ remove={remove}
+ projects={projects}
+ register={form.register}
+ watch={form.watch}
+ setValue={form.setValue}
+ />
+ </div>
+
+ <div className="flex justify-between">
+ <Button type="button" variant="outline" onClick={() => popModal()}>
+ Cancel
+ </Button>
+ <Button type="submit" disabled={isPending}>
+ {isPending ? 'Starting...' : 'Start Import'}
+ </Button>
+ </div>
+ </form>
+ );
+}
+
+interface MixpanelImportProps {
+ onSubmit: (config: IMixpanelImportConfig) => void;
+ isPending: boolean;
+ organizationId: string;
+}
+
+function MixpanelImport({
+ onSubmit,
+ isPending,
+ organizationId,
+}: MixpanelImportProps) {
+ const trpc = useTRPC();
+ const form = useForm<MixpanelFormData>({
+ resolver: zodResolver(zMixpanelImportConfig),
+ defaultValues: {
+ provider: 'mixpanel',
+ type: 'api',
+ serviceAccount: '',
+ serviceSecret: '',
+ projectId: '',
+ from: '',
+ to: '',
+ },
+ });
+
+ const handleDateRangeSelect = () => {
+ pushModal('DateRangerPicker', {
+ startDate: form.getValues('from')
+ ? new Date(form.getValues('from'))
+ : undefined,
+ endDate: form.getValues('to')
+ ? new Date(form.getValues('to'))
+ : undefined,
+ onChange: ({ startDate, endDate }) => {
+ form.setValue('from', format(startDate, 'yyyy-MM-dd'));
+ form.setValue('to', format(endDate, 'yyyy-MM-dd'));
+ form.trigger('from');
+ form.trigger('to');
+ },
+ });
+ };
+
+ const handleSubmit = form.handleSubmit((data) => {
+ onSubmit(data);
+ });
+
+ return (
+ <form onSubmit={handleSubmit} className="space-y-4">
+ <div className="space-y-4 py-4">
+ <InputWithLabel
+ label="Service Account"
+ placeholder="Eg. xxx.xxx.mp-service-account"
+ error={form.formState.errors.serviceAccount?.message}
+ {...form.register('serviceAccount')}
+ />
+
+ <InputWithLabel
+ label="Service Secret"
+ type="password"
+ placeholder="Your Mixpanel service secret"
+ error={form.formState.errors.serviceSecret?.message}
+ {...form.register('serviceSecret')}
+ />
+
+ <InputWithLabel
+ label="Project ID"
+ placeholder="Your Mixpanel project ID"
+ error={form.formState.errors.projectId?.message}
+ {...form.register('projectId')}
+ />
+
+ <WithLabel
+ label="Date Range"
+ info={
+ !form.getValues('from') || !form.getValues('to')
+ ? 'Select the date range for importing data'
+ : undefined
+ }
+ >
+ <Button
+ type="button"
+ variant="outline"
+ className={cn(
+ 'w-full justify-start text-left font-normal',
+ (!form.getValues('from') || !form.getValues('to')) &&
+ 'text-muted-foreground',
+ )}
+ onClick={handleDateRangeSelect}
+ >
+ <CalendarIcon className="mr-2 h-4 w-4" />
+ {form.getValues('from') && form.getValues('to') ? (
+ <>
+ {format(new Date(form.getValues('from')), 'LLL dd, y')} -{' '}
+ {format(new Date(form.getValues('to')), 'LLL dd, y')}
+ </>
+ ) : (
+ <span>Pick a date range</span>
+ )}
+ </Button>
+ </WithLabel>
+
+ <InputWithLabel
+ label="Screen View Property"
+ placeholder="Enter the name of the property that contains the screen name"
+ info="Leave empty if not applicable"
+ error={form.formState.errors.mapScreenViewProperty?.message}
+ {...form.register('mapScreenViewProperty')}
+ />
+ </div>
+
+ <div className="flex justify-between">
+ <Button type="button" variant="outline" onClick={() => popModal()}>
+ Cancel
+ </Button>
+ <Button type="submit" disabled={isPending}>
+ {isPending ? 'Starting...' : 'Start Import'}
+ </Button>
+ </div>
+ </form>
+ );
+}
+
+export default function AddImport({ provider, name }: AddImportProps) {
+ const { projectId, organizationId } = useAppParams();
+ const trpc = useTRPC();
+ const queryClient = useQueryClient();
+
+ const createImport = useMutation(
+ trpc.import.create.mutationOptions({
+ onSuccess() {
+ toast.success('Import started', {
+ description: 'Your data import has been queued for processing.',
+ });
+ popModal();
+ queryClient.invalidateQueries(trpc.import.list.pathFilter());
+ },
+ onError: (error) => {
+ toast.error('Import failed', {
+ description: error.message,
+ });
+ },
+ }),
+ );
+
+ const handleImportSubmit = (config: IImportConfig) => {
+ createImport.mutate({
+ projectId,
+ provider: config.provider,
+ config,
+ });
+ };
+
+ return (
+ <ModalContent>
+ <ModalHeader title={`Import from ${name}`} />
+
+ {provider === 'umami' && (
+ <UmamiImport
+ onSubmit={handleImportSubmit}
+ isPending={createImport.isPending}
+ organizationId={organizationId}
+ />
+ )}
+
+ {provider === 'mixpanel' && (
+ <MixpanelImport
+ onSubmit={handleImportSubmit}
+ isPending={createImport.isPending}
+ organizationId={organizationId}
+ />
+ )}
+ </ModalContent>
+ );
+}
diff --git a/apps/start/src/modals/date-ranger-picker.tsx b/apps/start/src/modals/date-ranger-picker.tsx
index 7c1142291..277fa163c 100644
--- a/apps/start/src/modals/date-ranger-picker.tsx
+++ b/apps/start/src/modals/date-ranger-picker.tsx
@@ -27,6 +27,7 @@ export default function DateRangerPicker({
return (
<ModalContent className="p-4 md:p-8 min-w-fit">
<Calendar
+ captionLayout="dropdown"
initialFocus
mode="range"
defaultMonth={subMonths(
diff --git a/apps/start/src/modals/index.tsx b/apps/start/src/modals/index.tsx
index 47ac6c5c1..709951d8f 100644
--- a/apps/start/src/modals/index.tsx
+++ b/apps/start/src/modals/index.tsx
@@ -5,6 +5,7 @@ import OverviewTopPagesModal from '@/components/overview/overview-top-pages-moda
import Instructions from './Instructions';
import AddClient from './add-client';
import AddDashboard from './add-dashboard';
+import AddImport from './add-import';
import AddIntegration from './add-integration';
import AddNotificationRule from './add-notification-rule';
import AddProject from './add-project';
@@ -38,6 +39,7 @@ const modals = {
EditClient: EditClient,
AddProject: AddProject,
AddClient: AddClient,
+ AddImport: AddImport,
Confirm: Confirm,
SaveReport: SaveReport,
AddDashboard: AddDashboard,
diff --git a/apps/start/src/routeTree.gen.ts b/apps/start/src/routeTree.gen.ts
index 9840b4d69..e221cf0ea 100644
--- a/apps/start/src/routeTree.gen.ts
+++ b/apps/start/src/routeTree.gen.ts
@@ -56,6 +56,7 @@ import { Route as AppOrganizationIdProjectIdSettingsTabsIndexRouteImport } from
import { Route as AppOrganizationIdProjectIdProfilesTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.profiles._tabs.index'
import { Route as AppOrganizationIdProjectIdNotificationsTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.notifications._tabs.index'
import { Route as AppOrganizationIdProjectIdEventsTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.events._tabs.index'
+import { Route as AppOrganizationIdProjectIdSettingsTabsImportsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.imports'
import { Route as AppOrganizationIdProjectIdSettingsTabsEventsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.events'
import { Route as AppOrganizationIdProjectIdSettingsTabsDetailsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.details'
import { Route as AppOrganizationIdProjectIdSettingsTabsClientsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.clients'
@@ -381,6 +382,12 @@ const AppOrganizationIdProjectIdEventsTabsIndexRoute =
path: '/',
getParentRoute: () => AppOrganizationIdProjectIdEventsTabsRoute,
} as any)
+const AppOrganizationIdProjectIdSettingsTabsImportsRoute =
+ AppOrganizationIdProjectIdSettingsTabsImportsRouteImport.update({
+ id: '/imports',
+ path: '/imports',
+ getParentRoute: () => AppOrganizationIdProjectIdSettingsTabsRoute,
+ } as any)
const AppOrganizationIdProjectIdSettingsTabsEventsRoute =
AppOrganizationIdProjectIdSettingsTabsEventsRouteImport.update({
id: '/events',
@@ -515,6 +522,7 @@ export interface FileRoutesByFullPath {
'/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
'/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
'/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
+ '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
'/$organizationId/$projectId/events/': typeof AppOrganizationIdProjectIdEventsTabsIndexRoute
'/$organizationId/$projectId/notifications/': typeof AppOrganizationIdProjectIdNotificationsTabsIndexRoute
'/$organizationId/$projectId/profiles/': typeof AppOrganizationIdProjectIdProfilesTabsIndexRoute
@@ -569,6 +577,7 @@ export interface FileRoutesByTo {
'/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
'/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
'/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
+ '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
'/$organizationId/$projectId/profiles/$profileId/events': typeof AppOrganizationIdProjectIdProfilesProfileIdTabsEventsRoute
}
export interface FileRoutesById {
@@ -633,6 +642,7 @@ export interface FileRoutesById {
'/_app/$organizationId/$projectId_/settings/_tabs/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
'/_app/$organizationId/$projectId_/settings/_tabs/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
'/_app/$organizationId/$projectId_/settings/_tabs/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
+ '/_app/$organizationId/$projectId_/settings/_tabs/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
'/_app/$organizationId/$projectId_/events/_tabs/': typeof AppOrganizationIdProjectIdEventsTabsIndexRoute
'/_app/$organizationId/$projectId_/notifications/_tabs/': typeof AppOrganizationIdProjectIdNotificationsTabsIndexRoute
'/_app/$organizationId/$projectId_/profiles/_tabs/': typeof AppOrganizationIdProjectIdProfilesTabsIndexRoute
@@ -692,6 +702,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/settings/clients'
| '/$organizationId/$projectId/settings/details'
| '/$organizationId/$projectId/settings/events'
+ | '/$organizationId/$projectId/settings/imports'
| '/$organizationId/$projectId/events/'
| '/$organizationId/$projectId/notifications/'
| '/$organizationId/$projectId/profiles/'
@@ -746,6 +757,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/settings/clients'
| '/$organizationId/$projectId/settings/details'
| '/$organizationId/$projectId/settings/events'
+ | '/$organizationId/$projectId/settings/imports'
| '/$organizationId/$projectId/profiles/$profileId/events'
id:
| '__root__'
@@ -809,6 +821,7 @@ export interface FileRouteTypes {
| '/_app/$organizationId/$projectId_/settings/_tabs/clients'
| '/_app/$organizationId/$projectId_/settings/_tabs/details'
| '/_app/$organizationId/$projectId_/settings/_tabs/events'
+ | '/_app/$organizationId/$projectId_/settings/_tabs/imports'
| '/_app/$organizationId/$projectId_/events/_tabs/'
| '/_app/$organizationId/$projectId_/notifications/_tabs/'
| '/_app/$organizationId/$projectId_/profiles/_tabs/'
@@ -1194,6 +1207,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppOrganizationIdProjectIdEventsTabsIndexRouteImport
parentRoute: typeof AppOrganizationIdProjectIdEventsTabsRoute
}
+ '/_app/$organizationId/$projectId_/settings/_tabs/imports': {
+ id: '/_app/$organizationId/$projectId_/settings/_tabs/imports'
+ path: '/imports'
+ fullPath: '/$organizationId/$projectId/settings/imports'
+ preLoaderRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRouteImport
+ parentRoute: typeof AppOrganizationIdProjectIdSettingsTabsRoute
+ }
'/_app/$organizationId/$projectId_/settings/_tabs/events': {
id: '/_app/$organizationId/$projectId_/settings/_tabs/events'
path: '/events'
@@ -1521,6 +1541,7 @@ interface AppOrganizationIdProjectIdSettingsTabsRouteChildren {
AppOrganizationIdProjectIdSettingsTabsClientsRoute: typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
AppOrganizationIdProjectIdSettingsTabsDetailsRoute: typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
AppOrganizationIdProjectIdSettingsTabsEventsRoute: typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
+ AppOrganizationIdProjectIdSettingsTabsImportsRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
AppOrganizationIdProjectIdSettingsTabsIndexRoute: typeof AppOrganizationIdProjectIdSettingsTabsIndexRoute
}
@@ -1532,6 +1553,8 @@ const AppOrganizationIdProjectIdSettingsTabsRouteChildren: AppOrganizationIdProj
AppOrganizationIdProjectIdSettingsTabsDetailsRoute,
AppOrganizationIdProjectIdSettingsTabsEventsRoute:
AppOrganizationIdProjectIdSettingsTabsEventsRoute,
+ AppOrganizationIdProjectIdSettingsTabsImportsRoute:
+ AppOrganizationIdProjectIdSettingsTabsImportsRoute,
AppOrganizationIdProjectIdSettingsTabsIndexRoute:
AppOrganizationIdProjectIdSettingsTabsIndexRoute,
}
diff --git a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx
new file mode 100644
index 000000000..123e78757
--- /dev/null
+++ b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx
@@ -0,0 +1,303 @@
+import { FullPageEmptyState } from '@/components/full-page-empty-state';
+import {
+ IntegrationCard,
+ IntegrationCardFooter,
+ IntegrationCardLogoImage,
+} from '@/components/integrations/integration-card';
+import { Skeleton } from '@/components/skeleton';
+import { Badge } from '@/components/ui/badge';
+import { Button } from '@/components/ui/button';
+import {
+ Table,
+ TableBody,
+ TableCell,
+ TableHead,
+ TableHeader,
+ TableRow,
+} from '@/components/ui/table';
+import { Tooltiper } from '@/components/ui/tooltip';
+import { useAppParams } from '@/hooks/use-app-params';
+import { useTRPC } from '@/integrations/trpc/react';
+import { pushModal } from '@/modals';
+import { IMPORT_PROVIDERS } from '@openpanel/importer/providers';
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
+import { createFileRoute } from '@tanstack/react-router';
+import { formatDistanceToNow } from 'date-fns';
+import {
+ CheckCircleIcon,
+ Download,
+ InfoIcon,
+ Loader2Icon,
+ XCircleIcon,
+} from 'lucide-react';
+import { toast } from 'sonner';
+
+export const Route = createFileRoute(
+ '/_app/$organizationId/$projectId_/settings/_tabs/imports',
+)({
+ component: ImportsSettings,
+});
+
+function ImportsSettings() {
+ const { projectId } = useAppParams();
+ const trpc = useTRPC();
+ const queryClient = useQueryClient();
+
+ const importsQuery = useQuery(
+ trpc.import.list.queryOptions(
+ { projectId },
+ {
+ refetchInterval: 5000,
+ },
+ ),
+ );
+ const imports = importsQuery.data ?? [];
+
+ const deleteImport = useMutation(
+ trpc.import.delete.mutationOptions({
+ onSuccess: () => {
+ toast.success('Import deleted', {
+ description: 'The import has been successfully deleted.',
+ });
+ queryClient.invalidateQueries(trpc.import.list.pathFilter());
+ },
+ }),
+ );
+
+ const retryImport = useMutation(
+ trpc.import.retry.mutationOptions({
+ onSuccess: () => {
+ toast.success('Import retried', {
+ description: 'The import has been queued for processing again.',
+ });
+ queryClient.invalidateQueries(trpc.import.list.pathFilter());
+ },
+ }),
+ );
+
+ const handleProviderSelect = (
+ provider: (typeof IMPORT_PROVIDERS)[number],
+ ) => {
+ pushModal('AddImport', {
+ provider: provider.id,
+ name: provider.name,
+ types: provider.types,
+ });
+ };
+
+ const getStatusBadge = (status: string, errorMessage: string | null) => {
+ const variants: Record<string, any> = {
+ pending: 'secondary',
+ processing: 'default',
+ completed: 'success',
+ failed: 'destructive',
+ };
+ const icons: Record<string, React.ReactNode> = {
+ pending: <Loader2Icon className="w-4 h-4 animate-spin" />,
+ processing: <Loader2Icon className="w-4 h-4 animate-spin" />,
+ completed: <CheckCircleIcon className="w-4 h-4" />,
+ failed: <XCircleIcon className="w-4 h-4" />,
+ };
+
+ if (status === 'failed') {
+ return (
+ <Tooltiper
+ content={errorMessage}
+ tooltipClassName="max-w-xs break-words"
+ >
+ <Badge variant={variants[status] || 'default'} className="capitalize">
+ {icons[status] || null}
+ {status}
+ </Badge>
+ </Tooltiper>
+ );
+ }
+
+ return (
+ <Badge variant={variants[status] || 'default'} className="capitalize">
+ {icons[status] || null}
+ {status}
+ </Badge>
+ );
+ };
+
+ return (
+ <div className="space-y-8">
+ <div>
+ <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
+ {IMPORT_PROVIDERS.map((provider) => (
+ <IntegrationCard
+ key={provider.id}
+ icon={
+ <IntegrationCardLogoImage
+ src={provider.logo}
+ backgroundColor={provider.backgroundColor}
+ className="p-4"
+ />
+ }
+ name={provider.name}
+ description={provider.description}
+ >
+ <IntegrationCardFooter className="row justify-end">
+ <Button
+ variant="ghost"
+ onClick={() => handleProviderSelect(provider)}
+ >
+ <Download className="w-4 h-4 mr-2" />
+ Import Data
+ </Button>
+ </IntegrationCardFooter>
+ </IntegrationCard>
+ ))}
+ </div>
+ </div>
+
+ <div>
+ <h3 className="text-lg font-medium mb-4">Import History</h3>
+
+ <div className="border rounded-lg">
+ <Table>
+ <TableHeader>
+ <TableRow>
+ <TableHead>Provider</TableHead>
+ <TableHead>Created</TableHead>
+ <TableHead>Status</TableHead>
+ <TableHead>Progress</TableHead>
+ <TableHead>Config</TableHead>
+ <TableHead className="text-right">Actions</TableHead>
+ </TableRow>
+ </TableHeader>
+ <TableBody>
+ {!importsQuery.isLoading && imports.length === 0 && (
+ <TableRow>
+ <TableCell colSpan={6}>
+ <FullPageEmptyState
+ title="No imports yet"
+ description="Your import history will appear here."
+ />
+ </TableCell>
+ </TableRow>
+ )}
+ {importsQuery.isLoading &&
+ [1, 2, 3, 4].map((index) => (
+ <TableRow key={index}>
+ <TableCell>
+ <Skeleton className="h-4 w-3/5" />
+ </TableCell>
+ <TableCell>
+ <Skeleton className="h-4 w-3/5" />
+ </TableCell>
+ <TableCell>
+ <Skeleton className="h-4 w-3/5" />
+ </TableCell>
+ <TableCell>
+ <Skeleton className="h-4 w-3/5" />
+ </TableCell>
+ <TableCell>
+ <Skeleton className="h-4 w-3/5" />
+ </TableCell>
+ <TableCell className="text-right justify-end row">
+ <Skeleton className="h-4 w-3/5" />
+ </TableCell>
+ </TableRow>
+ ))}
+ {imports.map((imp) => (
+ <TableRow key={imp.id}>
+ <TableCell className="font-medium capitalize">
+ <div className="row gap-2 items-center">
+ <div>{imp.config.provider}</div>
+ <Badge variant="outline" className="uppercase">
+ {imp.config.type}
+ </Badge>
+ </div>
+ </TableCell>
+ <TableCell>
+ {formatDistanceToNow(new Date(imp.createdAt), {
+ addSuffix: true,
+ })}
+ </TableCell>
+ <TableCell>
+ <div className="space-y-1">
+ {getStatusBadge(imp.status, imp.errorMessage)}
+ {imp.statusMessage && (
+ <div className="text-xs text-muted-foreground truncate">
+ {imp.statusMessage}
+ </div>
+ )}
+ </div>
+ </TableCell>
+ <TableCell>
+ {imp.totalEvents > 0 ? (
+ <div className="space-y-1 font-mono">
+ <div className="text-sm">
+ {imp.processedEvents.toLocaleString()}
+ {' / '}
+ <Tooltiper
+ content="Estimated number of events. Can be inaccurate depending on the provider."
+ tooltipClassName="max-w-xs"
+ >
+ {imp.totalEvents.toLocaleString()}{' '}
+ <InfoIcon className="w-4 h-4 inline-block relative -top-px" />
+ </Tooltiper>
+ </div>
+ {imp.status === 'processing' && (
+ <div className="w-full bg-secondary rounded-full h-1.5">
+ <div
+ className="bg-primary h-1.5 rounded-full transition-all"
+ style={{
+ width: `${Math.min(Math.round((imp.processedEvents / imp.totalEvents) * 100), 100)}%`,
+ }}
+ />
+ </div>
+ )}
+ </div>
+ ) : imp.totalEvents === -1 ? (
+ <div className="font-mono text-sm">
+ {imp.processedEvents.toLocaleString()}
+ {' / '}
+ N/A
+ </div>
+ ) : (
+ '-'
+ )}
+ </TableCell>
+
+ <TableCell>
+ <Tooltiper
+ content={
+ <pre className="font-mono text-sm leading-normal whitespace-pre-wrap break-words">
+ {JSON.stringify(imp.config, null, 2)}
+ </pre>
+ }
+ tooltipClassName="max-w-xs"
+ >
+ <Badge>Config</Badge>
+ </Tooltiper>
+ </TableCell>
+ <TableCell className="text-right space-x-2">
+ {imp.status === 'failed' && (
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={() => retryImport.mutate({ id: imp.id })}
+ >
+ Retry
+ </Button>
+ )}
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={() => deleteImport.mutate({ id: imp.id })}
+ >
+ Delete
+ </Button>
+ </TableCell>
+ </TableRow>
+ ))}
+ </TableBody>
+ </Table>
+ </div>
+ </div>
+ </div>
+ );
+}
diff --git a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx
index 39dc0cb31..0c7f217a8 100644
--- a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx
@@ -42,6 +42,7 @@ function ProjectDashboard() {
{ id: 'details', label: 'Details' },
{ id: 'events', label: 'Events' },
{ id: 'clients', label: 'Clients' },
+ { id: 'imports', label: 'Imports' },
];
const handleTabChange = (tabId: string) => {
diff --git a/apps/start/src/utils/math.ts b/apps/start/src/utils/math.ts
index e75c1964d..e41d4e1f5 100644
--- a/apps/start/src/utils/math.ts
+++ b/apps/start/src/utils/math.ts
@@ -1 +1 @@
-export * from '@openpanel/common/src/math';
+export * from '@openpanel/common';
diff --git a/apps/start/src/utils/op.ts b/apps/start/src/utils/op.ts
index 3e9cfee1e..0ba0fa951 100644
--- a/apps/start/src/utils/op.ts
+++ b/apps/start/src/utils/op.ts
@@ -1,8 +1,20 @@
import { OpenPanel } from '@openpanel/web';
-export const op = new OpenPanel({
- clientId: import.meta.env.VITE_OP_CLIENT_ID,
- trackScreenViews: true,
- trackOutgoingLinks: true,
- trackAttributes: true,
-});
+const clientId = import.meta.env.VITE_OP_CLIENT_ID;
+
+const createOpInstance = () => {
+ if (!clientId || clientId === 'undefined') {
+ return new Proxy({} as OpenPanel, {
+ get: () => () => {},
+ });
+ }
+
+ return new OpenPanel({
+ clientId,
+ trackScreenViews: true,
+ trackOutgoingLinks: true,
+ trackAttributes: true,
+ });
+};
+
+export const op = createOpInstance();
diff --git a/apps/start/src/utils/slug.ts b/apps/start/src/utils/slug.ts
index ebdad3b9f..e41d4e1f5 100644
--- a/apps/start/src/utils/slug.ts
+++ b/apps/start/src/utils/slug.ts
@@ -1 +1 @@
-export * from '@openpanel/common/src/slug';
+export * from '@openpanel/common';
diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile
index 7f953f59a..b816beef7 100644
--- a/apps/worker/Dockerfile
+++ b/apps/worker/Dockerfile
@@ -35,6 +35,7 @@ COPY packages/redis/package.json ./packages/redis/
COPY packages/queue/package.json ./packages/queue/
COPY packages/logger/package.json ./packages/logger/
COPY packages/common/package.json ./packages/common/
+COPY packages/importer/package.json ./packages/importer/
COPY packages/constants/package.json ./packages/constants/
COPY packages/validation/package.json ./packages/validation/
COPY packages/integrations/package.json packages/integrations/
@@ -80,9 +81,10 @@ COPY --from=build /app/packages/geo ./packages/geo
COPY --from=build /app/packages/json ./packages/json
COPY --from=build /app/packages/email ./packages/email
COPY --from=build /app/packages/redis ./packages/redis
-COPY --from=build /app/packages/logger ./packages/logger
COPY --from=build /app/packages/queue ./packages/queue
+COPY --from=build /app/packages/logger ./packages/logger
COPY --from=build /app/packages/common ./packages/common
+COPY --from=build /app/packages/importer ./packages/importer
COPY --from=build /app/packages/validation ./packages/validation
COPY --from=build /app/packages/integrations ./packages/integrations
COPY --from=build /app/tooling/typescript ./tooling/typescript
diff --git a/apps/worker/package.json b/apps/worker/package.json
index c9b548785..fc1b2d2e6 100644
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -8,8 +8,7 @@
"testing": "WORKER_PORT=9999 pnpm dev",
"start": "node dist/index.js",
"build": "rm -rf dist && tsdown",
- "typecheck": "tsc --noEmit",
- "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./src/referrers/index.ts"
+ "typecheck": "tsc --noEmit"
},
"dependencies": {
"@bull-board/api": "6.13.1",
@@ -20,6 +19,7 @@
"@openpanel/integrations": "workspace:^",
"@openpanel/json": "workspace:*",
"@openpanel/logger": "workspace:*",
+ "@openpanel/importer": "workspace:*",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*",
"bullmq": "^5.8.7",
@@ -38,7 +38,7 @@
"@types/source-map-support": "^0.5.10",
"@types/sqlstring": "^2.3.2",
"@types/uuid": "^9.0.8",
- "tsdown": "^0.14.2",
+ "tsdown": "0.14.2",
"typescript": "catalog:"
}
}
diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts
index d99e14b3f..d64bcf106 100644
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -5,6 +5,7 @@ import {
type EventsQueuePayloadIncomingEvent,
cronQueue,
eventsGroupQueue,
+ importQueue,
miscQueue,
notificationQueue,
queueLogger,
@@ -19,6 +20,7 @@ import { Worker as GroupWorker } from 'groupmq';
import { cronJob } from './jobs/cron';
import { eventsJob } from './jobs/events';
import { incomingEventPure } from './jobs/events.incoming-event';
+import { importJob } from './jobs/import';
import { miscJob } from './jobs/misc';
import { notificationJob } from './jobs/notification';
import { sessionsJob } from './jobs/sessions';
@@ -56,13 +58,18 @@ export async function bootWorkers() {
workerOptions,
);
const miscWorker = new Worker(miscQueue.name, miscJob, workerOptions);
+ const importWorker = new Worker(importQueue.name, importJob, {
+ ...workerOptions,
+ concurrency: Number.parseInt(process.env.IMPORT_JOB_CONCURRENCY || '1', 10),
+ });
const workers = [
sessionsWorker,
cronWorker,
notificationWorker,
miscWorker,
- eventsGroupWorker,
+ importWorker,
+ // eventsGroupWorker,
];
workers.forEach((worker) => {
@@ -148,7 +155,15 @@ export async function bootWorkers() {
['uncaughtException', 'unhandledRejection', 'SIGTERM', 'SIGINT'].forEach(
(evt) => {
process.on(evt, (code) => {
- exitHandler(evt, code);
+ if (process.env.NODE_ENV === 'production') {
+ exitHandler(evt, code);
+ } else {
+ logger.info('Shutting down for development', {
+ event: evt,
+ code,
+ });
+ process.exit(0);
+ }
});
},
);
diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts
index bb7793d7d..74bb3cb62 100644
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -5,6 +5,7 @@ import { createInitialSalts } from '@openpanel/db';
import {
cronQueue,
eventsGroupQueue,
+ importQueue,
miscQueue,
notificationQueue,
sessionsQueue,
@@ -31,13 +32,14 @@ async function start() {
if (process.env.DISABLE_BULLBOARD === undefined) {
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath('/');
- ({
+ createBullBoard({
queues: [
new BullBoardGroupMQAdapter(eventsGroupQueue) as any,
new BullMQAdapter(sessionsQueue),
new BullMQAdapter(cronQueue),
new BullMQAdapter(notificationQueue),
new BullMQAdapter(miscQueue),
+ new BullMQAdapter(importQueue),
],
serverAdapter: serverAdapter,
});
diff --git a/apps/worker/src/jobs/cron.delete-projects.ts b/apps/worker/src/jobs/cron.delete-projects.ts
index abb8f6848..eab4f84c5 100644
--- a/apps/worker/src/jobs/cron.delete-projects.ts
+++ b/apps/worker/src/jobs/cron.delete-projects.ts
@@ -54,7 +54,7 @@ export async function deleteProjects(job: Job<CronQueuePayload>) {
await ch.command({
query,
clickhouse_settings: {
- lightweight_deletes_sync: 0,
+ lightweight_deletes_sync: '0',
},
});
}
diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.ts
index ecd28a550..bb3f3743e 100644
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -1,12 +1,15 @@
import { logger as baseLogger } from '@/utils/logger';
-import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
import {
createSessionEndJob,
createSessionStart,
getSessionEnd,
} from '@/utils/session-handler';
import { isSameDomain, parsePath } from '@openpanel/common';
-import { parseUserAgent } from '@openpanel/common/server';
+import {
+ getReferrerWithQuery,
+ parseReferrer,
+ parseUserAgent,
+} from '@openpanel/common/server';
import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
import {
checkNotificationRulesForEvent,
@@ -15,10 +18,9 @@ import {
} from '@openpanel/db';
import type { ILogger } from '@openpanel/logger';
import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
-import { getLock } from '@openpanel/redis';
-import { DelayedError, type Job } from 'bullmq';
-import { omit } from 'ramda';
+import type { Job } from 'bullmq';
import * as R from 'ramda';
+import { omit } from 'ramda';
import { v4 as uuid } from 'uuid';
const GLOBAL_PROPERTIES = ['__path', '__referrer'];
@@ -115,9 +117,9 @@ export async function incomingEventPure(
latitude: geo.latitude,
path,
origin,
- referrer: utmReferrer?.url || referrer?.url || '',
+ referrer: referrer?.url || '',
referrerName: utmReferrer?.name || referrer?.name || '',
- referrerType: utmReferrer?.type || referrer?.type || '',
+ referrerType: referrer?.type || utmReferrer?.type || '',
os: uaInfo.os,
osVersion: uaInfo.osVersion,
browser: uaInfo.browser,
diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.ts
index 292453d4c..d5d9dd0c5 100644
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ b/apps/worker/src/jobs/events.incoming-events.test.ts
@@ -99,7 +99,7 @@ describe('incomingEvent', () => {
origin: 'https://example.com',
referrer: '',
referrerName: '',
- referrerType: 'unknown',
+ referrerType: '',
sdkName: jobData.payload.headers['openpanel-sdk-name'],
sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
};
@@ -207,7 +207,7 @@ describe('incomingEvent', () => {
origin: 'https://example.com',
referrer: '',
referrerName: '',
- referrerType: 'unknown',
+ referrerType: '',
sdkName: jobData.payload.headers['openpanel-sdk-name'],
sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
};
diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.ts
new file mode 100644
index 000000000..0346d5832
--- /dev/null
+++ b/apps/worker/src/jobs/import.ts
@@ -0,0 +1,332 @@
+import {
+ type IClickhouseEvent,
+ type ImportSteps,
+ type Prisma,
+ backfillSessionsToProduction,
+ createSessionsStartEndEvents,
+ db,
+ formatClickhouseDate,
+ generateSessionIds,
+ getImportDateBounds,
+ getImportProgress,
+ insertImportBatch,
+ markImportComplete,
+ moveImportsToProduction,
+ updateImportStatus,
+} from '@openpanel/db';
+import { MixpanelProvider, UmamiProvider } from '@openpanel/importer';
+import type { ILogger } from '@openpanel/logger';
+import type { ImportQueuePayload } from '@openpanel/queue';
+import type { Job } from 'bullmq';
+import { logger } from '../utils/logger';
+
+const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10);
+
+/**
+ * Yields control back to the event loop to prevent stalled jobs
+ */
+async function yieldToEventLoop(): Promise<void> {
+ return new Promise((resolve) => {
+ setTimeout(resolve, 100);
+ });
+}
+
+export async function importJob(job: Job<ImportQueuePayload>) {
+ const { importId } = job.data.payload;
+
+ const record = await db.import.findUniqueOrThrow({
+ where: { id: importId },
+ include: {
+ project: true,
+ },
+ });
+
+ const jobLogger = logger.child({
+ importId,
+ config: record.config,
+ });
+
+ type ValidStep = Exclude<ImportSteps, 'failed' | 'completed'>;
+ const steps: Record<ValidStep, number> = {
+ loading: 0,
+ generating_session_ids: 1,
+ creating_sessions: 2,
+ moving: 3,
+ backfilling_sessions: 4,
+ };
+
+ jobLogger.info('Starting import job');
+ const providerInstance = createProvider(record, jobLogger);
+
+ try {
+ // Check if this is a resume operation
+ const isNewImport = record.currentStep === null;
+
+ if (isNewImport) {
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'loading',
+ });
+ } else {
+ jobLogger.info('Resuming import from previous state', {
+ currentStep: record.currentStep,
+ currentBatch: record.currentBatch,
+ });
+ }
+
+ // Try to get a precomputed total for better progress reporting
+ const totalEvents = await providerInstance
+ .getTotalEventsCount()
+ .catch(() => -1);
+ let processedEvents = record.processedEvents;
+
+ const resumeLoadingFrom =
+ (record.currentStep === 'loading' && record.currentBatch) || undefined;
+
+ const resumeGeneratingSessionIdsFrom =
+ (record.currentStep === 'generating_session_ids' &&
+ record.currentBatch) ||
+ undefined;
+
+ const resumeCreatingSessionsFrom =
+ (record.currentStep === 'creating_sessions' && record.currentBatch) ||
+ undefined;
+
+ const resumeMovingFrom =
+ (record.currentStep === 'moving' && record.currentBatch) || undefined;
+
+ const resumeBackfillingSessionsFrom =
+ (record.currentStep === 'backfilling_sessions' && record.currentBatch) ||
+ undefined;
+
+ // Example:
+ // shouldRunStep(0) // currStep = 2 (should not run)
+ // shouldRunStep(1) // currStep = 2 (should not run)
+ // shouldRunStep(2) // currStep = 2 (should run)
+ // shouldRunStep(3) // currStep = 2 (should run)
+ const shouldRunStep = (step: ValidStep) => {
+ if (isNewImport) {
+ return true;
+ }
+
+ const stepToRunIndex = steps[step];
+ const currentStepIndex = steps[record.currentStep as ValidStep];
+ return stepToRunIndex >= currentStepIndex;
+ };
+
+ async function whileBounds(
+ from: string | undefined,
+ callback: (from: string, to: string) => Promise<void>,
+ ) {
+ const bounds = await getImportDateBounds(importId, from);
+ if (bounds.min && bounds.max) {
+ const start = new Date(bounds.min);
+ const end = new Date(bounds.max);
+ let cursor = new Date(start);
+ while (cursor < end) {
+ const next = new Date(cursor);
+ next.setDate(next.getDate() + 1);
+ await callback(
+ formatClickhouseDate(cursor, true),
+ formatClickhouseDate(next, true),
+ );
+ cursor = next;
+
+ // Yield control back to event loop after processing each day
+ await yieldToEventLoop();
+ }
+ }
+ }
+
+ // Phase 1: Fetch & Transform - Process events in batches
+ if (shouldRunStep('loading')) {
+ const eventBatch: any = [];
+ for await (const rawEvent of providerInstance.parseSource(
+ resumeLoadingFrom,
+ )) {
+ // Validate event
+ if (
+ !providerInstance.validate(
+ // @ts-expect-error
+ rawEvent,
+ )
+ ) {
+ jobLogger.warn('Skipping invalid event', { rawEvent });
+ continue;
+ }
+
+ eventBatch.push(rawEvent);
+
+ // Process batch when it reaches the batch size
+ if (eventBatch.length >= BATCH_SIZE) {
+ jobLogger.info('Processing batch', { batchSize: eventBatch.length });
+
+ const transformedEvents: IClickhouseEvent[] = eventBatch.map(
+ (
+ // @ts-expect-error
+ event,
+ ) => providerInstance!.transformEvent(event),
+ );
+
+ await insertImportBatch(transformedEvents, importId);
+
+ processedEvents += eventBatch.length;
+ eventBatch.length = 0;
+
+ const createdAt = new Date(transformedEvents[0]?.created_at || '')
+ .toISOString()
+ .split('T')[0];
+
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'loading',
+ batch: createdAt,
+ totalEvents,
+ processedEvents,
+ });
+
+ // Yield control back to event loop after processing each batch
+ await yieldToEventLoop();
+ }
+ }
+
+ // Process remaining events in the last batch
+ if (eventBatch.length > 0) {
+ const transformedEvents = eventBatch.map(
+ (
+ // @ts-expect-error
+ event,
+ ) => providerInstance!.transformEvent(event),
+ );
+
+ await insertImportBatch(transformedEvents, importId);
+
+ processedEvents += eventBatch.length;
+ eventBatch.length = 0;
+
+ const createdAt = new Date(transformedEvents[0]?.created_at || '')
+ .toISOString()
+ .split('T')[0];
+
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'loading',
+ batch: createdAt,
+ });
+
+ // Yield control back to event loop after processing final batch
+ await yieldToEventLoop();
+ }
+ }
+
+ // Phase 2: Generate session IDs if provider requires it
+ if (
+ shouldRunStep('generating_session_ids') &&
+ providerInstance.shouldGenerateSessionIds()
+ ) {
+ await whileBounds(resumeGeneratingSessionIdsFrom, async (from) => {
+ console.log('Generating session IDs', { from });
+ await generateSessionIds(importId, from);
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'generating_session_ids',
+ batch: from,
+ });
+
+ // Yield control back to event loop after processing each day
+ await yieldToEventLoop();
+ });
+
+ jobLogger.info('Session ID generation complete');
+ }
+
+ // Phase 3-5: Process in daily batches for robustness
+
+ if (shouldRunStep('creating_sessions')) {
+ await whileBounds(resumeCreatingSessionsFrom, async (from) => {
+ await createSessionsStartEndEvents(importId, from);
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'creating_sessions',
+ batch: from,
+ });
+
+ // Yield control back to event loop after processing each day
+ await yieldToEventLoop();
+ });
+ }
+
+ if (shouldRunStep('moving')) {
+ await whileBounds(resumeMovingFrom, async (from) => {
+ await moveImportsToProduction(importId, from);
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'moving',
+ batch: from,
+ });
+
+ // Yield control back to event loop after processing each day
+ await yieldToEventLoop();
+ });
+ }
+
+ if (shouldRunStep('backfilling_sessions')) {
+ await whileBounds(resumeBackfillingSessionsFrom, async (from) => {
+ await backfillSessionsToProduction(importId, from);
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'backfilling_sessions',
+ batch: from,
+ });
+
+ // Yield control back to event loop after processing each day
+ await yieldToEventLoop();
+ });
+ }
+
+ await markImportComplete(importId);
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'completed',
+ });
+ jobLogger.info('Import marked as complete');
+
+ // Get final progress
+ const finalProgress = await getImportProgress(importId);
+
+ jobLogger.info('Import job completed successfully', {
+ totalEvents: finalProgress.totalEvents,
+ insertedEvents: finalProgress.insertedEvents,
+ status: finalProgress.status,
+ });
+
+ return {
+ success: true,
+ totalEvents: finalProgress.totalEvents,
+ processedEvents: finalProgress.insertedEvents,
+ };
+ } catch (error) {
+ jobLogger.error('Import job failed', { error });
+
+ // Mark import as failed
+ try {
+ const errorMsg = error instanceof Error ? error.message : 'Unknown error';
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'failed',
+ errorMessage: errorMsg,
+ });
+ jobLogger.warn('Import marked as failed', { error: errorMsg });
+ } catch (markError) {
+ jobLogger.error('Failed to mark import as failed', { error, markError });
+ }
+
+ throw error;
+ }
+}
+
+function createProvider(
+ record: Prisma.ImportGetPayload<{ include: { project: true } }>,
+ jobLogger: ILogger,
+) {
+ const config = record.config;
+ switch (config.provider) {
+ case 'umami':
+ return new UmamiProvider(record.projectId, config, jobLogger);
+ case 'mixpanel':
+ return new MixpanelProvider(record.projectId, config, jobLogger);
+ default:
+ throw new Error(`Unknown provider: ${config.provider}`);
+ }
+}
diff --git a/package.json b/package.json
index 0860945fd..c29855854 100644
--- a/package.json
+++ b/package.json
@@ -9,7 +9,7 @@
"scripts": {
"test": "vitest run",
"gen:bots": "pnpm -r --filter api gen:bots",
- "gen:referrers": "pnpm -r --filter worker gen:referrers",
+ "gen:referrers": "pnpm -r --filter common gen:referrers",
"dock:up": "docker compose up -d",
"dock:down": "docker compose down",
"dock:ch": "docker compose exec -it op-ch clickhouse-client -d openpanel",
diff --git a/packages/cli/package.json b/packages/cli/package.json
deleted file mode 100644
index d352ce721..000000000
--- a/packages/cli/package.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "name": "@openpanel/cli",
- "version": "0.0.1-beta",
- "type": "module",
- "module": "index.ts",
- "bin": {
- "openpanel": "dist/bin/cli.js"
- },
- "scripts": {
- "build": "rm -rf dist && tsup",
- "typecheck": "tsc --noEmit"
- },
- "dependencies": {
- "@openpanel/common": "workspace:*",
- "arg": "^5.0.2",
- "glob": "^10.4.3",
- "inquirer": "^9.3.5",
- "p-limit": "^6.1.0",
- "progress": "^2.0.3",
- "ramda": "^0.29.1",
- "zod": "catalog:"
- },
- "devDependencies": {
- "@openpanel/db": "workspace:^",
- "@openpanel/sdk": "workspace:*",
- "@openpanel/tsconfig": "workspace:*",
- "@types/node": "catalog:",
- "@types/progress": "^2.0.7",
- "@types/ramda": "^0.30.1",
- "tsup": "^7.2.0",
- "typescript": "catalog:"
- }
-}
diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
deleted file mode 100644
index 872aa7311..000000000
--- a/packages/cli/src/cli.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import arg from 'arg';
-
-import importer from './importer';
-
-function cli() {
- const args = arg(
- {
- '--help': Boolean,
- },
- {
- permissive: true,
- },
- );
-
- const [command] = args._;
-
- switch (command) {
- case 'import': {
- return importer();
- }
- }
-}
-
-cli();
diff --git a/packages/cli/src/importer/importer.ts b/packages/cli/src/importer/importer.ts
deleted file mode 100644
index 27be17af6..000000000
--- a/packages/cli/src/importer/importer.ts
+++ /dev/null
@@ -1,467 +0,0 @@
-import { randomUUID } from 'node:crypto';
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-import readline from 'node:readline';
-import zlib from 'node:zlib';
-import Progress from 'progress';
-import { assocPath, prop, uniqBy } from 'ramda';
-
-import { isSameDomain, parsePath } from '@openpanel/common';
-import type { IImportedEvent } from '@openpanel/db';
-
-const BATCH_SIZE = 30_000;
-const SLEEP_TIME = 20;
-const MAX_CONCURRENT_REQUESTS = 8;
-
-type IMixpanelEvent = {
- event: string;
- properties: {
- [key: string]: unknown;
- time: number;
- $current_url?: string;
- distinct_id?: string;
- $device_id?: string;
- country_code?: string;
- $region?: string;
- $city?: string;
- $os?: string;
- $browser?: string;
- $browser_version?: string;
- $initial_referrer?: string;
- $search_engine?: string;
- };
-};
-
-function stripMixpanelProperties(obj: Record<string, unknown>) {
- return Object.fromEntries(
- Object.entries(obj).filter(
- ([key]) =>
- !key.match(/^(\$|mp_)/) && !['time', 'distinct_id'].includes(key),
- ),
- );
-}
-
-async function* parseJsonStream(
- fileStream: fs.ReadStream,
-): AsyncGenerator<any, void, unknown> {
- const rl = readline.createInterface({
- input: fileStream,
- crlfDelay: Number.POSITIVE_INFINITY,
- });
-
- let buffer = '';
- let bracketCount = 0;
-
- for await (const line of rl) {
- buffer += line;
- bracketCount +=
- (line.match(/{/g) || []).length - (line.match(/}/g) || []).length;
-
- if (bracketCount === 0 && buffer.trim()) {
- try {
- const json = JSON.parse(buffer);
- yield json;
- } catch (error) {
- console.log('Warning: Failed to parse JSON');
- console.log('Buffer:', buffer);
- }
- buffer = '';
- }
- }
-
- if (buffer.trim()) {
- try {
- const json = JSON.parse(buffer);
- yield json;
- } catch (error) {
- console.log('Warning: Failed to parse remaining JSON');
- console.log('Buffer:', buffer);
- }
- }
-}
-
-interface Session {
- start: number;
- end: number;
- profileId?: string;
- deviceId?: string;
- sessionId: string;
- firstEvent?: IImportedEvent;
- lastEvent?: IImportedEvent;
- events: IImportedEvent[];
-}
-
-function generateSessionEvents(events: IImportedEvent[]): Session[] {
- let sessionList: Session[] = [];
- const lastSessionByDevice: Record<string, Session> = {};
- const lastSessionByProfile: Record<string, Session> = {};
- const thirtyMinutes = 30 * 60 * 1000;
-
- events.sort(
- (a, b) =>
- new Date(a.created_at).getTime() - new Date(b.created_at).getTime(),
- );
-
- for (const event of events) {
- const eventTime = new Date(event.created_at).getTime();
- let deviceSession = event.device_id
- ? lastSessionByDevice[event.device_id]
- : undefined;
- let profileSession = event.profile_id
- ? lastSessionByProfile[event.profile_id]
- : undefined;
-
- if (
- event.device_id &&
- event.device_id !== event.profile_id &&
- (!deviceSession || eventTime > deviceSession.end + thirtyMinutes)
- ) {
- deviceSession = {
- start: eventTime,
- end: eventTime,
- deviceId: event.device_id,
- sessionId: randomUUID(),
- firstEvent: event,
- events: [event],
- };
- lastSessionByDevice[event.device_id] = deviceSession;
- sessionList.push(deviceSession);
- } else if (deviceSession) {
- deviceSession.end = eventTime;
- deviceSession.lastEvent = event;
- deviceSession.events.push(event);
- }
-
- if (
- event.profile_id &&
- event.device_id !== event.profile_id &&
- (!profileSession || eventTime > profileSession.end + thirtyMinutes)
- ) {
- profileSession = {
- start: eventTime,
- end: eventTime,
- profileId: event.profile_id,
- sessionId: randomUUID(),
- firstEvent: event,
- events: [event],
- };
- lastSessionByProfile[event.profile_id] = profileSession;
- sessionList.push(profileSession);
- } else if (profileSession) {
- profileSession.end = eventTime;
- profileSession.lastEvent = event;
- profileSession.events.push(event);
- }
-
- if (
- deviceSession &&
- profileSession &&
- deviceSession.sessionId !== profileSession.sessionId
- ) {
- const unifiedSession = {
- ...deviceSession,
- ...profileSession,
- events: [...deviceSession.events, ...profileSession.events],
- start: Math.min(deviceSession.start, profileSession.start),
- end: Math.max(deviceSession.end, profileSession.end),
- sessionId: deviceSession.sessionId,
- };
- lastSessionByDevice[event.device_id] = unifiedSession;
- lastSessionByProfile[event.profile_id] = unifiedSession;
- sessionList = sessionList.filter(
- (session) =>
- session.sessionId !== deviceSession?.sessionId &&
- session.sessionId !== profileSession?.sessionId,
- );
- sessionList.push(unifiedSession);
- }
- }
-
- return sessionList;
-}
-
-function createEventObject(event: IMixpanelEvent): IImportedEvent {
- const getReferrer = (referrer: string | undefined) => {
- if (!referrer) {
- return '';
- }
-
- if (referrer === '$direct') {
- return '';
- }
-
- if (isSameDomain(referrer, event.properties.$current_url)) {
- return '';
- }
-
- return referrer;
- };
- const url = parsePath(event.properties.$current_url);
- return {
- profile_id: event.properties.distinct_id
- ? String(event.properties.distinct_id).replace(/^\$device:/, '')
- : (event.properties.$device_id ?? ''),
- name: event.event,
- created_at: new Date(event.properties.time * 1000).toISOString(),
- properties: {
- ...stripMixpanelProperties(event.properties),
- ...(event.properties.$current_url
- ? {
- __query: url.query,
- __hash: url.hash,
- }
- : {}),
- },
- country: event.properties.country_code ?? '',
- region: event.properties.$region ?? '',
- city: event.properties.$city ?? '',
- os: event.properties.$os ?? '',
- browser: event.properties.$browser ?? '',
- browser_version: event.properties.$browser_version
- ? String(event.properties.$browser_version)
- : '',
- referrer: getReferrer(event.properties.$initial_referrer),
- referrer_type: event.properties.$search_engine ? 'search' : '',
- referrer_name: event.properties.$search_engine ?? '',
- device_id: event.properties.$device_id ?? '',
- session_id: '',
- project_id: '',
- path: url.path,
- origin: url.origin,
- os_version: '',
- model: '',
- longitude: null,
- latitude: null,
- id: randomUUID(),
- duration: 0,
- device: event.properties.$current_url ? '' : 'server',
- brand: '',
- sdk_name: '',
- sdk_version: '',
- };
-}
-
-function isMixpanelEvent(event: any): event is IMixpanelEvent {
- return (
- typeof event === 'object' &&
- event !== null &&
- typeof event?.event === 'string' &&
- typeof event?.properties === 'object' &&
- event?.properties !== null &&
- typeof event?.properties.time === 'number'
- );
-}
-
-async function processFile(file: string): Promise<IImportedEvent[]> {
- const fileStream = fs.createReadStream(file);
- const events: IImportedEvent[] = [];
- for await (const event of parseJsonStream(fileStream)) {
- if (Array.isArray(event)) {
- for (const item of event) {
- if (isMixpanelEvent(item)) {
- events.push(createEventObject(item));
- } else {
- console.log('Not a Mixpanel event', item);
- }
- }
- } else {
- if (isMixpanelEvent(event)) {
- events.push(createEventObject(event));
- } else {
- console.log('Not a Mixpanel event', event);
- }
- }
- }
- return events;
-}
-
-function processEvents(events: IImportedEvent[]): IImportedEvent[] {
- const sessions = generateSessionEvents(events);
- const processedEvents = sessions.flatMap((session) =>
- [
- session.firstEvent && {
- ...session.firstEvent,
- id: randomUUID(),
- created_at: new Date(
- new Date(session.firstEvent.created_at).getTime() - 1000,
- ).toISOString(),
- session_id: session.sessionId,
- name: 'session_start',
- },
- ...uniqBy(
- prop('id'),
- session.events.map((event) =>
- assocPath(['session_id'], session.sessionId, event),
- ),
- ),
- session.lastEvent && {
- ...session.lastEvent,
- id: randomUUID(),
- created_at: new Date(
- new Date(session.lastEvent.created_at).getTime() + 1000,
- ).toISOString(),
- session_id: session.sessionId,
- name: 'session_end',
- },
- ].filter((item): item is IImportedEvent => !!item),
- );
-
- return [
- ...processedEvents,
- ...events.filter((event) => {
- return !event.profile_id && !event.device_id;
- }),
- ];
-}
-
-async function sendBatchToAPI(
- batch: IImportedEvent[],
- {
- apiUrl,
- clientId,
- clientSecret,
- }: {
- apiUrl: string;
- clientId: string;
- clientSecret: string;
- },
-) {
- async function request() {
- const res = await fetch(`${apiUrl}/import/events`, {
- method: 'POST',
- headers: {
- 'Content-Encoding': 'gzip',
- 'Content-Type': 'application/json',
- 'openpanel-client-id': clientId,
- 'openpanel-client-secret': clientSecret,
- },
- body: Buffer.from(zlib.gzipSync(JSON.stringify(batch))),
- });
- if (!res.ok) {
- throw new Error(`Failed to send batch: ${await res.text()}`);
- }
- await new Promise((resolve) => setTimeout(resolve, SLEEP_TIME));
- }
-
- try {
- await request();
- } catch (e) {
- console.log('Error sending batch, retrying...');
- await new Promise((resolve) => setTimeout(resolve, 1000));
- try {
- await request();
- } catch (e) {
- console.log('Error sending batch, skipping...');
- fs.writeFileSync(
- path.join(
- os.tmpdir(),
- `openpanel/failed-import-batch-${batch[0]?.created_at ? new Date(batch[0]?.created_at).toISOString() : Date.now()}.json`,
- ),
- JSON.stringify(batch, null, 2),
- );
- }
- }
-}
-
-async function processFiles({
- files,
- apiUrl,
- clientId,
- clientSecret,
-}: {
- files: string[];
- apiUrl: string;
- clientId: string;
- clientSecret: string;
-}) {
- const progress = new Progress(
- 'Processing (:current/:total) :file [:bar] :percent | :savedEvents saved events | :status',
- {
- total: files.length,
- width: 20,
- },
- );
- let savedEvents = 0;
- let currentBatch: IImportedEvent[] = [];
- let apiBatching = [];
-
- for (const file of files) {
- progress.tick({
- file,
- savedEvents,
- status: 'reading file',
- });
- const events = await processFile(file);
- progress.render({
- file,
- savedEvents,
- status: 'processing events',
- });
- const processedEvents = processEvents(events);
- for (const event of processedEvents) {
- currentBatch.push(event);
- if (currentBatch.length >= BATCH_SIZE) {
- apiBatching.push(currentBatch);
- savedEvents += currentBatch.length;
- progress.render({ file, savedEvents, status: 'saving events' });
- currentBatch = [];
- }
-
- if (apiBatching.length >= MAX_CONCURRENT_REQUESTS) {
- await Promise.all(
- apiBatching.map((batch) =>
- sendBatchToAPI(batch, {
- apiUrl,
- clientId,
- clientSecret,
- }),
- ),
- );
- apiBatching = [];
- }
- }
- }
-
- if (currentBatch.length > 0) {
- await sendBatchToAPI(currentBatch, {
- apiUrl,
- clientId,
- clientSecret,
- });
- savedEvents += currentBatch.length;
- progress.render({ file: 'Complete', savedEvents, status: 'Complete' });
- }
-}
-
-export async function importFiles({
- files,
- apiUrl,
- clientId,
- clientSecret,
-}: {
- files: string[];
- apiUrl: string;
- clientId: string;
- clientSecret: string;
-}) {
- if (files.length === 0) {
- console.log('No files found');
- return;
- }
-
- console.log(`Found ${files.length} files to process`);
-
- const startTime = Date.now();
- await processFiles({
- files,
- apiUrl,
- clientId,
- clientSecret,
- });
- const endTime = Date.now();
-
- console.log(
- `\nProcessing completed in ${(endTime - startTime) / 1000} seconds`,
- );
-}
diff --git a/packages/cli/src/importer/index.ts b/packages/cli/src/importer/index.ts
deleted file mode 100644
index fcd1f196b..000000000
--- a/packages/cli/src/importer/index.ts
+++ /dev/null
@@ -1,59 +0,0 @@
-import path from 'node:path';
-import arg from 'arg';
-import { glob } from 'glob';
-
-import { importFiles } from './importer';
-
-export default async function importer() {
- const args = arg(
- {
- '--glob': String,
- '--api-url': String,
- '--client-id': String,
- '--client-secret': String,
- '--dry-run': Boolean,
- '--from': Number,
- '--to': Number,
- },
- {
- permissive: true,
- },
- );
-
- if (!args['--glob']) {
- throw new Error('Missing --glob argument');
- }
-
- if (!args['--client-id']) {
- throw new Error('Missing --client-id argument');
- }
-
- if (!args['--client-secret']) {
- throw new Error('Missing --client-secret argument');
- }
-
- const cwd = process.cwd();
-
- const fileMatcher = path.resolve(cwd, args['--glob']);
- const allFiles = await glob([fileMatcher], { root: '/' });
- allFiles.sort((a, b) => a.localeCompare(b));
-
- const files = allFiles.slice(
- args['--from'] ?? 0,
- args['--to'] ?? Number.MAX_SAFE_INTEGER,
- );
-
- if (args['--dry-run']) {
- files.forEach((file, index) => {
- console.log(`Would import (index: ${index}): ${file}`);
- });
- return;
- }
-
- return importFiles({
- files,
- clientId: args['--client-id'],
- clientSecret: args['--client-secret'],
- apiUrl: args['--api-url'] ?? 'https://api.openpanel.dev',
- });
-}
diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json
deleted file mode 100644
index fa4341f12..000000000
--- a/packages/cli/tsconfig.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "extends": "@openpanel/tsconfig/base.json",
- "compilerOptions": {
- "incremental": false,
- "outDir": "dist"
- },
- "exclude": ["dist"]
-}
diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts
deleted file mode 100644
index 628edb0ae..000000000
--- a/packages/cli/tsup.config.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-import { defineConfig } from 'tsup';
-
-export default defineConfig({
- entry: ['src/cli.ts'],
- format: ['cjs', 'esm'],
- dts: true,
- splitting: false,
- sourcemap: false,
- clean: true,
- minify: true,
-});
diff --git a/packages/common/package.json b/packages/common/package.json
index a8c914f7c..5eaab8d25 100644
--- a/packages/common/package.json
+++ b/packages/common/package.json
@@ -3,9 +3,14 @@
"version": "0.0.1",
"type": "module",
"main": "index.ts",
+ "exports": {
+ ".": "./index.ts",
+ "./server": "./server/index.ts"
+ },
"scripts": {
"test": "vitest",
- "typecheck": "tsc --noEmit"
+ "typecheck": "tsc --noEmit",
+ "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./server/referrers/index.ts"
},
"dependencies": {
"@openpanel/constants": "workspace:*",
diff --git a/packages/common/scripts/get-referrers.ts b/packages/common/scripts/get-referrers.ts
new file mode 100644
index 000000000..27302eb79
--- /dev/null
+++ b/packages/common/scripts/get-referrers.ts
@@ -0,0 +1,96 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// extras
+const extraReferrers = {
+ 'zoom.us': { type: 'social', name: 'Zoom' },
+ 'apple.com': { type: 'tech', name: 'Apple' },
+ 'adobe.com': { type: 'tech', name: 'Adobe' },
+ 'figma.com': { type: 'tech', name: 'Figma' },
+ 'wix.com': { type: 'commerce', name: 'Wix' },
+ 'gmail.com': { type: 'email', name: 'Gmail' },
+ 'notion.so': { type: 'tech', name: 'Notion' },
+ 'ebay.com': { type: 'commerce', name: 'eBay' },
+ 'github.com': { type: 'tech', name: 'GitHub' },
+ 'gitlab.com': { type: 'tech', name: 'GitLab' },
+ 'slack.com': { type: 'social', name: 'Slack' },
+ 'etsy.com': { type: 'commerce', name: 'Etsy' },
+ 'bsky.app': { type: 'social', name: 'Bluesky' },
+ 'twitch.tv': { type: 'content', name: 'Twitch' },
+ 'dropbox.com': { type: 'tech', name: 'Dropbox' },
+ 'outlook.com': { type: 'email', name: 'Outlook' },
+ 'medium.com': { type: 'content', name: 'Medium' },
+ 'paypal.com': { type: 'commerce', name: 'PayPal' },
+ 'discord.com': { type: 'social', name: 'Discord' },
+ 'stripe.com': { type: 'commerce', name: 'Stripe' },
+ 'spotify.com': { type: 'content', name: 'Spotify' },
+ 'netflix.com': { type: 'content', name: 'Netflix' },
+ 'whatsapp.com': { type: 'social', name: 'WhatsApp' },
+ 'shopify.com': { type: 'commerce', name: 'Shopify' },
+ 'microsoft.com': { type: 'tech', name: 'Microsoft' },
+ 'alibaba.com': { type: 'commerce', name: 'Alibaba' },
+ 'telegram.org': { type: 'social', name: 'Telegram' },
+ 'substack.com': { type: 'content', name: 'Substack' },
+ 'salesforce.com': { type: 'tech', name: 'Salesforce' },
+ 'instagram.com': { type: 'social', name: 'Instagram' },
+ 'wikipedia.org': { type: 'content', name: 'Wikipedia' },
+ 'mastodon.social': { type: 'social', name: 'Mastodon' },
+ 'office.com': { type: 'tech', name: 'Microsoft Office' },
+ 'squarespace.com': { type: 'commerce', name: 'Squarespace' },
+ 'stackoverflow.com': { type: 'tech', name: 'Stack Overflow' },
+ 'teams.microsoft.com': { type: 'social', name: 'Microsoft Teams' },
+};
+
+function transform(data: any) {
+ const obj: Record<string, unknown> = {};
+ for (const type in data) {
+ for (const name in data[type]) {
+ const domains = data[type][name].domains ?? [];
+ for (const domain of domains) {
+ obj[domain] = {
+ type,
+ name,
+ };
+ }
+ }
+ }
+
+ return obj;
+}
+
+async function main() {
+ // Get document, or throw exception on error
+ try {
+ const data = await fetch(
+ 'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json',
+ ).then((res) => res.json());
+
+ fs.writeFileSync(
+ path.resolve(__dirname, '../../worker/src/referrers/index.ts'),
+ [
+ '// This file is generated by the script get-referrers.ts',
+ '',
+ '// The data is fetch from snowplow-referer-parser https://github.com/snowplow-referer-parser/referer-parser',
+ `// The orginal referers.yml is based on Piwik's SearchEngines.php and Socials.php, copyright 2012 Matthieu Aubry and available under the GNU General Public License v3.`,
+ '',
+ `const referrers: Record<string, { type: string, name: string }> = ${JSON.stringify(
+ {
+ ...transform(data),
+ ...extraReferrers,
+ },
+ )} as const;`,
+ 'export default referrers;',
+ ].join('\n'),
+ 'utf-8',
+ );
+ } catch (e) {
+ console.log(e);
+ }
+}
+
+main();
diff --git a/packages/common/server/index.ts b/packages/common/server/index.ts
index 3bb5bb376..cb5409062 100644
--- a/packages/common/server/index.ts
+++ b/packages/common/server/index.ts
@@ -1,3 +1,5 @@
export * from './crypto';
export * from './profileId';
export * from './parser-user-agent';
+export * from './parse-referrer';
+export * from './id';
diff --git a/apps/worker/src/utils/parse-referrer.test.ts b/packages/common/server/parse-referrer.test.ts
similarity index 93%
rename from apps/worker/src/utils/parse-referrer.test.ts
rename to packages/common/server/parse-referrer.test.ts
index 66d17c54b..009118b63 100644
--- a/apps/worker/src/utils/parse-referrer.test.ts
+++ b/packages/common/server/parse-referrer.test.ts
@@ -5,13 +5,13 @@ describe('parseReferrer', () => {
it('should handle undefined or empty URLs', () => {
expect(parseReferrer(undefined)).toEqual({
name: '',
- type: 'unknown',
+ type: '',
url: '',
});
expect(parseReferrer('')).toEqual({
name: '',
- type: 'unknown',
+ type: '',
url: '',
});
});
@@ -41,7 +41,7 @@ describe('parseReferrer', () => {
it('should handle unknown referrers', () => {
expect(parseReferrer('https://unknown-site.com')).toEqual({
name: '',
- type: 'unknown',
+ type: '',
url: 'https://unknown-site.com',
});
});
@@ -49,7 +49,7 @@ describe('parseReferrer', () => {
it('should handle invalid URLs', () => {
expect(parseReferrer('not-a-url')).toEqual({
name: '',
- type: 'unknown',
+ type: '',
url: 'not-a-url',
});
});
@@ -64,7 +64,7 @@ describe('getReferrerWithQuery', () => {
it('should parse utm_source parameter', () => {
expect(getReferrerWithQuery({ utm_source: 'google' })).toEqual({
name: 'Google',
- type: 'unknown',
+ type: 'search',
url: '',
});
});
@@ -88,7 +88,7 @@ describe('getReferrerWithQuery', () => {
it('should handle case-insensitive matching', () => {
expect(getReferrerWithQuery({ utm_source: 'GoOgLe' })).toEqual({
name: 'Google',
- type: 'unknown',
+ type: 'search',
url: '',
});
});
@@ -96,7 +96,7 @@ describe('getReferrerWithQuery', () => {
it('should handle unknown sources', () => {
expect(getReferrerWithQuery({ utm_source: 'unknown-source' })).toEqual({
name: 'unknown-source',
- type: 'unknown',
+ type: '',
url: '',
});
});
@@ -110,7 +110,7 @@ describe('getReferrerWithQuery', () => {
}),
).toEqual({
name: 'Google',
- type: 'unknown',
+ type: 'search',
url: '',
});
});
diff --git a/apps/worker/src/utils/parse-referrer.ts b/packages/common/server/parse-referrer.ts
similarity index 68%
rename from apps/worker/src/utils/parse-referrer.ts
rename to packages/common/server/parse-referrer.ts
index 6e4d86af1..3fe35c5ee 100644
--- a/apps/worker/src/utils/parse-referrer.ts
+++ b/packages/common/server/parse-referrer.ts
@@ -1,6 +1,6 @@
-import { stripTrailingSlash } from '@openpanel/common';
+import { stripTrailingSlash } from '../src/string';
-import referrers from '../referrers';
+import referrers from './referrers';
function getHostname(url: string | undefined) {
if (!url) {
@@ -20,7 +20,7 @@ export function parseReferrer(url: string | undefined) {
return {
name: match?.name ?? '',
- type: match?.type ?? 'unknown',
+ type: match?.type ?? '',
url: stripTrailingSlash(url ?? ''),
};
}
@@ -32,16 +32,23 @@ export function getReferrerWithQuery(
return null;
}
- const source = query.utm_source ?? query.ref ?? query.utm_referrer ?? '';
+ const source = (
+ query.utm_source ??
+ query.ref ??
+ query.utm_referrer ??
+ ''
+ ).toLowerCase();
if (source === '') {
return null;
}
const match =
+ referrers[source] ||
+ referrers[`${source}.com`] ||
Object.values(referrers).find(
- (referrer) => referrer.name.toLowerCase() === source.toLowerCase(),
- ) || referrers[source];
+ (referrer) => referrer.name.toLowerCase() === source,
+ );
if (match) {
return {
@@ -53,7 +60,7 @@ export function getReferrerWithQuery(
return {
name: source,
- type: 'unknown',
+ type: '',
url: '',
};
}
diff --git a/packages/common/server/parser-user-agent.ts b/packages/common/server/parser-user-agent.ts
index b84b9768c..9238de3b8 100644
--- a/packages/common/server/parser-user-agent.ts
+++ b/packages/common/server/parser-user-agent.ts
@@ -68,6 +68,7 @@ const parse = (ua: string): UAParser.IResult => {
return res;
};
+export type UserAgentInfo = ReturnType<typeof parseUserAgent>;
export function parseUserAgent(
ua?: string | null,
overrides?: Record<string, unknown>,
@@ -80,13 +81,35 @@ export function parseUserAgent(
}
return {
- os: overrides?.__os || res.os.name,
- osVersion: overrides?.__osVersion || res.os.version,
- browser: overrides?.__browser || res.browser.name,
- browserVersion: overrides?.__browserVersion || res.browser.version,
- device: overrides?.__device || res.device.type || getDevice(ua),
- brand: overrides?.__brand || res.device.vendor,
- model: overrides?.__model || res.device.model,
+ os:
+ typeof overrides?.__os === 'string' && overrides?.__os
+ ? overrides?.__os
+ : res.os.name,
+ osVersion:
+ typeof overrides?.__osVersion === 'string' && overrides?.__osVersion
+ ? overrides?.__osVersion
+ : res.os.version,
+ browser:
+ typeof overrides?.__browser === 'string' && overrides?.__browser
+ ? overrides?.__browser
+ : res.browser.name,
+ browserVersion:
+ typeof overrides?.__browserVersion === 'string' &&
+ overrides?.__browserVersion
+ ? overrides?.__browserVersion
+ : res.browser.version,
+ device:
+ typeof overrides?.__device === 'string' && overrides?.__device
+ ? overrides?.__device
+ : res.device.type || getDevice(ua),
+ brand:
+ typeof overrides?.__brand === 'string' && overrides?.__brand
+ ? overrides?.__brand
+ : res.device.vendor,
+ model:
+ typeof overrides?.__model === 'string' && overrides?.__model
+ ? overrides?.__model
+ : res.device.model,
isServer: false,
} as const;
}
diff --git a/apps/worker/src/referrers/index.ts b/packages/common/server/referrers/index.ts
similarity index 100%
rename from apps/worker/src/referrers/index.ts
rename to packages/common/server/referrers/index.ts
diff --git a/apps/worker/src/referrers/referrers.readme.md b/packages/common/server/referrers/referrers.readme.md
similarity index 100%
rename from apps/worker/src/referrers/referrers.readme.md
rename to packages/common/server/referrers/referrers.readme.md
diff --git a/packages/common/src/object.test.ts b/packages/common/src/object.test.ts
new file mode 100644
index 000000000..d33dcb789
--- /dev/null
+++ b/packages/common/src/object.test.ts
@@ -0,0 +1,48 @@
+import { describe, expect, it } from 'vitest';
+import { toDots } from './object';
+
+describe('toDots', () => {
+ it('should convert an object to a dot object', () => {
+ const obj = {
+ a: 1,
+ b: 2,
+ array: ['1', '2', '3'],
+ arrayWithObjects: [{ a: 1 }, { b: 2 }, { c: 3 }],
+ objectWithArrays: { a: [1, 2, 3] },
+ null: null,
+ undefined: undefined,
+ empty: '',
+ jsonString: '{"a": 1, "b": 2}',
+ };
+ expect(toDots(obj)).toEqual({
+ a: '1',
+ b: '2',
+ 'array.0': '1',
+ 'array.1': '2',
+ 'array.2': '3',
+ 'arrayWithObjects.0.a': '1',
+ 'arrayWithObjects.1.b': '2',
+ 'arrayWithObjects.2.c': '3',
+ 'objectWithArrays.a.0': '1',
+ 'objectWithArrays.a.1': '2',
+ 'objectWithArrays.a.2': '3',
+ 'jsonString.a': '1',
+ 'jsonString.b': '2',
+ });
+ });
+
+ it('should handle malformed JSON strings gracefully', () => {
+ const obj = {
+ validJson: '{"key":"value"}',
+ malformedJson: '{"key":"unterminated string',
+ startsWithBrace: '{not json at all',
+ startsWithBracket: '[also not json',
+ regularString: 'normal string',
+ };
+
+ expect(toDots(obj)).toEqual({
+ 'validJson.key': 'value',
+ regularString: 'normal string',
+ });
+ });
+});
diff --git a/packages/common/src/object.ts b/packages/common/src/object.ts
index dd3a237e0..ebcb0917c 100644
--- a/packages/common/src/object.ts
+++ b/packages/common/src/object.ts
@@ -1,5 +1,18 @@
import { anyPass, assocPath, isEmpty, isNil, reject } from 'ramda';
+function isValidJsonString(value: string): boolean {
+ return (
+ (value.startsWith('{') && value.endsWith('}')) ||
+ (value.startsWith('[') && value.endsWith(']'))
+ );
+}
+function isMalformedJsonString(value: string): boolean {
+ return (
+ (value.startsWith('{') && !value.endsWith('}')) ||
+ (value.startsWith('[') && !value.endsWith(']'))
+ );
+}
+
export function toDots(
obj: Record<string, unknown>,
path = '',
@@ -19,10 +32,28 @@ export function toDots(
};
}
- if (value === undefined || value === null) {
+ if (value === undefined || value === null || value === '') {
+ return acc;
+ }
+
+ if (typeof value === 'string' && isMalformedJsonString(value)) {
+ // Skip it
return acc;
}
+ // Fix nested json strings - but catch parse errors for malformed JSON
+ if (typeof value === 'string' && isValidJsonString(value)) {
+ try {
+ return {
+ ...acc,
+ ...toDots(JSON.parse(value), `${path}${key}.`),
+ };
+ } catch {
+ // Skip it
+ return acc;
+ }
+ }
+
const cleanedValue =
typeof value === 'string'
? removeInvalidSurrogates(value).trim()
diff --git a/packages/db/code-migrations/5-add-imports-table.sql b/packages/db/code-migrations/5-add-imports-table.sql
new file mode 100644
index 000000000..b9144b41f
--- /dev/null
+++ b/packages/db/code-migrations/5-add-imports-table.sql
@@ -0,0 +1,47 @@
+CREATE TABLE IF NOT EXISTS events_imports_replicated ON CLUSTER '{cluster}' (
+ `id` UUID DEFAULT generateUUIDv4(),
+ `name` LowCardinality(String),
+ `sdk_name` LowCardinality(String),
+ `sdk_version` LowCardinality(String),
+ `device_id` String CODEC(ZSTD(3)),
+ `profile_id` String CODEC(ZSTD(3)),
+ `project_id` String CODEC(ZSTD(3)),
+ `session_id` String CODEC(LZ4),
+ `path` String CODEC(ZSTD(3)),
+ `origin` String CODEC(ZSTD(3)),
+ `referrer` String CODEC(ZSTD(3)),
+ `referrer_name` String CODEC(ZSTD(3)),
+ `referrer_type` LowCardinality(String),
+ `duration` UInt64 CODEC(Delta(4), LZ4),
+ `properties` Map(String, String) CODEC(ZSTD(3)),
+ `created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)),
+ `country` LowCardinality(FixedString(2)),
+ `city` String,
+ `region` LowCardinality(String),
+ `longitude` Nullable(Float32) CODEC(Gorilla, LZ4),
+ `latitude` Nullable(Float32) CODEC(Gorilla, LZ4),
+ `os` LowCardinality(String),
+ `os_version` LowCardinality(String),
+ `browser` LowCardinality(String),
+ `browser_version` LowCardinality(String),
+ `device` LowCardinality(String),
+ `brand` LowCardinality(String),
+ `model` LowCardinality(String),
+ `imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4),
+ `import_id` String CODEC(ZSTD(3)),
+ `import_status` LowCardinality(String) DEFAULT 'pending',
+ `imported_at_meta` DateTime DEFAULT now()
+)
+ENGINE = ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/openpanel/v1/{table}', '{replica}')
+PARTITION BY toYYYYMM(imported_at_meta)
+ORDER BY (import_id, created_at)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE TABLE IF NOT EXISTS events_imports ON CLUSTER '{cluster}' AS events_imports_replicated
+ENGINE = Distributed('{cluster}', currentDatabase(), events_imports_replicated, cityHash64(import_id));
+
+---
+
+ALTER TABLE events_imports_replicated ON CLUSTER '{cluster}' MODIFY TTL imported_at_meta + INTERVAL 7 DAY;
\ No newline at end of file
diff --git a/packages/db/code-migrations/5-add-imports-table.ts b/packages/db/code-migrations/5-add-imports-table.ts
new file mode 100644
index 000000000..e9062c74b
--- /dev/null
+++ b/packages/db/code-migrations/5-add-imports-table.ts
@@ -0,0 +1,90 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { TABLE_NAMES } from '../src/clickhouse/client';
+import {
+ createTable,
+ modifyTTL,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+export async function up() {
+ const isClustered = getIsCluster();
+
+ const sqls: string[] = [
+ ...createTable({
+ name: 'events_imports',
+ columns: [
+ // Same columns as events table
+ '`id` UUID DEFAULT generateUUIDv4()',
+ '`name` LowCardinality(String)',
+ '`sdk_name` LowCardinality(String)',
+ '`sdk_version` LowCardinality(String)',
+ '`device_id` String CODEC(ZSTD(3))',
+ '`profile_id` String CODEC(ZSTD(3))',
+ '`project_id` String CODEC(ZSTD(3))',
+ '`session_id` String CODEC(LZ4)',
+ '`path` String CODEC(ZSTD(3))',
+ '`origin` String CODEC(ZSTD(3))',
+ '`referrer` String CODEC(ZSTD(3))',
+ '`referrer_name` String CODEC(ZSTD(3))',
+ '`referrer_type` LowCardinality(String)',
+ '`duration` UInt64 CODEC(Delta(4), LZ4)',
+ '`properties` Map(String, String) CODEC(ZSTD(3))',
+ '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+ '`country` LowCardinality(FixedString(2))',
+ '`city` String',
+ '`region` LowCardinality(String)',
+ '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+ '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+ '`os` LowCardinality(String)',
+ '`os_version` LowCardinality(String)',
+ '`browser` LowCardinality(String)',
+ '`browser_version` LowCardinality(String)',
+ '`device` LowCardinality(String)',
+ '`brand` LowCardinality(String)',
+ '`model` LowCardinality(String)',
+ '`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4)',
+
+ // Additional metadata columns for import tracking
+ '`import_id` String CODEC(ZSTD(3))',
+ "`import_status` LowCardinality(String) DEFAULT 'pending'",
+ '`imported_at_meta` DateTime DEFAULT now()',
+ ],
+ orderBy: ['import_id', 'created_at'],
+ partitionBy: 'toYYYYMM(imported_at_meta)',
+ settings: {
+ index_granularity: 8192,
+ },
+ distributionHash: 'cityHash64(import_id)',
+ replicatedVersion: '1',
+ isClustered,
+ }),
+ ];
+
+ // Add TTL policy for auto-cleanup after 7 days
+ sqls.push(
+ modifyTTL({
+ tableName: 'events_imports',
+ isClustered,
+ ttl: 'imported_at_meta + INTERVAL 7 DAY',
+ }),
+ );
+
+ fs.writeFileSync(
+ path.join(__filename.replace('.ts', '.sql')),
+ sqls
+ .map((sql) =>
+ sql
+ .trim()
+ .replace(/;$/, '')
+ .replace(/\n{2,}/g, '\n')
+ .concat(';'),
+ )
+ .join('\n\n---\n\n'),
+ );
+
+ if (!process.argv.includes('--dry')) {
+ await runClickhouseMigrationCommands(sqls);
+ }
+}
diff --git a/packages/db/index.ts b/packages/db/index.ts
index b0aef4e3d..5a725e8c1 100644
--- a/packages/db/index.ts
+++ b/packages/db/index.ts
@@ -1,5 +1,6 @@
export * from './src/prisma-client';
export * from './src/clickhouse/client';
+export * from './src/clickhouse/csv';
export * from './src/sql-builder';
export * from './src/services/chart.service';
export * from './src/services/clients.service';
@@ -23,5 +24,6 @@ export * from './src/services/access.service';
export * from './src/buffers';
export * from './src/types';
export * from './src/clickhouse/query-builder';
+export * from './src/services/import.service';
export * from './src/services/overview.service';
export * from './src/session-context';
diff --git a/packages/db/package.json b/packages/db/package.json
index 56826516f..284b6d963 100644
--- a/packages/db/package.json
+++ b/packages/db/package.json
@@ -13,7 +13,7 @@
"with-env": "dotenv -e ../../.env -c --"
},
"dependencies": {
- "@clickhouse/client": "^1.2.0",
+ "@clickhouse/client": "^1.12.1",
"@openpanel/common": "workspace:*",
"@openpanel/constants": "workspace:*",
"@openpanel/json": "workspace:*",
diff --git a/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sql b/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sql
new file mode 100644
index 000000000..46fe97732
--- /dev/null
+++ b/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sql
@@ -0,0 +1,22 @@
+-- CreateTable
+CREATE TABLE "public"."imports" (
+ "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+ "projectId" TEXT NOT NULL,
+ "provider" TEXT NOT NULL,
+ "sourceType" TEXT NOT NULL,
+ "sourceLocation" TEXT NOT NULL,
+ "jobId" TEXT,
+ "status" TEXT NOT NULL,
+ "config" JSONB NOT NULL DEFAULT '{}',
+ "totalEvents" INTEGER NOT NULL DEFAULT 0,
+ "processedEvents" INTEGER NOT NULL DEFAULT 0,
+ "errorMessage" TEXT,
+ "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "completedAt" TIMESTAMP(3),
+ "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ CONSTRAINT "imports_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "public"."imports" ADD CONSTRAINT "imports_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
diff --git a/packages/db/prisma/migrations/20251018214030_fix/migration.sql b/packages/db/prisma/migrations/20251018214030_fix/migration.sql
new file mode 100644
index 000000000..1b8aebd8c
--- /dev/null
+++ b/packages/db/prisma/migrations/20251018214030_fix/migration.sql
@@ -0,0 +1,13 @@
+/*
+ Warnings:
+
+ - You are about to drop the column `provider` on the `imports` table. All the data in the column will be lost.
+ - You are about to drop the column `sourceLocation` on the `imports` table. All the data in the column will be lost.
+ - You are about to drop the column `sourceType` on the `imports` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "public"."imports" DROP COLUMN "provider",
+DROP COLUMN "sourceLocation",
+DROP COLUMN "sourceType",
+ALTER COLUMN "config" DROP DEFAULT;
diff --git a/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sql b/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sql
new file mode 100644
index 000000000..ece57d171
--- /dev/null
+++ b/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "public"."imports" ADD COLUMN "statusMessage" TEXT;
diff --git a/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sql b/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sql
new file mode 100644
index 000000000..b579e0f4b
--- /dev/null
+++ b/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sql
@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "public"."imports" ADD COLUMN "currentBatch" INTEGER NOT NULL DEFAULT 0,
+ADD COLUMN "currentStep" TEXT;
diff --git a/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sql b/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sql
new file mode 100644
index 000000000..c05db2aec
--- /dev/null
+++ b/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sql
@@ -0,0 +1,14 @@
+/*
+ Warnings:
+
+ - Changed the type of `status` on the `imports` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required.
+ - Made the column `currentStep` on table `imports` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- CreateEnum
+CREATE TYPE "public"."ImportStatus" AS ENUM ('pending', 'processing', 'completed', 'failed');
+
+-- AlterTable
+ALTER TABLE "public"."imports" DROP COLUMN "status",
+ADD COLUMN "status" "public"."ImportStatus" NOT NULL,
+ALTER COLUMN "currentStep" SET NOT NULL;
diff --git a/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sql b/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sql
new file mode 100644
index 000000000..6afc57871
--- /dev/null
+++ b/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "public"."imports" ALTER COLUMN "currentStep" DROP NOT NULL;
diff --git a/packages/db/prisma/migrations/20251028152531_imports_again/migration.sql b/packages/db/prisma/migrations/20251028152531_imports_again/migration.sql
new file mode 100644
index 000000000..3abc525e9
--- /dev/null
+++ b/packages/db/prisma/migrations/20251028152531_imports_again/migration.sql
@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "public"."imports" ALTER COLUMN "currentBatch" DROP NOT NULL,
+ALTER COLUMN "currentBatch" DROP DEFAULT,
+ALTER COLUMN "currentBatch" SET DATA TYPE TEXT;
diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma
index def5169e5..e6aee8aba 100644
--- a/packages/db/prisma/schema.prisma
+++ b/packages/db/prisma/schema.prisma
@@ -194,6 +194,7 @@ model Project {
notificationRules NotificationRule[]
notifications Notification[]
+ imports Import[]
// When deleteAt > now(), the project will be deleted
deleteAt DateTime?
@@ -467,3 +468,31 @@ model ResetPassword {
@@map("reset_password")
}
+
+enum ImportStatus {
+ pending
+ processing
+ completed
+ failed
+}
+
+model Import {
+ id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+ projectId String
+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+ jobId String? // BullMQ job ID
+ status ImportStatus
+ statusMessage String? // Human-readable current step like "Importing events (Feb 2025)", "Generating session IDs"
+ errorMessage String?
+ /// [IPrismaImportConfig]
+ config Json
+ totalEvents Int @default(0)
+ processedEvents Int @default(0)
+ currentStep String?
+ currentBatch String? // String date 2020-01-01
+ createdAt DateTime @default(now())
+ completedAt DateTime?
+ updatedAt DateTime @default(now()) @updatedAt
+
+ @@map("imports")
+}
diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.ts
index f87cc9037..d1be225c4 100644
--- a/packages/db/src/buffers/base-buffer.ts
+++ b/packages/db/src/buffers/base-buffer.ts
@@ -1,4 +1,4 @@
-import { generateSecureId } from '@openpanel/common/server/id';
+import { generateSecureId } from '@openpanel/common/server';
import { type ILogger, createLogger } from '@openpanel/logger';
import { getRedisCache, runEvery } from '@openpanel/redis';
diff --git a/packages/db/src/buffers/event-buffer.test.ts b/packages/db/src/buffers/event-buffer.test.ts
index 0df5b4aea..1babc51bf 100644
--- a/packages/db/src/buffers/event-buffer.test.ts
+++ b/packages/db/src/buffers/event-buffer.test.ts
@@ -10,6 +10,18 @@ import {
} from 'vitest';
import { ch } from '../clickhouse/client';
+const clickhouseSettings = {
+ async_insert: 1,
+ http_headers_progress_interval_ms: '50000',
+ input_format_parallel_parsing: 1,
+ max_execution_time: 300,
+ max_http_get_redirects: '0',
+ max_insert_block_size: '500000',
+ send_progress_in_http_headers: 1,
+ wait_end_of_query: 1,
+ wait_for_async_insert: 1,
+};
+
// Mock transformEvent to avoid circular dependency with buffers -> services -> buffers
vi.mock('../services/event.service', () => ({
transformEvent: (event: any) => ({
@@ -127,6 +139,7 @@ describe('EventBuffer with real Redis', () => {
duration: 1000,
},
],
+ clickhouse_settings: clickhouseSettings,
});
const sessionKey = `event_buffer:session:${first.session_id}`;
@@ -171,6 +184,7 @@ describe('EventBuffer with real Redis', () => {
format: 'JSONEachRow',
table: 'events',
values: [first, end],
+ clickhouse_settings: clickhouseSettings,
});
const sessionKey = `event_buffer:session:${first.session_id}`;
const storedEvents = await redis.lrange(sessionKey, 0, -1);
@@ -502,6 +516,7 @@ describe('EventBuffer with real Redis', () => {
format: 'JSONEachRow',
table: 'events',
values: [end],
+ clickhouse_settings: clickhouseSettings,
});
const sessionKey = `event_buffer:session:${s}`;
@@ -552,6 +567,7 @@ describe('EventBuffer with real Redis', () => {
format: 'JSONEachRow',
table: 'events',
values: [view1, view2, view3, end],
+ clickhouse_settings: clickhouseSettings,
});
// Session should be completely empty and removed
@@ -596,6 +612,7 @@ describe('EventBuffer with real Redis', () => {
format: 'JSONEachRow',
table: 'events',
values: [{ ...view1, duration: 1000 }],
+ clickhouse_settings: clickhouseSettings,
});
// Session should be REMOVED from ready_sessions (only 1 event left)
@@ -620,6 +637,7 @@ describe('EventBuffer with real Redis', () => {
format: 'JSONEachRow',
table: 'events',
values: [{ ...view2, duration: 1000 }],
+ clickhouse_settings: clickhouseSettings,
});
// Session should be REMOVED again (only 1 event left)
@@ -667,6 +685,7 @@ describe('EventBuffer with real Redis', () => {
format: 'JSONEachRow',
table: 'events',
values: [view, end],
+ clickhouse_settings: clickhouseSettings,
});
// NOW it should be removed from ready_sessions (because it's empty)
diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts
index b363cb44e..be55866a8 100644
--- a/packages/db/src/clickhouse/client.ts
+++ b/packages/db/src/clickhouse/client.ts
@@ -1,3 +1,4 @@
+import { Readable } from 'node:stream';
import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client';
import { ClickHouseLogLevel, createClient } from '@clickhouse/client';
import sqlstring from 'sqlstring';
@@ -23,13 +24,10 @@ type WarnLogParams = LogParams & { err?: Error };
class CustomLogger implements Logger {
trace({ message, args }: LogParams) {
- logger.debug(message, args);
+ logger.info(message, args);
}
debug({ message, args }: LogParams) {
- if (message.includes('Query:') && args?.response_status === 200) {
- return;
- }
- logger.debug(message, args);
+ logger.info(message, args);
}
info({ message, args }: LogParams) {
logger.info(message, args);
@@ -56,14 +54,15 @@ export const TABLE_NAMES = {
event_property_values_mv: 'event_property_values_mv',
cohort_events_mv: 'cohort_events_mv',
sessions: 'sessions',
+ events_imports: 'events_imports',
};
export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {
max_open_connections: 30,
- request_timeout: 60000,
+ request_timeout: 300000,
keep_alive: {
enabled: true,
- idle_socket_ttl: 8000,
+ idle_socket_ttl: 60000,
},
compression: {
request: true,
@@ -87,7 +86,7 @@ const cleanQuery = (query?: string) =>
? query.replace(/\n/g, '').replace(/\s+/g, ' ').trim()
: undefined;
-async function withRetry<T>(
+export async function withRetry<T>(
operation: () => Promise<T>,
maxRetries = 3,
baseDelay = 500,
@@ -132,7 +131,34 @@ export const ch = new Proxy(originalCh, {
const value = Reflect.get(target, property, receiver);
if (property === 'insert') {
- return (...args: any[]) => withRetry(() => value.apply(target, args));
+ return (...args: any[]) =>
+ withRetry(() => {
+ args[0].clickhouse_settings = {
+ // Allow bigger HTTP payloads/time to stream rows
+ async_insert: 1,
+ wait_for_async_insert: 1,
+ // Increase insert timeouts and buffer sizes for large batches
+ max_execution_time: 300,
+ max_insert_block_size: '500000',
+ max_http_get_redirects: '0',
+ // Ensure JSONEachRow stays efficient
+ input_format_parallel_parsing: 1,
+ // Keep long-running inserts/queries from idling out at proxies by sending progress headers
+ send_progress_in_http_headers: 1,
+ http_headers_progress_interval_ms: '50000',
+ // Ensure server holds the connection until the query is finished
+ wait_end_of_query: 1,
+ ...args[0].clickhouse_settings,
+ };
+ return value.apply(target, args);
+ });
+ }
+
+ if (property === 'command') {
+ return (...args: any[]) =>
+ withRetry(() => {
+ return value.apply(target, args);
+ });
}
return value;
@@ -177,6 +203,34 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
return response;
}
+export async function chInsertCSV(tableName: string, rows: string[]) {
+ try {
+ const now = performance.now();
+ // Create a readable stream in binary mode for CSV (similar to EventBuffer)
+ const csvStream = Readable.from(rows.join('\n'), {
+ objectMode: false,
+ });
+
+ await ch.insert({
+ table: tableName,
+ values: csvStream,
+ format: 'CSV',
+ clickhouse_settings: {
+ format_csv_allow_double_quotes: 1,
+ format_csv_allow_single_quotes: 0,
+ },
+ });
+
+ logger.info('CSV Insert successful', {
+ elapsed: performance.now() - now,
+ rows: rows.length,
+ });
+ } catch (error) {
+ logger.error('CSV Insert failed:', error);
+ throw error;
+ }
+}
+
export async function chQuery<T extends Record<string, any>>(
query: string,
clickhouseSettings?: ClickHouseSettings,
diff --git a/packages/db/src/clickhouse/csv.ts b/packages/db/src/clickhouse/csv.ts
new file mode 100644
index 000000000..21802f04d
--- /dev/null
+++ b/packages/db/src/clickhouse/csv.ts
@@ -0,0 +1,53 @@
+// ClickHouse Map(String, String) format in CSV uses single quotes, not JSON double quotes
+// Format: '{'key1':'value1','key2':'value2'}'
+// Single quotes inside values must be escaped with backslash: \'
+// We also need to escape newlines and control characters to prevent CSV parsing issues
+const escapeMapValue = (str: string) => {
+ return str
+ .replace(/\\/g, '\\\\') // Escape backslashes first
+ .replace(/'/g, "\\'") // Escape single quotes
+ .replace(/\n/g, '\\n') // Escape newlines
+ .replace(/\r/g, '\\r') // Escape carriage returns
+ .replace(/\t/g, '\\t') // Escape tabs
+ .replace(/\0/g, '\\0'); // Escape null bytes
+};
+
+export const csvEscapeJson = (
+ value: Record<string, unknown> | null | undefined,
+): string => {
+ if (value == null) return '';
+
+ // Normalize to strings if your column is Map(String,String)
+ const normalized: Record<string, string> = Object.fromEntries(
+ Object.entries(value).map(([k, v]) => [
+ String(k),
+ v == null ? '' : String(v),
+ ]),
+ );
+
+ // Empty object should return empty Map (without quotes, csvEscapeField will handle if needed)
+ if (Object.keys(normalized).length === 0) return '{}';
+
+ const pairs = Object.entries(normalized)
+ .map(([k, v]) => `'${escapeMapValue(k)}':'${escapeMapValue(v)}'`)
+ .join(',');
+
+ // Return Map format without outer quotes - csvEscapeField will handle CSV escaping
+ // This allows csvEscapeField to properly wrap/escape the entire field if it contains newlines/quotes
+ return csvEscapeField(`{${pairs}}`);
+};
+
+// Escape a CSV field - wrap in double quotes if it contains commas, quotes, or newlines
+// Double quotes inside must be doubled (""), per CSV standard
+export const csvEscapeField = (value: string | number): string => {
+ const str = String(value);
+
+ // If field contains commas, quotes, or newlines, it must be quoted
+ if (/[,"\n\r]/.test(str)) {
+ // Escape double quotes by doubling them
+ const escaped = str.replace(/"/g, '""');
+ return `"${escaped}"`;
+ }
+
+ return str;
+};
diff --git a/packages/db/src/clickhouse/migration.ts b/packages/db/src/clickhouse/migration.ts
index 2b65d5610..57f27d34f 100644
--- a/packages/db/src/clickhouse/migration.ts
+++ b/packages/db/src/clickhouse/migration.ts
@@ -115,6 +115,22 @@ ENGINE = Distributed('{cluster}', currentDatabase(), ${replicated(tableName)}, $
];
}
+export const modifyTTL = ({
+ tableName,
+ isClustered,
+ ttl,
+}: {
+ tableName: string;
+ isClustered: boolean;
+ ttl: string;
+}) => {
+ if (isClustered) {
+ return `ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' MODIFY TTL ${ttl}`;
+ }
+
+ return `ALTER TABLE ${tableName} MODIFY TTL ${ttl}`;
+};
+
/**
* Generates ALTER TABLE statements for adding columns
*/
diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts
index a182b69b3..6cd68a80a 100644
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -141,6 +141,10 @@ export type IServiceCreateEventPayload = Omit<
IServiceEvent,
'id' | 'importedAt' | 'profile' | 'meta'
>;
+export type IServiceImportedEventPayload = Omit<
+ IServiceEvent,
+ 'profile' | 'meta'
+>;
export interface IServiceEvent {
id: string;
diff --git a/packages/db/src/services/import.service.ts b/packages/db/src/services/import.service.ts
new file mode 100644
index 000000000..6d0b8f554
--- /dev/null
+++ b/packages/db/src/services/import.service.ts
@@ -0,0 +1,784 @@
+import type { ILogger } from '@openpanel/logger';
+import sqlstring from 'sqlstring';
+import {
+ TABLE_NAMES,
+ ch,
+ chInsertCSV,
+ convertClickhouseDateToJs,
+ formatClickhouseDate,
+} from '../clickhouse/client';
+import { csvEscapeField, csvEscapeJson } from '../clickhouse/csv';
+import { type Prisma, db } from '../prisma-client';
+import type { IClickhouseEvent } from './event.service';
+
+export interface ImportStageResult {
+ importId: string;
+ totalEvents: number;
+ insertedEvents: number;
+}
+
+export interface ImportProgress {
+ importId: string;
+ totalEvents: number;
+ insertedEvents: number;
+ status: 'pending' | 'processing' | 'processed' | 'failed';
+}
+
+/**
+ * Insert a batch of events into the imports staging table
+ */
+export async function insertImportBatch(
+ events: IClickhouseEvent[],
+ importId: string,
+): Promise<ImportStageResult> {
+ if (events.length === 0) {
+ return { importId, totalEvents: 0, insertedEvents: 0 };
+ }
+
+ // Important to have same order as events_imports table
+ // CSV format: properly quotes fields that need it
+ const csvRows = events.map((event) => {
+ // Properties need to be converted to JSON for Map(String, String)
+ // All fields must be CSV-escaped when joining with commas
+ const fields = [
+ csvEscapeField(event.id || ''),
+ csvEscapeField(event.name),
+ csvEscapeField(event.sdk_name || ''),
+ csvEscapeField(event.sdk_version || ''),
+ csvEscapeField(event.device_id || ''),
+ csvEscapeField(event.profile_id || ''),
+ csvEscapeField(event.project_id || ''),
+ csvEscapeField(event.session_id || ''),
+ csvEscapeField(event.path),
+ csvEscapeField(event.origin || ''),
+ csvEscapeField(event.referrer || ''),
+ csvEscapeField(event.referrer_name || ''),
+ csvEscapeField(event.referrer_type || ''),
+ csvEscapeField(event.duration ?? 0),
+ csvEscapeJson(event.properties),
+ csvEscapeField(event.created_at),
+ csvEscapeField(event.country || ''),
+ csvEscapeField(event.city || ''),
+ csvEscapeField(event.region || ''),
+ csvEscapeField(event.longitude != null ? event.longitude : '\\N'),
+ csvEscapeField(event.latitude != null ? event.latitude : '\\N'),
+ csvEscapeField(event.os || ''),
+ csvEscapeField(event.os_version || ''),
+ csvEscapeField(event.browser || ''),
+ csvEscapeField(event.browser_version || ''),
+ csvEscapeField(event.device || ''),
+ csvEscapeField(event.brand || ''),
+ csvEscapeField(event.model || ''),
+ csvEscapeField('\\N'), // imported_at (Nullable)
+ csvEscapeField(importId),
+ csvEscapeField('pending'), // import_status
+ csvEscapeField(formatClickhouseDate(new Date())), // imported_at_meta (DateTime, not DateTime64, so no milliseconds)
+ ];
+ return fields.join(',');
+ });
+
+ await chInsertCSV(TABLE_NAMES.events_imports, csvRows);
+
+ return {
+ importId,
+ totalEvents: events.length,
+ insertedEvents: events.length,
+ };
+}
+
+/**
+ * Generate deterministic session IDs for events that don't have them
+ * Uses 30-minute time windows to create consistent session IDs across imports
+ * Only processes events where device != 'server' and session_id = ''
+ */
+export async function generateSessionIds(
+ importId: string,
+ from: string,
+): Promise<void> {
+ const rangeWhere = [
+ 'import_id = {importId:String}',
+ "import_status = 'pending'",
+ "device != 'server'",
+ "session_id = ''",
+ from ? 'toDate(created_at) = {from:String}' : '',
+ ]
+ .filter(Boolean)
+ .join(' AND ');
+
+ // Use SQL to generate deterministic session IDs based on device_id + 30-min time windows
+ // This ensures same events always get same session IDs regardless of import order
+ const updateQuery = `
+ ALTER TABLE ${TABLE_NAMES.events_imports}
+ UPDATE session_id = lower(hex(MD5(concat(
+ device_id,
+ '-',
+ toString(toInt64(toUnixTimestamp(created_at) / 1800))
+ ))))
+ WHERE ${rangeWhere}
+ `;
+
+ await ch.command({
+ query: updateQuery,
+ query_params: { importId, from },
+ clickhouse_settings: {
+ wait_end_of_query: 1,
+ mutations_sync: '2', // Wait for mutation to complete on all replicas (critical!)
+ send_progress_in_http_headers: 1,
+ http_headers_progress_interval_ms: '50000',
+ },
+ });
+}
+
+/**
+ * Reconstruct sessions using SQL-based logic
+ * This identifies session boundaries and creates session_start/session_end events
+ * session_start inherits all properties from the first event in the session
+ * session_end inherits all properties from the last event in the session and calculates duration
+ */
+export async function createSessionsStartEndEvents(
+ importId: string,
+ from: string,
+): Promise<void> {
+ // First, let's identify session boundaries and get first/last events for each session
+ const rangeWhere = [
+ 'import_id = {importId:String}',
+ "import_status = 'pending'",
+ "session_id != ''", // Only process events that have session IDs
+ 'toDate(created_at) = {from:String}',
+ ]
+ .filter(Boolean)
+ .join(' AND ');
+
+ // Use window functions to efficiently get first event (all fields) and last event (only changing fields)
+ // session_end only needs: properties, path, origin, created_at - the rest can be inherited from session_start
+ const sessionEventsQuery = `
+ SELECT
+ device_id,
+ session_id,
+ project_id,
+ profile_id,
+ argMin((path, origin, referrer, referrer_name, referrer_type, properties, created_at, country, city, region, longitude, latitude, os, os_version, browser, browser_version, device, brand, model), created_at) AS first_event,
+ argMax((path, origin, properties, created_at), created_at) AS last_event_fields,
+ min(created_at) AS first_timestamp,
+ max(created_at) AS last_timestamp
+ FROM ${TABLE_NAMES.events_imports}
+ WHERE ${rangeWhere}
+ AND name NOT IN ('session_start', 'session_end')
+ GROUP BY session_id, device_id, project_id, profile_id
+ `;
+
+ const sessionEventsResult = await ch.query({
+ query: sessionEventsQuery,
+ query_params: { importId, from },
+ format: 'JSONEachRow',
+ });
+
+ const sessionData = (await sessionEventsResult.json()) as Array<{
+ device_id: string;
+ session_id: string;
+ project_id: string;
+ profile_id: string;
+ first_event: [
+ // string, // id
+ // string, // name
+ string, // path
+ string, // origin
+ string, // referrer
+ string, // referrer_name
+ string, // referrer_type
+ // number, // duration
+ Record<string, unknown>, // properties
+ string, // created_at
+ string, // country
+ string, // city
+ string, // region
+ number | null, // longitude
+ number | null, // latitude
+ string, // os
+ string, // os_version
+ string, // browser
+ string, // browser_version
+ string, // device
+ string, // brand
+ string, // model
+ // string, // sdk_name
+ // string, // sdk_version
+ // string, // imported_at
+ ];
+ last_event_fields: [
+ string, // path
+ string, // origin
+ Record<string, unknown>, // properties
+ string, // created_at
+ ];
+ first_timestamp: string;
+ last_timestamp: string;
+ }>;
+
+ // Create session_start and session_end events
+ const sessionEvents: IClickhouseEvent[] = [];
+
+ for (const session of sessionData) {
+ // Destructure first event tuple (all fields)
+ const [
+ // firstId,
+ // firstName,
+ firstPath,
+ firstOrigin,
+ firstReferrer,
+ firstReferrerName,
+ firstReferrerType,
+ // firstDuration,
+ firstProperties,
+ firstCreatedAt,
+ firstCountry,
+ firstCity,
+ firstRegion,
+ firstLongitude,
+ firstLatitude,
+ firstOs,
+ firstOsVersion,
+ firstBrowser,
+ firstBrowserVersion,
+ firstDevice,
+ firstBrand,
+ firstModel,
+ // firstSdkName,
+ // firstSdkVersion,
+ // firstImportedAt,
+ ] = session.first_event;
+
+ // Destructure last event fields (only the changing ones)
+ const [lastPath, lastOrigin, lastProperties, lastCreatedAt] =
+ session.last_event_fields;
+
+ // Calculate duration in milliseconds
+ // Parse timestamps as Date objects to calculate duration
+ const firstTime = new Date(session.first_timestamp).getTime();
+ const lastTime = new Date(session.last_timestamp).getTime();
+ const durationMs = lastTime - firstTime;
+
+ // Helper function to adjust timestamp by milliseconds without timezone conversion
+ const adjustTimestamp = (timestamp: string, offsetMs: number): string => {
+ // Parse the timestamp, adjust it, and format back to ClickHouse format
+ const date = convertClickhouseDateToJs(timestamp);
+ date.setTime(date.getTime() + offsetMs);
+ return formatClickhouseDate(date);
+ };
+
+ // Create session_start event - inherit everything from first event but change name
+ // Set created_at to 1 second before the first event
+ sessionEvents.push({
+ id: crypto.randomUUID(),
+ name: 'session_start',
+ device_id: session.device_id,
+ profile_id: session.profile_id,
+ project_id: session.project_id,
+ session_id: session.session_id,
+ path: firstPath,
+ origin: firstOrigin,
+ referrer: firstReferrer,
+ referrer_name: firstReferrerName,
+ referrer_type: firstReferrerType,
+ duration: 0, // session_start always has 0 duration
+ properties: firstProperties as Record<
+ string,
+ string | number | boolean | null | undefined
+ >,
+ created_at: adjustTimestamp(session.first_timestamp, -1000), // 1 second before first event
+ country: firstCountry,
+ city: firstCity,
+ region: firstRegion,
+ longitude: firstLongitude,
+ latitude: firstLatitude,
+ os: firstOs,
+ os_version: firstOsVersion,
+ browser: firstBrowser,
+ browser_version: firstBrowserVersion,
+ device: firstDevice,
+ brand: firstBrand,
+ model: firstModel,
+ imported_at: new Date().toISOString(),
+ sdk_name: 'import-session-reconstruction',
+ sdk_version: '1.0.0',
+ });
+
+ // Create session_end event - inherit most from session_start, but use last event's path, origin, properties
+ // Set created_at to 1 second after the last event
+ sessionEvents.push({
+ id: crypto.randomUUID(),
+ name: 'session_end',
+ device_id: session.device_id,
+ profile_id: session.profile_id,
+ project_id: session.project_id,
+ session_id: session.session_id,
+ path: lastPath, // From last event
+ origin: lastOrigin, // From last event
+ referrer: firstReferrer, // Same as session_start
+ referrer_name: firstReferrerName, // Same as session_start
+ referrer_type: firstReferrerType, // Same as session_start
+ duration: durationMs,
+ properties: lastProperties as Record<
+ string,
+ string | number | boolean | null | undefined
+ >, // From last event
+ created_at: adjustTimestamp(session.last_timestamp, 500), // 1 second after last event
+ country: firstCountry, // Same as session_start
+ city: firstCity, // Same as session_start
+ region: firstRegion, // Same as session_start
+ longitude: firstLongitude, // Same as session_start
+ latitude: firstLatitude, // Same as session_start
+ os: firstOs, // Same as session_start
+ os_version: firstOsVersion, // Same as session_start
+ browser: firstBrowser, // Same as session_start
+ browser_version: firstBrowserVersion, // Same as session_start
+ device: firstDevice, // Same as session_start
+ brand: firstBrand, // Same as session_start
+ model: firstModel, // Same as session_start
+ imported_at: new Date().toISOString(),
+ sdk_name: 'import-session-reconstruction',
+ sdk_version: '1.0.0',
+ });
+ }
+
+ // Insert session events into imports table
+ if (sessionEvents.length > 0) {
+ await insertImportBatch(sessionEvents, importId);
+ }
+}
+
+/**
+ * Migrate all events from imports table to production events table
+ * This includes both original events and generated session events
+ */
+export async function moveImportsToProduction(
+ importId: string,
+ from: string,
+): Promise<void> {
+ // Build the WHERE clause for migration
+ // For session events (session_start/session_end), we don't filter by their created_at
+ // because they're created with adjusted timestamps (±1 second) that might fall outside
+ // the date range. Instead, we include them if their session_id has events in this range.
+ let whereClause = 'import_id = {importId:String}';
+
+ if (from) {
+ whereClause += ` AND (
+ (toDate(created_at) = {from:String}) OR
+ (
+ name IN ('session_start', 'session_end') AND
+ session_id IN (
+ SELECT DISTINCT session_id
+ FROM ${TABLE_NAMES.events_imports}
+ WHERE import_id = {importId:String}
+ AND toDate(created_at) = {from:String}
+ AND name NOT IN ('session_start', 'session_end')
+ )
+ )
+ )`;
+ }
+
+ const migrationQuery = `
+ INSERT INTO ${TABLE_NAMES.events} (
+ id,
+ name,
+ sdk_name,
+ sdk_version,
+ device_id,
+ profile_id,
+ project_id,
+ session_id,
+ path,
+ origin,
+ referrer,
+ referrer_name,
+ referrer_type,
+ duration,
+ properties,
+ created_at,
+ country,
+ city,
+ region,
+ longitude,
+ latitude,
+ os,
+ os_version,
+ browser,
+ browser_version,
+ device,
+ brand,
+ model,
+ imported_at
+ )
+ SELECT
+ id,
+ name,
+ sdk_name,
+ sdk_version,
+ device_id,
+ profile_id,
+ project_id,
+ session_id,
+ path,
+ origin,
+ referrer,
+ referrer_name,
+ referrer_type,
+ duration,
+ properties,
+ created_at,
+ country,
+ city,
+ region,
+ longitude,
+ latitude,
+ os,
+ os_version,
+ browser,
+ browser_version,
+ device,
+ brand,
+ model,
+ imported_at
+ FROM ${TABLE_NAMES.events_imports}
+ WHERE ${whereClause}
+ ORDER BY created_at ASC
+ `;
+
+ await ch.command({
+ query: migrationQuery,
+ query_params: { importId, from },
+ clickhouse_settings: {
+ wait_end_of_query: 1,
+ // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.
+ send_progress_in_http_headers: 1,
+ // The interval of sending these progress headers. Here it is less than 60s,
+ http_headers_progress_interval_ms: '50000',
+ },
+ });
+}
+
+export async function backfillSessionsToProduction(
+ importId: string,
+ from: string,
+): Promise<void> {
+ // After migrating events, populate the sessions table based on the migrated sessions
+ // We detect all session_ids involved in this import from the imports table,
+ // then aggregate over the production events to construct session rows.
+ const sessionsInsertQuery = `
+ INSERT INTO ${TABLE_NAMES.sessions} (
+ id,
+ project_id,
+ profile_id,
+ device_id,
+ created_at,
+ ended_at,
+ is_bounce,
+ entry_origin,
+ entry_path,
+ exit_origin,
+ exit_path,
+ screen_view_count,
+ revenue,
+ event_count,
+ duration,
+ country,
+ region,
+ city,
+ longitude,
+ latitude,
+ device,
+ brand,
+ model,
+ browser,
+ browser_version,
+ os,
+ os_version,
+ sign,
+ version,
+ properties,
+ utm_medium,
+ utm_source,
+ utm_campaign,
+ utm_content,
+ utm_term,
+ referrer,
+ referrer_name,
+ referrer_type
+ )
+ SELECT
+ any(e.session_id) as id,
+ any(e.project_id) as project_id,
+ if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,
+ any(e.device_id) as device_id,
+ argMin(e.created_at, e.created_at) as created_at,
+ argMax(e.created_at, e.created_at) as ended_at,
+ if(
+ argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = '',
+ if(countIf(e.name = 'screen_view') > 1, true, false),
+ argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = 'true'
+ ) as is_bounce,
+ argMinIf(e.origin, e.created_at, e.name = 'session_start') as entry_origin,
+ argMinIf(e.path, e.created_at, e.name = 'session_start') as entry_path,
+ argMaxIf(e.origin, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_origin,
+ argMaxIf(e.path, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_path,
+ countIf(e.name = 'screen_view') as screen_view_count,
+ 0 as revenue,
+ countIf(e.name != 'screen_view' AND e.name != 'session_start' AND e.name != 'session_end') as event_count,
+ sumIf(e.duration, name = 'session_end') AS duration,
+ argMinIf(e.country, e.created_at, e.name = 'session_start') as country,
+ argMinIf(e.region, e.created_at, e.name = 'session_start') as region,
+ argMinIf(e.city, e.created_at, e.name = 'session_start') as city,
+ argMinIf(e.longitude, e.created_at, e.name = 'session_start') as longitude,
+ argMinIf(e.latitude, e.created_at, e.name = 'session_start') as latitude,
+ argMinIf(e.device, e.created_at, e.name = 'session_start') as device,
+ argMinIf(e.brand, e.created_at, e.name = 'session_start') as brand,
+ argMinIf(e.model, e.created_at, e.name = 'session_start') as model,
+ argMinIf(e.browser, e.created_at, e.name = 'session_start') as browser,
+ argMinIf(e.browser_version, e.created_at, e.name = 'session_start') as browser_version,
+ argMinIf(e.os, e.created_at, e.name = 'session_start') as os,
+ argMinIf(e.os_version, e.created_at, e.name = 'session_start') as os_version,
+ 1 as sign,
+ 1 as version,
+ argMinIf(e.properties, e.created_at, e.name = 'session_start') as properties,
+ argMinIf(e.properties['__query.utm_medium'], e.created_at, e.name = 'session_start') as utm_medium,
+ argMinIf(e.properties['__query.utm_source'], e.created_at, e.name = 'session_start') as utm_source,
+ argMinIf(e.properties['__query.utm_campaign'], e.created_at, e.name = 'session_start') as utm_campaign,
+ argMinIf(e.properties['__query.utm_content'], e.created_at, e.name = 'session_start') as utm_content,
+ argMinIf(e.properties['__query.utm_term'], e.created_at, e.name = 'session_start') as utm_term,
+ argMinIf(e.referrer, e.created_at, e.name = 'session_start') as referrer,
+ argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name,
+ argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_type
+ FROM ${TABLE_NAMES.events_imports} e
+ WHERE
+ e.import_id = ${sqlstring.escape(importId)}
+ AND toDate(e.created_at) = ${sqlstring.escape(from)}
+ AND e.session_id != ''
+ GROUP BY e.session_id
+ `;
+
+ await ch.command({
+ query: sessionsInsertQuery,
+ clickhouse_settings: {
+ wait_end_of_query: 1,
+ // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.
+ send_progress_in_http_headers: 1,
+ // The interval of sending these progress headers. Here it is less than 60s,
+ http_headers_progress_interval_ms: '50000',
+ },
+ });
+}
+
+/**
+ * Mark import as complete by updating status
+ */
+export async function markImportComplete(importId: string): Promise<void> {
+ const updateQuery = `
+ ALTER TABLE ${TABLE_NAMES.events_imports}
+ UPDATE import_status = 'processed'
+ WHERE import_id = {importId:String}
+ `;
+
+ await ch.command({
+ query: updateQuery,
+ query_params: { importId },
+ clickhouse_settings: {
+ wait_end_of_query: 1,
+ mutations_sync: '2', // Wait for mutation to complete
+ // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.
+ send_progress_in_http_headers: 1,
+ // The interval of sending these progress headers. Here it is less than 60s,
+ http_headers_progress_interval_ms: '50000',
+ },
+ });
+}
+
+/**
+ * Get import progress and status
+ */
+export async function getImportProgress(
+ importId: string,
+): Promise<ImportProgress> {
+ const progressQuery = `
+ SELECT
+ import_id,
+ COUNT(*) as total_events,
+ COUNTIf(import_status = 'pending') as pending_events,
+ COUNTIf(import_status = 'processed') as processed_events,
+ any(import_status) as status
+ FROM ${TABLE_NAMES.events_imports}
+ WHERE import_id = {importId:String}
+ AND name NOT IN ('session_start', 'session_end')
+ GROUP BY import_id
+ `;
+
+ const result = await ch.query({
+ query: progressQuery,
+ query_params: { importId },
+ format: 'JSONEachRow',
+ });
+
+ const data = (await result.json()) as Array<{
+ import_id: string;
+ total_events: number;
+ pending_events: number;
+ processed_events: number;
+ status: string;
+ }>;
+
+ if (data.length === 0) {
+ return {
+ importId,
+ totalEvents: 0,
+ insertedEvents: 0,
+ status: 'pending',
+ };
+ }
+
+ const row = data[0];
+ if (!row) {
+ return {
+ importId,
+ totalEvents: 0,
+ insertedEvents: 0,
+ status: 'pending',
+ };
+ }
+
+ return {
+ importId,
+ totalEvents: row.total_events,
+ insertedEvents: row.processed_events,
+ status: row.status as 'pending' | 'processing' | 'processed' | 'failed',
+ };
+}
+
+/**
+ * Utility: get min/max created_at for an import
+ */
+export async function getImportDateBounds(
+ importId: string,
+ fromCreatedAt?: string,
+): Promise<{ min: string | null; max: string | null }> {
+ const res = await ch.query({
+ query: `
+ SELECT min(created_at) AS min, max(created_at) AS max
+ FROM ${TABLE_NAMES.events_imports}
+ WHERE import_id = {importId:String}
+ ${fromCreatedAt ? 'AND created_at >= {fromCreatedAt:String}' : ''}
+ `,
+ query_params: { importId, fromCreatedAt },
+ format: 'JSONEachRow',
+ });
+ const rows = (await res.json()) as Array<{
+ min: string | null;
+ max: string | null;
+ }>;
+ return rows.length > 0
+ ? {
+ min: fromCreatedAt ?? rows[0]?.min ?? null,
+ max: rows[0]?.max ?? null,
+ }
+ : { min: null, max: null };
+}
+
+/**
+ * Unified method to update all import status information
+ * Combines step, batch, progress, and status message updates
+ */
+export type UpdateImportStatusOptions =
+ | {
+ step: 'loading';
+ batch?: string;
+ totalEvents?: number;
+ processedEvents?: number;
+ }
+ | {
+ step: 'generating_session_ids';
+ batch?: string;
+ }
+ | {
+ step: 'creating_sessions';
+ batch?: string;
+ }
+ | {
+ step: 'moving';
+ batch?: string;
+ }
+ | {
+ step: 'backfilling_sessions';
+ batch?: string;
+ }
+ | {
+ step: 'completed';
+ }
+ | {
+ step: 'failed';
+ errorMessage?: string;
+ };
+
+export type ImportSteps = UpdateImportStatusOptions['step'];
+
+export async function updateImportStatus(
+ jobLogger: ILogger,
+ job: {
+ updateProgress: (progress: Record<string, any>) => void;
+ },
+ importId: string,
+ options: UpdateImportStatusOptions,
+): Promise<void> {
+ const data: Prisma.ImportUpdateInput = {};
+ switch (options.step) {
+ case 'loading':
+ data.status = 'processing';
+ data.currentStep = 'loading';
+ data.currentBatch = options.batch;
+ data.statusMessage = options.batch
+ ? `Importing events from ${options.batch}`
+ : 'Initializing...';
+ data.totalEvents = options.totalEvents;
+ data.processedEvents = options.processedEvents;
+ break;
+ case 'generating_session_ids':
+ data.currentStep = 'generating_session_ids';
+ data.currentBatch = options.batch;
+ data.statusMessage = options.batch
+ ? `Generating session IDs for ${options.batch}`
+ : 'Generating session IDs...';
+ break;
+ case 'creating_sessions':
+ data.currentStep = 'creating_sessions';
+ data.currentBatch = options.batch;
+ data.statusMessage = `Creating sessions for ${options.batch}`;
+ break;
+ case 'moving':
+ data.currentStep = 'moving';
+ data.currentBatch = options.batch;
+ data.statusMessage = `Moving imports to production for ${options.batch}`;
+ break;
+ case 'backfilling_sessions':
+ data.currentStep = 'backfilling_sessions';
+ data.currentBatch = options.batch;
+ data.statusMessage = `Aggregating sessions for ${options.batch}`;
+ break;
+ case 'completed':
+ data.status = 'completed';
+ data.currentStep = 'completed';
+ data.statusMessage = 'Import completed';
+ data.completedAt = new Date();
+ break;
+ case 'failed':
+ data.status = 'failed';
+ data.statusMessage = 'Import failed';
+ data.errorMessage = options.errorMessage;
+ break;
+ }
+
+ jobLogger.info('Import status update', data);
+
+ await job.updateProgress(data);
+
+ await db.import.update({
+ where: { id: importId },
+ data,
+ });
+}
diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.ts
index 016e478e3..aa9fe2fbe 100644
--- a/packages/db/src/services/session.service.ts
+++ b/packages/db/src/services/session.service.ts
@@ -196,7 +196,7 @@ export async function getSessionList({
organization?.subscriptionPeriodEventsLimit &&
organization?.subscriptionPeriodEventsLimit > 1_000_000
? 1
- : 7;
+ : 360;
if (cursor) {
const cAt = sqlstring.escape(cursor.createdAt);
diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts
index 8d6db5860..7e525f25f 100644
--- a/packages/db/src/types.ts
+++ b/packages/db/src/types.ts
@@ -1,4 +1,5 @@
import type {
+ IImportConfig,
IIntegrationConfig,
INotificationRuleConfig,
IProjectFilters,
@@ -12,6 +13,7 @@ import type { IClickhouseProfile } from './services/profile.service';
declare global {
namespace PrismaJson {
+ type IPrismaImportConfig = IImportConfig;
type IPrismaNotificationRuleConfig = INotificationRuleConfig;
type IPrismaIntegrationConfig = IIntegrationConfig;
type IPrismaNotificationPayload = INotificationPayload;
diff --git a/packages/importer/package.json b/packages/importer/package.json
new file mode 100644
index 000000000..ff91d681a
--- /dev/null
+++ b/packages/importer/package.json
@@ -0,0 +1,35 @@
+{
+ "name": "@openpanel/importer",
+ "version": "0.0.0",
+ "type": "module",
+ "main": "src/index.ts",
+ "scripts": {
+ "build": "tsc",
+ "dev": "tsc --watch",
+ "test": "vitest",
+ "test:run": "vitest run"
+ },
+ "exports": {
+ ".": "./src/index.ts",
+ "./providers": "./src/providers/metadata.ts"
+ },
+ "dependencies": {
+ "@openpanel/common": "workspace:*",
+ "@openpanel/db": "workspace:*",
+ "@openpanel/queue": "workspace:*",
+ "@openpanel/validation": "workspace:*",
+ "csv-parse": "^6.1.0",
+ "ramda": "^0.29.1",
+ "uuid": "^9.0.1",
+ "zod": "catalog:"
+ },
+ "devDependencies": {
+ "@openpanel/logger": "workspace:*",
+ "@types/node": "^20.0.0",
+ "@types/ramda": "^0.31.1",
+ "@types/uuid": "^9.0.7",
+ "bullmq": "^5.8.7",
+ "typescript": "^5.0.0",
+ "vitest": "^1.0.0"
+ }
+}
diff --git a/packages/importer/src/base-provider.ts b/packages/importer/src/base-provider.ts
new file mode 100644
index 000000000..e7c6189fe
--- /dev/null
+++ b/packages/importer/src/base-provider.ts
@@ -0,0 +1,121 @@
+import type { IClickhouseEvent } from '@openpanel/db';
+import type { BaseRawEvent, ErrorContext, ImportJobMetadata } from './types';
+
+export abstract class BaseImportProvider<
+ TRawEvent extends BaseRawEvent = BaseRawEvent,
+> {
+ abstract provider: string;
+ abstract version: string;
+
+ /**
+ * Stream-read and parse source (file/API) → yields raw events
+ * This should be implemented as an async generator to handle large files efficiently
+ */
+ abstract parseSource(
+ overrideFrom?: string,
+ ): AsyncGenerator<TRawEvent, void, unknown>;
+
+ /**
+ * Convert provider format → IClickhouseEvent
+ */
+ abstract transformEvent(rawEvent: TRawEvent): IClickhouseEvent;
+
+ /**
+ * Validate raw event structure
+ */
+ abstract validate(rawEvent: TRawEvent): boolean;
+
+ /**
+ * Returns how many events will be imported
+ */
+ abstract getTotalEventsCount(): Promise<number>;
+
+ /**
+ * Optional hook: Pre-process batch
+ */
+ async beforeBatch?(events: TRawEvent[]): Promise<TRawEvent[]> {
+ return events;
+ }
+
+ /**
+ * Optional hook: Get import metadata for tracking
+ */
+ getImportMetadata?(): ImportJobMetadata;
+
+ /**
+ * Optional hook: Custom error handling
+ */
+ async onError?(error: Error, context?: ErrorContext): Promise<void> {
+ // Default: re-throw
+ throw error;
+ }
+
+ /**
+ * Get estimated total events (optional, for progress tracking)
+ */
+ async getEstimatedTotal?(): Promise<number> {
+ return 0;
+ }
+
+ /**
+ * Indicates whether session IDs should be generated in SQL after import
+ * If true, the import job will generate deterministic session IDs based on
+ * device_id and timestamp using SQL window functions
+ * If false, assumes the provider already generates session IDs during streaming
+ */
+ shouldGenerateSessionIds(): boolean {
+ return false; // Default: assume provider handles it
+ }
+
+ /**
+ * Utility: Split a date range into chunks to avoid timeout issues with large imports
+ * Returns array of [from, to] date pairs in YYYY-MM-DD format
+ *
+ * @param from - Start date in YYYY-MM-DD format
+ * @param to - End date in YYYY-MM-DD format
+ * @param chunkSizeDays - Number of days per chunk (default: 1)
+ */
+ public getDateChunks(
+ from: string,
+ to: string,
+ options?: {
+ chunkSizeDays?: number;
+ },
+ ): Array<[string, string]> {
+ const chunks: Array<[string, string]> = [];
+
+ const startDate = new Date(from);
+ const endDate = new Date(to);
+ const chunkSizeDays = options?.chunkSizeDays ?? 1;
+
+ // Handle case where from and to are the same date
+ if (startDate.getTime() === endDate.getTime()) {
+ return [[from, to]];
+ }
+
+ const cursor = new Date(startDate);
+
+ while (cursor <= endDate) {
+ const chunkStart = cursor.toISOString().split('T')[0]!;
+
+ // Calculate chunk end: move forward by (chunkSizeDays - 1) to get the last day of the chunk
+ const chunkEndDate = new Date(cursor);
+ chunkEndDate.setDate(chunkEndDate.getDate() + (chunkSizeDays - 1));
+
+ // Don't go past the end date
+ const chunkEnd =
+ chunkEndDate > endDate
+ ? endDate.toISOString().split('T')[0]!
+ : chunkEndDate.toISOString().split('T')[0]!;
+
+ chunks.push([chunkStart, chunkEnd]);
+
+ // Move cursor to the next chunk start (after the current chunk)
+ cursor.setDate(cursor.getDate() + chunkSizeDays);
+
+ if (cursor > endDate) break;
+ }
+
+ return chunks;
+ }
+}
diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.ts
new file mode 100644
index 000000000..52f4e80df
--- /dev/null
+++ b/packages/importer/src/index.ts
@@ -0,0 +1,13 @@
+export { UmamiProvider } from './providers/umami';
+export { MixpanelProvider } from './providers/mixpanel';
+export type {
+ ImportConfig,
+ ImportProgress,
+ ImportResult,
+ BatchResult,
+ BaseRawEvent,
+ ErrorContext,
+ EventProperties,
+ ImportJobMetadata,
+ ImportStageResult,
+} from './types';
diff --git a/packages/importer/src/providers/metadata.ts b/packages/importer/src/providers/metadata.ts
new file mode 100644
index 000000000..639394b72
--- /dev/null
+++ b/packages/importer/src/providers/metadata.ts
@@ -0,0 +1,30 @@
+export type ImportProviderId = 'umami' | 'mixpanel';
+export type ImportProviderType = 'file' | 'api';
+
+export interface ImportProviderMeta {
+ id: ImportProviderId;
+ name: string;
+ description: string;
+ logo: string;
+ backgroundColor: string;
+ types: ImportProviderType[];
+}
+
+export const IMPORT_PROVIDERS: ImportProviderMeta[] = [
+ {
+ id: 'umami',
+ name: 'Umami',
+ description: 'Import your analytics data from Umami',
+ logo: 'https://cdn.brandfetch.io/id_3VEohOm/w/180/h/180/theme/dark/logo.png?c=1dxbfHSJFAPEGdCLU4o5B',
+ backgroundColor: '#fff',
+ types: ['file'],
+ },
+ {
+ id: 'mixpanel',
+ name: 'Mixpanel',
+ description: 'Import your analytics data from Mixpanel API',
+ logo: 'https://cdn.brandfetch.io/idr_rhI2FS/theme/dark/idMJ8uODLv.svg?c=1dxbfHSJFAPEGdCLU4o5B',
+ backgroundColor: '#fff',
+ types: ['api'],
+ },
+];
diff --git a/packages/importer/src/providers/mixpanel.test.ts b/packages/importer/src/providers/mixpanel.test.ts
new file mode 100644
index 000000000..c3f1d0524
--- /dev/null
+++ b/packages/importer/src/providers/mixpanel.test.ts
@@ -0,0 +1,319 @@
+import { omit } from 'ramda';
+import { describe, expect, it } from 'vitest';
+import { MixpanelProvider } from './mixpanel';
+
+describe('mixpanel', () => {
+ it('should chunk date range into day chunks', async () => {
+ const provider = new MixpanelProvider('pid', {
+ from: '2025-01-01',
+ to: '2025-01-04',
+ serviceAccount: 'sa',
+ serviceSecret: 'ss',
+ projectId: '123',
+ provider: 'mixpanel',
+ type: 'api',
+ mapScreenViewProperty: undefined,
+ });
+
+ const chunks = provider.getDateChunks('2025-01-01', '2025-01-04');
+ expect(chunks).toEqual([
+ ['2025-01-01', '2025-01-01'],
+ ['2025-01-02', '2025-01-02'],
+ ['2025-01-03', '2025-01-03'],
+ ['2025-01-04', '2025-01-04'],
+ ]);
+ });
+
+ it('should transform event', async () => {
+ const provider = new MixpanelProvider('pid', {
+ from: '2025-01-01',
+ to: '2025-01-02',
+ serviceAccount: 'sa',
+ serviceSecret: 'ss',
+ projectId: '123',
+ provider: 'mixpanel',
+ type: 'api',
+ mapScreenViewProperty: undefined,
+ });
+
+ const rawEvent = {
+ event: '$mp_web_page_view',
+ properties: {
+ time: 1746097970,
+ distinct_id: '$device:123',
+ $browser: 'Chrome',
+ $browser_version: 135,
+ $city: 'Mumbai',
+ $current_url:
+ 'https://domain.com/state/maharashtra?utm_source=google&utm_medium=cpc&utm_campaignid=890&utm_adgroupid=&utm_adid=&utm_term=&utm_device=m&utm_network=x&utm_location=123&gclid=oqneoqow&gad_sour',
+ $device: 'Android',
+ $device_id: '123',
+ $initial_referrer: 'https://referrer.com/',
+ $initial_referring_domain: 'referrer.com',
+ $insert_id: 'source_id',
+ $lib_version: '2.60.0',
+ $mp_api_endpoint: 'api-js.mixpanel.com',
+ $mp_api_timestamp_ms: 1746078175363,
+ $mp_autocapture: true,
+ $os: 'Android',
+ $referrer: 'https://google.com/',
+ $referring_domain: 'referrer.com',
+ $region: 'Maharashtra',
+ $screen_height: 854,
+ $screen_width: 384,
+ current_domain: 'domain.com',
+ current_page_title:
+ 'Landeed: Satbara Utara, 7/12 Extract, Property Card & Index 2',
+ current_url_path: '/state/maharashtra',
+ current_url_protocol: 'https:',
+ current_url_search:
+ '?utm_source=google&utm_medium=cpc&utm_campaignid=890&utm_adgroupid=&utm_adid=&utm_term=&utm_device=m&utm_network=x&utm_location=123&gclid=oqneoqow&gad_source=5&gclid=EAIaIQobChMI6MnvhciBjQMVlS-DAx',
+ gclid: 'oqneoqow',
+ mp_country_code: 'IN',
+ mp_lib: 'web',
+ mp_processing_time_ms: 1746078175546,
+ mp_sent_by_lib_version: '2.60.0',
+ utm_medium: 'cpc',
+ utm_source: 'google',
+ },
+ };
+
+ const res = provider.transformEvent(rawEvent);
+
+ expect(res).toMatchObject({
+ id: expect.any(String),
+ name: 'screen_view',
+ device_id: '123',
+ profile_id: '123',
+ project_id: 'pid',
+ session_id: '',
+ properties: {
+ __source_insert_id: 'source_id',
+ __screen: '384x854',
+ __lib_version: '2.60.0',
+ '__query.utm_source': 'google',
+ '__query.utm_medium': 'cpc',
+ '__query.utm_campaignid': '890',
+ '__query.utm_device': 'm',
+ '__query.utm_network': 'x',
+ '__query.utm_location': '123',
+ '__query.gclid': 'oqneoqow',
+ __title:
+ 'Landeed: Satbara Utara, 7/12 Extract, Property Card & Index 2',
+ },
+ created_at: '2025-05-01T11:12:50.000Z',
+ country: 'IN',
+ city: 'Mumbai',
+ region: 'Maharashtra',
+ longitude: null,
+ latitude: null,
+ os: 'Android',
+ os_version: undefined,
+ browser: 'Chrome',
+ browser_version: '',
+ device: 'mobile',
+ brand: '',
+ model: '',
+ duration: 0,
+ path: '/state/maharashtra',
+ origin: 'https://domain.com',
+ referrer: 'https://referrer.com',
+ referrer_name: 'Google',
+ referrer_type: 'search',
+ imported_at: expect.any(String),
+ sdk_name: 'mixpanel (web)',
+ sdk_version: '1.0.0',
+ });
+ });
+
+ it('should parse stringified JSON in properties and flatten them', async () => {
+ const provider = new MixpanelProvider('pid', {
+ from: '2025-01-01',
+ to: '2025-01-02',
+ serviceAccount: 'sa',
+ serviceSecret: 'ss',
+ projectId: '123',
+ provider: 'mixpanel',
+ type: 'api',
+ mapScreenViewProperty: undefined,
+ });
+
+ const rawEvent = {
+ event: 'custom_event',
+ properties: {
+ time: 1746097970,
+ distinct_id: '$device:123',
+ $device_id: '123',
+ $user_id: 'user123',
+ mp_lib: 'web',
+ // Stringified JSON object - should be parsed and flattened
+ area: '{"displayText":"Malab, Nuh, Mewat","id":1189005}',
+ // Stringified JSON array - should be parsed and flattened
+ tags: '["tag1","tag2","tag3"]',
+ // Regular string - should remain as is
+ regularString: 'just a string',
+ // Number - should be converted to string
+ count: 42,
+ // Object - should be flattened
+ nested: { level1: { level2: 'value' } },
+ },
+ };
+
+ const res = provider.transformEvent(rawEvent);
+
+ expect(res.properties).toMatchObject({
+ // Parsed JSON object should be flattened with dot notation
+ 'area.displayText': 'Malab, Nuh, Mewat',
+ 'area.id': '1189005',
+ // Parsed JSON array should be flattened with numeric indices
+ 'tags.0': 'tag1',
+ 'tags.1': 'tag2',
+ 'tags.2': 'tag3',
+ // Regular values
+ regularString: 'just a string',
+ count: '42',
+ // Nested object flattened
+ 'nested.level1.level2': 'value',
+ });
+ });
+
+ it('should handle react-native referrer', async () => {
+ const provider = new MixpanelProvider('pid', {
+ from: '2025-01-01',
+ to: '2025-01-02',
+ serviceAccount: 'sa',
+ serviceSecret: 'ss',
+ projectId: '123',
+ provider: 'mixpanel',
+ type: 'api',
+ mapScreenViewProperty: undefined,
+ });
+
+ const rawEvent = {
+ event: 'ec_search_error',
+ properties: {
+ time: 1759947367,
+ distinct_id: '3385916',
+ $browser: 'Mobile Safari',
+ $browser_version: null,
+ $city: 'Bengaluru',
+ $current_url:
+ 'https://web.landeed.com/karnataka/ec-encumbrance-certificate',
+ $device: 'iPhone',
+ $device_id:
+ '199b498af1036c-0e943279a1292e-5c0f4368-51bf4-199b498af1036c',
+ $initial_referrer: 'https://www.google.com/',
+ $initial_referring_domain: 'www.google.com',
+ $insert_id: 'bclkaepeqcfuzt4v',
+ $lib_version: '2.60.0',
+ $mp_api_endpoint: 'api-js.mixpanel.com',
+ $mp_api_timestamp_ms: 1759927570699,
+ $os: 'iOS',
+ $region: 'Karnataka',
+ $screen_height: 852,
+ $screen_width: 393,
+ $search_engine: 'google',
+ $user_id: '3385916',
+ binaryReadableVersion: 'NA',
+ binaryVersion: 'NA',
+ component: '/karnataka/ec-encumbrance-certificate',
+ errMsg: 'Request failed with status code 500',
+ errType: 'SERVER_ERROR',
+ isSilentSearch: false,
+ isTimeout: false,
+ jsVersion: '0.42.0',
+ language: 'english',
+ mp_country_code: 'IN',
+ mp_lib: 'web',
+ mp_processing_time_ms: 1759927592421,
+ mp_sent_by_lib_version: '2.60.0',
+ os: 'web',
+ osVersion:
+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1',
+ phoneBrand: 'NA',
+ phoneManufacturer: 'NA',
+ phoneModel: 'NA',
+ searchUuid: '68e65d08-fd81-4ded-37d3-2b08d2bc70c3',
+ serverVersion: 'web2.0',
+ state: 17,
+ stateStr: '17',
+ statusCode: 500,
+ type: 'result_event',
+ utm_medium: 'cpc',
+ utm_source:
+ 'google%26utm_medium=cpc%26utm_campaignid=21380769590%26utm_adgroupid=%26utm_adid=%26utm_term=%26utm_device=m%26utm_network=%26utm_location=9062055%26gclid=%26gad_campaignid=21374496705%26gbraid=0AAAAAoV7mTM9mWFripzQ2Od0xXAfrW6p3%26wbraid=CmAKCQjwi4PHBhCUA',
+ },
+ };
+
+ const res = provider.transformEvent(rawEvent);
+
+ expect(res.id.length).toBeGreaterThan(30);
+ expect(res.imported_at).toMatch(
+ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/,
+ );
+ expect(omit(['id', 'imported_at'], res)).toEqual({
+ brand: 'Apple',
+ browser: 'GSA',
+ browser_version: 'null',
+ city: 'Bengaluru',
+ country: 'IN',
+ created_at: '2025-10-08T18:16:07.000Z',
+ device: 'mobile',
+ device_id: '199b498af1036c-0e943279a1292e-5c0f4368-51bf4-199b498af1036c',
+ duration: 0,
+ latitude: null,
+ longitude: null,
+ model: 'iPhone',
+ name: 'ec_search_error',
+ origin: 'https://web.landeed.com',
+ os: 'iOS',
+ os_version: '18.7.0',
+ path: '/karnataka/ec-encumbrance-certificate',
+ profile_id: '3385916',
+ project_id: 'pid',
+ properties: {
+ __lib_version: '2.60.0',
+ '__query.gad_campaignid': '21374496705',
+ '__query.gbraid': '0AAAAAoV7mTM9mWFripzQ2Od0xXAfrW6p3',
+ '__query.utm_campaignid': '21380769590',
+ '__query.utm_device': 'm',
+ '__query.utm_location': '9062055',
+ '__query.utm_medium': 'cpc',
+ '__query.utm_source': 'google',
+ '__query.wbraid': 'CmAKCQjwi4PHBhCUA',
+ __screen: '393x852',
+ __source_insert_id: 'bclkaepeqcfuzt4v',
+ __userAgent:
+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1',
+ binaryReadableVersion: 'NA',
+ binaryVersion: 'NA',
+ component: '/karnataka/ec-encumbrance-certificate',
+ errMsg: 'Request failed with status code 500',
+ errType: 'SERVER_ERROR',
+ isSilentSearch: 'false',
+ isTimeout: 'false',
+ jsVersion: '0.42.0',
+ language: 'english',
+ os: 'web',
+ osVersion:
+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1',
+ phoneBrand: 'NA',
+ phoneManufacturer: 'NA',
+ phoneModel: 'NA',
+ searchUuid: '68e65d08-fd81-4ded-37d3-2b08d2bc70c3',
+ serverVersion: 'web2.0',
+ state: '17',
+ stateStr: '17',
+ statusCode: '500',
+ type: 'result_event',
+ },
+ referrer: 'https://www.google.com',
+ referrer_name: 'Google',
+ referrer_type: 'search',
+ region: 'Karnataka',
+ sdk_name: 'mixpanel (web)',
+ sdk_version: '1.0.0',
+ session_id: '',
+ });
+ });
+});
diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/importer/src/providers/mixpanel.ts
new file mode 100644
index 000000000..1d29bb331
--- /dev/null
+++ b/packages/importer/src/providers/mixpanel.ts
@@ -0,0 +1,452 @@
+import { randomUUID } from 'node:crypto';
+import { isSameDomain, parsePath, toDots } from '@openpanel/common';
+import { type UserAgentInfo, parseUserAgent } from '@openpanel/common/server';
+import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';
+import type { IClickhouseEvent } from '@openpanel/db';
+import type { ILogger } from '@openpanel/logger';
+import type { IMixpanelImportConfig } from '@openpanel/validation';
+import { z } from 'zod';
+import { BaseImportProvider } from '../base-provider';
+
+export const zMixpanelRawEvent = z.object({
+ event: z.string(),
+ properties: z.record(z.unknown()),
+});
+
+export type MixpanelRawEvent = z.infer<typeof zMixpanelRawEvent>;
+
+export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {
+ provider = 'mixpanel';
+ version = '1.0.0';
+
+ constructor(
+ private readonly projectId: string,
+ private readonly config: IMixpanelImportConfig,
+ private readonly logger?: ILogger,
+ ) {
+ super();
+ }
+
+ async getTotalEventsCount(): Promise<number> {
+ // Mixpanel sucks and dont provide a good way to extract total event count within a period
+ // jql would work but not accurate and will be deprecated end of 2025
+ return -1;
+ }
+
+ /**
+ * Mixpanel doesn't provide session IDs, so we need to generate them in SQL
+ * after all events are imported to ensure deterministic results
+ */
+ shouldGenerateSessionIds(): boolean {
+ return true;
+ }
+
+ async *parseSource(
+ overrideFrom?: string,
+ ): AsyncGenerator<MixpanelRawEvent, void, unknown> {
+ yield* this.fetchEventsFromMixpanel(overrideFrom);
+ }
+
+ private async *fetchEventsFromMixpanel(
+ overrideFrom?: string,
+ ): AsyncGenerator<MixpanelRawEvent, void, unknown> {
+ const { serviceAccount, serviceSecret, projectId, from, to } = this.config;
+
+ // Split the date range into monthly chunks for reliability
+ // Uses base class utility to avoid timeout issues with large date ranges
+ const dateChunks = this.getDateChunks(overrideFrom ?? from, to); // 1 month per chunk
+
+ for (const [chunkFrom, chunkTo] of dateChunks) {
+ yield* this.fetchEventsForDateRange(
+ serviceAccount,
+ serviceSecret,
+ projectId,
+ chunkFrom,
+ chunkTo,
+ );
+ }
+ }
+
+ private async *fetchEventsForDateRange(
+ serviceAccount: string,
+ serviceSecret: string,
+ projectId: string,
+ from: string,
+ to: string,
+ ): AsyncGenerator<MixpanelRawEvent, void, unknown> {
+ const url = 'https://data.mixpanel.com/api/2.0/export';
+
+ const params = new URLSearchParams({
+ from_date: from,
+ to_date: to,
+ project_id: projectId,
+ });
+
+ this.logger?.info('Fetching events from Mixpanel', {
+ url: `${url}?${params}`,
+ from,
+ to,
+ projectId,
+ serviceAccount,
+ });
+
+ const response = await fetch(`${url}?${params}`, {
+ method: 'GET',
+ headers: {
+ Authorization: `Basic ${Buffer.from(`${serviceAccount}:${serviceSecret}`).toString('base64')}`,
+ Accept: 'application/json',
+ },
+ });
+
+ if (!response.ok) {
+ throw new Error(
+ `Failed to fetch events from Mixpanel: ${response.status} ${response.statusText}`,
+ );
+ }
+
+ if (!response.body) {
+ throw new Error('No response body from Mixpanel API');
+ }
+
+ // Stream the response line by line
+ const reader = response.body.getReader();
+ const decoder = new TextDecoder();
+ let buffer = '';
+
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+
+ if (done) break;
+
+ buffer += decoder.decode(value, { stream: true });
+
+ // Process complete lines
+ const lines = buffer.split('\n');
+ buffer = lines.pop() || ''; // Keep the last incomplete line in buffer
+
+ for (const line of lines) {
+ if (line.trim()) {
+ try {
+ const event = JSON.parse(line);
+ yield event;
+ } catch (error) {
+ console.warn('Failed to parse Mixpanel event:', line);
+ }
+ }
+ }
+ }
+
+ // Process any remaining line in buffer
+ if (buffer.trim()) {
+ try {
+ const event = JSON.parse(buffer);
+ yield event;
+ } catch (error) {
+ console.warn('Failed to parse final Mixpanel event:', buffer);
+ }
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ }
+
+ validate(rawEvent: MixpanelRawEvent): boolean {
+ const res = zMixpanelRawEvent.safeParse(rawEvent);
+ return res.success;
+ }
+
+ transformEvent(_rawEvent: MixpanelRawEvent): IClickhouseEvent {
+ const projectId = this.projectId;
+ const rawEvent = zMixpanelRawEvent.parse(_rawEvent);
+ const props = rawEvent.properties as Record<string, any>;
+ const deviceId = props.$device_id;
+ const profileId = String(props.$user_id || props.distinct_id).replace(
+ /^\$device:/,
+ '',
+ );
+
+ // Build full URL from current_url and current_url_search (web only)
+ const fullUrl = props.$current_url;
+ let path = '';
+ let origin = '';
+ let hash = '';
+ let query: Record<string, string> = {};
+
+ if (fullUrl) {
+ const parsed = parsePath(fullUrl);
+ path = parsed.path || '';
+ origin = parsed.origin || '';
+ hash = parsed.hash || '';
+ query = parsed.query || {};
+ } else if (this.config.mapScreenViewProperty) {
+ path = props[this.config.mapScreenViewProperty] || '';
+ }
+
+ // Extract referrer information (web only)
+ const referrerUrl = props.$initial_referrer || props.$referrer || '';
+ const referrer =
+ referrerUrl && !isSameDomain(referrerUrl, fullUrl)
+ ? parseReferrer(referrerUrl)
+ : null;
+
+ // Check for UTM referrer in query params (web only)
+ const utmReferrer = getReferrerWithQuery(query);
+
+ // Extract location data
+ const country = props.$country || props.mp_country_code || '';
+ const city = props.$city || '';
+ const region = props.$region || '';
+
+ // For web events, use the standard user agent parsing
+ const userAgent = props.osVersion || '';
+ const uaInfo = this.isWebEvent(props.mp_lib)
+ ? parseUserAgent(userAgent, props)
+ : this.parseServerDeviceInfo(props);
+
+ // Map event name - $mp_web_page_view should be screen_view
+ let eventName = rawEvent.event;
+ if (eventName === '$mp_web_page_view') {
+ eventName = 'screen_view';
+ }
+
+ // Build properties object - strip Mixpanel-specific properties
+ const properties = this.stripMixpanelProperties(props, query);
+
+ if (props.$insert_id) {
+ properties.__source_insert_id = String(props.$insert_id);
+ }
+ // Add useful properties
+ if (props.$screen_width && props.$screen_height) {
+ properties.__screen = `${props.$screen_width}x${props.$screen_height}`;
+ }
+ if (props.$screen_dpi) {
+ properties.__dpi = props.$screen_dpi;
+ }
+ if (props.$language) {
+ properties.__language = props.$language;
+ }
+ if (props.$timezone) {
+ properties.__timezone = props.$timezone;
+ }
+ if (props.$app_version) {
+ properties.__version = props.$app_version;
+ }
+ if (props.$app_build_number) {
+ properties.__buildNumber = props.$app_build_number;
+ }
+ if (props.$lib_version) {
+ properties.__lib_version = props.$lib_version;
+ }
+
+ if (hash) {
+ properties.__hash = hash;
+ }
+
+ if (Object.keys(query).length > 0) {
+ properties.__query = query;
+ }
+
+ if (props.current_page_title) {
+ properties.__title = props.current_page_title;
+ }
+
+ if (userAgent) {
+ properties.__userAgent = userAgent;
+ }
+
+ // Always use UUID for id to match ClickHouse UUID column
+ const event = {
+ id: randomUUID(),
+ name: eventName,
+ device_id: deviceId,
+ profile_id: profileId,
+ project_id: projectId,
+ session_id: '', // Will be generated in SQL after import
+ properties: toDots(properties), // Flatten nested objects/arrays to Map(String, String)
+ created_at: new Date(props.time * 1000).toISOString(),
+ country,
+ city,
+ region,
+ longitude: null,
+ latitude: null,
+ os: uaInfo.os || props.$os,
+ os_version: uaInfo.osVersion || props.$osVersion,
+ browser: uaInfo.browser || props.$browser,
+ browser_version:
+ uaInfo.browserVersion || props.$browserVersion
+ ? String(props.$browser_version)
+ : '',
+ device: this.getDeviceType(props.mp_lib, uaInfo, props),
+ brand: uaInfo.brand || '',
+ model: uaInfo.model || '',
+ duration: 0,
+ path,
+ origin,
+ referrer: referrer?.url || '',
+ referrer_name: utmReferrer?.name || referrer?.name || '',
+ referrer_type: referrer?.type || utmReferrer?.type || '',
+ imported_at: new Date().toISOString(),
+ sdk_name: props.mp_lib
+ ? `${this.provider} (${props.mp_lib})`
+ : this.provider,
+ sdk_version: this.version,
+ };
+
+ // TODO: Remove this
+ // Temporary fix for a client
+ const isMightBeScreenView = this.getMightBeScreenView(rawEvent);
+ if (isMightBeScreenView && event.name === 'Loaded a Screen') {
+ event.name = 'screen_view';
+ event.path = isMightBeScreenView;
+ }
+
+ // TODO: Remove this
+ // This is a hack to get utm tags (not sure if this is just the testing project or all mixpanel projects)
+ if (props.utm_source && !properties.__query?.utm_source) {
+ const split = decodeURIComponent(props.utm_source).split('&');
+ const query = Object.fromEntries(split.map((item) => item.split('=')));
+ for (const [key, value] of Object.entries(query)) {
+ if (key && value) {
+ event.properties[`__query.${key}`] = String(value);
+ } else if (
+ value === undefined &&
+ key &&
+ props.utm_source.startsWith(key)
+ ) {
+ event.properties['__query.utm_source'] = String(key);
+ }
+ }
+ }
+
+ return event;
+ }
+
+ private getDeviceType(
+ mp_lib: string,
+ uaInfo: UserAgentInfo,
+ props: Record<string, any>,
+ ) {
+ // Normalize lib/os/browser data
+ const lib = (mp_lib || '').toLowerCase();
+ const os = String(props.$os || uaInfo.os || '').toLowerCase();
+ const browser = String(
+ props.$browser || uaInfo.browser || '',
+ ).toLowerCase();
+
+ const isTabletOs = os === 'ipados' || os === 'ipad os' || os === 'ipad';
+
+ // Strong hint from SDK library
+ if (['android', 'iphone', 'react-native', 'swift', 'unity'].includes(lib)) {
+ return isTabletOs ? 'tablet' : 'mobile';
+ }
+
+ // Web or unknown SDKs: infer from OS/Browser
+ const isMobileSignal =
+ os === 'ios' ||
+ os === 'android' ||
+ browser.includes('mobile safari') ||
+ browser.includes('chrome ios') ||
+ browser.includes('android mobile') ||
+ browser.includes('samsung internet') ||
+ browser.includes('mobile');
+
+ if (isMobileSignal) {
+ return 'mobile';
+ }
+
+ const isTabletSignal =
+ isTabletOs ||
+ browser.includes('tablet') ||
+ // iPad often reports as Mac OS X with Mobile Safari
+ (browser.includes('mobile safari') &&
+ (os === 'mac os x' || os === 'macos'));
+
+ if (isTabletSignal) {
+ return 'tablet';
+ }
+
+ // Default to desktop
+ return this.isServerEvent(mp_lib) ? 'server' : 'desktop';
+ }
+
+ private isWebEvent(mp_lib: string) {
+ return [
+ 'web',
+ 'android',
+ 'iphone',
+ 'swift',
+ 'unity',
+ 'react-native',
+ ].includes(mp_lib);
+ }
+
+ private isServerEvent(mp_lib: string) {
+ return !this.isWebEvent(mp_lib);
+ }
+
+ private getMightBeScreenView(rawEvent: MixpanelRawEvent) {
+ const props = rawEvent.properties as Record<string, any>;
+ return Object.keys(props).find((key) => key.match(/^[A-Z1-9_]+$/));
+ }
+
+ private parseServerDeviceInfo(props: Record<string, any>): UserAgentInfo {
+ // For mobile events, extract device information from Mixpanel properties
+ const os = props.$os || props.os || '';
+ const osVersion = props.$os_version || props.osVersion || '';
+ const brand = props.$brand || props.phoneBrand || '';
+ const model = props.$model || props.phoneModel || '';
+ const device = os.toLowerCase();
+
+ return {
+ isServer: true,
+ os: os,
+ osVersion: osVersion,
+ browser: '',
+ browserVersion: '',
+ device: device,
+ brand: brand,
+ model: model,
+ };
+ }
+
+ private stripMixpanelProperties(
+ properties: Record<string, any>,
+ searchParams: Record<string, string>,
+ ): Record<string, any> {
+ const strip = [
+ 'time',
+ 'distinct_id',
+ 'current_page_title',
+ 'current_url_path',
+ 'current_url_protocol',
+ 'current_url_search',
+ 'current_domain',
+ ...Object.keys(searchParams),
+ ];
+ const filtered = Object.fromEntries(
+ Object.entries(properties).filter(
+ ([key]) => !key.match(/^(\$|mp_|utm_)/) && !strip.includes(key),
+ ),
+ );
+
+ // Parse JSON strings back to objects/arrays so toDots() can flatten them
+ const parsed: Record<string, any> = {};
+ for (const [key, value] of Object.entries(filtered)) {
+ if (
+ typeof value === 'string' &&
+ (value.startsWith('{') || value.startsWith('['))
+ ) {
+ try {
+ parsed[key] = JSON.parse(value);
+ } catch {
+ parsed[key] = value; // Keep as string if parsing fails
+ }
+ } else {
+ parsed[key] = value;
+ }
+ }
+
+ return parsed;
+ }
+}
diff --git a/packages/importer/src/providers/umami.ts b/packages/importer/src/providers/umami.ts
new file mode 100644
index 000000000..f232e1b8e
--- /dev/null
+++ b/packages/importer/src/providers/umami.ts
@@ -0,0 +1,382 @@
+import { randomUUID } from 'node:crypto';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { createBrotliDecompress, createGunzip } from 'node:zlib';
+import { isSameDomain, parsePath } from '@openpanel/common';
+import { generateDeviceId } from '@openpanel/common/server';
+import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';
+import type { IClickhouseEvent } from '@openpanel/db';
+import type { ILogger } from '@openpanel/logger';
+import type { IUmamiImportConfig } from '@openpanel/validation';
+import { parse } from 'csv-parse';
+import { assocPath } from 'ramda';
+import { z } from 'zod';
+import { BaseImportProvider } from '../base-provider';
+
+export const zUmamiRawEvent = z.object({
+ // Required fields
+ event_type: z.coerce.number(),
+ event_name: z.string(),
+ created_at: z.coerce.date(),
+ event_id: z.string().min(1),
+ session_id: z.string().min(1),
+ website_id: z.string().min(1),
+
+ // Optional fields that might be empty
+ visit_id: z.string().optional(),
+ distinct_id: z.string().optional(),
+ url_path: z.string().optional(),
+ hostname: z.string().optional(),
+ referrer_domain: z.string().optional(),
+ referrer_path: z.string().optional(),
+ referrer_query: z.string().optional(),
+ referrer_name: z.string().optional(),
+ referrer_type: z.string().optional(),
+ country: z.string().optional(),
+ city: z.string().optional(),
+ region: z.string().optional(),
+ browser: z.string().optional(),
+ os: z.string().optional(),
+ device: z.string().optional(),
+ screen: z.string().optional(),
+ language: z.string().optional(),
+ utm_source: z.string().optional(),
+ utm_medium: z.string().optional(),
+ utm_campaign: z.string().optional(),
+ utm_content: z.string().optional(),
+ utm_term: z.string().optional(),
+ page_title: z.string().optional(),
+ gclid: z.string().optional(),
+ fbclid: z.string().optional(),
+ msclkid: z.string().optional(),
+ ttclid: z.string().optional(),
+ li_fat_id: z.string().optional(),
+ twclid: z.string().optional(),
+ url_query: z.string().optional(),
+});
+export type UmamiRawEvent = z.infer<typeof zUmamiRawEvent>;
+
+export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {
+ provider = 'umami';
+ version = '1.0.0';
+
+ constructor(
+ private readonly projectId: string,
+ private readonly config: IUmamiImportConfig,
+ private readonly logger?: ILogger,
+ ) {
+ super();
+ }
+
+ async getTotalEventsCount(): Promise<number> {
+ return -1;
+ }
+
+ async *parseSource(): AsyncGenerator<UmamiRawEvent, void, unknown> {
+ yield* this.parseRemoteFile(this.config.fileUrl);
+ }
+
+ private async *parseRemoteFile(
+ url: string,
+ opts: {
+ signal?: AbortSignal;
+ maxBytes?: number;
+ maxRows?: number;
+ } = {},
+ ): AsyncGenerator<UmamiRawEvent, void, unknown> {
+ const { signal, maxBytes, maxRows } = opts;
+ const controller = new AbortController();
+
+ // Link to caller's signal for cancellation
+ if (signal) {
+ signal.addEventListener('abort', () => controller.abort(), {
+ once: true,
+ });
+ }
+
+ const res = await fetch(url, { signal: controller.signal });
+ if (!res.ok || !res.body) {
+ throw new Error(
+ `Failed to fetch remote file: ${res.status} ${res.statusText}`,
+ );
+ }
+
+ const contentType = res.headers.get('content-type') || '';
+ const contentEnc = res.headers.get('content-encoding') || '';
+ const contentLen = Number(res.headers.get('content-length') ?? 0);
+
+ if (
+ contentType &&
+ !/text\/csv|text\/plain|application\/gzip|application\/octet-stream/i.test(
+ contentType,
+ )
+ ) {
+ console.warn(`Warning: Content-Type is ${contentType}, expected CSV-ish`);
+ }
+
+ if (maxBytes && contentLen && contentLen > maxBytes) {
+ throw new Error(
+ `Remote file exceeds size limit (${contentLen} > ${maxBytes})`,
+ );
+ }
+
+ const looksGzip =
+ /\.gz($|\?)/i.test(url) ||
+ /gzip/i.test(contentEnc) ||
+ /application\/gzip/i.test(contentType);
+ const looksBr = /br/i.test(contentEnc) || /\.br($|\?)/i.test(url);
+
+ // WHATWG -> Node stream
+ const body = Readable.fromWeb(res.body as any);
+
+ // Optional size guard during stream
+ let seenBytes = 0;
+ if (maxBytes) {
+ body.on('data', (chunk: Buffer) => {
+ seenBytes += chunk.length;
+ if (seenBytes > maxBytes) {
+ controller.abort();
+ body.destroy(
+ new Error(
+ `Stream exceeded size limit (${seenBytes} > ${maxBytes})`,
+ ),
+ );
+ }
+ });
+ }
+
+ // Build decode chain (gzip/brotli -> CSV parser)
+ const decompress = looksGzip
+ ? createGunzip()
+ : looksBr
+ ? createBrotliDecompress()
+ : null;
+
+ const parser = parse({
+ columns: true, // objects per row
+ bom: true, // handle UTF-8 BOM
+ relax_column_count: true,
+ skip_empty_lines: true,
+ });
+
+ // Wire the pipeline for proper backpressure & error propagation
+ (async () => {
+ try {
+ if (decompress) {
+ await pipeline(body, decompress, parser, {
+ signal: controller.signal,
+ });
+ } else {
+ await pipeline(body, parser, { signal: controller.signal });
+ }
+ } catch (e) {
+ parser.destroy(e as Error);
+ }
+ })().catch(() => {
+ /* handled by iterator */
+ });
+
+ let rows = 0;
+ try {
+ for await (const record of parser) {
+ rows++;
+ if (maxRows && rows > maxRows) {
+ controller.abort();
+ throw new Error(`Row limit exceeded (${rows} > ${maxRows})`);
+ }
+ yield record as UmamiRawEvent;
+ }
+ } catch (err) {
+ throw new Error(
+ `Failed to parse remote file from ${url}: ${
+ err instanceof Error ? err.message : String(err)
+ }`,
+ );
+ } finally {
+ controller.abort(); // ensure fetch stream is torn down
+ }
+ }
+
+ validate(rawEvent: UmamiRawEvent): boolean {
+ const res = zUmamiRawEvent.safeParse(rawEvent);
+ return res.success;
+ }
+
+ transformEvent(_rawEvent: UmamiRawEvent): IClickhouseEvent {
+ const projectId =
+ this.config.projectMapper.find(
+ (mapper) => mapper.from === _rawEvent.website_id,
+ )?.to || this.projectId;
+
+ const rawEvent = zUmamiRawEvent.parse(_rawEvent);
+ // Extract device/profile ID - use visit_id as device_id, session_id for session tracking
+ const deviceId =
+ rawEvent.visit_id ||
+ generateDeviceId({
+ ip: rawEvent.visit_id!,
+ ua: rawEvent.visit_id!,
+ origin: projectId,
+ salt: 'xxx',
+ });
+ const profileId = rawEvent.distinct_id || deviceId;
+
+ // Parse URL if available - use same logic as real-time events
+ const url = rawEvent.url_path
+ ? `https://${[rawEvent.hostname, rawEvent.url_path, rawEvent.url_query]
+ .filter(Boolean)
+ .join('')}`
+ : '';
+ const { path, hash, query, origin } = parsePath(url);
+ // Extract referrer information - use same logic as real-time events
+ const referrerUrl = rawEvent.referrer_domain
+ ? `https://${rawEvent.referrer_domain}${rawEvent.referrer_path || ''}`
+ : '';
+
+ // Check if referrer is from same domain (like real-time events do)
+ const referrer = isSameDomain(referrerUrl, url)
+ ? null
+ : parseReferrer(referrerUrl);
+
+ // Check for UTM referrer in query params (like real-time events do)
+ const utmReferrer = getReferrerWithQuery(query);
+
+ // Extract location data
+ const country = rawEvent.country || '';
+ const city = rawEvent.city || '';
+ const region = rawEvent.region || '';
+
+ // Extract browser/device info
+ const browser = rawEvent.browser || '';
+ const browserVersion = ''; // Not available in Umami CSV
+ const os = rawEvent.os || '';
+ const osVersion = ''; // Not available in Umami CSV
+ const device = rawEvent.device || '';
+ const brand = ''; // Not available in Umami CSV
+ const model = ''; // Not available in Umami CSV
+
+ let properties: Record<string, any> = {};
+
+ if (query) {
+ properties.__query = query;
+ }
+
+ // Add useful properties from Umami data
+ if (rawEvent.page_title) properties.__title = rawEvent.page_title;
+ if (rawEvent.screen) properties.__screen = rawEvent.screen;
+ if (rawEvent.language) properties.__language = rawEvent.language;
+ if (rawEvent.utm_source)
+ properties = assocPath(
+ ['__query', 'utm_source'],
+ rawEvent.utm_source,
+ properties,
+ );
+ if (rawEvent.utm_medium)
+ properties = assocPath(
+ ['__query', 'utm_medium'],
+ rawEvent.utm_medium,
+ properties,
+ );
+ if (rawEvent.utm_campaign)
+ properties = assocPath(
+ ['__query', 'utm_campaign'],
+ rawEvent.utm_campaign,
+ properties,
+ );
+ if (rawEvent.utm_content)
+ properties = assocPath(
+ ['__query', 'utm_content'],
+ rawEvent.utm_content,
+ properties,
+ );
+ if (rawEvent.utm_term)
+ properties = assocPath(
+ ['__query', 'utm_term'],
+ rawEvent.utm_term,
+ properties,
+ );
+
+ return {
+ id: rawEvent.event_id || randomUUID(),
+ name: rawEvent.event_type === 1 ? 'screen_view' : rawEvent.event_name,
+ device_id: deviceId,
+ profile_id: profileId,
+ project_id: projectId,
+ session_id: rawEvent.session_id || '',
+ properties,
+ created_at: rawEvent.created_at.toISOString(),
+ country,
+ city,
+ region: this.mapRegion(region),
+ longitude: null,
+ latitude: null,
+ os,
+ os_version: osVersion,
+ browser: this.mapBrowser(browser),
+ browser_version: browserVersion,
+ device: this.mapDevice(device),
+ brand,
+ model,
+ duration: 0,
+ path,
+ origin,
+ referrer: utmReferrer?.url || referrer?.url || '',
+ referrer_name: utmReferrer?.name || referrer?.name || '',
+ referrer_type: utmReferrer?.type || referrer?.type || '',
+ imported_at: new Date().toISOString(),
+ sdk_name: this.provider,
+ sdk_version: this.version,
+ };
+ }
+
+ mapRegion(region: string): string {
+ return region.replace(/^[A-Z]{2}\-/, '');
+ }
+
+ mapDevice(device: string): string {
+ const mapping: Record<string, string> = {
+ desktop: 'desktop',
+ laptop: 'desktop',
+ mobile: 'mobile',
+ tablet: 'tablet',
+ smarttv: 'smarttv',
+ Unknown: 'desktop',
+ };
+
+ return mapping[device] || 'desktop';
+ }
+
+ mapBrowser(browser: string): string {
+ const mapping: Record<string, string> = {
+ android: 'Android',
+ aol: 'AOL',
+ bb10: 'BlackBerry 10',
+ beaker: 'Beaker',
+ chrome: 'Chrome',
+ 'chromium-webview': 'Chrome (webview)',
+ crios: 'Chrome (iOS)',
+ curl: 'Curl',
+ edge: 'Edge',
+ 'edge-chromium': 'Edge (Chromium)',
+ 'edge-ios': 'Edge (iOS)',
+ facebook: 'Facebook',
+ firefox: 'Firefox',
+ fxios: 'Firefox (iOS)',
+ ie: 'IE',
+ instagram: 'Instagram',
+ ios: 'iOS',
+ 'ios-webview': 'iOS (webview)',
+ kakaotalk: 'KakaoTalk',
+ miui: 'MIUI',
+ opera: 'Opera',
+ 'opera-mini': 'Opera Mini',
+ phantomjs: 'PhantomJS',
+ safari: 'Safari',
+ samsung: 'Samsung',
+ searchbot: 'Searchbot',
+ silk: 'Silk',
+ yandexbrowser: 'Yandex',
+ };
+
+ return mapping[browser] || browser || 'Unknown';
+ }
+}
diff --git a/packages/importer/src/types.ts b/packages/importer/src/types.ts
new file mode 100644
index 000000000..ae56bf29e
--- /dev/null
+++ b/packages/importer/src/types.ts
@@ -0,0 +1,80 @@
+import type {
+ IImportedEvent,
+ IServiceCreateEventPayload,
+ IServiceImportedEventPayload,
+} from '@openpanel/db';
+
+export interface ImportConfig {
+ projectId: string;
+ provider: string;
+ sourceType: 'file' | 'api';
+ sourceLocation: string;
+}
+
+export interface SessionInfo {
+ id: string;
+ lastTimestamp: number;
+ lastEvent: IServiceImportedEventPayload;
+}
+
+export interface ImportProgress {
+ totalEvents: number;
+ processedEvents: number;
+ currentBatch: number;
+ totalBatches: number;
+}
+
+export interface ImportResult {
+ success: boolean;
+ totalEvents: number;
+ processedEvents: number;
+ error?: string;
+}
+
+export interface BatchResult {
+ events: IServiceImportedEventPayload[];
+ sessionEvents: IServiceImportedEventPayload[];
+}
+
+// Generic types for raw events from different providers
+export interface BaseRawEvent {
+ [key: string]: unknown;
+}
+
+// Error context for better error handling
+export interface ErrorContext {
+ batchNumber?: number;
+ batchSize?: number;
+ eventIndex?: number;
+ rawEvent?: BaseRawEvent;
+ provider?: string;
+}
+
+// Properties type for events - more specific than Record<string, any>
+export interface EventProperties {
+ [key: string]:
+ | string
+ | number
+ | boolean
+ | null
+ | undefined
+ | Record<string, unknown>;
+ __query?: Record<string, unknown>;
+ __title?: string;
+ __screen?: string;
+ __language?: string;
+}
+
+// Import job metadata for tracking import progress
+export interface ImportJobMetadata {
+ importId: string;
+ importStatus: 'pending' | 'processing' | 'processed' | 'failed';
+ importedAt: Date;
+}
+
+// Result of import staging operations
+export interface ImportStageResult {
+ importId: string;
+ totalEvents: number;
+ insertedEvents: number;
+}
diff --git a/packages/importer/tsconfig.json b/packages/importer/tsconfig.json
new file mode 100644
index 000000000..1d1748e91
--- /dev/null
+++ b/packages/importer/tsconfig.json
@@ -0,0 +1,9 @@
+{
+ "extends": "../../tooling/typescript/base.json",
+ "compilerOptions": {
+ "outDir": "./dist",
+ "rootDir": "./src"
+ },
+ "include": ["src/**/*"],
+ "exclude": ["node_modules", "dist"]
+}
diff --git a/packages/importer/vitest.config.ts b/packages/importer/vitest.config.ts
new file mode 100644
index 000000000..f87a2039f
--- /dev/null
+++ b/packages/importer/vitest.config.ts
@@ -0,0 +1,3 @@
+import { getSharedVitestConfig } from '../../vitest.shared';
+
+export default getSharedVitestConfig({ __dirname });
diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts
index 7511cd09f..9d07372a0 100644
--- a/packages/queue/src/queues.ts
+++ b/packages/queue/src/queues.ts
@@ -110,7 +110,6 @@ export const eventsGroupQueue = new GroupQueue<
>({
logger: queueLogger,
namespace: 'group_events',
- // @ts-expect-error - TODO: Fix this in groupmq
redis: getRedisGroupQueue(),
orderingMethod: 'in-memory',
orderingWindowMs,
@@ -166,6 +165,21 @@ export const notificationQueue = new Queue<NotificationQueuePayload>(
},
);
+export type ImportQueuePayload = {
+ type: 'import';
+ payload: {
+ importId: string;
+ };
+};
+
+export const importQueue = new Queue<ImportQueuePayload>('import', {
+ connection: getRedisQueue(),
+ defaultJobOptions: {
+ removeOnComplete: 10,
+ removeOnFail: 50,
+ },
+});
+
export function addTrialEndingSoonJob(organizationId: string, delay: number) {
return miscQueue.add(
'misc',
diff --git a/packages/redis/package.json b/packages/redis/package.json
index f958f1d73..742886ab4 100644
--- a/packages/redis/package.json
+++ b/packages/redis/package.json
@@ -8,7 +8,7 @@
},
"dependencies": {
"@openpanel/json": "workspace:*",
- "ioredis": "^5.7.0"
+ "ioredis": "5.8.2"
},
"devDependencies": {
"@openpanel/db": "workspace:*",
diff --git a/packages/trpc/package.json b/packages/trpc/package.json
index 630e74d96..6632ce43f 100644
--- a/packages/trpc/package.json
+++ b/packages/trpc/package.json
@@ -16,6 +16,7 @@
"@openpanel/payments": "workspace:^",
"@openpanel/redis": "workspace:*",
"@openpanel/validation": "workspace:*",
+ "@openpanel/queue": "workspace:*",
"@trpc-limiter/redis": "^0.0.2",
"@trpc/client": "^11.6.0",
"@trpc/server": "^11.6.0",
diff --git a/packages/trpc/src/root.ts b/packages/trpc/src/root.ts
index 2d7888372..d852c9c1c 100644
--- a/packages/trpc/src/root.ts
+++ b/packages/trpc/src/root.ts
@@ -4,6 +4,7 @@ import { chatRouter } from './routers/chat';
import { clientRouter } from './routers/client';
import { dashboardRouter } from './routers/dashboard';
import { eventRouter } from './routers/event';
+import { importRouter } from './routers/import';
import { integrationRouter } from './routers/integration';
import { notificationRouter } from './routers/notification';
import { onboardingRouter } from './routers/onboarding';
@@ -40,6 +41,7 @@ export const appRouter = createTRPCRouter({
reference: referenceRouter,
notification: notificationRouter,
integration: integrationRouter,
+ import: importRouter,
auth: authRouter,
subscription: subscriptionRouter,
overview: overviewRouter,
diff --git a/packages/trpc/src/routers/auth.ts b/packages/trpc/src/routers/auth.ts
index 34a9d19c3..ccfc7f7b3 100644
--- a/packages/trpc/src/routers/auth.ts
+++ b/packages/trpc/src/routers/auth.ts
@@ -12,7 +12,7 @@ import {
validateSessionToken,
verifyPasswordHash,
} from '@openpanel/auth';
-import { generateSecureId } from '@openpanel/common/server/id';
+import { generateSecureId } from '@openpanel/common/server';
import {
connectUserToOrganization,
db,
diff --git a/packages/trpc/src/routers/import.ts b/packages/trpc/src/routers/import.ts
new file mode 100644
index 000000000..d77effa13
--- /dev/null
+++ b/packages/trpc/src/routers/import.ts
@@ -0,0 +1,178 @@
+import { z } from 'zod';
+
+import { db } from '@openpanel/db';
+import { importQueue } from '@openpanel/queue';
+import { zCreateImport } from '@openpanel/validation';
+
+import { getProjectAccess } from '../access';
+import { TRPCAccessError } from '../errors';
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export const importRouter = createTRPCRouter({
+ list: protectedProcedure
+ .input(z.object({ projectId: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const access = await getProjectAccess({
+ projectId: input.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this project');
+ }
+
+ return db.import.findMany({
+ where: {
+ projectId: input.projectId,
+ },
+ orderBy: {
+ createdAt: 'desc',
+ },
+ });
+ }),
+
+ get: protectedProcedure
+ .input(z.object({ id: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const importRecord = await db.import.findUniqueOrThrow({
+ where: {
+ id: input.id,
+ },
+ include: {
+ project: true,
+ },
+ });
+
+ const access = await getProjectAccess({
+ projectId: importRecord.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this import');
+ }
+
+ return importRecord;
+ }),
+
+ create: protectedProcedure
+ .input(zCreateImport)
+ .mutation(async ({ input, ctx }) => {
+ const access = await getProjectAccess({
+ projectId: input.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access || (typeof access !== 'boolean' && access.level === 'read')) {
+ throw TRPCAccessError(
+ 'You do not have permission to create imports for this project',
+ );
+ }
+
+ // Create import record
+ const importRecord = await db.import.create({
+ data: {
+ projectId: input.projectId,
+ config: input.config,
+ status: 'pending',
+ },
+ });
+
+ // Add job to queue
+ const job = await importQueue.add('import', {
+ type: 'import',
+ payload: {
+ importId: importRecord.id,
+ },
+ });
+
+ // Update import record with job ID
+ await db.import.update({
+ where: { id: importRecord.id },
+ data: { jobId: job.id },
+ });
+
+ return {
+ ...importRecord,
+ jobId: job.id,
+ };
+ }),
+
+ delete: protectedProcedure
+ .input(z.object({ id: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const importRecord = await db.import.findUniqueOrThrow({
+ where: {
+ id: input.id,
+ },
+ });
+
+ const access = await getProjectAccess({
+ projectId: importRecord.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access || (typeof access !== 'boolean' && access.level === 'read')) {
+ throw TRPCAccessError(
+ 'You do not have permission to delete imports for this project',
+ );
+ }
+
+ if (importRecord.jobId) {
+ const job = await importQueue.getJob(importRecord.jobId);
+ if (job) {
+ await job.remove();
+ }
+ }
+
+ return db.import.delete({
+ where: {
+ id: input.id,
+ },
+ });
+ }),
+
+ retry: protectedProcedure
+ .input(z.object({ id: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const importRecord = await db.import.findUniqueOrThrow({
+ where: {
+ id: input.id,
+ },
+ });
+
+ const access = await getProjectAccess({
+ projectId: importRecord.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access || (typeof access !== 'boolean' && access.level === 'read')) {
+ throw TRPCAccessError(
+ 'You do not have permission to retry imports for this project',
+ );
+ }
+
+ // Only allow retry for failed imports
+ if (importRecord.status !== 'failed') {
+ throw new Error('Only failed imports can be retried');
+ }
+
+ // Add new job to queue
+ const job = await importQueue.add('import', {
+ type: 'import',
+ payload: {
+ importId: importRecord.id,
+ },
+ });
+
+ // Update import record
+ return db.import.update({
+ where: { id: importRecord.id },
+ data: {
+ jobId: job.id,
+ status: 'pending',
+ errorMessage: null,
+ },
+ });
+ }),
+});
diff --git a/packages/trpc/src/routers/organization.ts b/packages/trpc/src/routers/organization.ts
index 21bf029a9..40ff4ea1d 100644
--- a/packages/trpc/src/routers/organization.ts
+++ b/packages/trpc/src/routers/organization.ts
@@ -11,7 +11,7 @@ import {
} from '@openpanel/db';
import { zEditOrganization, zInviteUser } from '@openpanel/validation';
-import { generateSecureId } from '@openpanel/common/server/id';
+import { generateSecureId } from '@openpanel/common/server';
import { sendEmail } from '@openpanel/email';
import { addDays } from 'date-fns';
import { getOrganizationAccess } from '../access';
diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts
index 44ae45dee..0b1cefe60 100644
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -435,3 +435,54 @@ export const zEditOrganization = z.object({
name: z.string().min(2),
timezone: z.string().min(1),
});
+
+const zProjectMapper = z.object({
+ from: z.string().min(1),
+ to: z.string().min(1),
+});
+
+const createFileImportConfig = <T extends string>(provider: T) =>
+ z.object({
+ provider: z.literal(provider),
+ type: z.literal('file'),
+ fileUrl: z.string().url(),
+ });
+
+// Import configs
+export const zUmamiImportConfig = createFileImportConfig('umami').extend({
+ projectMapper: z.array(zProjectMapper),
+});
+
+export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;
+
+export const zPlausibleImportConfig = createFileImportConfig('plausible');
+export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;
+
+export const zMixpanelImportConfig = z.object({
+ provider: z.literal('mixpanel'),
+ type: z.literal('api'),
+ serviceAccount: z.string().min(1),
+ serviceSecret: z.string().min(1),
+ projectId: z.string().min(1),
+ from: z.string().min(1),
+ to: z.string().min(1),
+ mapScreenViewProperty: z.string().optional(),
+});
+export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;
+
+export type IImportConfig =
+ | IUmamiImportConfig
+ | IPlausibleImportConfig
+ | IMixpanelImportConfig;
+
+export const zCreateImport = z.object({
+ projectId: z.string().min(1),
+ provider: z.enum(['umami', 'plausible', 'mixpanel']),
+ config: z.union([
+ zUmamiImportConfig,
+ zPlausibleImportConfig,
+ zMixpanelImportConfig,
+ ]),
+});
+
+export type ICreateImport = z.infer<typeof zCreateImport>;
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index f15819bd0..6b6e6377d 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -226,7 +226,7 @@ importers:
specifier: ^4.1.0
version: 4.1.0
tsdown:
- specifier: ^0.14.2
+ specifier: 0.14.2
version: 0.14.2([email protected])
typescript:
specifier: 'catalog:'
@@ -373,6 +373,9 @@ importers:
'@openpanel/constants':
specifier: workspace:^
version: link:../../packages/constants
+ '@openpanel/importer':
+ specifier: workspace:^
+ version: link:../../packages/importer
'@openpanel/integrations':
specifier: workspace:^
version: link:../../packages/integrations
@@ -815,6 +818,9 @@ importers:
'@openpanel/email':
specifier: workspace:*
version: link:../../packages/email
+ '@openpanel/importer':
+ specifier: workspace:*
+ version: link:../../packages/importer
'@openpanel/integrations':
specifier: workspace:^
version: link:../../packages/integrations
@@ -874,7 +880,7 @@ importers:
specifier: ^9.0.8
version: 9.0.8
tsdown:
- specifier: ^0.14.2
+ specifier: 0.14.2
version: 0.14.2([email protected])
typescript:
specifier: 'catalog:'
@@ -923,58 +929,6 @@ importers:
specifier: 'catalog:'
version: 5.9.3
- packages/cli:
- dependencies:
- '@openpanel/common':
- specifier: workspace:*
- version: link:../common
- arg:
- specifier: ^5.0.2
- version: 5.0.2
- glob:
- specifier: ^10.4.3
- version: 10.4.5
- inquirer:
- specifier: ^9.3.5
- version: 9.3.6
- p-limit:
- specifier: ^6.1.0
- version: 6.1.0
- progress:
- specifier: ^2.0.3
- version: 2.0.3
- ramda:
- specifier: ^0.29.1
- version: 0.29.1
- zod:
- specifier: 'catalog:'
- version: 3.24.2
- devDependencies:
- '@openpanel/db':
- specifier: workspace:^
- version: link:../db
- '@openpanel/sdk':
- specifier: workspace:*
- version: link:../sdks/sdk
- '@openpanel/tsconfig':
- specifier: workspace:*
- version: link:../../tooling/typescript
- '@types/node':
- specifier: 'catalog:'
- version: 24.7.1
- '@types/progress':
- specifier: ^2.0.7
- version: 2.0.7
- '@types/ramda':
- specifier: ^0.30.1
- version: 0.30.1
- tsup:
- specifier: ^7.2.0
- typescript:
- specifier: 'catalog:'
- version: 5.9.3
-
packages/common:
dependencies:
'@openpanel/constants':
@@ -1048,8 +1002,8 @@ importers:
packages/db:
dependencies:
'@clickhouse/client':
- specifier: ^1.2.0
- version: 1.2.0
+ specifier: ^1.12.1
+ version: 1.12.1
'@openpanel/common':
specifier: workspace:*
version: link:../common
@@ -1186,6 +1140,55 @@ importers:
specifier: 'catalog:'
version: 5.9.3
+ packages/importer:
+ dependencies:
+ '@openpanel/common':
+ specifier: workspace:*
+ version: link:../common
+ '@openpanel/db':
+ specifier: workspace:*
+ version: link:../db
+ '@openpanel/queue':
+ specifier: workspace:*
+ version: link:../queue
+ '@openpanel/validation':
+ specifier: workspace:*
+ version: link:../validation
+ csv-parse:
+ specifier: ^6.1.0
+ version: 6.1.0
+ ramda:
+ specifier: ^0.29.1
+ version: 0.29.1
+ uuid:
+ specifier: ^9.0.1
+ version: 9.0.1
+ zod:
+ specifier: 'catalog:'
+ version: 3.24.2
+ devDependencies:
+ '@openpanel/logger':
+ specifier: workspace:*
+ version: link:../logger
+ '@types/node':
+ specifier: ^20.0.0
+ version: 20.14.8
+ '@types/ramda':
+ specifier: ^0.31.1
+ version: 0.31.1
+ '@types/uuid':
+ specifier: ^9.0.7
+ version: 9.0.8
+ bullmq:
+ specifier: ^5.8.7
+ version: 5.8.7
+ typescript:
+ specifier: ^5.0.0
+ version: 5.9.3
+ vitest:
+ specifier: ^1.0.0
+
packages/integrations:
dependencies:
'@slack/bolt':
@@ -1320,8 +1323,8 @@ importers:
specifier: workspace:*
version: link:../json
ioredis:
- specifier: ^5.7.0
- version: 5.7.0
+ specifier: 5.8.2
+ version: 5.8.2
devDependencies:
'@openpanel/db':
specifier: workspace:*
@@ -1509,6 +1512,9 @@ importers:
'@openpanel/payments':
specifier: workspace:^
version: link:../payments
+ '@openpanel/queue':
+ specifier: workspace:*
+ version: link:../queue
'@openpanel/redis':
specifier: workspace:*
version: link:../redis
@@ -2710,9 +2716,16 @@ packages:
'@capsizecss/[email protected]':
resolution: {integrity: sha512-GrSU71meACqcmIUxPYOJvGKF0yryjN/L1aCuE9DViCTJI7bfkjgYDPD1zbNDcINJwSSP6UaBZY9GAbYDO7re0Q==}
+ '@clickhouse/[email protected]':
+ resolution: {integrity: sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==}
+
'@clickhouse/[email protected]':
resolution: {integrity: sha512-VfA/C/tVJ2eNe72CaQ7eXmai+yqFEvZjQZiNtvJoOMLP+Vtb6DzqH9nfkgsiHHMhUhhclvt2mFh6+euk1Ea5wA==}
+ '@clickhouse/[email protected]':
+ resolution: {integrity: sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==}
+ engines: {node: '>=16'}
+
'@clickhouse/[email protected]':
resolution: {integrity: sha512-zMp2EhMfp1IrFKr/NjDwNiLsf7nq68nW8lGKszwFe7Iglc6Z5PY9ZA9Hd0XqAk75Q1NmFrkGCP1r3JCM1Nm1Bw==}
engines: {node: '>=16'}
@@ -2866,6 +2879,12 @@ packages:
cpu: [ppc64]
os: [aix]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==}
+ engines: {node: '>=12'}
+ cpu: [ppc64]
+ os: [aix]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-WtKdFM7ls47zkKHFVzMz8opM7LkcsIp9amDUBIAWirg70RM71WRSjdILPsY5Uv1D42ZpUfaPILDlfactHgsRkw==}
engines: {node: '>=18'}
@@ -2908,6 +2927,12 @@ packages:
cpu: [arm64]
os: [android]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==}
+ engines: {node: '>=12'}
+ cpu: [arm64]
+ os: [android]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-Vsm497xFM7tTIPYK9bNTYJyF/lsP590Qc1WxJdlB6ljCbdZKU9SY8i7+Iin4kyhV/KV5J2rOKsBQbB77Ab7L/w==}
engines: {node: '>=18'}
@@ -2950,6 +2975,12 @@ packages:
cpu: [arm]
os: [android]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==}
+ engines: {node: '>=12'}
+ cpu: [arm]
+ os: [android]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-arAtTPo76fJ/ICkXWetLCc9EwEHKaeya4vMrReVlEIUCAUncH7M4bhMQ+M9Vf+FFOZJdTNMXNBrWwW+OXWpSew==}
engines: {node: '>=18'}
@@ -2992,6 +3023,12 @@ packages:
cpu: [x64]
os: [android]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==}
+ engines: {node: '>=12'}
+ cpu: [x64]
+ os: [android]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-t8GrvnFkiIY7pa7mMgJd7p8p8qqYIz1NYiAoKc75Zyv73L3DZW++oYMSHPRarcotTKuSs6m3hTOa5CKHaS02TQ==}
engines: {node: '>=18'}
@@ -3034,6 +3071,12 @@ packages:
cpu: [arm64]
os: [darwin]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==}
+ engines: {node: '>=12'}
+ cpu: [arm64]
+ os: [darwin]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw==}
engines: {node: '>=18'}
@@ -3076,6 +3119,12 @@ packages:
cpu: [x64]
os: [darwin]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==}
+ engines: {node: '>=12'}
+ cpu: [x64]
+ os: [darwin]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-rgtz6flkVkh58od4PwTRqxbKH9cOjaXCMZgWD905JOzjFKW+7EiUObfd/Kav+A6Gyud6WZk9w+xu6QLytdi2OA==}
engines: {node: '>=18'}
@@ -3118,6 +3167,12 @@ packages:
cpu: [arm64]
os: [freebsd]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==}
+ engines: {node: '>=12'}
+ cpu: [arm64]
+ os: [freebsd]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-6Mtdq5nHggwfDNLAHkPlyLBpE5L6hwsuXZX8XNmHno9JuL2+bg2BX5tRkwjyfn6sKbxZTq68suOjgWqCicvPXA==}
engines: {node: '>=18'}
@@ -3160,6 +3215,12 @@ packages:
cpu: [x64]
os: [freebsd]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==}
+ engines: {node: '>=12'}
+ cpu: [x64]
+ os: [freebsd]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-D3H+xh3/zphoX8ck4S2RxKR6gHlHDXXzOf6f/9dbFt/NRBDIE33+cVa49Kil4WUjxMGW0ZIYBYtaGCa2+OsQwQ==}
engines: {node: '>=18'}
@@ -3202,6 +3263,12 @@ packages:
cpu: [arm64]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==}
+ engines: {node: '>=12'}
+ cpu: [arm64]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-TDijPXTOeE3eaMkRYpcy3LarIg13dS9wWHRdwYRnzlwlA370rNdZqbcp0WTyyV/k2zSxfko52+C7jU5F9Tfj1g==}
engines: {node: '>=18'}
@@ -3244,6 +3311,12 @@ packages:
cpu: [arm]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==}
+ engines: {node: '>=12'}
+ cpu: [arm]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-gJKIi2IjRo5G6Glxb8d3DzYXlxdEj2NlkixPsqePSZMhLudqPhtZ4BUrpIuTjJYXxvF9njql+vRjB2oaC9XpBw==}
engines: {node: '>=18'}
@@ -3286,6 +3359,12 @@ packages:
cpu: [ia32]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==}
+ engines: {node: '>=12'}
+ cpu: [ia32]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-K40ip1LAcA0byL05TbCQ4yJ4swvnbzHscRmUilrmP9Am7//0UjPreh4lpYzvThT2Quw66MhjG//20mrufm40mA==}
engines: {node: '>=18'}
@@ -3328,6 +3407,12 @@ packages:
cpu: [loong64]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==}
+ engines: {node: '>=12'}
+ cpu: [loong64]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-0mswrYP/9ai+CU0BzBfPMZ8RVm3RGAN/lmOMgW4aFUSOQBjA31UP8Mr6DDhWSuMwj7jaWOT0p0WoZ6jeHhrD7g==}
engines: {node: '>=18'}
@@ -3370,6 +3455,12 @@ packages:
cpu: [mips64el]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==}
+ engines: {node: '>=12'}
+ cpu: [mips64el]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-hIKvXm0/3w/5+RDtCJeXqMZGkI2s4oMUGj3/jM0QzhgIASWrGO5/RlzAzm5nNh/awHE0A19h/CvHQe6FaBNrRA==}
engines: {node: '>=18'}
@@ -3412,6 +3503,12 @@ packages:
cpu: [ppc64]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==}
+ engines: {node: '>=12'}
+ cpu: [ppc64]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-HcZh5BNq0aC52UoocJxaKORfFODWXZxtBaaZNuN3PUX3MoDsChsZqopzi5UupRhPHSEHotoiptqikjN/B77mYQ==}
engines: {node: '>=18'}
@@ -3454,6 +3551,12 @@ packages:
cpu: [riscv64]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==}
+ engines: {node: '>=12'}
+ cpu: [riscv64]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-bEh7dMn/h3QxeR2KTy1DUszQjUrIHPZKyO6aN1X4BCnhfYhuQqedHaa5MxSQA/06j3GpiIlFGSsy1c7Gf9padw==}
engines: {node: '>=18'}
@@ -3496,6 +3599,12 @@ packages:
cpu: [s390x]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==}
+ engines: {node: '>=12'}
+ cpu: [s390x]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-ZcQ6+qRkw1UcZGPyrCiHHkmBaj9SiCD8Oqd556HldP+QlpUIe2Wgn3ehQGVoPOvZvtHm8HPx+bH20c9pvbkX3g==}
engines: {node: '>=18'}
@@ -3538,6 +3647,12 @@ packages:
cpu: [x64]
os: [linux]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==}
+ engines: {node: '>=12'}
+ cpu: [x64]
+ os: [linux]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-vbutsFqQ+foy3wSSbmjBXXIJ6PL3scghJoM8zCL142cGaZKAdCZHyf+Bpu/MmX9zT9Q0zFBVKb36Ma5Fzfa8xA==}
engines: {node: '>=18'}
@@ -3604,6 +3719,12 @@ packages:
cpu: [x64]
os: [netbsd]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==}
+ engines: {node: '>=12'}
+ cpu: [x64]
+ os: [netbsd]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-hjQ0R/ulkO8fCYFsG0FZoH+pWgTTDreqpqY7UnQntnaKv95uP5iW3+dChxnx7C3trQQU40S+OgWhUVwCjVFLvg==}
engines: {node: '>=18'}
@@ -3676,6 +3797,12 @@ packages:
cpu: [x64]
os: [openbsd]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==}
+ engines: {node: '>=12'}
+ cpu: [x64]
+ os: [openbsd]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-4ir0aY1NGUhIC1hdoCzr1+5b43mw99uNwVzhIq1OY3QcEwPDO3B7WNXBzaKY5Nsf1+N11i1eOfFcq+D/gOS15Q==}
engines: {node: '>=18'}
@@ -3730,6 +3857,12 @@ packages:
cpu: [x64]
os: [sunos]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==}
+ engines: {node: '>=12'}
+ cpu: [x64]
+ os: [sunos]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-jVzdzsbM5xrotH+W5f1s+JtUy1UWgjU0Cf4wMvffTB8m6wP5/kx0KiaLHlbJO+dMgtxKV8RQ/JvtlFcdZ1zCPA==}
engines: {node: '>=18'}
@@ -3772,6 +3905,12 @@ packages:
cpu: [arm64]
os: [win32]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==}
+ engines: {node: '>=12'}
+ cpu: [arm64]
+ os: [win32]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-iKc8GAslzRpBytO2/aN3d2yb2z8XTVfNV0PjGlCxKo5SgWmNXx82I/Q3aG1tFfS+A2igVCY97TJ8tnYwpUWLCA==}
engines: {node: '>=18'}
@@ -3814,6 +3953,12 @@ packages:
cpu: [ia32]
os: [win32]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==}
+ engines: {node: '>=12'}
+ cpu: [ia32]
+ os: [win32]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-vQW36KZolfIudCcTnaTpmLQ24Ha1RjygBo39/aLkM2kmjkWmZGEJ5Gn9l5/7tzXA42QGIoWbICfg6KLLkIw6yw==}
engines: {node: '>=18'}
@@ -3856,6 +4001,12 @@ packages:
cpu: [x64]
os: [win32]
+ '@esbuild/[email protected]':
+ resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==}
+ engines: {node: '>=12'}
+ cpu: [x64]
+ os: [win32]
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-7IAFPrjSQIJrGsK6flwg7NFmwBoSTyF3rl7If0hNUFQU4ilTsEPL6GuMuU9BfIWVVGuRnuIidkSMC+c0Otu8IA==}
engines: {node: '>=18'}
@@ -4178,9 +4329,6 @@ packages:
resolution: {integrity: sha512-R7Gsg6elpuqdn55fBH2y9oYzrU/yKrSmIsDX4ROT51vohrECFzTf2zw9BfUbOW8xjfmM2QbVoVYdTwhrtEKWSQ==}
engines: {node: '>=18'}
- '@ioredis/[email protected]':
- resolution: {integrity: sha512-M/T6Zewn7sDaBQEqIZ8Rb+i9y8qfGmq+5SDFSf9sA2lUZTmdDLVdOiQaeDp+Q4wElZ9HG1GAX5KhDaidp6LQsQ==}
-
'@ioredis/[email protected]':
resolution: {integrity: sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==}
@@ -8570,9 +8718,6 @@ packages:
resolution: {integrity: sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA==}
- '@types/[email protected]':
- resolution: {integrity: sha512-m5ObIqwsUp6BZzyiy4RdZpzWGub9bqLJMvZDD0QMXhxjqMHMENlj+SqF5QxoUwaQNFe+8kz8XM8ZQhqkQPTgMQ==}
-
resolution: {integrity: sha512-CmyhGZanP88uuC5GpWU9q+fI61j2SkhO3UGMUdfYRE6Bcy0ccyzn1Rqj9YAB/ZY4kOXmNf0ocah5GtphmLMP6Q==}
@@ -8588,9 +8733,6 @@ packages:
resolution: {integrity: sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==}
- '@types/[email protected]':
- resolution: {integrity: sha512-iadjw02vte8qWx7U0YM++EybBha2CQLPGu9iJ97whVgJUT5Zq9MjAPYUnbfRI2Kpehimf1QjFJYxD0t8nqzu5w==}
-
resolution: {integrity: sha512-wA0UT0HeT2fGHzIFV9kWpYz5mdoyLxKrTgMdZQM++5h6pYAFH73HXcQhefg24nD1yivUFEn5KU+EF4b+CXJ4Wg==}
@@ -8603,15 +8745,15 @@ packages:
resolution: {integrity: sha512-0BzWVKtSEtignlk+XBuK88Il5wzQwbRVfEkzE8iKm02NYHMGQ/9ffB05M+zXhTCqo50DOIAT9pNSJsjFMMG4rQ==}
- '@types/[email protected]':
- resolution: {integrity: sha512-aoyF/ADPL6N+/NXXfhPWF+Qj6w1Cql59m9wX0Gi15uyF+bpzXeLd63HPdiTDE2bmLXfNcVufsDPKmbfOrOzTBA==}
-
resolution: {integrity: sha512-PyzHvjCalm2BRYjAU6nIB3TprYwMNOUY/7P/N8bSzp9W/yM2YrtGtAnnVtaCNSeOZ8DzKyFDvaqQs7LnWwwmBA==}
resolution: {integrity: sha512-1lWWZ/2YiNttGcIUxQwnvMuh55GIEbn/zlpzzEojAsbxquI/TXQZCRaXsfxG1CHjlqGoqxWePkvaM/5qYHNuvQ==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-Vt6sFXnuRpzaEj+yeutA0q3bcAsK7wdPuASIzR9LXqL4gJPyFw8im9qchlbp4ltuf3kDEIRmPJTD/Fkg60dn7g==}
+
resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==}
@@ -8734,6 +8876,9 @@ packages:
peerDependencies:
vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0
+ '@vitest/[email protected]':
+ resolution: {integrity: sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog==}
+
'@vitest/[email protected]':
resolution: {integrity: sha512-7FTQQuuLKmN1Ig/h+h/GO+44Q1IlglPlR2es4ab7Yvfx+Uk5xsv+Ykk+MEt/M2Yn/xGmzaLKxGw2lgy2bwuYqg==}
@@ -8751,15 +8896,27 @@ packages:
'@vitest/[email protected]':
resolution: {integrity: sha512-i6FDiBeJUGLDKADw2Gb01UtUNb12yyXAqC/mmRWuYl+m/U9GS7s8us5ONmGkGpUUo7/iAYzI2ePVfOZTYvUifA==}
+ '@vitest/[email protected]':
+ resolution: {integrity: sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA==}
+
'@vitest/[email protected]':
resolution: {integrity: sha512-Tae+ogtlNfFei5DggOsSUvkIaSuVywujMj6HzR97AHK6XK8i3BuVyIifWAm/sE3a15lF5RH9yQIrbXYuo0IFyA==}
+ '@vitest/[email protected]':
+ resolution: {integrity: sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ==}
+
'@vitest/[email protected]':
resolution: {integrity: sha512-XVa5OPNTYUsyqG9skuUkFzAeFnEzDp8hQu7kZ0N25B1+6KjGm4hWLtURyBbsIAOekfWQ7Wuz/N/XXzgYO3deWQ==}
+ '@vitest/[email protected]':
+ resolution: {integrity: sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw==}
+
'@vitest/[email protected]':
resolution: {integrity: sha512-x6w+ctOEmEXdWaa6TO4ilb7l9DxPR5bwEb6hILKuxfU1NqWT2mpJD9NJN7t3OTfxmVlOMrvtoFJGdgyzZ605lQ==}
+ '@vitest/[email protected]':
+ resolution: {integrity: sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g==}
+
'@vitest/[email protected]':
resolution: {integrity: sha512-2Ltrpht4OmHO9+c/nmHtF09HWiyWdworqnHIwjfvDyWjuwKbdkcS9AnhsDn+8E2RM4x++foD1/tNuLPVvWG1Rg==}
@@ -8920,10 +9077,6 @@ packages:
resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==}
engines: {node: '>=12'}
- resolution: {integrity: sha512-BGcItUBWSMRgOCe+SVZJ+S7yTRG0eGt9cXAHev72yuGcY23hnLA7Bky5L/xLyPINoSN95geovfBkqoTlNZYa7w==}
- engines: {node: '>=14'}
-
resolution: {integrity: sha512-HqZ5rWlFjGiV0tDm3UxxgNRqsOTniqoKZu0pIAfh7TZQMGuZK+hH0drySty0si0QXj1ieop4+SkSfPZBPPkHig==}
engines: {node: '>=14'}
@@ -9009,6 +9162,9 @@ packages:
resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==}
+ resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==}
+
resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==}
engines: {node: '>=12'}
@@ -9399,6 +9555,10 @@ packages:
resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==}
+ resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==}
+ engines: {node: '>=4'}
+
resolution: {integrity: sha512-mCuXncKXk5iCLhfhwTc0izo0gtEmpz5CtG2y8GiOINBlMVS6v8TMRc5TaLWKS6692m9+dVVfzgeVxR5UxWHTYw==}
engines: {node: '>=12'}
@@ -9442,6 +9602,9 @@ packages:
resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==}
+ resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==}
+
resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==}
engines: {node: '>= 16'}
@@ -9836,6 +9999,9 @@ packages:
resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}
+ resolution: {integrity: sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==}
+
resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==}
@@ -10123,6 +10289,10 @@ packages:
resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==}
+ resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==}
+ engines: {node: '>=6'}
+
resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==}
engines: {node: '>=6'}
@@ -10245,6 +10415,10 @@ packages:
resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==}
+ resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==}
+ engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
+
resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==}
engines: {node: '>=0.3.1'}
@@ -10525,6 +10699,11 @@ packages:
engines: {node: '>=12'}
hasBin: true
+ resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==}
+ engines: {node: '>=12'}
+ hasBin: true
+
resolution: {integrity: sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ==}
engines: {node: '>=18'}
@@ -11122,6 +11301,9 @@ packages:
resolution: {integrity: sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==}
+
resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
engines: {node: '>= 0.4'}
@@ -11594,10 +11776,6 @@ packages:
resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==}
- resolution: {integrity: sha512-NUcA93i1lukyXU+riqEyPtSEkyFq8tX90uL659J+qpCZ3rEdViB/APC58oAhIh3+bJln2hzdlZbBZsGNrlsR8g==}
- engines: {node: '>=12.22.0'}
-
resolution: {integrity: sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==}
engines: {node: '>=12.22.0'}
@@ -12269,6 +12447,10 @@ packages:
resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+ resolution: {integrity: sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ==}
+ engines: {node: '>=14'}
+
resolution: {integrity: sha512-arhlxbFRmoQHl33a0Zkle/YWlmNwoyt6QNZEIJcqNbdrsix5Lvc4HyyI3EnwxTYlZYc32EbYrQ8SzEZ7dqgg9A==}
engines: {node: '>=14'}
@@ -12378,6 +12560,9 @@ packages:
resolution: {integrity: sha512-uvhvYPC8kGPjXT3MyKMrL3JitEAmDMp30lVkuq/590Mw9ok6pWcFCwXJveo0t5uqYw1UREQHofD+jVpdjBv8wg==}
+ resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==}
+
resolution: {integrity: sha512-kkIp7XSkP78ZxJEsSxW3712C6teJVoeHHwgo9zJ380de7IYyJ2ISlxojcH2pC5OFLewESmnRi/+XCDIEEVyoug==}
@@ -13329,8 +13514,8 @@ packages:
resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==}
engines: {node: '>=10'}
- resolution: {integrity: sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==}
+ resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==}
engines: {node: '>=18'}
@@ -13498,6 +13683,9 @@ packages:
resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==}
+ resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==}
+
resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==}
engines: {node: '>= 14.16'}
@@ -15107,6 +15295,9 @@ packages:
resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==}
engines: {node: '>=0.10.0'}
+ resolution: {integrity: sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q==}
+
resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==}
@@ -15374,6 +15565,10 @@ packages:
resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==}
engines: {node: '>=12.0.0'}
+ resolution: {integrity: sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==}
+ engines: {node: '>=14.0.0'}
+
resolution: {integrity: sha512-al6n+QEANGFOMf/dmUMsuS5/r9B06uwlyNjZZql/zv8J7ybHCgoihBNORZCY2mzUuAnomQa2JdhyHKzZxPCrFA==}
engines: {node: ^18.0.0 || >=20.0.0}
@@ -15382,6 +15577,10 @@ packages:
resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==}
engines: {node: '>=14.0.0'}
+ resolution: {integrity: sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==}
+ engines: {node: '>=14.0.0'}
+
resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==}
engines: {node: '>=14.0.0'}
@@ -15528,6 +15727,10 @@ packages:
resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==}
engines: {node: '>=4'}
+ resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==}
+ engines: {node: '>=4'}
+
resolution: {integrity: sha512-eaBzG6MxNzEn9kiwvtre90cXaNLkmadMWa1zQMs3XORCXNbsH/OewwbxC5ia9dCxIxnTAsSxXJaa/p5y8DlvJg==}
engines: {node: '>=10'}
@@ -15636,9 +15839,6 @@ packages:
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
- resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
-
resolution: {integrity: sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==}
@@ -16024,6 +16224,11 @@ packages:
resolution: {integrity: sha512-+pZIP+U3pEJdDCeFmsXwHzV7vNHQC/eIbHklfe2ZCZqayYRH7lQbHcVgsJ0XOOv27hWs4jH4MONgXxHMObTMSA==}
+ resolution: {integrity: sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA==}
+ engines: {node: ^18.0.0 || >=20.0.0}
+ hasBin: true
+
resolution: {integrity: sha512-uHV4plJ2IxCl4u1up1FQRrqclylKAogbtBfOTwcuJ28xFi+89PZ57BRh+naIRvH70HPwxy5QHYzg1OrEaC7AbA==}
engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}
@@ -16037,27 +16242,22 @@ packages:
vite:
optional: true
- resolution: {integrity: sha512-5nXH+QsELbFKhsEfWLkHrvgRpTdGJzqOZ+utSdmPTvwHmvU6ITTm3xx+mRusihkcI8GeC7lCDyn3kDtiki9scw==}
- engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}
+ resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==}
+ engines: {node: ^18.0.0 || >=20.0.0}
hasBin: true
peerDependencies:
- '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0
- jiti: '>=1.21.0'
+ '@types/node': ^18.0.0 || >=20.0.0
less: '*'
lightningcss: ^1.21.0
sass: '*'
sass-embedded: '*'
stylus: '*'
sugarss: '*'
- terser: ^5.16.0
- tsx: ^4.8.1
- yaml: ^2.4.2
+ terser: ^5.4.0
peerDependenciesMeta:
'@types/node':
optional: true
- jiti:
- optional: true
less:
optional: true
lightningcss:
@@ -16072,13 +16272,9 @@ packages:
optional: true
terser:
optional: true
- tsx:
- optional: true
- yaml:
- optional: true
- resolution: {integrity: sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==}
+ resolution: {integrity: sha512-5nXH+QsELbFKhsEfWLkHrvgRpTdGJzqOZ+utSdmPTvwHmvU6ITTm3xx+mRusihkcI8GeC7lCDyn3kDtiki9scw==}
engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}
hasBin: true
peerDependencies:
@@ -16117,19 +16313,59 @@ packages:
yaml:
optional: true
- resolution: {integrity: sha512-4nVGliEpxmhCL8DslSAUdxlB6+SMrhB0a1v5ijlh1xB1nEPuy1mxaHxysVucLHuWryAxLWg6a5ei+U4TLn/rFg==}
- engines: {node: ^20.19.0 || >=22.12.0}
+ resolution: {integrity: sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==}
+ engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}
hasBin: true
peerDependencies:
- '@types/node': ^20.19.0 || >=22.12.0
+ '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0
jiti: '>=1.21.0'
- less: ^4.0.0
+ less: '*'
lightningcss: ^1.21.0
- sass: ^1.70.0
- sass-embedded: ^1.70.0
- stylus: '>=0.54.8'
- sugarss: ^5.0.0
+ sass: '*'
+ sass-embedded: '*'
+ stylus: '*'
+ sugarss: '*'
+ terser: ^5.16.0
+ tsx: ^4.8.1
+ yaml: ^2.4.2
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+ jiti:
+ optional: true
+ less:
+ optional: true
+ lightningcss:
+ optional: true
+ sass:
+ optional: true
+ sass-embedded:
+ optional: true
+ stylus:
+ optional: true
+ sugarss:
+ optional: true
+ terser:
+ optional: true
+ tsx:
+ optional: true
+ yaml:
+ optional: true
+
+ resolution: {integrity: sha512-4nVGliEpxmhCL8DslSAUdxlB6+SMrhB0a1v5ijlh1xB1nEPuy1mxaHxysVucLHuWryAxLWg6a5ei+U4TLn/rFg==}
+ engines: {node: ^20.19.0 || >=22.12.0}
+ hasBin: true
+ peerDependencies:
+ '@types/node': ^20.19.0 || >=22.12.0
+ jiti: '>=1.21.0'
+ less: ^4.0.0
+ lightningcss: ^1.21.0
+ sass: ^1.70.0
+ sass-embedded: ^1.70.0
+ stylus: '>=0.54.8'
+ sugarss: ^5.0.0
terser: ^5.16.0
tsx: ^4.8.1
yaml: ^2.4.2
@@ -16173,6 +16409,31 @@ packages:
vite:
optional: true
+ resolution: {integrity: sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag==}
+ engines: {node: ^18.0.0 || >=20.0.0}
+ hasBin: true
+ peerDependencies:
+ '@edge-runtime/vm': '*'
+ '@types/node': ^18.0.0 || >=20.0.0
+ '@vitest/browser': 1.6.1
+ '@vitest/ui': 1.6.1
+ happy-dom: '*'
+ jsdom: '*'
+ peerDependenciesMeta:
+ '@edge-runtime/vm':
+ optional: true
+ '@types/node':
+ optional: true
+ '@vitest/browser':
+ optional: true
+ '@vitest/ui':
+ optional: true
+ happy-dom:
+ optional: true
+ jsdom:
+ optional: true
+
resolution: {integrity: sha512-188iM4hAHQ0km23TN/adso1q5hhwKqUpv+Sd6p5sOuh6FhQnRNW3IsiIpvxqahtBabsJ2SLZgmGSpcYK4wQYJw==}
engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}
@@ -17931,8 +18192,14 @@ snapshots:
transitivePeerDependencies:
- encoding
+ '@clickhouse/[email protected]': {}
+
'@clickhouse/[email protected]': {}
+ '@clickhouse/[email protected]':
+ dependencies:
+ '@clickhouse/client-common': 1.12.1
+
'@clickhouse/[email protected]':
dependencies:
'@clickhouse/client-common': 1.2.0
@@ -18084,6 +18351,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18105,6 +18375,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18126,6 +18399,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18147,6 +18423,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18168,6 +18447,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18189,6 +18471,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18210,6 +18495,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18231,6 +18519,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18252,6 +18543,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18273,6 +18567,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18294,6 +18591,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18315,6 +18615,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18336,6 +18639,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18357,6 +18663,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18378,6 +18687,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18399,6 +18711,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18420,6 +18735,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18453,6 +18771,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18489,6 +18810,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18516,6 +18840,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18537,6 +18864,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18558,6 +18888,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -18579,6 +18912,9 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ '@esbuild/[email protected]':
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -19150,8 +19486,6 @@ snapshots:
'@inquirer/[email protected]': {}
- '@ioredis/[email protected]': {}
-
'@ioredis/[email protected]': {}
'@isaacs/[email protected]':
@@ -19177,14 +19511,14 @@ snapshots:
dependencies:
'@jest/fake-timers': 29.7.0
'@jest/types': 29.6.3
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
jest-mock: 29.7.0
dependencies:
'@jest/types': 29.6.3
'@sinonjs/fake-timers': 10.3.0
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
jest-message-util: 29.7.0
jest-mock: 29.7.0
jest-util: 29.7.0
@@ -19197,7 +19531,7 @@ snapshots:
dependencies:
'@types/istanbul-lib-coverage': 2.0.6
'@types/istanbul-reports': 3.0.4
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
'@types/yargs': 15.0.19
chalk: 4.1.2
@@ -19206,13 +19540,13 @@ snapshots:
'@jest/schemas': 29.6.3
'@types/istanbul-lib-coverage': 2.0.6
'@types/istanbul-reports': 3.0.4
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
'@types/yargs': 17.0.32
chalk: 4.1.2
'@jridgewell/[email protected]':
dependencies:
- '@jridgewell/sourcemap-codec': 1.5.0
+ '@jridgewell/sourcemap-codec': 1.5.5
'@jridgewell/trace-mapping': 0.3.30
'@jridgewell/[email protected]':
@@ -19224,7 +19558,7 @@ snapshots:
'@jridgewell/[email protected]':
dependencies:
'@jridgewell/set-array': 1.2.1
- '@jridgewell/sourcemap-codec': 1.5.0
+ '@jridgewell/sourcemap-codec': 1.5.5
'@jridgewell/trace-mapping': 0.3.30
'@jridgewell/[email protected]':
@@ -19252,22 +19586,22 @@ snapshots:
'@jridgewell/[email protected]':
dependencies:
'@jridgewell/resolve-uri': 3.1.2
- '@jridgewell/sourcemap-codec': 1.5.0
+ '@jridgewell/sourcemap-codec': 1.5.5
'@jridgewell/[email protected]':
dependencies:
'@jridgewell/resolve-uri': 3.1.2
- '@jridgewell/sourcemap-codec': 1.5.0
+ '@jridgewell/sourcemap-codec': 1.5.5
'@jridgewell/[email protected]':
dependencies:
'@jridgewell/resolve-uri': 3.1.2
- '@jridgewell/sourcemap-codec': 1.5.0
+ '@jridgewell/sourcemap-codec': 1.5.5
'@jridgewell/[email protected]':
dependencies:
'@jridgewell/resolve-uri': 3.1.2
- '@jridgewell/sourcemap-codec': 1.5.0
+ '@jridgewell/sourcemap-codec': 1.5.5
'@js-sdsl/[email protected]': {}
@@ -23051,18 +23385,18 @@ snapshots:
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
dependencies:
'@slack/logger': 3.0.0
'@slack/web-api': 6.12.1
'@types/jsonwebtoken': 8.5.9
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
jsonwebtoken: 9.0.2
lodash.isstring: 4.0.1
transitivePeerDependencies:
@@ -23073,7 +23407,7 @@ snapshots:
'@slack/logger': 4.0.0
'@slack/web-api': 7.5.0
'@types/jsonwebtoken': 9.0.9
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
jsonwebtoken: 9.0.2
lodash.isstring: 4.0.1
transitivePeerDependencies:
@@ -23083,7 +23417,7 @@ snapshots:
dependencies:
'@slack/logger': 3.0.0
'@slack/web-api': 6.12.1
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
'@types/ws': 7.4.7
eventemitter3: 5.0.1
finity: 0.5.4
@@ -23100,7 +23434,7 @@ snapshots:
'@slack/logger': 3.0.0
'@slack/types': 2.14.0
'@types/is-stream': 1.1.0
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
axios: 1.7.7
eventemitter3: 3.1.2
form-data: 2.5.1
@@ -23115,7 +23449,7 @@ snapshots:
dependencies:
'@slack/logger': 4.0.0
'@slack/types': 2.14.0
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
'@types/retry': 0.12.0
axios: 1.7.7
eventemitter3: 5.0.1
@@ -23741,7 +24075,7 @@ snapshots:
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
@@ -23775,19 +24109,19 @@ snapshots:
dependencies:
'@types/connect': 3.4.38
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
'@types/[email protected]': {}
@@ -23798,11 +24132,11 @@ snapshots:
'@types/connect': 3.4.38
'@types/express': 4.17.21
'@types/keygrip': 1.0.6
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
'@types/[email protected]': {}
@@ -23956,14 +24290,14 @@ snapshots:
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
'@types/qs': 6.9.11
'@types/range-parser': 1.2.7
'@types/send': 0.17.4
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
'@types/qs': 6.9.11
'@types/range-parser': 1.2.7
'@types/send': 0.17.4
@@ -24011,7 +24345,7 @@ snapshots:
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
'@types/[email protected]': {}
@@ -24027,7 +24361,7 @@ snapshots:
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
dependencies:
@@ -24051,7 +24385,7 @@ snapshots:
'@types/http-errors': 2.0.4
'@types/keygrip': 1.0.6
'@types/koa-compose': 3.2.8
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
@@ -24085,7 +24419,7 @@ snapshots:
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
'@types/[email protected]': {}
@@ -24097,11 +24431,11 @@ snapshots:
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
@@ -24111,10 +24445,6 @@ snapshots:
dependencies:
undici-types: 5.26.5
- '@types/[email protected]':
- dependencies:
- undici-types: 6.21.0
-
dependencies:
undici-types: 7.14.0
@@ -24131,14 +24461,10 @@ snapshots:
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
pg-protocol: 1.6.1
pg-types: 2.2.0
- '@types/[email protected]':
- dependencies:
- '@types/node': 24.7.1
-
'@types/[email protected]': {}
'@types/[email protected]': {}
@@ -24149,10 +24475,6 @@ snapshots:
dependencies:
types-ramda: 0.29.7
- '@types/[email protected]':
- dependencies:
- types-ramda: 0.30.1
-
dependencies:
types-ramda: 0.30.1
@@ -24161,6 +24483,10 @@ snapshots:
dependencies:
types-ramda: 0.31.0
+ '@types/[email protected]':
+ dependencies:
+ types-ramda: 0.31.0
+
'@types/[email protected]': {}
@@ -24204,13 +24530,13 @@ snapshots:
dependencies:
'@types/mime': 1.3.5
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
'@types/http-errors': 2.0.4
'@types/mime': 3.0.4
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
'@types/[email protected]': {}
@@ -24224,11 +24550,11 @@ snapshots:
dependencies:
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
'@types/[email protected]': {}
@@ -24248,7 +24574,7 @@ snapshots:
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
dependencies:
@@ -24309,6 +24635,12 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ '@vitest/[email protected]':
+ dependencies:
+ '@vitest/spy': 1.6.1
+ '@vitest/utils': 1.6.1
+ chai: 4.5.0
+
'@vitest/[email protected]':
dependencies:
'@vitest/spy': 3.1.3
@@ -24328,21 +24660,44 @@ snapshots:
dependencies:
tinyrainbow: 2.0.0
+ '@vitest/[email protected]':
+ dependencies:
+ '@vitest/utils': 1.6.1
+ p-limit: 5.0.0
+ pathe: 1.1.2
+
'@vitest/[email protected]':
dependencies:
'@vitest/utils': 3.1.3
pathe: 2.0.3
+ '@vitest/[email protected]':
+ dependencies:
+ magic-string: 0.30.19
+ pathe: 1.1.2
+ pretty-format: 29.7.0
+
'@vitest/[email protected]':
dependencies:
'@vitest/pretty-format': 3.1.3
magic-string: 0.30.17
pathe: 2.0.3
+ '@vitest/[email protected]':
+ dependencies:
+ tinyspy: 2.2.1
+
'@vitest/[email protected]':
dependencies:
tinyspy: 3.0.2
+ '@vitest/[email protected]':
+ dependencies:
+ diff-sequences: 29.6.3
+ estree-walker: 3.0.3
+ loupe: 2.3.7
+ pretty-format: 29.7.0
+
'@vitest/[email protected]':
dependencies:
'@vitest/pretty-format': 3.1.3
@@ -24513,8 +24868,6 @@ snapshots:
-
@@ -24616,6 +24969,8 @@ snapshots:
+
@@ -24693,9 +25048,9 @@ snapshots:
xxhash-wasm: 1.1.0
yargs-parser: 21.1.1
yocto-spinner: 0.2.2
- zod: 3.24.2
- zod-to-json-schema: 3.24.5([email protected])
+ zod: 3.25.76
+ zod-to-json-schema: 3.24.5([email protected])
optionalDependencies:
sharp: 0.33.5
transitivePeerDependencies:
@@ -25078,11 +25433,11 @@ snapshots:
dependencies:
cron-parser: 4.9.0
- ioredis: 5.7.0
+ ioredis: 5.8.2
msgpackr: 1.10.1
node-abort-controller: 3.1.1
- semver: 7.6.0
- tslib: 2.6.2
+ semver: 7.7.2
+ tslib: 2.7.0
uuid: 9.0.1
transitivePeerDependencies:
- supports-color
@@ -25205,6 +25560,16 @@ snapshots:
+ dependencies:
+ assertion-error: 1.1.0
+ check-error: 1.0.3
+ deep-eql: 4.1.4
+ get-func-name: 2.0.2
+ loupe: 2.3.7
+ pathval: 1.1.1
+ type-detect: 4.1.0
+
dependencies:
assertion-error: 2.0.1
@@ -25244,6 +25609,10 @@ snapshots:
+ dependencies:
+ get-func-name: 2.0.2
+
@@ -25305,7 +25674,7 @@ snapshots:
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
escape-string-regexp: 4.0.0
is-wsl: 2.2.0
lighthouse-logger: 1.4.2
@@ -25314,7 +25683,7 @@ snapshots:
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
escape-string-regexp: 4.0.0
is-wsl: 2.2.0
lighthouse-logger: 1.4.2
@@ -25660,6 +26029,8 @@ snapshots:
+
dependencies:
internmap: 1.0.1
@@ -25929,6 +26300,10 @@ snapshots:
dependencies:
character-entities: 2.0.2
+ dependencies:
+ type-detect: 4.1.0
+
@@ -26053,6 +26428,8 @@ snapshots:
+
@@ -26217,7 +26594,7 @@ snapshots:
dependencies:
'@types/cookie': 0.4.1
'@types/cors': 2.8.17
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
accepts: 1.3.8
base64id: 2.0.0
cookie: 0.7.2
@@ -26471,6 +26848,32 @@ snapshots:
'@esbuild/win32-ia32': 0.19.12
'@esbuild/win32-x64': 0.19.12
+ optionalDependencies:
+ '@esbuild/aix-ppc64': 0.21.5
+ '@esbuild/android-arm': 0.21.5
+ '@esbuild/android-arm64': 0.21.5
+ '@esbuild/android-x64': 0.21.5
+ '@esbuild/darwin-arm64': 0.21.5
+ '@esbuild/darwin-x64': 0.21.5
+ '@esbuild/freebsd-arm64': 0.21.5
+ '@esbuild/freebsd-x64': 0.21.5
+ '@esbuild/linux-arm': 0.21.5
+ '@esbuild/linux-arm64': 0.21.5
+ '@esbuild/linux-ia32': 0.21.5
+ '@esbuild/linux-loong64': 0.21.5
+ '@esbuild/linux-mips64el': 0.21.5
+ '@esbuild/linux-ppc64': 0.21.5
+ '@esbuild/linux-riscv64': 0.21.5
+ '@esbuild/linux-s390x': 0.21.5
+ '@esbuild/linux-x64': 0.21.5
+ '@esbuild/netbsd-x64': 0.21.5
+ '@esbuild/openbsd-x64': 0.21.5
+ '@esbuild/sunos-x64': 0.21.5
+ '@esbuild/win32-arm64': 0.21.5
+ '@esbuild/win32-ia32': 0.21.5
+ '@esbuild/win32-x64': 0.21.5
+
optionalDependencies:
'@esbuild/aix-ppc64': 0.24.0
@@ -27441,6 +27844,8 @@ snapshots:
+
dependencies:
es-errors: 1.3.0
@@ -28092,20 +28497,6 @@ snapshots:
dependencies:
loose-envify: 1.4.0
- dependencies:
- '@ioredis/commands': 1.3.0
- cluster-key-slot: 1.1.2
- debug: 4.4.0
- denque: 2.1.0
- lodash.defaults: 4.2.0
- lodash.isarguments: 3.1.0
- redis-errors: 1.2.0
- redis-parser: 3.0.0
- standard-as-callback: 2.1.0
- transitivePeerDependencies:
- - supports-color
-
dependencies:
'@ioredis/commands': 1.4.0
@@ -28388,7 +28779,7 @@ snapshots:
'@jest/environment': 29.7.0
'@jest/fake-timers': 29.7.0
'@jest/types': 29.6.3
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
jest-mock: 29.7.0
jest-util: 29.7.0
@@ -28409,13 +28800,13 @@ snapshots:
dependencies:
'@jest/types': 29.6.3
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
jest-util: 29.7.0
dependencies:
'@jest/types': 29.6.3
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
chalk: 4.1.2
ci-info: 3.9.0
graceful-fs: 4.2.11
@@ -28432,7 +28823,7 @@ snapshots:
dependencies:
- '@types/node': 24.7.1
+ '@types/node': 20.14.8
jest-util: 29.7.0
merge-stream: 2.0.0
supports-color: 8.1.1
@@ -28763,6 +29154,11 @@ snapshots:
+ dependencies:
+ mlly: 1.8.0
+ pkg-types: 1.3.1
+
dependencies:
mlly: 1.8.0
@@ -28860,6 +29256,10 @@ snapshots:
+ dependencies:
+ get-func-name: 2.0.2
+
@@ -30282,7 +30682,7 @@ snapshots:
dependencies:
yocto-queue: 0.1.0
dependencies:
yocto-queue: 1.1.1
@@ -30455,6 +30855,8 @@ snapshots:
+
@@ -30757,7 +31159,7 @@ snapshots:
'@protobufjs/path': 1.1.2
'@protobufjs/pool': 1.1.0
'@protobufjs/utf8': 1.1.0
- '@types/node': 22.18.0
+ '@types/node': 20.14.8
long: 5.2.3
@@ -32462,6 +32864,10 @@ snapshots:
+ dependencies:
+ js-tokens: 9.0.1
+
dependencies:
js-tokens: 9.0.1
@@ -32787,10 +33193,14 @@ snapshots:
fdir: 6.5.0([email protected])
picomatch: 4.0.3
+
+
@@ -32853,7 +33263,7 @@ snapshots:
dependencies:
- ansis: 4.1.0
+ ansis: 4.2.0
cac: 6.7.14
chokidar: 4.0.3
debug: 4.4.1
@@ -32864,7 +33274,7 @@ snapshots:
rolldown-plugin-dts: 0.15.9([email protected])([email protected])
semver: 7.7.2
tinyexec: 1.0.1
- tinyglobby: 0.2.14
+ tinyglobby: 0.2.15
tree-kill: 1.2.2
unconfig: 7.3.3
optionalDependencies:
@@ -32915,6 +33325,8 @@ snapshots:
+
@@ -33047,8 +33459,6 @@ snapshots:
-
@@ -33387,6 +33797,24 @@ snapshots:
d3-time: 3.1.0
d3-timer: 3.0.1
+ dependencies:
+ cac: 6.7.14
+ debug: 4.4.1
+ pathe: 1.1.2
+ picocolors: 1.1.1
+ transitivePeerDependencies:
+ - '@types/node'
+ - less
+ - lightningcss
+ - sass
+ - sass-embedded
+ - stylus
+ - sugarss
+ - supports-color
+ - terser
+
dependencies:
cac: 6.7.14
@@ -33419,6 +33847,17 @@ snapshots:
- supports-color
- typescript
+ dependencies:
+ esbuild: 0.21.5
+ postcss: 8.5.6
+ rollup: 4.52.5
+ optionalDependencies:
+ '@types/node': 20.14.8
+ fsevents: 2.3.3
+ lightningcss: 1.30.1
+ terser: 5.27.1
+
dependencies:
esbuild: 0.25.9
@@ -33475,6 +33914,41 @@ snapshots:
optionalDependencies:
+ dependencies:
+ '@vitest/expect': 1.6.1
+ '@vitest/runner': 1.6.1
+ '@vitest/snapshot': 1.6.1
+ '@vitest/spy': 1.6.1
+ '@vitest/utils': 1.6.1
+ acorn-walk: 8.3.2
+ chai: 4.5.0
+ debug: 4.4.1
+ execa: 8.0.1
+ local-pkg: 0.5.1
+ magic-string: 0.30.19
+ pathe: 1.1.2
+ picocolors: 1.1.1
+ std-env: 3.9.0
+ strip-literal: 2.1.1
+ tinybench: 2.9.0
+ tinypool: 0.8.4
+ why-is-node-running: 2.3.0
+ optionalDependencies:
+ '@types/node': 20.14.8
+ jsdom: 26.1.0
+ transitivePeerDependencies:
+ - less
+ - lightningcss
+ - sass
+ - sass-embedded
+ - stylus
+ - sugarss
+ - supports-color
+ - terser
+
dependencies:
'@vitest/expect': 3.1.3
@@ -33867,10 +34341,14 @@ snapshots:
dependencies:
zod: 3.24.2
+ dependencies:
+ zod: 3.25.76
+
dependencies:
typescript: 5.9.3
- zod: 3.24.2
+ zod: 3.25.76