OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: redo how the importer works
Carl-Gerhard Lindesvärd647ac2a4afac
diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.tsindex f03a053cc..f390a72a7 100644--- a/apps/worker/src/jobs/import.ts+++ b/apps/worker/src/jobs/import.ts@@ -1,17 +1,16 @@import {- type IClickhouseEvent,- type ImportSteps,- type Prisma,backfillSessionsToProduction,+ cleanupStagingData,createSessionsStartEndEvents,db,- formatClickhouseDate,- generateSessionIds,+ generateGapBasedSessionIds,getImportDateBounds,- getImportProgress,+ type IClickhouseEvent,+ type IClickhouseProfile,insertImportBatch,- markImportComplete,+ insertProfilesBatch,moveImportsToProduction,+ type Prisma,updateImportStatus,} from '@openpanel/db';import { MixpanelProvider, UmamiProvider } from '@openpanel/importer';@@ -22,294 +21,245 @@ import { logger } from '../utils/logger';const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10);-/**- * Yields control back to the event loop to prevent stalled jobs- */-async function yieldToEventLoop(): Promise<void> {+function yieldToEventLoop(): Promise<void> {return new Promise((resolve) => {setTimeout(resolve, 100);});}+const PRODUCTION_STEPS = ['moving', 'backfilling_sessions'];+export async function importJob(job: Job<ImportQueuePayload>) {const { importId } = job.data.payload;const record = await db.$primary().import.findUniqueOrThrow({where: { id: importId },- include: {- project: true,- },+ include: { project: true },});- const jobLogger = logger.child({- importId,- config: record.config,- });-- type ValidStep = Exclude<ImportSteps, 'failed' | 'completed'>;- const steps: Record<ValidStep, number> = {- loading: 0,- generating_session_ids: 1,- creating_sessions: 2,- moving: 3,- backfilling_sessions: 4,- };-+ const jobLogger = logger.child({ importId, config: record.config });jobLogger.info('Starting import job');+const providerInstance = createProvider(record, jobLogger);+ const shouldGenerateSessionIds = providerInstance.shouldGenerateSessionIds();try {- // Check if this is a resume operation- const isNewImport = record.currentStep === null;-- if (isNewImport) {- await updateImportStatus(jobLogger, job, importId, {- step: 'loading',- });- } else {- jobLogger.info('Resuming import from previous state', {- currentStep: record.currentStep,- currentBatch: record.currentBatch,- });- }-- // Try to get a precomputed total for better progress reporting- const totalEvents = await providerInstance- .getTotalEventsCount()- .catch(() => -1);- let processedEvents = record.processedEvents;-- const resumeLoadingFrom =- (record.currentStep === 'loading' && record.currentBatch) || undefined;-- const resumeGeneratingSessionIdsFrom =- (record.currentStep === 'generating_session_ids' &&- record.currentBatch) ||- undefined;-- const resumeCreatingSessionsFrom =- (record.currentStep === 'creating_sessions' && record.currentBatch) ||- undefined;-- const resumeMovingFrom =- (record.currentStep === 'moving' && record.currentBatch) || undefined;-- const resumeBackfillingSessionsFrom =- (record.currentStep === 'backfilling_sessions' && record.currentBatch) ||- undefined;-- // Example:- // shouldRunStep(0) // currStep = 2 (should not run)- // shouldRunStep(1) // currStep = 2 (should not run)- // shouldRunStep(2) // currStep = 2 (should run)- // shouldRunStep(3) // currStep = 2 (should run)- const shouldRunStep = (step: ValidStep) => {- if (isNewImport) {- return true;+ const isRetry = record.currentStep !== null;+ const hasReachedProduction =+ isRetry && PRODUCTION_STEPS.includes(record.currentStep as string);++ // -------------------------------------------------------+ // STAGING PHASE: clean slate on failure, run from scratch+ // -------------------------------------------------------+ if (!hasReachedProduction) {+ if (isRetry) {+ jobLogger.info(+ 'Retry detected before production phase — cleaning staging data'+ );+ await cleanupStagingData(importId);}- const stepToRunIndex = steps[step];- const currentStepIndex = steps[record.currentStep as ValidStep];- return stepToRunIndex >= currentStepIndex;- };-- async function whileBounds(- from: string | undefined,- callback: (from: string, to: string) => Promise<void>,- ) {- const bounds = await getImportDateBounds(importId, from);- if (bounds.min && bounds.max) {- const start = new Date(bounds.min);- const end = new Date(bounds.max);- let cursor = new Date(start);- while (cursor < end) {- const next = new Date(cursor);- next.setDate(next.getDate() + 1);- await callback(- formatClickhouseDate(cursor, true),- formatClickhouseDate(next, true),- );- cursor = next;-- // Yield control back to event loop after processing each day- await yieldToEventLoop();- }- }- }+ // Phase 1: Load events into staging+ await updateImportStatus(jobLogger, job, importId, { step: 'loading' });- // Phase 1: Fetch & Transform - Process events in batches- if (shouldRunStep('loading')) {- const eventBatch: any = [];- for await (const rawEvent of providerInstance.parseSource(- resumeLoadingFrom,- )) {- // Validate event+ const totalEvents = await providerInstance+ .getTotalEventsCount()+ .catch(() => -1);+ let processedEvents = 0;+ const eventBatch: IClickhouseEvent[] = [];++ for await (const rawEvent of providerInstance.parseSource()) {if (!providerInstance.validate(- // @ts-expect-error- rawEvent,+ // @ts-expect-error -- provider-specific raw type+ rawEvent)) {jobLogger.warn('Skipping invalid event', { rawEvent });continue;}- eventBatch.push(rawEvent);-- // Process batch when it reaches the batch size- if (eventBatch.length >= BATCH_SIZE) {- jobLogger.info('Processing batch', { batchSize: eventBatch.length });-- const transformedEvents: IClickhouseEvent[] = eventBatch.map(- (- // @ts-expect-error- event,- ) => providerInstance!.transformEvent(event),- );+ const transformed: IClickhouseEvent = providerInstance.transformEvent(+ // @ts-expect-error -- provider-specific raw type+ rawEvent+ );- await insertImportBatch(transformedEvents, importId);+ // Session IDs for providers that need them (e.g. Mixpanel) are generated+ // in generateGapBasedSessionIds after loading, using gap-based logic.+ eventBatch.push(transformed);+ if (eventBatch.length >= BATCH_SIZE) {+ await insertImportBatch(eventBatch, importId);processedEvents += eventBatch.length;- eventBatch.length = 0;- const createdAt = new Date(transformedEvents[0]?.created_at || '')+ const batchDate = new Date(eventBatch[0]?.created_at || '').toISOString().split('T')[0];await updateImportStatus(jobLogger, job, importId, {step: 'loading',- batch: createdAt,+ batch: batchDate,totalEvents,processedEvents,});- // Yield control back to event loop after processing each batch+ eventBatch.length = 0;await yieldToEventLoop();}}- // Process remaining events in the last batchif (eventBatch.length > 0) {- const transformedEvents = eventBatch.map(- (- // @ts-expect-error- event,- ) => providerInstance!.transformEvent(event),- );-- await insertImportBatch(transformedEvents, importId);-+ await insertImportBatch(eventBatch, importId);processedEvents += eventBatch.length;- eventBatch.length = 0;- const createdAt = new Date(transformedEvents[0]?.created_at || '')+ const batchDate = new Date(eventBatch[0]?.created_at || '').toISOString().split('T')[0];await updateImportStatus(jobLogger, job, importId, {step: 'loading',- batch: createdAt,+ batch: batchDate,totalEvents,processedEvents,});-- // Yield control back to event loop after processing final batch- await yieldToEventLoop();+ eventBatch.length = 0;}- }- // Phase 2: Generate session IDs if provider requires it- if (- shouldRunStep('generating_session_ids') &&- providerInstance.shouldGenerateSessionIds()- ) {- await whileBounds(resumeGeneratingSessionIdsFrom, async (from) => {- console.log('Generating session IDs', { from });- await generateSessionIds(importId, from);+ jobLogger.info('Loading complete', { processedEvents });++ // Phase 1b: Load user profiles (Mixpanel only)+ const profileBatchSize = 5000;+ if (+ 'streamProfiles' in providerInstance &&+ typeof (providerInstance as MixpanelProvider).streamProfiles ===+ 'function'+ ) {await updateImportStatus(jobLogger, job, importId, {- step: 'generating_session_ids',- batch: from,+ step: 'loading_profiles',});- // Yield control back to event loop after processing each day- await yieldToEventLoop();- });+ const profileBatch: IClickhouseProfile[] = [];+ let processedProfiles = 0;++ for await (const rawProfile of (+ providerInstance as MixpanelProvider+ ).streamProfiles()) {+ const profile = (+ providerInstance as MixpanelProvider+ ).transformProfile(rawProfile);+ profileBatch.push(profile);++ if (profileBatch.length >= profileBatchSize) {+ await insertProfilesBatch(profileBatch, record.projectId);+ processedProfiles += profileBatch.length;+ await updateImportStatus(jobLogger, job, importId, {+ step: 'loading_profiles',+ processedProfiles,+ });+ profileBatch.length = 0;+ await yieldToEventLoop();+ }+ }- jobLogger.info('Session ID generation complete');- }+ if (profileBatch.length > 0) {+ await insertProfilesBatch(profileBatch, record.projectId);+ processedProfiles += profileBatch.length;+ await updateImportStatus(jobLogger, job, importId, {+ step: 'loading_profiles',+ processedProfiles,+ totalProfiles: processedProfiles,+ });+ }- // Phase 3-5: Process in daily batches for robustness+ jobLogger.info('Profile loading complete', { processedProfiles });+ }- if (shouldRunStep('creating_sessions')) {- await whileBounds(resumeCreatingSessionsFrom, async (from) => {- await createSessionsStartEndEvents(importId, from);+ // Phase 2: Generate gap-based session IDs (Mixpanel etc.)+ if (shouldGenerateSessionIds) {await updateImportStatus(jobLogger, job, importId, {- step: 'creating_sessions',- batch: from,+ step: 'generating_sessions',});-- // Yield control back to event loop after processing each day+ await generateGapBasedSessionIds(importId);await yieldToEventLoop();+ jobLogger.info('Session ID generation complete');+ }++ // Phase 3: Create session_start / session_end events+ await updateImportStatus(jobLogger, job, importId, {+ step: 'creating_sessions',+ batch: 'all sessions',});+ await createSessionsStartEndEvents(importId);+ await yieldToEventLoop();++ jobLogger.info('Session event creation complete');}- if (shouldRunStep('moving')) {- await whileBounds(resumeMovingFrom, async (from) => {- await moveImportsToProduction(importId, from);+ // -------------------------------------------------------+ // PRODUCTION PHASE: resume-safe, track progress per batch+ // -------------------------------------------------------++ // Phase 3: Move staging events to production (per-day)+ const resumeMovingFrom =+ hasReachedProduction && record.currentStep === 'moving'+ ? (record.currentBatch ?? undefined)+ : undefined;++ // currentBatch is the last successfully completed day — resume from the next day to avoid re-inserting it+ const moveFromDate = (() => {+ if (!resumeMovingFrom) return undefined;+ const next = new Date(`${resumeMovingFrom}T12:00:00Z`);+ next.setUTCDate(next.getUTCDate() + 1);+ return next.toISOString().split('T')[0]!;+ })();++ const bounds = await getImportDateBounds(importId, moveFromDate);+ if (bounds.min && bounds.max) {+ const startDate = bounds.min.split(' ')[0]!;+ const endDate = bounds.max.split(' ')[0]!;+ const cursor = new Date(`${startDate}T12:00:00Z`);+ const end = new Date(`${endDate}T12:00:00Z`);++ while (cursor <= end) {+ const dateStr = cursor.toISOString().split('T')[0]!;++ await moveImportsToProduction(importId, dateStr);await updateImportStatus(jobLogger, job, importId, {step: 'moving',- batch: from,+ batch: dateStr,});- // Yield control back to event loop after processing each dayawait yieldToEventLoop();- });+ cursor.setUTCDate(cursor.getUTCDate() + 1);+ }}- if (shouldRunStep('backfilling_sessions')) {- await whileBounds(resumeBackfillingSessionsFrom, async (from) => {- await backfillSessionsToProduction(importId, from);- await updateImportStatus(jobLogger, job, importId, {- step: 'backfilling_sessions',- batch: from,- });-- // Yield control back to event loop after processing each day- await yieldToEventLoop();- });- }+ jobLogger.info('Move to production complete');- await markImportComplete(importId);+ // Phase 4: Backfill sessions tableawait updateImportStatus(jobLogger, job, importId, {- step: 'completed',+ step: 'backfilling_sessions',+ batch: 'all sessions',});- jobLogger.info('Import marked as complete');+ await backfillSessionsToProduction(importId);+ await yieldToEventLoop();- // Get final progress- const finalProgress = await getImportProgress(importId);+ jobLogger.info('Session backfill complete');- jobLogger.info('Import job completed successfully', {- totalEvents: finalProgress.totalEvents,- insertedEvents: finalProgress.insertedEvents,- status: finalProgress.status,- });+ // Done+ await updateImportStatus(jobLogger, job, importId, { step: 'completed' });+ jobLogger.info('Import completed');- return {- success: true,- totalEvents: finalProgress.totalEvents,- processedEvents: finalProgress.insertedEvents,- };+ return { success: true };} catch (error) {jobLogger.error('Import job failed', { error });- // Mark import as failedtry {const errorMsg = error instanceof Error ? error.message : 'Unknown error';await updateImportStatus(jobLogger, job, importId, {step: 'failed',errorMessage: errorMsg,});- jobLogger.warn('Import marked as failed', { error: errorMsg });} catch (markError) {jobLogger.error('Failed to mark import as failed', { error, markError });}@@ -320,7 +270,7 @@ export async function importJob(job: Job<ImportQueuePayload>) {function createProvider(record: Prisma.ImportGetPayload<{ include: { project: true } }>,- jobLogger: ILogger,+ jobLogger: ILogger) {const config = record.config;switch (config.provider) {diff --git a/packages/db/index.ts b/packages/db/index.tsindex 06c41720c..f0e461c3b 100644--- a/packages/db/index.ts+++ b/packages/db/index.ts@@ -1,6 +1,5 @@export * from './src/prisma-client';export * from './src/clickhouse/client';-export * from './src/clickhouse/csv';export * from './src/sql-builder';export * from './src/services/chart.service';export * from './src/engine';diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex bcba30be0..c1d306a2d 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -1,11 +1,9 @@-import { Readable } from 'node:stream';import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client';import { ClickHouseLogLevel, createClient } from '@clickhouse/client';-import sqlstring from 'sqlstring';-import type { NodeClickHouseClientConfigOptions } from '@clickhouse/client/dist/config';import { createLogger } from '@openpanel/logger';import type { IInterval } from '@openpanel/validation';+import sqlstring from 'sqlstring';export { createClient };@@ -68,8 +66,11 @@ export const TABLE_NAMES = {* Non-clustered mode = self-hosted environments*/export function isClickhouseClustered(): boolean {- if (process.env.CLICKHOUSE_CLUSTER === 'true' || process.env.CLICKHOUSE_CLUSTER === '1') {- return true+ if (+ process.env.CLICKHOUSE_CLUSTER === 'true' ||+ process.env.CLICKHOUSE_CLUSTER === '1'+ ) {+ return true;}return !(@@ -97,21 +98,21 @@ function getClickhouseSettings(): ClickHouseSettings {return {distributed_product_mode: 'allow',date_time_input_format: 'best_effort',- ...(!process.env.CLICKHOUSE_SETTINGS_REMOVE_CONVERT_ANY_JOIN- ? {+ ...(process.env.CLICKHOUSE_SETTINGS_REMOVE_CONVERT_ANY_JOIN+ ? {}+ : {query_plan_convert_any_join_to_semi_or_anti_join: 0,- }- : {}),+ }),...additionalSettings,};}export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {max_open_connections: 30,- request_timeout: 300000,+ request_timeout: 300_000,keep_alive: {enabled: true,- idle_socket_ttl: 60000,+ idle_socket_ttl: 60_000,},compression: {request: true,@@ -138,7 +139,7 @@ const cleanQuery = (query?: string) =>export async function withRetry<T>(operation: () => Promise<T>,maxRetries = 3,- baseDelay = 500,+ baseDelay = 500): Promise<T> {let lastError: Error | undefined;@@ -162,7 +163,7 @@ export async function withRetry<T>(`Attempt ${attempt + 1}/${maxRetries} failed, retrying in ${delay}ms`,{error: error.message,- },+ });await new Promise((resolve) => setTimeout(resolve, delay));continue;@@ -213,7 +214,7 @@ export const ch = new Proxy(originalCh, {export async function chQueryWithMeta<T extends Record<string, any>>(query: string,- clickhouseSettings?: ClickHouseSettings,+ clickhouseSettings?: ClickHouseSettings): Promise<ResponseJSON<T>> {const start = Date.now();const res = await ch.query({@@ -249,44 +250,16 @@ export async function chQueryWithMeta<T extends Record<string, any>>(return response;}-export async function chInsertCSV(tableName: string, rows: string[]) {- try {- const now = performance.now();- // Create a readable stream in binary mode for CSV (similar to EventBuffer)- const csvStream = Readable.from(rows.join('\n'), {- objectMode: false,- });-- await ch.insert({- table: tableName,- values: csvStream,- format: 'CSV',- clickhouse_settings: {- format_csv_allow_double_quotes: 1,- format_csv_allow_single_quotes: 0,- },- });-- logger.info('CSV Insert successful', {- elapsed: performance.now() - now,- rows: rows.length,- });- } catch (error) {- logger.error('CSV Insert failed:', error);- throw error;- }-}-export async function chQuery<T extends Record<string, any>>(query: string,- clickhouseSettings?: ClickHouseSettings,+ clickhouseSettings?: ClickHouseSettings): Promise<T[]> {return (await chQueryWithMeta<T>(query, clickhouseSettings)).data;}export function formatClickhouseDate(date: Date | string,- skipTime = false,+ skipTime = false): string {if (skipTime) {return new Date(date).toISOString().split('T')[0]!;diff --git a/packages/db/src/clickhouse/csv.ts b/packages/db/src/clickhouse/csv.tsdeleted file mode 100644index 21802f04d..000000000--- a/packages/db/src/clickhouse/csv.ts+++ /dev/null@@ -1,53 +0,0 @@-// ClickHouse Map(String, String) format in CSV uses single quotes, not JSON double quotes-// Format: '{'key1':'value1','key2':'value2'}'-// Single quotes inside values must be escaped with backslash: \'-// We also need to escape newlines and control characters to prevent CSV parsing issues-const escapeMapValue = (str: string) => {- return str- .replace(/\\/g, '\\\\') // Escape backslashes first- .replace(/'/g, "\\'") // Escape single quotes- .replace(/\n/g, '\\n') // Escape newlines- .replace(/\r/g, '\\r') // Escape carriage returns- .replace(/\t/g, '\\t') // Escape tabs- .replace(/\0/g, '\\0'); // Escape null bytes-};--export const csvEscapeJson = (- value: Record<string, unknown> | null | undefined,-): string => {- if (value == null) return '';-- // Normalize to strings if your column is Map(String,String)- const normalized: Record<string, string> = Object.fromEntries(- Object.entries(value).map(([k, v]) => [- String(k),- v == null ? '' : String(v),- ]),- );-- // Empty object should return empty Map (without quotes, csvEscapeField will handle if needed)- if (Object.keys(normalized).length === 0) return '{}';-- const pairs = Object.entries(normalized)- .map(([k, v]) => `'${escapeMapValue(k)}':'${escapeMapValue(v)}'`)- .join(',');-- // Return Map format without outer quotes - csvEscapeField will handle CSV escaping- // This allows csvEscapeField to properly wrap/escape the entire field if it contains newlines/quotes- return csvEscapeField(`{${pairs}}`);-};--// Escape a CSV field - wrap in double quotes if it contains commas, quotes, or newlines-// Double quotes inside must be doubled (""), per CSV standard-export const csvEscapeField = (value: string | number): string => {- const str = String(value);-- // If field contains commas, quotes, or newlines, it must be quoted- if (/[,"\n\r]/.test(str)) {- // Escape double quotes by doubling them- const escaped = str.replace(/"/g, '""');- return `"${escaped}"`;- }-- return str;-};diff --git a/packages/db/src/services/import.service.ts b/packages/db/src/services/import.service.tsindex 36d371690..19132ab68 100644--- a/packages/db/src/services/import.service.ts+++ b/packages/db/src/services/import.service.ts@@ -1,15 +1,14 @@+import { createHash } from 'node:crypto';import type { ILogger } from '@openpanel/logger';-import sqlstring from 'sqlstring';import {- TABLE_NAMES,ch,- chInsertCSV,convertClickhouseDateToJs,formatClickhouseDate,getReplicatedTableName,+ TABLE_NAMES,} from '../clickhouse/client';-import { csvEscapeField, csvEscapeJson } from '../clickhouse/csv';-import { type Prisma, db } from '../prisma-client';+import { db, type Prisma } from '../prisma-client';+import type { IClickhouseProfile } from './profile.service';import type { IClickhouseEvent } from './event.service';export interface ImportStageResult {@@ -18,11 +17,89 @@ export interface ImportStageResult {insertedEvents: number;}-export interface ImportProgress {- importId: string;- totalEvents: number;- insertedEvents: number;- status: 'pending' | 'processing' | 'processed' | 'failed';+const SESSION_GAP_MS = 30 * 60 * 1000; // 30 minutes++/**+ * Generate gap-based session IDs for events that have none.+ * Streams events from staging (sorted by device_id, created_at), assigns a new+ * session when gap > 30 min, re-inserts with session_id, then deletes old rows.+ */+export async function generateGapBasedSessionIds(+ importId: string+): Promise<void> {+ let currentDeviceId = '';+ let currentSessionId = '';+ let currentLastTime = 0;+ let currentCounter = -1;+ const BATCH_SIZE = 5000;+ const batch: IClickhouseEvent[] = [];++ const result = await ch.query({+ query: `+ SELECT id, name, sdk_name, sdk_version, device_id, profile_id, project_id,+ session_id, path, origin, referrer, referrer_name, referrer_type,+ duration, properties, created_at, country, city, region,+ longitude, latitude, os, os_version, browser, browser_version,+ device, brand, model, imported_at+ FROM ${TABLE_NAMES.events_imports}+ WHERE import_id = {importId:String}+ AND session_id = ''+ AND device != 'server'+ ORDER BY device_id, created_at+ `,+ query_params: { importId },+ format: 'JSONEachRow',+ });++ const stream = result.stream();+ for await (const rows of stream) {+ for (const row of rows) {+ const event = row.json() as IClickhouseEvent;+ const time = new Date(event.created_at).getTime();++ if (event.device_id !== currentDeviceId) {+ currentDeviceId = event.device_id;+ currentSessionId = '';+ currentLastTime = 0;+ currentCounter = -1;+ }++ if (!currentSessionId || time - currentLastTime > SESSION_GAP_MS) {+ currentCounter++;+ currentSessionId = createHash('md5')+ .update(`${event.device_id}-${currentCounter}`)+ .digest('hex')+ .toLowerCase();+ }+ currentLastTime = time;+ event.session_id = currentSessionId;++ batch.push(event);+ if (batch.length >= BATCH_SIZE) {+ await insertImportBatch(batch, importId);+ batch.length = 0;+ }+ }+ }++ if (batch.length > 0) {+ await insertImportBatch(batch, importId);+ }++ const mutationTable = getReplicatedTableName(TABLE_NAMES.events_imports);+ await ch.command({+ query: `ALTER TABLE ${mutationTable} DELETE+ WHERE import_id = {importId:String}+ AND session_id = ''+ AND device != 'server'`,+ query_params: { importId },+ clickhouse_settings: {+ wait_end_of_query: 1,+ mutations_sync: '2',+ send_progress_in_http_headers: 1,+ http_headers_progress_interval_ms: '50000',+ },+ });}/**@@ -30,56 +107,27 @@ export interface ImportProgress {*/export async function insertImportBatch(events: IClickhouseEvent[],- importId: string,+ importId: string): Promise<ImportStageResult> {if (events.length === 0) {return { importId, totalEvents: 0, insertedEvents: 0 };}- // Important to have same order as events_imports table- // CSV format: properly quotes fields that need it- const csvRows = events.map((event) => {- // Properties need to be converted to JSON for Map(String, String)- // All fields must be CSV-escaped when joining with commas- const fields = [- csvEscapeField(event.id || ''),- csvEscapeField(event.name),- csvEscapeField(event.sdk_name || ''),- csvEscapeField(event.sdk_version || ''),- csvEscapeField(event.device_id || ''),- csvEscapeField(event.profile_id || ''),- csvEscapeField(event.project_id || ''),- csvEscapeField(event.session_id || ''),- csvEscapeField(event.path),- csvEscapeField(event.origin || ''),- csvEscapeField(event.referrer || ''),- csvEscapeField(event.referrer_name || ''),- csvEscapeField(event.referrer_type || ''),- csvEscapeField(event.duration ?? 0),- csvEscapeJson(event.properties),- csvEscapeField(event.created_at),- csvEscapeField(event.country || ''),- csvEscapeField(event.city || ''),- csvEscapeField(event.region || ''),- csvEscapeField(event.longitude != null ? event.longitude : '\\N'),- csvEscapeField(event.latitude != null ? event.latitude : '\\N'),- csvEscapeField(event.os || ''),- csvEscapeField(event.os_version || ''),- csvEscapeField(event.browser || ''),- csvEscapeField(event.browser_version || ''),- csvEscapeField(event.device || ''),- csvEscapeField(event.brand || ''),- csvEscapeField(event.model || ''),- csvEscapeField('\\N'), // imported_at (Nullable)- csvEscapeField(importId),- csvEscapeField('pending'), // import_status- csvEscapeField(formatClickhouseDate(new Date())), // imported_at_meta (DateTime, not DateTime64, so no milliseconds)- ];- return fields.join(',');+ const now = formatClickhouseDate(new Date());+ const rows = events.map((event) => ({+ ...event,+ import_id: importId,+ import_status: 'pending',+ imported_at: event.imported_at || now,+ imported_at_meta: now,+ }));++ await ch.insert({+ table: TABLE_NAMES.events_imports,+ values: rows,+ format: 'JSONEachRow',});- await chInsertCSV(TABLE_NAMES.events_imports, csvRows);-return {importId,totalEvents: events.length,@@ -88,44 +136,86 @@ export async function insertImportBatch(}/**- * Generate deterministic session IDs for events that don't have them- * Uses 30-minute time windows to create consistent session IDs across imports- * Only processes events where device != 'server' and session_id = ''+ * Insert a batch of profiles into the production profiles table.+ * Used by Mixpanel (and other providers) to import user profiles during an import job.*/-export async function generateSessionIds(- importId: string,- from: string,-): Promise<void> {- const rangeWhere = [- 'import_id = {importId:String}',- "import_status = 'pending'",- "device != 'server'",- "session_id = ''",- from ? 'toDate(created_at) = {from:String}' : '',- ]- .filter(Boolean)- .join(' AND ');-- // Use SQL to generate deterministic session IDs based on device_id + 30-min time windows- // This ensures same events always get same session IDs regardless of import order- // In clustered mode, we must use the replicated table for mutations- const mutationTableName = getReplicatedTableName(TABLE_NAMES.events_imports);- const updateQuery = `- ALTER TABLE ${mutationTableName}- UPDATE session_id = lower(hex(MD5(concat(- device_id,- '-',- toString(toInt64(toUnixTimestamp(created_at) / 1800))- ))))- WHERE ${rangeWhere}- `;+export async function insertProfilesBatch(+ profiles: IClickhouseProfile[],+ projectId: string+): Promise<{ inserted: number }> {+ if (profiles.length === 0) {+ return { inserted: 0 };+ }++ const normalized = profiles.map((p) => ({+ id: p.id,+ project_id: projectId,+ first_name: p.first_name ?? '',+ last_name: p.last_name ?? '',+ email: p.email ?? '',+ avatar: p.avatar ?? '',+ is_external: p.is_external ?? true,+ properties: Object.fromEntries(+ Object.entries(p.properties || {}).filter(+ (kv): kv is [string, string] => kv[1] != null && kv[1] !== ''+ )+ ) as Record<string, string>,+ created_at: p.created_at,+ }));++ await ch.insert({+ table: TABLE_NAMES.profiles,+ values: normalized,+ format: 'JSONEachRow',+ });++ return { inserted: normalized.length };+}+++++++++++++++++++++++++++++++++/**+ * Delete all staging data for an import. Used to get a clean slate on retry+ * when the failure happened before moving data to production.+ */+export async function cleanupStagingData(importId: string): Promise<void> {+ const mutationTableName = getReplicatedTableName(TABLE_NAMES.events_imports);await ch.command({- query: updateQuery,- query_params: { importId, from },+ query: `ALTER TABLE ${mutationTableName} DELETE WHERE import_id = {importId:String}`,+ query_params: { importId },clickhouse_settings: {wait_end_of_query: 1,- mutations_sync: '2', // Wait for mutation to complete on all replicas (critical!)+ mutations_sync: '2',send_progress_in_http_headers: 1,http_headers_progress_interval_ms: '50000',},@@ -133,315 +223,256 @@ export async function generateSessionIds(}/**- * Reconstruct sessions using SQL-based logic- * This identifies session boundaries and creates session_start/session_end events- * session_start inherits all properties from the first event in the session- * session_end inherits all properties from the last event in the session and calculates duration+ * Reconstruct sessions across ALL dates for the import.+ * Each session_id gets exactly one session_start and one session_end,+ * even if the session spans midnight.+ *+ * Batches by fetching distinct session_ids first, then running the+ * heavy aggregation only for that batch of IDs.*/export async function createSessionsStartEndEvents(- importId: string,- from: string,+ importId: string): Promise<void> {- // First, let's identify session boundaries and get first/last events for each session- const rangeWhere = [+ const SESSION_BATCH_SIZE = 5000;+ let lastSessionId = '';++ const baseWhere = ['import_id = {importId:String}',- "import_status = 'pending'",- "session_id != ''", // Only process events that have session IDs- 'toDate(created_at) = {from:String}',- ]- .filter(Boolean)- .join(' AND ');-- // Use window functions to efficiently get first event (all fields) and last event (only changing fields)- // session_end only needs: properties, path, origin, created_at - the rest can be inherited from session_start- const sessionEventsQuery = `- SELECT- device_id,- session_id,- project_id,- profile_id,- argMin((path, origin, referrer, referrer_name, referrer_type, properties, created_at, country, city, region, longitude, latitude, os, os_version, browser, browser_version, device, brand, model), created_at) AS first_event,- argMax((path, origin, properties, created_at), created_at) AS last_event_fields,- min(created_at) AS first_timestamp,- max(created_at) AS last_timestamp- FROM ${TABLE_NAMES.events_imports}- WHERE ${rangeWhere}- AND name NOT IN ('session_start', 'session_end')- GROUP BY session_id, device_id, project_id, profile_id- `;+ "session_id != ''",+ "name NOT IN ('session_start', 'session_end')",+ ].join(' AND ');++ while (true) {+ const idsResult = await ch.query({+ query: `+ SELECT DISTINCT session_id+ FROM ${TABLE_NAMES.events_imports}+ WHERE ${baseWhere}+ AND session_id > {lastSessionId:String}+ ORDER BY session_id+ LIMIT {limit:UInt32}+ `,+ query_params: { importId, lastSessionId, limit: SESSION_BATCH_SIZE },+ format: 'JSONEachRow',+ });- const sessionEventsResult = await ch.query({- query: sessionEventsQuery,- query_params: { importId, from },- format: 'JSONEachRow',- });+ const idRows = (await idsResult.json()) as Array<{ session_id: string }>;+ if (idRows.length === 0) {+ break;+ }- const sessionData = (await sessionEventsResult.json()) as Array<{- device_id: string;- session_id: string;- project_id: string;- profile_id: string;- first_event: [- // string, // id- // string, // name- string, // path- string, // origin- string, // referrer- string, // referrer_name- string, // referrer_type- // number, // duration- Record<string, unknown>, // properties- string, // created_at- string, // country- string, // city- string, // region- number | null, // longitude- number | null, // latitude- string, // os- string, // os_version- string, // browser- string, // browser_version- string, // device- string, // brand- string, // model- // string, // sdk_name- // string, // sdk_version- // string, // imported_at- ];- last_event_fields: [- string, // path- string, // origin- Record<string, unknown>, // properties- string, // created_at- ];- first_timestamp: string;- last_timestamp: string;- }>;+ const sessionIds = idRows.map((r) => r.session_id);++ const sessionEventsQuery = `+ SELECT+ device_id,+ session_id,+ project_id,+ if(+ any(nullIf(profile_id, device_id)) IS NULL,+ any(profile_id),+ any(nullIf(profile_id, device_id))+ ) AS profile_id,+ argMin((path, origin, referrer, referrer_name, referrer_type, properties, created_at, country, city, region, longitude, latitude, os, os_version, browser, browser_version, device, brand, model), created_at) AS first_event,+ argMax((path, origin, properties, created_at), created_at) AS last_event_fields,+ min(created_at) AS first_timestamp,+ max(created_at) AS last_timestamp+ FROM ${TABLE_NAMES.events_imports}+ WHERE ${baseWhere}+ AND session_id IN ({sessionIds:Array(String)})+ GROUP BY session_id, device_id, project_id+ `;++ const sessionEventsResult = await ch.query({+ query: sessionEventsQuery,+ query_params: { importId, sessionIds },+ format: 'JSONEachRow',+ });++ const sessionData = (await sessionEventsResult.json()) as Array<{+ device_id: string;+ session_id: string;+ project_id: string;+ profile_id: string;+ first_event: [+ string, // path+ string, // origin+ string, // referrer+ string, // referrer_name+ string, // referrer_type+ Record<string, unknown>, // properties+ string, // created_at+ string, // country+ string, // city+ string, // region+ number | null, // longitude+ number | null, // latitude+ string, // os+ string, // os_version+ string, // browser+ string, // browser_version+ string, // device+ string, // brand+ string, // model+ ];+ last_event_fields: [+ string, // path+ string, // origin+ Record<string, unknown>, // properties+ string, // created_at+ ];+ first_timestamp: string;+ last_timestamp: string;+ }>;++ const sessionEvents: IClickhouseEvent[] = [];- // Create session_start and session_end events- const sessionEvents: IClickhouseEvent[] = [];-- for (const session of sessionData) {- // Destructure first event tuple (all fields)- const [- // firstId,- // firstName,- firstPath,- firstOrigin,- firstReferrer,- firstReferrerName,- firstReferrerType,- // firstDuration,- firstProperties,- firstCreatedAt,- firstCountry,- firstCity,- firstRegion,- firstLongitude,- firstLatitude,- firstOs,- firstOsVersion,- firstBrowser,- firstBrowserVersion,- firstDevice,- firstBrand,- firstModel,- // firstSdkName,- // firstSdkVersion,- // firstImportedAt,- ] = session.first_event;-- // Destructure last event fields (only the changing ones)- const [lastPath, lastOrigin, lastProperties, lastCreatedAt] =- session.last_event_fields;-- // Calculate duration in milliseconds- // Parse timestamps as Date objects to calculate duration- const firstTime = new Date(session.first_timestamp).getTime();- const lastTime = new Date(session.last_timestamp).getTime();- const durationMs = Math.max(0, lastTime - firstTime); // Ensure non-negative duration-- // Helper function to adjust timestamp by milliseconds without timezone conversionconst adjustTimestamp = (timestamp: string, offsetMs: number): string => {- // Parse the timestamp, adjust it, and format back to ClickHouse formatconst date = convertClickhouseDateToJs(timestamp);date.setTime(date.getTime() + offsetMs);return formatClickhouseDate(date);};- // Create session_start event - inherit everything from first event but change name- // Set created_at to 1 second before the first event- sessionEvents.push({- id: crypto.randomUUID(),- name: 'session_start',- device_id: session.device_id,- profile_id: session.profile_id,- project_id: session.project_id,- session_id: session.session_id,- path: firstPath,- origin: firstOrigin,- referrer: firstReferrer,- referrer_name: firstReferrerName,- referrer_type: firstReferrerType,- duration: 0, // session_start always has 0 duration- properties: firstProperties as Record<- string,- string | number | boolean | null | undefined- >,- created_at: adjustTimestamp(session.first_timestamp, -1000), // 1 second before first event- country: firstCountry,- city: firstCity,- region: firstRegion,- longitude: firstLongitude,- latitude: firstLatitude,- os: firstOs,- os_version: firstOsVersion,- browser: firstBrowser,- browser_version: firstBrowserVersion,- device: firstDevice,- brand: firstBrand,- model: firstModel,- imported_at: new Date().toISOString(),- sdk_name: 'import-session-reconstruction',- sdk_version: '1.0.0',- });+ for (const session of sessionData) {+ const [+ firstPath,+ firstOrigin,+ firstReferrer,+ firstReferrerName,+ firstReferrerType,+ firstProperties,+ _firstCreatedAt,+ firstCountry,+ firstCity,+ firstRegion,+ firstLongitude,+ firstLatitude,+ firstOs,+ firstOsVersion,+ firstBrowser,+ firstBrowserVersion,+ firstDevice,+ firstBrand,+ firstModel,+ ] = session.first_event;++ const [lastPath, lastOrigin, lastProperties, _lastCreatedAt] =+ session.last_event_fields;++ const firstTime = new Date(session.first_timestamp).getTime();+ const lastTime = new Date(session.last_timestamp).getTime();+ const durationMs = Math.max(0, lastTime - firstTime);++ sessionEvents.push({+ id: crypto.randomUUID(),+ name: 'session_start',+ device_id: session.device_id,+ profile_id: session.profile_id,+ project_id: session.project_id,+ session_id: session.session_id,+ path: firstPath,+ origin: firstOrigin,+ referrer: firstReferrer,+ referrer_name: firstReferrerName,+ referrer_type: firstReferrerType,+ duration: 0,+ properties: firstProperties as Record<+ string,+ string | number | boolean | null | undefined+ >,+ created_at: adjustTimestamp(session.first_timestamp, -1000),+ country: firstCountry,+ city: firstCity,+ region: firstRegion,+ longitude: firstLongitude,+ latitude: firstLatitude,+ os: firstOs,+ os_version: firstOsVersion,+ browser: firstBrowser,+ browser_version: firstBrowserVersion,+ device: firstDevice,+ brand: firstBrand,+ model: firstModel,+ imported_at: new Date().toISOString(),+ sdk_name: 'import-session-reconstruction',+ sdk_version: '1.0.0',+ });++ sessionEvents.push({+ id: crypto.randomUUID(),+ name: 'session_end',+ device_id: session.device_id,+ profile_id: session.profile_id,+ project_id: session.project_id,+ session_id: session.session_id,+ path: lastPath,+ origin: lastOrigin,+ referrer: firstReferrer,+ referrer_name: firstReferrerName,+ referrer_type: firstReferrerType,+ duration: durationMs,+ properties: lastProperties as Record<+ string,+ string | number | boolean | null | undefined+ >,+ created_at: adjustTimestamp(session.last_timestamp, 1000),+ country: firstCountry,+ city: firstCity,+ region: firstRegion,+ longitude: firstLongitude,+ latitude: firstLatitude,+ os: firstOs,+ os_version: firstOsVersion,+ browser: firstBrowser,+ browser_version: firstBrowserVersion,+ device: firstDevice,+ brand: firstBrand,+ model: firstModel,+ imported_at: new Date().toISOString(),+ sdk_name: 'import-session-reconstruction',+ sdk_version: '1.0.0',+ });+ }- // Create session_end event - inherit most from session_start, but use last event's path, origin, properties- // Set created_at to 1 second after the last event- sessionEvents.push({- id: crypto.randomUUID(),- name: 'session_end',- device_id: session.device_id,- profile_id: session.profile_id,- project_id: session.project_id,- session_id: session.session_id,- path: lastPath, // From last event- origin: lastOrigin, // From last event- referrer: firstReferrer, // Same as session_start- referrer_name: firstReferrerName, // Same as session_start- referrer_type: firstReferrerType, // Same as session_start- duration: durationMs,- properties: lastProperties as Record<- string,- string | number | boolean | null | undefined- >, // From last event- created_at: adjustTimestamp(session.last_timestamp, 500), // 1 second after last event- country: firstCountry, // Same as session_start- city: firstCity, // Same as session_start- region: firstRegion, // Same as session_start- longitude: firstLongitude, // Same as session_start- latitude: firstLatitude, // Same as session_start- os: firstOs, // Same as session_start- os_version: firstOsVersion, // Same as session_start- browser: firstBrowser, // Same as session_start- browser_version: firstBrowserVersion, // Same as session_start- device: firstDevice, // Same as session_start- brand: firstBrand, // Same as session_start- model: firstModel, // Same as session_start- imported_at: new Date().toISOString(),- sdk_name: 'import-session-reconstruction',- sdk_version: '1.0.0',- });- }+ if (sessionEvents.length > 0) {+ await insertImportBatch(sessionEvents, importId);+ }- // Insert session events into imports table- if (sessionEvents.length > 0) {- await insertImportBatch(sessionEvents, importId);+ lastSessionId = idRows[idRows.length - 1]!.session_id;+ if (idRows.length < SESSION_BATCH_SIZE) {+ break;+ }}}/**- * Migrate all events from imports table to production events table- * This includes both original events and generated session events+ * Move events from staging to production events table.+ * Batched per-day using a simple date filter.*/export async function moveImportsToProduction(importId: string,- from: string,+ from: string): Promise<void> {- // Build the WHERE clause for migration- // For session events (session_start/session_end), we don't filter by their created_at- // because they're created with adjusted timestamps (±1 second) that might fall outside- // the date range. Instead, we include them if their session_id has events in this range.let whereClause = 'import_id = {importId:String}';if (from) {- whereClause += ` AND (- (toDate(created_at) = {from:String}) OR- (- name IN ('session_start', 'session_end') AND- session_id IN (- SELECT DISTINCT session_id- FROM ${TABLE_NAMES.events_imports}- WHERE import_id = {importId:String}- AND toDate(created_at) = {from:String}- AND name NOT IN ('session_start', 'session_end')- )- )- )`;+ whereClause += ' AND toDate(created_at) = {from:String}';}const migrationQuery = `INSERT INTO ${TABLE_NAMES.events} (- id,- name,- sdk_name,- sdk_version,- device_id,- profile_id,- project_id,- session_id,- path,- origin,- referrer,- referrer_name,- referrer_type,- duration,- properties,- created_at,- country,- city,- region,- longitude,- latitude,- os,- os_version,- browser,- browser_version,- device,- brand,- model,- imported_at+ id, name, sdk_name, sdk_version, device_id, profile_id, project_id,+ session_id, path, origin, referrer, referrer_name, referrer_type,+ duration, properties, created_at, country, city, region,+ longitude, latitude, os, os_version, browser, browser_version,+ device, brand, model, imported_at)SELECT- id,- name,- sdk_name,- sdk_version,- device_id,- profile_id,- project_id,- session_id,- path,- origin,- referrer,- referrer_name,- referrer_type,- duration,- properties,- created_at,- country,- city,- region,- longitude,- latitude,- os,- os_version,- browser,- browser_version,- device,- brand,- model,- imported_at+ id, name, sdk_name, sdk_version, device_id, profile_id, project_id,+ session_id, path, origin, referrer, referrer_name, referrer_type,+ duration, properties, created_at, country, city, region,+ longitude, latitude, os, os_version, browser, browser_version,+ device, brand, model, imported_atFROM ${TABLE_NAMES.events_imports}WHERE ${whereClause}ORDER BY created_at ASC@@ -452,227 +483,127 @@ export async function moveImportsToProduction(query_params: { importId, from },clickhouse_settings: {wait_end_of_query: 1,- // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.- send_progress_in_http_headers: 1,- // The interval of sending these progress headers. Here it is less than 60s,- http_headers_progress_interval_ms: '50000',- },- });-}--export async function backfillSessionsToProduction(- importId: string,- from: string,-): Promise<void> {- // After migrating events, populate the sessions table based on the migrated sessions- // We detect all session_ids involved in this import from the imports table,- // then aggregate over the production events to construct session rows.- const sessionsInsertQuery = `- INSERT INTO ${TABLE_NAMES.sessions} (- id,- project_id,- profile_id,- device_id,- created_at,- ended_at,- is_bounce,- entry_origin,- entry_path,- exit_origin,- exit_path,- screen_view_count,- revenue,- event_count,- duration,- country,- region,- city,- longitude,- latitude,- device,- brand,- model,- browser,- browser_version,- os,- os_version,- sign,- version,- utm_medium,- utm_source,- utm_campaign,- utm_content,- utm_term,- referrer,- referrer_name,- referrer_type- )- SELECT- any(e.session_id) as id,- any(e.project_id) as project_id,- if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,- any(e.device_id) as device_id,- argMin(e.created_at, e.created_at) as created_at,- argMax(e.created_at, e.created_at) as ended_at,- if(- argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = '',- if(countIf(e.name = 'screen_view') > 1, false, true),- argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = 'true'- ) as is_bounce,- argMinIf(e.origin, e.created_at, e.name = 'session_start') as entry_origin,- argMinIf(e.path, e.created_at, e.name = 'session_start') as entry_path,- argMaxIf(e.origin, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_origin,- argMaxIf(e.path, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_path,- countIf(e.name = 'screen_view') as screen_view_count,- 0 as revenue,- countIf(e.name != 'screen_view' AND e.name != 'session_start' AND e.name != 'session_end') as event_count,- sumIf(e.duration, name = 'session_end') AS duration,- argMinIf(e.country, e.created_at, e.name = 'session_start') as country,- argMinIf(e.region, e.created_at, e.name = 'session_start') as region,- argMinIf(e.city, e.created_at, e.name = 'session_start') as city,- argMinIf(e.longitude, e.created_at, e.name = 'session_start') as longitude,- argMinIf(e.latitude, e.created_at, e.name = 'session_start') as latitude,- argMinIf(e.device, e.created_at, e.name = 'session_start') as device,- argMinIf(e.brand, e.created_at, e.name = 'session_start') as brand,- argMinIf(e.model, e.created_at, e.name = 'session_start') as model,- argMinIf(e.browser, e.created_at, e.name = 'session_start') as browser,- argMinIf(e.browser_version, e.created_at, e.name = 'session_start') as browser_version,- argMinIf(e.os, e.created_at, e.name = 'session_start') as os,- argMinIf(e.os_version, e.created_at, e.name = 'session_start') as os_version,- 1 as sign,- 1 as version,- argMinIf(e.properties['__query.utm_medium'], e.created_at, e.name = 'session_start') as utm_medium,- argMinIf(e.properties['__query.utm_source'], e.created_at, e.name = 'session_start') as utm_source,- argMinIf(e.properties['__query.utm_campaign'], e.created_at, e.name = 'session_start') as utm_campaign,- argMinIf(e.properties['__query.utm_content'], e.created_at, e.name = 'session_start') as utm_content,- argMinIf(e.properties['__query.utm_term'], e.created_at, e.name = 'session_start') as utm_term,- argMinIf(e.referrer, e.created_at, e.name = 'session_start') as referrer,- argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name,- argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_type- FROM ${TABLE_NAMES.events_imports} e- WHERE- e.import_id = ${sqlstring.escape(importId)}- AND e.session_id != ''- AND (- (toDate(e.created_at) = ${sqlstring.escape(from)}) OR- (- e.name IN ('session_start', 'session_end') AND- e.session_id IN (- SELECT DISTINCT session_id- FROM ${TABLE_NAMES.events_imports}- WHERE import_id = ${sqlstring.escape(importId)}- AND toDate(created_at) = ${sqlstring.escape(from)}- AND name NOT IN ('session_start', 'session_end')- )- )- )- GROUP BY e.session_id- `;-- await ch.command({- query: sessionsInsertQuery,- clickhouse_settings: {- wait_end_of_query: 1,- // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.send_progress_in_http_headers: 1,- // The interval of sending these progress headers. Here it is less than 60s,http_headers_progress_interval_ms: '50000',},});}/**- * Mark import as complete by updating status+ * Aggregate sessions from staging into the sessions table.+ * Runs across all dates so cross-midnight sessions become one row.+ * Batches by session_ids to bound ClickHouse memory.*/-export async function markImportComplete(importId: string): Promise<void> {- // In clustered mode, we must use the replicated table for mutations- const mutationTableName = getReplicatedTableName(TABLE_NAMES.events_imports);- const updateQuery = `- ALTER TABLE ${mutationTableName}- UPDATE import_status = 'processed'- WHERE import_id = {importId:String}- `;-- await ch.command({- query: updateQuery,- query_params: { importId },- clickhouse_settings: {- wait_end_of_query: 1,- mutations_sync: '2', // Wait for mutation to complete- // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.- send_progress_in_http_headers: 1,- // The interval of sending these progress headers. Here it is less than 60s,- http_headers_progress_interval_ms: '50000',- },- });-}--/**- * Get import progress and status- */-export async function getImportProgress(- importId: string,-): Promise<ImportProgress> {- const progressQuery = `- SELECT- import_id,- COUNT(*) as total_events,- COUNTIf(import_status = 'pending') as pending_events,- COUNTIf(import_status = 'processed') as processed_events,- any(import_status) as status- FROM ${TABLE_NAMES.events_imports}- WHERE import_id = {importId:String}- AND name NOT IN ('session_start', 'session_end')- GROUP BY import_id- `;-- const result = await ch.query({- query: progressQuery,- query_params: { importId },- format: 'JSONEachRow',- });+export async function backfillSessionsToProduction(+ importId: string+): Promise<void> {+ const SESSION_BATCH_SIZE = 5000;+ let lastSessionId = '';++ while (true) {+ const idsResult = await ch.query({+ query: `+ SELECT DISTINCT session_id+ FROM ${TABLE_NAMES.events_imports}+ WHERE import_id = {importId:String}+ AND session_id > {lastSessionId:String}+ ORDER BY session_id+ LIMIT {limit:UInt32}+ `,+ query_params: { importId, lastSessionId, limit: SESSION_BATCH_SIZE },+ format: 'JSONEachRow',+ });- const data = (await result.json()) as Array<{- import_id: string;- total_events: number;- pending_events: number;- processed_events: number;- status: string;- }>;+ const idRows = (await idsResult.json()) as Array<{ session_id: string }>;+ if (idRows.length === 0) {+ break;+ }- if (data.length === 0) {- return {- importId,- totalEvents: 0,- insertedEvents: 0,- status: 'pending',- };- }+ const sessionIds = idRows.map((r) => r.session_id);++ const sessionsInsertQuery = `+ INSERT INTO ${TABLE_NAMES.sessions} (+ id, project_id, profile_id, device_id, created_at, ended_at,+ is_bounce, entry_origin, entry_path, exit_origin, exit_path,+ screen_view_count, revenue, event_count, duration,+ country, region, city, longitude, latitude,+ device, brand, model, browser, browser_version, os, os_version,+ sign, version,+ utm_medium, utm_source, utm_campaign, utm_content, utm_term,+ referrer, referrer_name, referrer_type+ )+ SELECT+ any(e.session_id) as id,+ any(e.project_id) as project_id,+ if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,+ any(e.device_id) as device_id,+ argMin(e.created_at, e.created_at) as created_at,+ argMax(e.created_at, e.created_at) as ended_at,+ if(+ argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = '',+ if(countIf(e.name = 'screen_view') > 1, false, true),+ argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = 'true'+ ) as is_bounce,+ argMinIf(e.origin, e.created_at, e.name = 'session_start') as entry_origin,+ argMinIf(e.path, e.created_at, e.name = 'session_start') as entry_path,+ argMaxIf(e.origin, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_origin,+ argMaxIf(e.path, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_path,+ countIf(e.name = 'screen_view') as screen_view_count,+ 0 as revenue,+ countIf(e.name != 'screen_view' AND e.name != 'session_start' AND e.name != 'session_end') as event_count,+ sumIf(e.duration, name = 'session_end') AS duration,+ argMinIf(e.country, e.created_at, e.name = 'session_start') as country,+ argMinIf(e.region, e.created_at, e.name = 'session_start') as region,+ argMinIf(e.city, e.created_at, e.name = 'session_start') as city,+ argMinIf(e.longitude, e.created_at, e.name = 'session_start') as longitude,+ argMinIf(e.latitude, e.created_at, e.name = 'session_start') as latitude,+ argMinIf(e.device, e.created_at, e.name = 'session_start') as device,+ argMinIf(e.brand, e.created_at, e.name = 'session_start') as brand,+ argMinIf(e.model, e.created_at, e.name = 'session_start') as model,+ argMinIf(e.browser, e.created_at, e.name = 'session_start') as browser,+ argMinIf(e.browser_version, e.created_at, e.name = 'session_start') as browser_version,+ argMinIf(e.os, e.created_at, e.name = 'session_start') as os,+ argMinIf(e.os_version, e.created_at, e.name = 'session_start') as os_version,+ 1 as sign,+ 1 as version,+ argMinIf(e.properties['__query.utm_medium'], e.created_at, e.name = 'session_start') as utm_medium,+ argMinIf(e.properties['__query.utm_source'], e.created_at, e.name = 'session_start') as utm_source,+ argMinIf(e.properties['__query.utm_campaign'], e.created_at, e.name = 'session_start') as utm_campaign,+ argMinIf(e.properties['__query.utm_content'], e.created_at, e.name = 'session_start') as utm_content,+ argMinIf(e.properties['__query.utm_term'], e.created_at, e.name = 'session_start') as utm_term,+ argMinIf(e.referrer, e.created_at, e.name = 'session_start') as referrer,+ argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name,+ argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_type+ FROM ${TABLE_NAMES.events_imports} e+ WHERE+ e.import_id = {importId:String}+ AND e.session_id IN ({sessionIds:Array(String)})+ GROUP BY e.session_id+ `;++ await ch.command({+ query: sessionsInsertQuery,+ query_params: { importId, sessionIds },+ clickhouse_settings: {+ wait_end_of_query: 1,+ send_progress_in_http_headers: 1,+ http_headers_progress_interval_ms: '50000',+ },+ });- const row = data[0];- if (!row) {- return {- importId,- totalEvents: 0,- insertedEvents: 0,- status: 'pending',- };+ lastSessionId = idRows[idRows.length - 1]!.session_id;+ if (idRows.length < SESSION_BATCH_SIZE) {+ break;+ }}-- return {- importId,- totalEvents: row.total_events,- insertedEvents: row.processed_events,- status: row.status as 'pending' | 'processing' | 'processed' | 'failed',- };}/**- * Utility: get min/max created_at for an import+ * Get min/max created_at for an import's staging data.*/export async function getImportDateBounds(importId: string,- fromCreatedAt?: string,+ fromCreatedAt?: string): Promise<{ min: string | null; max: string | null }> {const res = await ch.query({query: `@@ -697,10 +628,6 @@ export async function getImportDateBounds(: { min: null, max: null };}-/**- * Unified method to update all import status information- * Combines step, batch, progress, and status message updates- */export type UpdateImportStatusOptions =| {step: 'loading';@@ -709,13 +636,17 @@ export type UpdateImportStatusOptions =processedEvents?: number;}| {- step: 'generating_session_ids';- batch?: string;+ step: 'loading_profiles';+ processedProfiles?: number;+ totalProfiles?: number;}| {step: 'creating_sessions';batch?: string;}+ | {+ step: 'generating_sessions';+ }| {step: 'moving';batch?: string;@@ -740,7 +671,7 @@ export async function updateImportStatus(updateProgress: (progress: Record<string, any>) => void;},importId: string,- options: UpdateImportStatusOptions,+ options: UpdateImportStatusOptions): Promise<void> {const data: Prisma.ImportUpdateInput = {};switch (options.step) {@@ -754,27 +685,35 @@ export async function updateImportStatus(data.totalEvents = options.totalEvents;data.processedEvents = options.processedEvents;break;- case 'generating_session_ids':- data.currentStep = 'generating_session_ids';- data.currentBatch = options.batch;- data.statusMessage = options.batch- ? `Generating session IDs for ${options.batch}`- : 'Generating session IDs...';+ case 'loading_profiles':+ data.currentStep = 'loading_profiles';+ data.statusMessage =+ options.processedProfiles != null && options.totalProfiles != null+ ? `Importing user profiles (${options.processedProfiles} / ${options.totalProfiles})`+ : 'Importing user profiles...';break;case 'creating_sessions':data.currentStep = 'creating_sessions';data.currentBatch = options.batch;- data.statusMessage = `Creating sessions for ${options.batch}`;+ data.statusMessage = options.batch+ ? `Creating sessions (${options.batch})`+ : 'Creating sessions...';+ break;+ case 'generating_sessions':+ data.currentStep = 'generating_sessions';+ data.statusMessage = 'Generating session IDs...';break;case 'moving':data.currentStep = 'moving';data.currentBatch = options.batch;- data.statusMessage = `Moving imports to production for ${options.batch}`;+ data.statusMessage = `Moving events to production (${options.batch})`;break;case 'backfilling_sessions':data.currentStep = 'backfilling_sessions';data.currentBatch = options.batch;- data.statusMessage = `Aggregating sessions for ${options.batch}`;+ data.statusMessage = options.batch+ ? `Aggregating sessions (${options.batch})`+ : 'Aggregating sessions...';break;case 'completed':data.status = 'completed';@@ -787,6 +726,8 @@ export async function updateImportStatus(data.statusMessage = 'Import failed';data.errorMessage = options.errorMessage;break;+ default:+ break;}jobLogger.info('Import status update', data);diff --git a/packages/importer/src/providers/mixpanel.test.ts b/packages/importer/src/providers/mixpanel.test.tsindex c3f1d0524..da63b9d3a 100644--- a/packages/importer/src/providers/mixpanel.test.ts+++ b/packages/importer/src/providers/mixpanel.test.ts@@ -39,7 +39,7 @@ describe('mixpanel', () => {const rawEvent = {event: '$mp_web_page_view',properties: {- time: 1746097970,+ time: 1_746_097_970,distinct_id: '$device:123',$browser: 'Chrome',$browser_version: 135,@@ -53,7 +53,7 @@ describe('mixpanel', () => {$insert_id: 'source_id',$lib_version: '2.60.0',$mp_api_endpoint: 'api-js.mixpanel.com',- $mp_api_timestamp_ms: 1746078175363,+ $mp_api_timestamp_ms: 1_746_078_175_363,$mp_autocapture: true,$os: 'Android',$referrer: 'https://google.com/',@@ -71,7 +71,7 @@ describe('mixpanel', () => {gclid: 'oqneoqow',mp_country_code: 'IN',mp_lib: 'web',- mp_processing_time_ms: 1746078175546,+ mp_processing_time_ms: 1_746_078_175_546,mp_sent_by_lib_version: '2.60.0',utm_medium: 'cpc',utm_source: 'google',@@ -101,7 +101,7 @@ describe('mixpanel', () => {__title:'Landeed: Satbara Utara, 7/12 Extract, Property Card & Index 2',},- created_at: '2025-05-01T11:12:50.000Z',+ created_at: '2025-05-01 11:12:50',country: 'IN',city: 'Mumbai',region: 'Maharashtra',@@ -110,7 +110,7 @@ describe('mixpanel', () => {os: 'Android',os_version: undefined,browser: 'Chrome',- browser_version: '',+ browser_version: '135',device: 'mobile',brand: '',model: '',@@ -141,7 +141,7 @@ describe('mixpanel', () => {const rawEvent = {event: 'custom_event',properties: {- time: 1746097970,+ time: 1_746_097_970,distinct_id: '$device:123',$device_id: '123',$user_id: 'user123',@@ -192,7 +192,7 @@ describe('mixpanel', () => {const rawEvent = {event: 'ec_search_error',properties: {- time: 1759947367,+ time: 1_759_947_367,distinct_id: '3385916',$browser: 'Mobile Safari',$browser_version: null,@@ -207,7 +207,7 @@ describe('mixpanel', () => {$insert_id: 'bclkaepeqcfuzt4v',$lib_version: '2.60.0',$mp_api_endpoint: 'api-js.mixpanel.com',- $mp_api_timestamp_ms: 1759927570699,+ $mp_api_timestamp_ms: 1_759_927_570_699,$os: 'iOS',$region: 'Karnataka',$screen_height: 852,@@ -225,7 +225,7 @@ describe('mixpanel', () => {language: 'english',mp_country_code: 'IN',mp_lib: 'web',- mp_processing_time_ms: 1759927592421,+ mp_processing_time_ms: 1_759_927_592_421,mp_sent_by_lib_version: '2.60.0',os: 'web',osVersion:@@ -249,15 +249,15 @@ describe('mixpanel', () => {expect(res.id.length).toBeGreaterThan(30);expect(res.imported_at).toMatch(- /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/,+ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/);expect(omit(['id', 'imported_at'], res)).toEqual({brand: 'Apple',browser: 'GSA',- browser_version: 'null',+ browser_version: '388.0.811331708',city: 'Bengaluru',country: 'IN',- created_at: '2025-10-08T18:16:07.000Z',+ created_at: '2025-10-08 18:16:07',device: 'mobile',device_id: '199b498af1036c-0e943279a1292e-5c0f4368-51bf4-199b498af1036c',duration: 0,diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/importer/src/providers/mixpanel.tsindex 42e41c0aa..c3676b4e4 100644--- a/packages/importer/src/providers/mixpanel.ts+++ b/packages/importer/src/providers/mixpanel.ts@@ -1,8 +1,13 @@import { randomUUID } from 'node:crypto';import { isSameDomain, parsePath, toDots } from '@openpanel/common';-import { type UserAgentInfo, parseUserAgent } from '@openpanel/common/server';-import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';-import type { IClickhouseEvent } from '@openpanel/db';+import {+ getReferrerWithQuery,+ parseReferrer,+ parseUserAgent,+ type UserAgentInfo,+} from '@openpanel/common/server';+import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';+import type { IClickhouseProfile } from '@openpanel/db';import type { ILogger } from '@openpanel/logger';import type { IMixpanelImportConfig } from '@openpanel/validation';import { z } from 'zod';@@ -15,22 +20,88 @@ export const zMixpanelRawEvent = z.object({export type MixpanelRawEvent = z.infer<typeof zMixpanelRawEvent>;+/** Engage API profile: https://docs.mixpanel.com/docs/export-methods#exporting-profiles */+export const zMixpanelRawProfile = z.object({+ $distinct_id: z.union([z.string(), z.number()]),+ $properties: z.record(z.unknown()).optional().default({}),+});+export type MixpanelRawProfile = z.infer<typeof zMixpanelRawProfile>;++class MixpanelRateLimitError extends Error {+ readonly retryAfterMs?: number;++ constructor(message: string, retryAfterMs?: number) {+ super(message);+ this.name = 'MixpanelRateLimitError';+ this.retryAfterMs = retryAfterMs;+ }+}+export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {provider = 'mixpanel';version = '1.0.0';+ private static readonly MAX_REQUESTS_PER_HOUR = 100;+ private static readonly MIN_REQUEST_INTERVAL_MS = 334; // 3 QPS limit+ private requestTimestamps: number[] = [];+ private lastRequestTime = 0;+constructor(private readonly projectId: string,private readonly config: IMixpanelImportConfig,- private readonly logger?: ILogger,+ private readonly logger?: ILogger) {super();}- async getTotalEventsCount(): Promise<number> {+ private async waitForRateLimit(): Promise<void> {+ const now = Date.now();+ const oneHourAgo = now - 60 * 60 * 1000;++ // Prune timestamps older than 1 hour+ this.requestTimestamps = this.requestTimestamps.filter(+ (t) => t > oneHourAgo+ );++ // Enforce per-second limit (3 QPS → min 334ms gap)+ const timeSinceLast = now - this.lastRequestTime;+ if (timeSinceLast < MixpanelProvider.MIN_REQUEST_INTERVAL_MS) {+ const delay = MixpanelProvider.MIN_REQUEST_INTERVAL_MS - timeSinceLast;+ await new Promise((resolve) => setTimeout(resolve, delay));+ }++ // Enforce hourly limit+ if (+ this.requestTimestamps.length >= MixpanelProvider.MAX_REQUESTS_PER_HOUR+ ) {+ const oldestInWindow = this.requestTimestamps[0]!;+ const waitUntil = oldestInWindow + 60 * 60 * 1000;+ const waitMs = waitUntil - Date.now() + 1000; // +1s buffer++ if (waitMs > 0) {+ this.logger?.info(+ `Rate limit: ${this.requestTimestamps.length} requests in the last hour, waiting ${Math.ceil(waitMs / 1000)}s`,+ {+ requestsInWindow: this.requestTimestamps.length,+ waitMs,+ }+ );+ await new Promise((resolve) => setTimeout(resolve, waitMs));+ // Prune again after waiting+ this.requestTimestamps = this.requestTimestamps.filter(+ (t) => t > Date.now() - 60 * 60 * 1000+ );+ }+ }++ this.lastRequestTime = Date.now();+ this.requestTimestamps.push(Date.now());+ }++ getTotalEventsCount(): Promise<number> {// Mixpanel sucks and dont provide a good way to extract total event count within a period// jql would work but not accurate and will be deprecated end of 2025- return -1;+ return Promise.resolve(-1);}/**@@ -42,13 +113,13 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {}async *parseSource(- overrideFrom?: string,+ overrideFrom?: string): AsyncGenerator<MixpanelRawEvent, void, unknown> {yield* this.fetchEventsFromMixpanel(overrideFrom);}private async *fetchEventsFromMixpanel(- overrideFrom?: string,+ overrideFrom?: string): AsyncGenerator<MixpanelRawEvent, void, unknown> {const { serviceAccount, serviceSecret, projectId, from, to } = this.config;@@ -58,20 +129,24 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {for (const [chunkFrom, chunkTo] of dateChunks) {let retries = 0;- const maxRetries = 3;+ const maxRetries = 6;while (retries <= maxRetries) {try {+ await this.waitForRateLimit();yield* this.fetchEventsForDateRange(serviceAccount,serviceSecret,projectId,chunkFrom,- chunkTo,+ chunkTo);break; // Success, move to next chunk} catch (error) {retries++;+ const isRateLimit =+ error instanceof MixpanelRateLimitError ||+ (error instanceof Error && error.message.includes('429'));const isLastRetry = retries > maxRetries;this.logger?.warn('Failed to fetch events for date range', {@@ -80,22 +155,31 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {attempt: retries,maxRetries,error: (error as Error).message,+ isRateLimit,willRetry: !isLastRetry,});if (isLastRetry) {- // Final attempt failed, re-throwthrow new Error(- `Failed to fetch Mixpanel events for ${chunkFrom} to ${chunkTo} after ${maxRetries} retries: ${(error as Error).message}`,+ `Failed to fetch Mixpanel events for ${chunkFrom} to ${chunkTo} after ${maxRetries} retries: ${(error as Error).message}`);}- // Exponential backoff: wait before retrying- const delay = Math.min(1000 * 2 ** (retries - 1), 60_000); // Cap at 1 minute+ let delay: number;+ if (error instanceof MixpanelRateLimitError && error.retryAfterMs) {+ delay = error.retryAfterMs;+ } else if (isRateLimit) {+ // 5min → 10min → 15min → 15min → 15min = 60min total+ delay = Math.min(300_000 * 2 ** (retries - 1), 900_000);+ } else {+ delay = Math.min(1000 * 2 ** (retries - 1), 60_000);+ }+this.logger?.info('Retrying after delay', {delayMs: delay,chunkFrom,chunkTo,+ isRateLimit,});await new Promise((resolve) => setTimeout(resolve, delay));}@@ -108,7 +192,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {serviceSecret: string,projectId: string,from: string,- to: string,+ to: string): AsyncGenerator<MixpanelRawEvent, void, unknown> {const url = 'https://data.mixpanel.com/api/2.0/export';@@ -134,9 +218,18 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {},});+ if (response.status === 429) {+ const retryAfter = response.headers.get('Retry-After');+ const retryAfterMs = retryAfter ? Number(retryAfter) * 1000 : undefined;+ throw new MixpanelRateLimitError(+ 'Mixpanel rate limit exceeded (429)',+ retryAfterMs+ );+ }+if (!response.ok) {throw new Error(- `Failed to fetch events from Mixpanel: ${response.status} ${response.statusText}`,+ `Failed to fetch events from Mixpanel: ${response.status} ${response.statusText}`);}@@ -153,7 +246,9 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {while (true) {const { done, value } = await reader.read();- if (done) break;+ if (done) {+ break;+ }buffer += decoder.decode(value, { stream: true });@@ -187,7 +282,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {{line: buffer.substring(0, 100),error,- },+ });}}@@ -196,6 +291,114 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {}}+ /**+ * Stream user profiles from Mixpanel Engage API.+ * Paginates with page/page_size (5k per page) and yields each profile.+ */+ async *streamProfiles(): AsyncGenerator<MixpanelRawProfile, void, unknown> {+ const { serviceAccount, serviceSecret, projectId } = this.config;+ const pageSize = 5000;+ let page = 0;++ while (true) {+ await this.waitForRateLimit();++ const url = `https://mixpanel.com/api/query/engage?project_id=${encodeURIComponent(projectId)}`;+ const body = new URLSearchParams({+ page: String(page),+ page_size: String(pageSize),+ });++ this.logger?.info('Fetching profiles from Mixpanel Engage', {+ page,+ page_size: pageSize,+ projectId,+ });++ const response = await fetch(url, {+ method: 'POST',+ headers: {+ Authorization: `Basic ${Buffer.from(`${serviceAccount}:${serviceSecret}`).toString('base64')}`,+ Accept: 'application/json',+ 'Content-Type': 'application/x-www-form-urlencoded',+ },+ body: body.toString(),+ });++ if (response.status === 429) {+ const retryAfter = response.headers.get('Retry-After');+ const retryAfterMs = retryAfter ? Number(retryAfter) * 1000 : undefined;+ throw new MixpanelRateLimitError(+ 'Mixpanel rate limit exceeded (429)',+ retryAfterMs+ );+ }++ if (!response.ok) {+ const text = await response.text();+ throw new Error(+ `Failed to fetch profiles from Mixpanel: ${response.status} ${response.statusText} - ${text}`+ );+ }++ const data = (await response.json()) as {+ results?: Array<{ $distinct_id: string | number; $properties?: Record<string, unknown> }>;+ page?: number;+ total?: number;+ };++ const results = data.results ?? [];+ for (const row of results) {+ const parsed = zMixpanelRawProfile.safeParse(row);+ if (parsed.success) {+ yield parsed.data;+ } else {+ this.logger?.warn('Skipping invalid Mixpanel profile', {+ row: JSON.stringify(row).slice(0, 200),+ });+ }+ }++ if (results.length < pageSize) {+ break;+ }+ page++;+ }+ }++ /**+ * Map Mixpanel Engage profile to OpenPanel IClickhouseProfile.+ */+ transformProfile(raw: MixpanelRawProfile): IClickhouseProfile {+ const parsed = zMixpanelRawProfile.parse(raw);+ const props = (parsed.$properties || {}) as Record<string, unknown>;++ const id = String(parsed.$distinct_id).replace(/^\$device:/, '');+ const createdAt = props.$created+ ? formatClickhouseDate(new Date(String(props.$created)))+ : formatClickhouseDate(new Date());++ const properties: Record<string, string> = {};+ const stripPrefix = /^\$/;+ for (const [key, value] of Object.entries(props)) {+ if (stripPrefix.test(key)) continue;+ if (value == null) continue;+ properties[key] = typeof value === 'object' ? JSON.stringify(value) : String(value);+ }++ return {+ id,+ project_id: this.projectId,+ first_name: String(props.$first_name ?? ''),+ last_name: String(props.$last_name ?? ''),+ email: String(props.$email ?? ''),+ avatar: String(props.$avatar ?? props.$image ?? ''),+ properties,+ created_at: createdAt,+ is_external: true,+ };+ }+validate(rawEvent: MixpanelRawEvent): boolean {const res = zMixpanelRawEvent.safeParse(rawEvent);return res.success;@@ -208,7 +411,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {const deviceId = props.$device_id;const profileId = String(props.$user_id || props.distinct_id).replace(/^\$device:/,- '',+ '');// Build full URL from current_url and current_url_search (web only)@@ -309,7 +512,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {project_id: projectId,session_id: '', // Will be generated in SQL after importproperties: toDots(properties), // Flatten nested objects/arrays to Map(String, String)- created_at: new Date(props.time * 1000).toISOString(),+ created_at: formatClickhouseDate(new Date(props.time * 1000)),country,city,region,@@ -318,10 +521,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {os: uaInfo.os || props.$os,os_version: uaInfo.osVersion || props.$osVersion,browser: uaInfo.browser || props.$browser,- browser_version:- uaInfo.browserVersion || props.$browserVersion- ? String(props.$browser_version)- : '',+ browser_version: uaInfo.browserVersion || String(props.$browser_version ?? ''),device: this.getDeviceType(props.mp_lib, uaInfo, props),brand: uaInfo.brand || '',model: uaInfo.model || '',@@ -338,14 +538,6 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {sdk_version: this.version,};- // TODO: Remove this- // Temporary fix for a client- const isMightBeScreenView = this.getMightBeScreenView(rawEvent);- if (isMightBeScreenView && event.name === 'Loaded a Screen') {- event.name = 'screen_view';- event.path = isMightBeScreenView;- }-// TODO: Remove this// This is a hack to get utm tags (not sure if this is just the testing project or all mixpanel projects)if (props.utm_source && !properties.__query?.utm_source) {@@ -371,13 +563,13 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {private getDeviceType(mp_lib: string,uaInfo: UserAgentInfo,- props: Record<string, any>,+ props: Record<string, any>) {// Normalize lib/os/browser dataconst lib = (mp_lib || '').toLowerCase();const os = String(props.$os || uaInfo.os || '').toLowerCase();const browser = String(- props.$browser || uaInfo.browser || '',+ props.$browser || uaInfo.browser || '').toLowerCase();const isTabletOs = os === 'ipados' || os === 'ipad os' || os === 'ipad';@@ -431,11 +623,6 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {return !this.isWebEvent(mp_lib);}- private getMightBeScreenView(rawEvent: MixpanelRawEvent) {- const props = rawEvent.properties as Record<string, any>;- return Object.keys(props).find((key) => key.match(/^[A-Z1-9_]+$/));- }-private parseServerDeviceInfo(props: Record<string, any>): UserAgentInfo {// For mobile events, extract device information from Mixpanel propertiesconst os = props.$os || props.os || '';@@ -446,19 +633,19 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {return {isServer: true,- os: os,- osVersion: osVersion,+ os,+ osVersion,browser: '',browserVersion: '',- device: device,- brand: brand,- model: model,+ device,+ brand,+ model,};}private stripMixpanelProperties(properties: Record<string, any>,- searchParams: Record<string, string>,+ searchParams: Record<string, string>): Record<string, any> {const strip = ['time',@@ -472,8 +659,8 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {];const filtered = Object.fromEntries(Object.entries(properties).filter(- ([key]) => !key.match(/^(\$|mp_|utm_)/) && !strip.includes(key),- ),+ ([key]) => !(key.match(/^(\$|mp_|utm_)/) || strip.includes(key))+ ));// Parse JSON strings back to objects/arrays so toDots() can flatten themdiff --git a/packages/importer/src/providers/umami.ts b/packages/importer/src/providers/umami.tsindex f232e1b8e..c69473295 100644--- a/packages/importer/src/providers/umami.ts+++ b/packages/importer/src/providers/umami.ts@@ -2,10 +2,13 @@ import { randomUUID } from 'node:crypto';import { Readable } from 'node:stream';import { pipeline } from 'node:stream/promises';import { createBrotliDecompress, createGunzip } from 'node:zlib';-import { isSameDomain, parsePath } from '@openpanel/common';-import { generateDeviceId } from '@openpanel/common/server';-import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';-import type { IClickhouseEvent } from '@openpanel/db';+import { isSameDomain, parsePath, toDots } from '@openpanel/common';+import {+ generateDeviceId,+ getReferrerWithQuery,+ parseReferrer,+} from '@openpanel/common/server';+import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';import type { ILogger } from '@openpanel/logger';import type { IUmamiImportConfig } from '@openpanel/validation';import { parse } from 'csv-parse';@@ -63,7 +66,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {constructor(private readonly projectId: string,private readonly config: IUmamiImportConfig,- private readonly logger?: ILogger,+ private readonly logger?: ILogger) {super();}@@ -82,7 +85,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {signal?: AbortSignal;maxBytes?: number;maxRows?: number;- } = {},+ } = {}): AsyncGenerator<UmamiRawEvent, void, unknown> {const { signal, maxBytes, maxRows } = opts;const controller = new AbortController();@@ -95,9 +98,9 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {}const res = await fetch(url, { signal: controller.signal });- if (!res.ok || !res.body) {+ if (!(res.ok && res.body)) {throw new Error(- `Failed to fetch remote file: ${res.status} ${res.statusText}`,+ `Failed to fetch remote file: ${res.status} ${res.statusText}`);}@@ -108,15 +111,15 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {if (contentType &&!/text\/csv|text\/plain|application\/gzip|application\/octet-stream/i.test(- contentType,+ contentType)) {- console.warn(`Warning: Content-Type is ${contentType}, expected CSV-ish`);+ this.logger?.warn(`Warning: Content-Type is ${contentType}, expected CSV-ish`);}if (maxBytes && contentLen && contentLen > maxBytes) {throw new Error(- `Remote file exceeds size limit (${contentLen} > ${maxBytes})`,+ `Remote file exceeds size limit (${contentLen} > ${maxBytes})`);}@@ -137,9 +140,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {if (seenBytes > maxBytes) {controller.abort();body.destroy(- new Error(- `Stream exceeded size limit (${seenBytes} > ${maxBytes})`,- ),+ new Error(`Stream exceeded size limit (${seenBytes} > ${maxBytes})`));}});@@ -190,7 +191,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {throw new Error(`Failed to parse remote file from ${url}: ${err instanceof Error ? err.message : String(err)- }`,+ }`);} finally {controller.abort(); // ensure fetch stream is torn down@@ -205,7 +206,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {transformEvent(_rawEvent: UmamiRawEvent): IClickhouseEvent {const projectId =this.config.projectMapper.find(- (mapper) => mapper.from === _rawEvent.website_id,+ (mapper) => mapper.from === _rawEvent.website_id)?.to || this.projectId;const rawEvent = zUmamiRawEvent.parse(_rawEvent);@@ -261,39 +262,50 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {}// Add useful properties from Umami data- if (rawEvent.page_title) properties.__title = rawEvent.page_title;- if (rawEvent.screen) properties.__screen = rawEvent.screen;- if (rawEvent.language) properties.__language = rawEvent.language;- if (rawEvent.utm_source)+ if (rawEvent.page_title) {+ properties.__title = rawEvent.page_title;+ }+ if (rawEvent.screen) {+ properties.__screen = rawEvent.screen;+ }+ if (rawEvent.language) {+ properties.__language = rawEvent.language;+ }+ if (rawEvent.utm_source) {properties = assocPath(['__query', 'utm_source'],rawEvent.utm_source,- properties,+ properties);- if (rawEvent.utm_medium)+ }+ if (rawEvent.utm_medium) {properties = assocPath(['__query', 'utm_medium'],rawEvent.utm_medium,- properties,+ properties);- if (rawEvent.utm_campaign)+ }+ if (rawEvent.utm_campaign) {properties = assocPath(['__query', 'utm_campaign'],rawEvent.utm_campaign,- properties,+ properties);- if (rawEvent.utm_content)+ }+ if (rawEvent.utm_content) {properties = assocPath(['__query', 'utm_content'],rawEvent.utm_content,- properties,+ properties);- if (rawEvent.utm_term)+ }+ if (rawEvent.utm_term) {properties = assocPath(['__query', 'utm_term'],rawEvent.utm_term,- properties,+ properties);+ }return {id: rawEvent.event_id || randomUUID(),@@ -302,8 +314,8 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {profile_id: profileId,project_id: projectId,session_id: rawEvent.session_id || '',- properties,- created_at: rawEvent.created_at.toISOString(),+ properties: toDots(properties),+ created_at: formatClickhouseDate(rawEvent.created_at),country,city,region: this.mapRegion(region),@@ -329,7 +341,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {}mapRegion(region: string): string {- return region.replace(/^[A-Z]{2}\-/, '');+ return region.replace(/^[A-Z]{2}-/, '');}mapDevice(device: string): string {