OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat(importer): add data residency for mixpanel imports (#360)
Mayank Rajfb283cf9daf0
diff --git a/apps/start/src/modals/add-import.tsx b/apps/start/src/modals/add-import.tsxindex 03a77e354..c970cea5a 100644--- a/apps/start/src/modals/add-import.tsx+++ b/apps/start/src/modals/add-import.tsx@@ -17,6 +17,13 @@ import {zMixpanelImportConfig,zUmamiImportConfig,} from '@openpanel/validation';+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from '@/components/ui/select';import { format } from 'date-fns';import { CalendarIcon } from 'lucide-react';import { useFieldArray, useForm } from 'react-hook-form';@@ -214,6 +221,24 @@ function MixpanelImport({error={form.formState.errors.mapScreenViewProperty?.message}{...form.register('mapScreenViewProperty')}/>++ <WithLabel label="Data Residency" info="Select the Mixpanel data center your project uses">+ <Select+ value={form.watch('dataResidency') ?? 'us'}+ onValueChange={(value) =>+ form.setValue('dataResidency', value as 'us' | 'eu' | 'in')+ }+ >+ <SelectTrigger className="w-full">+ <SelectValue placeholder="US (default)" />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value="us">US (default)</SelectItem>+ <SelectItem value="eu">EU</SelectItem>+ <SelectItem value="in">India</SelectItem>+ </SelectContent>+ </Select>+ </WithLabel></div><div className="flex justify-between">diff --git a/packages/importer/src/providers/mixpanel.test.ts b/packages/importer/src/providers/mixpanel.test.tsindex 23de7bea5..31ee435d7 100644--- a/packages/importer/src/providers/mixpanel.test.ts+++ b/packages/importer/src/providers/mixpanel.test.ts@@ -317,4 +317,24 @@ describe('mixpanel', () => {groups: [],});});++ it('should accept dataResidency config', () => {+ const base = {+ from: '2025-01-01',+ to: '2025-01-02',+ serviceAccount: 'sa',+ serviceSecret: 'ss',+ projectId: '123',+ provider: 'mixpanel' as const,+ type: 'api' as const,+ };++ // All valid values should construct without throwing+ expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'us' })).not.toThrow();+ expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'eu' })).not.toThrow();+ expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'in' })).not.toThrow();++ // Undefined defaults to 'us' without throwing+ expect(() => new MixpanelProvider('pid', { ...base })).not.toThrow();+ });});diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/importer/src/providers/mixpanel.tsindex 99dafc177..b41ae0b25 100644--- a/packages/importer/src/providers/mixpanel.ts+++ b/packages/importer/src/providers/mixpanel.ts@@ -43,8 +43,27 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {private static readonly MAX_REQUESTS_PER_HOUR = 100;private static readonly MIN_REQUEST_INTERVAL_MS = 334; // 3 QPS limit+ private static readonly dataResidencyUrls: Record<+ string,+ { dataBase: string; apiBase: string }+ > = {+ us: {+ dataBase: 'https://data.mixpanel.com',+ apiBase: 'https://mixpanel.com',+ },+ eu: {+ dataBase: 'https://data-eu.mixpanel.com',+ apiBase: 'https://eu.mixpanel.com',+ },+ in: {+ dataBase: 'https://data-in.mixpanel.com',+ apiBase: 'https://in.mixpanel.com',+ },+ };+private requestTimestamps: number[] = [];private lastRequestTime = 0;+ private readonly residencyUrls: { dataBase: string; apiBase: string };constructor(private readonly projectId: string,@@ -52,6 +71,10 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {private readonly logger?: ILogger) {super();+ const residency = config.dataResidency ?? 'us';+ this.residencyUrls =+ MixpanelProvider.dataResidencyUrls[residency] ??+ MixpanelProvider.dataResidencyUrls.us!;}private async waitForRateLimit(): Promise<void> {@@ -200,7 +223,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {from: string,to: string): AsyncGenerator<MixpanelRawEvent, void, unknown> {- const url = 'https://data.mixpanel.com/api/2.0/export';+ const url = `${this.residencyUrls.dataBase}/api/2.0/export`;const params = new URLSearchParams({from_date: from,@@ -309,7 +332,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {while (true) {await this.waitForRateLimit();- const url = `https://mixpanel.com/api/query/engage?project_id=${encodeURIComponent(projectId)}`;+ const url = `${this.residencyUrls.apiBase}/api/query/engage?project_id=${encodeURIComponent(projectId)}`;const body = new URLSearchParams({page: String(page),page_size: String(pageSize),diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.tsindex 7d4ec29f5..18fc4b9f8 100644--- a/packages/validation/src/index.ts+++ b/packages/validation/src/index.ts@@ -627,6 +627,9 @@ export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;export const zPlausibleImportConfig = createFileImportConfig('plausible');export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;+export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);+export type IMixpanelDataResidency = z.infer<typeof zMixpanelDataResidency>;+export const zMixpanelImportConfig = z.object({provider: z.literal('mixpanel'),type: z.literal('api'),@@ -636,6 +639,7 @@ export const zMixpanelImportConfig = z.object({from: z.string().min(1),to: z.string().min(1),mapScreenViewProperty: z.string().optional(),+ dataResidency: zMixpanelDataResidency.optional(),});export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;