Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: amplitude importer

Carl-Gerhard Lindesvärdccbd2550f0b7
diff --git a/apps/start/src/modals/add-import.tsx b/apps/start/src/modals/add-import.tsx
index 03e718ea8..d2384304c 100644
--- a/apps/start/src/modals/add-import.tsx
+++ b/apps/start/src/modals/add-import.tsx
@@ -262,32 +262,35 @@ interface AmplitudeImportProps {
organizationId: string;
}
-function AmplitudeImport({
- onSubmit,
- isPending,
- organizationId,
-}: AmplitudeImportProps) {
- const trpc = useTRPC();
- const { data: projects = [] } = useQuery(
- trpc.project.list.queryOptions({
- organizationId,
- }),
- );
-
+function AmplitudeImport({ onSubmit, isPending }: AmplitudeImportProps) {
const form = useForm<AmplitudeFormData>({
resolver: zodResolver(zAmplitudeImportConfig),
defaultValues: {
provider: 'amplitude',
- type: 'file',
- fileUrl: '',
- projectMapper: [],
+ type: 'api',
+ apiKey: '',
+ secretKey: '',
+ from: '',
+ to: '',
},
});
- const { fields, append, remove } = useFieldArray({
- control: form.control,
- name: 'projectMapper',
- });
+ const handleDateRangeSelect = () => {
+ pushModal('DateRangerPicker', {
+ startDate: form.getValues('from')
+ ? new Date(form.getValues('from'))
+ : undefined,
+ endDate: form.getValues('to')
+ ? new Date(form.getValues('to'))
+ : undefined,
+ onChange: ({ startDate, endDate }) => {
+ form.setValue('from', format(startDate, 'yyyy-MM-dd'));
+ form.setValue('to', format(endDate, 'yyyy-MM-dd'));
+ form.trigger('from');
+ form.trigger('to');
+ },
+ });
+ };
const handleSubmit = form.handleSubmit((data) => {
onSubmit(data);
@@ -297,22 +300,78 @@ function AmplitudeImport({
<form onSubmit={handleSubmit} className="space-y-4">
<div className="space-y-4 py-4">
<InputWithLabel
- label="File URL"
- placeholder="https://example.com/export.json"
- error={form.formState.errors.fileUrl?.message}
- info="Provide a publicly accessible URL to your Amplitude export (newline-delimited JSON, optionally gzipped)."
- {...form.register('fileUrl')}
+ label="API Key"
+ placeholder="Your Amplitude API key"
+ error={form.formState.errors.apiKey?.message}
+ info="Settings → Projects → General in Amplitude."
+ {...form.register('apiKey')}
/>
- <ProjectMapper
- fields={fields}
- append={append}
- remove={remove}
- projects={projects}
- register={form.register}
- watch={form.watch}
- setValue={form.setValue}
+ <InputWithLabel
+ label="Secret Key"
+ type="password"
+ placeholder="Your Amplitude secret key"
+ error={form.formState.errors.secretKey?.message}
+ {...form.register('secretKey')}
+ />
+
+ <WithLabel
+ label="Date Range"
+ info={
+ !form.getValues('from') || !form.getValues('to')
+ ? 'Select the date range for importing data'
+ : undefined
+ }
+ >
+ <Button
+ type="button"
+ variant="outline"
+ className={cn(
+ 'w-full justify-start text-left font-normal',
+ (!form.getValues('from') || !form.getValues('to')) &&
+ 'text-muted-foreground',
+ )}
+ onClick={handleDateRangeSelect}
+ >
+ <CalendarIcon className="mr-2 h-4 w-4" />
+ {form.getValues('from') && form.getValues('to') ? (
+ <>
+ {format(new Date(form.getValues('from')), 'LLL dd, y')} -{' '}
+ {format(new Date(form.getValues('to')), 'LLL dd, y')}
+ </>
+ ) : (
+ <span>Pick a date range</span>
+ )}
+ </Button>
+ </WithLabel>
+
+ <InputWithLabel
+ label="Screen View Property"
+ placeholder="Enter the name of the property that contains the screen name"
+ info="Leave empty if not applicable"
+ error={form.formState.errors.mapScreenViewProperty?.message}
+ {...form.register('mapScreenViewProperty')}
/>
+
+ <WithLabel
+ label="Data Residency"
+ info="Select the Amplitude data region your project uses"
+ >
+ <Select
+ value={form.watch('dataResidency') ?? 'us'}
+ onValueChange={(value) =>
+ form.setValue('dataResidency', value as 'us' | 'eu')
+ }
+ >
+ <SelectTrigger className="w-full">
+ <SelectValue placeholder="US (default)" />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="us">US (default)</SelectItem>
+ <SelectItem value="eu">EU</SelectItem>
+ </SelectContent>
+ </Select>
+ </WithLabel>
</div>
<div className="flex justify-between">
diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.ts
index e2e79b3c5..ac40a244a 100644
--- a/apps/worker/src/jobs/import.ts
+++ b/apps/worker/src/jobs/import.ts
@@ -34,6 +34,40 @@ function yieldToEventLoop(): Promise<void> {
const RESUMABLE_STEPS = ['creating_sessions', 'moving', 'backfilling_sessions'];
+/**
+ * Merge a freshly-derived profile into the bounded dedup map: keep the earliest
+ * created_at (first seen), advance last_seen_at to the latest activity, and fill
+ * identity fields/properties, preferring the newer row.
+ */
+function mergeProfileInto(
+ map: Map<string, IClickhouseProfile>,
+ incoming: IClickhouseProfile
+): void {
+ const existing = map.get(incoming.id);
+ if (!existing) {
+ map.set(incoming.id, incoming);
+ return;
+ }
+
+ const createdAt =
+ existing.created_at < incoming.created_at
+ ? existing.created_at
+ : incoming.created_at;
+ const isIncomingNewer = incoming.last_seen_at >= existing.last_seen_at;
+ const base = isIncomingNewer ? incoming : existing;
+ const other = isIncomingNewer ? existing : incoming;
+
+ map.set(incoming.id, {
+ ...base,
+ created_at: createdAt,
+ first_name: base.first_name || other.first_name,
+ last_name: base.last_name || other.last_name,
+ email: base.email || other.email,
+ avatar: base.avatar || other.avatar,
+ properties: { ...other.properties, ...base.properties },
+ });
+}
+
export async function importJob(job: Job<ImportQueuePayload>) {
const { importId } = job.data.payload;
@@ -73,6 +107,33 @@ export async function importJob(job: Job<ImportQueuePayload>) {
let processedEvents = 0;
const eventBatch: IClickhouseEvent[] = [];
+ // Profiles derived inline from events (e.g. Amplitude, which has no
+ // profile export API). A bounded dedup map keeps memory flat regardless of
+ // event volume; the profiles table's ReplacingMergeTree(last_seen_at)
+ // collapses any duplicate rows that span flush boundaries to the latest
+ // activity per id.
+ const canProfileFromEvents =
+ typeof (providerInstance as AmplitudeProvider)
+ .transformEventToProfile === 'function';
+ const PROFILE_MAP_CAP = 50_000;
+ const profileMap = new Map<string, IClickhouseProfile>();
+ let processedProfiles = 0;
+
+ const flushProfiles = async () => {
+ if (profileMap.size === 0) {
+ return;
+ }
+ const values = Array.from(profileMap.values());
+ await insertProfilesBatch(values, record.projectId);
+ processedProfiles += values.length;
+ profileMap.clear();
+ await updateImportStatus(jobLogger, job, importId, {
+ step: 'loading_profiles',
+ processedProfiles,
+ });
+ await yieldToEventLoop();
+ };
+
for await (const rawEvent of providerInstance.parseSource()) {
if (
!providerInstance.validate(
@@ -93,6 +154,21 @@ export async function importJob(job: Job<ImportQueuePayload>) {
// in generateGapBasedSessionIds after loading, using gap-based logic.
eventBatch.push(transformed);
+ if (canProfileFromEvents) {
+ const profile = (
+ providerInstance as AmplitudeProvider
+ ).transformEventToProfile(
+ // @ts-expect-error -- provider-specific raw type
+ rawEvent
+ );
+ if (profile) {
+ mergeProfileInto(profileMap, profile);
+ if (profileMap.size >= PROFILE_MAP_CAP) {
+ await flushProfiles();
+ }
+ }
+ }
+
if (eventBatch.length >= BATCH_SIZE) {
await insertImportBatch(eventBatch, importId);
processedEvents += eventBatch.length;
@@ -132,6 +208,15 @@ export async function importJob(job: Job<ImportQueuePayload>) {
jobLogger.info({ processedEvents }, 'Loading complete');
+ // Phase 1a: Flush profiles derived inline from events (Amplitude)
+ if (canProfileFromEvents) {
+ await flushProfiles();
+ jobLogger.info(
+ { processedProfiles },
+ 'Inline profile derivation complete'
+ );
+ }
+
// Phase 1b: Load user profiles (Mixpanel only)
const profileBatchSize = 5000;
if (
diff --git a/packages/importer/package.json b/packages/importer/package.json
index ff91d681a..c7fe3eeb6 100644
--- a/packages/importer/package.json
+++ b/packages/importer/package.json
@@ -20,6 +20,7 @@
"@openpanel/validation": "workspace:*",
"csv-parse": "^6.1.0",
"ramda": "^0.29.1",
+ "unzipper": "^0.12.3",
"uuid": "^9.0.1",
"zod": "catalog:"
},
@@ -27,6 +28,7 @@
"@openpanel/logger": "workspace:*",
"@types/node": "^20.0.0",
"@types/ramda": "^0.31.1",
+ "@types/unzipper": "^0.10.10",
"@types/uuid": "^9.0.7",
"bullmq": "^5.8.7",
"typescript": "^5.0.0",
diff --git a/packages/importer/src/providers/amplitude.test.ts b/packages/importer/src/providers/amplitude.test.ts
index 0702a3826..c50541df7 100644
--- a/packages/importer/src/providers/amplitude.test.ts
+++ b/packages/importer/src/providers/amplitude.test.ts
@@ -18,9 +18,11 @@ async function collect(
const baseConfig = {
provider: 'amplitude' as const,
- type: 'file' as const,
- fileUrl: 'https://example.com/export.json',
- projectMapper: [],
+ type: 'api' as const,
+ apiKey: 'test-api-key',
+ secretKey: 'test-secret-key',
+ from: '2025-02-01',
+ to: '2025-02-28',
};
function loadFixture(): AmplitudeRawEvent[] {
@@ -137,7 +139,10 @@ describe('amplitude', () => {
)}\n`;
const events = await collect(
- provider.streamNdjson(Readable.from(Buffer.from(ndjson)), baseConfig.fileUrl)
+ provider.streamNdjson(
+ Readable.from(Buffer.from(ndjson)),
+ 'https://example.com/export.json'
+ )
);
expect(events.map((e) => e.event_type)).toEqual(['A', 'B']);
@@ -171,6 +176,65 @@ describe('amplitude', () => {
expect(events.map((e) => e.event_type)).toEqual(['A']);
});
+ it('derives a profile from an event with a user_id', () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+ const profile = provider.transformEventToProfile({
+ event_type: 'Custom',
+ event_properties: {},
+ user_properties: {
+ email: '[email protected]',
+ firstName: 'Jane',
+ last_name: 'Doe',
+ plan: 'pro',
+ nested: { a: 1 },
+ },
+ device_id: 'd1',
+ user_id: 'user-001',
+ event_time: '2025-02-05 22:58:30.568000',
+ } as AmplitudeRawEvent);
+
+ expect(profile).not.toBeNull();
+ expect(profile).toMatchObject({
+ id: 'user-001',
+ project_id: 'pid',
+ email: '[email protected]',
+ first_name: 'Jane',
+ last_name: 'Doe',
+ is_external: true,
+ created_at: '2025-02-05 22:58:30',
+ last_seen_at: '2025-02-05 22:58:30',
+ properties: { plan: 'pro', nested: '{"a":1}' },
+ });
+ // known identity keys are not duplicated into the properties bag
+ expect(profile?.properties.email).toBeUndefined();
+ expect(profile?.properties.firstName).toBeUndefined();
+ });
+
+ it('returns null for anonymous events (no user_id or user_id === device_id)', () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+
+ expect(
+ provider.transformEventToProfile({
+ event_type: 'Custom',
+ event_properties: {},
+ user_properties: {},
+ device_id: 'd1',
+ event_time: '2025-02-05 22:58:30.568000',
+ } as AmplitudeRawEvent)
+ ).toBeNull();
+
+ expect(
+ provider.transformEventToProfile({
+ event_type: 'Custom',
+ event_properties: {},
+ user_properties: {},
+ device_id: 'd1',
+ user_id: 'd1',
+ event_time: '2025-02-05 22:58:30.568000',
+ } as AmplitudeRawEvent)
+ ).toBeNull();
+ });
+
it('maps Amplitude country names to ISO alpha-2 (FixedString(2) safe)', () => {
expect(toCountryCode('Nigeria')).toBe('NG');
expect(toCountryCode('United States')).toBe('US');
diff --git a/packages/importer/src/providers/amplitude.ts b/packages/importer/src/providers/amplitude.ts
index 94d4ab7fd..ff9a837ab 100644
--- a/packages/importer/src/providers/amplitude.ts
+++ b/packages/importer/src/providers/amplitude.ts
@@ -7,9 +7,14 @@ import {
getReferrerWithQuery,
parseReferrer,
} from '@openpanel/common/server';
-import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';
+import {
+ formatClickhouseDate,
+ type IClickhouseEvent,
+ type IClickhouseProfile,
+} from '@openpanel/db';
import type { ILogger } from '@openpanel/logger';
import type { IAmplitudeImportConfig } from '@openpanel/validation';
+import unzipper from 'unzipper';
import { z } from 'zod';
import { BaseImportProvider } from '../base-provider';
import { toCountryCode } from './country-codes';
@@ -61,12 +66,27 @@ export class AmplitudeProvider extends BaseImportProvider<AmplitudeRawEvent> {
provider = 'amplitude';
version = '1.0.0';
+ /**
+ * Amplitude Export API base URLs by data region. The export endpoint lives at
+ * `${base}/api/2/export`. See https://amplitude.com/docs/apis/analytics/export
+ */
+ private static readonly dataResidencyUrls: Record<string, string> = {
+ us: 'https://amplitude.com',
+ eu: 'https://analytics.eu.amplitude.com',
+ };
+
+ private readonly exportBaseUrl: string;
+
constructor(
private readonly projectId: string,
private readonly config: IAmplitudeImportConfig,
private readonly logger?: ILogger
) {
super();
+ const residency = config.dataResidency ?? 'us';
+ this.exportBaseUrl =
+ AmplitudeProvider.dataResidencyUrls[residency] ??
+ AmplitudeProvider.dataResidencyUrls.us!;
}
async getTotalEventsCount(): Promise<number> {
@@ -82,37 +102,137 @@ export class AmplitudeProvider extends BaseImportProvider<AmplitudeRawEvent> {
return true;
}
- async *parseSource(): AsyncGenerator<AmplitudeRawEvent, void, unknown> {
- yield* this.parseRemoteFile(this.config.fileUrl);
+ /**
+ * Pull events from the Amplitude Export API. The date range is split into
+ * daily chunks (Export API caps a single request at ~4GB) and each chunk is
+ * downloaded as a ZIP of gzipped newline-delimited JSON files.
+ */
+ async *parseSource(
+ overrideFrom?: string
+ ): AsyncGenerator<AmplitudeRawEvent, void, unknown> {
+ const { from, to } = this.config;
+ const dateChunks = this.getDateChunks(overrideFrom ?? from, to);
+
+ for (const [chunkFrom, chunkTo] of dateChunks) {
+ // Amplitude expects YYYYMMDDTHH (hour inclusive on both ends).
+ const start = `${chunkFrom.replaceAll('-', '')}T00`;
+ const end = `${chunkTo.replaceAll('-', '')}T23`;
+
+ const response = await this.fetchExportWithRetry(start, end);
+ if (!response) {
+ // 404 → no data in this window, skip it.
+ continue;
+ }
+
+ yield* this.streamExportZip(response, `${start}-${end}`);
+ }
}
/**
- * Stream a remote newline-delimited JSON file.
- *
- * Note: we do NOT inspect the Content-Encoding/Content-Type headers to decide
- * whether to decompress. fetch/undici already transparently decompresses
- * transport-level gzip/br (e.g. a dev server that gzips responses on the fly),
- * yet leaves the Content-Encoding header in place. Trusting it would gunzip
- * already-plain JSON and throw "incorrect header check". Instead we detect a
- * genuinely-gzipped file by its magic bytes (see streamNdjson).
+ * Fetch a single export window with retries. Returns the response on success,
+ * or null when Amplitude reports no data (404) for the window. Throws after
+ * exhausting retries for retryable failures (429 / 5xx / network).
*/
- private async *parseRemoteFile(
- url: string
- ): AsyncGenerator<AmplitudeRawEvent, void, unknown> {
- const controller = new AbortController();
- const res = await fetch(url, { signal: controller.signal });
- if (!(res.ok && res.body)) {
- throw new Error(
- `Failed to fetch remote file: ${res.status} ${res.statusText}`
+ private async fetchExportWithRetry(
+ start: string,
+ end: string
+ ): Promise<Response | null> {
+ const url = `${this.exportBaseUrl}/api/2/export?start=${start}&end=${end}`;
+ const auth = Buffer.from(
+ `${this.config.apiKey}:${this.config.secretKey}`
+ ).toString('base64');
+ const maxRetries = 6;
+ let retries = 0;
+
+ while (true) {
+ let response: Response;
+ try {
+ this.logger?.info({ url, start, end }, 'Fetching Amplitude export');
+ response = await fetch(url, {
+ headers: { Authorization: `Basic ${auth}` },
+ });
+ } catch (error) {
+ retries++;
+ if (retries > maxRetries) {
+ throw new Error(
+ `Failed to fetch Amplitude export for ${start}-${end} after ${maxRetries} retries: ${(error as Error).message}`
+ );
+ }
+ const delay = Math.min(1000 * 2 ** (retries - 1), 60_000);
+ this.logger?.warn(
+ { err: error, start, end, attempt: retries, delay },
+ 'Network error fetching Amplitude export, retrying'
+ );
+ await new Promise((resolve) => setTimeout(resolve, delay));
+ continue;
+ }
+
+ // No data available for this window.
+ if (response.status === 404) {
+ this.logger?.info(
+ { start, end },
+ 'No Amplitude data for window, skipping'
+ );
+ return null;
+ }
+
+ if (response.ok) {
+ if (!response.body) {
+ throw new Error('No response body from Amplitude export');
+ }
+ return response;
+ }
+
+ const retryable = response.status === 429 || response.status >= 500;
+ retries++;
+ if (!retryable || retries > maxRetries) {
+ const text = await response.text().catch(() => '');
+ throw new Error(
+ `Failed to fetch Amplitude export for ${start}-${end}: ${response.status} ${response.statusText} ${text.slice(0, 300)}`
+ );
+ }
+
+ let delay: number;
+ if (response.status === 429) {
+ const retryAfter = response.headers.get('Retry-After');
+ delay = retryAfter
+ ? Number(retryAfter) * 1000
+ : Math.min(60_000 * 2 ** (retries - 1), 300_000);
+ } else {
+ delay = Math.min(1000 * 2 ** (retries - 1), 60_000);
+ }
+ this.logger?.warn(
+ { status: response.status, start, end, attempt: retries, delay },
+ 'Retryable error fetching Amplitude export, retrying'
);
+ await new Promise((resolve) => setTimeout(resolve, delay));
}
+ }
- const body = Readable.fromWeb(res.body as any);
- try {
- yield* this.streamNdjson(body, url);
- } finally {
- controller.abort(); // tear down the fetch stream
+ /**
+ * Stream an Amplitude export ZIP. Each entry is a gzipped newline-delimited
+ * JSON file; we feed each one through streamNdjson (which sniffs gzip magic
+ * bytes) and yield its events. Non-matching entries are drained so the ZIP
+ * parser can advance.
+ */
+ private async *streamExportZip(
+ response: Response,
+ window: string
+ ): AsyncGenerator<AmplitudeRawEvent, void, unknown> {
+ const body = Readable.fromWeb(response.body as any);
+ const zip = body.pipe(unzipper.Parse({ forceStream: true }));
+
+ for await (const entry of zip) {
+ const isDataFile =
+ entry.type === 'File' && /\.json(\.gz)?$/i.test(entry.path);
+ if (isDataFile) {
+ yield* this.streamNdjson(entry as unknown as Readable, entry.path);
+ } else {
+ entry.autodrain();
+ }
}
+
+ this.logger?.info({ window }, 'Finished streaming Amplitude export window');
}
/**
@@ -275,6 +395,77 @@ export class AmplitudeProvider extends BaseImportProvider<AmplitudeRawEvent> {
};
}
+ /**
+ * Derive an OpenPanel profile from a single Amplitude event. Amplitude has no
+ * profile export API, so identity lives inside each event's user_properties.
+ * Returns null for anonymous events (no user_id, or user_id === device_id) so
+ * we don't create profiles for purely-device-scoped traffic.
+ *
+ * The caller (import worker) dedupes these in a bounded map and relies on the
+ * profiles table's ReplacingMergeTree(last_seen_at) to keep the latest row per
+ * id, so this method stays stateless.
+ */
+ transformEventToProfile(
+ _rawEvent: AmplitudeRawEvent
+ ): IClickhouseProfile | null {
+ const raw = zAmplitudeRawEvent.parse(_rawEvent);
+ const deviceId = raw.device_id || String(raw.amplitude_id ?? '');
+ const userId = raw.user_id;
+ if (!userId || userId === deviceId) {
+ return null;
+ }
+
+ const up = (raw.user_properties ?? {}) as Record<string, unknown>;
+ const timestamp = formatClickhouseDate(
+ this.parseTimestamp(raw.event_time || raw.client_event_time)
+ );
+
+ const pick = (...keys: string[]): string => {
+ for (const key of keys) {
+ const value = up[key];
+ if (value != null && value !== '') {
+ return String(value);
+ }
+ }
+ return '';
+ };
+
+ const known = new Set([
+ 'email',
+ '$email',
+ 'first_name',
+ 'firstName',
+ '$first_name',
+ 'last_name',
+ 'lastName',
+ '$last_name',
+ 'avatar',
+ '$avatar',
+ ]);
+ const properties: Record<string, string> = {};
+ for (const [key, value] of Object.entries(up)) {
+ if (known.has(key) || value == null) {
+ continue;
+ }
+ properties[key] =
+ typeof value === 'object' ? JSON.stringify(value) : String(value);
+ }
+
+ return {
+ id: String(userId),
+ project_id: this.projectId,
+ first_name: pick('first_name', 'firstName', '$first_name'),
+ last_name: pick('last_name', 'lastName', '$last_name'),
+ email: pick('email', '$email'),
+ avatar: pick('avatar', '$avatar'),
+ properties,
+ created_at: timestamp,
+ last_seen_at: timestamp,
+ is_external: true,
+ groups: [],
+ };
+ }
+
/**
* Map Amplitude system event names to OpenPanel conventions. Custom event
* names pass through unchanged.
diff --git a/packages/importer/src/providers/metadata.ts b/packages/importer/src/providers/metadata.ts
index 5f81c7c90..578e3fa30 100644
--- a/packages/importer/src/providers/metadata.ts
+++ b/packages/importer/src/providers/metadata.ts
@@ -30,9 +30,9 @@ export const IMPORT_PROVIDERS: ImportProviderMeta[] = [
{
id: 'amplitude',
name: 'Amplitude',
- description: 'Import your analytics data from an Amplitude export',
+ description: 'Import your analytics data from the Amplitude Export API',
logo: 'https://cdn.brandfetch.io/idnYrZGER0/theme/dark/symbol.svg?c=1dxbfHSJFAPEGdCLU4o5B',
backgroundColor: '#fff',
- types: ['file'],
+ types: ['api'],
},
];
diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts
index 3392561e3..bd1c7a45e 100644
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -673,12 +673,19 @@ export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;
export const zPlausibleImportConfig = createFileImportConfig('plausible');
export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;
-export const zAmplitudeImportConfig = createFileImportConfig('amplitude').extend(
- {
- projectMapper: z.array(zProjectMapper),
- mapScreenViewProperty: z.string().optional(),
- },
-);
+export const zAmplitudeDataResidency = z.enum(['us', 'eu']);
+export type IAmplitudeDataResidency = z.infer<typeof zAmplitudeDataResidency>;
+
+export const zAmplitudeImportConfig = z.object({
+ provider: z.literal('amplitude'),
+ type: z.literal('api'),
+ apiKey: z.string().min(1),
+ secretKey: z.string().min(1),
+ from: z.string().min(1),
+ to: z.string().min(1),
+ mapScreenViewProperty: z.string().optional(),
+ dataResidency: zAmplitudeDataResidency.optional(),
+});
export type IAmplitudeImportConfig = z.infer<typeof zAmplitudeImportConfig>;
export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index e85fa9f8e..d0048263b 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1419,6 +1419,9 @@ importers:
ramda:
specifier: ^0.29.1
version: 0.29.1
+ unzipper:
+ specifier: ^0.12.3
+ version: 0.12.5
uuid:
specifier: ^9.0.1
version: 9.0.1
@@ -1435,6 +1438,9 @@ importers:
'@types/ramda':
specifier: ^0.31.1
version: 0.31.1
+ '@types/unzipper':
+ specifier: ^0.10.10
+ version: 0.10.11
'@types/uuid':
specifier: ^9.0.7
version: 9.0.8
@@ -9941,6 +9947,9 @@ packages:
resolution: {integrity: sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-D25im2zjyMCcgL9ag6N46+wbtJBnXIr7SI4zHf9eJD2Dw2tEB5e+p5MYkrxKIVRscs5QV0EhtU9rgXSPx90oJg==}
+
resolution: {integrity: sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==}
@@ -10702,6 +10711,9 @@ packages:
resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==}
+ resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
+
resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==}
@@ -12006,6 +12018,9 @@ packages:
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==}
+
resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==}
@@ -12835,6 +12850,10 @@ packages:
resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
+ resolution: {integrity: sha512-eXvGGwZ5CL17ZSwHWd3bbgk7UUpF6IFHtP57NYYakPvHOs8GDgDe5KJI36jIJzDkJ6eJjuzRA8eBQb6SkKue0g==}
+ engines: {node: '>=14.14'}
+
resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==}
engines: {node: '>=6 <7 || >=8'}
@@ -18845,6 +18864,9 @@ packages:
resolution: {integrity: sha512-Vhp5gb1tusSQw5of/g3Q697srYgMXvwMgXMjcG4ZNga02fDX9coxJ9fAb0Ci38hM2Hv/U1FXRPGgjP2BYqhNoQ==}
+ resolution: {integrity: sha512-tXYOi9R57Uj/2Z25SOs5RRSzq886MBQj2gY8dPL+xl/kv6s6SvByoKfAtvfVeEuhntWDgjd2o9p2lb4TVPAz0A==}
+
resolution: {integrity: sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==}
hasBin: true
@@ -30559,6 +30581,10 @@ snapshots:
'@types/[email protected]': {}
+ '@types/[email protected]':
+ dependencies:
+ '@types/node': 20.19.24
+
'@types/[email protected]': {}
'@types/[email protected]': {}
@@ -31663,6 +31689,8 @@ snapshots:
+
@@ -33088,6 +33116,10 @@ snapshots:
es-errors: 1.3.0
gopd: 1.2.0
+ dependencies:
+ readable-stream: 2.3.8
+
@@ -34454,6 +34486,12 @@ snapshots:
+ dependencies:
+ graceful-fs: 4.2.11
+ jsonfile: 6.2.0
+ universalify: 2.0.1
+
dependencies:
graceful-fs: 4.2.11
@@ -41873,6 +41911,14 @@ snapshots:
pkg-types: 2.3.0
unplugin: 2.3.10
+ dependencies:
+ bluebird: 3.7.2
+ duplexer2: 0.1.4
+ fs-extra: 11.3.1
+ graceful-fs: 4.2.11
+ node-int64: 0.4.0
+
dependencies:
browserslist: 4.28.0