Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

feat: add amplitude importer

Carl-Gerhard Lindesvärdf8a79da629b3
diff --git a/apps/start/src/modals/add-import.tsx b/apps/start/src/modals/add-import.tsx
index c970cea5a..03e718ea8 100644
--- a/apps/start/src/modals/add-import.tsx
+++ b/apps/start/src/modals/add-import.tsx
@@ -9,11 +9,13 @@ import { useTRPC } from '@/integrations/trpc/react';
import { cn } from '@/lib/utils';
import { zodResolver } from '@hookform/resolvers/zod';
import type {
+ IAmplitudeImportConfig,
IImportConfig,
IMixpanelImportConfig,
IUmamiImportConfig,
} from '@openpanel/validation';
import {
+ zAmplitudeImportConfig,
zMixpanelImportConfig,
zUmamiImportConfig,
} from '@openpanel/validation';
@@ -31,7 +33,7 @@ import type { z } from 'zod';
import { popModal, pushModal } from '.';
import { ModalContent, ModalHeader } from './Modal/Container';
-type Provider = 'umami' | 'plausible' | 'mixpanel';
+type Provider = 'umami' | 'plausible' | 'mixpanel' | 'amplitude';
interface AddImportProps {
provider: Provider;
@@ -41,6 +43,7 @@ interface AddImportProps {
type UmamiFormData = z.infer<typeof zUmamiImportConfig>;
type MixpanelFormData = z.infer<typeof zMixpanelImportConfig>;
+type AmplitudeFormData = z.infer<typeof zAmplitudeImportConfig>;
interface UmamiImportProps {
onSubmit: (config: IUmamiImportConfig) => void;
@@ -253,6 +256,77 @@ function MixpanelImport({
);
}
+interface AmplitudeImportProps {
+ onSubmit: (config: IAmplitudeImportConfig) => void;
+ isPending: boolean;
+ organizationId: string;
+}
+
+function AmplitudeImport({
+ onSubmit,
+ isPending,
+ organizationId,
+}: AmplitudeImportProps) {
+ const trpc = useTRPC();
+ const { data: projects = [] } = useQuery(
+ trpc.project.list.queryOptions({
+ organizationId,
+ }),
+ );
+
+ const form = useForm<AmplitudeFormData>({
+ resolver: zodResolver(zAmplitudeImportConfig),
+ defaultValues: {
+ provider: 'amplitude',
+ type: 'file',
+ fileUrl: '',
+ projectMapper: [],
+ },
+ });
+
+ const { fields, append, remove } = useFieldArray({
+ control: form.control,
+ name: 'projectMapper',
+ });
+
+ const handleSubmit = form.handleSubmit((data) => {
+ onSubmit(data);
+ });
+
+ return (
+ <form onSubmit={handleSubmit} className="space-y-4">
+ <div className="space-y-4 py-4">
+ <InputWithLabel
+ label="File URL"
+ placeholder="https://example.com/export.json"
+ error={form.formState.errors.fileUrl?.message}
+ info="Provide a publicly accessible URL to your Amplitude export (newline-delimited JSON, optionally gzipped)."
+ {...form.register('fileUrl')}
+ />
+
+ <ProjectMapper
+ fields={fields}
+ append={append}
+ remove={remove}
+ projects={projects}
+ register={form.register}
+ watch={form.watch}
+ setValue={form.setValue}
+ />
+ </div>
+
+ <div className="flex justify-between">
+ <Button type="button" variant="outline" onClick={() => popModal()}>
+ Cancel
+ </Button>
+ <Button type="submit" disabled={isPending}>
+ {isPending ? 'Starting...' : 'Start Import'}
+ </Button>
+ </div>
+ </form>
+ );
+}
+
export default function AddImport({ provider, name }: AddImportProps) {
const { projectId, organizationId } = useAppParams();
const trpc = useTRPC();
@@ -302,6 +376,14 @@ export default function AddImport({ provider, name }: AddImportProps) {
organizationId={organizationId}
/>
)}
+
+ {provider === 'amplitude' && (
+ <AmplitudeImport
+ onSubmit={handleImportSubmit}
+ isPending={createImport.isPending}
+ organizationId={organizationId}
+ />
+ )}
</ModalContent>
);
}
diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.ts
index 3676958f7..e2e79b3c5 100644
--- a/apps/worker/src/jobs/import.ts
+++ b/apps/worker/src/jobs/import.ts
@@ -14,7 +14,11 @@ import {
type Prisma,
updateImportStatus,
} from '@openpanel/db';
-import { MixpanelProvider, UmamiProvider } from '@openpanel/importer';
+import {
+ AmplitudeProvider,
+ MixpanelProvider,
+ UmamiProvider,
+} from '@openpanel/importer';
import type { ILogger } from '@openpanel/logger';
import type { ImportQueuePayload } from '@openpanel/queue';
import type { Job } from 'bullmq';
@@ -298,6 +302,8 @@ function createProvider(
return new UmamiProvider(record.projectId, config, jobLogger);
case 'mixpanel':
return new MixpanelProvider(record.projectId, config, jobLogger);
+ case 'amplitude':
+ return new AmplitudeProvider(record.projectId, config, jobLogger);
default:
throw new Error(`Unknown provider: ${config.provider}`);
}
diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts
index 4a393c4d7..b31c5bafa 100644
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -487,7 +487,7 @@ export async function getEventList(options: GetEventListOptions) {
} = options;
const { sb, getSql, join } = createSqlBuilder();
- const MAX_DATE_INTERVAL_IN_DAYS = 365;
+ const MAX_DATE_INTERVAL_IN_DAYS = 365 * 5;
// Cap the date interval to prevent infinity
const safeDateIntervalInDays = Math.min(
dateIntervalInDays,
@@ -1297,7 +1297,7 @@ export interface QueryEventsInput {
}
export async function queryEventsCore(
- input: QueryEventsInput,
+ input: QueryEventsInput
): Promise<IClickhouseEvent[]> {
const builder = clix(ch)
.select<IClickhouseEvent>([])
@@ -1358,7 +1358,9 @@ export async function queryEventsCore(
if (input.properties) {
for (const [key, value] of Object.entries(input.properties)) {
- builder.rawWhere(`properties[${sqlstring.escape(key)}] = ${sqlstring.escape(value)}`);
+ builder.rawWhere(
+ `properties[${sqlstring.escape(key)}] = ${sqlstring.escape(value)}`
+ );
}
}
@@ -1368,7 +1370,7 @@ export async function queryEventsCore(
if (!input.sessionId) {
const { startDate: start, endDate: end } = resolveDateRange(
input.startDate,
- input.endDate,
+ input.endDate
);
builder.where('created_at', 'BETWEEN', [
clix.datetime(start),
@@ -1378,7 +1380,7 @@ export async function queryEventsCore(
// If caller still wants to scope by date, honor it.
const { startDate: start, endDate: end } = resolveDateRange(
input.startDate,
- input.endDate,
+ input.endDate
);
builder.where('created_at', 'BETWEEN', [
clix.datetime(start),
diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.ts
index 52f4e80df..61bb8d9cc 100644
--- a/packages/importer/src/index.ts
+++ b/packages/importer/src/index.ts
@@ -1,5 +1,6 @@
export { UmamiProvider } from './providers/umami';
export { MixpanelProvider } from './providers/mixpanel';
+export { AmplitudeProvider } from './providers/amplitude';
export type {
ImportConfig,
ImportProgress,
diff --git a/packages/importer/src/providers/__fixtures__/amplitude-export.json b/packages/importer/src/providers/__fixtures__/amplitude-export.json
new file mode 100644
index 000000000..117eece99
--- /dev/null
+++ b/packages/importer/src/providers/__fixtures__/amplitude-export.json
@@ -0,0 +1,7 @@
+{"$insert_id":"00000000-0000-4000-8000-000000000001","amplitude_id":100000000001,"app":111111,"city":"Springfield","client_event_time":"2025-02-05 22:57:31.078000","client_upload_time":"2025-02-05 22:58:14.311000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"AAAAAAAA-0000-0000-0000-000000000001","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone 12 Mini","event_id":1,"event_properties":{"userId":"user-001"},"event_time":"2025-02-05 22:57:31.078000","event_type":"LoggedIn","ip_address":"192.0.2.1","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"17.6.1","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-001","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a1","version_name":"201"}
+{"$insert_id":"00000000-0000-4000-8000-000000000002","amplitude_id":100000000001,"app":111111,"city":"Springfield","client_event_time":"2025-02-05 22:57:31.148000","client_upload_time":"2025-02-05 22:58:13.930000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"AAAAAAAA-0000-0000-0000-000000000001","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone 12 Mini","event_id":2,"event_properties":{"page":"LoginForm with password","email":"[email protected]"},"event_time":"2025-02-05 22:57:31.148000","event_type":"User logged in","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"17.6.1","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-001","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a2","version_name":"201"}
+{"$insert_id":"00000000-0000-4000-8000-000000000003","amplitude_id":100000000002,"app":111111,"city":"Rivertown","client_event_time":"2025-02-05 23:01:10.000000","client_upload_time":"2025-02-05 23:01:20.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"BBBBBBBB-0000-0000-0000-000000000002","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone 15 Pro Max","event_id":3,"event_properties":{"cardType":"virtual","section":"wallet"},"event_time":"2025-02-05 23:01:10.000000","event_type":"Get Card","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"18.1.0","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-002","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a3","version_name":"201"}
+{"$insert_id":"00000000-0000-4000-8000-000000000004","amplitude_id":100000000002,"app":111111,"city":"Rivertown","client_event_time":"2025-02-05 23:05:00.000000","client_upload_time":"2025-02-05 23:05:10.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"BBBBBBBB-0000-0000-0000-000000000002","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone 15 Pro Max","event_id":4,"event_properties":{"from_background":true,"build":"5","version":"201"},"event_time":"2025-02-05 23:05:00.000000","event_type":"Application Backgrounded","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"18.1.0","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-002","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a4","version_name":"201"}
+{"$insert_id":"00000000-0000-4000-8000-000000000005","amplitude_id":100000000003,"app":111111,"city":"Laketown","client_event_time":"2025-02-05 23:10:00.000000","client_upload_time":"2025-02-05 23:10:10.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"CCCCCCCC-0000-0000-0000-000000000003","device_manufacturer":null,"device_model":null,"device_type":"iOS Simulator","event_id":5,"event_properties":{"from_background":false,"build":"5","version":"201"},"event_time":"2025-02-05 23:10:00.000000","event_type":"Application Opened","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"17.6.1","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-003","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a5","version_name":"201"}
+{"$insert_id":"00000000-0000-4000-8000-000000000006","amplitude_id":100000000003,"app":111111,"city":"Laketown","client_event_time":"2025-02-05 23:11:00.000000","client_upload_time":"2025-02-05 23:11:10.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"CCCCCCCC-0000-0000-0000-000000000003","device_manufacturer":null,"device_model":null,"device_type":"iOS Simulator","event_id":6,"event_properties":{},"event_time":"2025-02-05 23:11:00.000000","event_type":"Application Installed","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"17.6.1","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-003","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a6","version_name":"201"}
+{"$insert_id":"00000000-0000-4000-8000-000000000007","amplitude_id":100000000004,"app":111111,"city":"Hilltown","client_event_time":"2025-02-05 23:20:00.000000","client_upload_time":"2025-02-05 23:20:10.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"DDDDDDDD-0000-0000-0000-000000000004","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone SE (3rd generation)","event_id":7,"event_properties":{"section":"signup"},"event_time":"2025-02-05 23:20:00.000000","event_type":"Create Account Button","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"16.4.0","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-004","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a7","version_name":"201"}
diff --git a/packages/importer/src/providers/amplitude.test.ts b/packages/importer/src/providers/amplitude.test.ts
new file mode 100644
index 000000000..0702a3826
--- /dev/null
+++ b/packages/importer/src/providers/amplitude.test.ts
@@ -0,0 +1,206 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { gzipSync } from 'node:zlib';
+import { describe, expect, it } from 'vitest';
+import { AmplitudeProvider, type AmplitudeRawEvent } from './amplitude';
+import { toCountryCode } from './country-codes';
+
+async function collect(
+ gen: AsyncGenerator<AmplitudeRawEvent, void, unknown>
+): Promise<AmplitudeRawEvent[]> {
+ const out: AmplitudeRawEvent[] = [];
+ for await (const event of gen) {
+ out.push(event);
+ }
+ return out;
+}
+
+const baseConfig = {
+ provider: 'amplitude' as const,
+ type: 'file' as const,
+ fileUrl: 'https://example.com/export.json',
+ projectMapper: [],
+};
+
+function loadFixture(): AmplitudeRawEvent[] {
+ const raw = readFileSync(
+ join(__dirname, '__fixtures__', 'amplitude-export.json'),
+ 'utf-8'
+ );
+ return raw
+ .split('\n')
+ .map((line) => line.trim())
+ .filter(Boolean)
+ .map((line) => JSON.parse(line) as AmplitudeRawEvent);
+}
+
+describe('amplitude', () => {
+ it('generates session IDs in SQL (Segment exports have none)', () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+ expect(provider.shouldGenerateSessionIds()).toBe(true);
+ });
+
+ it('transforms a mobile (Segment iOS) event', () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+
+ const rawEvent = {
+ event_type: 'Application Opened',
+ event_properties: { build: '5', from_background: true, version: '201' },
+ user_properties: {},
+ device_id: 'AAAAAAAA-0000-0000-0000-000000000001',
+ user_id: 'user-001',
+ amplitude_id: 100_000_000_001,
+ event_time: '2025-02-05 22:58:30.568000',
+ client_event_time: '2025-02-05 22:58:30.568000',
+ country: 'Sweden',
+ city: 'Stockholm',
+ region: 'Stockholm County',
+ platform: 'iOS',
+ os_name: 'ios',
+ os_version: '17.6.1',
+ device_family: 'Apple iPhone',
+ device_type: 'Apple iPhone 12 Mini',
+ language: 'English',
+ version_name: '201',
+ library: 'segment',
+ $insert_id: '00000000-0000-4000-8000-000000000005',
+ uuid: '00000000-0000-4000-8000-0000000000a5',
+ } satisfies AmplitudeRawEvent;
+
+ const res = provider.transformEvent(rawEvent);
+
+ expect(res.id.length).toBeGreaterThan(30);
+ expect(res.imported_at).toMatch(
+ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
+ );
+ expect(res).toMatchObject({
+ name: 'Application Opened',
+ device_id: 'AAAAAAAA-0000-0000-0000-000000000001',
+ profile_id: 'user-001',
+ project_id: 'pid',
+ session_id: '',
+ created_at: '2025-02-05 22:58:30',
+ country: 'SE',
+ city: 'Stockholm',
+ region: 'Stockholm County',
+ os: 'iOS',
+ os_version: '17.6.1',
+ browser: '',
+ device: 'mobile',
+ brand: 'Apple',
+ model: 'Apple iPhone 12 Mini',
+ path: '',
+ sdk_name: 'amplitude (segment)',
+ sdk_version: '1.0.0',
+ properties: {
+ build: '5',
+ from_background: 'true',
+ version: '201',
+ __source_insert_id: '00000000-0000-4000-8000-000000000005',
+ __language: 'English',
+ __version: '201',
+ },
+ });
+ });
+
+ it('falls back to amplitude_id for device, device for profile', () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+ const res = provider.transformEvent({
+ event_type: 'Custom',
+ event_properties: {},
+ user_properties: {},
+ amplitude_id: 123,
+ event_time: '2025-02-05 22:58:30.568000',
+ } as AmplitudeRawEvent);
+
+ expect(res.device_id).toBe('123');
+ expect(res.profile_id).toBe('123');
+ });
+
+ it('maps page-view system events to screen_view', () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+ const res = provider.transformEvent({
+ event_type: '[Amplitude] Page Viewed',
+ event_properties: {},
+ user_properties: {},
+ device_id: 'd1',
+ event_time: '2025-02-05 22:58:30.568000',
+ } as AmplitudeRawEvent);
+ expect(res.name).toBe('screen_view');
+ });
+
+ it('streams plain (uncompressed) NDJSON', async () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+ const ndjson = `${JSON.stringify({ event_type: 'A', device_id: 'd1' })}\n${JSON.stringify(
+ { event_type: 'B', device_id: 'd2' }
+ )}\n`;
+
+ const events = await collect(
+ provider.streamNdjson(Readable.from(Buffer.from(ndjson)), baseConfig.fileUrl)
+ );
+
+ expect(events.map((e) => e.event_type)).toEqual(['A', 'B']);
+ });
+
+ it('streams gzipped NDJSON by detecting magic bytes', async () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+ const ndjson = `${JSON.stringify({ event_type: 'A', device_id: 'd1' })}\n`;
+ const gz = gzipSync(Buffer.from(ndjson));
+
+ const events = await collect(
+ provider.streamNdjson(Readable.from(gz), 'https://example.com/export.json.gz')
+ );
+
+ expect(events.map((e) => e.event_type)).toEqual(['A']);
+ });
+
+ it('does not double-decompress when fetch already decoded transport gzip', async () => {
+ // Reproduces "incorrect header check": a .json URL whose bytes are already
+ // plain JSON must not be gunzipped just because the server used gzip transport.
+ const provider = new AmplitudeProvider('pid', baseConfig);
+ const ndjson = `${JSON.stringify({ event_type: 'A', device_id: 'd1' })}\n`;
+
+ const events = await collect(
+ provider.streamNdjson(
+ Readable.from(Buffer.from(ndjson)),
+ 'https://example.com/amplitude.json'
+ )
+ );
+
+ expect(events.map((e) => e.event_type)).toEqual(['A']);
+ });
+
+ it('maps Amplitude country names to ISO alpha-2 (FixedString(2) safe)', () => {
+ expect(toCountryCode('Nigeria')).toBe('NG');
+ expect(toCountryCode('United States')).toBe('US');
+ expect(toCountryCode('Russia')).toBe('RU');
+ expect(toCountryCode('South Korea')).toBe('KR');
+ expect(toCountryCode('Czechia')).toBe('CZ');
+ // Already a code
+ expect(toCountryCode('US')).toBe('US');
+ expect(toCountryCode('se')).toBe('SE');
+ // Unknown / empty → '' so the FixedString(2) insert never overflows
+ expect(toCountryCode('Testland')).toBe('');
+ expect(toCountryCode('')).toBe('');
+ expect(toCountryCode(null)).toBe('');
+ expect(toCountryCode(undefined)).toBe('');
+ });
+
+ it('transforms every event in the real export without throwing', () => {
+ const provider = new AmplitudeProvider('pid', baseConfig);
+ const events = loadFixture();
+ expect(events.length).toBeGreaterThan(0);
+
+ for (const raw of events) {
+ expect(provider.validate(raw)).toBe(true);
+ const res = provider.transformEvent(raw);
+ expect(res.name).toBeTruthy();
+ expect(res.device_id).toBeTruthy();
+ expect(res.session_id).toBe('');
+ expect(res.created_at).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/);
+ // country must always be storable in FixedString(2)
+ expect(res.country.length).toBeLessThanOrEqual(2);
+ }
+ });
+});
diff --git a/packages/importer/src/providers/amplitude.ts b/packages/importer/src/providers/amplitude.ts
new file mode 100644
index 000000000..94d4ab7fd
--- /dev/null
+++ b/packages/importer/src/providers/amplitude.ts
@@ -0,0 +1,393 @@
+import { randomUUID } from 'node:crypto';
+import { Readable } from 'node:stream';
+import { createInterface } from 'node:readline';
+import { createBrotliDecompress, createGunzip } from 'node:zlib';
+import { isSameDomain, parsePath, toDots } from '@openpanel/common';
+import {
+ getReferrerWithQuery,
+ parseReferrer,
+} from '@openpanel/common/server';
+import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';
+import type { ILogger } from '@openpanel/logger';
+import type { IAmplitudeImportConfig } from '@openpanel/validation';
+import { z } from 'zod';
+import { BaseImportProvider } from '../base-provider';
+import { toCountryCode } from './country-codes';
+
+/**
+ * Amplitude / Segment export rows are newline-delimited JSON. Each line is a
+ * flat object with top-level metadata and nested `event_properties` /
+ * `user_properties`. We only validate the fields we consume; everything else
+ * is passed through untouched.
+ */
+export const zAmplitudeRawEvent = z.object({
+ event_type: z.string().min(1),
+ event_properties: z.record(z.string(), z.unknown()).optional().default({}),
+ user_properties: z.record(z.string(), z.unknown()).optional().default({}),
+ // Identity
+ device_id: z.string().optional().nullable(),
+ user_id: z.string().optional().nullable(),
+ amplitude_id: z.union([z.string(), z.number()]).optional().nullable(),
+ // Timestamps (UTC, "YYYY-MM-DD HH:mm:ss.SSSSSS")
+ event_time: z.string().optional().nullable(),
+ client_event_time: z.string().optional().nullable(),
+ // Geo
+ country: z.string().optional().nullable(),
+ city: z.string().optional().nullable(),
+ region: z.string().optional().nullable(),
+ location_lat: z.number().optional().nullable(),
+ location_lng: z.number().optional().nullable(),
+ // Device / platform
+ platform: z.string().optional().nullable(),
+ os_name: z.string().optional().nullable(),
+ os_version: z.string().optional().nullable(),
+ device_brand: z.string().optional().nullable(),
+ device_manufacturer: z.string().optional().nullable(),
+ device_model: z.string().optional().nullable(),
+ device_family: z.string().optional().nullable(),
+ device_type: z.string().optional().nullable(),
+ language: z.string().optional().nullable(),
+ version_name: z.string().optional().nullable(),
+ // Source
+ library: z.string().optional().nullable(),
+ $insert_id: z.string().optional().nullable(),
+ uuid: z.string().optional().nullable(),
+});
+export type AmplitudeRawEvent = z.infer<typeof zAmplitudeRawEvent>;
+
+const URL_LIKE_KEYS = ['url', '$current_url', 'current_url', 'page_url'];
+
+export class AmplitudeProvider extends BaseImportProvider<AmplitudeRawEvent> {
+ provider = 'amplitude';
+ version = '1.0.0';
+
+ constructor(
+ private readonly projectId: string,
+ private readonly config: IAmplitudeImportConfig,
+ private readonly logger?: ILogger
+ ) {
+ super();
+ }
+
+ async getTotalEventsCount(): Promise<number> {
+ return -1;
+ }
+
+ /**
+ * Amplitude exports carry no usable session IDs (Segment-forwarded events
+ * report session_id = -1). Generate gap-based sessions in SQL after load,
+ * the same way the Mixpanel importer does.
+ */
+ shouldGenerateSessionIds(): boolean {
+ return true;
+ }
+
+ async *parseSource(): AsyncGenerator<AmplitudeRawEvent, void, unknown> {
+ yield* this.parseRemoteFile(this.config.fileUrl);
+ }
+
+ /**
+ * Stream a remote newline-delimited JSON file.
+ *
+ * Note: we do NOT inspect the Content-Encoding/Content-Type headers to decide
+ * whether to decompress. fetch/undici already transparently decompresses
+ * transport-level gzip/br (e.g. a dev server that gzips responses on the fly),
+ * yet leaves the Content-Encoding header in place. Trusting it would gunzip
+ * already-plain JSON and throw "incorrect header check". Instead we detect a
+ * genuinely-gzipped file by its magic bytes (see streamNdjson).
+ */
+ private async *parseRemoteFile(
+ url: string
+ ): AsyncGenerator<AmplitudeRawEvent, void, unknown> {
+ const controller = new AbortController();
+ const res = await fetch(url, { signal: controller.signal });
+ if (!(res.ok && res.body)) {
+ throw new Error(
+ `Failed to fetch remote file: ${res.status} ${res.statusText}`
+ );
+ }
+
+ const body = Readable.fromWeb(res.body as any);
+ try {
+ yield* this.streamNdjson(body, url);
+ } finally {
+ controller.abort(); // tear down the fetch stream
+ }
+ }
+
+ /**
+ * Decode a binary stream of newline-delimited JSON into parsed events.
+ * Detects gzip by peeking the first chunk's magic bytes (0x1f 0x8b) rather
+ * than trusting response headers; brotli is only applied for explicit `.br`
+ * URLs since the stream may already be transport-decompressed.
+ */
+ async *streamNdjson(
+ input: Readable,
+ url: string
+ ): AsyncGenerator<AmplitudeRawEvent, void, unknown> {
+ const iterator = input[Symbol.asyncIterator]();
+ const first = await iterator.next();
+ const firstChunk = first.done ? undefined : (first.value as Buffer);
+
+ const isGzip =
+ !!firstChunk &&
+ firstChunk.length >= 2 &&
+ firstChunk[0] === 0x1f &&
+ firstChunk[1] === 0x8b;
+ const isBrotli = !isGzip && /\.br($|\?)/i.test(url);
+
+ async function* sourceChunks() {
+ if (firstChunk !== undefined) {
+ yield firstChunk;
+ }
+ for (
+ let next = await iterator.next();
+ !next.done;
+ next = await iterator.next()
+ ) {
+ yield next.value;
+ }
+ }
+
+ const source = Readable.from(sourceChunks());
+ const stream = isGzip
+ ? source.pipe(createGunzip())
+ : isBrotli
+ ? source.pipe(createBrotliDecompress())
+ : source;
+
+ const rl = createInterface({
+ input: stream,
+ crlfDelay: Number.POSITIVE_INFINITY,
+ });
+
+ try {
+ for await (const line of rl) {
+ const trimmed = line.trim();
+ if (!trimmed) {
+ continue;
+ }
+ try {
+ yield JSON.parse(trimmed) as AmplitudeRawEvent;
+ } catch (error) {
+ this.logger?.warn(
+ { err: error, line: trimmed.substring(0, 100) },
+ 'Failed to parse Amplitude event'
+ );
+ }
+ }
+ } finally {
+ rl.close();
+ }
+ }
+
+ validate(rawEvent: AmplitudeRawEvent): boolean {
+ return zAmplitudeRawEvent.safeParse(rawEvent).success;
+ }
+
+ transformEvent(_rawEvent: AmplitudeRawEvent): IClickhouseEvent {
+ const raw = zAmplitudeRawEvent.parse(_rawEvent);
+ const props = raw.event_properties as Record<string, unknown>;
+
+ const deviceId = raw.device_id || String(raw.amplitude_id ?? '');
+ const profileId = raw.user_id || deviceId;
+
+ // Web URL handling (rare for mobile exports, but supported for web shards)
+ const rawUrl = URL_LIKE_KEYS.map((key) => props[key]).find(
+ (value): value is string => typeof value === 'string' && value.length > 0
+ );
+ let path = '';
+ let origin = '';
+ let hash = '';
+ let query: Record<string, string> = {};
+ if (rawUrl) {
+ const parsed = parsePath(rawUrl);
+ path = parsed.path || '';
+ origin = parsed.origin || '';
+ hash = parsed.hash || '';
+ query = parsed.query || {};
+ } else if (this.config.mapScreenViewProperty) {
+ path = String(props[this.config.mapScreenViewProperty] ?? '');
+ }
+
+ // Referrer (web only)
+ const referrerUrl = String(props.referrer || props.$referrer || '');
+ const referrer =
+ referrerUrl && !isSameDomain(referrerUrl, rawUrl || '')
+ ? parseReferrer(referrerUrl)
+ : null;
+ const utmReferrer = getReferrerWithQuery(query);
+
+ const properties = this.buildProperties(props, query);
+ const insertId = raw.$insert_id || raw.uuid;
+ if (insertId) {
+ properties.__source_insert_id = insertId;
+ }
+ if (raw.language) {
+ properties.__language = raw.language;
+ }
+ if (raw.version_name) {
+ properties.__version = raw.version_name;
+ }
+ if (hash) {
+ properties.__hash = hash;
+ }
+ if (Object.keys(query).length > 0) {
+ properties.__query = query;
+ }
+
+ const eventName = this.mapEventName(raw.event_type);
+ const timestamp = this.parseTimestamp(raw.event_time || raw.client_event_time);
+
+ return {
+ id: randomUUID(),
+ name: eventName,
+ device_id: deviceId,
+ profile_id: profileId,
+ project_id: this.projectId,
+ session_id: '', // Generated in SQL after import
+ properties: toDots(properties),
+ created_at: formatClickhouseDate(timestamp),
+ country: toCountryCode(raw.country),
+ city: raw.city || '',
+ region: raw.region || '',
+ longitude: raw.location_lng ?? null,
+ latitude: raw.location_lat ?? null,
+ os: this.mapOs(raw.os_name),
+ os_version: raw.os_version || '',
+ browser: '',
+ browser_version: '',
+ device: this.getDeviceType(raw),
+ brand: this.getBrand(raw),
+ model: raw.device_model || raw.device_type || '',
+ duration: 0,
+ path,
+ origin,
+ referrer: referrer?.url || '',
+ referrer_name: utmReferrer?.name || referrer?.name || '',
+ referrer_type: referrer?.type || utmReferrer?.type || '',
+ imported_at: new Date().toISOString(),
+ sdk_name: raw.library
+ ? `${this.provider} (${raw.library})`
+ : this.provider,
+ sdk_version: this.version,
+ groups: [],
+ };
+ }
+
+ /**
+ * Map Amplitude system event names to OpenPanel conventions. Custom event
+ * names pass through unchanged.
+ */
+ private mapEventName(eventType: string): string {
+ if (
+ eventType === '[Amplitude] Page Viewed' ||
+ eventType === 'Page Viewed' ||
+ eventType === 'Screen Viewed'
+ ) {
+ return 'screen_view';
+ }
+ return eventType;
+ }
+
+ /**
+ * Parse Amplitude's "YYYY-MM-DD HH:mm:ss.SSSSSS" UTC timestamps. Trims the
+ * microseconds to milliseconds and pins the zone to UTC so we don't fall back
+ * to the host's local timezone.
+ */
+ private parseTimestamp(value?: string | null): Date {
+ if (!value) {
+ return new Date();
+ }
+ const normalized = `${value.replace(' ', 'T').replace(/(\.\d{3})\d*$/, '$1')}Z`;
+ const date = new Date(normalized);
+ return Number.isNaN(date.getTime()) ? new Date(value) : date;
+ }
+
+ private mapOs(osName?: string | null): string {
+ const os = (osName || '').toLowerCase();
+ const mapping: Record<string, string> = {
+ ios: 'iOS',
+ android: 'Android',
+ 'mac os x': 'Mac OS',
+ macos: 'Mac OS',
+ windows: 'Windows',
+ linux: 'Linux',
+ };
+ return mapping[os] || osName || '';
+ }
+
+ private getBrand(raw: AmplitudeRawEvent): string {
+ if (raw.device_brand) {
+ return raw.device_brand;
+ }
+ if (raw.device_manufacturer) {
+ return raw.device_manufacturer;
+ }
+ // Derive from device_family (e.g. "Apple iPhone" -> "Apple")
+ const family = raw.device_family || raw.device_type || '';
+ if (/apple|iphone|ipad|ios/i.test(family)) {
+ return 'Apple';
+ }
+ return '';
+ }
+
+ private getDeviceType(raw: AmplitudeRawEvent): string {
+ const os = (raw.os_name || '').toLowerCase();
+ const platform = (raw.platform || '').toLowerCase();
+ const family = `${raw.device_family || ''} ${raw.device_type || ''}`.toLowerCase();
+
+ const isMobileOs =
+ os === 'ios' ||
+ os === 'android' ||
+ platform === 'ios' ||
+ platform === 'android';
+
+ if (isMobileOs) {
+ return family.includes('ipad') || family.includes('tablet')
+ ? 'tablet'
+ : 'mobile';
+ }
+
+ if (platform === 'web' || /chrome|safari|firefox|edge/.test(os)) {
+ return 'desktop';
+ }
+
+ // Unknown platform: Amplitude exports are overwhelmingly mobile, but a
+ // present platform that isn't web is a stronger mobile signal than not.
+ return platform ? 'mobile' : 'desktop';
+ }
+
+ /**
+ * Build the OpenPanel properties bag from Amplitude event_properties: drop
+ * keys already represented elsewhere (URLs, query params) and parse
+ * stringified JSON so toDots() can flatten nested structures.
+ */
+ private buildProperties(
+ props: Record<string, unknown>,
+ query: Record<string, string>
+ ): Record<string, any> {
+ const strip = new Set([...URL_LIKE_KEYS, 'referrer', '$referrer', ...Object.keys(query)]);
+ const result: Record<string, any> = {};
+
+ for (const [key, value] of Object.entries(props)) {
+ if (strip.has(key) || value == null) {
+ continue;
+ }
+ if (
+ typeof value === 'string' &&
+ (value.startsWith('{') || value.startsWith('['))
+ ) {
+ try {
+ result[key] = JSON.parse(value);
+ } catch {
+ result[key] = value;
+ }
+ } else {
+ result[key] = value;
+ }
+ }
+
+ return result;
+ }
+}
diff --git a/packages/importer/src/providers/country-codes.ts b/packages/importer/src/providers/country-codes.ts
new file mode 100644
index 000000000..ab2c84b2c
--- /dev/null
+++ b/packages/importer/src/providers/country-codes.ts
@@ -0,0 +1,302 @@
+/**
+ * Map a country *name* (as exported by Amplitude / MaxMind) to its ISO 3166-1
+ * alpha-2 code. OpenPanel's ClickHouse `country` column is FixedString(2), so
+ * anything that isn't a clean 2-letter code must be converted or dropped —
+ * inserting a full name like "Nigeria" throws "Too large value for
+ * FixedString(2)".
+ *
+ * Names are matched case-insensitively and include the common aliases MaxMind
+ * uses (e.g. "Russia", "South Korea", "Czechia"). Unknown names return '' so a
+ * bad/rare value never breaks the import.
+ */
+
+// Canonical ISO 3166-1 alpha-2 → English name, plus alias names that mapping
+// providers commonly emit. Keys are lowercased at build time.
+const NAME_TO_ALPHA2: Record<string, string> = {};
+
+const CANONICAL: [string, string[]][] = [
+ ['AD', ['Andorra']],
+ ['AE', ['United Arab Emirates']],
+ ['AF', ['Afghanistan']],
+ ['AG', ['Antigua and Barbuda']],
+ ['AI', ['Anguilla']],
+ ['AL', ['Albania']],
+ ['AM', ['Armenia']],
+ ['AO', ['Angola']],
+ ['AQ', ['Antarctica']],
+ ['AR', ['Argentina']],
+ ['AS', ['American Samoa']],
+ ['AT', ['Austria']],
+ ['AU', ['Australia']],
+ ['AW', ['Aruba']],
+ ['AX', ['Åland Islands', 'Aland Islands']],
+ ['AZ', ['Azerbaijan']],
+ ['BA', ['Bosnia and Herzegovina']],
+ ['BB', ['Barbados']],
+ ['BD', ['Bangladesh']],
+ ['BE', ['Belgium']],
+ ['BF', ['Burkina Faso']],
+ ['BG', ['Bulgaria']],
+ ['BH', ['Bahrain']],
+ ['BI', ['Burundi']],
+ ['BJ', ['Benin']],
+ ['BL', ['Saint Barthélemy', 'Saint Barthelemy']],
+ ['BM', ['Bermuda']],
+ ['BN', ['Brunei', 'Brunei Darussalam']],
+ ['BO', ['Bolivia']],
+ ['BQ', ['Bonaire, Sint Eustatius and Saba', 'Caribbean Netherlands']],
+ ['BR', ['Brazil']],
+ ['BS', ['Bahamas', 'The Bahamas']],
+ ['BT', ['Bhutan']],
+ ['BV', ['Bouvet Island']],
+ ['BW', ['Botswana']],
+ ['BY', ['Belarus']],
+ ['BZ', ['Belize']],
+ ['CA', ['Canada']],
+ ['CC', ['Cocos (Keeling) Islands']],
+ [
+ 'CD',
+ [
+ 'Congo, The Democratic Republic of the',
+ 'Democratic Republic of the Congo',
+ 'DR Congo',
+ ],
+ ],
+ ['CF', ['Central African Republic']],
+ ['CG', ['Congo', 'Republic of the Congo']],
+ ['CH', ['Switzerland']],
+ ['CI', ["Côte d'Ivoire", "Cote d'Ivoire", 'Ivory Coast']],
+ ['CK', ['Cook Islands']],
+ ['CL', ['Chile']],
+ ['CM', ['Cameroon']],
+ ['CN', ['China']],
+ ['CO', ['Colombia']],
+ ['CR', ['Costa Rica']],
+ ['CU', ['Cuba']],
+ ['CV', ['Cape Verde', 'Cabo Verde']],
+ ['CW', ['Curaçao', 'Curacao']],
+ ['CX', ['Christmas Island']],
+ ['CY', ['Cyprus']],
+ ['CZ', ['Czechia', 'Czech Republic']],
+ ['DE', ['Germany']],
+ ['DJ', ['Djibouti']],
+ ['DK', ['Denmark']],
+ ['DM', ['Dominica']],
+ ['DO', ['Dominican Republic']],
+ ['DZ', ['Algeria']],
+ ['EC', ['Ecuador']],
+ ['EE', ['Estonia']],
+ ['EG', ['Egypt']],
+ ['EH', ['Western Sahara']],
+ ['ER', ['Eritrea']],
+ ['ES', ['Spain']],
+ ['ET', ['Ethiopia']],
+ ['FI', ['Finland']],
+ ['FJ', ['Fiji']],
+ ['FK', ['Falkland Islands', 'Falkland Islands (Malvinas)']],
+ ['FM', ['Micronesia', 'Micronesia, Federated States of']],
+ ['FO', ['Faroe Islands']],
+ ['FR', ['France']],
+ ['GA', ['Gabon']],
+ ['GB', ['United Kingdom', 'UK', 'Great Britain']],
+ ['GD', ['Grenada']],
+ ['GE', ['Georgia']],
+ ['GF', ['French Guiana']],
+ ['GG', ['Guernsey']],
+ ['GH', ['Ghana']],
+ ['GI', ['Gibraltar']],
+ ['GL', ['Greenland']],
+ ['GM', ['Gambia', 'The Gambia']],
+ ['GN', ['Guinea']],
+ ['GP', ['Guadeloupe']],
+ ['GQ', ['Equatorial Guinea']],
+ ['GR', ['Greece']],
+ ['GS', ['South Georgia and the South Sandwich Islands']],
+ ['GT', ['Guatemala']],
+ ['GU', ['Guam']],
+ ['GW', ['Guinea-Bissau']],
+ ['GY', ['Guyana']],
+ ['HK', ['Hong Kong']],
+ ['HM', ['Heard Island and McDonald Islands']],
+ ['HN', ['Honduras']],
+ ['HR', ['Croatia']],
+ ['HT', ['Haiti']],
+ ['HU', ['Hungary']],
+ ['ID', ['Indonesia']],
+ ['IE', ['Ireland']],
+ ['IL', ['Israel']],
+ ['IM', ['Isle of Man']],
+ ['IN', ['India']],
+ ['IO', ['British Indian Ocean Territory']],
+ ['IQ', ['Iraq']],
+ ['IR', ['Iran', 'Iran, Islamic Republic of']],
+ ['IS', ['Iceland']],
+ ['IT', ['Italy']],
+ ['JE', ['Jersey']],
+ ['JM', ['Jamaica']],
+ ['JO', ['Jordan']],
+ ['JP', ['Japan']],
+ ['KE', ['Kenya']],
+ ['KG', ['Kyrgyzstan']],
+ ['KH', ['Cambodia']],
+ ['KI', ['Kiribati']],
+ ['KM', ['Comoros']],
+ ['KN', ['Saint Kitts and Nevis']],
+ ['KP', ['North Korea', "Korea, Democratic People's Republic of"]],
+ ['KR', ['South Korea', 'Korea, Republic of', 'Republic of Korea']],
+ ['KW', ['Kuwait']],
+ ['KY', ['Cayman Islands']],
+ ['KZ', ['Kazakhstan']],
+ ['LA', ['Laos', "Lao People's Democratic Republic"]],
+ ['LB', ['Lebanon']],
+ ['LC', ['Saint Lucia']],
+ ['LI', ['Liechtenstein']],
+ ['LK', ['Sri Lanka']],
+ ['LR', ['Liberia']],
+ ['LS', ['Lesotho']],
+ ['LT', ['Lithuania']],
+ ['LU', ['Luxembourg']],
+ ['LV', ['Latvia']],
+ ['LY', ['Libya']],
+ ['MA', ['Morocco']],
+ ['MC', ['Monaco']],
+ ['MD', ['Moldova', 'Moldova, Republic of']],
+ ['ME', ['Montenegro']],
+ ['MF', ['Saint Martin', 'Saint Martin (French part)']],
+ ['MG', ['Madagascar']],
+ ['MH', ['Marshall Islands']],
+ ['MK', ['North Macedonia', 'Macedonia']],
+ ['ML', ['Mali']],
+ ['MM', ['Myanmar', 'Burma']],
+ ['MN', ['Mongolia']],
+ ['MO', ['Macao', 'Macau']],
+ ['MP', ['Northern Mariana Islands']],
+ ['MQ', ['Martinique']],
+ ['MR', ['Mauritania']],
+ ['MS', ['Montserrat']],
+ ['MT', ['Malta']],
+ ['MU', ['Mauritius']],
+ ['MV', ['Maldives']],
+ ['MW', ['Malawi']],
+ ['MX', ['Mexico']],
+ ['MY', ['Malaysia']],
+ ['MZ', ['Mozambique']],
+ ['NA', ['Namibia']],
+ ['NC', ['New Caledonia']],
+ ['NE', ['Niger']],
+ ['NF', ['Norfolk Island']],
+ ['NG', ['Nigeria']],
+ ['NI', ['Nicaragua']],
+ ['NL', ['Netherlands', 'The Netherlands']],
+ ['NO', ['Norway']],
+ ['NP', ['Nepal']],
+ ['NR', ['Nauru']],
+ ['NU', ['Niue']],
+ ['NZ', ['New Zealand']],
+ ['OM', ['Oman']],
+ ['PA', ['Panama']],
+ ['PE', ['Peru']],
+ ['PF', ['French Polynesia']],
+ ['PG', ['Papua New Guinea']],
+ ['PH', ['Philippines']],
+ ['PK', ['Pakistan']],
+ ['PL', ['Poland']],
+ ['PM', ['Saint Pierre and Miquelon']],
+ ['PN', ['Pitcairn', 'Pitcairn Islands']],
+ ['PR', ['Puerto Rico']],
+ ['PS', ['Palestine', 'Palestine, State of', 'Palestinian Territory']],
+ ['PT', ['Portugal']],
+ ['PW', ['Palau']],
+ ['PY', ['Paraguay']],
+ ['QA', ['Qatar']],
+ ['RE', ['Réunion', 'Reunion']],
+ ['RO', ['Romania']],
+ ['RS', ['Serbia']],
+ ['RU', ['Russia', 'Russian Federation']],
+ ['RW', ['Rwanda']],
+ ['SA', ['Saudi Arabia']],
+ ['SB', ['Solomon Islands']],
+ ['SC', ['Seychelles']],
+ ['SD', ['Sudan']],
+ ['SE', ['Sweden']],
+ ['SG', ['Singapore']],
+ ['SH', ['Saint Helena', 'Saint Helena, Ascension and Tristan da Cunha']],
+ ['SI', ['Slovenia']],
+ ['SJ', ['Svalbard and Jan Mayen']],
+ ['SK', ['Slovakia']],
+ ['SL', ['Sierra Leone']],
+ ['SM', ['San Marino']],
+ ['SN', ['Senegal']],
+ ['SO', ['Somalia']],
+ ['SR', ['Suriname']],
+ ['SS', ['South Sudan']],
+ ['ST', ['Sao Tome and Principe', 'São Tomé and Príncipe']],
+ ['SV', ['El Salvador']],
+ ['SX', ['Sint Maarten', 'Sint Maarten (Dutch part)']],
+ ['SY', ['Syria', 'Syrian Arab Republic']],
+ ['SZ', ['Eswatini', 'Swaziland']],
+ ['TC', ['Turks and Caicos Islands']],
+ ['TD', ['Chad']],
+ ['TF', ['French Southern Territories']],
+ ['TG', ['Togo']],
+ ['TH', ['Thailand']],
+ ['TJ', ['Tajikistan']],
+ ['TK', ['Tokelau']],
+ ['TL', ['Timor-Leste', 'East Timor']],
+ ['TM', ['Turkmenistan']],
+ ['TN', ['Tunisia']],
+ ['TO', ['Tonga']],
+ ['TR', ['Turkey', 'Türkiye', 'Turkiye']],
+ ['TT', ['Trinidad and Tobago']],
+ ['TV', ['Tuvalu']],
+ ['TW', ['Taiwan', 'Taiwan, Province of China']],
+ ['TZ', ['Tanzania', 'Tanzania, United Republic of']],
+ ['UA', ['Ukraine']],
+ ['UG', ['Uganda']],
+ ['UM', ['United States Minor Outlying Islands']],
+ ['US', ['United States', 'United States of America', 'USA']],
+ ['UY', ['Uruguay']],
+ ['UZ', ['Uzbekistan']],
+ ['VA', ['Vatican City', 'Holy See (Vatican City State)']],
+ ['VC', ['Saint Vincent and the Grenadines']],
+ ['VE', ['Venezuela', 'Venezuela, Bolivarian Republic of']],
+ ['VG', ['British Virgin Islands', 'Virgin Islands, British']],
+ ['VI', ['U.S. Virgin Islands', 'Virgin Islands, U.S.']],
+ ['VN', ['Vietnam', 'Viet Nam']],
+ ['VU', ['Vanuatu']],
+ ['WF', ['Wallis and Futuna']],
+ ['WS', ['Samoa']],
+ ['XK', ['Kosovo']],
+ ['YE', ['Yemen']],
+ ['YT', ['Mayotte']],
+ ['ZA', ['South Africa']],
+ ['ZM', ['Zambia']],
+ ['ZW', ['Zimbabwe']],
+];
+
+const ALPHA2_SET = new Set<string>();
+for (const [code, names] of CANONICAL) {
+ ALPHA2_SET.add(code);
+ for (const name of names) {
+ NAME_TO_ALPHA2[name.toLowerCase()] = code;
+ }
+}
+
+/**
+ * Convert a country value to an ISO 3166-1 alpha-2 code suitable for the
+ * ClickHouse FixedString(2) column.
+ *
+ * - Already a valid 2-letter code → returned uppercased.
+ * - A known country name (or alias) → its alpha-2 code.
+ * - Anything else → '' (safe to store, never overflows).
+ */
+export function toCountryCode(value?: string | null): string {
+ if (!value) {
+ return '';
+ }
+ const trimmed = value.trim();
+ if (trimmed.length === 2 && ALPHA2_SET.has(trimmed.toUpperCase())) {
+ return trimmed.toUpperCase();
+ }
+ return NAME_TO_ALPHA2[trimmed.toLowerCase()] ?? '';
+}
diff --git a/packages/importer/src/providers/metadata.ts b/packages/importer/src/providers/metadata.ts
index 639394b72..5f81c7c90 100644
--- a/packages/importer/src/providers/metadata.ts
+++ b/packages/importer/src/providers/metadata.ts
@@ -1,4 +1,4 @@
-export type ImportProviderId = 'umami' | 'mixpanel';
+export type ImportProviderId = 'umami' | 'mixpanel' | 'amplitude';
export type ImportProviderType = 'file' | 'api';
export interface ImportProviderMeta {
@@ -27,4 +27,12 @@ export const IMPORT_PROVIDERS: ImportProviderMeta[] = [
backgroundColor: '#fff',
types: ['api'],
},
+ {
+ id: 'amplitude',
+ name: 'Amplitude',
+ description: 'Import your analytics data from an Amplitude export',
+ logo: 'https://cdn.brandfetch.io/idnYrZGER0/theme/dark/symbol.svg?c=1dxbfHSJFAPEGdCLU4o5B',
+ backgroundColor: '#fff',
+ types: ['file'],
+ },
];
diff --git a/packages/importer/src/providers/umami.ts b/packages/importer/src/providers/umami.ts
index d2b5b0431..71eb14bff 100644
--- a/packages/importer/src/providers/umami.ts
+++ b/packages/importer/src/providers/umami.ts
@@ -105,7 +105,6 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {
}
const contentType = res.headers.get('content-type') || '';
- const contentEnc = res.headers.get('content-encoding') || '';
const contentLen = Number(res.headers.get('content-length') ?? 0);
if (
@@ -123,33 +122,52 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {
);
}
- const looksGzip =
- /\.gz($|\?)/i.test(url) ||
- /gzip/i.test(contentEnc) ||
- /application\/gzip/i.test(contentType);
- const looksBr = /br/i.test(contentEnc) || /\.br($|\?)/i.test(url);
-
// WHATWG -> Node stream
- const body = Readable.fromWeb(res.body as any);
-
- // Optional size guard during stream
+ const rawBody = Readable.fromWeb(res.body as any);
+
+ // Detect gzip by peeking the first chunk's magic bytes (0x1f 0x8b) rather
+ // than trusting Content-Encoding/Type. fetch/undici already decompresses
+ // transport-level gzip/br while leaving the header in place; trusting it
+ // would gunzip already-plain bytes and throw "incorrect header check".
+ const iterator = rawBody[Symbol.asyncIterator]();
+ const first = await iterator.next();
+ const firstChunk = first.done ? undefined : (first.value as Buffer);
+
+ const isGzip =
+ !!firstChunk &&
+ firstChunk.length >= 2 &&
+ firstChunk[0] === 0x1f &&
+ firstChunk[1] === 0x8b;
+ const isBrotli = !isGzip && /\.br($|\?)/i.test(url);
+
+ // Replay the peeked chunk and enforce the optional size guard inline.
let seenBytes = 0;
- if (maxBytes) {
- body.on('data', (chunk: Buffer) => {
+ async function* sourceChunks() {
+ if (firstChunk !== undefined) {
+ seenBytes += firstChunk.length;
+ yield firstChunk;
+ }
+ for (
+ let next = await iterator.next();
+ !next.done;
+ next = await iterator.next()
+ ) {
+ const chunk = next.value as Buffer;
seenBytes += chunk.length;
- if (seenBytes > maxBytes) {
- controller.abort();
- body.destroy(
- new Error(`Stream exceeded size limit (${seenBytes} > ${maxBytes})`)
+ if (maxBytes && seenBytes > maxBytes) {
+ throw new Error(
+ `Stream exceeded size limit (${seenBytes} > ${maxBytes})`
);
}
- });
+ yield chunk;
+ }
}
+ const body = Readable.from(sourceChunks());
// Build decode chain (gzip/brotli -> CSV parser)
- const decompress = looksGzip
+ const decompress = isGzip
? createGunzip()
- : looksBr
+ : isBrotli
? createBrotliDecompress()
: null;
diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts
index 4b2fd87cb..b02813cff 100644
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -659,6 +659,14 @@ export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;
export const zPlausibleImportConfig = createFileImportConfig('plausible');
export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;
+export const zAmplitudeImportConfig = createFileImportConfig('amplitude').extend(
+ {
+ projectMapper: z.array(zProjectMapper),
+ mapScreenViewProperty: z.string().optional(),
+ },
+);
+export type IAmplitudeImportConfig = z.infer<typeof zAmplitudeImportConfig>;
+
export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);
export type IMixpanelDataResidency = z.infer<typeof zMixpanelDataResidency>;
@@ -678,15 +686,17 @@ export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;
export type IImportConfig =
| IUmamiImportConfig
| IPlausibleImportConfig
- | IMixpanelImportConfig;
+ | IMixpanelImportConfig
+ | IAmplitudeImportConfig;
export const zCreateImport = z.object({
projectId: z.string().min(1),
- provider: z.enum(['umami', 'plausible', 'mixpanel']),
+ provider: z.enum(['umami', 'plausible', 'mixpanel', 'amplitude']),
config: z.union([
zUmamiImportConfig,
zPlausibleImportConfig,
zMixpanelImportConfig,
+ zAmplitudeImportConfig,
]),
});