OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat: add amplitude importer
Carl-Gerhard Lindesvärdf8a79da629b3
diff --git a/apps/start/src/modals/add-import.tsx b/apps/start/src/modals/add-import.tsxindex c970cea5a..03e718ea8 100644--- a/apps/start/src/modals/add-import.tsx+++ b/apps/start/src/modals/add-import.tsx@@ -9,11 +9,13 @@ import { useTRPC } from '@/integrations/trpc/react';import { cn } from '@/lib/utils';import { zodResolver } from '@hookform/resolvers/zod';import type {+ IAmplitudeImportConfig,IImportConfig,IMixpanelImportConfig,IUmamiImportConfig,} from '@openpanel/validation';import {+ zAmplitudeImportConfig,zMixpanelImportConfig,zUmamiImportConfig,} from '@openpanel/validation';@@ -31,7 +33,7 @@ import type { z } from 'zod';import { popModal, pushModal } from '.';import { ModalContent, ModalHeader } from './Modal/Container';-type Provider = 'umami' | 'plausible' | 'mixpanel';+type Provider = 'umami' | 'plausible' | 'mixpanel' | 'amplitude';interface AddImportProps {provider: Provider;@@ -41,6 +43,7 @@ interface AddImportProps {type UmamiFormData = z.infer<typeof zUmamiImportConfig>;type MixpanelFormData = z.infer<typeof zMixpanelImportConfig>;+type AmplitudeFormData = z.infer<typeof zAmplitudeImportConfig>;interface UmamiImportProps {onSubmit: (config: IUmamiImportConfig) => void;@@ -253,6 +256,77 @@ function MixpanelImport({);}+interface AmplitudeImportProps {+ onSubmit: (config: IAmplitudeImportConfig) => void;+ isPending: boolean;+ organizationId: string;+}++function AmplitudeImport({+ onSubmit,+ isPending,+ organizationId,+}: AmplitudeImportProps) {+ const trpc = useTRPC();+ const { data: projects = [] } = useQuery(+ trpc.project.list.queryOptions({+ organizationId,+ }),+ );++ const form = useForm<AmplitudeFormData>({+ resolver: zodResolver(zAmplitudeImportConfig),+ defaultValues: {+ provider: 'amplitude',+ type: 'file',+ fileUrl: '',+ projectMapper: [],+ },+ });++ const { fields, append, remove } = useFieldArray({+ control: form.control,+ name: 'projectMapper',+ });++ const handleSubmit = form.handleSubmit((data) => {+ onSubmit(data);+ });++ return (+ <form onSubmit={handleSubmit} className="space-y-4">+ <div className="space-y-4 py-4">+ <InputWithLabel+ label="File URL"+ placeholder="https://example.com/export.json"+ error={form.formState.errors.fileUrl?.message}+ info="Provide a publicly accessible URL to your Amplitude export (newline-delimited JSON, optionally gzipped)."+ {...form.register('fileUrl')}+ />++ <ProjectMapper+ fields={fields}+ append={append}+ remove={remove}+ projects={projects}+ register={form.register}+ watch={form.watch}+ setValue={form.setValue}+ />+ </div>++ <div className="flex justify-between">+ <Button type="button" variant="outline" onClick={() => popModal()}>+ Cancel+ </Button>+ <Button type="submit" disabled={isPending}>+ {isPending ? 'Starting...' : 'Start Import'}+ </Button>+ </div>+ </form>+ );+}+export default function AddImport({ provider, name }: AddImportProps) {const { projectId, organizationId } = useAppParams();const trpc = useTRPC();@@ -302,6 +376,14 @@ export default function AddImport({ provider, name }: AddImportProps) {organizationId={organizationId}/>)}++ {provider === 'amplitude' && (+ <AmplitudeImport+ onSubmit={handleImportSubmit}+ isPending={createImport.isPending}+ organizationId={organizationId}+ />+ )}</ModalContent>);}diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.tsindex 3676958f7..e2e79b3c5 100644--- a/apps/worker/src/jobs/import.ts+++ b/apps/worker/src/jobs/import.ts@@ -14,7 +14,11 @@ import {type Prisma,updateImportStatus,} from '@openpanel/db';-import { MixpanelProvider, UmamiProvider } from '@openpanel/importer';+import {+ AmplitudeProvider,+ MixpanelProvider,+ UmamiProvider,+} from '@openpanel/importer';import type { ILogger } from '@openpanel/logger';import type { ImportQueuePayload } from '@openpanel/queue';import type { Job } from 'bullmq';@@ -298,6 +302,8 @@ function createProvider(return new UmamiProvider(record.projectId, config, jobLogger);case 'mixpanel':return new MixpanelProvider(record.projectId, config, jobLogger);+ case 'amplitude':+ return new AmplitudeProvider(record.projectId, config, jobLogger);default:throw new Error(`Unknown provider: ${config.provider}`);}diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 4a393c4d7..b31c5bafa 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -487,7 +487,7 @@ export async function getEventList(options: GetEventListOptions) {} = options;const { sb, getSql, join } = createSqlBuilder();- const MAX_DATE_INTERVAL_IN_DAYS = 365;+ const MAX_DATE_INTERVAL_IN_DAYS = 365 * 5;// Cap the date interval to prevent infinityconst safeDateIntervalInDays = Math.min(dateIntervalInDays,@@ -1297,7 +1297,7 @@ export interface QueryEventsInput {}export async function queryEventsCore(- input: QueryEventsInput,+ input: QueryEventsInput): Promise<IClickhouseEvent[]> {const builder = clix(ch).select<IClickhouseEvent>([])@@ -1358,7 +1358,9 @@ export async function queryEventsCore(if (input.properties) {for (const [key, value] of Object.entries(input.properties)) {- builder.rawWhere(`properties[${sqlstring.escape(key)}] = ${sqlstring.escape(value)}`);+ builder.rawWhere(+ `properties[${sqlstring.escape(key)}] = ${sqlstring.escape(value)}`+ );}}@@ -1368,7 +1370,7 @@ export async function queryEventsCore(if (!input.sessionId) {const { startDate: start, endDate: end } = resolveDateRange(input.startDate,- input.endDate,+ input.endDate);builder.where('created_at', 'BETWEEN', [clix.datetime(start),@@ -1378,7 +1380,7 @@ export async function queryEventsCore(// If caller still wants to scope by date, honor it.const { startDate: start, endDate: end } = resolveDateRange(input.startDate,- input.endDate,+ input.endDate);builder.where('created_at', 'BETWEEN', [clix.datetime(start),diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.tsindex 52f4e80df..61bb8d9cc 100644--- a/packages/importer/src/index.ts+++ b/packages/importer/src/index.ts@@ -1,5 +1,6 @@export { UmamiProvider } from './providers/umami';export { MixpanelProvider } from './providers/mixpanel';+export { AmplitudeProvider } from './providers/amplitude';export type {ImportConfig,ImportProgress,diff --git a/packages/importer/src/providers/__fixtures__/amplitude-export.json b/packages/importer/src/providers/__fixtures__/amplitude-export.jsonnew file mode 100644index 000000000..117eece99--- /dev/null+++ b/packages/importer/src/providers/__fixtures__/amplitude-export.json@@ -0,0 +1,7 @@+{"$insert_id":"00000000-0000-4000-8000-000000000001","amplitude_id":100000000001,"app":111111,"city":"Springfield","client_event_time":"2025-02-05 22:57:31.078000","client_upload_time":"2025-02-05 22:58:14.311000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"AAAAAAAA-0000-0000-0000-000000000001","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone 12 Mini","event_id":1,"event_properties":{"userId":"user-001"},"event_time":"2025-02-05 22:57:31.078000","event_type":"LoggedIn","ip_address":"192.0.2.1","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"17.6.1","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-001","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a1","version_name":"201"}+{"$insert_id":"00000000-0000-4000-8000-000000000002","amplitude_id":100000000001,"app":111111,"city":"Springfield","client_event_time":"2025-02-05 22:57:31.148000","client_upload_time":"2025-02-05 22:58:13.930000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"AAAAAAAA-0000-0000-0000-000000000001","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone 12 Mini","event_id":2,"event_properties":{"page":"LoginForm with password","email":"[email protected]"},"event_time":"2025-02-05 22:57:31.148000","event_type":"User logged in","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"17.6.1","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-001","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a2","version_name":"201"}+{"$insert_id":"00000000-0000-4000-8000-000000000003","amplitude_id":100000000002,"app":111111,"city":"Rivertown","client_event_time":"2025-02-05 23:01:10.000000","client_upload_time":"2025-02-05 23:01:20.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"BBBBBBBB-0000-0000-0000-000000000002","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone 15 Pro Max","event_id":3,"event_properties":{"cardType":"virtual","section":"wallet"},"event_time":"2025-02-05 23:01:10.000000","event_type":"Get Card","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"18.1.0","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-002","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a3","version_name":"201"}+{"$insert_id":"00000000-0000-4000-8000-000000000004","amplitude_id":100000000002,"app":111111,"city":"Rivertown","client_event_time":"2025-02-05 23:05:00.000000","client_upload_time":"2025-02-05 23:05:10.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"BBBBBBBB-0000-0000-0000-000000000002","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone 15 Pro Max","event_id":4,"event_properties":{"from_background":true,"build":"5","version":"201"},"event_time":"2025-02-05 23:05:00.000000","event_type":"Application Backgrounded","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"18.1.0","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-002","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a4","version_name":"201"}+{"$insert_id":"00000000-0000-4000-8000-000000000005","amplitude_id":100000000003,"app":111111,"city":"Laketown","client_event_time":"2025-02-05 23:10:00.000000","client_upload_time":"2025-02-05 23:10:10.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"CCCCCCCC-0000-0000-0000-000000000003","device_manufacturer":null,"device_model":null,"device_type":"iOS Simulator","event_id":5,"event_properties":{"from_background":false,"build":"5","version":"201"},"event_time":"2025-02-05 23:10:00.000000","event_type":"Application Opened","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"17.6.1","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-003","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a5","version_name":"201"}+{"$insert_id":"00000000-0000-4000-8000-000000000006","amplitude_id":100000000003,"app":111111,"city":"Laketown","client_event_time":"2025-02-05 23:11:00.000000","client_upload_time":"2025-02-05 23:11:10.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"CCCCCCCC-0000-0000-0000-000000000003","device_manufacturer":null,"device_model":null,"device_type":"iOS Simulator","event_id":6,"event_properties":{},"event_time":"2025-02-05 23:11:00.000000","event_type":"Application Installed","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"17.6.1","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-003","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a6","version_name":"201"}+{"$insert_id":"00000000-0000-4000-8000-000000000007","amplitude_id":100000000004,"app":111111,"city":"Hilltown","client_event_time":"2025-02-05 23:20:00.000000","client_upload_time":"2025-02-05 23:20:10.000000","country":"Testland","data":{"path":"/2/httpapi"},"data_type":"event","device_brand":null,"device_family":"Apple iPhone","device_id":"DDDDDDDD-0000-0000-0000-000000000004","device_manufacturer":null,"device_model":null,"device_type":"Apple iPhone SE (3rd generation)","event_id":7,"event_properties":{"section":"signup"},"event_time":"2025-02-05 23:20:00.000000","event_type":"Create Account Button","language":"English","library":"segment","location_lat":null,"location_lng":null,"os_name":"ios","os_version":"16.4.0","platform":"iOS","region":"Test Region","session_id":-1,"start_version":"201","user_id":"user-004","user_properties":{},"uuid":"00000000-0000-4000-8000-0000000000a7","version_name":"201"}diff --git a/packages/importer/src/providers/amplitude.test.ts b/packages/importer/src/providers/amplitude.test.tsnew file mode 100644index 000000000..0702a3826--- /dev/null+++ b/packages/importer/src/providers/amplitude.test.ts@@ -0,0 +1,206 @@+import { readFileSync } from 'node:fs';+import { join } from 'node:path';+import { Readable } from 'node:stream';+import { gzipSync } from 'node:zlib';+import { describe, expect, it } from 'vitest';+import { AmplitudeProvider, type AmplitudeRawEvent } from './amplitude';+import { toCountryCode } from './country-codes';++async function collect(+ gen: AsyncGenerator<AmplitudeRawEvent, void, unknown>+): Promise<AmplitudeRawEvent[]> {+ const out: AmplitudeRawEvent[] = [];+ for await (const event of gen) {+ out.push(event);+ }+ return out;+}++const baseConfig = {+ provider: 'amplitude' as const,+ type: 'file' as const,+ fileUrl: 'https://example.com/export.json',+ projectMapper: [],+};++function loadFixture(): AmplitudeRawEvent[] {+ const raw = readFileSync(+ join(__dirname, '__fixtures__', 'amplitude-export.json'),+ 'utf-8'+ );+ return raw+ .split('\n')+ .map((line) => line.trim())+ .filter(Boolean)+ .map((line) => JSON.parse(line) as AmplitudeRawEvent);+}++describe('amplitude', () => {+ it('generates session IDs in SQL (Segment exports have none)', () => {+ const provider = new AmplitudeProvider('pid', baseConfig);+ expect(provider.shouldGenerateSessionIds()).toBe(true);+ });++ it('transforms a mobile (Segment iOS) event', () => {+ const provider = new AmplitudeProvider('pid', baseConfig);++ const rawEvent = {+ event_type: 'Application Opened',+ event_properties: { build: '5', from_background: true, version: '201' },+ user_properties: {},+ device_id: 'AAAAAAAA-0000-0000-0000-000000000001',+ user_id: 'user-001',+ amplitude_id: 100_000_000_001,+ event_time: '2025-02-05 22:58:30.568000',+ client_event_time: '2025-02-05 22:58:30.568000',+ country: 'Sweden',+ city: 'Stockholm',+ region: 'Stockholm County',+ platform: 'iOS',+ os_name: 'ios',+ os_version: '17.6.1',+ device_family: 'Apple iPhone',+ device_type: 'Apple iPhone 12 Mini',+ language: 'English',+ version_name: '201',+ library: 'segment',+ $insert_id: '00000000-0000-4000-8000-000000000005',+ uuid: '00000000-0000-4000-8000-0000000000a5',+ } satisfies AmplitudeRawEvent;++ const res = provider.transformEvent(rawEvent);++ expect(res.id.length).toBeGreaterThan(30);+ expect(res.imported_at).toMatch(+ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/+ );+ expect(res).toMatchObject({+ name: 'Application Opened',+ device_id: 'AAAAAAAA-0000-0000-0000-000000000001',+ profile_id: 'user-001',+ project_id: 'pid',+ session_id: '',+ created_at: '2025-02-05 22:58:30',+ country: 'SE',+ city: 'Stockholm',+ region: 'Stockholm County',+ os: 'iOS',+ os_version: '17.6.1',+ browser: '',+ device: 'mobile',+ brand: 'Apple',+ model: 'Apple iPhone 12 Mini',+ path: '',+ sdk_name: 'amplitude (segment)',+ sdk_version: '1.0.0',+ properties: {+ build: '5',+ from_background: 'true',+ version: '201',+ __source_insert_id: '00000000-0000-4000-8000-000000000005',+ __language: 'English',+ __version: '201',+ },+ });+ });++ it('falls back to amplitude_id for device, device for profile', () => {+ const provider = new AmplitudeProvider('pid', baseConfig);+ const res = provider.transformEvent({+ event_type: 'Custom',+ event_properties: {},+ user_properties: {},+ amplitude_id: 123,+ event_time: '2025-02-05 22:58:30.568000',+ } as AmplitudeRawEvent);++ expect(res.device_id).toBe('123');+ expect(res.profile_id).toBe('123');+ });++ it('maps page-view system events to screen_view', () => {+ const provider = new AmplitudeProvider('pid', baseConfig);+ const res = provider.transformEvent({+ event_type: '[Amplitude] Page Viewed',+ event_properties: {},+ user_properties: {},+ device_id: 'd1',+ event_time: '2025-02-05 22:58:30.568000',+ } as AmplitudeRawEvent);+ expect(res.name).toBe('screen_view');+ });++ it('streams plain (uncompressed) NDJSON', async () => {+ const provider = new AmplitudeProvider('pid', baseConfig);+ const ndjson = `${JSON.stringify({ event_type: 'A', device_id: 'd1' })}\n${JSON.stringify(+ { event_type: 'B', device_id: 'd2' }+ )}\n`;++ const events = await collect(+ provider.streamNdjson(Readable.from(Buffer.from(ndjson)), baseConfig.fileUrl)+ );++ expect(events.map((e) => e.event_type)).toEqual(['A', 'B']);+ });++ it('streams gzipped NDJSON by detecting magic bytes', async () => {+ const provider = new AmplitudeProvider('pid', baseConfig);+ const ndjson = `${JSON.stringify({ event_type: 'A', device_id: 'd1' })}\n`;+ const gz = gzipSync(Buffer.from(ndjson));++ const events = await collect(+ provider.streamNdjson(Readable.from(gz), 'https://example.com/export.json.gz')+ );++ expect(events.map((e) => e.event_type)).toEqual(['A']);+ });++ it('does not double-decompress when fetch already decoded transport gzip', async () => {+ // Reproduces "incorrect header check": a .json URL whose bytes are already+ // plain JSON must not be gunzipped just because the server used gzip transport.+ const provider = new AmplitudeProvider('pid', baseConfig);+ const ndjson = `${JSON.stringify({ event_type: 'A', device_id: 'd1' })}\n`;++ const events = await collect(+ provider.streamNdjson(+ Readable.from(Buffer.from(ndjson)),+ 'https://example.com/amplitude.json'+ )+ );++ expect(events.map((e) => e.event_type)).toEqual(['A']);+ });++ it('maps Amplitude country names to ISO alpha-2 (FixedString(2) safe)', () => {+ expect(toCountryCode('Nigeria')).toBe('NG');+ expect(toCountryCode('United States')).toBe('US');+ expect(toCountryCode('Russia')).toBe('RU');+ expect(toCountryCode('South Korea')).toBe('KR');+ expect(toCountryCode('Czechia')).toBe('CZ');+ // Already a code+ expect(toCountryCode('US')).toBe('US');+ expect(toCountryCode('se')).toBe('SE');+ // Unknown / empty → '' so the FixedString(2) insert never overflows+ expect(toCountryCode('Testland')).toBe('');+ expect(toCountryCode('')).toBe('');+ expect(toCountryCode(null)).toBe('');+ expect(toCountryCode(undefined)).toBe('');+ });++ it('transforms every event in the real export without throwing', () => {+ const provider = new AmplitudeProvider('pid', baseConfig);+ const events = loadFixture();+ expect(events.length).toBeGreaterThan(0);++ for (const raw of events) {+ expect(provider.validate(raw)).toBe(true);+ const res = provider.transformEvent(raw);+ expect(res.name).toBeTruthy();+ expect(res.device_id).toBeTruthy();+ expect(res.session_id).toBe('');+ expect(res.created_at).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/);+ // country must always be storable in FixedString(2)+ expect(res.country.length).toBeLessThanOrEqual(2);+ }+ });+});diff --git a/packages/importer/src/providers/amplitude.ts b/packages/importer/src/providers/amplitude.tsnew file mode 100644index 000000000..94d4ab7fd--- /dev/null+++ b/packages/importer/src/providers/amplitude.ts@@ -0,0 +1,393 @@+import { randomUUID } from 'node:crypto';+import { Readable } from 'node:stream';+import { createInterface } from 'node:readline';+import { createBrotliDecompress, createGunzip } from 'node:zlib';+import { isSameDomain, parsePath, toDots } from '@openpanel/common';+import {+ getReferrerWithQuery,+ parseReferrer,+} from '@openpanel/common/server';+import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';+import type { ILogger } from '@openpanel/logger';+import type { IAmplitudeImportConfig } from '@openpanel/validation';+import { z } from 'zod';+import { BaseImportProvider } from '../base-provider';+import { toCountryCode } from './country-codes';++/**+ * Amplitude / Segment export rows are newline-delimited JSON. Each line is a+ * flat object with top-level metadata and nested `event_properties` /+ * `user_properties`. We only validate the fields we consume; everything else+ * is passed through untouched.+ */+export const zAmplitudeRawEvent = z.object({+ event_type: z.string().min(1),+ event_properties: z.record(z.string(), z.unknown()).optional().default({}),+ user_properties: z.record(z.string(), z.unknown()).optional().default({}),+ // Identity+ device_id: z.string().optional().nullable(),+ user_id: z.string().optional().nullable(),+ amplitude_id: z.union([z.string(), z.number()]).optional().nullable(),+ // Timestamps (UTC, "YYYY-MM-DD HH:mm:ss.SSSSSS")+ event_time: z.string().optional().nullable(),+ client_event_time: z.string().optional().nullable(),+ // Geo+ country: z.string().optional().nullable(),+ city: z.string().optional().nullable(),+ region: z.string().optional().nullable(),+ location_lat: z.number().optional().nullable(),+ location_lng: z.number().optional().nullable(),+ // Device / platform+ platform: z.string().optional().nullable(),+ os_name: z.string().optional().nullable(),+ os_version: z.string().optional().nullable(),+ device_brand: z.string().optional().nullable(),+ device_manufacturer: z.string().optional().nullable(),+ device_model: z.string().optional().nullable(),+ device_family: z.string().optional().nullable(),+ device_type: z.string().optional().nullable(),+ language: z.string().optional().nullable(),+ version_name: z.string().optional().nullable(),+ // Source+ library: z.string().optional().nullable(),+ $insert_id: z.string().optional().nullable(),+ uuid: z.string().optional().nullable(),+});+export type AmplitudeRawEvent = z.infer<typeof zAmplitudeRawEvent>;++const URL_LIKE_KEYS = ['url', '$current_url', 'current_url', 'page_url'];++export class AmplitudeProvider extends BaseImportProvider<AmplitudeRawEvent> {+ provider = 'amplitude';+ version = '1.0.0';++ constructor(+ private readonly projectId: string,+ private readonly config: IAmplitudeImportConfig,+ private readonly logger?: ILogger+ ) {+ super();+ }++ async getTotalEventsCount(): Promise<number> {+ return -1;+ }++ /**+ * Amplitude exports carry no usable session IDs (Segment-forwarded events+ * report session_id = -1). Generate gap-based sessions in SQL after load,+ * the same way the Mixpanel importer does.+ */+ shouldGenerateSessionIds(): boolean {+ return true;+ }++ async *parseSource(): AsyncGenerator<AmplitudeRawEvent, void, unknown> {+ yield* this.parseRemoteFile(this.config.fileUrl);+ }++ /**+ * Stream a remote newline-delimited JSON file.+ *+ * Note: we do NOT inspect the Content-Encoding/Content-Type headers to decide+ * whether to decompress. fetch/undici already transparently decompresses+ * transport-level gzip/br (e.g. a dev server that gzips responses on the fly),+ * yet leaves the Content-Encoding header in place. Trusting it would gunzip+ * already-plain JSON and throw "incorrect header check". Instead we detect a+ * genuinely-gzipped file by its magic bytes (see streamNdjson).+ */+ private async *parseRemoteFile(+ url: string+ ): AsyncGenerator<AmplitudeRawEvent, void, unknown> {+ const controller = new AbortController();+ const res = await fetch(url, { signal: controller.signal });+ if (!(res.ok && res.body)) {+ throw new Error(+ `Failed to fetch remote file: ${res.status} ${res.statusText}`+ );+ }++ const body = Readable.fromWeb(res.body as any);+ try {+ yield* this.streamNdjson(body, url);+ } finally {+ controller.abort(); // tear down the fetch stream+ }+ }++ /**+ * Decode a binary stream of newline-delimited JSON into parsed events.+ * Detects gzip by peeking the first chunk's magic bytes (0x1f 0x8b) rather+ * than trusting response headers; brotli is only applied for explicit `.br`+ * URLs since the stream may already be transport-decompressed.+ */+ async *streamNdjson(+ input: Readable,+ url: string+ ): AsyncGenerator<AmplitudeRawEvent, void, unknown> {+ const iterator = input[Symbol.asyncIterator]();+ const first = await iterator.next();+ const firstChunk = first.done ? undefined : (first.value as Buffer);++ const isGzip =+ !!firstChunk &&+ firstChunk.length >= 2 &&+ firstChunk[0] === 0x1f &&+ firstChunk[1] === 0x8b;+ const isBrotli = !isGzip && /\.br($|\?)/i.test(url);++ async function* sourceChunks() {+ if (firstChunk !== undefined) {+ yield firstChunk;+ }+ for (+ let next = await iterator.next();+ !next.done;+ next = await iterator.next()+ ) {+ yield next.value;+ }+ }++ const source = Readable.from(sourceChunks());+ const stream = isGzip+ ? source.pipe(createGunzip())+ : isBrotli+ ? source.pipe(createBrotliDecompress())+ : source;++ const rl = createInterface({+ input: stream,+ crlfDelay: Number.POSITIVE_INFINITY,+ });++ try {+ for await (const line of rl) {+ const trimmed = line.trim();+ if (!trimmed) {+ continue;+ }+ try {+ yield JSON.parse(trimmed) as AmplitudeRawEvent;+ } catch (error) {+ this.logger?.warn(+ { err: error, line: trimmed.substring(0, 100) },+ 'Failed to parse Amplitude event'+ );+ }+ }+ } finally {+ rl.close();+ }+ }++ validate(rawEvent: AmplitudeRawEvent): boolean {+ return zAmplitudeRawEvent.safeParse(rawEvent).success;+ }++ transformEvent(_rawEvent: AmplitudeRawEvent): IClickhouseEvent {+ const raw = zAmplitudeRawEvent.parse(_rawEvent);+ const props = raw.event_properties as Record<string, unknown>;++ const deviceId = raw.device_id || String(raw.amplitude_id ?? '');+ const profileId = raw.user_id || deviceId;++ // Web URL handling (rare for mobile exports, but supported for web shards)+ const rawUrl = URL_LIKE_KEYS.map((key) => props[key]).find(+ (value): value is string => typeof value === 'string' && value.length > 0+ );+ let path = '';+ let origin = '';+ let hash = '';+ let query: Record<string, string> = {};+ if (rawUrl) {+ const parsed = parsePath(rawUrl);+ path = parsed.path || '';+ origin = parsed.origin || '';+ hash = parsed.hash || '';+ query = parsed.query || {};+ } else if (this.config.mapScreenViewProperty) {+ path = String(props[this.config.mapScreenViewProperty] ?? '');+ }++ // Referrer (web only)+ const referrerUrl = String(props.referrer || props.$referrer || '');+ const referrer =+ referrerUrl && !isSameDomain(referrerUrl, rawUrl || '')+ ? parseReferrer(referrerUrl)+ : null;+ const utmReferrer = getReferrerWithQuery(query);++ const properties = this.buildProperties(props, query);+ const insertId = raw.$insert_id || raw.uuid;+ if (insertId) {+ properties.__source_insert_id = insertId;+ }+ if (raw.language) {+ properties.__language = raw.language;+ }+ if (raw.version_name) {+ properties.__version = raw.version_name;+ }+ if (hash) {+ properties.__hash = hash;+ }+ if (Object.keys(query).length > 0) {+ properties.__query = query;+ }++ const eventName = this.mapEventName(raw.event_type);+ const timestamp = this.parseTimestamp(raw.event_time || raw.client_event_time);++ return {+ id: randomUUID(),+ name: eventName,+ device_id: deviceId,+ profile_id: profileId,+ project_id: this.projectId,+ session_id: '', // Generated in SQL after import+ properties: toDots(properties),+ created_at: formatClickhouseDate(timestamp),+ country: toCountryCode(raw.country),+ city: raw.city || '',+ region: raw.region || '',+ longitude: raw.location_lng ?? null,+ latitude: raw.location_lat ?? null,+ os: this.mapOs(raw.os_name),+ os_version: raw.os_version || '',+ browser: '',+ browser_version: '',+ device: this.getDeviceType(raw),+ brand: this.getBrand(raw),+ model: raw.device_model || raw.device_type || '',+ duration: 0,+ path,+ origin,+ referrer: referrer?.url || '',+ referrer_name: utmReferrer?.name || referrer?.name || '',+ referrer_type: referrer?.type || utmReferrer?.type || '',+ imported_at: new Date().toISOString(),+ sdk_name: raw.library+ ? `${this.provider} (${raw.library})`+ : this.provider,+ sdk_version: this.version,+ groups: [],+ };+ }++ /**+ * Map Amplitude system event names to OpenPanel conventions. Custom event+ * names pass through unchanged.+ */+ private mapEventName(eventType: string): string {+ if (+ eventType === '[Amplitude] Page Viewed' ||+ eventType === 'Page Viewed' ||+ eventType === 'Screen Viewed'+ ) {+ return 'screen_view';+ }+ return eventType;+ }++ /**+ * Parse Amplitude's "YYYY-MM-DD HH:mm:ss.SSSSSS" UTC timestamps. Trims the+ * microseconds to milliseconds and pins the zone to UTC so we don't fall back+ * to the host's local timezone.+ */+ private parseTimestamp(value?: string | null): Date {+ if (!value) {+ return new Date();+ }+ const normalized = `${value.replace(' ', 'T').replace(/(\.\d{3})\d*$/, '$1')}Z`;+ const date = new Date(normalized);+ return Number.isNaN(date.getTime()) ? new Date(value) : date;+ }++ private mapOs(osName?: string | null): string {+ const os = (osName || '').toLowerCase();+ const mapping: Record<string, string> = {+ ios: 'iOS',+ android: 'Android',+ 'mac os x': 'Mac OS',+ macos: 'Mac OS',+ windows: 'Windows',+ linux: 'Linux',+ };+ return mapping[os] || osName || '';+ }++ private getBrand(raw: AmplitudeRawEvent): string {+ if (raw.device_brand) {+ return raw.device_brand;+ }+ if (raw.device_manufacturer) {+ return raw.device_manufacturer;+ }+ // Derive from device_family (e.g. "Apple iPhone" -> "Apple")+ const family = raw.device_family || raw.device_type || '';+ if (/apple|iphone|ipad|ios/i.test(family)) {+ return 'Apple';+ }+ return '';+ }++ private getDeviceType(raw: AmplitudeRawEvent): string {+ const os = (raw.os_name || '').toLowerCase();+ const platform = (raw.platform || '').toLowerCase();+ const family = `${raw.device_family || ''} ${raw.device_type || ''}`.toLowerCase();++ const isMobileOs =+ os === 'ios' ||+ os === 'android' ||+ platform === 'ios' ||+ platform === 'android';++ if (isMobileOs) {+ return family.includes('ipad') || family.includes('tablet')+ ? 'tablet'+ : 'mobile';+ }++ if (platform === 'web' || /chrome|safari|firefox|edge/.test(os)) {+ return 'desktop';+ }++ // Unknown platform: Amplitude exports are overwhelmingly mobile, but a+ // present platform that isn't web is a stronger mobile signal than not.+ return platform ? 'mobile' : 'desktop';+ }++ /**+ * Build the OpenPanel properties bag from Amplitude event_properties: drop+ * keys already represented elsewhere (URLs, query params) and parse+ * stringified JSON so toDots() can flatten nested structures.+ */+ private buildProperties(+ props: Record<string, unknown>,+ query: Record<string, string>+ ): Record<string, any> {+ const strip = new Set([...URL_LIKE_KEYS, 'referrer', '$referrer', ...Object.keys(query)]);+ const result: Record<string, any> = {};++ for (const [key, value] of Object.entries(props)) {+ if (strip.has(key) || value == null) {+ continue;+ }+ if (+ typeof value === 'string' &&+ (value.startsWith('{') || value.startsWith('['))+ ) {+ try {+ result[key] = JSON.parse(value);+ } catch {+ result[key] = value;+ }+ } else {+ result[key] = value;+ }+ }++ return result;+ }+}diff --git a/packages/importer/src/providers/country-codes.ts b/packages/importer/src/providers/country-codes.tsnew file mode 100644index 000000000..ab2c84b2c--- /dev/null+++ b/packages/importer/src/providers/country-codes.ts@@ -0,0 +1,302 @@+/**+ * Map a country *name* (as exported by Amplitude / MaxMind) to its ISO 3166-1+ * alpha-2 code. OpenPanel's ClickHouse `country` column is FixedString(2), so+ * anything that isn't a clean 2-letter code must be converted or dropped —+ * inserting a full name like "Nigeria" throws "Too large value for+ * FixedString(2)".+ *+ * Names are matched case-insensitively and include the common aliases MaxMind+ * uses (e.g. "Russia", "South Korea", "Czechia"). Unknown names return '' so a+ * bad/rare value never breaks the import.+ */++// Canonical ISO 3166-1 alpha-2 → English name, plus alias names that mapping+// providers commonly emit. Keys are lowercased at build time.+const NAME_TO_ALPHA2: Record<string, string> = {};++const CANONICAL: [string, string[]][] = [+ ['AD', ['Andorra']],+ ['AE', ['United Arab Emirates']],+ ['AF', ['Afghanistan']],+ ['AG', ['Antigua and Barbuda']],+ ['AI', ['Anguilla']],+ ['AL', ['Albania']],+ ['AM', ['Armenia']],+ ['AO', ['Angola']],+ ['AQ', ['Antarctica']],+ ['AR', ['Argentina']],+ ['AS', ['American Samoa']],+ ['AT', ['Austria']],+ ['AU', ['Australia']],+ ['AW', ['Aruba']],+ ['AX', ['Åland Islands', 'Aland Islands']],+ ['AZ', ['Azerbaijan']],+ ['BA', ['Bosnia and Herzegovina']],+ ['BB', ['Barbados']],+ ['BD', ['Bangladesh']],+ ['BE', ['Belgium']],+ ['BF', ['Burkina Faso']],+ ['BG', ['Bulgaria']],+ ['BH', ['Bahrain']],+ ['BI', ['Burundi']],+ ['BJ', ['Benin']],+ ['BL', ['Saint Barthélemy', 'Saint Barthelemy']],+ ['BM', ['Bermuda']],+ ['BN', ['Brunei', 'Brunei Darussalam']],+ ['BO', ['Bolivia']],+ ['BQ', ['Bonaire, Sint Eustatius and Saba', 'Caribbean Netherlands']],+ ['BR', ['Brazil']],+ ['BS', ['Bahamas', 'The Bahamas']],+ ['BT', ['Bhutan']],+ ['BV', ['Bouvet Island']],+ ['BW', ['Botswana']],+ ['BY', ['Belarus']],+ ['BZ', ['Belize']],+ ['CA', ['Canada']],+ ['CC', ['Cocos (Keeling) Islands']],+ [+ 'CD',+ [+ 'Congo, The Democratic Republic of the',+ 'Democratic Republic of the Congo',+ 'DR Congo',+ ],+ ],+ ['CF', ['Central African Republic']],+ ['CG', ['Congo', 'Republic of the Congo']],+ ['CH', ['Switzerland']],+ ['CI', ["Côte d'Ivoire", "Cote d'Ivoire", 'Ivory Coast']],+ ['CK', ['Cook Islands']],+ ['CL', ['Chile']],+ ['CM', ['Cameroon']],+ ['CN', ['China']],+ ['CO', ['Colombia']],+ ['CR', ['Costa Rica']],+ ['CU', ['Cuba']],+ ['CV', ['Cape Verde', 'Cabo Verde']],+ ['CW', ['Curaçao', 'Curacao']],+ ['CX', ['Christmas Island']],+ ['CY', ['Cyprus']],+ ['CZ', ['Czechia', 'Czech Republic']],+ ['DE', ['Germany']],+ ['DJ', ['Djibouti']],+ ['DK', ['Denmark']],+ ['DM', ['Dominica']],+ ['DO', ['Dominican Republic']],+ ['DZ', ['Algeria']],+ ['EC', ['Ecuador']],+ ['EE', ['Estonia']],+ ['EG', ['Egypt']],+ ['EH', ['Western Sahara']],+ ['ER', ['Eritrea']],+ ['ES', ['Spain']],+ ['ET', ['Ethiopia']],+ ['FI', ['Finland']],+ ['FJ', ['Fiji']],+ ['FK', ['Falkland Islands', 'Falkland Islands (Malvinas)']],+ ['FM', ['Micronesia', 'Micronesia, Federated States of']],+ ['FO', ['Faroe Islands']],+ ['FR', ['France']],+ ['GA', ['Gabon']],+ ['GB', ['United Kingdom', 'UK', 'Great Britain']],+ ['GD', ['Grenada']],+ ['GE', ['Georgia']],+ ['GF', ['French Guiana']],+ ['GG', ['Guernsey']],+ ['GH', ['Ghana']],+ ['GI', ['Gibraltar']],+ ['GL', ['Greenland']],+ ['GM', ['Gambia', 'The Gambia']],+ ['GN', ['Guinea']],+ ['GP', ['Guadeloupe']],+ ['GQ', ['Equatorial Guinea']],+ ['GR', ['Greece']],+ ['GS', ['South Georgia and the South Sandwich Islands']],+ ['GT', ['Guatemala']],+ ['GU', ['Guam']],+ ['GW', ['Guinea-Bissau']],+ ['GY', ['Guyana']],+ ['HK', ['Hong Kong']],+ ['HM', ['Heard Island and McDonald Islands']],+ ['HN', ['Honduras']],+ ['HR', ['Croatia']],+ ['HT', ['Haiti']],+ ['HU', ['Hungary']],+ ['ID', ['Indonesia']],+ ['IE', ['Ireland']],+ ['IL', ['Israel']],+ ['IM', ['Isle of Man']],+ ['IN', ['India']],+ ['IO', ['British Indian Ocean Territory']],+ ['IQ', ['Iraq']],+ ['IR', ['Iran', 'Iran, Islamic Republic of']],+ ['IS', ['Iceland']],+ ['IT', ['Italy']],+ ['JE', ['Jersey']],+ ['JM', ['Jamaica']],+ ['JO', ['Jordan']],+ ['JP', ['Japan']],+ ['KE', ['Kenya']],+ ['KG', ['Kyrgyzstan']],+ ['KH', ['Cambodia']],+ ['KI', ['Kiribati']],+ ['KM', ['Comoros']],+ ['KN', ['Saint Kitts and Nevis']],+ ['KP', ['North Korea', "Korea, Democratic People's Republic of"]],+ ['KR', ['South Korea', 'Korea, Republic of', 'Republic of Korea']],+ ['KW', ['Kuwait']],+ ['KY', ['Cayman Islands']],+ ['KZ', ['Kazakhstan']],+ ['LA', ['Laos', "Lao People's Democratic Republic"]],+ ['LB', ['Lebanon']],+ ['LC', ['Saint Lucia']],+ ['LI', ['Liechtenstein']],+ ['LK', ['Sri Lanka']],+ ['LR', ['Liberia']],+ ['LS', ['Lesotho']],+ ['LT', ['Lithuania']],+ ['LU', ['Luxembourg']],+ ['LV', ['Latvia']],+ ['LY', ['Libya']],+ ['MA', ['Morocco']],+ ['MC', ['Monaco']],+ ['MD', ['Moldova', 'Moldova, Republic of']],+ ['ME', ['Montenegro']],+ ['MF', ['Saint Martin', 'Saint Martin (French part)']],+ ['MG', ['Madagascar']],+ ['MH', ['Marshall Islands']],+ ['MK', ['North Macedonia', 'Macedonia']],+ ['ML', ['Mali']],+ ['MM', ['Myanmar', 'Burma']],+ ['MN', ['Mongolia']],+ ['MO', ['Macao', 'Macau']],+ ['MP', ['Northern Mariana Islands']],+ ['MQ', ['Martinique']],+ ['MR', ['Mauritania']],+ ['MS', ['Montserrat']],+ ['MT', ['Malta']],+ ['MU', ['Mauritius']],+ ['MV', ['Maldives']],+ ['MW', ['Malawi']],+ ['MX', ['Mexico']],+ ['MY', ['Malaysia']],+ ['MZ', ['Mozambique']],+ ['NA', ['Namibia']],+ ['NC', ['New Caledonia']],+ ['NE', ['Niger']],+ ['NF', ['Norfolk Island']],+ ['NG', ['Nigeria']],+ ['NI', ['Nicaragua']],+ ['NL', ['Netherlands', 'The Netherlands']],+ ['NO', ['Norway']],+ ['NP', ['Nepal']],+ ['NR', ['Nauru']],+ ['NU', ['Niue']],+ ['NZ', ['New Zealand']],+ ['OM', ['Oman']],+ ['PA', ['Panama']],+ ['PE', ['Peru']],+ ['PF', ['French Polynesia']],+ ['PG', ['Papua New Guinea']],+ ['PH', ['Philippines']],+ ['PK', ['Pakistan']],+ ['PL', ['Poland']],+ ['PM', ['Saint Pierre and Miquelon']],+ ['PN', ['Pitcairn', 'Pitcairn Islands']],+ ['PR', ['Puerto Rico']],+ ['PS', ['Palestine', 'Palestine, State of', 'Palestinian Territory']],+ ['PT', ['Portugal']],+ ['PW', ['Palau']],+ ['PY', ['Paraguay']],+ ['QA', ['Qatar']],+ ['RE', ['Réunion', 'Reunion']],+ ['RO', ['Romania']],+ ['RS', ['Serbia']],+ ['RU', ['Russia', 'Russian Federation']],+ ['RW', ['Rwanda']],+ ['SA', ['Saudi Arabia']],+ ['SB', ['Solomon Islands']],+ ['SC', ['Seychelles']],+ ['SD', ['Sudan']],+ ['SE', ['Sweden']],+ ['SG', ['Singapore']],+ ['SH', ['Saint Helena', 'Saint Helena, Ascension and Tristan da Cunha']],+ ['SI', ['Slovenia']],+ ['SJ', ['Svalbard and Jan Mayen']],+ ['SK', ['Slovakia']],+ ['SL', ['Sierra Leone']],+ ['SM', ['San Marino']],+ ['SN', ['Senegal']],+ ['SO', ['Somalia']],+ ['SR', ['Suriname']],+ ['SS', ['South Sudan']],+ ['ST', ['Sao Tome and Principe', 'São Tomé and Príncipe']],+ ['SV', ['El Salvador']],+ ['SX', ['Sint Maarten', 'Sint Maarten (Dutch part)']],+ ['SY', ['Syria', 'Syrian Arab Republic']],+ ['SZ', ['Eswatini', 'Swaziland']],+ ['TC', ['Turks and Caicos Islands']],+ ['TD', ['Chad']],+ ['TF', ['French Southern Territories']],+ ['TG', ['Togo']],+ ['TH', ['Thailand']],+ ['TJ', ['Tajikistan']],+ ['TK', ['Tokelau']],+ ['TL', ['Timor-Leste', 'East Timor']],+ ['TM', ['Turkmenistan']],+ ['TN', ['Tunisia']],+ ['TO', ['Tonga']],+ ['TR', ['Turkey', 'Türkiye', 'Turkiye']],+ ['TT', ['Trinidad and Tobago']],+ ['TV', ['Tuvalu']],+ ['TW', ['Taiwan', 'Taiwan, Province of China']],+ ['TZ', ['Tanzania', 'Tanzania, United Republic of']],+ ['UA', ['Ukraine']],+ ['UG', ['Uganda']],+ ['UM', ['United States Minor Outlying Islands']],+ ['US', ['United States', 'United States of America', 'USA']],+ ['UY', ['Uruguay']],+ ['UZ', ['Uzbekistan']],+ ['VA', ['Vatican City', 'Holy See (Vatican City State)']],+ ['VC', ['Saint Vincent and the Grenadines']],+ ['VE', ['Venezuela', 'Venezuela, Bolivarian Republic of']],+ ['VG', ['British Virgin Islands', 'Virgin Islands, British']],+ ['VI', ['U.S. Virgin Islands', 'Virgin Islands, U.S.']],+ ['VN', ['Vietnam', 'Viet Nam']],+ ['VU', ['Vanuatu']],+ ['WF', ['Wallis and Futuna']],+ ['WS', ['Samoa']],+ ['XK', ['Kosovo']],+ ['YE', ['Yemen']],+ ['YT', ['Mayotte']],+ ['ZA', ['South Africa']],+ ['ZM', ['Zambia']],+ ['ZW', ['Zimbabwe']],+];++const ALPHA2_SET = new Set<string>();+for (const [code, names] of CANONICAL) {+ ALPHA2_SET.add(code);+ for (const name of names) {+ NAME_TO_ALPHA2[name.toLowerCase()] = code;+ }+}++/**+ * Convert a country value to an ISO 3166-1 alpha-2 code suitable for the+ * ClickHouse FixedString(2) column.+ *+ * - Already a valid 2-letter code → returned uppercased.+ * - A known country name (or alias) → its alpha-2 code.+ * - Anything else → '' (safe to store, never overflows).+ */+export function toCountryCode(value?: string | null): string {+ if (!value) {+ return '';+ }+ const trimmed = value.trim();+ if (trimmed.length === 2 && ALPHA2_SET.has(trimmed.toUpperCase())) {+ return trimmed.toUpperCase();+ }+ return NAME_TO_ALPHA2[trimmed.toLowerCase()] ?? '';+}diff --git a/packages/importer/src/providers/metadata.ts b/packages/importer/src/providers/metadata.tsindex 639394b72..5f81c7c90 100644--- a/packages/importer/src/providers/metadata.ts+++ b/packages/importer/src/providers/metadata.ts@@ -1,4 +1,4 @@-export type ImportProviderId = 'umami' | 'mixpanel';+export type ImportProviderId = 'umami' | 'mixpanel' | 'amplitude';export type ImportProviderType = 'file' | 'api';export interface ImportProviderMeta {@@ -27,4 +27,12 @@ export const IMPORT_PROVIDERS: ImportProviderMeta[] = [backgroundColor: '#fff',types: ['api'],},+ {+ id: 'amplitude',+ name: 'Amplitude',+ description: 'Import your analytics data from an Amplitude export',+ logo: 'https://cdn.brandfetch.io/idnYrZGER0/theme/dark/symbol.svg?c=1dxbfHSJFAPEGdCLU4o5B',+ backgroundColor: '#fff',+ types: ['file'],+ },];diff --git a/packages/importer/src/providers/umami.ts b/packages/importer/src/providers/umami.tsindex d2b5b0431..71eb14bff 100644--- a/packages/importer/src/providers/umami.ts+++ b/packages/importer/src/providers/umami.ts@@ -105,7 +105,6 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {}const contentType = res.headers.get('content-type') || '';- const contentEnc = res.headers.get('content-encoding') || '';const contentLen = Number(res.headers.get('content-length') ?? 0);if (@@ -123,33 +122,52 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {);}- const looksGzip =- /\.gz($|\?)/i.test(url) ||- /gzip/i.test(contentEnc) ||- /application\/gzip/i.test(contentType);- const looksBr = /br/i.test(contentEnc) || /\.br($|\?)/i.test(url);-// WHATWG -> Node stream- const body = Readable.fromWeb(res.body as any);-- // Optional size guard during stream+ const rawBody = Readable.fromWeb(res.body as any);++ // Detect gzip by peeking the first chunk's magic bytes (0x1f 0x8b) rather+ // than trusting Content-Encoding/Type. fetch/undici already decompresses+ // transport-level gzip/br while leaving the header in place; trusting it+ // would gunzip already-plain bytes and throw "incorrect header check".+ const iterator = rawBody[Symbol.asyncIterator]();+ const first = await iterator.next();+ const firstChunk = first.done ? undefined : (first.value as Buffer);++ const isGzip =+ !!firstChunk &&+ firstChunk.length >= 2 &&+ firstChunk[0] === 0x1f &&+ firstChunk[1] === 0x8b;+ const isBrotli = !isGzip && /\.br($|\?)/i.test(url);++ // Replay the peeked chunk and enforce the optional size guard inline.let seenBytes = 0;- if (maxBytes) {- body.on('data', (chunk: Buffer) => {+ async function* sourceChunks() {+ if (firstChunk !== undefined) {+ seenBytes += firstChunk.length;+ yield firstChunk;+ }+ for (+ let next = await iterator.next();+ !next.done;+ next = await iterator.next()+ ) {+ const chunk = next.value as Buffer;seenBytes += chunk.length;- if (seenBytes > maxBytes) {- controller.abort();- body.destroy(- new Error(`Stream exceeded size limit (${seenBytes} > ${maxBytes})`)+ if (maxBytes && seenBytes > maxBytes) {+ throw new Error(+ `Stream exceeded size limit (${seenBytes} > ${maxBytes})`);}- });+ yield chunk;+ }}+ const body = Readable.from(sourceChunks());// Build decode chain (gzip/brotli -> CSV parser)- const decompress = looksGzip+ const decompress = isGzip? createGunzip()- : looksBr+ : isBrotli? createBrotliDecompress(): null;diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.tsindex 4b2fd87cb..b02813cff 100644--- a/packages/validation/src/index.ts+++ b/packages/validation/src/index.ts@@ -659,6 +659,14 @@ export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;export const zPlausibleImportConfig = createFileImportConfig('plausible');export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;+export const zAmplitudeImportConfig = createFileImportConfig('amplitude').extend(+ {+ projectMapper: z.array(zProjectMapper),+ mapScreenViewProperty: z.string().optional(),+ },+);+export type IAmplitudeImportConfig = z.infer<typeof zAmplitudeImportConfig>;+export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);export type IMixpanelDataResidency = z.infer<typeof zMixpanelDataResidency>;@@ -678,15 +686,17 @@ export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;export type IImportConfig =| IUmamiImportConfig| IPlausibleImportConfig- | IMixpanelImportConfig;+ | IMixpanelImportConfig+ | IAmplitudeImportConfig;export const zCreateImport = z.object({projectId: z.string().min(1),- provider: z.enum(['umami', 'plausible', 'mixpanel']),+ provider: z.enum(['umami', 'plausible', 'mixpanel', 'amplitude']),config: z.union([zUmamiImportConfig,zPlausibleImportConfig,zMixpanelImportConfig,+ zAmplitudeImportConfig,]),});