Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M5-004): import module: packages/importer absorbed + import.service + trpc + /import REST + import queue

ralph315f0b5f73a0
diff --git a/apps/api/src/controllers/import.controller.ts b/apps/api/src/controllers/import.controller.ts
index 188aa7253..ae246b5ad 100644
--- a/apps/api/src/controllers/import.controller.ts
+++ b/apps/api/src/controllers/import.controller.ts
@@ -1,39 +1,28 @@
-import type { FastifyReply, FastifyRequest } from 'fastify';
-
-import { toDots } from '@openpanel/common';
+import { insertRawEventsBatch } from '@openpanel/core';
import type { IClickhouseEvent } from '@openpanel/db';
-import { TABLE_NAMES, ch, formatClickhouseDate } from '@openpanel/db';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+// Dissolved into @openpanel/core's import module (M5-004): the
+// toDots/project_id/imported_at stamping and the ClickHouse insert moved to
+// import.service.ts's `insertRawEventsBatch`. This controller stays
+// (DELEGATE PATTERN) — it is the live Fastify handler, a thin wrapper
+// resolving `request.client` and translating the result into a Fastify
+// reply.
export async function importEvents(
request: FastifyRequest<{
Body: IClickhouseEvent[];
}>,
- reply: FastifyReply,
+ reply: FastifyReply
) {
const projectId = request.client?.projectId;
if (!projectId) {
throw new Error('Project ID is required');
}
- const importedAt = formatClickhouseDate(new Date());
- const values: IClickhouseEvent[] = request.body.map((event) => {
- return {
- ...event,
- properties: toDots(event.properties),
- project_id: projectId,
- created_at: formatClickhouseDate(event.created_at),
- imported_at: importedAt,
- };
- });
-
try {
- const res = await ch.insert({
- table: TABLE_NAMES.events,
- values,
- format: 'JSONEachRow',
- });
+ const { writtenRows } = await insertRawEventsBatch(projectId, request.body);
- console.log(res.summary?.written_rows, 'events imported');
+ console.log(writtenRows, 'events imported');
reply.send('OK');
} catch (e) {
console.error(e);
diff --git a/apps/start/Dockerfile b/apps/start/Dockerfile
index 02e4bd0a7..ee139ae8f 100644
--- a/apps/start/Dockerfile
+++ b/apps/start/Dockerfile
@@ -29,7 +29,6 @@ COPY apps/start/package.json ./apps/start/
COPY packages/trpc/package.json packages/trpc/
COPY packages/json/package.json packages/json/
COPY packages/common/package.json packages/common/
-COPY packages/importer/package.json packages/importer/
COPY packages/payments/package.json packages/payments/
COPY packages/constants/package.json packages/constants/
COPY packages/validation/package.json packages/validation/
@@ -86,7 +85,6 @@ COPY --from=build /app/packages/trpc/package.json ./packages/trpc/
COPY --from=build /app/packages/auth/package.json ./packages/auth/
COPY --from=build /app/packages/json/package.json ./packages/json/
COPY --from=build /app/packages/common/package.json ./packages/common/
-COPY --from=build /app/packages/importer/package.json ./packages/importer/
COPY --from=build /app/packages/payments/package.json ./packages/payments/
COPY --from=build /app/packages/constants/package.json ./packages/constants/
COPY --from=build /app/packages/validation/package.json ./packages/validation/
diff --git a/apps/start/package.json b/apps/start/package.json
index 3421bea54..a9fba591b 100644
--- a/apps/start/package.json
+++ b/apps/start/package.json
@@ -33,7 +33,7 @@
"@number-flow/react": "0.5.10",
"@openpanel/common": "workspace:^",
"@openpanel/constants": "workspace:^",
- "@openpanel/importer": "workspace:^",
+ "@openpanel/core": "workspace:^",
"@openpanel/json": "workspace:*",
"@openpanel/payments": "workspace:*",
"@openpanel/sdk-info": "workspace:^",
diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.imports.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.imports.tsx
index 602ceb440..f2e9943cf 100644
--- a/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.imports.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.imports.tsx
@@ -1,4 +1,4 @@
-import { IMPORT_PROVIDERS } from '@openpanel/importer/providers';
+import { IMPORT_PROVIDERS } from '@openpanel/core/modules/import/import.constants';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { formatDistanceToNow } from 'date-fns';
diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile
index bc485eb89..dd496926b 100644
--- a/apps/worker/Dockerfile
+++ b/apps/worker/Dockerfile
@@ -34,7 +34,6 @@ COPY packages/redis/package.json ./packages/redis/
COPY packages/queue/package.json ./packages/queue/
COPY packages/logger/package.json ./packages/logger/
COPY packages/common/package.json ./packages/common/
-COPY packages/importer/package.json ./packages/importer/
COPY packages/payments/package.json ./packages/payments/
COPY packages/constants/package.json ./packages/constants/
COPY packages/validation/package.json ./packages/validation/
@@ -82,7 +81,6 @@ COPY --from=build /app/packages/redis ./packages/redis
COPY --from=build /app/packages/queue ./packages/queue
COPY --from=build /app/packages/logger ./packages/logger
COPY --from=build /app/packages/common ./packages/common
-COPY --from=build /app/packages/importer ./packages/importer
COPY --from=build /app/packages/payments ./packages/payments
COPY --from=build /app/packages/constants ./packages/constants
COPY --from=build /app/packages/validation ./packages/validation
diff --git a/apps/worker/package.json b/apps/worker/package.json
index f7b38a3f7..0b8f59a2d 100644
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -18,7 +18,6 @@
"@openpanel/core": "workspace:*",
"@openpanel/db": "workspace:*",
"@openpanel/email": "workspace:*",
- "@openpanel/importer": "workspace:*",
"@openpanel/payments": "workspace:*",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*",
diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.ts
index a767c7e30..6941ea53b 100644
--- a/apps/worker/src/jobs/import.ts
+++ b/apps/worker/src/jobs/import.ts
@@ -1,395 +1,15 @@
-import type { ILogger } from '@openpanel/core';
-import {
- backfillSessionsToProduction,
- cleanupSessionStartEndEvents,
- cleanupStagingData,
- createSessionsStartEndEvents,
- db,
- generateGapBasedSessionIds,
- getImportDateBounds,
- type IClickhouseEvent,
- type IClickhouseProfile,
- insertImportBatch,
- insertProfilesBatch,
- moveImportsToProduction,
- type Prisma,
- updateImportStatus,
-} from '@openpanel/db';
-import {
- AmplitudeProvider,
- MixpanelProvider,
- UmamiProvider,
-} from '@openpanel/importer';
+// Dissolved into @openpanel/core's import module (M5-004): the ClickHouse
+// staging/session pipeline, the provider dispatch and the job body itself
+// moved to packages/core/src/modules/import/import.service.ts. This file
+// stays (DELEGATE PATTERN) — it is the BullMQ job body V1's worker registers
+// (boot-workers.ts), a thin wrapper handing the real BullMQ `Job` to
+// `runImportJob` as its progress reporter (`ImportJobProgress` — a BullMQ
+// `Job` satisfies it structurally, see import.service.ts's header).
+import { runImportJob } from '@openpanel/core';
import type { ImportQueuePayload } from '@openpanel/queue';
import type { Job } from 'bullmq';
-import { logger } from '../utils/logger';
-
-const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10);
-
-function yieldToEventLoop(): Promise<void> {
- return new Promise((resolve) => {
- setTimeout(resolve, 100);
- });
-}
-
-const RESUMABLE_STEPS = ['creating_sessions', 'moving', 'backfilling_sessions'];
-
-/**
- * Merge a freshly-derived profile into the bounded dedup map: keep the earliest
- * created_at (first seen), advance last_seen_at to the latest activity, and fill
- * identity fields/properties, preferring the newer row.
- */
-function mergeProfileInto(
- map: Map<string, IClickhouseProfile>,
- incoming: IClickhouseProfile
-): void {
- const existing = map.get(incoming.id);
- if (!existing) {
- map.set(incoming.id, incoming);
- return;
- }
-
- const createdAt =
- existing.created_at < incoming.created_at
- ? existing.created_at
- : incoming.created_at;
- const isIncomingNewer = incoming.last_seen_at >= existing.last_seen_at;
- const base = isIncomingNewer ? incoming : existing;
- const other = isIncomingNewer ? existing : incoming;
-
- map.set(incoming.id, {
- ...base,
- created_at: createdAt,
- first_name: base.first_name || other.first_name,
- last_name: base.last_name || other.last_name,
- email: base.email || other.email,
- avatar: base.avatar || other.avatar,
- properties: { ...other.properties, ...base.properties },
- });
-}
export async function importJob(job: Job<ImportQueuePayload>) {
const { importId } = job.data.payload;
-
- const record = await db.import.findUniqueOrThrow({
- where: { id: importId },
- include: { project: true },
- });
-
- const jobLogger = logger.child({ importId, config: record.config });
- jobLogger.info('Starting import job');
-
- const providerInstance = createProvider(record, jobLogger);
- const shouldGenerateSessionIds = providerInstance.shouldGenerateSessionIds();
-
- try {
- const isRetry = record.currentStep !== null;
- const canResume =
- isRetry && RESUMABLE_STEPS.includes(record.currentStep as string);
-
- // -------------------------------------------------------
- // STAGING PHASE: clean slate on failure, run from scratch
- // -------------------------------------------------------
- if (!canResume) {
- if (isRetry) {
- jobLogger.info(
- 'Retry detected before resumable phase — cleaning staging data'
- );
- await cleanupStagingData(importId);
- }
-
- // Phase 1: Load events into staging
- await updateImportStatus(jobLogger, job, importId, { step: 'loading' });
-
- const totalEvents = await providerInstance
- .getTotalEventsCount()
- .catch(() => -1);
- let processedEvents = 0;
- const eventBatch: IClickhouseEvent[] = [];
-
- // Profiles derived inline from events (e.g. Amplitude, which has no
- // profile export API). A bounded dedup map keeps memory flat regardless of
- // event volume; the profiles table's ReplacingMergeTree(last_seen_at)
- // collapses any duplicate rows that span flush boundaries to the latest
- // activity per id.
- const canProfileFromEvents =
- typeof (providerInstance as AmplitudeProvider)
- .transformEventToProfile === 'function';
- const PROFILE_MAP_CAP = 50_000;
- const profileMap = new Map<string, IClickhouseProfile>();
- let processedProfiles = 0;
-
- const flushProfiles = async () => {
- if (profileMap.size === 0) {
- return;
- }
- const values = Array.from(profileMap.values());
- await insertProfilesBatch(values, record.projectId);
- processedProfiles += values.length;
- profileMap.clear();
- await updateImportStatus(jobLogger, job, importId, {
- step: 'loading_profiles',
- processedProfiles,
- });
- await yieldToEventLoop();
- };
-
- for await (const rawEvent of providerInstance.parseSource()) {
- if (
- !providerInstance.validate(
- // @ts-expect-error -- provider-specific raw type
- rawEvent
- )
- ) {
- jobLogger.warn({ rawEvent }, 'Skipping invalid event');
- continue;
- }
-
- const transformed: IClickhouseEvent = providerInstance.transformEvent(
- // @ts-expect-error -- provider-specific raw type
- rawEvent
- );
-
- // Session IDs for providers that need them (e.g. Mixpanel) are generated
- // in generateGapBasedSessionIds after loading, using gap-based logic.
- eventBatch.push(transformed);
-
- if (canProfileFromEvents) {
- const profile = (
- providerInstance as AmplitudeProvider
- ).transformEventToProfile(
- // @ts-expect-error -- provider-specific raw type
- rawEvent
- );
- if (profile) {
- mergeProfileInto(profileMap, profile);
- if (profileMap.size >= PROFILE_MAP_CAP) {
- await flushProfiles();
- }
- }
- }
-
- if (eventBatch.length >= BATCH_SIZE) {
- await insertImportBatch(eventBatch, importId);
- processedEvents += eventBatch.length;
-
- const batchDate = new Date(eventBatch[0]?.created_at || '')
- .toISOString()
- .split('T')[0];
-
- await updateImportStatus(jobLogger, job, importId, {
- step: 'loading',
- batch: batchDate,
- totalEvents,
- processedEvents,
- });
-
- eventBatch.length = 0;
- await yieldToEventLoop();
- }
- }
-
- if (eventBatch.length > 0) {
- await insertImportBatch(eventBatch, importId);
- processedEvents += eventBatch.length;
-
- const batchDate = new Date(eventBatch[0]?.created_at || '')
- .toISOString()
- .split('T')[0];
-
- await updateImportStatus(jobLogger, job, importId, {
- step: 'loading',
- batch: batchDate,
- totalEvents,
- processedEvents,
- });
- eventBatch.length = 0;
- }
-
- jobLogger.info({ processedEvents }, 'Loading complete');
-
- // Phase 1a: Flush profiles derived inline from events (Amplitude)
- if (canProfileFromEvents) {
- await flushProfiles();
- jobLogger.info(
- { processedProfiles },
- 'Inline profile derivation complete'
- );
- }
-
- // Phase 1b: Load user profiles (Mixpanel only)
- const profileBatchSize = 5000;
- if (
- 'streamProfiles' in providerInstance &&
- typeof (providerInstance as MixpanelProvider).streamProfiles ===
- 'function'
- ) {
- await updateImportStatus(jobLogger, job, importId, {
- step: 'loading_profiles',
- });
-
- const profileBatch: IClickhouseProfile[] = [];
- let processedProfiles = 0;
-
- for await (const rawProfile of (
- providerInstance as MixpanelProvider
- ).streamProfiles()) {
- const profile = (
- providerInstance as MixpanelProvider
- ).transformProfile(rawProfile);
- profileBatch.push(profile);
-
- if (profileBatch.length >= profileBatchSize) {
- await insertProfilesBatch(profileBatch, record.projectId);
- processedProfiles += profileBatch.length;
- await updateImportStatus(jobLogger, job, importId, {
- step: 'loading_profiles',
- processedProfiles,
- });
- profileBatch.length = 0;
- await yieldToEventLoop();
- }
- }
-
- if (profileBatch.length > 0) {
- await insertProfilesBatch(profileBatch, record.projectId);
- processedProfiles += profileBatch.length;
- await updateImportStatus(jobLogger, job, importId, {
- step: 'loading_profiles',
- processedProfiles,
- totalProfiles: processedProfiles,
- });
- }
-
- jobLogger.info({ processedProfiles }, 'Profile loading complete');
- }
-
- // Phase 2: Generate gap-based session IDs (Mixpanel etc.)
- if (shouldGenerateSessionIds) {
- await updateImportStatus(jobLogger, job, importId, {
- step: 'generating_sessions',
- });
- await generateGapBasedSessionIds(importId);
- await yieldToEventLoop();
- jobLogger.info('Session ID generation complete');
- }
- }
-
- // -------------------------------------------------------
- // SESSION CREATION PHASE: resumable by cleaning session_start/end
- // -------------------------------------------------------
- const skipSessionCreation =
- canResume && record.currentStep !== 'creating_sessions';
-
- if (!skipSessionCreation) {
- if (canResume && record.currentStep === 'creating_sessions') {
- jobLogger.info(
- 'Retry at creating_sessions — cleaning existing session_start/end events'
- );
- await cleanupSessionStartEndEvents(importId);
- }
-
- await updateImportStatus(jobLogger, job, importId, {
- step: 'creating_sessions',
- batch: 'all sessions',
- });
- await createSessionsStartEndEvents(importId);
- await yieldToEventLoop();
-
- jobLogger.info('Session event creation complete');
- }
-
- // -------------------------------------------------------
- // PRODUCTION PHASE: resume-safe, track progress per batch
- // -------------------------------------------------------
-
- // Phase 3: Move staging events to production (per-day)
- const resumeMovingFrom =
- canResume && record.currentStep === 'moving'
- ? (record.currentBatch ?? undefined)
- : undefined;
-
- // currentBatch is the last successfully completed day — resume from the next day to avoid re-inserting it
- const moveFromDate = (() => {
- if (!resumeMovingFrom) {
- return undefined;
- }
- const next = new Date(`${resumeMovingFrom}T12:00:00Z`);
- next.setUTCDate(next.getUTCDate() + 1);
- return next.toISOString().split('T')[0]!;
- })();
-
- const bounds = await getImportDateBounds(importId, moveFromDate);
- if (bounds.min && bounds.max) {
- const startDate = bounds.min.split(' ')[0]!;
- const endDate = bounds.max.split(' ')[0]!;
- const cursor = new Date(`${startDate}T12:00:00Z`);
- const end = new Date(`${endDate}T12:00:00Z`);
-
- while (cursor <= end) {
- const dateStr = cursor.toISOString().split('T')[0]!;
-
- await moveImportsToProduction(importId, dateStr);
- await updateImportStatus(jobLogger, job, importId, {
- step: 'moving',
- batch: dateStr,
- });
-
- await yieldToEventLoop();
- cursor.setUTCDate(cursor.getUTCDate() + 1);
- }
- }
-
- jobLogger.info('Move to production complete');
-
- // Phase 4: Backfill sessions table
- await updateImportStatus(jobLogger, job, importId, {
- step: 'backfilling_sessions',
- batch: 'all sessions',
- });
- await backfillSessionsToProduction(importId);
- await yieldToEventLoop();
-
- jobLogger.info('Session backfill complete');
-
- // Done
- await updateImportStatus(jobLogger, job, importId, { step: 'completed' });
- jobLogger.info('Import completed');
-
- return { success: true };
- } catch (error) {
- jobLogger.error({ err: error }, 'Import job failed');
-
- try {
- const errorMsg = error instanceof Error ? error.message : 'Unknown error';
- await updateImportStatus(jobLogger, job, importId, {
- step: 'failed',
- errorMessage: errorMsg,
- });
- } catch (markError) {
- jobLogger.error(
- { err: error, markError },
- 'Failed to mark import as failed'
- );
- }
-
- throw error;
- }
-}
-
-function createProvider(
- record: Prisma.ImportGetPayload<{ include: { project: true } }>,
- jobLogger: ILogger
-) {
- const config = record.config;
- switch (config.provider) {
- case 'umami':
- return new UmamiProvider(record.projectId, config, jobLogger);
- case 'mixpanel':
- return new MixpanelProvider(record.projectId, config, jobLogger);
- case 'amplitude':
- return new AmplitudeProvider(record.projectId, config, jobLogger);
- default:
- throw new Error(`Unknown provider: ${config.provider}`);
- }
+ return runImportJob(importId, job);
}
diff --git a/packages/core/package.json b/packages/core/package.json
index 5eba5c77e..82ae8262e 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -41,6 +41,7 @@
"@trpc/server": "^11.17.0",
"arctic": "^2.3.0",
"bullmq": "6.3.3",
+ "csv-parse": "^6.1.0",
"elysia": "1.4.30",
"ipaddr.js": "^2.5.0",
"lru-cache": "^11.2.4",
@@ -49,17 +50,21 @@
"pino-pretty": "catalog:",
"prom-client": "^15.1.3",
"qrcode": "^1.5.4",
+ "ramda": "^0.29.1",
"sqlstring": "^2.3.3",
"superjson": "^1.13.3",
"ua-parser-js": "^2.0.6",
"undici": "^8.10.0",
+ "unzipper": "^0.12.3",
"zod": "catalog:"
},
"devDependencies": {
"@clickhouse/client": "^1.18.5",
"@openpanel/tsconfig": "workspace:*",
"@types/qrcode": "^1.5.6",
+ "@types/ramda": "^0.31.1",
"@types/sqlstring": "^2.3.2",
+ "@types/unzipper": "^0.10.10",
"bun-types": "1.4.0",
"tar": "^7.4.3",
"typescript": "catalog:"
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index dbf5153c4..0d7a96fcd 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -209,6 +209,34 @@ export {
selectGscSite,
syncGscData,
} from './modules/gsc/gsc.service';
+// Dissolved from @openpanel/db's services/import.service.ts +
+// apps/worker's job file + apps/api's /import controller (M5-004) —
+// apps/worker's import job file and apps/api's import controller call these
+// directly, the same way V1 reaches every other dissolved service here.
+// packages/db/src/services/import.service.ts is deleted outright: nothing
+// else reached it through @openpanel/db's barrel (same as cohort, M5-003).
+export type {
+ ImportJobProgress,
+ ImportService,
+ ImportStageResult,
+ ImportSteps,
+ InsertRawEventsResult,
+ UpdateImportStatusOptions,
+} from './modules/import/import.service';
+export {
+ backfillSessionsToProduction,
+ cleanupSessionStartEndEvents,
+ cleanupStagingData,
+ createSessionsStartEndEvents,
+ generateGapBasedSessionIds,
+ getImportDateBounds,
+ insertImportBatch,
+ insertProfilesBatch,
+ insertRawEventsBatch,
+ moveImportsToProduction,
+ runImportJob,
+ updateImportStatus,
+} from './modules/import/import.service';
// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service
// (M5-001) — apps/worker's insight job files and packages/trpc's insight
// router call these directly, the same way V1 reaches every other dissolved
diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.ts
index d853841a0..4b154cbf2 100644
--- a/packages/core/src/jobs.registry.test.ts
+++ b/packages/core/src/jobs.registry.test.ts
@@ -135,3 +135,15 @@ test('the cron queue carries the cohort module cron fragment', () => {
name: 'cohortRefresh',
});
});
+
+// M5-004: the import module's own job. No cron fragment — imports are always
+// user-triggered (import.rpc.ts's create/retry), unlike gsc/cohort/insight.
+// legacyCompat.import already discriminates on this exact name
+// (jobs/compat.ts), so this pins the two in agreement.
+test('the import queue carries the import module job', () => {
+ expect(Object.keys(queues.import.jobs)).toEqual(['import']);
+ expect(queues.import.jobs.import).toMatchObject({
+ queue: 'import',
+ name: 'import',
+ });
+});
diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.ts
index 5444a7d2d..f5f8905d0 100644
--- a/packages/core/src/jobs.registry.ts
+++ b/packages/core/src/jobs.registry.ts
@@ -16,6 +16,7 @@ import { defineQueue } from './jobs/define';
import type { ProducerHandle } from './jobs/producers';
import { cohortCronJobs, cohortQueueJobs } from './modules/cohort/cohort.jobs';
import { gscCronJobs, gscQueueJobs } from './modules/gsc/gsc.jobs';
+import { importQueueJobs } from './modules/import/import.jobs';
import {
insightCronJobs,
insightQueueJobs,
@@ -71,7 +72,9 @@ const importQueue = defineQueue('import', {
defaults: { removeOnComplete: 10, removeOnFail: 50 },
worker: { concurrency: CONCURRENCY.import },
compat: legacyCompat.import,
- jobs: {},
+ jobs: {
+ ...importQueueJobs,
+ },
});
const insights = defineQueue('insights', {
diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.ts
index a31f7d4ee..3db8fc483 100644
--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts
+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts
@@ -33,6 +33,7 @@ function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {
auth: {} as Services['auth'],
insight: {} as Services['insight'],
gsc: {} as Services['gsc'],
+ import: {} as Services['import'],
cohort: cohort as CohortService,
};
return {
diff --git a/packages/core/src/modules/cohort/cohort.service.test.ts b/packages/core/src/modules/cohort/cohort.service.test.ts
index 5f4ff3f53..d70b0d701 100644
--- a/packages/core/src/modules/cohort/cohort.service.test.ts
+++ b/packages/core/src/modules/cohort/cohort.service.test.ts
@@ -60,10 +60,12 @@ const chQuery = mock(async () => [] as unknown[]);
const chInsert = mock(async (_args: { table: string }) => undefined);
const chCommand = mock(async (_args: { query: string }) => undefined);
const getReplicatedTableName = mock((table: string) => table);
-// `originalCh`/`TABLE_NAMES` are unused here but included because
-// `mock.module` replaces this specifier process-wide (bun runs every test
-// file in one shared module registry without `--isolate` — see AGENTS.md) —
-// gsc.service.test.ts and insight.service.test.ts mock the same path, so
+// `originalCh`/`TABLE_NAMES`/`formatClickhouseDate`/
+// `convertClickhouseDateToJs` and the extra TABLE_NAMES keys are unused here
+// but included because `mock.module` replaces this specifier process-wide
+// (bun runs every test file in one shared module registry without
+// `--isolate` — see AGENTS.md) — gsc.service.test.ts,
+// insight.service.test.ts and import.service.test.ts mock the same path, so
// every factory must be a superset of every consumer's needs, whichever one
// ends up registered last.
mock.module('@openpanel/db/src/clickhouse/client', () => ({
@@ -74,7 +76,19 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({
},
chQuery,
getReplicatedTableName,
- TABLE_NAMES: { sessions: 'sessions' },
+ TABLE_NAMES: {
+ events: 'events',
+ events_imports: 'events_imports',
+ profiles: 'profiles',
+ sessions: 'sessions',
+ },
+ formatClickhouseDate: (date: Date | string) =>
+ new Date(date)
+ .toISOString()
+ .replace('T', ' ')
+ .replace(/(\.\d{3})?Z+$/, ''),
+ convertClickhouseDateToJs: (date: string) =>
+ new Date(`${date.replace(' ', 'T')}Z`),
}));
let subject: typeof import('./cohort.service');
diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.ts
index ce356a350..8a6fefd86 100644
--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts
+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts
@@ -29,6 +29,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {
auth: {} as Services['auth'],
insight: {} as Services['insight'],
cohort: {} as Services['cohort'],
+ import: {} as Services['import'],
gsc: gsc as GscService,
};
return {
@@ -120,6 +121,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', async
auth: {} as Services['auth'],
insight: {} as Services['insight'],
cohort: {} as Services['cohort'],
+ import: {} as Services['import'],
gsc: {
listConnectionsForSync: async () => [
{ projectId: 'p1' },
diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.ts
index e535f7743..e1fbb0159 100644
--- a/packages/core/src/modules/gsc/gsc.service.test.ts
+++ b/packages/core/src/modules/gsc/gsc.service.test.ts
@@ -95,10 +95,12 @@ const originalCh = {
query: mock(async () => ({ json: async () => [] as unknown[] })),
insert: mock(async () => undefined),
};
-// `ch`/`getReplicatedTableName` are unused here but included because
-// `mock.module` replaces this specifier process-wide (bun runs every test
-// file in one shared module registry without `--isolate` — see AGENTS.md) —
-// insight.service.test.ts and cohort.service.test.ts mock the same path, so
+// `ch`/`getReplicatedTableName`/`formatClickhouseDate`/
+// `convertClickhouseDateToJs` and the extra TABLE_NAMES keys are unused here
+// but included because `mock.module` replaces this specifier process-wide
+// (bun runs every test file in one shared module registry without
+// `--isolate` — see AGENTS.md) — insight.service.test.ts,
+// cohort.service.test.ts and import.service.test.ts mock the same path, so
// every factory must be a superset of every consumer's needs, whichever one
// ends up registered last.
mock.module('@openpanel/db/src/clickhouse/client', () => ({
@@ -109,7 +111,19 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({
originalCh,
chQuery,
getReplicatedTableName: mock((table: string) => table),
- TABLE_NAMES: { sessions: 'sessions' },
+ TABLE_NAMES: {
+ events: 'events',
+ events_imports: 'events_imports',
+ profiles: 'profiles',
+ sessions: 'sessions',
+ },
+ formatClickhouseDate: (date: Date | string) =>
+ new Date(date)
+ .toISOString()
+ .replace('T', ' ')
+ .replace(/(\.\d{3})?Z+$/, ''),
+ convertClickhouseDateToJs: (date: string) =>
+ new Date(`${date.replace(' ', 'T')}Z`),
}));
// Bypasses the Redis cache-aside entirely — `getGscCannibalization`'s own
diff --git a/packages/core/src/modules/import/import.constants.ts b/packages/core/src/modules/import/import.constants.ts
new file mode 100644
index 000000000..5ee8b2755
--- /dev/null
+++ b/packages/core/src/modules/import/import.constants.ts
@@ -0,0 +1,143 @@
+// Moved from packages/validation/src/index.ts + packages/importer/src/
+// providers/metadata.ts (M5-004, ADR-008's module map: import owns "C").
+// packages/validation/src/import.validation.ts becomes a re-export shim of
+// this file (same shape as packages/validation/src/cohort.validation.ts since
+// M5-003), so apps/start and packages/db/src/types.ts keep resolving these
+// symbols through @openpanel/validation's existing barrel unchanged.
+//
+// Isomorphic by the AGENTS.md rule: zod and nothing else.
+
+import { z } from 'zod';
+
+export type ImportProviderId = 'umami' | 'mixpanel' | 'amplitude';
+export type ImportProviderType = 'file' | 'api';
+
+export interface ImportProviderMeta {
+ id: ImportProviderId;
+ name: string;
+ description: string;
+ logo: string;
+ backgroundColor: string;
+ types: ImportProviderType[];
+}
+
+export const IMPORT_PROVIDERS: ImportProviderMeta[] = [
+ {
+ id: 'umami',
+ name: 'Umami',
+ description: 'Import your analytics data from Umami',
+ logo: 'https://cdn.brandfetch.io/id_3VEohOm/w/180/h/180/theme/dark/logo.png?c=1dxbfHSJFAPEGdCLU4o5B',
+ backgroundColor: '#fff',
+ types: ['file'],
+ },
+ {
+ id: 'mixpanel',
+ name: 'Mixpanel',
+ description: 'Import your analytics data from Mixpanel API',
+ logo: 'https://cdn.brandfetch.io/idr_rhI2FS/theme/dark/idMJ8uODLv.svg?c=1dxbfHSJFAPEGdCLU4o5B',
+ backgroundColor: '#fff',
+ types: ['api'],
+ },
+ {
+ id: 'amplitude',
+ name: 'Amplitude',
+ description: 'Import your analytics data from the Amplitude Export API',
+ logo: 'https://cdn.brandfetch.io/idnYrZGER0/theme/dark/symbol.svg?c=1dxbfHSJFAPEGdCLU4o5B',
+ backgroundColor: '#fff',
+ types: ['api'],
+ },
+];
+
+const zProjectMapper = z.object({
+ from: z.string().min(1),
+ to: z.string().min(1),
+});
+
+/**
+ * `z.string().url()` alone accepts `file:`, `gopher:` and friends. Restricting
+ * the scheme here gives the user an immediate form error instead of a job that
+ * fails later. It is NOT the SSRF control - the value is stored and fetched
+ * afterwards, so the destination is re-validated at fetch time by
+ * `safeFetchStream`.
+ */
+export const zHttpUrl = z
+ .string()
+ .url()
+ .refine(
+ (value) => {
+ try {
+ const { protocol } = new URL(value);
+ return protocol === 'http:' || protocol === 'https:';
+ } catch {
+ return false;
+ }
+ },
+ { message: 'Only http and https URLs are allowed' }
+ );
+
+const createFileImportConfig = <T extends string>(provider: T) =>
+ z.object({
+ provider: z.literal(provider),
+ type: z.literal('file'),
+ fileUrl: zHttpUrl,
+ });
+
+// Import configs
+export const zUmamiImportConfig = createFileImportConfig('umami').extend({
+ projectMapper: z.array(zProjectMapper),
+});
+
+export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;
+
+export const zPlausibleImportConfig = createFileImportConfig('plausible');
+export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;
+
+export const zAmplitudeDataResidency = z.enum(['us', 'eu']);
+export type IAmplitudeDataResidency = z.infer<typeof zAmplitudeDataResidency>;
+
+export const zAmplitudeImportConfig = z.object({
+ provider: z.literal('amplitude'),
+ type: z.literal('api'),
+ apiKey: z.string().min(1),
+ secretKey: z.string().min(1),
+ from: z.string().min(1),
+ to: z.string().min(1),
+ mapScreenViewProperty: z.string().optional(),
+ dataResidency: zAmplitudeDataResidency.optional(),
+});
+export type IAmplitudeImportConfig = z.infer<typeof zAmplitudeImportConfig>;
+
+export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);
+export type IMixpanelDataResidency = z.infer<typeof zMixpanelDataResidency>;
+
+export const zMixpanelImportConfig = z.object({
+ provider: z.literal('mixpanel'),
+ type: z.literal('api'),
+ serviceAccount: z.string().min(1),
+ serviceSecret: z.string().min(1),
+ projectId: z.string().min(1),
+ from: z.string().min(1),
+ to: z.string().min(1),
+ mapScreenViewProperty: z.string().optional(),
+ dataResidency: zMixpanelDataResidency.optional(),
+});
+export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;
+
+export type IImportConfig =
+ | IUmamiImportConfig
+ | IPlausibleImportConfig
+ | IMixpanelImportConfig
+ | IAmplitudeImportConfig;
+
+export const zCreateImport = z.object({
+ projectId: z.string().min(1),
+ provider: z.enum(['umami', 'plausible', 'mixpanel', 'amplitude']),
+ config: z.union([
+ zUmamiImportConfig,
+ zPlausibleImportConfig,
+ zMixpanelImportConfig,
+ zAmplitudeImportConfig,
+ ]),
+});
+
+export type ICreateImport = z.infer<typeof zCreateImport>;
diff --git a/packages/core/src/modules/import/import.jobs.test.ts b/packages/core/src/modules/import/import.jobs.test.ts
new file mode 100644
index 000000000..209c4d1a2
--- /dev/null
+++ b/packages/core/src/modules/import/import.jobs.test.ts
@@ -0,0 +1,79 @@
+// Job wiring — no ClickHouse/Postgres touched. The service methods
+// themselves are exercised in import.service.test.ts.
+
+import { expect, test } from 'bun:test';
+import type { JobCtx } from '../../context';
+import { createRecordingProducers } from '../../jobs/testing';
+import { queues } from '../../jobs.registry';
+import type { Logger } from '../../logger';
+import type { Services } from '../../services';
+import { importQueueJobs } from './import.jobs';
+import type { ImportService } from './import.service';
+
+function stubLogger(): Logger {
+ const noop = () => undefined;
+ const logger: Logger = {
+ fatal: noop,
+ error: noop,
+ warn: noop,
+ info: noop,
+ debug: noop,
+ trace: noop,
+ child: () => logger,
+ };
+ return logger;
+}
+
+function stubJobCtx(importService: Partial<ImportService>): JobCtx {
+ const services: Services = {
+ auth: {} as Services['auth'],
+ insight: {} as Services['insight'],
+ cohort: {} as Services['cohort'],
+ gsc: {} as Services['gsc'],
+ import: importService as ImportService,
+ };
+ return {
+ db: {},
+ ch: {},
+ redis: {},
+ clients: {},
+ buffers: {},
+ logger: stubLogger(),
+ queues: createRecordingProducers(queues).queues,
+ services,
+ requestId: 'req_1',
+ job: { id: 'job_1', attempt: 0, queue: 'import', name: 'import' },
+ };
+}
+
+test('the import queue declares exactly the import job', () => {
+ expect(Object.keys(importQueueJobs)).toEqual(['import']);
+ expect(queues.import.jobs.import).toMatchObject({
+ queue: 'import',
+ name: 'import',
+ });
+});
+
+test('import validates its payload', () => {
+ expect(() => importQueueJobs.import.payload.parse({})).toThrow();
+ expect(importQueueJobs.import.payload.parse({ importId: 'imp_1' })).toEqual({
+ importId: 'imp_1',
+ });
+});
+
+test('import delegates to ctx.services.import.run', async () => {
+ const calls: unknown[] = [];
+ const ctx = stubJobCtx({
+ run: async (importId) => {
+ calls.push(importId);
+ return { success: true };
+ },
+ });
+
+ await importQueueJobs.import.handler({
+ payload: { importId: 'imp_1' },
+ ctx,
+ });
+
+ expect(calls).toEqual(['imp_1']);
+});
diff --git a/packages/core/src/modules/import/import.jobs.ts b/packages/core/src/modules/import/import.jobs.ts
new file mode 100644
index 000000000..7e33d34dc
--- /dev/null
+++ b/packages/core/src/modules/import/import.jobs.ts
@@ -0,0 +1,27 @@
+// Ported from apps/worker/src/jobs/import.ts (M5-004).
+//
+// `import` is this module's own queue AND its only job — the registry key,
+// the Redis name and the BullMQ job name are all `import`
+// (legacyCompat.import already discriminates on `{type: 'import', payload}`,
+// jobs/compat.ts). Unlike gsc/cohort/insight, this queue has no cron
+// fragment: imports are always user-triggered (import.rpc.ts's create/retry).
+//
+// Core's own worker has no live BullMQ job to hand `updateImportStatus` for
+// progress reporting (see import.service.ts's `ImportJobProgress` header),
+// so this handler runs with the no-op default — inert until ADR-007's P9
+// worker cutover, same as gsc/cohort/insight's job handlers today.
+
+import { z } from 'zod';
+import { defineJob } from '../../jobs/define';
+
+const importJobPayload = z.object({ importId: z.string() });
+
+/** The `import` queue's own job. */
+export const importQueueJobs = {
+ import: defineJob({
+ payload: importJobPayload,
+ handler: async ({ payload, ctx }) => {
+ await ctx.services.import.run(payload.importId);
+ },
+ }),
+};
diff --git a/packages/core/src/modules/import/import.routes.ts b/packages/core/src/modules/import/import.routes.ts
new file mode 100644
index 000000000..e7ebf7fa5
--- /dev/null
+++ b/packages/core/src/modules/import/import.routes.ts
@@ -0,0 +1,48 @@
+// Ported from apps/api/src/routes/import.router.ts +
+// apps/api/src/controllers/import.controller.ts's business logic (M5-004).
+// V1's Fastify controller stays the LIVE route (DELEGATE PATTERN) and
+// delegates its ClickHouse insert to import.service.ts's
+// `insertRawEventsBatch` — the same function this route calls.
+//
+// NAMED GAP, same as gsc.routes.ts: this route is not yet reachable.
+// `authenticateClient` (http/client-auth.ts) is a P8 stub that always
+// returns null, so `clientAuth` 401s every request until it is filled in;
+// main.ts also does not mount `publicApiRoutes` until a real `AppDeps`
+// exists. `allow: ['read', 'root']` mirrors V1's rule (utils/auth.ts's
+// `validateImportRequest`: a `write`-type client may not import).
+//
+// V1 has no request-body schema for this route (ADR-003: "/profile,
+// /import, /event and /tools have no request schemas today, and adding them
+// would be a behaviour change"), so the body is typed but not shape-validated.
+
+import type { IClickhouseEvent } from '@openpanel/db';
+import { z } from 'zod';
+import { defineRoutes } from '../../http/define';
+import { insertRawEventsBatch } from './import.service';
+
+export const importRoutes = defineRoutes((app) =>
+ app.post(
+ '/import/events',
+ async ({ body, client, ctx, status }) => {
+ try {
+ const { writtenRows } = await insertRawEventsBatch(
+ client.projectId,
+ body as IClickhouseEvent[]
+ );
+ ctx.logger.info({ writtenRows }, 'events imported');
+ return 'OK';
+ } catch (error) {
+ ctx.logger.error({ err: error }, 'Failed to import events');
+ return status(500, 'Error');
+ }
+ },
+ {
+ clientAuth: { allow: ['read', 'root'] },
+ body: z.array(z.unknown()),
+ detail: {
+ tags: ['Import'],
+ description: 'Bulk import historical events.',
+ },
+ }
+ )
+);
diff --git a/packages/core/src/modules/import/import.rpc.test.ts b/packages/core/src/modules/import/import.rpc.test.ts
new file mode 100644
index 000000000..4d992438d
--- /dev/null
+++ b/packages/core/src/modules/import/import.rpc.test.ts
@@ -0,0 +1,75 @@
+// Only the "is anyone logged in" boundary is exercised here — no database.
+// The access-check + business logic ride on @openpanel/db (lazy-loaded, see
+// import.service.ts's header) and ctx.services.import; wiring this router
+// end-to-end against a real Postgres is P6's (protectedProcedure) job, not
+// this one's — see import.rpc.ts's header.
+
+import { expect, test } from 'bun:test';
+import { stubHttpCtx } from '../../../test/rpc-fixtures';
+import { makeTrpcContext } from '../../rpc/base';
+import type { CookieOptions } from '../../shared/cookie';
+import { importRouter } from './import.rpc';
+
+const COOKIE_OPTIONS: CookieOptions = {
+ domain: '.openpanel.dev',
+ secure: true,
+ sameSite: 'lax',
+ httpOnly: true,
+ path: '/',
+};
+
+// EMPTY_SESSION's shape (packages/db/src/services/auth-session.service.ts) —
+// `TrpcContext.session` is never literally `null`, only its `userId` is.
+const EMPTY_SESSION = { session: null, user: null, userId: null };
+
+async function anonCaller() {
+ const { ctx } = stubHttpCtx({}, EMPTY_SESSION);
+ const trpcCtx = await makeTrpcContext(ctx, new Headers(), {
+ cookieOptions: COOKIE_OPTIONS,
+ });
+ return importRouter.createCaller(trpcCtx);
+}
+
+test('list rejects an unauthenticated caller before touching a project', async () => {
+ const caller = await anonCaller();
+ await expect(caller.list({ projectId: 'proj_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
+
+test('get rejects an unauthenticated caller before reading an import', async () => {
+ const caller = await anonCaller();
+ await expect(caller.get({ id: 'imp_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
+
+test('create rejects an unauthenticated caller before writing an import', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.create({
+ projectId: 'proj_1',
+ provider: 'umami',
+ config: {
+ provider: 'umami',
+ type: 'file',
+ fileUrl: 'https://example.com/export.csv',
+ projectMapper: [],
+ },
+ })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('delete rejects an unauthenticated caller before removing an import', async () => {
+ const caller = await anonCaller();
+ await expect(caller.delete({ id: 'imp_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
+
+test('retry rejects an unauthenticated caller before re-enqueuing', async () => {
+ const caller = await anonCaller();
+ await expect(caller.retry({ id: 'imp_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
diff --git a/packages/core/src/modules/import/import.rpc.ts b/packages/core/src/modules/import/import.rpc.ts
new file mode 100644
index 000000000..80fab08a6
--- /dev/null
+++ b/packages/core/src/modules/import/import.rpc.ts
@@ -0,0 +1,177 @@
+// Ported from packages/trpc/src/routers/import.ts (M5-004).
+//
+// V1's `protectedProcedure` — the logger/session-scope/rate-limit middleware
+// stack — lands in core with auth (rpc/base.ts: "those need the resolved
+// Session shape and the access rules, so they land with auth (P6)"). Until
+// then this router does its own minimal "is anyone logged in" check inline,
+// exactly like V1's `enforceUserIsAuthed` (same as cohort.rpc.ts / gsc.rpc.ts).
+//
+// V1's router has no compute-heavy or ClickHouse-touching logic to delegate
+// (it is Prisma CRUD + a BullMQ enqueue, unlike gsc/cohort), so
+// packages/trpc/src/routers/import.ts stays completely unmodified — there is
+// nothing here for it to delegate to. This router exists to satisfy the
+// module map ("R") and to serve core's own (not yet live) appRouter.
+//
+// The per-project access ladder itself IS shared: `./src/access.ts` binds
+// core's shared/access.ts ladder to @openpanel/db's real lookups, the same
+// way packages/trpc/src/access.ts does for V1.
+
+import { z } from 'zod';
+import { createTRPCRouter, procedure } from '../../rpc/base';
+import {
+ TRPCAccessError,
+ TRPCBadRequestError,
+ TRPCNotFoundError,
+} from '../../rpc/errors';
+import { zCreateImport } from './import.constants';
+
+function loadAccessChecks() {
+ return import('./src/access');
+}
+
+function loadDb() {
+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);
+}
+
+function requireLogin(userId: string | null | undefined): string {
+ if (!userId) {
+ throw new TRPCAccessError('Not authenticated');
+ }
+ return userId;
+}
+
+export const importRouter = createTRPCRouter({
+ list: procedure
+ .input(z.object({ projectId: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: input.projectId,
+ level: 'read',
+ });
+
+ const db = await loadDb();
+ return db.import.findMany({
+ where: { projectId: input.projectId },
+ orderBy: { createdAt: 'desc' },
+ });
+ }),
+
+ get: procedure
+ .input(z.object({ id: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const db = await loadDb();
+ const importRecord = await db.import.findUniqueOrThrow({
+ where: { id: input.id },
+ include: { project: true },
+ });
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: importRecord.projectId,
+ level: 'read',
+ });
+
+ return importRecord;
+ }),
+
+ create: procedure.input(zCreateImport).mutation(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: input.projectId,
+ level: 'write',
+ });
+
+ const db = await loadDb();
+ const organization = await db.organization.findFirst({
+ where: { projects: { some: { id: input.projectId } } },
+ });
+
+ if (!organization) {
+ throw new TRPCNotFoundError(
+ 'Could not start import, organization not found'
+ );
+ }
+
+ if (!organization.isActive) {
+ throw new TRPCBadRequestError(
+ 'You cannot start an import without an active subscription!'
+ );
+ }
+
+ const importRecord = await db.import.create({
+ data: {
+ projectId: input.projectId,
+ config: input.config,
+ status: 'pending',
+ },
+ });
+
+ const jobId = await ctx.services.import.enqueue(importRecord.id);
+
+ await db.import.update({
+ where: { id: importRecord.id },
+ data: { jobId },
+ });
+
+ return { ...importRecord, jobId };
+ }),
+
+ delete: procedure
+ .input(z.object({ id: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const db = await loadDb();
+ const importRecord = await db.import.findUniqueOrThrow({
+ where: { id: input.id },
+ });
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: importRecord.projectId,
+ level: 'write',
+ });
+
+ if (importRecord.jobId) {
+ await ctx.queues.import.import.remove(importRecord.jobId);
+ }
+
+ return db.import.delete({ where: { id: input.id } });
+ }),
+
+ retry: procedure
+ .input(z.object({ id: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const db = await loadDb();
+ const importRecord = await db.import.findUniqueOrThrow({
+ where: { id: input.id },
+ });
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: importRecord.projectId,
+ level: 'write',
+ });
+
+ // Only allow retry for failed imports
+ if (importRecord.status !== 'failed') {
+ throw new Error('Only failed imports can be retried');
+ }
+
+ const jobId = await ctx.services.import.enqueue(importRecord.id);
+
+ return db.import.update({
+ where: { id: importRecord.id },
+ data: { jobId, status: 'pending', errorMessage: null },
+ });
+ }),
+});
diff --git a/packages/core/src/modules/import/import.service.test.ts b/packages/core/src/modules/import/import.service.test.ts
new file mode 100644
index 000000000..546376c0d
--- /dev/null
+++ b/packages/core/src/modules/import/import.service.test.ts
@@ -0,0 +1,181 @@
+// import.service.ts's db/ch access is lazy (`await import(...)` inside each
+// function — see the file's header), which is exactly what makes
+// `mock.module` work here with no import-time side effects to race: every
+// mock below is registered before the subject's first call, not before its
+// (side-effect-free) import.
+
+import { beforeAll, expect, mock, test } from 'bun:test';
+import { createRecordingProducers } from '../../jobs/testing';
+import { queues } from '../../jobs.registry';
+import type { Logger } from '../../logger';
+
+function stubLogger(): Logger {
+ const noop = () => undefined;
+ const logger: Logger = {
+ fatal: noop,
+ error: noop,
+ warn: noop,
+ info: noop,
+ debug: noop,
+ trace: noop,
+ child: () => logger,
+ };
+ return logger;
+}
+
+const chInsert = mock(async (_opts: { values: unknown[] }) => ({
+ summary: { written_rows: '2' },
+}));
+// `ch.command`/`originalCh`/`chQuery`/`getReplicatedTableName`/
+// `convertClickhouseDateToJs` and the extra TABLE_NAMES keys are unused here
+// but included because `mock.module` replaces this specifier process-wide
+// (bun runs every test file in one shared module registry without
+// `--isolate` — see AGENTS.md) — gsc.service.test.ts, cohort.service.test.ts
+// and insight.service.test.ts mock the same path, so every factory must be a
+// superset of every consumer's needs, whichever one ends up registered last.
+mock.module('@openpanel/db/src/clickhouse/client', () => ({
+ ch: {
+ insert: chInsert,
+ command: mock(async () => undefined),
+ },
+ originalCh: {
+ query: mock(async () => ({ json: async () => [] as unknown[] })),
+ insert: mock(async () => undefined),
+ },
+ chQuery: mock(async () => [] as unknown[]),
+ getReplicatedTableName: mock((table: string) => table),
+ TABLE_NAMES: {
+ events: 'events',
+ events_imports: 'events_imports',
+ profiles: 'profiles',
+ sessions: 'sessions',
+ },
+ formatClickhouseDate: (date: Date | string) =>
+ new Date(date)
+ .toISOString()
+ .replace('T', ' ')
+ .replace(/(\.\d{3})?Z+$/, ''),
+ convertClickhouseDateToJs: (date: string) =>
+ new Date(`${date.replace(' ', 'T')}Z`),
+}));
+
+const importUpdate = mock(
+ async (_opts: { where: { id: string }; data: unknown }) => undefined
+);
+mock.module('@openpanel/db/src/prisma-client', () => ({
+ db: { import: { update: importUpdate } },
+}));
+
+let insertRawEventsBatch: typeof import('./import.service').insertRawEventsBatch;
+let updateImportStatus: typeof import('./import.service').updateImportStatus;
+let createImportService: typeof import('./import.service').createImportService;
+
+beforeAll(async () => {
+ ({ insertRawEventsBatch, updateImportStatus, createImportService } =
+ await import('./import.service'));
+});
+
+test('insertRawEventsBatch stamps project_id/imported_at and flattens properties', async () => {
+ chInsert.mockClear();
+
+ const result = await insertRawEventsBatch(
+ 'proj_1',
+ [
+ {
+ id: 'evt_1',
+ name: 'screen_view',
+ project_id: 'wrong_project',
+ properties: { nested: { a: 1 } },
+ created_at: '2025-01-01 00:00:00',
+ } as never,
+ ],
+ stubLogger()
+ );
+
+ expect(result).toEqual({ writtenRows: 2 });
+ expect(chInsert).toHaveBeenCalledTimes(1);
+
+ const [{ values }] = chInsert.mock.calls[0] as [{ values: any[] }];
+ expect(values).toHaveLength(1);
+ expect(values[0]).toMatchObject({
+ id: 'evt_1',
+ project_id: 'proj_1',
+ properties: { 'nested.a': '1' },
+ });
+ expect(values[0].imported_at).toBeTruthy();
+});
+
+test('updateImportStatus writes the loading step through both the progress reporter and Postgres', async () => {
+ importUpdate.mockClear();
+ const progressed: unknown[] = [];
+
+ await updateImportStatus(
+ stubLogger(),
+ { updateProgress: (p) => progressed.push(p) },
+ 'imp_1',
+ {
+ step: 'loading',
+ batch: '2025-01-01',
+ totalEvents: 10,
+ processedEvents: 5,
+ }
+ );
+
+ expect(progressed).toEqual([
+ {
+ status: 'processing',
+ currentStep: 'loading',
+ currentBatch: '2025-01-01',
+ statusMessage: 'Importing events from 2025-01-01',
+ totalEvents: 10,
+ processedEvents: 5,
+ },
+ ]);
+ expect(importUpdate).toHaveBeenCalledTimes(1);
+ const [{ where, data }] = importUpdate.mock.calls[0] as [
+ { where: { id: string }; data: unknown },
+ ];
+ expect(where).toEqual({ id: 'imp_1' });
+ expect(data).toEqual(progressed[0]);
+});
+
+test('updateImportStatus marks a failed step with its error message', async () => {
+ importUpdate.mockClear();
+ const progressed: unknown[] = [];
+
+ await updateImportStatus(
+ stubLogger(),
+ { updateProgress: (p) => progressed.push(p) },
+ 'imp_1',
+ { step: 'failed', errorMessage: 'boom' }
+ );
+
+ expect(progressed).toEqual([
+ { status: 'failed', statusMessage: 'Import failed', errorMessage: 'boom' },
+ ]);
+});
+
+test('ImportService.enqueue adds the import job onto the import queue', async () => {
+ const producers = createRecordingProducers(queues);
+ const service = createImportService({
+ db: {},
+ ch: {},
+ redis: {},
+ clients: {},
+ buffers: {},
+ logger: stubLogger(),
+ queues: producers.queues,
+ });
+
+ const jobId = await service.enqueue('imp_1');
+
+ expect(typeof jobId).toBe('string');
+ expect(producers.recorded).toEqual([
+ {
+ queue: 'import',
+ job: 'import',
+ payload: { importId: 'imp_1' },
+ meta: {},
+ },
+ ]);
+});
diff --git a/packages/db/src/services/import.service.ts b/packages/core/src/modules/import/import.service.ts
similarity index 54%
rename from packages/db/src/services/import.service.ts
rename to packages/core/src/modules/import/import.service.ts
index ec1d83546..0a6b22836 100644
--- a/packages/db/src/services/import.service.ts
+++ b/packages/core/src/modules/import/import.service.ts
@@ -1,15 +1,66 @@
+// Moved from packages/db/src/services/import.service.ts (the ClickHouse
+// staging/session pipeline) + apps/worker/src/jobs/import.ts (the job body
+// and provider dispatch) + apps/api/src/controllers/import.controller.ts
+// (the /import/events bulk-insert path) — M5-004, ADR-008's module map:
+// import owns "S". packages/db/src/services/import.service.ts is deleted
+// outright: nothing outside the worker job file it moves with reached it
+// through @openpanel/db's barrel (same as cohort.service.ts, M5-003).
+//
+// db/ch access is LAZY (`load*` below), not a static top-level import — see
+// gsc.service.ts's header for the full reasoning (jobs.registry.ts and
+// services.ts pull this module into the eager barrel chain nearly every core
+// test file reaches). The provider classes are behind a dynamic `import()`
+// for the same reason: they eagerly import `formatClickhouseDate` from
+// @openpanel/db's clickhouse client, which constructs a real pino transport
+// at import time.
+//
+// ClickHouse queries here still go through raw `ch`/`chQuery` calls, not the
+// `sql` tag: ADR-013 converts the analytics read path one query per P7 task,
+// and this module's queries haven't been converted yet.
+
import { createHash } from 'node:crypto';
-import {
- ch,
- convertClickhouseDateToJs,
- formatClickhouseDate,
- getReplicatedTableName,
- TABLE_NAMES,
-} from '../clickhouse/client';
-import type { ILogger } from '../logger';
-import { db, type Prisma } from '../prisma-client';
-import type { IClickhouseEvent } from './event.service';
-import type { IClickhouseProfile } from './profile.service';
+import { toDots } from '@openpanel/common';
+import type { IClickhouseEvent, IClickhouseProfile } from '@openpanel/db';
+import type { Prisma } from '@openpanel/db/src/prisma-client';
+import { createLogger, type ILogger } from '../../clients/logger';
+import type { Logger } from '../../logger';
+import type { ServiceDeps } from '../../services';
+import type { IImportConfig } from './import.constants';
+
+const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10);
+const SESSION_BATCH_SIZE = 5000;
+const PROFILE_BATCH_SIZE = 5000;
+// Profiles derived inline from events (Amplitude, which has no profile export
+// API) are deduped in a bounded map so memory stays flat regardless of event
+// volume; the profiles table's ReplacingMergeTree(last_seen_at) collapses any
+// duplicate rows that span flush boundaries to the latest activity per id.
+const PROFILE_MAP_CAP = 50_000;
+const RESUMABLE_STEPS = ['creating_sessions', 'moving', 'backfilling_sessions'];
+
+let _logger: ILogger | undefined;
+function getLogger(): ILogger {
+ _logger ??= createLogger({ name: 'core:import' });
+ return _logger;
+}
+
+function loadDb() {
+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);
+}
+
+function loadChClient() {
+ return import('@openpanel/db/src/clickhouse/client');
+}
+
+function yieldToEventLoop(): Promise<void> {
+ return new Promise((resolve) => {
+ setTimeout(resolve, 100);
+ });
+}
+
+// ---------------------------------------------------------------------------
+// ClickHouse staging pipeline — moved from packages/db/src/services/
+// import.service.ts.
+// ---------------------------------------------------------------------------
export interface ImportStageResult {
importId: string;
@@ -17,21 +68,94 @@ export interface ImportStageResult {
insertedEvents: number;
}
-const SESSION_GAP_MS = 30 * 60 * 1000; // 30 minutes
+/**
+ * Insert a batch of events into the imports staging table
+ */
+export async function insertImportBatch(
+ events: IClickhouseEvent[],
+ importId: string
+): Promise<ImportStageResult> {
+ if (events.length === 0) {
+ return { importId, totalEvents: 0, insertedEvents: 0 };
+ }
+
+ const { ch, TABLE_NAMES, formatClickhouseDate } = await loadChClient();
+ const now = formatClickhouseDate(new Date());
+ const rows = events.map((event) => ({
+ ...event,
+ import_id: importId,
+ import_status: 'pending',
+ imported_at: event.imported_at || now,
+ imported_at_meta: now,
+ }));
+
+ await ch.insert({
+ table: TABLE_NAMES.events_imports,
+ values: rows,
+ format: 'JSONEachRow',
+ });
+
+ return {
+ importId,
+ totalEvents: events.length,
+ insertedEvents: events.length,
+ };
+}
+
+/**
+ * Insert a batch of profiles into the production profiles table.
+ * Used by Mixpanel (and other providers) to import user profiles during an import job.
+ */
+export async function insertProfilesBatch(
+ profiles: IClickhouseProfile[],
+ projectId: string
+): Promise<{ inserted: number }> {
+ if (profiles.length === 0) {
+ return { inserted: 0 };
+ }
+
+ const { ch, TABLE_NAMES } = await loadChClient();
+ const normalized = profiles.map((p) => ({
+ id: p.id,
+ project_id: projectId,
+ first_name: p.first_name ?? '',
+ last_name: p.last_name ?? '',
+ email: p.email ?? '',
+ avatar: p.avatar ?? '',
+ is_external: p.is_external ?? true,
+ properties: Object.fromEntries(
+ Object.entries(p.properties || {}).filter(
+ (kv): kv is [string, string] => kv[1] != null && kv[1] !== ''
+ )
+ ) as Record<string, string>,
+ created_at: p.created_at,
+ last_seen_at: p.last_seen_at ?? p.created_at,
+ }));
+
+ await ch.insert({
+ table: TABLE_NAMES.profiles,
+ values: normalized,
+ format: 'JSONEachRow',
+ });
+
+ return { inserted: normalized.length };
+}
/**
* Generate gap-based session IDs for events that have none.
* Streams events from staging (sorted by device_id, created_at), assigns a new
* session when gap > 30 min, re-inserts with session_id, then deletes old rows.
*/
+const SESSION_GAP_MS = 30 * 60 * 1000; // 30 minutes
+
export async function generateGapBasedSessionIds(
importId: string
): Promise<void> {
+ const { ch, TABLE_NAMES, getReplicatedTableName } = await loadChClient();
let currentDeviceId = '';
let currentSessionId = '';
let currentLastTime = 0;
let currentCounter = -1;
- const BATCH_SIZE = 5000;
const batch: IClickhouseEvent[] = [];
const result = await ch.query({
@@ -75,7 +199,7 @@ export async function generateGapBasedSessionIds(
event.session_id = currentSessionId;
batch.push(event);
- if (batch.length >= BATCH_SIZE) {
+ if (batch.length >= SESSION_BATCH_SIZE) {
await insertImportBatch(batch, importId);
batch.length = 0;
}
@@ -102,82 +226,12 @@ export async function generateGapBasedSessionIds(
});
}
-/**
- * Insert a batch of events into the imports staging table
- */
-export async function insertImportBatch(
- events: IClickhouseEvent[],
- importId: string
-): Promise<ImportStageResult> {
- if (events.length === 0) {
- return { importId, totalEvents: 0, insertedEvents: 0 };
- }
-
- const now = formatClickhouseDate(new Date());
- const rows = events.map((event) => ({
- ...event,
- import_id: importId,
- import_status: 'pending',
- imported_at: event.imported_at || now,
- imported_at_meta: now,
- }));
-
- await ch.insert({
- table: TABLE_NAMES.events_imports,
- values: rows,
- format: 'JSONEachRow',
- });
-
- return {
- importId,
- totalEvents: events.length,
- insertedEvents: events.length,
- };
-}
-
-/**
- * Insert a batch of profiles into the production profiles table.
- * Used by Mixpanel (and other providers) to import user profiles during an import job.
- */
-export async function insertProfilesBatch(
- profiles: IClickhouseProfile[],
- projectId: string
-): Promise<{ inserted: number }> {
- if (profiles.length === 0) {
- return { inserted: 0 };
- }
-
- const normalized = profiles.map((p) => ({
- id: p.id,
- project_id: projectId,
- first_name: p.first_name ?? '',
- last_name: p.last_name ?? '',
- email: p.email ?? '',
- avatar: p.avatar ?? '',
- is_external: p.is_external ?? true,
- properties: Object.fromEntries(
- Object.entries(p.properties || {}).filter(
- (kv): kv is [string, string] => kv[1] != null && kv[1] !== ''
- )
- ) as Record<string, string>,
- created_at: p.created_at,
- last_seen_at: p.last_seen_at ?? p.created_at,
- }));
-
- await ch.insert({
- table: TABLE_NAMES.profiles,
- values: normalized,
- format: 'JSONEachRow',
- });
-
- return { inserted: normalized.length };
-}
-
/**
* Delete all staging data for an import. Used to get a clean slate on retry
* when the failure happened before moving data to production.
*/
export async function cleanupStagingData(importId: string): Promise<void> {
+ const { ch, TABLE_NAMES, getReplicatedTableName } = await loadChClient();
const mutationTableName = getReplicatedTableName(TABLE_NAMES.events_imports);
await ch.command({
query: `ALTER TABLE ${mutationTableName} DELETE WHERE import_id = {importId:String}`,
@@ -194,6 +248,7 @@ export async function cleanupStagingData(importId: string): Promise<void> {
export async function cleanupSessionStartEndEvents(
importId: string
): Promise<void> {
+ const { ch, TABLE_NAMES, getReplicatedTableName } = await loadChClient();
const mutationTableName = getReplicatedTableName(TABLE_NAMES.events_imports);
await ch.command({
query: `ALTER TABLE ${mutationTableName} DELETE WHERE import_id = {importId:String} AND name IN ('session_start', 'session_end')`,
@@ -218,7 +273,8 @@ export async function cleanupSessionStartEndEvents(
export async function createSessionsStartEndEvents(
importId: string
): Promise<void> {
- const SESSION_BATCH_SIZE = 5000;
+ const { ch, TABLE_NAMES, convertClickhouseDateToJs, formatClickhouseDate } =
+ await loadChClient();
let lastSessionId = '';
const baseWhere = [
@@ -441,6 +497,7 @@ export async function moveImportsToProduction(
importId: string,
from: string
): Promise<void> {
+ const { ch, TABLE_NAMES } = await loadChClient();
let whereClause = 'import_id = {importId:String}';
if (from) {
@@ -485,7 +542,7 @@ export async function moveImportsToProduction(
export async function backfillSessionsToProduction(
importId: string
): Promise<void> {
- const SESSION_BATCH_SIZE = 5000;
+ const { ch, TABLE_NAMES } = await loadChClient();
let lastSessionId = '';
while (true) {
@@ -520,7 +577,7 @@ export async function backfillSessionsToProduction(
utm_medium, utm_source, utm_campaign, utm_content, utm_term,
referrer, referrer_name, referrer_type
)
- SELECT
+ SELECT
any(e.session_id) as id,
any(e.project_id) as project_id,
if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,
@@ -563,7 +620,7 @@ export async function backfillSessionsToProduction(
argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name,
argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_type
FROM ${TABLE_NAMES.events_imports} e
- WHERE
+ WHERE
e.import_id = {importId:String}
AND e.session_id > {lastSessionId:String}
AND e.session_id <= {maxSessionId:String}
@@ -594,6 +651,7 @@ export async function getImportDateBounds(
importId: string,
fromCreatedAt?: string
): Promise<{ min: string | null; max: string | null }> {
+ const { ch, TABLE_NAMES } = await loadChClient();
const res = await ch.query({
query: `
SELECT min(created_at) AS min, max(created_at) AS max
@@ -617,6 +675,20 @@ export async function getImportDateBounds(
: { min: null, max: null };
}
+/**
+ * Reports progress on a running import. Wraps V1's `job.updateProgress` — a
+ * BullMQ `Job` satisfies this structurally, so the V1 worker delegate passes
+ * its real job straight through. Core's own job runner (import.jobs.ts) has
+ * no BullMQ job object to hand over (`JobCtx.job` is the erased
+ * `{id, attempt, queue, name}`, not the live BullMQ handle), so it falls back
+ * to the no-op default.
+ */
+export interface ImportJobProgress {
+ updateProgress(progress: Record<string, unknown>): unknown;
+}
+
+const NOOP_PROGRESS: ImportJobProgress = { updateProgress: () => undefined };
+
export type UpdateImportStatusOptions =
| {
step: 'loading';
@@ -655,13 +727,12 @@ export type UpdateImportStatusOptions =
export type ImportSteps = UpdateImportStatusOptions['step'];
export async function updateImportStatus(
- jobLogger: ILogger,
- job: {
- updateProgress: (progress: Record<string, any>) => void;
- },
+ jobLogger: Logger,
+ progress: ImportJobProgress,
importId: string,
options: UpdateImportStatusOptions
): Promise<void> {
+ const db = await loadDb();
const data: Prisma.ImportUpdateInput = {};
switch (options.step) {
case 'loading':
@@ -721,10 +792,450 @@ export async function updateImportStatus(
jobLogger.info({ data }, 'Import status update');
- await job.updateProgress(data);
+ await progress.updateProgress(data);
await db.import.update({
where: { id: importId },
data,
});
}
+
+// ---------------------------------------------------------------------------
+// Provider dispatch + job body — moved from apps/worker/src/jobs/import.ts.
+// ---------------------------------------------------------------------------
+
+/**
+ * Merge a freshly-derived profile into the bounded dedup map: keep the earliest
+ * created_at (first seen), advance last_seen_at to the latest activity, and fill
+ * identity fields/properties, preferring the newer row.
+ */
+function mergeProfileInto(
+ map: Map<string, IClickhouseProfile>,
+ incoming: IClickhouseProfile
+): void {
+ const existing = map.get(incoming.id);
+ if (!existing) {
+ map.set(incoming.id, incoming);
+ return;
+ }
+
+ const createdAt =
+ existing.created_at < incoming.created_at
+ ? existing.created_at
+ : incoming.created_at;
+ const isIncomingNewer = incoming.last_seen_at >= existing.last_seen_at;
+ const base = isIncomingNewer ? incoming : existing;
+ const other = isIncomingNewer ? existing : incoming;
+
+ map.set(incoming.id, {
+ ...base,
+ created_at: createdAt,
+ first_name: base.first_name || other.first_name,
+ last_name: base.last_name || other.last_name,
+ email: base.email || other.email,
+ avatar: base.avatar || other.avatar,
+ properties: { ...other.properties, ...base.properties },
+ });
+}
+
+/** Structural subset of a provider instance the job body actually drives. */
+interface RunnableProvider {
+ shouldGenerateSessionIds(): boolean;
+ getTotalEventsCount(): Promise<number>;
+ parseSource(): AsyncGenerator<unknown, void, unknown>;
+ validate(rawEvent: unknown): boolean;
+ transformEvent(rawEvent: unknown): IClickhouseEvent;
+ transformEventToProfile?(rawEvent: unknown): IClickhouseProfile | null;
+ streamProfiles?(): AsyncGenerator<unknown, void, unknown>;
+ transformProfile?(rawProfile: unknown): IClickhouseProfile;
+}
+
+/** Dynamic import keeps @openpanel/db's clickhouse client off this file's eager path (see header). */
+async function createImportProvider(
+ projectId: string,
+ config: IImportConfig,
+ jobLogger: Logger
+): Promise<RunnableProvider> {
+ switch (config.provider) {
+ case 'umami': {
+ const { UmamiProvider } = await import('./src/providers/umami');
+ return new UmamiProvider(projectId, config, jobLogger);
+ }
+ case 'mixpanel': {
+ const { MixpanelProvider } = await import('./src/providers/mixpanel');
+ return new MixpanelProvider(projectId, config, jobLogger);
+ }
+ case 'amplitude': {
+ const { AmplitudeProvider } = await import('./src/providers/amplitude');
+ return new AmplitudeProvider(projectId, config, jobLogger);
+ }
+ default:
+ throw new Error(
+ `Unknown provider: ${(config as { provider: string }).provider}`
+ );
+ }
+}
+
+/**
+ * The `import` queue job body. Ported verbatim from apps/worker/src/jobs/
+ * import.ts's `importJob`, with the BullMQ `Job` split into `importId` +
+ * `progress` (see `ImportJobProgress`'s header) so the same body serves V1's
+ * worker delegate and core's own (not yet live) `import` queue worker.
+ */
+export async function runImportJob(
+ importId: string,
+ progress: ImportJobProgress = NOOP_PROGRESS,
+ logger: Logger = getLogger()
+): Promise<{ success: true }> {
+ const db = await loadDb();
+ const record = await db.import.findUniqueOrThrow({
+ where: { id: importId },
+ include: { project: true },
+ });
+
+ const jobLogger = logger.child({ importId, config: record.config });
+ jobLogger.info('Starting import job');
+
+ const providerInstance = await createImportProvider(
+ record.projectId,
+ record.config,
+ jobLogger
+ );
+ const shouldGenerateSessionIds = providerInstance.shouldGenerateSessionIds();
+
+ try {
+ const isRetry = record.currentStep !== null;
+ const canResume =
+ isRetry && RESUMABLE_STEPS.includes(record.currentStep as string);
+
+ // -------------------------------------------------------
+ // STAGING PHASE: clean slate on failure, run from scratch
+ // -------------------------------------------------------
+ if (!canResume) {
+ if (isRetry) {
+ jobLogger.info(
+ 'Retry detected before resumable phase — cleaning staging data'
+ );
+ await cleanupStagingData(importId);
+ }
+
+ // Phase 1: Load events into staging
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'loading',
+ });
+
+ const totalEvents = await providerInstance
+ .getTotalEventsCount()
+ .catch(() => -1);
+ let processedEvents = 0;
+ const eventBatch: IClickhouseEvent[] = [];
+
+ const canProfileFromEvents =
+ typeof providerInstance.transformEventToProfile === 'function';
+ const profileMap = new Map<string, IClickhouseProfile>();
+ let processedProfiles = 0;
+
+ const flushProfiles = async () => {
+ if (profileMap.size === 0) {
+ return;
+ }
+ const values = Array.from(profileMap.values());
+ await insertProfilesBatch(values, record.projectId);
+ processedProfiles += values.length;
+ profileMap.clear();
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'loading_profiles',
+ processedProfiles,
+ });
+ await yieldToEventLoop();
+ };
+
+ for await (const rawEvent of providerInstance.parseSource()) {
+ if (!providerInstance.validate(rawEvent)) {
+ jobLogger.warn({ rawEvent }, 'Skipping invalid event');
+ continue;
+ }
+
+ const transformed = providerInstance.transformEvent(rawEvent);
+
+ // Session IDs for providers that need them (e.g. Mixpanel) are generated
+ // in generateGapBasedSessionIds after loading, using gap-based logic.
+ eventBatch.push(transformed);
+
+ if (canProfileFromEvents) {
+ const profile = providerInstance.transformEventToProfile?.(rawEvent);
+ if (profile) {
+ mergeProfileInto(profileMap, profile);
+ if (profileMap.size >= PROFILE_MAP_CAP) {
+ await flushProfiles();
+ }
+ }
+ }
+
+ if (eventBatch.length >= BATCH_SIZE) {
+ await insertImportBatch(eventBatch, importId);
+ processedEvents += eventBatch.length;
+
+ const batchDate = new Date(eventBatch[0]?.created_at || '')
+ .toISOString()
+ .split('T')[0];
+
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'loading',
+ batch: batchDate,
+ totalEvents,
+ processedEvents,
+ });
+
+ eventBatch.length = 0;
+ await yieldToEventLoop();
+ }
+ }
+
+ if (eventBatch.length > 0) {
+ await insertImportBatch(eventBatch, importId);
+ processedEvents += eventBatch.length;
+
+ const batchDate = new Date(eventBatch[0]?.created_at || '')
+ .toISOString()
+ .split('T')[0];
+
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'loading',
+ batch: batchDate,
+ totalEvents,
+ processedEvents,
+ });
+ eventBatch.length = 0;
+ }
+
+ jobLogger.info({ processedEvents }, 'Loading complete');
+
+ // Phase 1a: Flush profiles derived inline from events (Amplitude)
+ if (canProfileFromEvents) {
+ await flushProfiles();
+ jobLogger.info(
+ { processedProfiles },
+ 'Inline profile derivation complete'
+ );
+ }
+
+ // Phase 1b: Load user profiles (Mixpanel only)
+ if (typeof providerInstance.streamProfiles === 'function') {
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'loading_profiles',
+ });
+
+ const profileBatch: IClickhouseProfile[] = [];
+ let profilesLoaded = 0;
+
+ for await (const rawProfile of providerInstance.streamProfiles()) {
+ const profile = providerInstance.transformProfile?.(rawProfile);
+ if (!profile) {
+ continue;
+ }
+ profileBatch.push(profile);
+
+ if (profileBatch.length >= PROFILE_BATCH_SIZE) {
+ await insertProfilesBatch(profileBatch, record.projectId);
+ profilesLoaded += profileBatch.length;
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'loading_profiles',
+ processedProfiles: profilesLoaded,
+ });
+ profileBatch.length = 0;
+ await yieldToEventLoop();
+ }
+ }
+
+ if (profileBatch.length > 0) {
+ await insertProfilesBatch(profileBatch, record.projectId);
+ profilesLoaded += profileBatch.length;
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'loading_profiles',
+ processedProfiles: profilesLoaded,
+ totalProfiles: profilesLoaded,
+ });
+ }
+
+ jobLogger.info(
+ { processedProfiles: profilesLoaded },
+ 'Profile loading complete'
+ );
+ }
+
+ // Phase 2: Generate gap-based session IDs (Mixpanel etc.)
+ if (shouldGenerateSessionIds) {
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'generating_sessions',
+ });
+ await generateGapBasedSessionIds(importId);
+ await yieldToEventLoop();
+ jobLogger.info('Session ID generation complete');
+ }
+ }
+
+ // -------------------------------------------------------
+ // SESSION CREATION PHASE: resumable by cleaning session_start/end
+ // -------------------------------------------------------
+ const skipSessionCreation =
+ canResume && record.currentStep !== 'creating_sessions';
+
+ if (!skipSessionCreation) {
+ if (canResume && record.currentStep === 'creating_sessions') {
+ jobLogger.info(
+ 'Retry at creating_sessions — cleaning existing session_start/end events'
+ );
+ await cleanupSessionStartEndEvents(importId);
+ }
+
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'creating_sessions',
+ batch: 'all sessions',
+ });
+ await createSessionsStartEndEvents(importId);
+ await yieldToEventLoop();
+
+ jobLogger.info('Session event creation complete');
+ }
+
+ // -------------------------------------------------------
+ // PRODUCTION PHASE: resume-safe, track progress per batch
+ // -------------------------------------------------------
+
+ // Phase 3: Move staging events to production (per-day)
+ const resumeMovingFrom =
+ canResume && record.currentStep === 'moving'
+ ? (record.currentBatch ?? undefined)
+ : undefined;
+
+ // currentBatch is the last successfully completed day — resume from the next day to avoid re-inserting it
+ const moveFromDate = (() => {
+ if (!resumeMovingFrom) {
+ return undefined;
+ }
+ const next = new Date(`${resumeMovingFrom}T12:00:00Z`);
+ next.setUTCDate(next.getUTCDate() + 1);
+ return next.toISOString().split('T')[0]!;
+ })();
+
+ const bounds = await getImportDateBounds(importId, moveFromDate);
+ if (bounds.min && bounds.max) {
+ const startDate = bounds.min.split(' ')[0]!;
+ const endDate = bounds.max.split(' ')[0]!;
+ const cursor = new Date(`${startDate}T12:00:00Z`);
+ const end = new Date(`${endDate}T12:00:00Z`);
+
+ while (cursor <= end) {
+ const dateStr = cursor.toISOString().split('T')[0]!;
+
+ await moveImportsToProduction(importId, dateStr);
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'moving',
+ batch: dateStr,
+ });
+
+ await yieldToEventLoop();
+ cursor.setUTCDate(cursor.getUTCDate() + 1);
+ }
+ }
+
+ jobLogger.info('Move to production complete');
+
+ // Phase 4: Backfill sessions table
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'backfilling_sessions',
+ batch: 'all sessions',
+ });
+ await backfillSessionsToProduction(importId);
+ await yieldToEventLoop();
+
+ jobLogger.info('Session backfill complete');
+
+ // Done
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'completed',
+ });
+ jobLogger.info('Import completed');
+
+ return { success: true };
+ } catch (error) {
+ jobLogger.error({ err: error }, 'Import job failed');
+
+ try {
+ const errorMsg = error instanceof Error ? error.message : 'Unknown error';
+ await updateImportStatus(jobLogger, progress, importId, {
+ step: 'failed',
+ errorMessage: errorMsg,
+ });
+ } catch (markError) {
+ jobLogger.error(
+ { err: error, markError },
+ 'Failed to mark import as failed'
+ );
+ }
+
+ throw error;
+ }
+}
+
+// ---------------------------------------------------------------------------
+// /import/events — moved from apps/api/src/controllers/import.controller.ts.
+// Bulk-inserts already-shaped events straight into production; unrelated to
+// the provider/staging pipeline above (no Import record, no ClickHouse
+// staging table).
+// ---------------------------------------------------------------------------
+
+export interface InsertRawEventsResult {
+ writtenRows: number;
+}
+
+export async function insertRawEventsBatch(
+ projectId: string,
+ events: IClickhouseEvent[],
+ logger: Logger = getLogger()
+): Promise<InsertRawEventsResult> {
+ const { ch, TABLE_NAMES, formatClickhouseDate } = await loadChClient();
+ const importedAt = formatClickhouseDate(new Date());
+ const values: IClickhouseEvent[] = events.map((event) => ({
+ ...event,
+ properties: toDots(event.properties),
+ project_id: projectId,
+ created_at: formatClickhouseDate(event.created_at),
+ imported_at: importedAt,
+ }));
+
+ const res = await ch.insert({
+ table: TABLE_NAMES.events,
+ values,
+ format: 'JSONEachRow',
+ });
+
+ // ClickHouse's insert summary reports written_rows as a string.
+ const writtenRows = Number(res.summary?.written_rows ?? 0);
+ logger.info({ writtenRows, projectId }, 'events imported');
+ return { writtenRows };
+}
+
+// ---------------------------------------------------------------------------
+// `ctx.services.import` binding.
+// ---------------------------------------------------------------------------
+
+export interface ImportService {
+ run(
+ importId: string,
+ progress?: ImportJobProgress
+ ): Promise<{ success: true }>;
+ /** Enqueues the `import` job and returns its BullMQ job id, for `Import.jobId`. */
+ enqueue(importId: string): Promise<string>;
+}
+
+/** `ctx.services.import` — a thin binding of the job body above to a Ctx's queues. */
+export function createImportService(deps: ServiceDeps): ImportService {
+ const logger = deps.logger.child({ module: 'import' });
+
+ return {
+ run: (importId, progress) => runImportJob(importId, progress, logger),
+ enqueue: (importId) => deps.queues.import.import.add({ importId }),
+ };
+}
diff --git a/packages/core/src/modules/import/src/access.ts b/packages/core/src/modules/import/src/access.ts
new file mode 100644
index 000000000..de4a4168e
--- /dev/null
+++ b/packages/core/src/modules/import/src/access.ts
@@ -0,0 +1,21 @@
+// Binds core's shared/access.ts ladder to @openpanel/db's real lookups, for
+// this module's own router — the same binding packages/trpc/src/access.ts
+// does for V1, self-contained here because the general core -> db binding
+// itself moves with auth (P6), not this wave.
+
+import {
+ canWriteProject,
+ getOrganizationAccess,
+ getProjectAccess,
+} from '@openpanel/db/src/services/access.service';
+import { getProjectById } from '@openpanel/db/src/services/project.service';
+import { createAccessChecks } from '../../../shared/access';
+
+const checks = createAccessChecks({
+ getProjectAccess,
+ canWriteProject,
+ getOrganizationAccess,
+ getProjectById,
+});
+
+export const requireProjectAccess = checks.requireProjectAccess;
diff --git a/packages/importer/src/base-provider.ts b/packages/core/src/modules/import/src/base-provider.ts
similarity index 93%
rename from packages/importer/src/base-provider.ts
rename to packages/core/src/modules/import/src/base-provider.ts
index e7c6189fe..056ff20f5 100644
--- a/packages/importer/src/base-provider.ts
+++ b/packages/core/src/modules/import/src/base-provider.ts
@@ -12,7 +12,7 @@ export abstract class BaseImportProvider<
* This should be implemented as an async generator to handle large files efficiently
*/
abstract parseSource(
- overrideFrom?: string,
+ overrideFrom?: string
): AsyncGenerator<TRawEvent, void, unknown>;
/**
@@ -45,7 +45,7 @@ export abstract class BaseImportProvider<
/**
* Optional hook: Custom error handling
*/
- async onError?(error: Error, context?: ErrorContext): Promise<void> {
+ async onError?(error: Error, _context?: ErrorContext): Promise<void> {
// Default: re-throw
throw error;
}
@@ -80,9 +80,9 @@ export abstract class BaseImportProvider<
to: string,
options?: {
chunkSizeDays?: number;
- },
- ): Array<[string, string]> {
- const chunks: Array<[string, string]> = [];
+ }
+ ): [string, string][] {
+ const chunks: [string, string][] = [];
const startDate = new Date(from);
const endDate = new Date(to);
@@ -113,7 +113,9 @@ export abstract class BaseImportProvider<
// Move cursor to the next chunk start (after the current chunk)
cursor.setDate(cursor.getDate() + chunkSizeDays);
- if (cursor > endDate) break;
+ if (cursor > endDate) {
+ break;
+ }
}
return chunks;
diff --git a/packages/importer/src/providers/__fixtures__/amplitude-export.json b/packages/core/src/modules/import/src/providers/__fixtures__/amplitude-export.json
similarity index 100%
rename from packages/importer/src/providers/__fixtures__/amplitude-export.json
rename to packages/core/src/modules/import/src/providers/__fixtures__/amplitude-export.json
diff --git a/packages/importer/src/providers/amplitude.test.ts b/packages/core/src/modules/import/src/providers/amplitude.test.ts
similarity index 98%
rename from packages/importer/src/providers/amplitude.test.ts
rename to packages/core/src/modules/import/src/providers/amplitude.test.ts
index c50541df7..c3e3a1ba9 100644
--- a/packages/importer/src/providers/amplitude.test.ts
+++ b/packages/core/src/modules/import/src/providers/amplitude.test.ts
@@ -1,8 +1,8 @@
+import { describe, expect, it } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { Readable } from 'node:stream';
import { gzipSync } from 'node:zlib';
-import { describe, expect, it } from 'vitest';
import { AmplitudeProvider, type AmplitudeRawEvent } from './amplitude';
import { toCountryCode } from './country-codes';
@@ -154,7 +154,10 @@ describe('amplitude', () => {
const gz = gzipSync(Buffer.from(ndjson));
const events = await collect(
- provider.streamNdjson(Readable.from(gz), 'https://example.com/export.json.gz')
+ provider.streamNdjson(
+ Readable.from(gz),
+ 'https://example.com/export.json.gz'
+ )
);
expect(events.map((e) => e.event_type)).toEqual(['A']);
diff --git a/packages/importer/src/providers/amplitude.ts b/packages/core/src/modules/import/src/providers/amplitude.ts
similarity index 97%
rename from packages/importer/src/providers/amplitude.ts
rename to packages/core/src/modules/import/src/providers/amplitude.ts
index 6c68e603a..2a1e1bfa5 100644
--- a/packages/importer/src/providers/amplitude.ts
+++ b/packages/core/src/modules/import/src/providers/amplitude.ts
@@ -3,19 +3,18 @@ import { createInterface } from 'node:readline';
import { Readable } from 'node:stream';
import { createBrotliDecompress, createGunzip } from 'node:zlib';
import { isSameDomain, parsePath, toDots } from '@openpanel/common';
+// Deep imports, not @openpanel/db's full barrel — see umami.ts's header.
+import { formatClickhouseDate } from '@openpanel/db/src/clickhouse/client';
+import type { IClickhouseEvent } from '@openpanel/db/src/services/event.service';
+import type { IClickhouseProfile } from '@openpanel/db/src/services/profile.service';
+import unzipper from 'unzipper';
+import { z } from 'zod';
+import type { Logger } from '../../../../logger';
import {
getReferrerWithQuery,
- type ILogger,
parseReferrer,
-} from '@openpanel/core';
-import {
- formatClickhouseDate,
- type IClickhouseEvent,
- type IClickhouseProfile,
-} from '@openpanel/db';
-import type { IAmplitudeImportConfig } from '@openpanel/validation';
-import unzipper from 'unzipper';
-import { z } from 'zod';
+} from '../../../../shared/parse-referrer';
+import type { IAmplitudeImportConfig } from '../../import.constants';
import { BaseImportProvider } from '../base-provider';
import { toCountryCode } from './country-codes';
@@ -80,7 +79,7 @@ export class AmplitudeProvider extends BaseImportProvider<AmplitudeRawEvent> {
constructor(
private readonly projectId: string,
private readonly config: IAmplitudeImportConfig,
- private readonly logger?: ILogger
+ private readonly logger?: Logger
) {
super();
const residency = config.dataResidency ?? 'us';
diff --git a/packages/importer/src/providers/country-codes.ts b/packages/core/src/modules/import/src/providers/country-codes.ts
similarity index 100%
rename from packages/importer/src/providers/country-codes.ts
rename to packages/core/src/modules/import/src/providers/country-codes.ts
diff --git a/packages/importer/src/providers/mixpanel.test.ts b/packages/core/src/modules/import/src/providers/mixpanel.test.ts
similarity index 96%
rename from packages/importer/src/providers/mixpanel.test.ts
rename to packages/core/src/modules/import/src/providers/mixpanel.test.ts
index 31ee435d7..08f9987d5 100644
--- a/packages/importer/src/providers/mixpanel.test.ts
+++ b/packages/core/src/modules/import/src/providers/mixpanel.test.ts
@@ -1,5 +1,5 @@
+import { describe, expect, it } from 'bun:test';
import { omit } from 'ramda';
-import { describe, expect, it } from 'vitest';
import { MixpanelProvider } from './mixpanel';
describe('mixpanel', () => {
@@ -330,9 +330,15 @@ describe('mixpanel', () => {
};
// All valid values should construct without throwing
- expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'us' })).not.toThrow();
- expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'eu' })).not.toThrow();
- expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'in' })).not.toThrow();
+ expect(
+ () => new MixpanelProvider('pid', { ...base, dataResidency: 'us' })
+ ).not.toThrow();
+ expect(
+ () => new MixpanelProvider('pid', { ...base, dataResidency: 'eu' })
+ ).not.toThrow();
+ expect(
+ () => new MixpanelProvider('pid', { ...base, dataResidency: 'in' })
+ ).not.toThrow();
// Undefined defaults to 'us' without throwing
expect(() => new MixpanelProvider('pid', { ...base })).not.toThrow();
diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/core/src/modules/import/src/providers/mixpanel.ts
similarity index 97%
rename from packages/importer/src/providers/mixpanel.ts
rename to packages/core/src/modules/import/src/providers/mixpanel.ts
index 6d09223c6..de8976608 100644
--- a/packages/importer/src/providers/mixpanel.ts
+++ b/packages/core/src/modules/import/src/providers/mixpanel.ts
@@ -1,16 +1,20 @@
import { randomUUID } from 'node:crypto';
import { isSameDomain, parsePath, toDots } from '@openpanel/common';
+// Deep imports, not @openpanel/db's full barrel — see umami.ts's header.
+import { formatClickhouseDate } from '@openpanel/db/src/clickhouse/client';
+import type { IClickhouseEvent } from '@openpanel/db/src/services/event.service';
+import type { IClickhouseProfile } from '@openpanel/db/src/services/profile.service';
+import { z } from 'zod';
+import type { Logger } from '../../../../logger';
import {
getReferrerWithQuery,
- type ILogger,
parseReferrer,
+} from '../../../../shared/parse-referrer';
+import {
parseUserAgent,
type UserAgentInfo,
-} from '@openpanel/core';
-import type { IClickhouseProfile } from '@openpanel/db';
-import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';
-import type { IMixpanelImportConfig } from '@openpanel/validation';
-import { z } from 'zod';
+} from '../../../../shared/parser-user-agent';
+import type { IMixpanelImportConfig } from '../../import.constants';
import { BaseImportProvider } from '../base-provider';
export const zMixpanelRawEvent = z.object({
@@ -68,7 +72,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {
constructor(
private readonly projectId: string,
private readonly config: IMixpanelImportConfig,
- private readonly logger?: ILogger
+ private readonly logger?: Logger
) {
super();
const residency = config.dataResidency ?? 'us';
diff --git a/packages/importer/src/providers/umami.test.ts b/packages/core/src/modules/import/src/providers/umami.test.ts
similarity index 84%
rename from packages/importer/src/providers/umami.test.ts
rename to packages/core/src/modules/import/src/providers/umami.test.ts
index a386a6ae2..e75956696 100644
--- a/packages/importer/src/providers/umami.test.ts
+++ b/packages/core/src/modules/import/src/providers/umami.test.ts
@@ -1,6 +1,6 @@
+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { UmamiProvider } from './umami';
/**
@@ -24,7 +24,7 @@ describe('umami remote file SSRF guard', () => {
res.end('INTERNAL_ONLY_SECRET\n');
});
await new Promise<void>((resolve) =>
- internalServer.listen(0, '127.0.0.1', resolve),
+ internalServer.listen(0, '127.0.0.1', resolve)
);
internalPort = (internalServer.address() as AddressInfo).port;
});
@@ -50,7 +50,7 @@ describe('umami remote file SSRF guard', () => {
hits = [];
await expect(
- drain(`http://127.0.0.1:${internalPort}/secret.csv`),
+ drain(`http://127.0.0.1:${internalPort}/secret.csv`)
).rejects.toThrow();
expect(hits).toEqual([]);
@@ -58,7 +58,7 @@ describe('umami remote file SSRF guard', () => {
it('refuses cloud instance metadata', async () => {
await expect(
- drain('http://169.254.169.254/latest/meta-data/'),
+ drain('http://169.254.169.254/latest/meta-data/')
).rejects.toThrow();
});
@@ -66,9 +66,10 @@ describe('umami remote file SSRF guard', () => {
hits = [];
const target = `http://127.0.0.1:${internalPort}/secret.csv`;
- const error = await drain(target).catch((err: unknown) => err as Error);
+ const caught = await drain(target).catch((err: unknown) => err as Error);
- expect(error).toBeInstanceOf(Error);
+ expect(caught).toBeInstanceOf(Error);
+ const error = caught as Error;
// The old message was `Failed to fetch remote file: 418 I'm a Teapot`.
expect(error.message).not.toContain('418');
expect(error.message).not.toContain('Teapot');
diff --git a/packages/importer/src/providers/umami.ts b/packages/core/src/modules/import/src/providers/umami.ts
similarity index 93%
rename from packages/importer/src/providers/umami.ts
rename to packages/core/src/modules/import/src/providers/umami.ts
index c873eeb40..6022de72d 100644
--- a/packages/importer/src/providers/umami.ts
+++ b/packages/core/src/modules/import/src/providers/umami.ts
@@ -3,18 +3,24 @@ import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { createBrotliDecompress, createGunzip } from 'node:zlib';
import { isSameDomain, parsePath, toDots } from '@openpanel/common';
-import {
- generateDeviceId,
- getReferrerWithQuery,
- type ILogger,
- parseReferrer,
- safeFetchStream,
-} from '@openpanel/core';
-import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';
-import type { IUmamiImportConfig } from '@openpanel/validation';
+// Deep imports, not @openpanel/db's full barrel: the barrel's
+// `export * from './src/services/notification.service'` /
+// `'./src/buffers'` eagerly reach @openpanel/queue, which constructs a
+// pino-pretty transport at import time — fatal under bun:test's
+// `--isolate` worker threads (AGENTS.md's eager-barrel-chain hazard).
+import { formatClickhouseDate } from '@openpanel/db/src/clickhouse/client';
+import type { IClickhouseEvent } from '@openpanel/db/src/services/event.service';
import { parse } from 'csv-parse';
import { assocPath } from 'ramda';
import { z } from 'zod';
+import type { Logger } from '../../../../logger';
+import {
+ getReferrerWithQuery,
+ parseReferrer,
+} from '../../../../shared/parse-referrer';
+import { generateDeviceId } from '../../../../shared/profileId';
+import { safeFetchStream } from '../../../../shared/safe-fetch';
+import type { IUmamiImportConfig } from '../../import.constants';
import { BaseImportProvider } from '../base-provider';
/**
@@ -77,7 +83,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {
constructor(
private readonly projectId: string,
private readonly config: IUmamiImportConfig,
- private readonly logger?: ILogger
+ private readonly logger?: Logger
) {
super();
}
@@ -275,7 +281,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {
.filter(Boolean)
.join('')}`
: '';
- const { path, hash, query, origin } = parsePath(url);
+ const { path, query, origin } = parsePath(url);
// Extract referrer information - use same logic as real-time events
const referrerUrl = rawEvent.referrer_domain
? `https://${rawEvent.referrer_domain}${rawEvent.referrer_path || ''}`
diff --git a/packages/importer/src/types.ts b/packages/core/src/modules/import/src/types.ts
similarity index 93%
rename from packages/importer/src/types.ts
rename to packages/core/src/modules/import/src/types.ts
index ae56bf29e..5f7c466fe 100644
--- a/packages/importer/src/types.ts
+++ b/packages/core/src/modules/import/src/types.ts
@@ -1,8 +1,4 @@
-import type {
- IImportedEvent,
- IServiceCreateEventPayload,
- IServiceImportedEventPayload,
-} from '@openpanel/db';
+import type { IServiceImportedEventPayload } from '@openpanel/db';
export interface ImportConfig {
projectId: string;
diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.ts
index b47c44a5e..37c52bf5a 100644
--- a/packages/core/src/modules/insight/insight.jobs.test.ts
+++ b/packages/core/src/modules/insight/insight.jobs.test.ts
@@ -41,6 +41,7 @@ function stubJobCtx(
insight: insight as InsightService,
gsc: {} as Services['gsc'],
cohort: {} as Services['cohort'],
+ import: {} as Services['import'],
};
return {
db: {},
@@ -120,6 +121,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId de
} as InsightService,
gsc: {} as Services['gsc'],
cohort: {} as Services['cohort'],
+ import: {} as Services['import'],
},
requestId: 'req_1',
job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'insightsDaily' },
diff --git a/packages/core/src/modules/insight/insight.service.test.ts b/packages/core/src/modules/insight/insight.service.test.ts
index 9eee88827..99bc77878 100644
--- a/packages/core/src/modules/insight/insight.service.test.ts
+++ b/packages/core/src/modules/insight/insight.service.test.ts
@@ -22,11 +22,12 @@ mock.module('@openpanel/db/src/prisma-client', () => ({
db: { $executeRaw },
}));
-// `originalCh`/`chQuery`/`TABLE_NAMES`/`getReplicatedTableName` are unused
-// here but included because `mock.module` replaces this specifier
-// process-wide (bun runs every test file in one shared module registry
-// without `--isolate` — see AGENTS.md) — gsc.service.test.ts and
-// cohort.service.test.ts mock the same path, so every factory must be a
+// `originalCh`/`chQuery`/`TABLE_NAMES`/`getReplicatedTableName`/
+// `formatClickhouseDate`/`convertClickhouseDateToJs` are unused here but
+// included because `mock.module` replaces this specifier process-wide (bun
+// runs every test file in one shared module registry without `--isolate` —
+// see AGENTS.md) — gsc.service.test.ts, cohort.service.test.ts and
+// import.service.test.ts mock the same path, so every factory must be a
// superset of every consumer's needs, whichever one ends up registered last.
mock.module('@openpanel/db/src/clickhouse/client', () => ({
ch: {
@@ -39,7 +40,19 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({
},
chQuery: mock(async () => []),
getReplicatedTableName: mock((table: string) => table),
- TABLE_NAMES: { sessions: 'sessions' },
+ TABLE_NAMES: {
+ events: 'events',
+ events_imports: 'events_imports',
+ profiles: 'profiles',
+ sessions: 'sessions',
+ },
+ formatClickhouseDate: (date: Date | string) =>
+ new Date(date)
+ .toISOString()
+ .replace('T', ' ')
+ .replace(/(\.\d{3})?Z+$/, ''),
+ convertClickhouseDateToJs: (date: string) =>
+ new Date(`${date.replace(' ', 'T')}Z`),
}));
const spikesQuery = mock(async () => [
diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.ts
index 6a1cd109d..6e16e343f 100644
--- a/packages/core/src/rest.routes.ts
+++ b/packages/core/src/rest.routes.ts
@@ -10,6 +10,7 @@ import type { AppDeps } from './context';
import { registry } from './metrics';
import { gscRoutes } from './modules/gsc/gsc.routes';
import { healthRoutes } from './modules/health/health.routes';
+import { importRoutes } from './modules/import/import.routes';
// ADR-002/ADR-003 pin: @elysiajs/openapi at the 1.4.15 fallback (2.0 is
// NO-GO per spike 6). `specPath` is set explicitly rather than taking the
@@ -20,12 +21,17 @@ const OPENAPI_SPEC_PATH = '/openapi.json';
// endpoint (ADR-003: "the transform that hides /metrics from the spec").
const OPENAPI_EXCLUDED_PATHS = ['/metrics'];
-// Both surfaces still take `deps` even where unused so a module addition
-// never changes the factory's signature, only its body. Return types are
-// left inferred, as `defineRoutes`'s `RouteApp` is: `.use()`'d plugins widen
-// Elysia's type parameters in a way the bare `Elysia` type cannot express.
-export const publicApiRoutes = (_deps: AppDeps) =>
- new Elysia({ name: 'core/public-api-routes' });
+// A factory always takes `deps` even where its body is empty so a module
+// addition never changes the signature. Return types are left inferred, as
+// `defineRoutes`'s `RouteApp` is: `.use()`'d plugins widen Elysia's type
+// parameters in a way the bare `Elysia` type cannot express.
+//
+// import's /import/events is the first module to land here (M5-004). It is
+// not yet reachable: the clientAuth macro's authenticator is a P8 stub that
+// always returns null (import.routes.ts's header), and main.ts does not
+// mount `publicApiRoutes` until a real `AppDeps` exists (P3/P4/P8).
+export const publicApiRoutes = (deps: AppDeps) =>
+ new Elysia({ name: 'core/public-api-routes' }).use(importRoutes(deps));
// gsc's callback is the first module to land here (M5-002). It is not yet
// reachable in production: main.ts only mounts `dashboardRoutes` once a real
diff --git a/packages/core/src/rpc.router.ts b/packages/core/src/rpc.router.ts
index 3e8801b8d..d45ede326 100644
--- a/packages/core/src/rpc.router.ts
+++ b/packages/core/src/rpc.router.ts
@@ -7,6 +7,7 @@
import { cohortRouter } from './modules/cohort/cohort.rpc';
import { gscRouter } from './modules/gsc/gsc.rpc';
import { healthRouter } from './modules/health/health.rpc';
+import { importRouter } from './modules/import/import.rpc';
import { insightRouter } from './modules/insight/insight.rpc';
import { createTRPCRouter } from './rpc/base';
@@ -15,6 +16,7 @@ export const appRouter = createTRPCRouter({
insight: insightRouter,
gsc: gscRouter,
cohort: cohortRouter,
+ import: importRouter,
});
export type AppRouter = typeof appRouter;
diff --git a/packages/core/src/services.ts b/packages/core/src/services.ts
index 73b4aa00f..64a22c33d 100644
--- a/packages/core/src/services.ts
+++ b/packages/core/src/services.ts
@@ -12,6 +12,10 @@ import {
createCohortService,
} from './modules/cohort/cohort.service';
import { createGscService, type GscService } from './modules/gsc/gsc.service';
+import {
+ createImportService,
+ type ImportService,
+} from './modules/import/import.service';
import {
createInsightService,
type InsightService,
@@ -37,6 +41,7 @@ export interface Services {
insight: InsightService;
gsc: GscService;
cohort: CohortService;
+ import: ImportService;
}
export function createServices(deps: ServiceDeps): Services {
@@ -48,5 +53,6 @@ export function createServices(deps: ServiceDeps): Services {
insight: createInsightService(deps),
gsc: createGscService(deps),
cohort: createCohortService(deps),
+ import: createImportService(deps),
};
}
diff --git a/packages/db/index.ts b/packages/db/index.ts
index 1eb06c2f8..300617dcd 100644
--- a/packages/db/index.ts
+++ b/packages/db/index.ts
@@ -22,7 +22,6 @@ export * from './src/services/funnel.service';
export * from './src/services/group.service';
export * from './src/services/gsc.service';
export * from './src/services/id.service';
-export * from './src/services/import.service';
export * from './src/services/notification.service';
export * from './src/services/organization.service';
export * from './src/services/overview.service';
diff --git a/packages/importer/package.json b/packages/importer/package.json
deleted file mode 100644
index 732079c3e..000000000
--- a/packages/importer/package.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "name": "@openpanel/importer",
- "version": "0.0.0",
- "type": "module",
- "main": "src/index.ts",
- "scripts": {
- "build": "tsc",
- "dev": "tsc --watch",
- "test": "vitest",
- "test:run": "vitest run"
- },
- "exports": {
- ".": "./src/index.ts",
- "./providers": "./src/providers/metadata.ts"
- },
- "dependencies": {
- "@openpanel/common": "workspace:*",
- "@openpanel/core": "workspace:*",
- "@openpanel/db": "workspace:*",
- "@openpanel/queue": "workspace:*",
- "@openpanel/validation": "workspace:*",
- "csv-parse": "^6.1.0",
- "ramda": "^0.29.1",
- "unzipper": "^0.12.3",
- "uuid": "^9.0.1",
- "zod": "catalog:"
- },
- "devDependencies": {
- "@types/node": "^20.0.0",
- "@types/ramda": "^0.31.1",
- "@types/unzipper": "^0.10.10",
- "@types/uuid": "^9.0.7",
- "bullmq": "^5.8.7",
- "typescript": "^5.0.0",
- "vitest": "^1.0.0"
- }
-}
diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.ts
deleted file mode 100644
index 61bb8d9cc..000000000
--- a/packages/importer/src/index.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-export { UmamiProvider } from './providers/umami';
-export { MixpanelProvider } from './providers/mixpanel';
-export { AmplitudeProvider } from './providers/amplitude';
-export type {
- ImportConfig,
- ImportProgress,
- ImportResult,
- BatchResult,
- BaseRawEvent,
- ErrorContext,
- EventProperties,
- ImportJobMetadata,
- ImportStageResult,
-} from './types';
diff --git a/packages/importer/src/providers/metadata.ts b/packages/importer/src/providers/metadata.ts
deleted file mode 100644
index 578e3fa30..000000000
--- a/packages/importer/src/providers/metadata.ts
+++ /dev/null
@@ -1,38 +0,0 @@
-export type ImportProviderId = 'umami' | 'mixpanel' | 'amplitude';
-export type ImportProviderType = 'file' | 'api';
-
-export interface ImportProviderMeta {
- id: ImportProviderId;
- name: string;
- description: string;
- logo: string;
- backgroundColor: string;
- types: ImportProviderType[];
-}
-
-export const IMPORT_PROVIDERS: ImportProviderMeta[] = [
- {
- id: 'umami',
- name: 'Umami',
- description: 'Import your analytics data from Umami',
- logo: 'https://cdn.brandfetch.io/id_3VEohOm/w/180/h/180/theme/dark/logo.png?c=1dxbfHSJFAPEGdCLU4o5B',
- backgroundColor: '#fff',
- types: ['file'],
- },
- {
- id: 'mixpanel',
- name: 'Mixpanel',
- description: 'Import your analytics data from Mixpanel API',
- logo: 'https://cdn.brandfetch.io/idr_rhI2FS/theme/dark/idMJ8uODLv.svg?c=1dxbfHSJFAPEGdCLU4o5B',
- backgroundColor: '#fff',
- types: ['api'],
- },
- {
- id: 'amplitude',
- name: 'Amplitude',
- description: 'Import your analytics data from the Amplitude Export API',
- logo: 'https://cdn.brandfetch.io/idnYrZGER0/theme/dark/symbol.svg?c=1dxbfHSJFAPEGdCLU4o5B',
- backgroundColor: '#fff',
- types: ['api'],
- },
-];
diff --git a/packages/importer/tsconfig.json b/packages/importer/tsconfig.json
deleted file mode 100644
index 29cedb070..000000000
--- a/packages/importer/tsconfig.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "extends": "../../tooling/typescript/base.json",
- "compilerOptions": {
- "outDir": "./dist",
- "rootDir": "./src",
- "strictNullChecks": true
- },
- "include": ["src/**/*"],
- "exclude": ["node_modules", "dist"]
-}
diff --git a/packages/importer/vitest.config.ts b/packages/importer/vitest.config.ts
deleted file mode 100644
index f87a2039f..000000000
--- a/packages/importer/vitest.config.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-import { getSharedVitestConfig } from '../../vitest.shared';
-
-export default getSharedVitestConfig({ __dirname });
diff --git a/packages/validation/src/import.validation.ts b/packages/validation/src/import.validation.ts
new file mode 100644
index 000000000..662da343a
--- /dev/null
+++ b/packages/validation/src/import.validation.ts
@@ -0,0 +1,9 @@
+// Moved into @openpanel/core's import module (M5-004, ADR-008's module map:
+// import owns "C"). Re-exported here for existing @openpanel/validation
+// importers (apps/start's import UI, packages/db/src/types.ts) — same shape
+// as packages/validation/src/cohort.validation.ts since M5-003. The
+// `./modules/import/import.constants` subpath is a leaf, isomorphic file
+// with no back-imports into @openpanel/validation, so this re-export carries
+// no runtime cycle despite @openpanel/core already depending on
+// @openpanel/validation elsewhere.
+export * from '@openpanel/core/modules/import/import.constants';
diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts
index c4aec23fa..22bcd1e6c 100644
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -1,5 +1,3 @@
-import { z } from 'zod';
-
import {
chartSegments,
chartTypes,
@@ -10,6 +8,7 @@ import {
operators,
timeWindows,
} from '@openpanel/constants';
+import { z } from 'zod';
/**
* Chart formulas are plain arithmetic over series references (A, B, C, ...).
@@ -20,7 +19,7 @@ import {
const CHART_FORMULA_PATTERN = /^[A-Za-z0-9_ .,+\-*/()%^]*$/;
export function objectToZodEnums<K extends string>(
- obj: Record<K, any>,
+ obj: Record<K, any>
): [K, ...K[]] {
const [firstKey, ...otherKeys] = Object.keys(obj) as K[];
return [firstKey!, ...otherKeys];
@@ -42,21 +41,21 @@ export const zChartEventFilter = z.object({
.optional()
.describe(
'Cast type for the column/value in equality & comparison operators ' +
- '(string/number/date/datetime/boolean). Absent = legacy behavior.',
+ '(string/number/date/datetime/boolean). Absent = legacy behavior.'
),
cohortId: z
.string()
.optional()
.describe(
'DEPRECATED: legacy single-cohort id, kept for saved reports. ' +
- 'New code reads cohortIds via getCohortIds(filter).',
+ 'New code reads cohortIds via getCohortIds(filter).'
),
cohortIds: z
.array(z.string())
.optional()
.describe(
'Cohort IDs for inCohort/notInCohort. Multiple ids OR-match ' +
- '(matches profiles in any of the listed cohorts).',
+ '(matches profiles in any of the listed cohorts).'
),
});
@@ -100,7 +99,7 @@ export const zChartEvent = z.object({
.string()
.optional()
.describe(
- 'Optional property of the event used for specific segment calculations (e.g., value for property_sum/average)',
+ 'Optional property of the event used for specific segment calculations (e.g., value for property_sum/average)'
),
segment: zChartEventSegment,
filters: z
@@ -120,7 +119,7 @@ export const zChartFormula = z.object({
.max(1000)
.regex(
CHART_FORMULA_PATTERN,
- 'Formula may only contain series references, numbers and arithmetic operators',
+ 'Formula may only contain series references, numbers and arithmetic operators'
)
.describe('The formula expression (e.g., A+B, A/B)'),
displayName: z
@@ -131,7 +130,7 @@ export const zChartFormula = z.object({
.array(z.string())
.optional()
.describe(
- 'Alpha IDs (e.g. ["A", "B"]) of series referenced by this formula that should be hidden from the chart while still being used in the formula computation',
+ 'Alpha IDs (e.g. ["A", "B"]) of series referenced by this formula that should be hidden from the chart while still being used in the formula computation'
),
});
@@ -153,7 +152,7 @@ export const zChartBreakdown = z.object({
export const zChartSeries = z
.array(zChartEventItem)
.describe(
- 'Array of series (events or formulas) to be tracked and displayed in the chart',
+ 'Array of series (events or formulas) to be tracked and displayed in the chart'
);
export const zChartBreakdowns = z.array(zChartBreakdown);
@@ -238,10 +237,10 @@ export const zReportInput = z.object({
interval: zTimeInterval
.default('day')
.describe(
- 'The time interval for data aggregation (e.g., day, week, month)',
+ 'The time interval for data aggregation (e.g., day, week, month)'
),
series: zChartSeries.describe(
- 'Array of series (events or formulas) to be tracked and displayed in the chart',
+ 'Array of series (events or formulas) to be tracked and displayed in the chart'
),
breakdowns: zChartBreakdowns
.default([])
@@ -250,7 +249,7 @@ export const zReportInput = z.object({
.array(zChartEventFilter)
.optional()
.describe(
- 'Filters applied to ALL event series in this report (combined with each series own filters using AND)',
+ 'Filters applied to ALL event series in this report (combined with each series own filters using AND)'
),
range: zRange
.default('30d')
@@ -259,13 +258,13 @@ export const zReportInput = z.object({
.string()
.nullish()
.describe(
- 'Custom start date for the data range (overrides range if provided)',
+ 'Custom start date for the data range (overrides range if provided)'
),
endDate: z
.string()
.nullish()
.describe(
- 'Custom end date for the data range (overrides range if provided)',
+ 'Custom end date for the data range (overrides range if provided)'
),
previous: z
.boolean()
@@ -278,7 +277,7 @@ export const zReportInput = z.object({
metric: zMetric
.default('sum')
.describe(
- 'The aggregation method for the metric (e.g., sum, count, average)',
+ 'The aggregation method for the metric (e.g., sum, count, average)'
),
limit: z
.number()
@@ -304,7 +303,7 @@ export const zReportInput = z.object({
.string()
.optional()
.describe(
- "Optional unit of measurement for the chart's Y-axis (e.g., $, %, users)",
+ "Optional unit of measurement for the chart's Y-axis (e.g., $, %, users)"
),
});
@@ -390,7 +389,7 @@ export const zOnboardingProject = z
timezone: z.string().optional(),
})
.superRefine((data, ctx) => {
- if (!data.organization && !data.organizationId) {
+ if (!(data.organization || data.organizationId)) {
ctx.addIssue({
code: 'custom',
message: 'Organization is required',
@@ -426,7 +425,6 @@ export const zOnboardingProject = z
}
});
-
export * from './integrations';
export const zNotificationRuleEventConfig = z.object({
@@ -608,7 +606,7 @@ export const zGroupId = z
.min(1)
.regex(
/^[a-z0-9_-]+$/,
- 'ID must only contain lowercase letters, digits, hyphens, or underscores',
+ 'ID must only contain lowercase letters, digits, hyphens, or underscores'
);
export const zCreateGroup = z.object({
@@ -635,103 +633,10 @@ export const zEditOrganization = z.object({
timezone: z.string().min(1),
});
-const zProjectMapper = z.object({
- from: z.string().min(1),
- to: z.string().min(1),
-});
-
-/**
- * `z.string().url()` alone accepts `file:`, `gopher:` and friends. Restricting
- * the scheme here gives the user an immediate form error instead of a job that
- * fails later. It is NOT the SSRF control - the value is stored and fetched
- * afterwards, so the destination is re-validated at fetch time by
- * `safeFetchStream`.
- */
-export const zHttpUrl = z
- .string()
- .url()
- .refine(
- (value) => {
- try {
- const { protocol } = new URL(value);
- return protocol === 'http:' || protocol === 'https:';
- } catch {
- return false;
- }
- },
- { message: 'Only http and https URLs are allowed' },
- );
-
-const createFileImportConfig = <T extends string>(provider: T) =>
- z.object({
- provider: z.literal(provider),
- type: z.literal('file'),
- fileUrl: zHttpUrl,
- });
-
-// Import configs
-export const zUmamiImportConfig = createFileImportConfig('umami').extend({
- projectMapper: z.array(zProjectMapper),
-});
-
-export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;
-
-export const zPlausibleImportConfig = createFileImportConfig('plausible');
-export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;
-
-export const zAmplitudeDataResidency = z.enum(['us', 'eu']);
-export type IAmplitudeDataResidency = z.infer<typeof zAmplitudeDataResidency>;
-
-export const zAmplitudeImportConfig = z.object({
- provider: z.literal('amplitude'),
- type: z.literal('api'),
- apiKey: z.string().min(1),
- secretKey: z.string().min(1),
- from: z.string().min(1),
- to: z.string().min(1),
- mapScreenViewProperty: z.string().optional(),
- dataResidency: zAmplitudeDataResidency.optional(),
-});
-export type IAmplitudeImportConfig = z.infer<typeof zAmplitudeImportConfig>;
-
-export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);
-export type IMixpanelDataResidency = z.infer<typeof zMixpanelDataResidency>;
-
-export const zMixpanelImportConfig = z.object({
- provider: z.literal('mixpanel'),
- type: z.literal('api'),
- serviceAccount: z.string().min(1),
- serviceSecret: z.string().min(1),
- projectId: z.string().min(1),
- from: z.string().min(1),
- to: z.string().min(1),
- mapScreenViewProperty: z.string().optional(),
- dataResidency: zMixpanelDataResidency.optional(),
-});
-export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;
-
-export type IImportConfig =
- | IUmamiImportConfig
- | IPlausibleImportConfig
- | IMixpanelImportConfig
- | IAmplitudeImportConfig;
-
-export const zCreateImport = z.object({
- projectId: z.string().min(1),
- provider: z.enum(['umami', 'plausible', 'mixpanel', 'amplitude']),
- config: z.union([
- zUmamiImportConfig,
- zPlausibleImportConfig,
- zMixpanelImportConfig,
- zAmplitudeImportConfig,
- ]),
-});
-
-export type ICreateImport = z.infer<typeof zCreateImport>;
-
-export * from './types.insights';
-export * from './types.validation';
-export * from './track.validation';
-export * from './event-blocklist';
export * from './chat';
export * from './cohort.validation';
+export * from './event-blocklist';
+export * from './import.validation';
+export * from './track.validation';
+export * from './types.insights';
+export * from './types.validation';
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 41bc01939..bf08bcfe2 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -429,9 +429,9 @@ importers:
'@openpanel/constants':
specifier: workspace:^
version: link:../../packages/constants
- '@openpanel/importer':
+ '@openpanel/core':
specifier: workspace:^
- version: link:../../packages/importer
+ version: link:../../packages/core
'@openpanel/json':
specifier: workspace:*
version: link:../../packages/json
@@ -953,9 +953,6 @@ importers:
'@openpanel/email':
specifier: workspace:*
version: link:../../packages/email
- '@openpanel/importer':
- specifier: workspace:*
- version: link:../../packages/importer
'@openpanel/payments':
specifier: workspace:*
version: link:../../packages/payments
@@ -1153,6 +1150,9 @@ importers:
bullmq:
specifier: 6.3.3
+ csv-parse:
+ specifier: ^6.1.0
+ version: 6.1.0
elysia:
specifier: 1.4.30
@@ -1177,6 +1177,9 @@ importers:
qrcode:
specifier: ^1.5.4
version: 1.5.4
+ ramda:
+ specifier: ^0.29.1
+ version: 0.29.1
sqlstring:
specifier: ^2.3.3
version: 2.3.3
@@ -1189,6 +1192,9 @@ importers:
undici:
specifier: ^8.10.0
version: 8.10.0
+ unzipper:
+ specifier: ^0.12.3
+ version: 0.12.5
zod:
specifier: 'catalog:'
version: 4.3.6
@@ -1202,9 +1208,15 @@ importers:
'@types/qrcode':
specifier: ^1.5.6
version: 1.5.6
+ '@types/ramda':
+ specifier: ^0.31.1
+ version: 0.31.1
'@types/sqlstring':
specifier: ^2.3.2
version: 2.3.2
+ '@types/unzipper':
+ specifier: ^0.10.10
+ version: 0.10.11
bun-types:
specifier: 1.4.0
version: 1.4.0
@@ -1358,61 +1370,6 @@ importers:
specifier: 'catalog:'
version: 5.9.3
- packages/importer:
- dependencies:
- '@openpanel/common':
- specifier: workspace:*
- version: link:../common
- '@openpanel/core':
- specifier: workspace:*
- version: link:../core
- '@openpanel/db':
- specifier: workspace:*
- version: link:../db
- '@openpanel/queue':
- specifier: workspace:*
- version: link:../queue
- '@openpanel/validation':
- specifier: workspace:*
- version: link:../validation
- csv-parse:
- specifier: ^6.1.0
- version: 6.1.0
- ramda:
- specifier: ^0.29.1
- version: 0.29.1
- unzipper:
- specifier: ^0.12.3
- version: 0.12.5
- uuid:
- specifier: ^9.0.1
- version: 9.0.1
- zod:
- specifier: 'catalog:'
- version: 4.3.6
- devDependencies:
- '@types/node':
- specifier: ^20.0.0
- version: 20.19.24
- '@types/ramda':
- specifier: ^0.31.1
- version: 0.31.1
- '@types/unzipper':
- specifier: ^0.10.10
- version: 0.10.11
- '@types/uuid':
- specifier: ^9.0.7
- version: 9.0.8
- bullmq:
- specifier: ^5.8.7
- version: 5.63.0([email protected])
- typescript:
- specifier: ^5.0.0
- version: 5.9.3
- vitest:
- specifier: ^1.0.0
-
packages/json:
dependencies:
superjson:
@@ -24068,14 +24025,14 @@ snapshots:
dependencies:
'@jest/fake-timers': 29.7.0
'@jest/types': 29.6.3
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
jest-mock: 29.7.0
dependencies:
'@jest/types': 29.6.3
'@sinonjs/fake-timers': 10.3.0
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
jest-message-util: 29.7.0
jest-mock: 29.7.0
jest-util: 29.7.0
@@ -24088,7 +24045,7 @@ snapshots:
dependencies:
'@types/istanbul-lib-coverage': 2.0.6
'@types/istanbul-reports': 3.0.4
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/yargs': 15.0.19
chalk: 4.1.2
@@ -24097,7 +24054,7 @@ snapshots:
'@jest/schemas': 29.6.3
'@types/istanbul-lib-coverage': 2.0.6
'@types/istanbul-reports': 3.0.4
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/yargs': 17.0.32
chalk: 4.1.2
@@ -28168,18 +28125,18 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
'@slack/logger': 3.0.0
'@slack/web-api': 6.12.1([email protected]([email protected]))
'@types/jsonwebtoken': 8.5.9
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
jsonwebtoken: 9.0.2
lodash.isstring: 4.0.1
transitivePeerDependencies:
@@ -28200,7 +28157,7 @@ snapshots:
dependencies:
'@slack/logger': 3.0.0
'@slack/web-api': 6.12.1([email protected]([email protected]))
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/ws': 7.4.7
eventemitter3: 5.0.4
finity: 0.5.4
@@ -28217,7 +28174,7 @@ snapshots:
'@slack/logger': 3.0.0
'@slack/types': 2.14.0
'@types/is-stream': 1.1.0
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
eventemitter3: 3.1.2
form-data: 2.5.6
@@ -28232,7 +28189,7 @@ snapshots:
dependencies:
'@slack/logger': 4.0.0
'@slack/types': 2.14.0
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/retry': 0.12.0
eventemitter3: 5.0.4
@@ -29325,11 +29282,11 @@ snapshots:
dependencies:
'@types/connect': 3.4.38
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/[email protected]': {}
@@ -29339,13 +29296,13 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/[email protected]': {}
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/[email protected]': {}
@@ -29533,18 +29490,18 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/qs': 6.9.11
'@types/range-parser': 1.2.7
'@types/send': 0.17.4
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/qs': 6.9.11
'@types/range-parser': 1.2.7
'@types/send': 0.17.4
@@ -29596,7 +29553,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/[email protected]': {}
@@ -29614,12 +29571,12 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
'@types/ms': 0.7.34
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/[email protected]': {}
@@ -29655,7 +29612,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/[email protected]': {}
@@ -29667,7 +29624,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
@@ -29675,7 +29632,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
form-data: 4.0.5
'@types/[email protected]': {}
@@ -29712,13 +29669,13 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
pg-protocol: 1.6.1
pg-types: 2.2.0
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
pg-protocol: 1.6.1
pg-types: 2.2.0
@@ -29780,7 +29737,7 @@ snapshots:
dependencies:
'@types/caseless': 0.12.5
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/tough-cookie': 4.0.5
form-data: 2.5.6
@@ -29793,18 +29750,18 @@ snapshots:
dependencies:
'@types/mime': 1.3.5
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
'@types/http-errors': 2.0.4
'@types/mime': 3.0.4
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
'@types/http-errors': 2.0.4
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/send': 0.17.4
'@types/[email protected]': {}
@@ -29823,7 +29780,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
@@ -29832,7 +29789,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/[email protected]': {}
@@ -29850,7 +29807,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/[email protected]': {}
@@ -29869,13 +29826,13 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/source-list-map': 0.1.6
source-map: 0.7.6
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
'@types/tapable': 1.0.12
'@types/uglify-js': 3.17.5
'@types/webpack-sources': 3.2.3
@@ -29884,7 +29841,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
dependencies:
@@ -31310,7 +31267,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
escape-string-regexp: 4.0.0
is-wsl: 2.2.0
lighthouse-logger: 1.4.2([email protected])
@@ -31319,7 +31276,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
escape-string-regexp: 4.0.0
is-wsl: 2.2.0
lighthouse-logger: 1.4.2([email protected])
@@ -32405,7 +32362,7 @@ snapshots:
dependencies:
'@types/cookie': 0.4.1
'@types/cors': 2.8.17
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
accepts: 1.3.8
base64id: 2.0.0
cookie: 0.7.2
@@ -34792,7 +34749,7 @@ snapshots:
'@jest/environment': 29.7.0
'@jest/fake-timers': 29.7.0
'@jest/types': 29.6.3
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
jest-mock: 29.7.0
jest-util: 29.7.0
@@ -34813,13 +34770,13 @@ snapshots:
dependencies:
'@jest/types': 29.6.3
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
jest-util: 29.7.0
dependencies:
'@jest/types': 29.6.3
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
chalk: 4.1.2
ci-info: 3.9.0
graceful-fs: 4.2.11
@@ -34836,7 +34793,7 @@ snapshots:
dependencies:
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
jest-util: 29.7.0
merge-stream: 2.0.0
supports-color: 8.1.1
@@ -37935,7 +37892,7 @@ snapshots:
'@protobufjs/path': 1.1.2
'@protobufjs/pool': 1.1.0
'@protobufjs/utf8': 1.1.0
- '@types/node': 20.19.24
+ '@types/node': 24.10.1
long: 5.2.3
@@ -40782,24 +40739,6 @@ snapshots:
dependencies:
- dependencies:
- cac: 6.7.14
- debug: 4.4.3([email protected])
- pathe: 1.1.2
- picocolors: 1.1.1
- transitivePeerDependencies:
- - '@types/node'
- - less
- - lightningcss
- - sass
- - sass-embedded
- - stylus
- - sugarss
- - supports-color
- - terser
-
dependencies:
cac: 6.7.14
@@ -40912,17 +40851,6 @@ snapshots:
- supports-color
- typescript
- dependencies:
- esbuild: 0.21.5
- postcss: 8.5.6
- rollup: 4.52.5
- optionalDependencies:
- '@types/node': 20.19.24
- fsevents: 2.3.3
- lightningcss: 1.30.2
- terser: 5.27.1
-
dependencies:
esbuild: 0.21.5
@@ -40993,41 +40921,6 @@ snapshots:
optionalDependencies:
- dependencies:
- '@vitest/expect': 1.6.1
- '@vitest/runner': 1.6.1
- '@vitest/snapshot': 1.6.1
- '@vitest/spy': 1.6.1
- '@vitest/utils': 1.6.1
- acorn-walk: 8.3.2
- chai: 4.5.0
- debug: 4.4.3([email protected])
- execa: 8.0.1
- local-pkg: 0.5.1
- magic-string: 0.30.21
- pathe: 1.1.2
- picocolors: 1.1.1
- std-env: 3.10.0
- strip-literal: 2.1.1
- tinybench: 2.9.0
- tinypool: 0.8.4
- why-is-node-running: 2.3.0
- optionalDependencies:
- '@types/node': 20.19.24
- jsdom: 26.1.0([email protected])
- transitivePeerDependencies:
- - less
- - lightningcss
- - sass
- - sass-embedded
- - stylus
- - sugarss
- - supports-color
- - terser
-
dependencies:
'@vitest/expect': 1.6.1