OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M5-004): import module: packages/importer absorbed + import.service + trpc + /import REST + import queue
ralph315f0b5f73a0
diff --git a/apps/api/src/controllers/import.controller.ts b/apps/api/src/controllers/import.controller.tsindex 188aa7253..ae246b5ad 100644--- a/apps/api/src/controllers/import.controller.ts+++ b/apps/api/src/controllers/import.controller.ts@@ -1,39 +1,28 @@-import type { FastifyReply, FastifyRequest } from 'fastify';--import { toDots } from '@openpanel/common';+import { insertRawEventsBatch } from '@openpanel/core';import type { IClickhouseEvent } from '@openpanel/db';-import { TABLE_NAMES, ch, formatClickhouseDate } from '@openpanel/db';+import type { FastifyReply, FastifyRequest } from 'fastify';+// Dissolved into @openpanel/core's import module (M5-004): the+// toDots/project_id/imported_at stamping and the ClickHouse insert moved to+// import.service.ts's `insertRawEventsBatch`. This controller stays+// (DELEGATE PATTERN) — it is the live Fastify handler, a thin wrapper+// resolving `request.client` and translating the result into a Fastify+// reply.export async function importEvents(request: FastifyRequest<{Body: IClickhouseEvent[];}>,- reply: FastifyReply,+ reply: FastifyReply) {const projectId = request.client?.projectId;if (!projectId) {throw new Error('Project ID is required');}- const importedAt = formatClickhouseDate(new Date());- const values: IClickhouseEvent[] = request.body.map((event) => {- return {- ...event,- properties: toDots(event.properties),- project_id: projectId,- created_at: formatClickhouseDate(event.created_at),- imported_at: importedAt,- };- });-try {- const res = await ch.insert({- table: TABLE_NAMES.events,- values,- format: 'JSONEachRow',- });+ const { writtenRows } = await insertRawEventsBatch(projectId, request.body);- console.log(res.summary?.written_rows, 'events imported');+ console.log(writtenRows, 'events imported');reply.send('OK');} catch (e) {console.error(e);diff --git a/apps/start/Dockerfile b/apps/start/Dockerfileindex 02e4bd0a7..ee139ae8f 100644--- a/apps/start/Dockerfile+++ b/apps/start/Dockerfile@@ -29,7 +29,6 @@ COPY apps/start/package.json ./apps/start/COPY packages/trpc/package.json packages/trpc/COPY packages/json/package.json packages/json/COPY packages/common/package.json packages/common/-COPY packages/importer/package.json packages/importer/COPY packages/payments/package.json packages/payments/COPY packages/constants/package.json packages/constants/COPY packages/validation/package.json packages/validation/@@ -86,7 +85,6 @@ COPY --from=build /app/packages/trpc/package.json ./packages/trpc/COPY --from=build /app/packages/auth/package.json ./packages/auth/COPY --from=build /app/packages/json/package.json ./packages/json/COPY --from=build /app/packages/common/package.json ./packages/common/-COPY --from=build /app/packages/importer/package.json ./packages/importer/COPY --from=build /app/packages/payments/package.json ./packages/payments/COPY --from=build /app/packages/constants/package.json ./packages/constants/COPY --from=build /app/packages/validation/package.json ./packages/validation/diff --git a/apps/start/package.json b/apps/start/package.jsonindex 3421bea54..a9fba591b 100644--- a/apps/start/package.json+++ b/apps/start/package.json@@ -33,7 +33,7 @@"@number-flow/react": "0.5.10","@openpanel/common": "workspace:^","@openpanel/constants": "workspace:^",- "@openpanel/importer": "workspace:^",+ "@openpanel/core": "workspace:^","@openpanel/json": "workspace:*","@openpanel/payments": "workspace:*","@openpanel/sdk-info": "workspace:^",diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.imports.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.imports.tsxindex 602ceb440..f2e9943cf 100644--- a/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.imports.tsx+++ b/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.imports.tsx@@ -1,4 +1,4 @@-import { IMPORT_PROVIDERS } from '@openpanel/importer/providers';+import { IMPORT_PROVIDERS } from '@openpanel/core/modules/import/import.constants';import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';import { createFileRoute } from '@tanstack/react-router';import { formatDistanceToNow } from 'date-fns';diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfileindex bc485eb89..dd496926b 100644--- a/apps/worker/Dockerfile+++ b/apps/worker/Dockerfile@@ -34,7 +34,6 @@ COPY packages/redis/package.json ./packages/redis/COPY packages/queue/package.json ./packages/queue/COPY packages/logger/package.json ./packages/logger/COPY packages/common/package.json ./packages/common/-COPY packages/importer/package.json ./packages/importer/COPY packages/payments/package.json ./packages/payments/COPY packages/constants/package.json ./packages/constants/COPY packages/validation/package.json ./packages/validation/@@ -82,7 +81,6 @@ COPY --from=build /app/packages/redis ./packages/redisCOPY --from=build /app/packages/queue ./packages/queueCOPY --from=build /app/packages/logger ./packages/loggerCOPY --from=build /app/packages/common ./packages/common-COPY --from=build /app/packages/importer ./packages/importerCOPY --from=build /app/packages/payments ./packages/paymentsCOPY --from=build /app/packages/constants ./packages/constantsCOPY --from=build /app/packages/validation ./packages/validationdiff --git a/apps/worker/package.json b/apps/worker/package.jsonindex f7b38a3f7..0b8f59a2d 100644--- a/apps/worker/package.json+++ b/apps/worker/package.json@@ -18,7 +18,6 @@"@openpanel/core": "workspace:*","@openpanel/db": "workspace:*","@openpanel/email": "workspace:*",- "@openpanel/importer": "workspace:*","@openpanel/payments": "workspace:*","@openpanel/queue": "workspace:*","@openpanel/redis": "workspace:*",diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.tsindex a767c7e30..6941ea53b 100644--- a/apps/worker/src/jobs/import.ts+++ b/apps/worker/src/jobs/import.ts@@ -1,395 +1,15 @@-import type { ILogger } from '@openpanel/core';-import {- backfillSessionsToProduction,- cleanupSessionStartEndEvents,- cleanupStagingData,- createSessionsStartEndEvents,- db,- generateGapBasedSessionIds,- getImportDateBounds,- type IClickhouseEvent,- type IClickhouseProfile,- insertImportBatch,- insertProfilesBatch,- moveImportsToProduction,- type Prisma,- updateImportStatus,-} from '@openpanel/db';-import {- AmplitudeProvider,- MixpanelProvider,- UmamiProvider,-} from '@openpanel/importer';+// Dissolved into @openpanel/core's import module (M5-004): the ClickHouse+// staging/session pipeline, the provider dispatch and the job body itself+// moved to packages/core/src/modules/import/import.service.ts. This file+// stays (DELEGATE PATTERN) — it is the BullMQ job body V1's worker registers+// (boot-workers.ts), a thin wrapper handing the real BullMQ `Job` to+// `runImportJob` as its progress reporter (`ImportJobProgress` — a BullMQ+// `Job` satisfies it structurally, see import.service.ts's header).+import { runImportJob } from '@openpanel/core';import type { ImportQueuePayload } from '@openpanel/queue';import type { Job } from 'bullmq';-import { logger } from '../utils/logger';--const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10);--function yieldToEventLoop(): Promise<void> {- return new Promise((resolve) => {- setTimeout(resolve, 100);- });-}--const RESUMABLE_STEPS = ['creating_sessions', 'moving', 'backfilling_sessions'];--/**- * Merge a freshly-derived profile into the bounded dedup map: keep the earliest- * created_at (first seen), advance last_seen_at to the latest activity, and fill- * identity fields/properties, preferring the newer row.- */-function mergeProfileInto(- map: Map<string, IClickhouseProfile>,- incoming: IClickhouseProfile-): void {- const existing = map.get(incoming.id);- if (!existing) {- map.set(incoming.id, incoming);- return;- }-- const createdAt =- existing.created_at < incoming.created_at- ? existing.created_at- : incoming.created_at;- const isIncomingNewer = incoming.last_seen_at >= existing.last_seen_at;- const base = isIncomingNewer ? incoming : existing;- const other = isIncomingNewer ? existing : incoming;-- map.set(incoming.id, {- ...base,- created_at: createdAt,- first_name: base.first_name || other.first_name,- last_name: base.last_name || other.last_name,- email: base.email || other.email,- avatar: base.avatar || other.avatar,- properties: { ...other.properties, ...base.properties },- });-}export async function importJob(job: Job<ImportQueuePayload>) {const { importId } = job.data.payload;-- const record = await db.import.findUniqueOrThrow({- where: { id: importId },- include: { project: true },- });-- const jobLogger = logger.child({ importId, config: record.config });- jobLogger.info('Starting import job');-- const providerInstance = createProvider(record, jobLogger);- const shouldGenerateSessionIds = providerInstance.shouldGenerateSessionIds();-- try {- const isRetry = record.currentStep !== null;- const canResume =- isRetry && RESUMABLE_STEPS.includes(record.currentStep as string);-- // -------------------------------------------------------- // STAGING PHASE: clean slate on failure, run from scratch- // -------------------------------------------------------- if (!canResume) {- if (isRetry) {- jobLogger.info(- 'Retry detected before resumable phase — cleaning staging data'- );- await cleanupStagingData(importId);- }-- // Phase 1: Load events into staging- await updateImportStatus(jobLogger, job, importId, { step: 'loading' });-- const totalEvents = await providerInstance- .getTotalEventsCount()- .catch(() => -1);- let processedEvents = 0;- const eventBatch: IClickhouseEvent[] = [];-- // Profiles derived inline from events (e.g. Amplitude, which has no- // profile export API). A bounded dedup map keeps memory flat regardless of- // event volume; the profiles table's ReplacingMergeTree(last_seen_at)- // collapses any duplicate rows that span flush boundaries to the latest- // activity per id.- const canProfileFromEvents =- typeof (providerInstance as AmplitudeProvider)- .transformEventToProfile === 'function';- const PROFILE_MAP_CAP = 50_000;- const profileMap = new Map<string, IClickhouseProfile>();- let processedProfiles = 0;-- const flushProfiles = async () => {- if (profileMap.size === 0) {- return;- }- const values = Array.from(profileMap.values());- await insertProfilesBatch(values, record.projectId);- processedProfiles += values.length;- profileMap.clear();- await updateImportStatus(jobLogger, job, importId, {- step: 'loading_profiles',- processedProfiles,- });- await yieldToEventLoop();- };-- for await (const rawEvent of providerInstance.parseSource()) {- if (- !providerInstance.validate(- // @ts-expect-error -- provider-specific raw type- rawEvent- )- ) {- jobLogger.warn({ rawEvent }, 'Skipping invalid event');- continue;- }-- const transformed: IClickhouseEvent = providerInstance.transformEvent(- // @ts-expect-error -- provider-specific raw type- rawEvent- );-- // Session IDs for providers that need them (e.g. Mixpanel) are generated- // in generateGapBasedSessionIds after loading, using gap-based logic.- eventBatch.push(transformed);-- if (canProfileFromEvents) {- const profile = (- providerInstance as AmplitudeProvider- ).transformEventToProfile(- // @ts-expect-error -- provider-specific raw type- rawEvent- );- if (profile) {- mergeProfileInto(profileMap, profile);- if (profileMap.size >= PROFILE_MAP_CAP) {- await flushProfiles();- }- }- }-- if (eventBatch.length >= BATCH_SIZE) {- await insertImportBatch(eventBatch, importId);- processedEvents += eventBatch.length;-- const batchDate = new Date(eventBatch[0]?.created_at || '')- .toISOString()- .split('T')[0];-- await updateImportStatus(jobLogger, job, importId, {- step: 'loading',- batch: batchDate,- totalEvents,- processedEvents,- });-- eventBatch.length = 0;- await yieldToEventLoop();- }- }-- if (eventBatch.length > 0) {- await insertImportBatch(eventBatch, importId);- processedEvents += eventBatch.length;-- const batchDate = new Date(eventBatch[0]?.created_at || '')- .toISOString()- .split('T')[0];-- await updateImportStatus(jobLogger, job, importId, {- step: 'loading',- batch: batchDate,- totalEvents,- processedEvents,- });- eventBatch.length = 0;- }-- jobLogger.info({ processedEvents }, 'Loading complete');-- // Phase 1a: Flush profiles derived inline from events (Amplitude)- if (canProfileFromEvents) {- await flushProfiles();- jobLogger.info(- { processedProfiles },- 'Inline profile derivation complete'- );- }-- // Phase 1b: Load user profiles (Mixpanel only)- const profileBatchSize = 5000;- if (- 'streamProfiles' in providerInstance &&- typeof (providerInstance as MixpanelProvider).streamProfiles ===- 'function'- ) {- await updateImportStatus(jobLogger, job, importId, {- step: 'loading_profiles',- });-- const profileBatch: IClickhouseProfile[] = [];- let processedProfiles = 0;-- for await (const rawProfile of (- providerInstance as MixpanelProvider- ).streamProfiles()) {- const profile = (- providerInstance as MixpanelProvider- ).transformProfile(rawProfile);- profileBatch.push(profile);-- if (profileBatch.length >= profileBatchSize) {- await insertProfilesBatch(profileBatch, record.projectId);- processedProfiles += profileBatch.length;- await updateImportStatus(jobLogger, job, importId, {- step: 'loading_profiles',- processedProfiles,- });- profileBatch.length = 0;- await yieldToEventLoop();- }- }-- if (profileBatch.length > 0) {- await insertProfilesBatch(profileBatch, record.projectId);- processedProfiles += profileBatch.length;- await updateImportStatus(jobLogger, job, importId, {- step: 'loading_profiles',- processedProfiles,- totalProfiles: processedProfiles,- });- }-- jobLogger.info({ processedProfiles }, 'Profile loading complete');- }-- // Phase 2: Generate gap-based session IDs (Mixpanel etc.)- if (shouldGenerateSessionIds) {- await updateImportStatus(jobLogger, job, importId, {- step: 'generating_sessions',- });- await generateGapBasedSessionIds(importId);- await yieldToEventLoop();- jobLogger.info('Session ID generation complete');- }- }-- // -------------------------------------------------------- // SESSION CREATION PHASE: resumable by cleaning session_start/end- // -------------------------------------------------------- const skipSessionCreation =- canResume && record.currentStep !== 'creating_sessions';-- if (!skipSessionCreation) {- if (canResume && record.currentStep === 'creating_sessions') {- jobLogger.info(- 'Retry at creating_sessions — cleaning existing session_start/end events'- );- await cleanupSessionStartEndEvents(importId);- }-- await updateImportStatus(jobLogger, job, importId, {- step: 'creating_sessions',- batch: 'all sessions',- });- await createSessionsStartEndEvents(importId);- await yieldToEventLoop();-- jobLogger.info('Session event creation complete');- }-- // -------------------------------------------------------- // PRODUCTION PHASE: resume-safe, track progress per batch- // --------------------------------------------------------- // Phase 3: Move staging events to production (per-day)- const resumeMovingFrom =- canResume && record.currentStep === 'moving'- ? (record.currentBatch ?? undefined)- : undefined;-- // currentBatch is the last successfully completed day — resume from the next day to avoid re-inserting it- const moveFromDate = (() => {- if (!resumeMovingFrom) {- return undefined;- }- const next = new Date(`${resumeMovingFrom}T12:00:00Z`);- next.setUTCDate(next.getUTCDate() + 1);- return next.toISOString().split('T')[0]!;- })();-- const bounds = await getImportDateBounds(importId, moveFromDate);- if (bounds.min && bounds.max) {- const startDate = bounds.min.split(' ')[0]!;- const endDate = bounds.max.split(' ')[0]!;- const cursor = new Date(`${startDate}T12:00:00Z`);- const end = new Date(`${endDate}T12:00:00Z`);-- while (cursor <= end) {- const dateStr = cursor.toISOString().split('T')[0]!;-- await moveImportsToProduction(importId, dateStr);- await updateImportStatus(jobLogger, job, importId, {- step: 'moving',- batch: dateStr,- });-- await yieldToEventLoop();- cursor.setUTCDate(cursor.getUTCDate() + 1);- }- }-- jobLogger.info('Move to production complete');-- // Phase 4: Backfill sessions table- await updateImportStatus(jobLogger, job, importId, {- step: 'backfilling_sessions',- batch: 'all sessions',- });- await backfillSessionsToProduction(importId);- await yieldToEventLoop();-- jobLogger.info('Session backfill complete');-- // Done- await updateImportStatus(jobLogger, job, importId, { step: 'completed' });- jobLogger.info('Import completed');-- return { success: true };- } catch (error) {- jobLogger.error({ err: error }, 'Import job failed');-- try {- const errorMsg = error instanceof Error ? error.message : 'Unknown error';- await updateImportStatus(jobLogger, job, importId, {- step: 'failed',- errorMessage: errorMsg,- });- } catch (markError) {- jobLogger.error(- { err: error, markError },- 'Failed to mark import as failed'- );- }-- throw error;- }-}--function createProvider(- record: Prisma.ImportGetPayload<{ include: { project: true } }>,- jobLogger: ILogger-) {- const config = record.config;- switch (config.provider) {- case 'umami':- return new UmamiProvider(record.projectId, config, jobLogger);- case 'mixpanel':- return new MixpanelProvider(record.projectId, config, jobLogger);- case 'amplitude':- return new AmplitudeProvider(record.projectId, config, jobLogger);- default:- throw new Error(`Unknown provider: ${config.provider}`);- }+ return runImportJob(importId, job);}diff --git a/packages/core/package.json b/packages/core/package.jsonindex 5eba5c77e..82ae8262e 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -41,6 +41,7 @@"@trpc/server": "^11.17.0","arctic": "^2.3.0","bullmq": "6.3.3",+ "csv-parse": "^6.1.0","elysia": "1.4.30","ipaddr.js": "^2.5.0","lru-cache": "^11.2.4",@@ -49,17 +50,21 @@"pino-pretty": "catalog:","prom-client": "^15.1.3","qrcode": "^1.5.4",+ "ramda": "^0.29.1","sqlstring": "^2.3.3","superjson": "^1.13.3","ua-parser-js": "^2.0.6","undici": "^8.10.0",+ "unzipper": "^0.12.3","zod": "catalog:"},"devDependencies": {"@clickhouse/client": "^1.18.5","@openpanel/tsconfig": "workspace:*","@types/qrcode": "^1.5.6",+ "@types/ramda": "^0.31.1","@types/sqlstring": "^2.3.2",+ "@types/unzipper": "^0.10.10","bun-types": "1.4.0","tar": "^7.4.3","typescript": "catalog:"diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex dbf5153c4..0d7a96fcd 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -209,6 +209,34 @@ export {selectGscSite,syncGscData,} from './modules/gsc/gsc.service';+// Dissolved from @openpanel/db's services/import.service.ts ++// apps/worker's job file + apps/api's /import controller (M5-004) —+// apps/worker's import job file and apps/api's import controller call these+// directly, the same way V1 reaches every other dissolved service here.+// packages/db/src/services/import.service.ts is deleted outright: nothing+// else reached it through @openpanel/db's barrel (same as cohort, M5-003).+export type {+ ImportJobProgress,+ ImportService,+ ImportStageResult,+ ImportSteps,+ InsertRawEventsResult,+ UpdateImportStatusOptions,+} from './modules/import/import.service';+export {+ backfillSessionsToProduction,+ cleanupSessionStartEndEvents,+ cleanupStagingData,+ createSessionsStartEndEvents,+ generateGapBasedSessionIds,+ getImportDateBounds,+ insertImportBatch,+ insertProfilesBatch,+ insertRawEventsBatch,+ moveImportsToProduction,+ runImportJob,+ updateImportStatus,+} from './modules/import/import.service';// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service// (M5-001) — apps/worker's insight job files and packages/trpc's insight// router call these directly, the same way V1 reaches every other dissolveddiff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.tsindex d853841a0..4b154cbf2 100644--- a/packages/core/src/jobs.registry.test.ts+++ b/packages/core/src/jobs.registry.test.ts@@ -135,3 +135,15 @@ test('the cron queue carries the cohort module cron fragment', () => {name: 'cohortRefresh',});});++// M5-004: the import module's own job. No cron fragment — imports are always+// user-triggered (import.rpc.ts's create/retry), unlike gsc/cohort/insight.+// legacyCompat.import already discriminates on this exact name+// (jobs/compat.ts), so this pins the two in agreement.+test('the import queue carries the import module job', () => {+ expect(Object.keys(queues.import.jobs)).toEqual(['import']);+ expect(queues.import.jobs.import).toMatchObject({+ queue: 'import',+ name: 'import',+ });+});diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.tsindex 5444a7d2d..f5f8905d0 100644--- a/packages/core/src/jobs.registry.ts+++ b/packages/core/src/jobs.registry.ts@@ -16,6 +16,7 @@ import { defineQueue } from './jobs/define';import type { ProducerHandle } from './jobs/producers';import { cohortCronJobs, cohortQueueJobs } from './modules/cohort/cohort.jobs';import { gscCronJobs, gscQueueJobs } from './modules/gsc/gsc.jobs';+import { importQueueJobs } from './modules/import/import.jobs';import {insightCronJobs,insightQueueJobs,@@ -71,7 +72,9 @@ const importQueue = defineQueue('import', {defaults: { removeOnComplete: 10, removeOnFail: 50 },worker: { concurrency: CONCURRENCY.import },compat: legacyCompat.import,- jobs: {},+ jobs: {+ ...importQueueJobs,+ },});const insights = defineQueue('insights', {diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.tsindex a31f7d4ee..3db8fc483 100644--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts@@ -33,6 +33,7 @@ function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {auth: {} as Services['auth'],insight: {} as Services['insight'],gsc: {} as Services['gsc'],+ import: {} as Services['import'],cohort: cohort as CohortService,};return {diff --git a/packages/core/src/modules/cohort/cohort.service.test.ts b/packages/core/src/modules/cohort/cohort.service.test.tsindex 5f4ff3f53..d70b0d701 100644--- a/packages/core/src/modules/cohort/cohort.service.test.ts+++ b/packages/core/src/modules/cohort/cohort.service.test.ts@@ -60,10 +60,12 @@ const chQuery = mock(async () => [] as unknown[]);const chInsert = mock(async (_args: { table: string }) => undefined);const chCommand = mock(async (_args: { query: string }) => undefined);const getReplicatedTableName = mock((table: string) => table);-// `originalCh`/`TABLE_NAMES` are unused here but included because-// `mock.module` replaces this specifier process-wide (bun runs every test-// file in one shared module registry without `--isolate` — see AGENTS.md) —-// gsc.service.test.ts and insight.service.test.ts mock the same path, so+// `originalCh`/`TABLE_NAMES`/`formatClickhouseDate`/+// `convertClickhouseDateToJs` and the extra TABLE_NAMES keys are unused here+// but included because `mock.module` replaces this specifier process-wide+// (bun runs every test file in one shared module registry without+// `--isolate` — see AGENTS.md) — gsc.service.test.ts,+// insight.service.test.ts and import.service.test.ts mock the same path, so// every factory must be a superset of every consumer's needs, whichever one// ends up registered last.mock.module('@openpanel/db/src/clickhouse/client', () => ({@@ -74,7 +76,19 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({},chQuery,getReplicatedTableName,- TABLE_NAMES: { sessions: 'sessions' },+ TABLE_NAMES: {+ events: 'events',+ events_imports: 'events_imports',+ profiles: 'profiles',+ sessions: 'sessions',+ },+ formatClickhouseDate: (date: Date | string) =>+ new Date(date)+ .toISOString()+ .replace('T', ' ')+ .replace(/(\.\d{3})?Z+$/, ''),+ convertClickhouseDateToJs: (date: string) =>+ new Date(`${date.replace(' ', 'T')}Z`),}));let subject: typeof import('./cohort.service');diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.tsindex ce356a350..8a6fefd86 100644--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts@@ -29,6 +29,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {auth: {} as Services['auth'],insight: {} as Services['insight'],cohort: {} as Services['cohort'],+ import: {} as Services['import'],gsc: gsc as GscService,};return {@@ -120,6 +121,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', asyncauth: {} as Services['auth'],insight: {} as Services['insight'],cohort: {} as Services['cohort'],+ import: {} as Services['import'],gsc: {listConnectionsForSync: async () => [{ projectId: 'p1' },diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.tsindex e535f7743..e1fbb0159 100644--- a/packages/core/src/modules/gsc/gsc.service.test.ts+++ b/packages/core/src/modules/gsc/gsc.service.test.ts@@ -95,10 +95,12 @@ const originalCh = {query: mock(async () => ({ json: async () => [] as unknown[] })),insert: mock(async () => undefined),};-// `ch`/`getReplicatedTableName` are unused here but included because-// `mock.module` replaces this specifier process-wide (bun runs every test-// file in one shared module registry without `--isolate` — see AGENTS.md) —-// insight.service.test.ts and cohort.service.test.ts mock the same path, so+// `ch`/`getReplicatedTableName`/`formatClickhouseDate`/+// `convertClickhouseDateToJs` and the extra TABLE_NAMES keys are unused here+// but included because `mock.module` replaces this specifier process-wide+// (bun runs every test file in one shared module registry without+// `--isolate` — see AGENTS.md) — insight.service.test.ts,+// cohort.service.test.ts and import.service.test.ts mock the same path, so// every factory must be a superset of every consumer's needs, whichever one// ends up registered last.mock.module('@openpanel/db/src/clickhouse/client', () => ({@@ -109,7 +111,19 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({originalCh,chQuery,getReplicatedTableName: mock((table: string) => table),- TABLE_NAMES: { sessions: 'sessions' },+ TABLE_NAMES: {+ events: 'events',+ events_imports: 'events_imports',+ profiles: 'profiles',+ sessions: 'sessions',+ },+ formatClickhouseDate: (date: Date | string) =>+ new Date(date)+ .toISOString()+ .replace('T', ' ')+ .replace(/(\.\d{3})?Z+$/, ''),+ convertClickhouseDateToJs: (date: string) =>+ new Date(`${date.replace(' ', 'T')}Z`),}));// Bypasses the Redis cache-aside entirely — `getGscCannibalization`'s owndiff --git a/packages/core/src/modules/import/import.constants.ts b/packages/core/src/modules/import/import.constants.tsnew file mode 100644index 000000000..5ee8b2755--- /dev/null+++ b/packages/core/src/modules/import/import.constants.ts@@ -0,0 +1,143 @@+// Moved from packages/validation/src/index.ts + packages/importer/src/+// providers/metadata.ts (M5-004, ADR-008's module map: import owns "C").+// packages/validation/src/import.validation.ts becomes a re-export shim of+// this file (same shape as packages/validation/src/cohort.validation.ts since+// M5-003), so apps/start and packages/db/src/types.ts keep resolving these+// symbols through @openpanel/validation's existing barrel unchanged.+//+// Isomorphic by the AGENTS.md rule: zod and nothing else.++import { z } from 'zod';++export type ImportProviderId = 'umami' | 'mixpanel' | 'amplitude';+export type ImportProviderType = 'file' | 'api';++export interface ImportProviderMeta {+ id: ImportProviderId;+ name: string;+ description: string;+ logo: string;+ backgroundColor: string;+ types: ImportProviderType[];+}++export const IMPORT_PROVIDERS: ImportProviderMeta[] = [+ {+ id: 'umami',+ name: 'Umami',+ description: 'Import your analytics data from Umami',+ logo: 'https://cdn.brandfetch.io/id_3VEohOm/w/180/h/180/theme/dark/logo.png?c=1dxbfHSJFAPEGdCLU4o5B',+ backgroundColor: '#fff',+ types: ['file'],+ },+ {+ id: 'mixpanel',+ name: 'Mixpanel',+ description: 'Import your analytics data from Mixpanel API',+ logo: 'https://cdn.brandfetch.io/idr_rhI2FS/theme/dark/idMJ8uODLv.svg?c=1dxbfHSJFAPEGdCLU4o5B',+ backgroundColor: '#fff',+ types: ['api'],+ },+ {+ id: 'amplitude',+ name: 'Amplitude',+ description: 'Import your analytics data from the Amplitude Export API',+ logo: 'https://cdn.brandfetch.io/idnYrZGER0/theme/dark/symbol.svg?c=1dxbfHSJFAPEGdCLU4o5B',+ backgroundColor: '#fff',+ types: ['api'],+ },+];++const zProjectMapper = z.object({+ from: z.string().min(1),+ to: z.string().min(1),+});++/**+ * `z.string().url()` alone accepts `file:`, `gopher:` and friends. Restricting+ * the scheme here gives the user an immediate form error instead of a job that+ * fails later. It is NOT the SSRF control - the value is stored and fetched+ * afterwards, so the destination is re-validated at fetch time by+ * `safeFetchStream`.+ */+export const zHttpUrl = z+ .string()+ .url()+ .refine(+ (value) => {+ try {+ const { protocol } = new URL(value);+ return protocol === 'http:' || protocol === 'https:';+ } catch {+ return false;+ }+ },+ { message: 'Only http and https URLs are allowed' }+ );++const createFileImportConfig = <T extends string>(provider: T) =>+ z.object({+ provider: z.literal(provider),+ type: z.literal('file'),+ fileUrl: zHttpUrl,+ });++// Import configs+export const zUmamiImportConfig = createFileImportConfig('umami').extend({+ projectMapper: z.array(zProjectMapper),+});++export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;++export const zPlausibleImportConfig = createFileImportConfig('plausible');+export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;++export const zAmplitudeDataResidency = z.enum(['us', 'eu']);+export type IAmplitudeDataResidency = z.infer<typeof zAmplitudeDataResidency>;++export const zAmplitudeImportConfig = z.object({+ provider: z.literal('amplitude'),+ type: z.literal('api'),+ apiKey: z.string().min(1),+ secretKey: z.string().min(1),+ from: z.string().min(1),+ to: z.string().min(1),+ mapScreenViewProperty: z.string().optional(),+ dataResidency: zAmplitudeDataResidency.optional(),+});+export type IAmplitudeImportConfig = z.infer<typeof zAmplitudeImportConfig>;++export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);+export type IMixpanelDataResidency = z.infer<typeof zMixpanelDataResidency>;++export const zMixpanelImportConfig = z.object({+ provider: z.literal('mixpanel'),+ type: z.literal('api'),+ serviceAccount: z.string().min(1),+ serviceSecret: z.string().min(1),+ projectId: z.string().min(1),+ from: z.string().min(1),+ to: z.string().min(1),+ mapScreenViewProperty: z.string().optional(),+ dataResidency: zMixpanelDataResidency.optional(),+});+export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;++export type IImportConfig =+ | IUmamiImportConfig+ | IPlausibleImportConfig+ | IMixpanelImportConfig+ | IAmplitudeImportConfig;++export const zCreateImport = z.object({+ projectId: z.string().min(1),+ provider: z.enum(['umami', 'plausible', 'mixpanel', 'amplitude']),+ config: z.union([+ zUmamiImportConfig,+ zPlausibleImportConfig,+ zMixpanelImportConfig,+ zAmplitudeImportConfig,+ ]),+});++export type ICreateImport = z.infer<typeof zCreateImport>;diff --git a/packages/core/src/modules/import/import.jobs.test.ts b/packages/core/src/modules/import/import.jobs.test.tsnew file mode 100644index 000000000..209c4d1a2--- /dev/null+++ b/packages/core/src/modules/import/import.jobs.test.ts@@ -0,0 +1,79 @@+// Job wiring — no ClickHouse/Postgres touched. The service methods+// themselves are exercised in import.service.test.ts.++import { expect, test } from 'bun:test';+import type { JobCtx } from '../../context';+import { createRecordingProducers } from '../../jobs/testing';+import { queues } from '../../jobs.registry';+import type { Logger } from '../../logger';+import type { Services } from '../../services';+import { importQueueJobs } from './import.jobs';+import type { ImportService } from './import.service';++function stubLogger(): Logger {+ const noop = () => undefined;+ const logger: Logger = {+ fatal: noop,+ error: noop,+ warn: noop,+ info: noop,+ debug: noop,+ trace: noop,+ child: () => logger,+ };+ return logger;+}++function stubJobCtx(importService: Partial<ImportService>): JobCtx {+ const services: Services = {+ auth: {} as Services['auth'],+ insight: {} as Services['insight'],+ cohort: {} as Services['cohort'],+ gsc: {} as Services['gsc'],+ import: importService as ImportService,+ };+ return {+ db: {},+ ch: {},+ redis: {},+ clients: {},+ buffers: {},+ logger: stubLogger(),+ queues: createRecordingProducers(queues).queues,+ services,+ requestId: 'req_1',+ job: { id: 'job_1', attempt: 0, queue: 'import', name: 'import' },+ };+}++test('the import queue declares exactly the import job', () => {+ expect(Object.keys(importQueueJobs)).toEqual(['import']);+ expect(queues.import.jobs.import).toMatchObject({+ queue: 'import',+ name: 'import',+ });+});++test('import validates its payload', () => {+ expect(() => importQueueJobs.import.payload.parse({})).toThrow();+ expect(importQueueJobs.import.payload.parse({ importId: 'imp_1' })).toEqual({+ importId: 'imp_1',+ });+});++test('import delegates to ctx.services.import.run', async () => {+ const calls: unknown[] = [];+ const ctx = stubJobCtx({+ run: async (importId) => {+ calls.push(importId);+ return { success: true };+ },+ });++ await importQueueJobs.import.handler({+ payload: { importId: 'imp_1' },+ ctx,+ });++ expect(calls).toEqual(['imp_1']);+});diff --git a/packages/core/src/modules/import/import.jobs.ts b/packages/core/src/modules/import/import.jobs.tsnew file mode 100644index 000000000..7e33d34dc--- /dev/null+++ b/packages/core/src/modules/import/import.jobs.ts@@ -0,0 +1,27 @@+// Ported from apps/worker/src/jobs/import.ts (M5-004).+//+// `import` is this module's own queue AND its only job — the registry key,+// the Redis name and the BullMQ job name are all `import`+// (legacyCompat.import already discriminates on `{type: 'import', payload}`,+// jobs/compat.ts). Unlike gsc/cohort/insight, this queue has no cron+// fragment: imports are always user-triggered (import.rpc.ts's create/retry).+//+// Core's own worker has no live BullMQ job to hand `updateImportStatus` for+// progress reporting (see import.service.ts's `ImportJobProgress` header),+// so this handler runs with the no-op default — inert until ADR-007's P9+// worker cutover, same as gsc/cohort/insight's job handlers today.++import { z } from 'zod';+import { defineJob } from '../../jobs/define';++const importJobPayload = z.object({ importId: z.string() });++/** The `import` queue's own job. */+export const importQueueJobs = {+ import: defineJob({+ payload: importJobPayload,+ handler: async ({ payload, ctx }) => {+ await ctx.services.import.run(payload.importId);+ },+ }),+};diff --git a/packages/core/src/modules/import/import.routes.ts b/packages/core/src/modules/import/import.routes.tsnew file mode 100644index 000000000..e7ebf7fa5--- /dev/null+++ b/packages/core/src/modules/import/import.routes.ts@@ -0,0 +1,48 @@+// Ported from apps/api/src/routes/import.router.ts ++// apps/api/src/controllers/import.controller.ts's business logic (M5-004).+// V1's Fastify controller stays the LIVE route (DELEGATE PATTERN) and+// delegates its ClickHouse insert to import.service.ts's+// `insertRawEventsBatch` — the same function this route calls.+//+// NAMED GAP, same as gsc.routes.ts: this route is not yet reachable.+// `authenticateClient` (http/client-auth.ts) is a P8 stub that always+// returns null, so `clientAuth` 401s every request until it is filled in;+// main.ts also does not mount `publicApiRoutes` until a real `AppDeps`+// exists. `allow: ['read', 'root']` mirrors V1's rule (utils/auth.ts's+// `validateImportRequest`: a `write`-type client may not import).+//+// V1 has no request-body schema for this route (ADR-003: "/profile,+// /import, /event and /tools have no request schemas today, and adding them+// would be a behaviour change"), so the body is typed but not shape-validated.++import type { IClickhouseEvent } from '@openpanel/db';+import { z } from 'zod';+import { defineRoutes } from '../../http/define';+import { insertRawEventsBatch } from './import.service';++export const importRoutes = defineRoutes((app) =>+ app.post(+ '/import/events',+ async ({ body, client, ctx, status }) => {+ try {+ const { writtenRows } = await insertRawEventsBatch(+ client.projectId,+ body as IClickhouseEvent[]+ );+ ctx.logger.info({ writtenRows }, 'events imported');+ return 'OK';+ } catch (error) {+ ctx.logger.error({ err: error }, 'Failed to import events');+ return status(500, 'Error');+ }+ },+ {+ clientAuth: { allow: ['read', 'root'] },+ body: z.array(z.unknown()),+ detail: {+ tags: ['Import'],+ description: 'Bulk import historical events.',+ },+ }+ )+);diff --git a/packages/core/src/modules/import/import.rpc.test.ts b/packages/core/src/modules/import/import.rpc.test.tsnew file mode 100644index 000000000..4d992438d--- /dev/null+++ b/packages/core/src/modules/import/import.rpc.test.ts@@ -0,0 +1,75 @@+// Only the "is anyone logged in" boundary is exercised here — no database.+// The access-check + business logic ride on @openpanel/db (lazy-loaded, see+// import.service.ts's header) and ctx.services.import; wiring this router+// end-to-end against a real Postgres is P6's (protectedProcedure) job, not+// this one's — see import.rpc.ts's header.++import { expect, test } from 'bun:test';+import { stubHttpCtx } from '../../../test/rpc-fixtures';+import { makeTrpcContext } from '../../rpc/base';+import type { CookieOptions } from '../../shared/cookie';+import { importRouter } from './import.rpc';++const COOKIE_OPTIONS: CookieOptions = {+ domain: '.openpanel.dev',+ secure: true,+ sameSite: 'lax',+ httpOnly: true,+ path: '/',+};++// EMPTY_SESSION's shape (packages/db/src/services/auth-session.service.ts) —+// `TrpcContext.session` is never literally `null`, only its `userId` is.+const EMPTY_SESSION = { session: null, user: null, userId: null };++async function anonCaller() {+ const { ctx } = stubHttpCtx({}, EMPTY_SESSION);+ const trpcCtx = await makeTrpcContext(ctx, new Headers(), {+ cookieOptions: COOKIE_OPTIONS,+ });+ return importRouter.createCaller(trpcCtx);+}++test('list rejects an unauthenticated caller before touching a project', async () => {+ const caller = await anonCaller();+ await expect(caller.list({ projectId: 'proj_1' })).rejects.toMatchObject({+ code: 'UNAUTHORIZED',+ });+});++test('get rejects an unauthenticated caller before reading an import', async () => {+ const caller = await anonCaller();+ await expect(caller.get({ id: 'imp_1' })).rejects.toMatchObject({+ code: 'UNAUTHORIZED',+ });+});++test('create rejects an unauthenticated caller before writing an import', async () => {+ const caller = await anonCaller();+ await expect(+ caller.create({+ projectId: 'proj_1',+ provider: 'umami',+ config: {+ provider: 'umami',+ type: 'file',+ fileUrl: 'https://example.com/export.csv',+ projectMapper: [],+ },+ })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});++test('delete rejects an unauthenticated caller before removing an import', async () => {+ const caller = await anonCaller();+ await expect(caller.delete({ id: 'imp_1' })).rejects.toMatchObject({+ code: 'UNAUTHORIZED',+ });+});++test('retry rejects an unauthenticated caller before re-enqueuing', async () => {+ const caller = await anonCaller();+ await expect(caller.retry({ id: 'imp_1' })).rejects.toMatchObject({+ code: 'UNAUTHORIZED',+ });+});diff --git a/packages/core/src/modules/import/import.rpc.ts b/packages/core/src/modules/import/import.rpc.tsnew file mode 100644index 000000000..80fab08a6--- /dev/null+++ b/packages/core/src/modules/import/import.rpc.ts@@ -0,0 +1,177 @@+// Ported from packages/trpc/src/routers/import.ts (M5-004).+//+// V1's `protectedProcedure` — the logger/session-scope/rate-limit middleware+// stack — lands in core with auth (rpc/base.ts: "those need the resolved+// Session shape and the access rules, so they land with auth (P6)"). Until+// then this router does its own minimal "is anyone logged in" check inline,+// exactly like V1's `enforceUserIsAuthed` (same as cohort.rpc.ts / gsc.rpc.ts).+//+// V1's router has no compute-heavy or ClickHouse-touching logic to delegate+// (it is Prisma CRUD + a BullMQ enqueue, unlike gsc/cohort), so+// packages/trpc/src/routers/import.ts stays completely unmodified — there is+// nothing here for it to delegate to. This router exists to satisfy the+// module map ("R") and to serve core's own (not yet live) appRouter.+//+// The per-project access ladder itself IS shared: `./src/access.ts` binds+// core's shared/access.ts ladder to @openpanel/db's real lookups, the same+// way packages/trpc/src/access.ts does for V1.++import { z } from 'zod';+import { createTRPCRouter, procedure } from '../../rpc/base';+import {+ TRPCAccessError,+ TRPCBadRequestError,+ TRPCNotFoundError,+} from '../../rpc/errors';+import { zCreateImport } from './import.constants';++function loadAccessChecks() {+ return import('./src/access');+}++function loadDb() {+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);+}++function requireLogin(userId: string | null | undefined): string {+ if (!userId) {+ throw new TRPCAccessError('Not authenticated');+ }+ return userId;+}++export const importRouter = createTRPCRouter({+ list: procedure+ .input(z.object({ projectId: z.string() }))+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ const { requireProjectAccess } = await loadAccessChecks();+ await requireProjectAccess({+ userId,+ projectId: input.projectId,+ level: 'read',+ });++ const db = await loadDb();+ return db.import.findMany({+ where: { projectId: input.projectId },+ orderBy: { createdAt: 'desc' },+ });+ }),++ get: procedure+ .input(z.object({ id: z.string() }))+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ const db = await loadDb();+ const importRecord = await db.import.findUniqueOrThrow({+ where: { id: input.id },+ include: { project: true },+ });++ const { requireProjectAccess } = await loadAccessChecks();+ await requireProjectAccess({+ userId,+ projectId: importRecord.projectId,+ level: 'read',+ });++ return importRecord;+ }),++ create: procedure.input(zCreateImport).mutation(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ const { requireProjectAccess } = await loadAccessChecks();+ await requireProjectAccess({+ userId,+ projectId: input.projectId,+ level: 'write',+ });++ const db = await loadDb();+ const organization = await db.organization.findFirst({+ where: { projects: { some: { id: input.projectId } } },+ });++ if (!organization) {+ throw new TRPCNotFoundError(+ 'Could not start import, organization not found'+ );+ }++ if (!organization.isActive) {+ throw new TRPCBadRequestError(+ 'You cannot start an import without an active subscription!'+ );+ }++ const importRecord = await db.import.create({+ data: {+ projectId: input.projectId,+ config: input.config,+ status: 'pending',+ },+ });++ const jobId = await ctx.services.import.enqueue(importRecord.id);++ await db.import.update({+ where: { id: importRecord.id },+ data: { jobId },+ });++ return { ...importRecord, jobId };+ }),++ delete: procedure+ .input(z.object({ id: z.string() }))+ .mutation(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ const db = await loadDb();+ const importRecord = await db.import.findUniqueOrThrow({+ where: { id: input.id },+ });++ const { requireProjectAccess } = await loadAccessChecks();+ await requireProjectAccess({+ userId,+ projectId: importRecord.projectId,+ level: 'write',+ });++ if (importRecord.jobId) {+ await ctx.queues.import.import.remove(importRecord.jobId);+ }++ return db.import.delete({ where: { id: input.id } });+ }),++ retry: procedure+ .input(z.object({ id: z.string() }))+ .mutation(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ const db = await loadDb();+ const importRecord = await db.import.findUniqueOrThrow({+ where: { id: input.id },+ });++ const { requireProjectAccess } = await loadAccessChecks();+ await requireProjectAccess({+ userId,+ projectId: importRecord.projectId,+ level: 'write',+ });++ // Only allow retry for failed imports+ if (importRecord.status !== 'failed') {+ throw new Error('Only failed imports can be retried');+ }++ const jobId = await ctx.services.import.enqueue(importRecord.id);++ return db.import.update({+ where: { id: importRecord.id },+ data: { jobId, status: 'pending', errorMessage: null },+ });+ }),+});diff --git a/packages/core/src/modules/import/import.service.test.ts b/packages/core/src/modules/import/import.service.test.tsnew file mode 100644index 000000000..546376c0d--- /dev/null+++ b/packages/core/src/modules/import/import.service.test.ts@@ -0,0 +1,181 @@+// import.service.ts's db/ch access is lazy (`await import(...)` inside each+// function — see the file's header), which is exactly what makes+// `mock.module` work here with no import-time side effects to race: every+// mock below is registered before the subject's first call, not before its+// (side-effect-free) import.++import { beforeAll, expect, mock, test } from 'bun:test';+import { createRecordingProducers } from '../../jobs/testing';+import { queues } from '../../jobs.registry';+import type { Logger } from '../../logger';++function stubLogger(): Logger {+ const noop = () => undefined;+ const logger: Logger = {+ fatal: noop,+ error: noop,+ warn: noop,+ info: noop,+ debug: noop,+ trace: noop,+ child: () => logger,+ };+ return logger;+}++const chInsert = mock(async (_opts: { values: unknown[] }) => ({+ summary: { written_rows: '2' },+}));+// `ch.command`/`originalCh`/`chQuery`/`getReplicatedTableName`/+// `convertClickhouseDateToJs` and the extra TABLE_NAMES keys are unused here+// but included because `mock.module` replaces this specifier process-wide+// (bun runs every test file in one shared module registry without+// `--isolate` — see AGENTS.md) — gsc.service.test.ts, cohort.service.test.ts+// and insight.service.test.ts mock the same path, so every factory must be a+// superset of every consumer's needs, whichever one ends up registered last.+mock.module('@openpanel/db/src/clickhouse/client', () => ({+ ch: {+ insert: chInsert,+ command: mock(async () => undefined),+ },+ originalCh: {+ query: mock(async () => ({ json: async () => [] as unknown[] })),+ insert: mock(async () => undefined),+ },+ chQuery: mock(async () => [] as unknown[]),+ getReplicatedTableName: mock((table: string) => table),+ TABLE_NAMES: {+ events: 'events',+ events_imports: 'events_imports',+ profiles: 'profiles',+ sessions: 'sessions',+ },+ formatClickhouseDate: (date: Date | string) =>+ new Date(date)+ .toISOString()+ .replace('T', ' ')+ .replace(/(\.\d{3})?Z+$/, ''),+ convertClickhouseDateToJs: (date: string) =>+ new Date(`${date.replace(' ', 'T')}Z`),+}));++const importUpdate = mock(+ async (_opts: { where: { id: string }; data: unknown }) => undefined+);+mock.module('@openpanel/db/src/prisma-client', () => ({+ db: { import: { update: importUpdate } },+}));++let insertRawEventsBatch: typeof import('./import.service').insertRawEventsBatch;+let updateImportStatus: typeof import('./import.service').updateImportStatus;+let createImportService: typeof import('./import.service').createImportService;++beforeAll(async () => {+ ({ insertRawEventsBatch, updateImportStatus, createImportService } =+ await import('./import.service'));+});++test('insertRawEventsBatch stamps project_id/imported_at and flattens properties', async () => {+ chInsert.mockClear();++ const result = await insertRawEventsBatch(+ 'proj_1',+ [+ {+ id: 'evt_1',+ name: 'screen_view',+ project_id: 'wrong_project',+ properties: { nested: { a: 1 } },+ created_at: '2025-01-01 00:00:00',+ } as never,+ ],+ stubLogger()+ );++ expect(result).toEqual({ writtenRows: 2 });+ expect(chInsert).toHaveBeenCalledTimes(1);++ const [{ values }] = chInsert.mock.calls[0] as [{ values: any[] }];+ expect(values).toHaveLength(1);+ expect(values[0]).toMatchObject({+ id: 'evt_1',+ project_id: 'proj_1',+ properties: { 'nested.a': '1' },+ });+ expect(values[0].imported_at).toBeTruthy();+});++test('updateImportStatus writes the loading step through both the progress reporter and Postgres', async () => {+ importUpdate.mockClear();+ const progressed: unknown[] = [];++ await updateImportStatus(+ stubLogger(),+ { updateProgress: (p) => progressed.push(p) },+ 'imp_1',+ {+ step: 'loading',+ batch: '2025-01-01',+ totalEvents: 10,+ processedEvents: 5,+ }+ );++ expect(progressed).toEqual([+ {+ status: 'processing',+ currentStep: 'loading',+ currentBatch: '2025-01-01',+ statusMessage: 'Importing events from 2025-01-01',+ totalEvents: 10,+ processedEvents: 5,+ },+ ]);+ expect(importUpdate).toHaveBeenCalledTimes(1);+ const [{ where, data }] = importUpdate.mock.calls[0] as [+ { where: { id: string }; data: unknown },+ ];+ expect(where).toEqual({ id: 'imp_1' });+ expect(data).toEqual(progressed[0]);+});++test('updateImportStatus marks a failed step with its error message', async () => {+ importUpdate.mockClear();+ const progressed: unknown[] = [];++ await updateImportStatus(+ stubLogger(),+ { updateProgress: (p) => progressed.push(p) },+ 'imp_1',+ { step: 'failed', errorMessage: 'boom' }+ );++ expect(progressed).toEqual([+ { status: 'failed', statusMessage: 'Import failed', errorMessage: 'boom' },+ ]);+});++test('ImportService.enqueue adds the import job onto the import queue', async () => {+ const producers = createRecordingProducers(queues);+ const service = createImportService({+ db: {},+ ch: {},+ redis: {},+ clients: {},+ buffers: {},+ logger: stubLogger(),+ queues: producers.queues,+ });++ const jobId = await service.enqueue('imp_1');++ expect(typeof jobId).toBe('string');+ expect(producers.recorded).toEqual([+ {+ queue: 'import',+ job: 'import',+ payload: { importId: 'imp_1' },+ meta: {},+ },+ ]);+});diff --git a/packages/db/src/services/import.service.ts b/packages/core/src/modules/import/import.service.tssimilarity index 54%rename from packages/db/src/services/import.service.tsrename to packages/core/src/modules/import/import.service.tsindex ec1d83546..0a6b22836 100644--- a/packages/db/src/services/import.service.ts+++ b/packages/core/src/modules/import/import.service.ts@@ -1,15 +1,66 @@+// Moved from packages/db/src/services/import.service.ts (the ClickHouse+// staging/session pipeline) + apps/worker/src/jobs/import.ts (the job body+// and provider dispatch) + apps/api/src/controllers/import.controller.ts+// (the /import/events bulk-insert path) — M5-004, ADR-008's module map:+// import owns "S". packages/db/src/services/import.service.ts is deleted+// outright: nothing outside the worker job file it moves with reached it+// through @openpanel/db's barrel (same as cohort.service.ts, M5-003).+//+// db/ch access is LAZY (`load*` below), not a static top-level import — see+// gsc.service.ts's header for the full reasoning (jobs.registry.ts and+// services.ts pull this module into the eager barrel chain nearly every core+// test file reaches). The provider classes are behind a dynamic `import()`+// for the same reason: they eagerly import `formatClickhouseDate` from+// @openpanel/db's clickhouse client, which constructs a real pino transport+// at import time.+//+// ClickHouse queries here still go through raw `ch`/`chQuery` calls, not the+// `sql` tag: ADR-013 converts the analytics read path one query per P7 task,+// and this module's queries haven't been converted yet.+import { createHash } from 'node:crypto';-import {- ch,- convertClickhouseDateToJs,- formatClickhouseDate,- getReplicatedTableName,- TABLE_NAMES,-} from '../clickhouse/client';-import type { ILogger } from '../logger';-import { db, type Prisma } from '../prisma-client';-import type { IClickhouseEvent } from './event.service';-import type { IClickhouseProfile } from './profile.service';+import { toDots } from '@openpanel/common';+import type { IClickhouseEvent, IClickhouseProfile } from '@openpanel/db';+import type { Prisma } from '@openpanel/db/src/prisma-client';+import { createLogger, type ILogger } from '../../clients/logger';+import type { Logger } from '../../logger';+import type { ServiceDeps } from '../../services';+import type { IImportConfig } from './import.constants';++const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10);+const SESSION_BATCH_SIZE = 5000;+const PROFILE_BATCH_SIZE = 5000;+// Profiles derived inline from events (Amplitude, which has no profile export+// API) are deduped in a bounded map so memory stays flat regardless of event+// volume; the profiles table's ReplacingMergeTree(last_seen_at) collapses any+// duplicate rows that span flush boundaries to the latest activity per id.+const PROFILE_MAP_CAP = 50_000;+const RESUMABLE_STEPS = ['creating_sessions', 'moving', 'backfilling_sessions'];++let _logger: ILogger | undefined;+function getLogger(): ILogger {+ _logger ??= createLogger({ name: 'core:import' });+ return _logger;+}++function loadDb() {+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);+}++function loadChClient() {+ return import('@openpanel/db/src/clickhouse/client');+}++function yieldToEventLoop(): Promise<void> {+ return new Promise((resolve) => {+ setTimeout(resolve, 100);+ });+}++// ---------------------------------------------------------------------------+// ClickHouse staging pipeline — moved from packages/db/src/services/+// import.service.ts.+// ---------------------------------------------------------------------------export interface ImportStageResult {importId: string;@@ -17,21 +68,94 @@ export interface ImportStageResult {insertedEvents: number;}-const SESSION_GAP_MS = 30 * 60 * 1000; // 30 minutes+/**+ * Insert a batch of events into the imports staging table+ */+export async function insertImportBatch(+ events: IClickhouseEvent[],+ importId: string+): Promise<ImportStageResult> {+ if (events.length === 0) {+ return { importId, totalEvents: 0, insertedEvents: 0 };+ }++ const { ch, TABLE_NAMES, formatClickhouseDate } = await loadChClient();+ const now = formatClickhouseDate(new Date());+ const rows = events.map((event) => ({+ ...event,+ import_id: importId,+ import_status: 'pending',+ imported_at: event.imported_at || now,+ imported_at_meta: now,+ }));++ await ch.insert({+ table: TABLE_NAMES.events_imports,+ values: rows,+ format: 'JSONEachRow',+ });++ return {+ importId,+ totalEvents: events.length,+ insertedEvents: events.length,+ };+}++/**+ * Insert a batch of profiles into the production profiles table.+ * Used by Mixpanel (and other providers) to import user profiles during an import job.+ */+export async function insertProfilesBatch(+ profiles: IClickhouseProfile[],+ projectId: string+): Promise<{ inserted: number }> {+ if (profiles.length === 0) {+ return { inserted: 0 };+ }++ const { ch, TABLE_NAMES } = await loadChClient();+ const normalized = profiles.map((p) => ({+ id: p.id,+ project_id: projectId,+ first_name: p.first_name ?? '',+ last_name: p.last_name ?? '',+ email: p.email ?? '',+ avatar: p.avatar ?? '',+ is_external: p.is_external ?? true,+ properties: Object.fromEntries(+ Object.entries(p.properties || {}).filter(+ (kv): kv is [string, string] => kv[1] != null && kv[1] !== ''+ )+ ) as Record<string, string>,+ created_at: p.created_at,+ last_seen_at: p.last_seen_at ?? p.created_at,+ }));++ await ch.insert({+ table: TABLE_NAMES.profiles,+ values: normalized,+ format: 'JSONEachRow',+ });++ return { inserted: normalized.length };+}/*** Generate gap-based session IDs for events that have none.* Streams events from staging (sorted by device_id, created_at), assigns a new* session when gap > 30 min, re-inserts with session_id, then deletes old rows.*/+const SESSION_GAP_MS = 30 * 60 * 1000; // 30 minutes+export async function generateGapBasedSessionIds(importId: string): Promise<void> {+ const { ch, TABLE_NAMES, getReplicatedTableName } = await loadChClient();let currentDeviceId = '';let currentSessionId = '';let currentLastTime = 0;let currentCounter = -1;- const BATCH_SIZE = 5000;const batch: IClickhouseEvent[] = [];const result = await ch.query({@@ -75,7 +199,7 @@ export async function generateGapBasedSessionIds(event.session_id = currentSessionId;batch.push(event);- if (batch.length >= BATCH_SIZE) {+ if (batch.length >= SESSION_BATCH_SIZE) {await insertImportBatch(batch, importId);batch.length = 0;}@@ -102,82 +226,12 @@ export async function generateGapBasedSessionIds(});}-/**- * Insert a batch of events into the imports staging table- */-export async function insertImportBatch(- events: IClickhouseEvent[],- importId: string-): Promise<ImportStageResult> {- if (events.length === 0) {- return { importId, totalEvents: 0, insertedEvents: 0 };- }-- const now = formatClickhouseDate(new Date());- const rows = events.map((event) => ({- ...event,- import_id: importId,- import_status: 'pending',- imported_at: event.imported_at || now,- imported_at_meta: now,- }));-- await ch.insert({- table: TABLE_NAMES.events_imports,- values: rows,- format: 'JSONEachRow',- });-- return {- importId,- totalEvents: events.length,- insertedEvents: events.length,- };-}--/**- * Insert a batch of profiles into the production profiles table.- * Used by Mixpanel (and other providers) to import user profiles during an import job.- */-export async function insertProfilesBatch(- profiles: IClickhouseProfile[],- projectId: string-): Promise<{ inserted: number }> {- if (profiles.length === 0) {- return { inserted: 0 };- }-- const normalized = profiles.map((p) => ({- id: p.id,- project_id: projectId,- first_name: p.first_name ?? '',- last_name: p.last_name ?? '',- email: p.email ?? '',- avatar: p.avatar ?? '',- is_external: p.is_external ?? true,- properties: Object.fromEntries(- Object.entries(p.properties || {}).filter(- (kv): kv is [string, string] => kv[1] != null && kv[1] !== ''- )- ) as Record<string, string>,- created_at: p.created_at,- last_seen_at: p.last_seen_at ?? p.created_at,- }));-- await ch.insert({- table: TABLE_NAMES.profiles,- values: normalized,- format: 'JSONEachRow',- });-- return { inserted: normalized.length };-}-/*** Delete all staging data for an import. Used to get a clean slate on retry* when the failure happened before moving data to production.*/export async function cleanupStagingData(importId: string): Promise<void> {+ const { ch, TABLE_NAMES, getReplicatedTableName } = await loadChClient();const mutationTableName = getReplicatedTableName(TABLE_NAMES.events_imports);await ch.command({query: `ALTER TABLE ${mutationTableName} DELETE WHERE import_id = {importId:String}`,@@ -194,6 +248,7 @@ export async function cleanupStagingData(importId: string): Promise<void> {export async function cleanupSessionStartEndEvents(importId: string): Promise<void> {+ const { ch, TABLE_NAMES, getReplicatedTableName } = await loadChClient();const mutationTableName = getReplicatedTableName(TABLE_NAMES.events_imports);await ch.command({query: `ALTER TABLE ${mutationTableName} DELETE WHERE import_id = {importId:String} AND name IN ('session_start', 'session_end')`,@@ -218,7 +273,8 @@ export async function cleanupSessionStartEndEvents(export async function createSessionsStartEndEvents(importId: string): Promise<void> {- const SESSION_BATCH_SIZE = 5000;+ const { ch, TABLE_NAMES, convertClickhouseDateToJs, formatClickhouseDate } =+ await loadChClient();let lastSessionId = '';const baseWhere = [@@ -441,6 +497,7 @@ export async function moveImportsToProduction(importId: string,from: string): Promise<void> {+ const { ch, TABLE_NAMES } = await loadChClient();let whereClause = 'import_id = {importId:String}';if (from) {@@ -485,7 +542,7 @@ export async function moveImportsToProduction(export async function backfillSessionsToProduction(importId: string): Promise<void> {- const SESSION_BATCH_SIZE = 5000;+ const { ch, TABLE_NAMES } = await loadChClient();let lastSessionId = '';while (true) {@@ -520,7 +577,7 @@ export async function backfillSessionsToProduction(utm_medium, utm_source, utm_campaign, utm_content, utm_term,referrer, referrer_name, referrer_type)- SELECT+ SELECTany(e.session_id) as id,any(e.project_id) as project_id,if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,@@ -563,7 +620,7 @@ export async function backfillSessionsToProduction(argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name,argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_typeFROM ${TABLE_NAMES.events_imports} e- WHERE+ WHEREe.import_id = {importId:String}AND e.session_id > {lastSessionId:String}AND e.session_id <= {maxSessionId:String}@@ -594,6 +651,7 @@ export async function getImportDateBounds(importId: string,fromCreatedAt?: string): Promise<{ min: string | null; max: string | null }> {+ const { ch, TABLE_NAMES } = await loadChClient();const res = await ch.query({query: `SELECT min(created_at) AS min, max(created_at) AS max@@ -617,6 +675,20 @@ export async function getImportDateBounds(: { min: null, max: null };}+/**+ * Reports progress on a running import. Wraps V1's `job.updateProgress` — a+ * BullMQ `Job` satisfies this structurally, so the V1 worker delegate passes+ * its real job straight through. Core's own job runner (import.jobs.ts) has+ * no BullMQ job object to hand over (`JobCtx.job` is the erased+ * `{id, attempt, queue, name}`, not the live BullMQ handle), so it falls back+ * to the no-op default.+ */+export interface ImportJobProgress {+ updateProgress(progress: Record<string, unknown>): unknown;+}++const NOOP_PROGRESS: ImportJobProgress = { updateProgress: () => undefined };+export type UpdateImportStatusOptions =| {step: 'loading';@@ -655,13 +727,12 @@ export type UpdateImportStatusOptions =export type ImportSteps = UpdateImportStatusOptions['step'];export async function updateImportStatus(- jobLogger: ILogger,- job: {- updateProgress: (progress: Record<string, any>) => void;- },+ jobLogger: Logger,+ progress: ImportJobProgress,importId: string,options: UpdateImportStatusOptions): Promise<void> {+ const db = await loadDb();const data: Prisma.ImportUpdateInput = {};switch (options.step) {case 'loading':@@ -721,10 +792,450 @@ export async function updateImportStatus(jobLogger.info({ data }, 'Import status update');- await job.updateProgress(data);+ await progress.updateProgress(data);await db.import.update({where: { id: importId },data,});}++// ---------------------------------------------------------------------------+// Provider dispatch + job body — moved from apps/worker/src/jobs/import.ts.+// ---------------------------------------------------------------------------++/**+ * Merge a freshly-derived profile into the bounded dedup map: keep the earliest+ * created_at (first seen), advance last_seen_at to the latest activity, and fill+ * identity fields/properties, preferring the newer row.+ */+function mergeProfileInto(+ map: Map<string, IClickhouseProfile>,+ incoming: IClickhouseProfile+): void {+ const existing = map.get(incoming.id);+ if (!existing) {+ map.set(incoming.id, incoming);+ return;+ }++ const createdAt =+ existing.created_at < incoming.created_at+ ? existing.created_at+ : incoming.created_at;+ const isIncomingNewer = incoming.last_seen_at >= existing.last_seen_at;+ const base = isIncomingNewer ? incoming : existing;+ const other = isIncomingNewer ? existing : incoming;++ map.set(incoming.id, {+ ...base,+ created_at: createdAt,+ first_name: base.first_name || other.first_name,+ last_name: base.last_name || other.last_name,+ email: base.email || other.email,+ avatar: base.avatar || other.avatar,+ properties: { ...other.properties, ...base.properties },+ });+}++/** Structural subset of a provider instance the job body actually drives. */+interface RunnableProvider {+ shouldGenerateSessionIds(): boolean;+ getTotalEventsCount(): Promise<number>;+ parseSource(): AsyncGenerator<unknown, void, unknown>;+ validate(rawEvent: unknown): boolean;+ transformEvent(rawEvent: unknown): IClickhouseEvent;+ transformEventToProfile?(rawEvent: unknown): IClickhouseProfile | null;+ streamProfiles?(): AsyncGenerator<unknown, void, unknown>;+ transformProfile?(rawProfile: unknown): IClickhouseProfile;+}++/** Dynamic import keeps @openpanel/db's clickhouse client off this file's eager path (see header). */+async function createImportProvider(+ projectId: string,+ config: IImportConfig,+ jobLogger: Logger+): Promise<RunnableProvider> {+ switch (config.provider) {+ case 'umami': {+ const { UmamiProvider } = await import('./src/providers/umami');+ return new UmamiProvider(projectId, config, jobLogger);+ }+ case 'mixpanel': {+ const { MixpanelProvider } = await import('./src/providers/mixpanel');+ return new MixpanelProvider(projectId, config, jobLogger);+ }+ case 'amplitude': {+ const { AmplitudeProvider } = await import('./src/providers/amplitude');+ return new AmplitudeProvider(projectId, config, jobLogger);+ }+ default:+ throw new Error(+ `Unknown provider: ${(config as { provider: string }).provider}`+ );+ }+}++/**+ * The `import` queue job body. Ported verbatim from apps/worker/src/jobs/+ * import.ts's `importJob`, with the BullMQ `Job` split into `importId` ++ * `progress` (see `ImportJobProgress`'s header) so the same body serves V1's+ * worker delegate and core's own (not yet live) `import` queue worker.+ */+export async function runImportJob(+ importId: string,+ progress: ImportJobProgress = NOOP_PROGRESS,+ logger: Logger = getLogger()+): Promise<{ success: true }> {+ const db = await loadDb();+ const record = await db.import.findUniqueOrThrow({+ where: { id: importId },+ include: { project: true },+ });++ const jobLogger = logger.child({ importId, config: record.config });+ jobLogger.info('Starting import job');++ const providerInstance = await createImportProvider(+ record.projectId,+ record.config,+ jobLogger+ );+ const shouldGenerateSessionIds = providerInstance.shouldGenerateSessionIds();++ try {+ const isRetry = record.currentStep !== null;+ const canResume =+ isRetry && RESUMABLE_STEPS.includes(record.currentStep as string);++ // -------------------------------------------------------+ // STAGING PHASE: clean slate on failure, run from scratch+ // -------------------------------------------------------+ if (!canResume) {+ if (isRetry) {+ jobLogger.info(+ 'Retry detected before resumable phase — cleaning staging data'+ );+ await cleanupStagingData(importId);+ }++ // Phase 1: Load events into staging+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'loading',+ });++ const totalEvents = await providerInstance+ .getTotalEventsCount()+ .catch(() => -1);+ let processedEvents = 0;+ const eventBatch: IClickhouseEvent[] = [];++ const canProfileFromEvents =+ typeof providerInstance.transformEventToProfile === 'function';+ const profileMap = new Map<string, IClickhouseProfile>();+ let processedProfiles = 0;++ const flushProfiles = async () => {+ if (profileMap.size === 0) {+ return;+ }+ const values = Array.from(profileMap.values());+ await insertProfilesBatch(values, record.projectId);+ processedProfiles += values.length;+ profileMap.clear();+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'loading_profiles',+ processedProfiles,+ });+ await yieldToEventLoop();+ };++ for await (const rawEvent of providerInstance.parseSource()) {+ if (!providerInstance.validate(rawEvent)) {+ jobLogger.warn({ rawEvent }, 'Skipping invalid event');+ continue;+ }++ const transformed = providerInstance.transformEvent(rawEvent);++ // Session IDs for providers that need them (e.g. Mixpanel) are generated+ // in generateGapBasedSessionIds after loading, using gap-based logic.+ eventBatch.push(transformed);++ if (canProfileFromEvents) {+ const profile = providerInstance.transformEventToProfile?.(rawEvent);+ if (profile) {+ mergeProfileInto(profileMap, profile);+ if (profileMap.size >= PROFILE_MAP_CAP) {+ await flushProfiles();+ }+ }+ }++ if (eventBatch.length >= BATCH_SIZE) {+ await insertImportBatch(eventBatch, importId);+ processedEvents += eventBatch.length;++ const batchDate = new Date(eventBatch[0]?.created_at || '')+ .toISOString()+ .split('T')[0];++ await updateImportStatus(jobLogger, progress, importId, {+ step: 'loading',+ batch: batchDate,+ totalEvents,+ processedEvents,+ });++ eventBatch.length = 0;+ await yieldToEventLoop();+ }+ }++ if (eventBatch.length > 0) {+ await insertImportBatch(eventBatch, importId);+ processedEvents += eventBatch.length;++ const batchDate = new Date(eventBatch[0]?.created_at || '')+ .toISOString()+ .split('T')[0];++ await updateImportStatus(jobLogger, progress, importId, {+ step: 'loading',+ batch: batchDate,+ totalEvents,+ processedEvents,+ });+ eventBatch.length = 0;+ }++ jobLogger.info({ processedEvents }, 'Loading complete');++ // Phase 1a: Flush profiles derived inline from events (Amplitude)+ if (canProfileFromEvents) {+ await flushProfiles();+ jobLogger.info(+ { processedProfiles },+ 'Inline profile derivation complete'+ );+ }++ // Phase 1b: Load user profiles (Mixpanel only)+ if (typeof providerInstance.streamProfiles === 'function') {+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'loading_profiles',+ });++ const profileBatch: IClickhouseProfile[] = [];+ let profilesLoaded = 0;++ for await (const rawProfile of providerInstance.streamProfiles()) {+ const profile = providerInstance.transformProfile?.(rawProfile);+ if (!profile) {+ continue;+ }+ profileBatch.push(profile);++ if (profileBatch.length >= PROFILE_BATCH_SIZE) {+ await insertProfilesBatch(profileBatch, record.projectId);+ profilesLoaded += profileBatch.length;+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'loading_profiles',+ processedProfiles: profilesLoaded,+ });+ profileBatch.length = 0;+ await yieldToEventLoop();+ }+ }++ if (profileBatch.length > 0) {+ await insertProfilesBatch(profileBatch, record.projectId);+ profilesLoaded += profileBatch.length;+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'loading_profiles',+ processedProfiles: profilesLoaded,+ totalProfiles: profilesLoaded,+ });+ }++ jobLogger.info(+ { processedProfiles: profilesLoaded },+ 'Profile loading complete'+ );+ }++ // Phase 2: Generate gap-based session IDs (Mixpanel etc.)+ if (shouldGenerateSessionIds) {+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'generating_sessions',+ });+ await generateGapBasedSessionIds(importId);+ await yieldToEventLoop();+ jobLogger.info('Session ID generation complete');+ }+ }++ // -------------------------------------------------------+ // SESSION CREATION PHASE: resumable by cleaning session_start/end+ // -------------------------------------------------------+ const skipSessionCreation =+ canResume && record.currentStep !== 'creating_sessions';++ if (!skipSessionCreation) {+ if (canResume && record.currentStep === 'creating_sessions') {+ jobLogger.info(+ 'Retry at creating_sessions — cleaning existing session_start/end events'+ );+ await cleanupSessionStartEndEvents(importId);+ }++ await updateImportStatus(jobLogger, progress, importId, {+ step: 'creating_sessions',+ batch: 'all sessions',+ });+ await createSessionsStartEndEvents(importId);+ await yieldToEventLoop();++ jobLogger.info('Session event creation complete');+ }++ // -------------------------------------------------------+ // PRODUCTION PHASE: resume-safe, track progress per batch+ // -------------------------------------------------------++ // Phase 3: Move staging events to production (per-day)+ const resumeMovingFrom =+ canResume && record.currentStep === 'moving'+ ? (record.currentBatch ?? undefined)+ : undefined;++ // currentBatch is the last successfully completed day — resume from the next day to avoid re-inserting it+ const moveFromDate = (() => {+ if (!resumeMovingFrom) {+ return undefined;+ }+ const next = new Date(`${resumeMovingFrom}T12:00:00Z`);+ next.setUTCDate(next.getUTCDate() + 1);+ return next.toISOString().split('T')[0]!;+ })();++ const bounds = await getImportDateBounds(importId, moveFromDate);+ if (bounds.min && bounds.max) {+ const startDate = bounds.min.split(' ')[0]!;+ const endDate = bounds.max.split(' ')[0]!;+ const cursor = new Date(`${startDate}T12:00:00Z`);+ const end = new Date(`${endDate}T12:00:00Z`);++ while (cursor <= end) {+ const dateStr = cursor.toISOString().split('T')[0]!;++ await moveImportsToProduction(importId, dateStr);+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'moving',+ batch: dateStr,+ });++ await yieldToEventLoop();+ cursor.setUTCDate(cursor.getUTCDate() + 1);+ }+ }++ jobLogger.info('Move to production complete');++ // Phase 4: Backfill sessions table+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'backfilling_sessions',+ batch: 'all sessions',+ });+ await backfillSessionsToProduction(importId);+ await yieldToEventLoop();++ jobLogger.info('Session backfill complete');++ // Done+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'completed',+ });+ jobLogger.info('Import completed');++ return { success: true };+ } catch (error) {+ jobLogger.error({ err: error }, 'Import job failed');++ try {+ const errorMsg = error instanceof Error ? error.message : 'Unknown error';+ await updateImportStatus(jobLogger, progress, importId, {+ step: 'failed',+ errorMessage: errorMsg,+ });+ } catch (markError) {+ jobLogger.error(+ { err: error, markError },+ 'Failed to mark import as failed'+ );+ }++ throw error;+ }+}++// ---------------------------------------------------------------------------+// /import/events — moved from apps/api/src/controllers/import.controller.ts.+// Bulk-inserts already-shaped events straight into production; unrelated to+// the provider/staging pipeline above (no Import record, no ClickHouse+// staging table).+// ---------------------------------------------------------------------------++export interface InsertRawEventsResult {+ writtenRows: number;+}++export async function insertRawEventsBatch(+ projectId: string,+ events: IClickhouseEvent[],+ logger: Logger = getLogger()+): Promise<InsertRawEventsResult> {+ const { ch, TABLE_NAMES, formatClickhouseDate } = await loadChClient();+ const importedAt = formatClickhouseDate(new Date());+ const values: IClickhouseEvent[] = events.map((event) => ({+ ...event,+ properties: toDots(event.properties),+ project_id: projectId,+ created_at: formatClickhouseDate(event.created_at),+ imported_at: importedAt,+ }));++ const res = await ch.insert({+ table: TABLE_NAMES.events,+ values,+ format: 'JSONEachRow',+ });++ // ClickHouse's insert summary reports written_rows as a string.+ const writtenRows = Number(res.summary?.written_rows ?? 0);+ logger.info({ writtenRows, projectId }, 'events imported');+ return { writtenRows };+}++// ---------------------------------------------------------------------------+// `ctx.services.import` binding.+// ---------------------------------------------------------------------------++export interface ImportService {+ run(+ importId: string,+ progress?: ImportJobProgress+ ): Promise<{ success: true }>;+ /** Enqueues the `import` job and returns its BullMQ job id, for `Import.jobId`. */+ enqueue(importId: string): Promise<string>;+}++/** `ctx.services.import` — a thin binding of the job body above to a Ctx's queues. */+export function createImportService(deps: ServiceDeps): ImportService {+ const logger = deps.logger.child({ module: 'import' });++ return {+ run: (importId, progress) => runImportJob(importId, progress, logger),+ enqueue: (importId) => deps.queues.import.import.add({ importId }),+ };+}diff --git a/packages/core/src/modules/import/src/access.ts b/packages/core/src/modules/import/src/access.tsnew file mode 100644index 000000000..de4a4168e--- /dev/null+++ b/packages/core/src/modules/import/src/access.ts@@ -0,0 +1,21 @@+// Binds core's shared/access.ts ladder to @openpanel/db's real lookups, for+// this module's own router — the same binding packages/trpc/src/access.ts+// does for V1, self-contained here because the general core -> db binding+// itself moves with auth (P6), not this wave.++import {+ canWriteProject,+ getOrganizationAccess,+ getProjectAccess,+} from '@openpanel/db/src/services/access.service';+import { getProjectById } from '@openpanel/db/src/services/project.service';+import { createAccessChecks } from '../../../shared/access';++const checks = createAccessChecks({+ getProjectAccess,+ canWriteProject,+ getOrganizationAccess,+ getProjectById,+});++export const requireProjectAccess = checks.requireProjectAccess;diff --git a/packages/importer/src/base-provider.ts b/packages/core/src/modules/import/src/base-provider.tssimilarity index 93%rename from packages/importer/src/base-provider.tsrename to packages/core/src/modules/import/src/base-provider.tsindex e7c6189fe..056ff20f5 100644--- a/packages/importer/src/base-provider.ts+++ b/packages/core/src/modules/import/src/base-provider.ts@@ -12,7 +12,7 @@ export abstract class BaseImportProvider<* This should be implemented as an async generator to handle large files efficiently*/abstract parseSource(- overrideFrom?: string,+ overrideFrom?: string): AsyncGenerator<TRawEvent, void, unknown>;/**@@ -45,7 +45,7 @@ export abstract class BaseImportProvider</*** Optional hook: Custom error handling*/- async onError?(error: Error, context?: ErrorContext): Promise<void> {+ async onError?(error: Error, _context?: ErrorContext): Promise<void> {// Default: re-throwthrow error;}@@ -80,9 +80,9 @@ export abstract class BaseImportProvider<to: string,options?: {chunkSizeDays?: number;- },- ): Array<[string, string]> {- const chunks: Array<[string, string]> = [];+ }+ ): [string, string][] {+ const chunks: [string, string][] = [];const startDate = new Date(from);const endDate = new Date(to);@@ -113,7 +113,9 @@ export abstract class BaseImportProvider<// Move cursor to the next chunk start (after the current chunk)cursor.setDate(cursor.getDate() + chunkSizeDays);- if (cursor > endDate) break;+ if (cursor > endDate) {+ break;+ }}return chunks;diff --git a/packages/importer/src/providers/__fixtures__/amplitude-export.json b/packages/core/src/modules/import/src/providers/__fixtures__/amplitude-export.jsonsimilarity index 100%rename from packages/importer/src/providers/__fixtures__/amplitude-export.jsonrename to packages/core/src/modules/import/src/providers/__fixtures__/amplitude-export.jsondiff --git a/packages/importer/src/providers/amplitude.test.ts b/packages/core/src/modules/import/src/providers/amplitude.test.tssimilarity index 98%rename from packages/importer/src/providers/amplitude.test.tsrename to packages/core/src/modules/import/src/providers/amplitude.test.tsindex c50541df7..c3e3a1ba9 100644--- a/packages/importer/src/providers/amplitude.test.ts+++ b/packages/core/src/modules/import/src/providers/amplitude.test.ts@@ -1,8 +1,8 @@+import { describe, expect, it } from 'bun:test';import { readFileSync } from 'node:fs';import { join } from 'node:path';import { Readable } from 'node:stream';import { gzipSync } from 'node:zlib';-import { describe, expect, it } from 'vitest';import { AmplitudeProvider, type AmplitudeRawEvent } from './amplitude';import { toCountryCode } from './country-codes';@@ -154,7 +154,10 @@ describe('amplitude', () => {const gz = gzipSync(Buffer.from(ndjson));const events = await collect(- provider.streamNdjson(Readable.from(gz), 'https://example.com/export.json.gz')+ provider.streamNdjson(+ Readable.from(gz),+ 'https://example.com/export.json.gz'+ ));expect(events.map((e) => e.event_type)).toEqual(['A']);diff --git a/packages/importer/src/providers/amplitude.ts b/packages/core/src/modules/import/src/providers/amplitude.tssimilarity index 97%rename from packages/importer/src/providers/amplitude.tsrename to packages/core/src/modules/import/src/providers/amplitude.tsindex 6c68e603a..2a1e1bfa5 100644--- a/packages/importer/src/providers/amplitude.ts+++ b/packages/core/src/modules/import/src/providers/amplitude.ts@@ -3,19 +3,18 @@ import { createInterface } from 'node:readline';import { Readable } from 'node:stream';import { createBrotliDecompress, createGunzip } from 'node:zlib';import { isSameDomain, parsePath, toDots } from '@openpanel/common';+// Deep imports, not @openpanel/db's full barrel — see umami.ts's header.+import { formatClickhouseDate } from '@openpanel/db/src/clickhouse/client';+import type { IClickhouseEvent } from '@openpanel/db/src/services/event.service';+import type { IClickhouseProfile } from '@openpanel/db/src/services/profile.service';+import unzipper from 'unzipper';+import { z } from 'zod';+import type { Logger } from '../../../../logger';import {getReferrerWithQuery,- type ILogger,parseReferrer,-} from '@openpanel/core';-import {- formatClickhouseDate,- type IClickhouseEvent,- type IClickhouseProfile,-} from '@openpanel/db';-import type { IAmplitudeImportConfig } from '@openpanel/validation';-import unzipper from 'unzipper';-import { z } from 'zod';+} from '../../../../shared/parse-referrer';+import type { IAmplitudeImportConfig } from '../../import.constants';import { BaseImportProvider } from '../base-provider';import { toCountryCode } from './country-codes';@@ -80,7 +79,7 @@ export class AmplitudeProvider extends BaseImportProvider<AmplitudeRawEvent> {constructor(private readonly projectId: string,private readonly config: IAmplitudeImportConfig,- private readonly logger?: ILogger+ private readonly logger?: Logger) {super();const residency = config.dataResidency ?? 'us';diff --git a/packages/importer/src/providers/country-codes.ts b/packages/core/src/modules/import/src/providers/country-codes.tssimilarity index 100%rename from packages/importer/src/providers/country-codes.tsrename to packages/core/src/modules/import/src/providers/country-codes.tsdiff --git a/packages/importer/src/providers/mixpanel.test.ts b/packages/core/src/modules/import/src/providers/mixpanel.test.tssimilarity index 96%rename from packages/importer/src/providers/mixpanel.test.tsrename to packages/core/src/modules/import/src/providers/mixpanel.test.tsindex 31ee435d7..08f9987d5 100644--- a/packages/importer/src/providers/mixpanel.test.ts+++ b/packages/core/src/modules/import/src/providers/mixpanel.test.ts@@ -1,5 +1,5 @@+import { describe, expect, it } from 'bun:test';import { omit } from 'ramda';-import { describe, expect, it } from 'vitest';import { MixpanelProvider } from './mixpanel';describe('mixpanel', () => {@@ -330,9 +330,15 @@ describe('mixpanel', () => {};// All valid values should construct without throwing- expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'us' })).not.toThrow();- expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'eu' })).not.toThrow();- expect(() => new MixpanelProvider('pid', { ...base, dataResidency: 'in' })).not.toThrow();+ expect(+ () => new MixpanelProvider('pid', { ...base, dataResidency: 'us' })+ ).not.toThrow();+ expect(+ () => new MixpanelProvider('pid', { ...base, dataResidency: 'eu' })+ ).not.toThrow();+ expect(+ () => new MixpanelProvider('pid', { ...base, dataResidency: 'in' })+ ).not.toThrow();// Undefined defaults to 'us' without throwingexpect(() => new MixpanelProvider('pid', { ...base })).not.toThrow();diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/core/src/modules/import/src/providers/mixpanel.tssimilarity index 97%rename from packages/importer/src/providers/mixpanel.tsrename to packages/core/src/modules/import/src/providers/mixpanel.tsindex 6d09223c6..de8976608 100644--- a/packages/importer/src/providers/mixpanel.ts+++ b/packages/core/src/modules/import/src/providers/mixpanel.ts@@ -1,16 +1,20 @@import { randomUUID } from 'node:crypto';import { isSameDomain, parsePath, toDots } from '@openpanel/common';+// Deep imports, not @openpanel/db's full barrel — see umami.ts's header.+import { formatClickhouseDate } from '@openpanel/db/src/clickhouse/client';+import type { IClickhouseEvent } from '@openpanel/db/src/services/event.service';+import type { IClickhouseProfile } from '@openpanel/db/src/services/profile.service';+import { z } from 'zod';+import type { Logger } from '../../../../logger';import {getReferrerWithQuery,- type ILogger,parseReferrer,+} from '../../../../shared/parse-referrer';+import {parseUserAgent,type UserAgentInfo,-} from '@openpanel/core';-import type { IClickhouseProfile } from '@openpanel/db';-import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';-import type { IMixpanelImportConfig } from '@openpanel/validation';-import { z } from 'zod';+} from '../../../../shared/parser-user-agent';+import type { IMixpanelImportConfig } from '../../import.constants';import { BaseImportProvider } from '../base-provider';export const zMixpanelRawEvent = z.object({@@ -68,7 +72,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {constructor(private readonly projectId: string,private readonly config: IMixpanelImportConfig,- private readonly logger?: ILogger+ private readonly logger?: Logger) {super();const residency = config.dataResidency ?? 'us';diff --git a/packages/importer/src/providers/umami.test.ts b/packages/core/src/modules/import/src/providers/umami.test.tssimilarity index 84%rename from packages/importer/src/providers/umami.test.tsrename to packages/core/src/modules/import/src/providers/umami.test.tsindex a386a6ae2..e75956696 100644--- a/packages/importer/src/providers/umami.test.ts+++ b/packages/core/src/modules/import/src/providers/umami.test.ts@@ -1,6 +1,6 @@+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';import http from 'node:http';import type { AddressInfo } from 'node:net';-import { afterAll, beforeAll, describe, expect, it } from 'vitest';import { UmamiProvider } from './umami';/**@@ -24,7 +24,7 @@ describe('umami remote file SSRF guard', () => {res.end('INTERNAL_ONLY_SECRET\n');});await new Promise<void>((resolve) =>- internalServer.listen(0, '127.0.0.1', resolve),+ internalServer.listen(0, '127.0.0.1', resolve));internalPort = (internalServer.address() as AddressInfo).port;});@@ -50,7 +50,7 @@ describe('umami remote file SSRF guard', () => {hits = [];await expect(- drain(`http://127.0.0.1:${internalPort}/secret.csv`),+ drain(`http://127.0.0.1:${internalPort}/secret.csv`)).rejects.toThrow();expect(hits).toEqual([]);@@ -58,7 +58,7 @@ describe('umami remote file SSRF guard', () => {it('refuses cloud instance metadata', async () => {await expect(- drain('http://169.254.169.254/latest/meta-data/'),+ drain('http://169.254.169.254/latest/meta-data/')).rejects.toThrow();});@@ -66,9 +66,10 @@ describe('umami remote file SSRF guard', () => {hits = [];const target = `http://127.0.0.1:${internalPort}/secret.csv`;- const error = await drain(target).catch((err: unknown) => err as Error);+ const caught = await drain(target).catch((err: unknown) => err as Error);- expect(error).toBeInstanceOf(Error);+ expect(caught).toBeInstanceOf(Error);+ const error = caught as Error;// The old message was `Failed to fetch remote file: 418 I'm a Teapot`.expect(error.message).not.toContain('418');expect(error.message).not.toContain('Teapot');diff --git a/packages/importer/src/providers/umami.ts b/packages/core/src/modules/import/src/providers/umami.tssimilarity index 93%rename from packages/importer/src/providers/umami.tsrename to packages/core/src/modules/import/src/providers/umami.tsindex c873eeb40..6022de72d 100644--- a/packages/importer/src/providers/umami.ts+++ b/packages/core/src/modules/import/src/providers/umami.ts@@ -3,18 +3,24 @@ import { Readable } from 'node:stream';import { pipeline } from 'node:stream/promises';import { createBrotliDecompress, createGunzip } from 'node:zlib';import { isSameDomain, parsePath, toDots } from '@openpanel/common';-import {- generateDeviceId,- getReferrerWithQuery,- type ILogger,- parseReferrer,- safeFetchStream,-} from '@openpanel/core';-import { formatClickhouseDate, type IClickhouseEvent } from '@openpanel/db';-import type { IUmamiImportConfig } from '@openpanel/validation';+// Deep imports, not @openpanel/db's full barrel: the barrel's+// `export * from './src/services/notification.service'` /+// `'./src/buffers'` eagerly reach @openpanel/queue, which constructs a+// pino-pretty transport at import time — fatal under bun:test's+// `--isolate` worker threads (AGENTS.md's eager-barrel-chain hazard).+import { formatClickhouseDate } from '@openpanel/db/src/clickhouse/client';+import type { IClickhouseEvent } from '@openpanel/db/src/services/event.service';import { parse } from 'csv-parse';import { assocPath } from 'ramda';import { z } from 'zod';+import type { Logger } from '../../../../logger';+import {+ getReferrerWithQuery,+ parseReferrer,+} from '../../../../shared/parse-referrer';+import { generateDeviceId } from '../../../../shared/profileId';+import { safeFetchStream } from '../../../../shared/safe-fetch';+import type { IUmamiImportConfig } from '../../import.constants';import { BaseImportProvider } from '../base-provider';/**@@ -77,7 +83,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {constructor(private readonly projectId: string,private readonly config: IUmamiImportConfig,- private readonly logger?: ILogger+ private readonly logger?: Logger) {super();}@@ -275,7 +281,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {.filter(Boolean).join('')}`: '';- const { path, hash, query, origin } = parsePath(url);+ const { path, query, origin } = parsePath(url);// Extract referrer information - use same logic as real-time eventsconst referrerUrl = rawEvent.referrer_domain? `https://${rawEvent.referrer_domain}${rawEvent.referrer_path || ''}`diff --git a/packages/importer/src/types.ts b/packages/core/src/modules/import/src/types.tssimilarity index 93%rename from packages/importer/src/types.tsrename to packages/core/src/modules/import/src/types.tsindex ae56bf29e..5f7c466fe 100644--- a/packages/importer/src/types.ts+++ b/packages/core/src/modules/import/src/types.ts@@ -1,8 +1,4 @@-import type {- IImportedEvent,- IServiceCreateEventPayload,- IServiceImportedEventPayload,-} from '@openpanel/db';+import type { IServiceImportedEventPayload } from '@openpanel/db';export interface ImportConfig {projectId: string;diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.tsindex b47c44a5e..37c52bf5a 100644--- a/packages/core/src/modules/insight/insight.jobs.test.ts+++ b/packages/core/src/modules/insight/insight.jobs.test.ts@@ -41,6 +41,7 @@ function stubJobCtx(insight: insight as InsightService,gsc: {} as Services['gsc'],cohort: {} as Services['cohort'],+ import: {} as Services['import'],};return {db: {},@@ -120,6 +121,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId de} as InsightService,gsc: {} as Services['gsc'],cohort: {} as Services['cohort'],+ import: {} as Services['import'],},requestId: 'req_1',job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'insightsDaily' },diff --git a/packages/core/src/modules/insight/insight.service.test.ts b/packages/core/src/modules/insight/insight.service.test.tsindex 9eee88827..99bc77878 100644--- a/packages/core/src/modules/insight/insight.service.test.ts+++ b/packages/core/src/modules/insight/insight.service.test.ts@@ -22,11 +22,12 @@ mock.module('@openpanel/db/src/prisma-client', () => ({db: { $executeRaw },}));-// `originalCh`/`chQuery`/`TABLE_NAMES`/`getReplicatedTableName` are unused-// here but included because `mock.module` replaces this specifier-// process-wide (bun runs every test file in one shared module registry-// without `--isolate` — see AGENTS.md) — gsc.service.test.ts and-// cohort.service.test.ts mock the same path, so every factory must be a+// `originalCh`/`chQuery`/`TABLE_NAMES`/`getReplicatedTableName`/+// `formatClickhouseDate`/`convertClickhouseDateToJs` are unused here but+// included because `mock.module` replaces this specifier process-wide (bun+// runs every test file in one shared module registry without `--isolate` —+// see AGENTS.md) — gsc.service.test.ts, cohort.service.test.ts and+// import.service.test.ts mock the same path, so every factory must be a// superset of every consumer's needs, whichever one ends up registered last.mock.module('@openpanel/db/src/clickhouse/client', () => ({ch: {@@ -39,7 +40,19 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({},chQuery: mock(async () => []),getReplicatedTableName: mock((table: string) => table),- TABLE_NAMES: { sessions: 'sessions' },+ TABLE_NAMES: {+ events: 'events',+ events_imports: 'events_imports',+ profiles: 'profiles',+ sessions: 'sessions',+ },+ formatClickhouseDate: (date: Date | string) =>+ new Date(date)+ .toISOString()+ .replace('T', ' ')+ .replace(/(\.\d{3})?Z+$/, ''),+ convertClickhouseDateToJs: (date: string) =>+ new Date(`${date.replace(' ', 'T')}Z`),}));const spikesQuery = mock(async () => [diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.tsindex 6a1cd109d..6e16e343f 100644--- a/packages/core/src/rest.routes.ts+++ b/packages/core/src/rest.routes.ts@@ -10,6 +10,7 @@ import type { AppDeps } from './context';import { registry } from './metrics';import { gscRoutes } from './modules/gsc/gsc.routes';import { healthRoutes } from './modules/health/health.routes';+import { importRoutes } from './modules/import/import.routes';// ADR-002/ADR-003 pin: @elysiajs/openapi at the 1.4.15 fallback (2.0 is// NO-GO per spike 6). `specPath` is set explicitly rather than taking the@@ -20,12 +21,17 @@ const OPENAPI_SPEC_PATH = '/openapi.json';// endpoint (ADR-003: "the transform that hides /metrics from the spec").const OPENAPI_EXCLUDED_PATHS = ['/metrics'];-// Both surfaces still take `deps` even where unused so a module addition-// never changes the factory's signature, only its body. Return types are-// left inferred, as `defineRoutes`'s `RouteApp` is: `.use()`'d plugins widen-// Elysia's type parameters in a way the bare `Elysia` type cannot express.-export const publicApiRoutes = (_deps: AppDeps) =>- new Elysia({ name: 'core/public-api-routes' });+// A factory always takes `deps` even where its body is empty so a module+// addition never changes the signature. Return types are left inferred, as+// `defineRoutes`'s `RouteApp` is: `.use()`'d plugins widen Elysia's type+// parameters in a way the bare `Elysia` type cannot express.+//+// import's /import/events is the first module to land here (M5-004). It is+// not yet reachable: the clientAuth macro's authenticator is a P8 stub that+// always returns null (import.routes.ts's header), and main.ts does not+// mount `publicApiRoutes` until a real `AppDeps` exists (P3/P4/P8).+export const publicApiRoutes = (deps: AppDeps) =>+ new Elysia({ name: 'core/public-api-routes' }).use(importRoutes(deps));// gsc's callback is the first module to land here (M5-002). It is not yet// reachable in production: main.ts only mounts `dashboardRoutes` once a realdiff --git a/packages/core/src/rpc.router.ts b/packages/core/src/rpc.router.tsindex 3e8801b8d..d45ede326 100644--- a/packages/core/src/rpc.router.ts+++ b/packages/core/src/rpc.router.ts@@ -7,6 +7,7 @@import { cohortRouter } from './modules/cohort/cohort.rpc';import { gscRouter } from './modules/gsc/gsc.rpc';import { healthRouter } from './modules/health/health.rpc';+import { importRouter } from './modules/import/import.rpc';import { insightRouter } from './modules/insight/insight.rpc';import { createTRPCRouter } from './rpc/base';@@ -15,6 +16,7 @@ export const appRouter = createTRPCRouter({insight: insightRouter,gsc: gscRouter,cohort: cohortRouter,+ import: importRouter,});export type AppRouter = typeof appRouter;diff --git a/packages/core/src/services.ts b/packages/core/src/services.tsindex 73b4aa00f..64a22c33d 100644--- a/packages/core/src/services.ts+++ b/packages/core/src/services.ts@@ -12,6 +12,10 @@ import {createCohortService,} from './modules/cohort/cohort.service';import { createGscService, type GscService } from './modules/gsc/gsc.service';+import {+ createImportService,+ type ImportService,+} from './modules/import/import.service';import {createInsightService,type InsightService,@@ -37,6 +41,7 @@ export interface Services {insight: InsightService;gsc: GscService;cohort: CohortService;+ import: ImportService;}export function createServices(deps: ServiceDeps): Services {@@ -48,5 +53,6 @@ export function createServices(deps: ServiceDeps): Services {insight: createInsightService(deps),gsc: createGscService(deps),cohort: createCohortService(deps),+ import: createImportService(deps),};}diff --git a/packages/db/index.ts b/packages/db/index.tsindex 1eb06c2f8..300617dcd 100644--- a/packages/db/index.ts+++ b/packages/db/index.ts@@ -22,7 +22,6 @@ export * from './src/services/funnel.service';export * from './src/services/group.service';export * from './src/services/gsc.service';export * from './src/services/id.service';-export * from './src/services/import.service';export * from './src/services/notification.service';export * from './src/services/organization.service';export * from './src/services/overview.service';diff --git a/packages/importer/package.json b/packages/importer/package.jsondeleted file mode 100644index 732079c3e..000000000--- a/packages/importer/package.json+++ /dev/null@@ -1,37 +0,0 @@-{- "name": "@openpanel/importer",- "version": "0.0.0",- "type": "module",- "main": "src/index.ts",- "scripts": {- "build": "tsc",- "dev": "tsc --watch",- "test": "vitest",- "test:run": "vitest run"- },- "exports": {- ".": "./src/index.ts",- "./providers": "./src/providers/metadata.ts"- },- "dependencies": {- "@openpanel/common": "workspace:*",- "@openpanel/core": "workspace:*",- "@openpanel/db": "workspace:*",- "@openpanel/queue": "workspace:*",- "@openpanel/validation": "workspace:*",- "csv-parse": "^6.1.0",- "ramda": "^0.29.1",- "unzipper": "^0.12.3",- "uuid": "^9.0.1",- "zod": "catalog:"- },- "devDependencies": {- "@types/node": "^20.0.0",- "@types/ramda": "^0.31.1",- "@types/unzipper": "^0.10.10",- "@types/uuid": "^9.0.7",- "bullmq": "^5.8.7",- "typescript": "^5.0.0",- "vitest": "^1.0.0"- }-}diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.tsdeleted file mode 100644index 61bb8d9cc..000000000--- a/packages/importer/src/index.ts+++ /dev/null@@ -1,14 +0,0 @@-export { UmamiProvider } from './providers/umami';-export { MixpanelProvider } from './providers/mixpanel';-export { AmplitudeProvider } from './providers/amplitude';-export type {- ImportConfig,- ImportProgress,- ImportResult,- BatchResult,- BaseRawEvent,- ErrorContext,- EventProperties,- ImportJobMetadata,- ImportStageResult,-} from './types';diff --git a/packages/importer/src/providers/metadata.ts b/packages/importer/src/providers/metadata.tsdeleted file mode 100644index 578e3fa30..000000000--- a/packages/importer/src/providers/metadata.ts+++ /dev/null@@ -1,38 +0,0 @@-export type ImportProviderId = 'umami' | 'mixpanel' | 'amplitude';-export type ImportProviderType = 'file' | 'api';--export interface ImportProviderMeta {- id: ImportProviderId;- name: string;- description: string;- logo: string;- backgroundColor: string;- types: ImportProviderType[];-}--export const IMPORT_PROVIDERS: ImportProviderMeta[] = [- {- id: 'umami',- name: 'Umami',- description: 'Import your analytics data from Umami',- logo: 'https://cdn.brandfetch.io/id_3VEohOm/w/180/h/180/theme/dark/logo.png?c=1dxbfHSJFAPEGdCLU4o5B',- backgroundColor: '#fff',- types: ['file'],- },- {- id: 'mixpanel',- name: 'Mixpanel',- description: 'Import your analytics data from Mixpanel API',- logo: 'https://cdn.brandfetch.io/idr_rhI2FS/theme/dark/idMJ8uODLv.svg?c=1dxbfHSJFAPEGdCLU4o5B',- backgroundColor: '#fff',- types: ['api'],- },- {- id: 'amplitude',- name: 'Amplitude',- description: 'Import your analytics data from the Amplitude Export API',- logo: 'https://cdn.brandfetch.io/idnYrZGER0/theme/dark/symbol.svg?c=1dxbfHSJFAPEGdCLU4o5B',- backgroundColor: '#fff',- types: ['api'],- },-];diff --git a/packages/importer/tsconfig.json b/packages/importer/tsconfig.jsondeleted file mode 100644index 29cedb070..000000000--- a/packages/importer/tsconfig.json+++ /dev/null@@ -1,10 +0,0 @@-{- "extends": "../../tooling/typescript/base.json",- "compilerOptions": {- "outDir": "./dist",- "rootDir": "./src",- "strictNullChecks": true- },- "include": ["src/**/*"],- "exclude": ["node_modules", "dist"]-}diff --git a/packages/importer/vitest.config.ts b/packages/importer/vitest.config.tsdeleted file mode 100644index f87a2039f..000000000--- a/packages/importer/vitest.config.ts+++ /dev/null@@ -1,3 +0,0 @@-import { getSharedVitestConfig } from '../../vitest.shared';--export default getSharedVitestConfig({ __dirname });diff --git a/packages/validation/src/import.validation.ts b/packages/validation/src/import.validation.tsnew file mode 100644index 000000000..662da343a--- /dev/null+++ b/packages/validation/src/import.validation.ts@@ -0,0 +1,9 @@+// Moved into @openpanel/core's import module (M5-004, ADR-008's module map:+// import owns "C"). Re-exported here for existing @openpanel/validation+// importers (apps/start's import UI, packages/db/src/types.ts) — same shape+// as packages/validation/src/cohort.validation.ts since M5-003. The+// `./modules/import/import.constants` subpath is a leaf, isomorphic file+// with no back-imports into @openpanel/validation, so this re-export carries+// no runtime cycle despite @openpanel/core already depending on+// @openpanel/validation elsewhere.+export * from '@openpanel/core/modules/import/import.constants';diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.tsindex c4aec23fa..22bcd1e6c 100644--- a/packages/validation/src/index.ts+++ b/packages/validation/src/index.ts@@ -1,5 +1,3 @@-import { z } from 'zod';-import {chartSegments,chartTypes,@@ -10,6 +8,7 @@ import {operators,timeWindows,} from '@openpanel/constants';+import { z } from 'zod';/*** Chart formulas are plain arithmetic over series references (A, B, C, ...).@@ -20,7 +19,7 @@ import {const CHART_FORMULA_PATTERN = /^[A-Za-z0-9_ .,+\-*/()%^]*$/;export function objectToZodEnums<K extends string>(- obj: Record<K, any>,+ obj: Record<K, any>): [K, ...K[]] {const [firstKey, ...otherKeys] = Object.keys(obj) as K[];return [firstKey!, ...otherKeys];@@ -42,21 +41,21 @@ export const zChartEventFilter = z.object({.optional().describe('Cast type for the column/value in equality & comparison operators ' +- '(string/number/date/datetime/boolean). Absent = legacy behavior.',+ '(string/number/date/datetime/boolean). Absent = legacy behavior.'),cohortId: z.string().optional().describe('DEPRECATED: legacy single-cohort id, kept for saved reports. ' +- 'New code reads cohortIds via getCohortIds(filter).',+ 'New code reads cohortIds via getCohortIds(filter).'),cohortIds: z.array(z.string()).optional().describe('Cohort IDs for inCohort/notInCohort. Multiple ids OR-match ' +- '(matches profiles in any of the listed cohorts).',+ '(matches profiles in any of the listed cohorts).'),});@@ -100,7 +99,7 @@ export const zChartEvent = z.object({.string().optional().describe(- 'Optional property of the event used for specific segment calculations (e.g., value for property_sum/average)',+ 'Optional property of the event used for specific segment calculations (e.g., value for property_sum/average)'),segment: zChartEventSegment,filters: z@@ -120,7 +119,7 @@ export const zChartFormula = z.object({.max(1000).regex(CHART_FORMULA_PATTERN,- 'Formula may only contain series references, numbers and arithmetic operators',+ 'Formula may only contain series references, numbers and arithmetic operators').describe('The formula expression (e.g., A+B, A/B)'),displayName: z@@ -131,7 +130,7 @@ export const zChartFormula = z.object({.array(z.string()).optional().describe(- 'Alpha IDs (e.g. ["A", "B"]) of series referenced by this formula that should be hidden from the chart while still being used in the formula computation',+ 'Alpha IDs (e.g. ["A", "B"]) of series referenced by this formula that should be hidden from the chart while still being used in the formula computation'),});@@ -153,7 +152,7 @@ export const zChartBreakdown = z.object({export const zChartSeries = z.array(zChartEventItem).describe(- 'Array of series (events or formulas) to be tracked and displayed in the chart',+ 'Array of series (events or formulas) to be tracked and displayed in the chart');export const zChartBreakdowns = z.array(zChartBreakdown);@@ -238,10 +237,10 @@ export const zReportInput = z.object({interval: zTimeInterval.default('day').describe(- 'The time interval for data aggregation (e.g., day, week, month)',+ 'The time interval for data aggregation (e.g., day, week, month)'),series: zChartSeries.describe(- 'Array of series (events or formulas) to be tracked and displayed in the chart',+ 'Array of series (events or formulas) to be tracked and displayed in the chart'),breakdowns: zChartBreakdowns.default([])@@ -250,7 +249,7 @@ export const zReportInput = z.object({.array(zChartEventFilter).optional().describe(- 'Filters applied to ALL event series in this report (combined with each series own filters using AND)',+ 'Filters applied to ALL event series in this report (combined with each series own filters using AND)'),range: zRange.default('30d')@@ -259,13 +258,13 @@ export const zReportInput = z.object({.string().nullish().describe(- 'Custom start date for the data range (overrides range if provided)',+ 'Custom start date for the data range (overrides range if provided)'),endDate: z.string().nullish().describe(- 'Custom end date for the data range (overrides range if provided)',+ 'Custom end date for the data range (overrides range if provided)'),previous: z.boolean()@@ -278,7 +277,7 @@ export const zReportInput = z.object({metric: zMetric.default('sum').describe(- 'The aggregation method for the metric (e.g., sum, count, average)',+ 'The aggregation method for the metric (e.g., sum, count, average)'),limit: z.number()@@ -304,7 +303,7 @@ export const zReportInput = z.object({.string().optional().describe(- "Optional unit of measurement for the chart's Y-axis (e.g., $, %, users)",+ "Optional unit of measurement for the chart's Y-axis (e.g., $, %, users)"),});@@ -390,7 +389,7 @@ export const zOnboardingProject = ztimezone: z.string().optional(),}).superRefine((data, ctx) => {- if (!data.organization && !data.organizationId) {+ if (!(data.organization || data.organizationId)) {ctx.addIssue({code: 'custom',message: 'Organization is required',@@ -426,7 +425,6 @@ export const zOnboardingProject = z}});-export * from './integrations';export const zNotificationRuleEventConfig = z.object({@@ -608,7 +606,7 @@ export const zGroupId = z.min(1).regex(/^[a-z0-9_-]+$/,- 'ID must only contain lowercase letters, digits, hyphens, or underscores',+ 'ID must only contain lowercase letters, digits, hyphens, or underscores');export const zCreateGroup = z.object({@@ -635,103 +633,10 @@ export const zEditOrganization = z.object({timezone: z.string().min(1),});-const zProjectMapper = z.object({- from: z.string().min(1),- to: z.string().min(1),-});--/**- * `z.string().url()` alone accepts `file:`, `gopher:` and friends. Restricting- * the scheme here gives the user an immediate form error instead of a job that- * fails later. It is NOT the SSRF control - the value is stored and fetched- * afterwards, so the destination is re-validated at fetch time by- * `safeFetchStream`.- */-export const zHttpUrl = z- .string()- .url()- .refine(- (value) => {- try {- const { protocol } = new URL(value);- return protocol === 'http:' || protocol === 'https:';- } catch {- return false;- }- },- { message: 'Only http and https URLs are allowed' },- );--const createFileImportConfig = <T extends string>(provider: T) =>- z.object({- provider: z.literal(provider),- type: z.literal('file'),- fileUrl: zHttpUrl,- });--// Import configs-export const zUmamiImportConfig = createFileImportConfig('umami').extend({- projectMapper: z.array(zProjectMapper),-});--export type IUmamiImportConfig = z.infer<typeof zUmamiImportConfig>;--export const zPlausibleImportConfig = createFileImportConfig('plausible');-export type IPlausibleImportConfig = z.infer<typeof zPlausibleImportConfig>;--export const zAmplitudeDataResidency = z.enum(['us', 'eu']);-export type IAmplitudeDataResidency = z.infer<typeof zAmplitudeDataResidency>;--export const zAmplitudeImportConfig = z.object({- provider: z.literal('amplitude'),- type: z.literal('api'),- apiKey: z.string().min(1),- secretKey: z.string().min(1),- from: z.string().min(1),- to: z.string().min(1),- mapScreenViewProperty: z.string().optional(),- dataResidency: zAmplitudeDataResidency.optional(),-});-export type IAmplitudeImportConfig = z.infer<typeof zAmplitudeImportConfig>;--export const zMixpanelDataResidency = z.enum(['us', 'eu', 'in']);-export type IMixpanelDataResidency = z.infer<typeof zMixpanelDataResidency>;--export const zMixpanelImportConfig = z.object({- provider: z.literal('mixpanel'),- type: z.literal('api'),- serviceAccount: z.string().min(1),- serviceSecret: z.string().min(1),- projectId: z.string().min(1),- from: z.string().min(1),- to: z.string().min(1),- mapScreenViewProperty: z.string().optional(),- dataResidency: zMixpanelDataResidency.optional(),-});-export type IMixpanelImportConfig = z.infer<typeof zMixpanelImportConfig>;--export type IImportConfig =- | IUmamiImportConfig- | IPlausibleImportConfig- | IMixpanelImportConfig- | IAmplitudeImportConfig;--export const zCreateImport = z.object({- projectId: z.string().min(1),- provider: z.enum(['umami', 'plausible', 'mixpanel', 'amplitude']),- config: z.union([- zUmamiImportConfig,- zPlausibleImportConfig,- zMixpanelImportConfig,- zAmplitudeImportConfig,- ]),-});--export type ICreateImport = z.infer<typeof zCreateImport>;--export * from './types.insights';-export * from './types.validation';-export * from './track.validation';-export * from './event-blocklist';export * from './chat';export * from './cohort.validation';+export * from './event-blocklist';+export * from './import.validation';+export * from './track.validation';+export * from './types.insights';+export * from './types.validation';diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 41bc01939..bf08bcfe2 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -429,9 +429,9 @@ importers:'@openpanel/constants':specifier: workspace:^version: link:../../packages/constants- '@openpanel/importer':+ '@openpanel/core':specifier: workspace:^- version: link:../../packages/importer+ version: link:../../packages/core'@openpanel/json':specifier: workspace:*version: link:../../packages/json@@ -953,9 +953,6 @@ importers:'@openpanel/email':specifier: workspace:*version: link:../../packages/email- '@openpanel/importer':- specifier: workspace:*- version: link:../../packages/importer'@openpanel/payments':specifier: workspace:*version: link:../../packages/payments@@ -1153,6 +1150,9 @@ importers:bullmq:specifier: 6.3.3version: 6.3.3([email protected]([email protected]))+ csv-parse:+ specifier: ^6.1.0+ version: 6.1.0elysia:specifier: 1.4.30version: 1.4.30(@sinclair/[email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])@@ -1177,6 +1177,9 @@ importers:qrcode:specifier: ^1.5.4version: 1.5.4+ ramda:+ specifier: ^0.29.1+ version: 0.29.1sqlstring:specifier: ^2.3.3version: 2.3.3@@ -1189,6 +1192,9 @@ importers:undici:specifier: ^8.10.0version: 8.10.0+ unzipper:+ specifier: ^0.12.3+ version: 0.12.5zod:specifier: 'catalog:'version: 4.3.6@@ -1202,9 +1208,15 @@ importers:'@types/qrcode':specifier: ^1.5.6version: 1.5.6+ '@types/ramda':+ specifier: ^0.31.1+ version: 0.31.1'@types/sqlstring':specifier: ^2.3.2version: 2.3.2+ '@types/unzipper':+ specifier: ^0.10.10+ version: 0.10.11bun-types:specifier: 1.4.0version: 1.4.0@@ -1358,61 +1370,6 @@ importers:specifier: 'catalog:'version: 5.9.3- packages/importer:- dependencies:- '@openpanel/common':- specifier: workspace:*- version: link:../common- '@openpanel/core':- specifier: workspace:*- version: link:../core- '@openpanel/db':- specifier: workspace:*- version: link:../db- '@openpanel/queue':- specifier: workspace:*- version: link:../queue- '@openpanel/validation':- specifier: workspace:*- version: link:../validation- csv-parse:- specifier: ^6.1.0- version: 6.1.0- ramda:- specifier: ^0.29.1- version: 0.29.1- unzipper:- specifier: ^0.12.3- version: 0.12.5- uuid:- specifier: ^9.0.1- version: 9.0.1- zod:- specifier: 'catalog:'- version: 4.3.6- devDependencies:- '@types/node':- specifier: ^20.0.0- version: 20.19.24- '@types/ramda':- specifier: ^0.31.1- version: 0.31.1- '@types/unzipper':- specifier: ^0.10.10- version: 0.10.11- '@types/uuid':- specifier: ^9.0.7- version: 9.0.8- bullmq:- specifier: ^5.8.7- version: 5.63.0([email protected])- typescript:- specifier: ^5.0.0- version: 5.9.3- vitest:- specifier: ^1.0.0- version: 1.6.1(@types/[email protected])([email protected]([email protected]))([email protected])([email protected])([email protected])-packages/json:dependencies:superjson:@@ -24068,14 +24025,14 @@ snapshots:dependencies:'@jest/fake-timers': 29.7.0'@jest/types': 29.6.3- '@types/node': 20.19.24+ '@types/node': 24.10.1jest-mock: 29.7.0'@jest/[email protected]':dependencies:'@jest/types': 29.6.3'@sinonjs/fake-timers': 10.3.0- '@types/node': 20.19.24+ '@types/node': 24.10.1jest-message-util: 29.7.0jest-mock: 29.7.0jest-util: 29.7.0@@ -24088,7 +24045,7 @@ snapshots:dependencies:'@types/istanbul-lib-coverage': 2.0.6'@types/istanbul-reports': 3.0.4- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/yargs': 15.0.19chalk: 4.1.2@@ -24097,7 +24054,7 @@ snapshots:'@jest/schemas': 29.6.3'@types/istanbul-lib-coverage': 2.0.6'@types/istanbul-reports': 3.0.4- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/yargs': 17.0.32chalk: 4.1.2@@ -28168,18 +28125,18 @@ snapshots:'@slack/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@slack/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1dependencies:'@slack/logger': 3.0.0'@slack/web-api': 6.12.1([email protected]([email protected]))'@types/jsonwebtoken': 8.5.9- '@types/node': 20.19.24+ '@types/node': 24.10.1jsonwebtoken: 9.0.2lodash.isstring: 4.0.1transitivePeerDependencies:@@ -28200,7 +28157,7 @@ snapshots:dependencies:'@slack/logger': 3.0.0'@slack/web-api': 6.12.1([email protected]([email protected]))- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/ws': 7.4.7eventemitter3: 5.0.4finity: 0.5.4@@ -28217,7 +28174,7 @@ snapshots:'@slack/logger': 3.0.0'@slack/types': 2.14.0'@types/is-stream': 1.1.0- '@types/node': 20.19.24+ '@types/node': 24.10.1axios: 1.7.7([email protected]([email protected]))eventemitter3: 3.1.2form-data: 2.5.6@@ -28232,7 +28189,7 @@ snapshots:dependencies:'@slack/logger': 4.0.0'@slack/types': 2.14.0- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/retry': 0.12.0axios: 1.7.7([email protected]([email protected]))eventemitter3: 5.0.4@@ -29325,11 +29282,11 @@ snapshots:'@types/[email protected]':dependencies:'@types/connect': 3.4.38- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]': {}@@ -29339,13 +29296,13 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]': {}'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]': {}@@ -29533,18 +29490,18 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/qs': 6.9.11'@types/range-parser': 1.2.7'@types/send': 0.17.4'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/qs': 6.9.11'@types/range-parser': 1.2.7'@types/send': 0.17.4@@ -29596,7 +29553,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]': {}@@ -29614,12 +29571,12 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]':dependencies:'@types/ms': 0.7.34- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]': {}@@ -29655,7 +29612,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]': {}@@ -29667,7 +29624,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]':dependencies:@@ -29675,7 +29632,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1form-data: 4.0.5'@types/[email protected]': {}@@ -29712,13 +29669,13 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1pg-protocol: 1.6.1pg-types: 2.2.0'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1pg-protocol: 1.6.1pg-types: 2.2.0@@ -29780,7 +29737,7 @@ snapshots:'@types/[email protected]':dependencies:'@types/caseless': 0.12.5- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/tough-cookie': 4.0.5form-data: 2.5.6@@ -29793,18 +29750,18 @@ snapshots:'@types/[email protected]':dependencies:'@types/mime': 1.3.5- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]':dependencies:'@types/http-errors': 2.0.4'@types/mime': 3.0.4- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]':dependencies:'@types/http-errors': 2.0.4- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/send': 0.17.4'@types/[email protected]': {}@@ -29823,7 +29780,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]':dependencies:@@ -29832,7 +29789,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]': {}@@ -29850,7 +29807,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]': {}@@ -29869,13 +29826,13 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/source-list-map': 0.1.6source-map: 0.7.6'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/tapable': 1.0.12'@types/uglify-js': 3.17.5'@types/webpack-sources': 3.2.3@@ -29884,7 +29841,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1'@types/[email protected]':dependencies:@@ -31310,7 +31267,7 @@ snapshots:dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1escape-string-regexp: 4.0.0is-wsl: 2.2.0lighthouse-logger: 1.4.2([email protected])@@ -31319,7 +31276,7 @@ snapshots:dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1escape-string-regexp: 4.0.0is-wsl: 2.2.0lighthouse-logger: 1.4.2([email protected])@@ -32405,7 +32362,7 @@ snapshots:dependencies:'@types/cookie': 0.4.1'@types/cors': 2.8.17- '@types/node': 20.19.24+ '@types/node': 24.10.1accepts: 1.3.8base64id: 2.0.0cookie: 0.7.2@@ -34792,7 +34749,7 @@ snapshots:'@jest/environment': 29.7.0'@jest/fake-timers': 29.7.0'@jest/types': 29.6.3- '@types/node': 20.19.24+ '@types/node': 24.10.1jest-mock: 29.7.0jest-util: 29.7.0@@ -34813,13 +34770,13 @@ snapshots:dependencies:'@jest/types': 29.6.3- '@types/node': 20.19.24+ '@types/node': 24.10.1jest-util: 29.7.0dependencies:'@jest/types': 29.6.3- '@types/node': 20.19.24+ '@types/node': 24.10.1chalk: 4.1.2ci-info: 3.9.0graceful-fs: 4.2.11@@ -34836,7 +34793,7 @@ snapshots:dependencies:- '@types/node': 20.19.24+ '@types/node': 24.10.1jest-util: 29.7.0merge-stream: 2.0.0supports-color: 8.1.1@@ -37935,7 +37892,7 @@ snapshots:'@protobufjs/path': 1.1.2'@protobufjs/pool': 1.1.0'@protobufjs/utf8': 1.1.0- '@types/node': 20.19.24+ '@types/node': 24.10.1long: 5.2.3@@ -40782,24 +40739,6 @@ snapshots:dependencies:vite: 7.3.5(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])- [email protected](@types/[email protected])([email protected])([email protected])([email protected]):- dependencies:- cac: 6.7.14- debug: 4.4.3([email protected])- pathe: 1.1.2- picocolors: 1.1.1- transitivePeerDependencies:- - '@types/node'- - less- - lightningcss- - sass- - sass-embedded- - stylus- - sugarss- - supports-color- - terser-[email protected](@types/[email protected])([email protected])([email protected])([email protected]):dependencies:cac: 6.7.14@@ -40912,17 +40851,6 @@ snapshots:- supports-color- typescript- dependencies:- esbuild: 0.21.5- postcss: 8.5.6- rollup: 4.52.5- optionalDependencies:- '@types/node': 20.19.24- fsevents: 2.3.3- lightningcss: 1.30.2- terser: 5.27.1-dependencies:esbuild: 0.21.5@@ -40993,41 +40921,6 @@ snapshots:optionalDependencies:vite: 7.3.5(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])- [email protected](@types/[email protected])([email protected]([email protected]))([email protected])([email protected])([email protected]):- dependencies:- '@vitest/expect': 1.6.1- '@vitest/runner': 1.6.1- '@vitest/snapshot': 1.6.1- '@vitest/spy': 1.6.1- '@vitest/utils': 1.6.1- acorn-walk: 8.3.2- chai: 4.5.0- debug: 4.4.3([email protected])- execa: 8.0.1- local-pkg: 0.5.1- magic-string: 0.30.21- pathe: 1.1.2- picocolors: 1.1.1- std-env: 3.10.0- strip-literal: 2.1.1- tinybench: 2.9.0- tinypool: 0.8.4- vite-node: 1.6.1(@types/[email protected])([email protected])([email protected])([email protected])- why-is-node-running: 2.3.0- optionalDependencies:- '@types/node': 20.19.24- jsdom: 26.1.0([email protected])- transitivePeerDependencies:- - less- - lightningcss- - sass- - sass-embedded- - stylus- - sugarss- - supports-color- - terser-[email protected](@types/[email protected])([email protected]([email protected]))([email protected])([email protected])([email protected]):dependencies:'@vitest/expect': 1.6.1