🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Add import chunk processor
Edward Cuic578960ce32f
diff --git a/server/src/workers/processImportChunk.ts b/server/src/workers/processImportChunk.tsnew file mode 100644index 000000000..173cd3cbd--- /dev/null+++ b/server/src/workers/processImportChunk.ts@@ -0,0 +1,29 @@+import boss from "../lib/boss.js";+import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";+import { ProcessImportChunkJob, processImportChunkQueue } from "../types/import.js";+import { clickhouse } from "../db/clickhouse/clickhouse.js";+import { Job } from "pg-boss";++await boss.work(processImportChunkQueue, async (job: Job<ProcessImportChunkJob<UmamiEvent>>[]) => {+ const { site, importId, source, chunk, chunkNumber } = job[0].data;++ const dataMapper = getImportDataMapping(source);+ const transformedRecords = dataMapper.transform(chunk, site, importId);++ await clickhouse.insert({+ table: "events",+ values: transformedRecords,+ format: "JSONEachRow",+ });++ console.log(`📦 Processed chunk ${chunkNumber} for import ${importId}`);+});++function getImportDataMapping(source: string) {+ switch (source) {+ case "umami":+ return new UmamiImportMapper();+ default:+ throw new Error(`Unsupported import source: ${source}`);+ }+}