Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Add import chunk processor

Edward Cuic578960ce32f
diff --git a/server/src/workers/processImportChunk.ts b/server/src/workers/processImportChunk.ts
new file mode 100644
index 000000000..173cd3cbd
--- /dev/null
+++ b/server/src/workers/processImportChunk.ts
@@ -0,0 +1,29 @@
+import boss from "../lib/boss.js";
+import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";
+import { ProcessImportChunkJob, processImportChunkQueue } from "../types/import.js";
+import { clickhouse } from "../db/clickhouse/clickhouse.js";
+import { Job } from "pg-boss";
+
+await boss.work(processImportChunkQueue, async (job: Job<ProcessImportChunkJob<UmamiEvent>>[]) => {
+ const { site, importId, source, chunk, chunkNumber } = job[0].data;
+
+ const dataMapper = getImportDataMapping(source);
+ const transformedRecords = dataMapper.transform(chunk, site, importId);
+
+ await clickhouse.insert({
+ table: "events",
+ values: transformedRecords,
+ format: "JSONEachRow",
+ });
+
+ console.log(`📦 Processed chunk ${chunkNumber} for import ${importId}`);
+});
+
+function getImportDataMapping(source: string) {
+ switch (source) {
+ case "umami":
+ return new UmamiImportMapper();
+ default:
+ throw new Error(`Unsupported import source: ${source}`);
+ }
+}