Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Better dataInsertWorker.ts

Edward Cuif793f21e3fbd
diff --git a/server/src/types/import.ts b/server/src/types/import.ts
index cfc73a783..93931cd20 100644
--- a/server/src/types/import.ts
+++ b/server/src/types/import.ts
@@ -16,6 +16,7 @@ export interface CsvParseJob extends ImportJob {
export interface DataInsertJob<T> extends ImportJob {
chunk: T[];
chunkNumber: number;
+ finalChunk: boolean;
}
export interface ImportMapper<T> {
diff --git a/server/src/workers/dataInsertWorker.ts b/server/src/workers/dataInsertWorker.ts
index ac0d77882..e0c70840b 100644
--- a/server/src/workers/dataInsertWorker.ts
+++ b/server/src/workers/dataInsertWorker.ts
@@ -3,11 +3,21 @@ import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";
import { DataInsertJob, DATA_INSERT_QUEUE } from "../types/import.js";
import { clickhouse } from "../db/clickhouse/clickhouse.js";
import { Job } from "pg-boss";
+import { ImportStatusManager } from "../lib/importStatus.js";
export async function registerDataInsertWorker() {
await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<DataInsertJob<UmamiEvent>>[]) => {
+ const { site, importId, source, chunk, chunkNumber, finalChunk } = job.data;
+
try {
- const { site, importId, source, chunk, chunkNumber } = job.data;
+ const getImportDataMapping = (source: string) => {
+ switch (source) {
+ case "umami":
+ return new UmamiImportMapper();
+ default:
+ throw new Error(`Unsupported import source: ${source}`);
+ }
+ }
const dataMapper = getImportDataMapping(source);
const transformedRecords = dataMapper.transform(chunk, site, importId);
@@ -18,18 +28,21 @@ export async function registerDataInsertWorker() {
format: "JSONEachRow",
});
- console.log(`Processed chunk ${chunkNumber} for import ${importId}`);
+ console.log(`Successfully processed chunk ${chunkNumber} for import ${importId} (${transformedRecords.length} records)`);
+
+ await ImportStatusManager.updateProgress(
+ importId,
+ chunkNumber,
+ transformedRecords.length
+ );
+
+ if (finalChunk) {
+ await ImportStatusManager.updateStatus(importId, "completed");
+ }
} catch (error) {
- console.error("Error in processImportChunkQueue worker:", error);
+ console.error("Error in data insert worker:", error);
+ await ImportStatusManager.updateStatus(importId, "failed", "Error inserting chunk");
+ // throw error;
}
});
-
- function getImportDataMapping(source: string) {
- switch (source) {
- case "umami":
- return new UmamiImportMapper();
- default:
- throw new Error(`Unsupported import source: ${source}`);
- }
- }
}