🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Better dataInsertWorker.ts
Edward Cuif793f21e3fbd
diff --git a/server/src/types/import.ts b/server/src/types/import.tsindex cfc73a783..93931cd20 100644--- a/server/src/types/import.ts+++ b/server/src/types/import.ts@@ -16,6 +16,7 @@ export interface CsvParseJob extends ImportJob {export interface DataInsertJob<T> extends ImportJob {chunk: T[];chunkNumber: number;+ finalChunk: boolean;}export interface ImportMapper<T> {diff --git a/server/src/workers/dataInsertWorker.ts b/server/src/workers/dataInsertWorker.tsindex ac0d77882..e0c70840b 100644--- a/server/src/workers/dataInsertWorker.ts+++ b/server/src/workers/dataInsertWorker.ts@@ -3,11 +3,21 @@ import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";import { DataInsertJob, DATA_INSERT_QUEUE } from "../types/import.js";import { clickhouse } from "../db/clickhouse/clickhouse.js";import { Job } from "pg-boss";+import { ImportStatusManager } from "../lib/importStatus.js";export async function registerDataInsertWorker() {await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<DataInsertJob<UmamiEvent>>[]) => {+ const { site, importId, source, chunk, chunkNumber, finalChunk } = job.data;+try {- const { site, importId, source, chunk, chunkNumber } = job.data;+ const getImportDataMapping = (source: string) => {+ switch (source) {+ case "umami":+ return new UmamiImportMapper();+ default:+ throw new Error(`Unsupported import source: ${source}`);+ }+ }const dataMapper = getImportDataMapping(source);const transformedRecords = dataMapper.transform(chunk, site, importId);@@ -18,18 +28,21 @@ export async function registerDataInsertWorker() {format: "JSONEachRow",});- console.log(`Processed chunk ${chunkNumber} for import ${importId}`);+ console.log(`Successfully processed chunk ${chunkNumber} for import ${importId} (${transformedRecords.length} records)`);++ await ImportStatusManager.updateProgress(+ importId,+ chunkNumber,+ transformedRecords.length+ );++ if (finalChunk) {+ await ImportStatusManager.updateStatus(importId, "completed");+ }} catch (error) {- console.error("Error in processImportChunkQueue worker:", error);+ console.error("Error in data insert worker:", error);+ await ImportStatusManager.updateStatus(importId, "failed", "Error inserting chunk");+ // throw error;}});-- function getImportDataMapping(source: string) {- switch (source) {- case "umami":- return new UmamiImportMapper();- default:- throw new Error(`Unsupported import source: ${source}`);- }- }}