🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
wip
Edward Cui5ba91d3bb469
diff --git a/server/src/index.ts b/server/src/index.tsindex 6abe1e628..c62b38d02 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -79,7 +79,6 @@ import boss from "./db/postgres/boss.js";import { registerCsvParseWorker } from "./services/import/workers/csvParseWorker.js";import { registerDataInsertWorker } from "./services/import/workers/dataInsertWorker.js";import { listSiteImports } from "./api/sites/listSiteImports.js";-import { registerImportCompletionWorker } from "./services/import/workers/importCompletionWorker.js";const __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);@@ -337,7 +336,6 @@ const start = async () => {await boss.start();await registerCsvParseWorker();await registerDataInsertWorker();- await registerImportCompletionWorker();// Start the serverawait server.listen({ port: 3001, host: "0.0.0.0" });diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 388dd8235..85fe153c9 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -1,7 +1,7 @@import fs from "fs";import { parse } from "@fast-csv/parse";import boss from "../../../db/postgres/boss.js";-import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE, IMPORT_COMPLETION_QUEUE } from "./jobs.js";+import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";import { Job } from "pg-boss";import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";import { ImportStatusManager } from "../importStatusManager.js";@@ -40,6 +40,7 @@ export async function registerCsvParseWorker() {importId,source,chunk,+ allChunksSent: false,});chunk = [];}@@ -51,11 +52,16 @@ export async function registerCsvParseWorker() {importId,source,chunk,+ allChunksSent: false,});}- await boss.send(IMPORT_COMPLETION_QUEUE, {+ await boss.send(DATA_INSERT_QUEUE, {+ site,importId,+ source,+ chunk: [],+ allChunksSent: true,});} catch (error) {console.error("Error in CSV parse worker:", error);diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex 46fa60fe1..cb1342bf4 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -16,7 +16,19 @@ const getImportDataMapping = (source: string) => {export async function registerDataInsertWorker() {await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([ job ]: Job<DataInsertJob>[]) => {- const { site, importId, source, chunk } = job.data;+ const { site, importId, source, chunk, allChunksSent } = job.data;++ if (allChunksSent) {+ try {+ await ImportStatusManager.updateStatus(importId, "completed");+ console.log(`Import ${importId} completed`);+ return;+ } catch (error) {+ console.error(`Failed to mark import ${importId} as completed:`, error);+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");+ throw error;+ }+ }try {const dataMapper = getImportDataMapping(source);diff --git a/server/src/services/import/workers/importCompletionWorker.ts b/server/src/services/import/workers/importCompletionWorker.tsdeleted file mode 100644index dce68931a..000000000--- a/server/src/services/import/workers/importCompletionWorker.ts+++ /dev/null@@ -1,19 +0,0 @@-import boss from "../../../db/postgres/boss.js";-import { IMPORT_COMPLETION_QUEUE, ImportCompletionJob } from "./jobs.js";-import { Job } from "pg-boss";-import { ImportStatusManager } from "../importStatusManager.js";--export async function registerImportCompletionWorker() {- await boss.work(IMPORT_COMPLETION_QUEUE, { batchSize: 1, pollingIntervalSeconds: 30 }, async ([ job ]: Job<ImportCompletionJob>[]) => {- const { importId } = job.data;-- try {- await ImportStatusManager.updateStatus(importId, "completed");- console.log(`Import ${importId} completed`);- } catch (error) {- console.error(`Failed to mark import ${importId} as completed:`, error);- await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");- throw error;- }- });-}diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.tsindex c0c25847b..d344b961d 100644--- a/server/src/services/import/workers/jobs.ts+++ b/server/src/services/import/workers/jobs.ts@@ -4,8 +4,6 @@ export const CSV_PARSE_QUEUE = "csv-parse";export const DATA_INSERT_QUEUE = "data-insert";-export const IMPORT_COMPLETION_QUEUE = "import-completion";-interface ImportJob {site: string;importId: string;@@ -19,8 +17,5 @@ export interface CsvParseJob extends ImportJob {export interface DataInsertJob extends ImportJob {chunk: UmamiEvent[];-}--export interface ImportCompletionJob {- importId: string;+ allChunksSent: boolean;}