Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Edward Cui5ba91d3bb469
diff --git a/server/src/index.ts b/server/src/index.ts
index 6abe1e628..c62b38d02 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -79,7 +79,6 @@ import boss from "./db/postgres/boss.js";
import { registerCsvParseWorker } from "./services/import/workers/csvParseWorker.js";
import { registerDataInsertWorker } from "./services/import/workers/dataInsertWorker.js";
import { listSiteImports } from "./api/sites/listSiteImports.js";
-import { registerImportCompletionWorker } from "./services/import/workers/importCompletionWorker.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -337,7 +336,6 @@ const start = async () => {
await boss.start();
await registerCsvParseWorker();
await registerDataInsertWorker();
- await registerImportCompletionWorker();
// Start the server
await server.listen({ port: 3001, host: "0.0.0.0" });
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 388dd8235..85fe153c9 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -1,7 +1,7 @@
import fs from "fs";
import { parse } from "@fast-csv/parse";
import boss from "../../../db/postgres/boss.js";
-import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE, IMPORT_COMPLETION_QUEUE } from "./jobs.js";
+import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { Job } from "pg-boss";
import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
import { ImportStatusManager } from "../importStatusManager.js";
@@ -40,6 +40,7 @@ export async function registerCsvParseWorker() {
importId,
source,
chunk,
+ allChunksSent: false,
});
chunk = [];
}
@@ -51,11 +52,16 @@ export async function registerCsvParseWorker() {
importId,
source,
chunk,
+ allChunksSent: false,
});
}
- await boss.send(IMPORT_COMPLETION_QUEUE, {
+ await boss.send(DATA_INSERT_QUEUE, {
+ site,
importId,
+ source,
+ chunk: [],
+ allChunksSent: true,
});
} catch (error) {
console.error("Error in CSV parse worker:", error);
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 46fa60fe1..cb1342bf4 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -16,7 +16,19 @@ const getImportDataMapping = (source: string) => {
export async function registerDataInsertWorker() {
await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([ job ]: Job<DataInsertJob>[]) => {
- const { site, importId, source, chunk } = job.data;
+ const { site, importId, source, chunk, allChunksSent } = job.data;
+
+ if (allChunksSent) {
+ try {
+ await ImportStatusManager.updateStatus(importId, "completed");
+ console.log(`Import ${importId} completed`);
+ return;
+ } catch (error) {
+ console.error(`Failed to mark import ${importId} as completed:`, error);
+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");
+ throw error;
+ }
+ }
try {
const dataMapper = getImportDataMapping(source);
diff --git a/server/src/services/import/workers/importCompletionWorker.ts b/server/src/services/import/workers/importCompletionWorker.ts
deleted file mode 100644
index dce68931a..000000000
--- a/server/src/services/import/workers/importCompletionWorker.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import boss from "../../../db/postgres/boss.js";
-import { IMPORT_COMPLETION_QUEUE, ImportCompletionJob } from "./jobs.js";
-import { Job } from "pg-boss";
-import { ImportStatusManager } from "../importStatusManager.js";
-
-export async function registerImportCompletionWorker() {
- await boss.work(IMPORT_COMPLETION_QUEUE, { batchSize: 1, pollingIntervalSeconds: 30 }, async ([ job ]: Job<ImportCompletionJob>[]) => {
- const { importId } = job.data;
-
- try {
- await ImportStatusManager.updateStatus(importId, "completed");
- console.log(`Import ${importId} completed`);
- } catch (error) {
- console.error(`Failed to mark import ${importId} as completed:`, error);
- await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");
- throw error;
- }
- });
-}
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
index c0c25847b..d344b961d 100644
--- a/server/src/services/import/workers/jobs.ts
+++ b/server/src/services/import/workers/jobs.ts
@@ -4,8 +4,6 @@ export const CSV_PARSE_QUEUE = "csv-parse";
export const DATA_INSERT_QUEUE = "data-insert";
-export const IMPORT_COMPLETION_QUEUE = "import-completion";
-
interface ImportJob {
site: string;
importId: string;
@@ -19,8 +17,5 @@ export interface CsvParseJob extends ImportJob {
export interface DataInsertJob extends ImportJob {
chunk: UmamiEvent[];
-}
-
-export interface ImportCompletionJob {
- importId: string;
+ allChunksSent: boolean;
}