Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Various improvements

Edward Cuif27382c5af51
diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.ts
index 352cc149f..cdac32c98 100644
--- a/server/src/workers/importInitiation.ts
+++ b/server/src/workers/importInitiation.ts
@@ -3,44 +3,108 @@ import { parseStream } from "@fast-csv/parse";
import boss from "../lib/boss.js";
import { importInitiationQueue, ImportInitiationJob, processImportChunkQueue } from "../types/import.js";
import { Job } from "pg-boss";
+import { umamiHeaders } from "./mappings/umami.js";
await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {
const { tempFilePath, site, importId, source } = job[0].data;
const chunkSize = 1000;
+ const maxChunks = 1000; // 1 million rows limit
let chunkNumber = 0;
let chunk: any[] = [];
+ let shouldStop = false;
const stream = fs.createReadStream(tempFilePath);
+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {
+ if (arr1.length !== arr2.length) {
+ return false;
+ }
+ return arr1.every((value, index) => value === arr2[index]);
+ }
+
+ const validateHeaders = (headers: string[]): boolean => {
+ switch (source) {
+ case "umami":
+ return arraysAreEqual(umamiHeaders, headers);
+ default:
+ return false;
+ }
+ };
+
+ const cleanup = async (reason: string) => {
+ console.log(`πŸ›‘ Stopping CSV processing: ${reason}`);
+ shouldStop = true;
+
+ if (!stream.destroyed) {
+ stream.destroy();
+ }
+
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`πŸ—‘οΈ Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
+ }
+ };
+
parseStream(stream, { headers: true })
- .on("headers", headers => console.log(headers)) // validate headers
+ .on("headers", async (headers) => {
+ if (!validateHeaders(headers)) {
+ await cleanup("Header validation failed");
+ return;
+ }
+ })
.on("data", async (row) => {
+ if (shouldStop) {
+ return;
+ }
+
chunk.push(row);
if (chunk.length >= chunkSize) {
+ chunkNumber++;
+
+ if (chunkNumber > maxChunks) {
+ await cleanup(`Maximum chunks limit reached (${maxChunks} chunks / 1 million rows)`);
+ return;
+ }
+
await boss.send(processImportChunkQueue, {
+ site,
importId,
source,
- site,
- chunkNumber: ++chunkNumber,
- records: chunk,
+ chunk,
+ chunkNumber,
});
chunk = [];
}
})
.on("end", async (rowCount: number) => {
- if (chunk.length > 0) {
+ if (!shouldStop && chunk.length > 0) {
+ chunkNumber++;
+
await boss.send(processImportChunkQueue, {
+ site,
importId,
source,
- site,
- chunkNumber: ++chunkNumber,
- records: chunk,
+ chunk,
+ chunkNumber,
});
}
- console.log(`βœ… Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);
- })
- .on("error", error => console.error("CSV parsing error:", error));
- await fs.promises.unlink(tempFilePath);
+ if (!shouldStop) {
+ console.log(`βœ… Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);
+
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`πŸ—‘οΈ Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
+ }
+ }
+ })
+ .on("error", async (error) => {
+ console.error("CSV parsing error:", error);
+ await cleanup("CSV parsing error occurred");
+ });
});
diff --git a/server/src/workers/mappings/umami.ts b/server/src/workers/mappings/umami.ts
index d55d4a0c9..f939141d8 100644
--- a/server/src/workers/mappings/umami.ts
+++ b/server/src/workers/mappings/umami.ts
@@ -1,4 +1,4 @@
-import { ImportMapping } from "../../types/import.js";
+import { ImportMapper } from "../../types/import.js";
interface UmamiEvent {
website_id: string; // Ignore
@@ -43,7 +43,46 @@ interface UmamiEvent {
job_id: string | null; // Ignore
}
-export class UmamiImportMapping implements ImportMapping<UmamiEvent> {
+export const umamiHeaders = [
+ "website_id",
+ "session_id",
+ "visit_id",
+ "event_id",
+ "hostname",
+ "browser",
+ "os",
+ "device",
+ "screen",
+ "language",
+ "country",
+ "region",
+ "city",
+ "url_path",
+ "url_query",
+ "utm_source",
+ "utm_medium",
+ "utm_campaign",
+ "utm_content",
+ "utm_term",
+ "referrer_path",
+ "referrer_query",
+ "referrer_domain",
+ "page_title",
+ "gclid",
+ "fbclid",
+ "msclkid",
+ "ttclid",
+ "li_fat_id",
+ "twclid",
+ "event_type",
+ "event_name",
+ "tag",
+ "distinct_id",
+ "created_at",
+ "job_id"
+];
+
+export class UmamiImportMapper implements ImportMapper<UmamiEvent> {
transform(row: UmamiEvent, headers: string[]): any {
const [screenWidth, screenHeight] = row.screen?.split("x") || [null, null];