Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Edward Cui8d3a3a6f7984
diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.ts
index f15ca5657..f60cab7d2 100644
--- a/server/src/services/import/mappings/umami.ts
+++ b/server/src/services/import/mappings/umami.ts
@@ -4,10 +4,10 @@ import { RybbitEvent } from "./rybbit.js";
import { z } from "zod";
export interface UmamiEvent {
- website_id: string; // Ignore
+ // website_id: string; // Ignore
session_id: string;
- visit_id: string; // Ignore
- event_id: string; // Ignore
+ // visit_id: string; // Ignore
+ // event_id: string; // Ignore
hostname: string;
browser: string;
@@ -21,36 +21,36 @@ export interface UmamiEvent {
url_path: string;
url_query: string;
- utm_source: string; // Ignore, part of url_query
- utm_medium: string; // Ignore, part of url_query
- utm_campaign: string; // Ignore, part of url_query
- utm_content: string; // Ignore, part of url_query
- utm_term: string; // Ignore, part of url_query
+ // utm_source: string; // Ignore, part of url_query
+ // utm_medium: string; // Ignore, part of url_query
+ // utm_campaign: string; // Ignore, part of url_query
+ // utm_content: string; // Ignore, part of url_query
+ // utm_term: string; // Ignore, part of url_query
referrer_path: string;
referrer_query: string;
referrer_domain: string;
page_title: string;
- gclid: string; // Ignore, part of url_query
- fbclid: string; // Ignore, part of url_query
- msclkid: string; // Ignore, part of url_query
- ttclid: string; // Ignore, part of url_query
- li_fat_id: string; // Ignore, part of url_query
- twclid: string; // Ignore, part of url_query
+ // gclid: string; // Ignore, part of url_query
+ // fbclid: string; // Ignore, part of url_query
+ // msclkid: string; // Ignore, part of url_query
+ // ttclid: string; // Ignore, part of url_query
+ // li_fat_id: string; // Ignore, part of url_query
+ // twclid: string; // Ignore, part of url_query
event_type: number;
event_name: string;
- tag: string; // Ignore
+ // tag: string; // Ignore
distinct_id: string;
created_at: string;
- job_id: string | null; // Ignore
+ // job_id: string | null; // Ignore
}
export const umamiHeaders = [
- "website_id",
+ undefined,
"session_id",
- "visit_id",
- "event_id",
+ undefined,
+ undefined,
"hostname",
"browser",
"os",
@@ -62,27 +62,27 @@ export const umamiHeaders = [
"city",
"url_path",
"url_query",
- "utm_source",
- "utm_medium",
- "utm_campaign",
- "utm_content",
- "utm_term",
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
"referrer_path",
"referrer_query",
"referrer_domain",
"page_title",
- "gclid",
- "fbclid",
- "msclkid",
- "ttclid",
- "li_fat_id",
- "twclid",
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
"event_type",
"event_name",
- "tag",
+ undefined,
"distinct_id",
"created_at",
- "job_id"
+ undefined,
];
export class UmamiImportMapper {
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 4922ab5b8..8d6688af5 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -11,20 +11,6 @@ export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
const { site, importId, source, tempFilePath, organization } = job.data;
- const validHeaders = (headers: string[]): boolean => {
- const arraysAreEqual = (arr1: string[], arr2: string[]) => {
- return arr1.length === arr2.length &&
- arr1.every((value, index) => value === arr2[index]);
- };
-
- switch (source) {
- case "umami":
- return arraysAreEqual(umamiHeaders, headers);
- default:
- return false;
- }
- };
-
try {
const importableEvents = await ImportLimiter.countImportableEvents(organization);
if (importableEvents <= 0) {
@@ -42,23 +28,12 @@ export async function registerCsvParseWorker() {
await ImportStatusManager.updateStatus(importId, "processing");
const csvStream = parseStream(stream, {
- headers: true,
- maxRows: importableEvents,
- discardUnmappedColumns: true,
+ headers: umamiHeaders,
+ // maxRows: importableEvents,
+ // strictColumnHandling: true,
ignoreEmpty: true
});
- await new Promise<void>((resolve, reject) => {
- csvStream.once("headers", (headers) => {
- if (!validHeaders(headers)) {
- reject(new Error(`Invalid ${source} headers`));
- return;
- }
- resolve();
- });
- csvStream.once("error", reject);
- });
-
for await (const data of csvStream) {
if (rowsProcessed >= importableEvents) {
break;