Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor

Edward Cui9f3f194d5cd6
diff --git a/server/src/services/import/mappings/rybbit.ts b/server/src/services/import/mappings/rybbit.ts
new file mode 100644
index 000000000..2168f0f2c
--- /dev/null
+++ b/server/src/services/import/mappings/rybbit.ts
@@ -0,0 +1,35 @@
+export interface RybbitEvent {
+ site_id: number;
+ timestamp: string;
+ session_id: string;
+ user_id: string;
+ hostname: string;
+ pathname: string;
+ querystring: string;
+ url_parameters: Record<string, string>;
+ page_title: string;
+ referrer: string;
+ channel: string;
+ browser: string;
+ browser_version: string;
+ operating_system: string;
+ operating_system_version: string;
+ language: string;
+ country: string;
+ region: string;
+ city: string;
+ lat: number;
+ lon: number;
+ screen_width: number;
+ screen_height: number;
+ device_type: string;
+ type: string;
+ event_name: string;
+ props: Record<string, unknown>;
+ lcp: number | null,
+ cls: number | null,
+ inp: number | null,
+ fcp: number | null,
+ ttfb: number | null,
+ import_id: string | null,
+}
diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.ts
index 4d98ab777..10f74b8c3 100644
--- a/server/src/services/import/mappings/umami.ts
+++ b/server/src/services/import/mappings/umami.ts
@@ -1,6 +1,6 @@
-import { ImportMapper } from "../workers/jobs.js";
import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";
import { getChannel } from "../../tracker/getChannel.js";
+import { RybbitEvent } from "./rybbit.js";
export interface UmamiEvent {
website_id: string; // Ignore
@@ -84,8 +84,8 @@ export const umamiHeaders = [
"job_id"
];
-export class UmamiImportMapper implements ImportMapper<UmamiEvent[]> {
- transform(rows: UmamiEvent[], site: string, importId: string) {
+export class UmamiImportMapper {
+ transform(rows: UmamiEvent[], site: string, importId: string): RybbitEvent[] {
return rows.map(row => {
const querystring = row.url_query ? `?${row.url_query}` : "";
const referrer = row.referrer_domain ? `https://${row.referrer_domain}` : "";
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 433e11785..c8fd919fd 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -129,7 +129,6 @@ export async function registerCsvParseWorker() {
reject(error);
});
});
-
} catch (error) {
console.error("Error in CSV parse worker:", error);
await cleanup(error instanceof Error ? error.message : "Unknown error occurred");
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 8262af87c..078b57b2a 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -1,12 +1,12 @@
import boss from "../../../db/postgres/boss.js";
-import { UmamiEvent, UmamiImportMapper } from "../mappings/umami.js";
+import { UmamiImportMapper } from "../mappings/umami.js";
import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { Job } from "pg-boss";
import { ImportStatusManager } from "../importStatusManager.js";
export async function registerDataInsertWorker() {
- await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<DataInsertJob<UmamiEvent>>[]) => {
+ await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([ job ]: Job<DataInsertJob>[]) => {
const { site, importId, source, chunk, chunkNumber, finalChunk } = job.data;
try {
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
index 93931cd20..555e9bfac 100644
--- a/server/src/services/import/workers/jobs.ts
+++ b/server/src/services/import/workers/jobs.ts
@@ -1,3 +1,5 @@
+import { UmamiEvent } from "../mappings/umami.js";
+
export const CSV_PARSE_QUEUE = "csv-parse";
export const DATA_INSERT_QUEUE = "data-insert";
@@ -13,48 +15,8 @@ export interface CsvParseJob extends ImportJob {
organization: string;
}
-export interface DataInsertJob<T> extends ImportJob {
- chunk: T[];
+export interface DataInsertJob extends ImportJob {
+ chunk: UmamiEvent[];
chunkNumber: number;
finalChunk: boolean;
}
-
-export interface ImportMapper<T> {
- transform(rows: T, site: string, importId: string): RybbitEvent[];
-}
-
-interface RybbitEvent {
- site_id: number;
- timestamp: string;
- session_id: string;
- user_id: string;
- hostname: string;
- pathname: string;
- querystring: string;
- url_parameters: Record<string, string>;
- page_title: string;
- referrer: string;
- channel: string;
- browser: string;
- browser_version: string;
- operating_system: string;
- operating_system_version: string;
- language: string;
- country: string;
- region: string;
- city: string;
- lat: number;
- lon: number;
- screen_width: number;
- screen_height: number;
- device_type: string;
- type: string;
- event_name: string;
- props: Record<string, unknown>;
- lcp: number | null,
- cls: number | null,
- inp: number | null,
- fcp: number | null,
- ttfb: number | null,
- import_id: string | null,
-}