🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor
Edward Cui9f3f194d5cd6
diff --git a/server/src/services/import/mappings/rybbit.ts b/server/src/services/import/mappings/rybbit.tsnew file mode 100644index 000000000..2168f0f2c--- /dev/null+++ b/server/src/services/import/mappings/rybbit.ts@@ -0,0 +1,35 @@+export interface RybbitEvent {+ site_id: number;+ timestamp: string;+ session_id: string;+ user_id: string;+ hostname: string;+ pathname: string;+ querystring: string;+ url_parameters: Record<string, string>;+ page_title: string;+ referrer: string;+ channel: string;+ browser: string;+ browser_version: string;+ operating_system: string;+ operating_system_version: string;+ language: string;+ country: string;+ region: string;+ city: string;+ lat: number;+ lon: number;+ screen_width: number;+ screen_height: number;+ device_type: string;+ type: string;+ event_name: string;+ props: Record<string, unknown>;+ lcp: number | null,+ cls: number | null,+ inp: number | null,+ fcp: number | null,+ ttfb: number | null,+ import_id: string | null,+}diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.tsindex 4d98ab777..10f74b8c3 100644--- a/server/src/services/import/mappings/umami.ts+++ b/server/src/services/import/mappings/umami.ts@@ -1,6 +1,6 @@-import { ImportMapper } from "../workers/jobs.js";import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";import { getChannel } from "../../tracker/getChannel.js";+import { RybbitEvent } from "./rybbit.js";export interface UmamiEvent {website_id: string; // Ignore@@ -84,8 +84,8 @@ export const umamiHeaders = ["job_id"];-export class UmamiImportMapper implements ImportMapper<UmamiEvent[]> {- transform(rows: UmamiEvent[], site: string, importId: string) {+export class UmamiImportMapper {+ transform(rows: UmamiEvent[], site: string, importId: string): RybbitEvent[] {return rows.map(row => {const querystring = row.url_query ? `?${row.url_query}` : "";const referrer = row.referrer_domain ? `https://${row.referrer_domain}` : "";diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 433e11785..c8fd919fd 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -129,7 +129,6 @@ export async function registerCsvParseWorker() {reject(error);});});-} catch (error) {console.error("Error in CSV parse worker:", error);await cleanup(error instanceof Error ? error.message : "Unknown error occurred");diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex 8262af87c..078b57b2a 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -1,12 +1,12 @@import boss from "../../../db/postgres/boss.js";-import { UmamiEvent, UmamiImportMapper } from "../mappings/umami.js";+import { UmamiImportMapper } from "../mappings/umami.js";import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { Job } from "pg-boss";import { ImportStatusManager } from "../importStatusManager.js";export async function registerDataInsertWorker() {- await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<DataInsertJob<UmamiEvent>>[]) => {+ await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([ job ]: Job<DataInsertJob>[]) => {const { site, importId, source, chunk, chunkNumber, finalChunk } = job.data;try {diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.tsindex 93931cd20..555e9bfac 100644--- a/server/src/services/import/workers/jobs.ts+++ b/server/src/services/import/workers/jobs.ts@@ -1,3 +1,5 @@+import { UmamiEvent } from "../mappings/umami.js";+export const CSV_PARSE_QUEUE = "csv-parse";export const DATA_INSERT_QUEUE = "data-insert";@@ -13,48 +15,8 @@ export interface CsvParseJob extends ImportJob {organization: string;}-export interface DataInsertJob<T> extends ImportJob {- chunk: T[];+export interface DataInsertJob extends ImportJob {+ chunk: UmamiEvent[];chunkNumber: number;finalChunk: boolean;}--export interface ImportMapper<T> {- transform(rows: T, site: string, importId: string): RybbitEvent[];-}--interface RybbitEvent {- site_id: number;- timestamp: string;- session_id: string;- user_id: string;- hostname: string;- pathname: string;- querystring: string;- url_parameters: Record<string, string>;- page_title: string;- referrer: string;- channel: string;- browser: string;- browser_version: string;- operating_system: string;- operating_system_version: string;- language: string;- country: string;- region: string;- city: string;- lat: number;- lon: number;- screen_width: number;- screen_height: number;- device_type: string;- type: string;- event_name: string;- props: Record<string, unknown>;- lcp: number | null,- cls: number | null,- inp: number | null,- fcp: number | null,- ttfb: number | null,- import_id: string | null,-}