🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor
Edward Cuiaefb63f3a53a
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsindex 3d5a38843..f9c51e999 100644--- a/server/src/api/sites/importSiteData.ts+++ b/server/src/api/sites/importSiteData.ts@@ -7,8 +7,8 @@ import crypto from "crypto";import boss from "../../db/postgres/boss.js";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { CSV_PARSE_QUEUE } from "../../types/import.js";-import { ImportRateLimiter } from "../../lib/rateLimiter.js";-import { ImportStatusManager } from "../../lib/importStatus.js";+import { ImportStatusManager } from "../../services/import/importStatusManager.js";+import { ImportLimiter } from "../../services/import/importLimiter.js";const importDataRequestSchema = z.object({params: z.object({@@ -50,7 +50,7 @@ export async function importSiteData(return reply.status(403).send({ error: "Forbidden" });}- const concurrentImportLimitResult = await ImportRateLimiter.checkConcurrentImportLimit(organization);+ const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(organization);if (!concurrentImportLimitResult.allowed) {return reply.status(429).send({error: "Organization limit exceeded",diff --git a/server/src/api/sites/listSiteImports.ts b/server/src/api/sites/listSiteImports.tsindex 423f0720d..97b8fbb30 100644--- a/server/src/api/sites/listSiteImports.ts+++ b/server/src/api/sites/listSiteImports.ts@@ -1,6 +1,6 @@import { FastifyReply, FastifyRequest } from "fastify";import { getUserHasAccessToSite } from "../../lib/auth-utils.js";-import { ImportStatusManager } from "../../lib/importStatus.js";+import { ImportStatusManager } from "../../services/import/importStatusManager.js";import { z } from "zod";const listSiteImportsRequestSchema = z.object({diff --git a/server/src/lib/rateLimiter.ts b/server/src/lib/rateLimiter.tsindex c47e92930..697770bce 100644--- a/server/src/lib/rateLimiter.ts+++ b/server/src/lib/rateLimiter.ts@@ -1,7 +1,4 @@import { IS_CLOUD } from "./const.js";-import { db } from "../db/postgres/postgres.js";-import { eq, and, inArray, count, sum, isNotNull } from "drizzle-orm";-import { importStatus } from "../db/postgres/schema.js";// In-memory rate limiter for API keysclass ApiKeyRateLimiter {@@ -54,54 +51,3 @@ if (IS_CLOUD) {apiKeyRateLimiter.cleanup();}, 5 * 60 * 1000);}--export class ImportRateLimiter {- private static readonly IMPORTED_EVENTS_LIMIT = 1_000_000;- private static readonly CONCURRENT_LIMIT = 1;-- static async checkConcurrentImportLimit(organizationId: string): Promise<{- allowed: boolean;- reason?: string;- }> {- if (!IS_CLOUD) {- return { allowed: true };- }-- const [concurrentCountResult] = await db- .select({ count: count() })- .from(importStatus)- .where(- and(- eq(importStatus.organizationId, organizationId),- inArray(importStatus.status, ["pending", "processing"])- )- );-- if (concurrentCountResult.count >= this.CONCURRENT_LIMIT) {- return {- allowed: false,- reason: `Only ${this.CONCURRENT_LIMIT} concurrent import allowed per organization.`,- };- }-- return { allowed: true };- }-- static async countImportableEvents(organizationId: string): Promise<number> {- if (!IS_CLOUD) {- return Infinity;- }-- const [importedEvents] = await db- .select({ total: sum(importStatus.importedEvents) })- .from(importStatus)- .where(- and(- eq(importStatus.organizationId, organizationId),- isNotNull(importStatus.importedEvents)- )- );-- return this.IMPORTED_EVENTS_LIMIT - Number(importedEvents.total ?? 0);- }-}diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.tsnew file mode 100644index 000000000..6a987a1be--- /dev/null+++ b/server/src/services/import/importLimiter.ts@@ -0,0 +1,55 @@+import { IS_CLOUD } from "../../lib/const.js";+import { db } from "../../db/postgres/postgres.js";+import { and, count, eq, inArray, isNotNull, sum } from "drizzle-orm";+import { importStatus } from "../../db/postgres/schema.js";++export class ImportLimiter {+ private static readonly IMPORTED_EVENTS_LIMIT = 1_000_000;+ private static readonly CONCURRENT_LIMIT = 1;++ static async checkConcurrentImportLimit(organizationId: string): Promise<{+ allowed: boolean;+ reason?: string;+ }> {+ if (!IS_CLOUD) {+ return { allowed: true };+ }++ const [concurrentCountResult] = await db+ .select({ count: count() })+ .from(importStatus)+ .where(+ and(+ eq(importStatus.organizationId, organizationId),+ inArray(importStatus.status, ["pending", "processing"])+ )+ );++ if (concurrentCountResult.count >= this.CONCURRENT_LIMIT) {+ return {+ allowed: false,+ reason: `Only ${this.CONCURRENT_LIMIT} concurrent import allowed per organization.`,+ };+ }++ return { allowed: true };+ }++ static async countImportableEvents(organizationId: string): Promise<number> {+ if (!IS_CLOUD) {+ return Infinity;+ }++ const [importedEvents] = await db+ .select({ total: sum(importStatus.importedEvents) })+ .from(importStatus)+ .where(+ and(+ eq(importStatus.organizationId, organizationId),+ isNotNull(importStatus.importedEvents)+ )+ );++ return this.IMPORTED_EVENTS_LIMIT - Number(importedEvents.total ?? 0);+ }+}diff --git a/server/src/lib/importStatus.ts b/server/src/services/import/importStatusManager.tssimilarity index 91%rename from server/src/lib/importStatus.tsrename to server/src/services/import/importStatusManager.tsindex 125dc606d..a7db25412 100644--- a/server/src/lib/importStatus.ts+++ b/server/src/services/import/importStatusManager.ts@@ -1,6 +1,6 @@-import { db } from "../db/postgres/postgres.js";+import { db } from "../../db/postgres/postgres.js";import { eq, desc, sql } from "drizzle-orm";-import { importStatus } from "../db/postgres/schema.js";+import { importStatus } from "../../db/postgres/schema.js";type SelectImportStatus = typeof importStatus.$inferSelect;type InsertImportStatus = typeof importStatus.$inferInsert;diff --git a/server/src/workers/csvParseWorker.ts b/server/src/workers/csvParseWorker.tsindex a9ca73aad..a5d0b686c 100644--- a/server/src/workers/csvParseWorker.ts+++ b/server/src/workers/csvParseWorker.ts@@ -4,8 +4,8 @@ import boss from "../db/postgres/boss.js";import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "../types/import.js";import { Job } from "pg-boss";import { UmamiEvent, umamiHeaders } from "./mappings/umami.js";-import { ImportStatusManager } from "../lib/importStatus.js";-import { ImportRateLimiter } from "../lib/rateLimiter.js";+import { ImportStatusManager } from "../services/import/importStatusManager.js";+import { ImportLimiter } from "../services/import/importLimiter.js";export async function registerCsvParseWorker() {await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {@@ -27,7 +27,7 @@ export async function registerCsvParseWorker() {};try {- const importableEvents = await ImportRateLimiter.countImportableEvents(organization);+ const importableEvents = await ImportLimiter.countImportableEvents(organization);if (importableEvents <= 0) {await cleanup("Event import limit reached");return;diff --git a/server/src/workers/dataInsertWorker.ts b/server/src/workers/dataInsertWorker.tsindex a63861e3d..cca1ba80f 100644--- a/server/src/workers/dataInsertWorker.ts+++ b/server/src/workers/dataInsertWorker.ts@@ -3,7 +3,7 @@ import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";import { DataInsertJob, DATA_INSERT_QUEUE } from "../types/import.js";import { clickhouse } from "../db/clickhouse/clickhouse.js";import { Job } from "pg-boss";-import { ImportStatusManager } from "../lib/importStatus.js";+import { ImportStatusManager } from "../services/import/importStatusManager.js";export async function registerDataInsertWorker() {await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<DataInsertJob<UmamiEvent>>[]) => {