Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor

Edward Cuiaefb63f3a53a
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
index 3d5a38843..f9c51e999 100644
--- a/server/src/api/sites/importSiteData.ts
+++ b/server/src/api/sites/importSiteData.ts
@@ -7,8 +7,8 @@ import crypto from "crypto";
import boss from "../../db/postgres/boss.js";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { CSV_PARSE_QUEUE } from "../../types/import.js";
-import { ImportRateLimiter } from "../../lib/rateLimiter.js";
-import { ImportStatusManager } from "../../lib/importStatus.js";
+import { ImportStatusManager } from "../../services/import/importStatusManager.js";
+import { ImportLimiter } from "../../services/import/importLimiter.js";
const importDataRequestSchema = z.object({
params: z.object({
@@ -50,7 +50,7 @@ export async function importSiteData(
return reply.status(403).send({ error: "Forbidden" });
}
- const concurrentImportLimitResult = await ImportRateLimiter.checkConcurrentImportLimit(organization);
+ const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(organization);
if (!concurrentImportLimitResult.allowed) {
return reply.status(429).send({
error: "Organization limit exceeded",
diff --git a/server/src/api/sites/listSiteImports.ts b/server/src/api/sites/listSiteImports.ts
index 423f0720d..97b8fbb30 100644
--- a/server/src/api/sites/listSiteImports.ts
+++ b/server/src/api/sites/listSiteImports.ts
@@ -1,6 +1,6 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { getUserHasAccessToSite } from "../../lib/auth-utils.js";
-import { ImportStatusManager } from "../../lib/importStatus.js";
+import { ImportStatusManager } from "../../services/import/importStatusManager.js";
import { z } from "zod";
const listSiteImportsRequestSchema = z.object({
diff --git a/server/src/lib/rateLimiter.ts b/server/src/lib/rateLimiter.ts
index c47e92930..697770bce 100644
--- a/server/src/lib/rateLimiter.ts
+++ b/server/src/lib/rateLimiter.ts
@@ -1,7 +1,4 @@
import { IS_CLOUD } from "./const.js";
-import { db } from "../db/postgres/postgres.js";
-import { eq, and, inArray, count, sum, isNotNull } from "drizzle-orm";
-import { importStatus } from "../db/postgres/schema.js";
// In-memory rate limiter for API keys
class ApiKeyRateLimiter {
@@ -54,54 +51,3 @@ if (IS_CLOUD) {
apiKeyRateLimiter.cleanup();
}, 5 * 60 * 1000);
}
-
-export class ImportRateLimiter {
- private static readonly IMPORTED_EVENTS_LIMIT = 1_000_000;
- private static readonly CONCURRENT_LIMIT = 1;
-
- static async checkConcurrentImportLimit(organizationId: string): Promise<{
- allowed: boolean;
- reason?: string;
- }> {
- if (!IS_CLOUD) {
- return { allowed: true };
- }
-
- const [concurrentCountResult] = await db
- .select({ count: count() })
- .from(importStatus)
- .where(
- and(
- eq(importStatus.organizationId, organizationId),
- inArray(importStatus.status, ["pending", "processing"])
- )
- );
-
- if (concurrentCountResult.count >= this.CONCURRENT_LIMIT) {
- return {
- allowed: false,
- reason: `Only ${this.CONCURRENT_LIMIT} concurrent import allowed per organization.`,
- };
- }
-
- return { allowed: true };
- }
-
- static async countImportableEvents(organizationId: string): Promise<number> {
- if (!IS_CLOUD) {
- return Infinity;
- }
-
- const [importedEvents] = await db
- .select({ total: sum(importStatus.importedEvents) })
- .from(importStatus)
- .where(
- and(
- eq(importStatus.organizationId, organizationId),
- isNotNull(importStatus.importedEvents)
- )
- );
-
- return this.IMPORTED_EVENTS_LIMIT - Number(importedEvents.total ?? 0);
- }
-}
diff --git a/server/src/services/import/importLimiter.ts b/server/src/services/import/importLimiter.ts
new file mode 100644
index 000000000..6a987a1be
--- /dev/null
+++ b/server/src/services/import/importLimiter.ts
@@ -0,0 +1,55 @@
+import { IS_CLOUD } from "../../lib/const.js";
+import { db } from "../../db/postgres/postgres.js";
+import { and, count, eq, inArray, isNotNull, sum } from "drizzle-orm";
+import { importStatus } from "../../db/postgres/schema.js";
+
+export class ImportLimiter {
+ private static readonly IMPORTED_EVENTS_LIMIT = 1_000_000;
+ private static readonly CONCURRENT_LIMIT = 1;
+
+ static async checkConcurrentImportLimit(organizationId: string): Promise<{
+ allowed: boolean;
+ reason?: string;
+ }> {
+ if (!IS_CLOUD) {
+ return { allowed: true };
+ }
+
+ const [concurrentCountResult] = await db
+ .select({ count: count() })
+ .from(importStatus)
+ .where(
+ and(
+ eq(importStatus.organizationId, organizationId),
+ inArray(importStatus.status, ["pending", "processing"])
+ )
+ );
+
+ if (concurrentCountResult.count >= this.CONCURRENT_LIMIT) {
+ return {
+ allowed: false,
+ reason: `Only ${this.CONCURRENT_LIMIT} concurrent import allowed per organization.`,
+ };
+ }
+
+ return { allowed: true };
+ }
+
+ static async countImportableEvents(organizationId: string): Promise<number> {
+ if (!IS_CLOUD) {
+ return Infinity;
+ }
+
+ const [importedEvents] = await db
+ .select({ total: sum(importStatus.importedEvents) })
+ .from(importStatus)
+ .where(
+ and(
+ eq(importStatus.organizationId, organizationId),
+ isNotNull(importStatus.importedEvents)
+ )
+ );
+
+ return this.IMPORTED_EVENTS_LIMIT - Number(importedEvents.total ?? 0);
+ }
+}
diff --git a/server/src/lib/importStatus.ts b/server/src/services/import/importStatusManager.ts
similarity index 91%
rename from server/src/lib/importStatus.ts
rename to server/src/services/import/importStatusManager.ts
index 125dc606d..a7db25412 100644
--- a/server/src/lib/importStatus.ts
+++ b/server/src/services/import/importStatusManager.ts
@@ -1,6 +1,6 @@
-import { db } from "../db/postgres/postgres.js";
+import { db } from "../../db/postgres/postgres.js";
import { eq, desc, sql } from "drizzle-orm";
-import { importStatus } from "../db/postgres/schema.js";
+import { importStatus } from "../../db/postgres/schema.js";
type SelectImportStatus = typeof importStatus.$inferSelect;
type InsertImportStatus = typeof importStatus.$inferInsert;
diff --git a/server/src/workers/csvParseWorker.ts b/server/src/workers/csvParseWorker.ts
index a9ca73aad..a5d0b686c 100644
--- a/server/src/workers/csvParseWorker.ts
+++ b/server/src/workers/csvParseWorker.ts
@@ -4,8 +4,8 @@ import boss from "../db/postgres/boss.js";
import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "../types/import.js";
import { Job } from "pg-boss";
import { UmamiEvent, umamiHeaders } from "./mappings/umami.js";
-import { ImportStatusManager } from "../lib/importStatus.js";
-import { ImportRateLimiter } from "../lib/rateLimiter.js";
+import { ImportStatusManager } from "../services/import/importStatusManager.js";
+import { ImportLimiter } from "../services/import/importLimiter.js";
export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
@@ -27,7 +27,7 @@ export async function registerCsvParseWorker() {
};
try {
- const importableEvents = await ImportRateLimiter.countImportableEvents(organization);
+ const importableEvents = await ImportLimiter.countImportableEvents(organization);
if (importableEvents <= 0) {
await cleanup("Event import limit reached");
return;
diff --git a/server/src/workers/dataInsertWorker.ts b/server/src/workers/dataInsertWorker.ts
index a63861e3d..cca1ba80f 100644
--- a/server/src/workers/dataInsertWorker.ts
+++ b/server/src/workers/dataInsertWorker.ts
@@ -3,7 +3,7 @@ import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";
import { DataInsertJob, DATA_INSERT_QUEUE } from "../types/import.js";
import { clickhouse } from "../db/clickhouse/clickhouse.js";
import { Job } from "pg-boss";
-import { ImportStatusManager } from "../lib/importStatus.js";
+import { ImportStatusManager } from "../services/import/importStatusManager.js";
export async function registerDataInsertWorker() {
await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<DataInsertJob<UmamiEvent>>[]) => {