Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Merge branch 'master' into umami-import

Edward Cui2a37416b46e4
diff --cc server/src/db/postgres/schema.ts
index 523986e78,8aeae61be..b47222555
--- a/server/src/db/postgres/schema.ts
+++ b/server/src/db/postgres/schema.ts
@@@ -8,8 -8,7 +8,9 @@@ import
timestamp,
foreignKey,
unique,
+ real,
+ bigint,
+ pgEnum,
} from "drizzle-orm/pg-core";
// User table
@@@ -252,46 -248,15 +250,56 @@@ export const goals = pgTable
foreignColumns: [sites.siteId],
name: "goals_site_id_sites_site_id_fk",
}),
- ]
+ ],
);
+ // Telemetry table for tracking self-hosted instances
+ export const telemetry = pgTable("telemetry", {
+ id: serial("id").primaryKey().notNull(),
+ instanceId: text("instance_id").notNull(),
+ timestamp: timestamp("timestamp", { mode: "string" }).notNull().defaultNow(),
+ version: text("version").notNull(),
+ tableCounts: jsonb("table_counts").notNull().$type<Record<string, number>>(),
+ clickhouseSizeGb: real("clickhouse_size_gb").notNull(),
+ });
++
+const importSourceEnum = pgEnum("import_source_enum", [
+ "umami",
+]);
+
+const importStatusEnum = pgEnum("import_status_enum", [
+ "pending",
+ "processing",
+ "completed",
+ "failed",
+]);
+
+// Import status table for tracking data import progress
+export const importStatus = pgTable(
+ "import_status",
+ {
+ importId: text("import_id").primaryKey().notNull(),
+ siteId: integer("site_id").notNull(),
+ organizationId: text("organization_id").notNull(),
+ source: importSourceEnum("source").notNull(),
+ status: importStatusEnum("status").notNull().default("pending"),
+ importedEvents: integer("imported_events").notNull().default(0),
+ errorMessage: text("error_message"),
+ startedAt: timestamp("started_at").notNull().defaultNow(),
+ completedAt: timestamp("completed_at"),
+ fileName: text("file_name").notNull(),
+ fileSize: bigint("file_size", { mode: "number" }).notNull(),
+ },
+ (table) => [
+ foreignKey({
+ columns: [table.siteId],
+ foreignColumns: [sites.siteId],
+ name: "import_status_site_id_sites_site_id_fk",
+ }),
+ foreignKey({
+ columns: [table.organizationId],
+ foreignColumns: [organization.id],
+ name: "import_status_organization_id_organization_id_fk",
+ }),
+ ]
+);
diff --cc server/src/index.ts
index 4e854278c,a2da04148..557d12224
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@@ -67,13 -69,10 +70,15 @@@ import { getSessionFromReq, mapHeaders
import { auth } from "./lib/auth.js";
import { IS_CLOUD } from "./lib/const.js";
import { siteConfig } from "./lib/siteConfig.js";
- import { trackEvent } from "./tracker/trackEvent.js";
+ import { trackEvent } from "./services/tracker/trackEvent.js";
+ // need to import telemetry service here to start it
+ import { telemetryService } from "./services/telemetryService.js";
import { extractSiteId, isSitePublic } from "./utils.js";
+import { importData } from "./api/import/importData.js";
+import boss from "./lib/boss.js";
+import { registerCsvParseWorker } from "./workers/csvParseWorker.js";
+import { registerDataInsertWorker } from "./workers/dataInsertWorker.js";
+import { listSiteImports } from "./api/import/listSiteImports.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@@ -340,17 -312,10 +331,14 @@@ server.get("/api/health", { logLevel: "
const start = async () => {
try {
console.info("Starting server...");
- // Initialize the database
- await Promise.all([initializeClickhouse()]);
- await loadAllowedDomains();
+ await Promise.all([initializeClickhouse(), loadAllowedDomains(), siteConfig.loadSiteConfigs(), initPostgres()]);
- // Load site configurations cache
- await siteConfig.loadSiteConfigs();
+ telemetryService.startTelemetryCron();
+ await boss.start();
+ await registerCsvParseWorker();
+ await registerDataInsertWorker();
+
// Start the server
await server.listen({ port: 3001, host: "0.0.0.0" });
} catch (err) {