🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
wip
Edward Cui7873ec3bb085
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsindex cd0fed0a7..994203756 100644--- a/server/src/api/sites/importSiteData.ts+++ b/server/src/api/sites/importSiteData.ts@@ -9,6 +9,8 @@ import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { CSV_PARSE_QUEUE } from "../../services/import/workers/jobs.js";import { ImportStatusManager } from "../../services/import/importStatusManager.js";import { ImportLimiter } from "../../services/import/importLimiter.js";+import { r2Storage } from "../../services/storage/r2StorageService.js";+import { IS_CLOUD } from "../../lib/const.js";const importDataRequestSchema = z.object({params: z.object({@@ -73,10 +75,7 @@ export async function importSiteData(}const organization = concurrentImportLimitResult.organizationId;- const importDir = "/tmp/imports"; // TODO: ./tmp/imports?const importId = crypto.randomUUID();- const savedFileName = `${importId}.csv`;- const tempFilePath = path.join(importDir, savedFileName);await ImportStatusManager.createImportStatus({importId,@@ -88,18 +87,43 @@ export async function importSiteData(fileSize: fileData.file.readableLength || 0,});+ let storageLocation: string;+try {- await fs.promises.mkdir(importDir, { recursive: true });- await pipeline(fileData.file, fs.createWriteStream(tempFilePath));+ if (IS_CLOUD && r2Storage.isEnabled()) {+ const r2Key = `imports/${importId}/${fileData.filename}`;++ const chunks: Buffer[] = [];+ for await (const chunk of fileData.file) {+ chunks.push(chunk);+ }+ const fileBuffer = Buffer.concat(chunks);++ await r2Storage.storeImportFile(r2Key, fileBuffer);+ storageLocation = r2Key;++ console.log(`[Import] File stored in R2: ${r2Key}`);+ } else {+ const importDir = "/tmp/imports";+ const savedFileName = `${importId}.csv`;+ const tempFilePath = path.join(importDir, savedFileName);++ await fs.promises.mkdir(importDir, { recursive: true });+ await pipeline(fileData.file, fs.createWriteStream(tempFilePath));+ storageLocation = tempFilePath;++ console.log(`[Import] File stored locally: ${tempFilePath}`);+ }} catch (fileError) {await ImportStatusManager.updateStatus(importId, "failed", "Failed to save uploaded file");- console.error("Failed to save uploaded file to disk:", fileError);+ console.error("Failed to save uploaded file:", fileError);return reply.status(500).send({ error: "Could not process file upload." });}try {await boss.send(CSV_PARSE_QUEUE, {- tempFilePath,+ storageLocation,+ isR2Storage: IS_CLOUD && r2Storage.isEnabled(),organization,site,importId,diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 85fe153c9..1ed20fb1a 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -1,33 +1,54 @@import fs from "fs";import { parse } from "@fast-csv/parse";+import { Readable } from "stream";import boss from "../../../db/postgres/boss.js";import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";import { Job } from "pg-boss";import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";import { ImportStatusManager } from "../importStatusManager.js";import { ImportLimiter } from "../importLimiter.js";+import { r2Storage } from "../../storage/r2StorageService.js";export async function registerCsvParseWorker() {await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {- const { site, importId, source, tempFilePath, organization } = job.data;+ const { site, importId, source, storageLocation, isR2Storage, organization } = job.data;try {const importableEvents = await ImportLimiter.countImportableEvents(organization);if (importableEvents <= 0) {await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");- await fs.promises.unlink(tempFilePath);++ if (isR2Storage) {+ await r2Storage.deleteImportFile(storageLocation);+ } else {+ await fs.promises.unlink(storageLocation);+ }return;}const chunkSize = 1000;let chunk: UmamiEvent[] = [];- const stream = fs.createReadStream(tempFilePath).pipe(parse({- headers: umamiHeaders,- renameHeaders: true,- ignoreEmpty: true,- maxRows: importableEvents,- }));+ let stream: Readable;++ if (isR2Storage) {+ console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);+ const fileBuffer = await r2Storage.getImportFile(storageLocation);+ stream = Readable.from(fileBuffer).pipe(parse({+ headers: umamiHeaders,+ renameHeaders: true,+ ignoreEmpty: true,+ maxRows: importableEvents,+ }));+ } else {+ console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);+ stream = fs.createReadStream(storageLocation).pipe(parse({+ headers: umamiHeaders,+ renameHeaders: true,+ ignoreEmpty: true,+ maxRows: importableEvents,+ }));+ }await ImportStatusManager.updateStatus(importId, "processing");@@ -72,7 +93,18 @@ export async function registerCsvParseWorker() {);throw error;} finally {- await fs.promises.unlink(tempFilePath);+ try {+ if (isR2Storage) {+ await r2Storage.deleteImportFile(storageLocation);+ console.log(`[CSV Parser] Cleaned up R2 file: ${storageLocation}`);+ } else {+ await fs.promises.unlink(storageLocation);+ console.log(`[CSV Parser] Cleaned up local file: ${storageLocation}`);+ }+ } catch (cleanupError) {+ console.error("Error cleaning up import file:", cleanupError);+ // Don't throw here as the main processing might have succeeded+ }}});}diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.tsindex d344b961d..17c2abbd2 100644--- a/server/src/services/import/workers/jobs.ts+++ b/server/src/services/import/workers/jobs.ts@@ -11,7 +11,8 @@ interface ImportJob {}export interface CsvParseJob extends ImportJob {- tempFilePath: string;+ storageLocation: string; // Either local file path or R2 key+ isR2Storage: boolean;organization: string;}diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.tsindex 4bebebdf4..7f7f546d4 100644--- a/server/src/services/storage/r2StorageService.ts+++ b/server/src/services/storage/r2StorageService.ts@@ -143,6 +143,96 @@ export class R2StorageService {// Non-critical error, log but don't throw}}++ /**+ * Store an import CSV file in R2+ * Used for temporary storage during import processing+ */+ async storeImportFile(key: string, fileBuffer: Buffer): Promise<void> {+ if (!this.enabled || !this.client) {+ throw new Error("R2 storage is not enabled");+ }++ try {+ await this.client.send(+ new PutObjectCommand({+ Bucket: this.bucketName,+ Key: key,+ Body: fileBuffer,+ ContentType: "text/csv",+ Metadata: {+ type: "import-file",+ uploadTime: Date.now().toString(),+ },+ }),+ );++ console.log(`[R2Storage] Successfully stored import file: ${key}`);+ } catch (error) {+ console.error("[R2Storage] Failed to store import file:", error);+ throw error;+ }+ }++ /**+ * Retrieve an import CSV file from R2+ * Returns the file buffer+ */+ async getImportFile(key: string): Promise<Buffer> {+ if (!this.enabled || !this.client) {+ throw new Error("R2 storage is not enabled");+ }++ try {+ const response = await this.client.send(+ new GetObjectCommand({+ Bucket: this.bucketName,+ Key: key,+ }),+ );++ if (!response.Body) {+ throw new Error("Empty response body");+ }++ // Convert stream to buffer+ const chunks: Uint8Array[] = [];+ const stream = response.Body as Readable;++ for await (const chunk of stream) {+ chunks.push(chunk);+ }++ const buffer = Buffer.concat(chunks);+ console.log(`[R2Storage] Successfully retrieved import file: ${key}`);+ return buffer;+ } catch (error) {+ console.error("[R2Storage] Failed to retrieve import file:", error);+ throw error;+ }+ }++ /**+ * Delete an import file from R2 (for cleanup after processing)+ */+ async deleteImportFile(key: string): Promise<void> {+ if (!this.enabled || !this.client) {+ return;+ }++ try {+ await this.client.send(+ new DeleteObjectCommand({+ Bucket: this.bucketName,+ Key: key,+ }),+ );+ console.log(`[R2Storage] Successfully deleted import file: ${key}`);+ } catch (error) {+ console.error("[R2Storage] Failed to delete import file:", error);+ // Non-critical error, log but don't throw+ }+ }}// Singleton instance