🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Better logging
Edward Cui5121c654b147
diff --git a/server/src/services/import/importCleanupService.ts b/server/src/services/import/importCleanupService.tsindex b3118fcf3..e61fb2b87 100644--- a/server/src/services/import/importCleanupService.ts+++ b/server/src/services/import/importCleanupService.ts@@ -1,5 +1,8 @@import * as cron from "node-cron";import { r2Storage } from "../storage/r2StorageService.js";+import { createServiceLogger } from "../../lib/logger/logger.js";++const logger = createServiceLogger("import:cleanup");class ImportCleanupService {private cleanupTask: cron.ScheduledTask | null = null;@@ -11,7 +14,7 @@ class ImportCleanupService {* Runs daily at 2 AM UTC to clean up orphaned import files.*/initializeCleanupCron() {- console.info("[ImportCleanup] Initializing cleanup cron");+ logger.info("Initializing cleanup cron");// Schedule cleanup to run daily at 2 AM UTCthis.cleanupTask = cron.schedule(@@ -20,13 +23,13 @@ class ImportCleanupService {try {await this.cleanupOrphanedFiles();} catch (error) {- console.error("[ImportCleanup] Error during cleanup:", error);+ logger.error({ error }, "Error during cleanup");}},{ timezone: "UTC" });- console.info("[ImportCleanup] Cleanup initialized (runs daily at 2 AM UTC)");+ logger.info("Cleanup initialized (runs daily at 2 AM UTC)");}/**@@ -34,25 +37,25 @@ class ImportCleanupService {*/private async cleanupOrphanedFiles() {if (r2Storage.isEnabled()) {- console.info("[ImportCleanup] Starting cleanup of old R2 import files");+ logger.info("Starting cleanup of old R2 import files");try {const r2DeletedCount = await r2Storage.deleteOldImportFiles(1);- console.info(`[ImportCleanup] Deleted ${r2DeletedCount} old files from R2`);+ logger.info({ deletedCount: r2DeletedCount }, "Deleted old files from R2");} catch (error) {- console.error("[ImportCleanup] Error cleaning up R2 files:", error);+ logger.error({ error }, "Error cleaning up R2 files");}}}async triggerManualCleanup() {- console.info("[ImportCleanup] Manual cleanup triggered");+ logger.info("Manual cleanup triggered");await this.cleanupOrphanedFiles();}stopCleanupCron() {if (this.cleanupTask) {this.cleanupTask.stop();- console.info("[ImportCleanup] Cleanup cron stopped");+ logger.info("Cleanup cron stopped");}}}diff --git a/server/src/services/import/importQuotaChecker.ts b/server/src/services/import/importQuotaChecker.tsindex 8fec3ecc2..94c2730b7 100644--- a/server/src/services/import/importQuotaChecker.ts+++ b/server/src/services/import/importQuotaChecker.ts@@ -6,6 +6,9 @@ import { eq } from "drizzle-orm";import { processResults } from "../../api/analytics/utils.js";import { getBestSubscription, type SubscriptionInfo } from "../../lib/subscriptionUtils.js";import { IS_CLOUD } from "../../lib/const.js";+import { createServiceLogger } from "../../lib/logger/logger.js";++const logger = createServiceLogger("import:quota-checker");/*** Get the number of months of historical data allowed for import based on subscription tier@@ -156,7 +159,7 @@ export class ImportQuotaTracker {return monthlyUsage;} catch (error) {- console.error(`Error querying ClickHouse for monthly usage:`, error);+ logger.error({ error }, "Error querying ClickHouse for monthly usage");throw new Error(`Failed to query monthly usage for quota check: ${error instanceof Error ? error.message : "Unknown error"}`);@@ -170,7 +173,7 @@ export class ImportQuotaTracker {const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });if (!dt.isValid) {- console.warn(`Invalid timestamp format: ${timestamp}`);+ logger.warn({ timestamp }, "Invalid timestamp format");return false;}diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.tsindex 00841d37a..5a9258aa2 100644--- a/server/src/services/import/mappings/umami.ts+++ b/server/src/services/import/mappings/umami.ts@@ -2,6 +2,9 @@ import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";import { getChannel } from "../../tracker/getChannel.js";import { RybbitEvent } from "./rybbit.js";import { z } from "zod";+import { createServiceLogger } from "../../../lib/logger/logger.js";++const logger = createServiceLogger("import:umami-mapper");export interface UmamiEvent {// website_id: string; // Ignore@@ -194,7 +197,7 @@ export class UmamiImportMapper {return events.reduce<RybbitEvent[]>((acc, event) => {const parsed = UmamiImportMapper.umamiEventSchema.safeParse(event);if (!parsed.success) {- console.error(`Invalid Umami event for site ${site} during import ${importId}:`, parsed.error.flatten());+ logger.error({ site, importId, error: parsed.error.flatten() }, "Invalid Umami event");return acc;}diff --git a/server/src/services/import/queues/adapters/bullMQAdapter.ts b/server/src/services/import/queues/adapters/bullMQAdapter.tsindex 4eda5c2f6..0fb3d12a4 100644--- a/server/src/services/import/queues/adapters/bullMQAdapter.ts+++ b/server/src/services/import/queues/adapters/bullMQAdapter.ts@@ -3,6 +3,9 @@ import { IJobQueue } from "../jobQueue.js";import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../workers/jobs.js";import { createCsvParseWorker } from "../../workers/csvParseWorker.js";import { createDataInsertWorker } from "../../workers/dataInsertWorker.js";+import { createServiceLogger } from "../../../../lib/logger/logger.js";++const logger = createServiceLogger("import:bullmq");export class BullMQAdapter implements IJobQueue {private readonly connection: { host: string; port: number; password?: string };@@ -28,16 +31,16 @@ export class BullMQAdapter implements IJobQueue {await createDataInsertWorker(this);this.ready = true;- console.info("[BullMQ] Started successfully");+ logger.info("Started successfully");} catch (error) {this.ready = false;- console.error("[BullMQ] Failed to initialize:", error);+ logger.error({ error }, "Failed to initialize");throw error;}}async stop(): Promise<void> {- console.info("[BullMQ] Stopping...");+ logger.info("Stopping...");const errors: Error[] = [];// Close all workers first@@ -47,7 +50,7 @@ export class BullMQAdapter implements IJobQueue {const workerName = Array.from(this.workers.keys())[index];const error = new Error(`Failed to close worker ${workerName}: ${result.reason}`);errors.push(error);- console.error(`[BullMQ] ${error.message}`);+ logger.error({ workerName, error: result.reason }, error.message);}});this.workers.clear();@@ -59,7 +62,7 @@ export class BullMQAdapter implements IJobQueue {const queueName = Array.from(this.queueEvents.keys())[index];const error = new Error(`Failed to close queue events for ${queueName}: ${result.reason}`);errors.push(error);- console.error(`[BullMQ] ${error.message}`);+ logger.error({ queueName, error: result.reason }, error.message);}});this.queueEvents.clear();@@ -71,7 +74,7 @@ export class BullMQAdapter implements IJobQueue {const queueName = Array.from(this.queues.keys())[index];const error = new Error(`Failed to close queue ${queueName}: ${result.reason}`);errors.push(error);- console.error(`[BullMQ] ${error.message}`);+ logger.error({ queueName, error: result.reason }, error.message);}});this.queues.clear();@@ -80,11 +83,11 @@ export class BullMQAdapter implements IJobQueue {const aggregatedError = new Error(`BullMQ stop encountered ${errors.length} error(s): ${errors.map(e => e.message).join("; ")}`);- console.error("[BullMQ] Stopped with errors");+ logger.error({ errorCount: errors.length }, "Stopped with errors");throw aggregatedError;}- console.info("[BullMQ] Stopped successfully");+ logger.info("Stopped successfully");}async createQueue(queueName: string): Promise<void> {@@ -111,14 +114,14 @@ export class BullMQAdapter implements IJobQueue {this.queueEvents.set(queueName, queueEvents);queueEvents.on("completed", ({ jobId }) => {- console.info(`[BullMQ] Job ${jobId} completed in queue ${queueName}`);+ logger.info({ jobId, queueName }, "Job completed");});queueEvents.on("failed", ({ jobId, failedReason }) => {- console.error(`[BullMQ] Job ${jobId} failed in queue ${queueName}:`, failedReason);+ logger.error({ jobId, queueName, failedReason }, "Job failed");});} catch (error) {- console.error(`[BullMQ] Failed to create queue ${queueName}:`, error);+ logger.error({ queueName, error }, "Failed to create queue");const cleanupPromises: Promise<void>[] = [];@@ -126,7 +129,7 @@ export class BullMQAdapter implements IJobQueue {this.queueEvents.delete(queueName);cleanupPromises.push(queueEvents.close().catch(closeError => {- console.error(`[BullMQ] Failed to close queue events during cleanup for ${queueName}:`, closeError);+ logger.error({ queueName, error: closeError }, "Failed to close queue events during cleanup");}));}@@ -135,7 +138,7 @@ export class BullMQAdapter implements IJobQueue {this.queues.delete(queueName);cleanupPromises.push(queue.close().catch(closeError => {- console.error(`[BullMQ] Failed to close queue during cleanup for ${queueName}:`, closeError);+ logger.error({ queueName, error: closeError }, "Failed to close queue during cleanup");}));}@@ -157,7 +160,7 @@ export class BullMQAdapter implements IJobQueue {await queue.add(queueName, data);} catch (error) {const errorMessage = error instanceof Error ? error.message : String(error);- console.error(`[BullMQ] Failed to enqueue job to ${queueName}:`, error);+ logger.error({ queueName, error }, "Failed to enqueue job");throw new Error(`Failed to enqueue job to ${queueName}: ${errorMessage}`);}}@@ -176,16 +179,16 @@ export class BullMQAdapter implements IJobQueue {);worker.on("error", error => {- console.error(`[BullMQ] Worker error in queue ${queueName}:`, error);+ logger.error({ queueName, error }, "Worker error");});worker.on("failed", (job, error) => {- console.error(`[BullMQ] Job ${job?.id} failed in queue ${queueName}:`, error);+ logger.error({ queueName, jobId: job?.id, error }, "Job failed");});this.workers.set(queueName, worker);} catch (error) {- console.error(`[BullMQ] Failed to create worker for queue ${queueName}`, error);+ logger.error({ queueName, error }, "Failed to create worker");throw error;}}diff --git a/server/src/services/import/queues/adapters/pgBossAdapter.ts b/server/src/services/import/queues/adapters/pgBossAdapter.tsindex 335159abb..720f8c60d 100644--- a/server/src/services/import/queues/adapters/pgBossAdapter.ts+++ b/server/src/services/import/queues/adapters/pgBossAdapter.ts@@ -3,6 +3,9 @@ import { IJobQueue } from "../jobQueue.js";import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../workers/jobs.js";import { createCsvParseWorker } from "../../workers/csvParseWorker.js";import { createDataInsertWorker } from "../../workers/dataInsertWorker.js";+import { createServiceLogger } from "../../../../lib/logger/logger.js";++const logger = createServiceLogger("import:pgboss");export class PgBossAdapter implements IJobQueue {private boss: PgBoss;@@ -21,7 +24,7 @@ export class PgBossAdapter implements IJobQueue {});this.boss.on("error", error => {- console.error("[PgBoss] Error:", error);+ logger.error({ error }, "PgBoss error");});}@@ -36,17 +39,17 @@ export class PgBossAdapter implements IJobQueue {await createDataInsertWorker(this);this.ready = true;- console.info("[PgBoss] Started successfully");+ logger.info("Started successfully");} catch (error) {this.ready = false;- console.error("[PgBoss] Failed to initialize:", error);+ logger.error({ error }, "Failed to initialize");throw error;}}async stop(): Promise<void> {await this.boss.stop();- console.info("[PgBoss] Stopped successfully");+ logger.info("Stopped successfully");}async createQueue(queueName: string): Promise<void> {diff --git a/server/src/services/import/queues/jobQueueFactory.ts b/server/src/services/import/queues/jobQueueFactory.tsindex 4740eb09c..bd59d628d 100644--- a/server/src/services/import/queues/jobQueueFactory.ts+++ b/server/src/services/import/queues/jobQueueFactory.ts@@ -12,10 +12,8 @@ let queueInstance: IJobQueue | null = null;export const getJobQueue = (): IJobQueue => {if (!queueInstance) {if (IS_CLOUD) {- console.info("[JobQueue] Initializing BullMQ for cloud deployment");queueInstance = new BullMQAdapter();} else {- console.info("[JobQueue] Initializing pg-boss for self-hosted deployment");queueInstance = new PgBossAdapter();}}diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.tsindex 281228d31..aed56eddd 100644--- a/server/src/services/import/utils.ts+++ b/server/src/services/import/utils.ts@@ -1,6 +1,9 @@import { unlink } from "node:fs/promises";import { IS_CLOUD } from "../../lib/const.js";import { r2Storage } from "../storage/r2StorageService.js";+import { createServiceLogger } from "../../lib/logger/logger.js";++const logger = createServiceLogger("import:utils");export interface DeleteFileResult {success: boolean;@@ -34,15 +37,15 @@ export const deleteImportFile = async (storageLocation: string, isR2Storage: bootry {if (isR2Storage) {await r2Storage.deleteImportFile(storageLocation);- console.log(`[Import] Deleted R2 file: ${storageLocation}`);+ logger.info({ storageLocation }, "Deleted R2 file");} else {await unlink(storageLocation);- console.log(`[Import] Deleted local file: ${storageLocation}`);+ logger.info({ storageLocation }, "Deleted local file");}return { success: true };} catch (error) {const errorMsg = error instanceof Error ? error.message : "Unknown error";- console.error(`[Import] Failed to delete file ${storageLocation}:`, errorMsg);+ logger.error({ storageLocation, error: errorMsg }, "Failed to delete file");// DON'T throw - return error info instead to prevent worker crashesreturn { success: false, error: errorMsg };diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 7abe97dcf..973686ab8 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -9,6 +9,9 @@ import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";import { updateImportStatus } from "../importStatusManager.js";import { deleteImportFile } from "../utils.js";import { ImportQuotaTracker } from "../importQuotaChecker.js";+import { createServiceLogger } from "../../../lib/logger/logger.js";++const logger = createServiceLogger("import:csv-parser");const getImportDataHeaders = (platform: string) => {switch (platform) {@@ -20,7 +23,7 @@ const getImportDataHeaders = (platform: string) => {};const createR2FileStream = async (storageLocation: string, platform: string) => {- console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);+ logger.info({ storageLocation }, "Reading from R2");const fileStream = await r2Storage.getImportFileStream(storageLocation);return fileStream.pipe(parse({@@ -32,7 +35,7 @@ const createR2FileStream = async (storageLocation: string, platform: string) =>};const createLocalFileStream = async (storageLocation: string, platform: string) => {- console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);+ logger.info({ storageLocation }, "Reading from local disk");await access(storageLocation, constants.F_OK | constants.R_OK);return createReadStream(storageLocation).pipe(parse({@@ -97,7 +100,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {// Add explicit error handler before starting to consume the streamstream.on("error", error => {- console.error(`[Import ${importId}] Stream error:`, error);+ logger.error({ importId, error }, "Stream error");// Error will be caught by the outer try/catch});@@ -165,7 +168,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {await updateImportStatus(importId, "failed", errorMessage);const deleteResult = await deleteImportFile(storageLocation, isR2Storage);if (!deleteResult.success) {- console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);+ logger.warn({ importId, error: deleteResult.error }, "File cleanup failed");}return;}@@ -179,12 +182,12 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {allChunksSent: true,});} catch (error) {- console.error(`[Import ${importId}] Error in CSV parse worker:`, error);+ logger.error({ importId, error }, "Error in CSV parse worker");await updateImportStatus(importId, "failed", "An unexpected error occurred during import processing");// Don't re-throw - worker should continue processing other jobs- console.error(`[Import ${importId}] Import failed, worker continuing`);+ logger.error({ importId }, "Import failed, worker continuing");} finally {// Clean up timeoutif (processingTimeout) {@@ -196,14 +199,14 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {try {stream.destroy();} catch (streamError) {- console.warn(`[Import ${importId}] Failed to destroy stream:`, streamError);+ logger.warn({ importId, error: streamError }, "Failed to destroy stream");}}// Clean up file - don't throw on failure to prevent worker crashesconst deleteResult = await deleteImportFile(storageLocation, isR2Storage);if (!deleteResult.success) {- console.warn(`[Import ${importId}] File cleanup failed, will remain in storage: ${deleteResult.error}`);+ logger.warn({ importId, error: deleteResult.error }, "File cleanup failed, will remain in storage");// File will be orphaned but import status is already recorded// importCleanupService.ts handles orphans}diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex 8155f3434..41bc79488 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -3,6 +3,9 @@ import { UmamiImportMapper } from "../mappings/umami.js";import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { updateImportStatus, updateImportProgress } from "../importStatusManager.js";+import { createServiceLogger } from "../../../lib/logger/logger.js";++const logger = createServiceLogger("import:data-insert");const getImportDataMapping = (platform: string) => {switch (platform) {@@ -22,12 +25,12 @@ export async function createDataInsertWorker(jobQueue: IJobQueue) {await updateImportStatus(importId, "completed");return;} catch (error) {- console.error(`[Import ${importId}] Failed to mark as completed:`, error);+ logger.error({ importId, error }, "Failed to mark as completed");// Try to update to failed status, but don't crash workertry {await updateImportStatus(importId, "failed", "Failed to complete import");} catch (updateError) {- console.error(`[Import ${importId}] Could not update status to failed:`, updateError);+ logger.error({ importId, error: updateError }, "Could not update status to failed");}// Don't re-throw - worker should continuereturn;@@ -49,23 +52,23 @@ export async function createDataInsertWorker(jobQueue: IJobQueue) {try {await updateImportProgress(importId, transformedRecords.length);} catch (progressError) {- console.warn(- `[Import ${importId}] Progress update failed (data inserted successfully):`,- progressError instanceof Error ? progressError.message : progressError+ logger.warn(+ { importId, error: progressError instanceof Error ? progressError.message : progressError },+ "Progress update failed (data inserted successfully)");// Don't throw - data is safely in ClickHouse, progress can be off slightly}} catch (error) {- console.error(`[Import ${importId}] ClickHouse insert failed:`, error);+ logger.error({ importId, error }, "ClickHouse insert failed");try {await updateImportStatus(importId, "failed", "Data insertion failed due to unknown error");} catch (updateError) {- console.error(`[Import ${importId}] Could not update status to failed:`, updateError);+ logger.error({ importId, error: updateError }, "Could not update status to failed");}// Don't re-throw - worker should continue processing other jobs- console.error(`[Import ${importId}] Import chunk failed, worker continuing`);+ logger.error({ importId }, "Import chunk failed, worker continuing");}});}