Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Better logging

Edward Cui5121c654b147
diff --git a/server/src/services/import/importCleanupService.ts b/server/src/services/import/importCleanupService.ts
index b3118fcf3..e61fb2b87 100644
--- a/server/src/services/import/importCleanupService.ts
+++ b/server/src/services/import/importCleanupService.ts
@@ -1,5 +1,8 @@
import * as cron from "node-cron";
import { r2Storage } from "../storage/r2StorageService.js";
+import { createServiceLogger } from "../../lib/logger/logger.js";
+
+const logger = createServiceLogger("import:cleanup");
class ImportCleanupService {
private cleanupTask: cron.ScheduledTask | null = null;
@@ -11,7 +14,7 @@ class ImportCleanupService {
* Runs daily at 2 AM UTC to clean up orphaned import files.
*/
initializeCleanupCron() {
- console.info("[ImportCleanup] Initializing cleanup cron");
+ logger.info("Initializing cleanup cron");
// Schedule cleanup to run daily at 2 AM UTC
this.cleanupTask = cron.schedule(
@@ -20,13 +23,13 @@ class ImportCleanupService {
try {
await this.cleanupOrphanedFiles();
} catch (error) {
- console.error("[ImportCleanup] Error during cleanup:", error);
+ logger.error({ error }, "Error during cleanup");
}
},
{ timezone: "UTC" }
);
- console.info("[ImportCleanup] Cleanup initialized (runs daily at 2 AM UTC)");
+ logger.info("Cleanup initialized (runs daily at 2 AM UTC)");
}
/**
@@ -34,25 +37,25 @@ class ImportCleanupService {
*/
private async cleanupOrphanedFiles() {
if (r2Storage.isEnabled()) {
- console.info("[ImportCleanup] Starting cleanup of old R2 import files");
+ logger.info("Starting cleanup of old R2 import files");
try {
const r2DeletedCount = await r2Storage.deleteOldImportFiles(1);
- console.info(`[ImportCleanup] Deleted ${r2DeletedCount} old files from R2`);
+ logger.info({ deletedCount: r2DeletedCount }, "Deleted old files from R2");
} catch (error) {
- console.error("[ImportCleanup] Error cleaning up R2 files:", error);
+ logger.error({ error }, "Error cleaning up R2 files");
}
}
}
async triggerManualCleanup() {
- console.info("[ImportCleanup] Manual cleanup triggered");
+ logger.info("Manual cleanup triggered");
await this.cleanupOrphanedFiles();
}
stopCleanupCron() {
if (this.cleanupTask) {
this.cleanupTask.stop();
- console.info("[ImportCleanup] Cleanup cron stopped");
+ logger.info("Cleanup cron stopped");
}
}
}
diff --git a/server/src/services/import/importQuotaChecker.ts b/server/src/services/import/importQuotaChecker.ts
index 8fec3ecc2..94c2730b7 100644
--- a/server/src/services/import/importQuotaChecker.ts
+++ b/server/src/services/import/importQuotaChecker.ts
@@ -6,6 +6,9 @@ import { eq } from "drizzle-orm";
import { processResults } from "../../api/analytics/utils.js";
import { getBestSubscription, type SubscriptionInfo } from "../../lib/subscriptionUtils.js";
import { IS_CLOUD } from "../../lib/const.js";
+import { createServiceLogger } from "../../lib/logger/logger.js";
+
+const logger = createServiceLogger("import:quota-checker");
/**
* Get the number of months of historical data allowed for import based on subscription tier
@@ -156,7 +159,7 @@ export class ImportQuotaTracker {
return monthlyUsage;
} catch (error) {
- console.error(`Error querying ClickHouse for monthly usage:`, error);
+ logger.error({ error }, "Error querying ClickHouse for monthly usage");
throw new Error(
`Failed to query monthly usage for quota check: ${error instanceof Error ? error.message : "Unknown error"}`
);
@@ -170,7 +173,7 @@ export class ImportQuotaTracker {
const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
if (!dt.isValid) {
- console.warn(`Invalid timestamp format: ${timestamp}`);
+ logger.warn({ timestamp }, "Invalid timestamp format");
return false;
}
diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.ts
index 00841d37a..5a9258aa2 100644
--- a/server/src/services/import/mappings/umami.ts
+++ b/server/src/services/import/mappings/umami.ts
@@ -2,6 +2,9 @@ import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";
import { getChannel } from "../../tracker/getChannel.js";
import { RybbitEvent } from "./rybbit.js";
import { z } from "zod";
+import { createServiceLogger } from "../../../lib/logger/logger.js";
+
+const logger = createServiceLogger("import:umami-mapper");
export interface UmamiEvent {
// website_id: string; // Ignore
@@ -194,7 +197,7 @@ export class UmamiImportMapper {
return events.reduce<RybbitEvent[]>((acc, event) => {
const parsed = UmamiImportMapper.umamiEventSchema.safeParse(event);
if (!parsed.success) {
- console.error(`Invalid Umami event for site ${site} during import ${importId}:`, parsed.error.flatten());
+ logger.error({ site, importId, error: parsed.error.flatten() }, "Invalid Umami event");
return acc;
}
diff --git a/server/src/services/import/queues/adapters/bullMQAdapter.ts b/server/src/services/import/queues/adapters/bullMQAdapter.ts
index 4eda5c2f6..0fb3d12a4 100644
--- a/server/src/services/import/queues/adapters/bullMQAdapter.ts
+++ b/server/src/services/import/queues/adapters/bullMQAdapter.ts
@@ -3,6 +3,9 @@ import { IJobQueue } from "../jobQueue.js";
import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../workers/jobs.js";
import { createCsvParseWorker } from "../../workers/csvParseWorker.js";
import { createDataInsertWorker } from "../../workers/dataInsertWorker.js";
+import { createServiceLogger } from "../../../../lib/logger/logger.js";
+
+const logger = createServiceLogger("import:bullmq");
export class BullMQAdapter implements IJobQueue {
private readonly connection: { host: string; port: number; password?: string };
@@ -28,16 +31,16 @@ export class BullMQAdapter implements IJobQueue {
await createDataInsertWorker(this);
this.ready = true;
- console.info("[BullMQ] Started successfully");
+ logger.info("Started successfully");
} catch (error) {
this.ready = false;
- console.error("[BullMQ] Failed to initialize:", error);
+ logger.error({ error }, "Failed to initialize");
throw error;
}
}
async stop(): Promise<void> {
- console.info("[BullMQ] Stopping...");
+ logger.info("Stopping...");
const errors: Error[] = [];
// Close all workers first
@@ -47,7 +50,7 @@ export class BullMQAdapter implements IJobQueue {
const workerName = Array.from(this.workers.keys())[index];
const error = new Error(`Failed to close worker ${workerName}: ${result.reason}`);
errors.push(error);
- console.error(`[BullMQ] ${error.message}`);
+ logger.error({ workerName, error: result.reason }, error.message);
}
});
this.workers.clear();
@@ -59,7 +62,7 @@ export class BullMQAdapter implements IJobQueue {
const queueName = Array.from(this.queueEvents.keys())[index];
const error = new Error(`Failed to close queue events for ${queueName}: ${result.reason}`);
errors.push(error);
- console.error(`[BullMQ] ${error.message}`);
+ logger.error({ queueName, error: result.reason }, error.message);
}
});
this.queueEvents.clear();
@@ -71,7 +74,7 @@ export class BullMQAdapter implements IJobQueue {
const queueName = Array.from(this.queues.keys())[index];
const error = new Error(`Failed to close queue ${queueName}: ${result.reason}`);
errors.push(error);
- console.error(`[BullMQ] ${error.message}`);
+ logger.error({ queueName, error: result.reason }, error.message);
}
});
this.queues.clear();
@@ -80,11 +83,11 @@ export class BullMQAdapter implements IJobQueue {
const aggregatedError = new Error(
`BullMQ stop encountered ${errors.length} error(s): ${errors.map(e => e.message).join("; ")}`
);
- console.error("[BullMQ] Stopped with errors");
+ logger.error({ errorCount: errors.length }, "Stopped with errors");
throw aggregatedError;
}
- console.info("[BullMQ] Stopped successfully");
+ logger.info("Stopped successfully");
}
async createQueue(queueName: string): Promise<void> {
@@ -111,14 +114,14 @@ export class BullMQAdapter implements IJobQueue {
this.queueEvents.set(queueName, queueEvents);
queueEvents.on("completed", ({ jobId }) => {
- console.info(`[BullMQ] Job ${jobId} completed in queue ${queueName}`);
+ logger.info({ jobId, queueName }, "Job completed");
});
queueEvents.on("failed", ({ jobId, failedReason }) => {
- console.error(`[BullMQ] Job ${jobId} failed in queue ${queueName}:`, failedReason);
+ logger.error({ jobId, queueName, failedReason }, "Job failed");
});
} catch (error) {
- console.error(`[BullMQ] Failed to create queue ${queueName}:`, error);
+ logger.error({ queueName, error }, "Failed to create queue");
const cleanupPromises: Promise<void>[] = [];
@@ -126,7 +129,7 @@ export class BullMQAdapter implements IJobQueue {
this.queueEvents.delete(queueName);
cleanupPromises.push(
queueEvents.close().catch(closeError => {
- console.error(`[BullMQ] Failed to close queue events during cleanup for ${queueName}:`, closeError);
+ logger.error({ queueName, error: closeError }, "Failed to close queue events during cleanup");
})
);
}
@@ -135,7 +138,7 @@ export class BullMQAdapter implements IJobQueue {
this.queues.delete(queueName);
cleanupPromises.push(
queue.close().catch(closeError => {
- console.error(`[BullMQ] Failed to close queue during cleanup for ${queueName}:`, closeError);
+ logger.error({ queueName, error: closeError }, "Failed to close queue during cleanup");
})
);
}
@@ -157,7 +160,7 @@ export class BullMQAdapter implements IJobQueue {
await queue.add(queueName, data);
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
- console.error(`[BullMQ] Failed to enqueue job to ${queueName}:`, error);
+ logger.error({ queueName, error }, "Failed to enqueue job");
throw new Error(`Failed to enqueue job to ${queueName}: ${errorMessage}`);
}
}
@@ -176,16 +179,16 @@ export class BullMQAdapter implements IJobQueue {
);
worker.on("error", error => {
- console.error(`[BullMQ] Worker error in queue ${queueName}:`, error);
+ logger.error({ queueName, error }, "Worker error");
});
worker.on("failed", (job, error) => {
- console.error(`[BullMQ] Job ${job?.id} failed in queue ${queueName}:`, error);
+ logger.error({ queueName, jobId: job?.id, error }, "Job failed");
});
this.workers.set(queueName, worker);
} catch (error) {
- console.error(`[BullMQ] Failed to create worker for queue ${queueName}`, error);
+ logger.error({ queueName, error }, "Failed to create worker");
throw error;
}
}
diff --git a/server/src/services/import/queues/adapters/pgBossAdapter.ts b/server/src/services/import/queues/adapters/pgBossAdapter.ts
index 335159abb..720f8c60d 100644
--- a/server/src/services/import/queues/adapters/pgBossAdapter.ts
+++ b/server/src/services/import/queues/adapters/pgBossAdapter.ts
@@ -3,6 +3,9 @@ import { IJobQueue } from "../jobQueue.js";
import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../workers/jobs.js";
import { createCsvParseWorker } from "../../workers/csvParseWorker.js";
import { createDataInsertWorker } from "../../workers/dataInsertWorker.js";
+import { createServiceLogger } from "../../../../lib/logger/logger.js";
+
+const logger = createServiceLogger("import:pgboss");
export class PgBossAdapter implements IJobQueue {
private boss: PgBoss;
@@ -21,7 +24,7 @@ export class PgBossAdapter implements IJobQueue {
});
this.boss.on("error", error => {
- console.error("[PgBoss] Error:", error);
+ logger.error({ error }, "PgBoss error");
});
}
@@ -36,17 +39,17 @@ export class PgBossAdapter implements IJobQueue {
await createDataInsertWorker(this);
this.ready = true;
- console.info("[PgBoss] Started successfully");
+ logger.info("Started successfully");
} catch (error) {
this.ready = false;
- console.error("[PgBoss] Failed to initialize:", error);
+ logger.error({ error }, "Failed to initialize");
throw error;
}
}
async stop(): Promise<void> {
await this.boss.stop();
- console.info("[PgBoss] Stopped successfully");
+ logger.info("Stopped successfully");
}
async createQueue(queueName: string): Promise<void> {
diff --git a/server/src/services/import/queues/jobQueueFactory.ts b/server/src/services/import/queues/jobQueueFactory.ts
index 4740eb09c..bd59d628d 100644
--- a/server/src/services/import/queues/jobQueueFactory.ts
+++ b/server/src/services/import/queues/jobQueueFactory.ts
@@ -12,10 +12,8 @@ let queueInstance: IJobQueue | null = null;
export const getJobQueue = (): IJobQueue => {
if (!queueInstance) {
if (IS_CLOUD) {
- console.info("[JobQueue] Initializing BullMQ for cloud deployment");
queueInstance = new BullMQAdapter();
} else {
- console.info("[JobQueue] Initializing pg-boss for self-hosted deployment");
queueInstance = new PgBossAdapter();
}
}
diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.ts
index 281228d31..aed56eddd 100644
--- a/server/src/services/import/utils.ts
+++ b/server/src/services/import/utils.ts
@@ -1,6 +1,9 @@
import { unlink } from "node:fs/promises";
import { IS_CLOUD } from "../../lib/const.js";
import { r2Storage } from "../storage/r2StorageService.js";
+import { createServiceLogger } from "../../lib/logger/logger.js";
+
+const logger = createServiceLogger("import:utils");
export interface DeleteFileResult {
success: boolean;
@@ -34,15 +37,15 @@ export const deleteImportFile = async (storageLocation: string, isR2Storage: boo
try {
if (isR2Storage) {
await r2Storage.deleteImportFile(storageLocation);
- console.log(`[Import] Deleted R2 file: ${storageLocation}`);
+ logger.info({ storageLocation }, "Deleted R2 file");
} else {
await unlink(storageLocation);
- console.log(`[Import] Deleted local file: ${storageLocation}`);
+ logger.info({ storageLocation }, "Deleted local file");
}
return { success: true };
} catch (error) {
const errorMsg = error instanceof Error ? error.message : "Unknown error";
- console.error(`[Import] Failed to delete file ${storageLocation}:`, errorMsg);
+ logger.error({ storageLocation, error: errorMsg }, "Failed to delete file");
// DON'T throw - return error info instead to prevent worker crashes
return { success: false, error: errorMsg };
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 7abe97dcf..973686ab8 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -9,6 +9,9 @@ import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
import { updateImportStatus } from "../importStatusManager.js";
import { deleteImportFile } from "../utils.js";
import { ImportQuotaTracker } from "../importQuotaChecker.js";
+import { createServiceLogger } from "../../../lib/logger/logger.js";
+
+const logger = createServiceLogger("import:csv-parser");
const getImportDataHeaders = (platform: string) => {
switch (platform) {
@@ -20,7 +23,7 @@ const getImportDataHeaders = (platform: string) => {
};
const createR2FileStream = async (storageLocation: string, platform: string) => {
- console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);
+ logger.info({ storageLocation }, "Reading from R2");
const fileStream = await r2Storage.getImportFileStream(storageLocation);
return fileStream.pipe(
parse({
@@ -32,7 +35,7 @@ const createR2FileStream = async (storageLocation: string, platform: string) =>
};
const createLocalFileStream = async (storageLocation: string, platform: string) => {
- console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);
+ logger.info({ storageLocation }, "Reading from local disk");
await access(storageLocation, constants.F_OK | constants.R_OK);
return createReadStream(storageLocation).pipe(
parse({
@@ -97,7 +100,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {
// Add explicit error handler before starting to consume the stream
stream.on("error", error => {
- console.error(`[Import ${importId}] Stream error:`, error);
+ logger.error({ importId, error }, "Stream error");
// Error will be caught by the outer try/catch
});
@@ -165,7 +168,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {
await updateImportStatus(importId, "failed", errorMessage);
const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
if (!deleteResult.success) {
- console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);
+ logger.warn({ importId, error: deleteResult.error }, "File cleanup failed");
}
return;
}
@@ -179,12 +182,12 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {
allChunksSent: true,
});
} catch (error) {
- console.error(`[Import ${importId}] Error in CSV parse worker:`, error);
+ logger.error({ importId, error }, "Error in CSV parse worker");
await updateImportStatus(importId, "failed", "An unexpected error occurred during import processing");
// Don't re-throw - worker should continue processing other jobs
- console.error(`[Import ${importId}] Import failed, worker continuing`);
+ logger.error({ importId }, "Import failed, worker continuing");
} finally {
// Clean up timeout
if (processingTimeout) {
@@ -196,14 +199,14 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {
try {
stream.destroy();
} catch (streamError) {
- console.warn(`[Import ${importId}] Failed to destroy stream:`, streamError);
+ logger.warn({ importId, error: streamError }, "Failed to destroy stream");
}
}
// Clean up file - don't throw on failure to prevent worker crashes
const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
if (!deleteResult.success) {
- console.warn(`[Import ${importId}] File cleanup failed, will remain in storage: ${deleteResult.error}`);
+ logger.warn({ importId, error: deleteResult.error }, "File cleanup failed, will remain in storage");
// File will be orphaned but import status is already recorded
// importCleanupService.ts handles orphans
}
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 8155f3434..41bc79488 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -3,6 +3,9 @@ import { UmamiImportMapper } from "../mappings/umami.js";
import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { updateImportStatus, updateImportProgress } from "../importStatusManager.js";
+import { createServiceLogger } from "../../../lib/logger/logger.js";
+
+const logger = createServiceLogger("import:data-insert");
const getImportDataMapping = (platform: string) => {
switch (platform) {
@@ -22,12 +25,12 @@ export async function createDataInsertWorker(jobQueue: IJobQueue) {
await updateImportStatus(importId, "completed");
return;
} catch (error) {
- console.error(`[Import ${importId}] Failed to mark as completed:`, error);
+ logger.error({ importId, error }, "Failed to mark as completed");
// Try to update to failed status, but don't crash worker
try {
await updateImportStatus(importId, "failed", "Failed to complete import");
} catch (updateError) {
- console.error(`[Import ${importId}] Could not update status to failed:`, updateError);
+ logger.error({ importId, error: updateError }, "Could not update status to failed");
}
// Don't re-throw - worker should continue
return;
@@ -49,23 +52,23 @@ export async function createDataInsertWorker(jobQueue: IJobQueue) {
try {
await updateImportProgress(importId, transformedRecords.length);
} catch (progressError) {
- console.warn(
- `[Import ${importId}] Progress update failed (data inserted successfully):`,
- progressError instanceof Error ? progressError.message : progressError
+ logger.warn(
+ { importId, error: progressError instanceof Error ? progressError.message : progressError },
+ "Progress update failed (data inserted successfully)"
);
// Don't throw - data is safely in ClickHouse, progress can be off slightly
}
} catch (error) {
- console.error(`[Import ${importId}] ClickHouse insert failed:`, error);
+ logger.error({ importId, error }, "ClickHouse insert failed");
try {
await updateImportStatus(importId, "failed", "Data insertion failed due to unknown error");
} catch (updateError) {
- console.error(`[Import ${importId}] Could not update status to failed:`, updateError);
+ logger.error({ importId, error: updateError }, "Could not update status to failed");
}
// Don't re-throw - worker should continue processing other jobs
- console.error(`[Import ${importId}] Import chunk failed, worker continuing`);
+ logger.error({ importId }, "Import chunk failed, worker continuing");
}
});
}