Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Adjust logging

Edward Cui097ff2fd8a91
diff --git a/server/src/services/import/queues/adapters/bullMQAdapter.ts b/server/src/services/import/queues/adapters/bullMQAdapter.ts
index 0fb3d12a4..8bacbf93d 100644
--- a/server/src/services/import/queues/adapters/bullMQAdapter.ts
+++ b/server/src/services/import/queues/adapters/bullMQAdapter.ts
@@ -114,7 +114,7 @@ export class BullMQAdapter implements IJobQueue {
this.queueEvents.set(queueName, queueEvents);
queueEvents.on("completed", ({ jobId }) => {
- logger.info({ jobId, queueName }, "Job completed");
+ logger.debug({ jobId, queueName }, "Job completed");
});
queueEvents.on("failed", ({ jobId, failedReason }) => {
diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.ts
index aed56eddd..62bec0a42 100644
--- a/server/src/services/import/utils.ts
+++ b/server/src/services/import/utils.ts
@@ -37,10 +37,10 @@ export const deleteImportFile = async (storageLocation: string, isR2Storage: boo
try {
if (isR2Storage) {
await r2Storage.deleteImportFile(storageLocation);
- logger.info({ storageLocation }, "Deleted R2 file");
+ logger.debug({ storageLocation }, "Deleted R2 file");
} else {
await unlink(storageLocation);
- logger.info({ storageLocation }, "Deleted local file");
+ logger.debug({ storageLocation }, "Deleted local file");
}
return { success: true };
} catch (error) {
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 973686ab8..ec1080a91 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -93,6 +93,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {
let chunk: UmamiEvent[] = [];
let totalSkippedQuota = 0;
+ let totalProcessed = 0;
stream = isR2Storage
? await createR2FileStream(storageLocation, platform)
@@ -105,6 +106,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {
});
await updateImportStatus(importId, "processing");
+ logger.info({ importId, platform, organization }, "Started processing CSV import");
// Set timeout to prevent indefinite processing
processingTimeout = setTimeout(() => {
@@ -134,6 +136,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {
// Event passed all filters - add to chunk
chunk.push(data);
+ totalProcessed++;
if (chunk.length >= chunkSize) {
await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
@@ -174,6 +177,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {
}
// Send finalization signal with total chunk count
+ logger.info({ importId, totalProcessed, totalSkippedQuota }, "CSV parsing completed successfully");
await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
site,
importId,
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 41bc79488..ef311f1a6 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -23,6 +23,7 @@ export async function createDataInsertWorker(jobQueue: IJobQueue) {
if (allChunksSent) {
try {
await updateImportStatus(importId, "completed");
+ logger.info({ importId }, "Import completed successfully");
return;
} catch (error) {
logger.error({ importId, error }, "Failed to mark as completed");
@@ -48,6 +49,8 @@ export async function createDataInsertWorker(jobQueue: IJobQueue) {
format: "JSONEachRow",
});
+ logger.debug({ importId, recordCount: transformedRecords.length }, "Inserted chunk to ClickHouse");
+
// Update progress (non-critical - log if fails but don't crash)
try {
await updateImportProgress(importId, transformedRecords.length);