🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
date to csv parse
Edward Cuief228b920ec7
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 38facfbbe..a0372d3e3 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -8,10 +8,11 @@ import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";import { ImportStatusManager } from "../importStatusManager.js";import { ImportLimiter } from "../importLimiter.js";import { r2Storage } from "../../storage/r2StorageService.js";+import { DateTime } from "luxon";export async function registerCsvParseWorker() {await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {- const { site, importId, source, storageLocation, isR2Storage, organization } = job.data;+ const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;try {const importableEvents = await ImportLimiter.countImportableEvents(organization);@@ -28,6 +29,7 @@ export async function registerCsvParseWorker() {const chunkSize = 1000;let chunk: UmamiEvent[] = [];+ let rowsProcessed = 0;let stream: Readable;@@ -38,7 +40,6 @@ export async function registerCsvParseWorker() {headers: umamiHeaders,renameHeaders: true,ignoreEmpty: true,- maxRows: importableEvents,}));} else {console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);@@ -46,14 +47,39 @@ export async function registerCsvParseWorker() {headers: umamiHeaders,renameHeaders: true,ignoreEmpty: true,- maxRows: importableEvents,}));}await ImportStatusManager.updateStatus(importId, "processing");for await (const data of stream) {+ if (!data.created_at) {+ continue;+ }+ const createdAt = DateTime.fromFormat(data.created_at, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });+ if (!createdAt.isValid) {+ continue;+ }++ if (startDate) {+ const start = DateTime.fromFormat(startDate, "yyyy-MM-dd", { zone: "utc" });+ if (!start.isValid || createdAt < start.startOf("day")) {+ continue;+ }+ }+ if (endDate) {+ const end = DateTime.fromFormat(endDate, "yyyy-MM-dd", { zone: "utc" });+ if (!end.isValid || createdAt > end.endOf("day")) {+ continue;+ }+ }++ if (rowsProcessed >= importableEvents) {+ break;+ }+chunk.push(data);+ rowsProcessed++;if (chunk.length >= chunkSize) {await boss.send(DATA_INSERT_QUEUE, {