Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

date to csv parse

Edward Cuief228b920ec7
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 38facfbbe..a0372d3e3 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -8,10 +8,11 @@ import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
import { ImportStatusManager } from "../importStatusManager.js";
import { ImportLimiter } from "../importLimiter.js";
import { r2Storage } from "../../storage/r2StorageService.js";
+import { DateTime } from "luxon";
export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
- const { site, importId, source, storageLocation, isR2Storage, organization } = job.data;
+ const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
try {
const importableEvents = await ImportLimiter.countImportableEvents(organization);
@@ -28,6 +29,7 @@ export async function registerCsvParseWorker() {
const chunkSize = 1000;
let chunk: UmamiEvent[] = [];
+ let rowsProcessed = 0;
let stream: Readable;
@@ -38,7 +40,6 @@ export async function registerCsvParseWorker() {
headers: umamiHeaders,
renameHeaders: true,
ignoreEmpty: true,
- maxRows: importableEvents,
}));
} else {
console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);
@@ -46,14 +47,39 @@ export async function registerCsvParseWorker() {
headers: umamiHeaders,
renameHeaders: true,
ignoreEmpty: true,
- maxRows: importableEvents,
}));
}
await ImportStatusManager.updateStatus(importId, "processing");
for await (const data of stream) {
+ if (!data.created_at) {
+ continue;
+ }
+ const createdAt = DateTime.fromFormat(data.created_at, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
+ if (!createdAt.isValid) {
+ continue;
+ }
+
+ if (startDate) {
+ const start = DateTime.fromFormat(startDate, "yyyy-MM-dd", { zone: "utc" });
+ if (!start.isValid || createdAt < start.startOf("day")) {
+ continue;
+ }
+ }
+ if (endDate) {
+ const end = DateTime.fromFormat(endDate, "yyyy-MM-dd", { zone: "utc" });
+ if (!end.isValid || createdAt > end.endOf("day")) {
+ continue;
+ }
+ }
+
+ if (rowsProcessed >= importableEvents) {
+ break;
+ }
+
chunk.push(data);
+ rowsProcessed++;
if (chunk.length >= chunkSize) {
await boss.send(DATA_INSERT_QUEUE, {