Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

fix: Umami mapper reads whole ZIP in memory

Blue Mousec6193c872b4f
diff --git a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts
index 783ebb734..a1158230b 100644
--- a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts
+++ b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts
@@ -2,12 +2,13 @@ import * as fs from 'fs'
import * as path from 'path'
import * as os from 'os'
import * as crypto from 'crypto'
-import { unzipSync } from 'fflate'
+import { Unzip, UnzipInflate } from 'fflate'
import { parse } from 'csv-parse'
import { ImportMapper, AnalyticsImportRow } from './mapper.interface'
const WEBSITE_EVENT_CSV = 'website_event.csv'
+const MAX_UMAMI_CSV_BYTES = 512 * 1024 * 1024
const BROWSER_MAP: Record<string, string> = {
chrome: 'Chrome',
@@ -135,24 +136,172 @@ export class UmamiMapper implements ImportMapper {
readonly provider = 'umami'
readonly expectedFileExtension = '.zip'
- private extractCsvFromZip(zipPath: string): string {
- const zipBuffer = fs.readFileSync(zipPath)
- const files = unzipSync(new Uint8Array(zipBuffer))
+ private async extractCsvFromZip(zipPath: string): Promise<string> {
+ const tempCsv = path.join(os.tmpdir(), `swetrix-umami-${Date.now()}.csv`)
- const matchKey = Object.keys(files).find((name) => {
- const basename = name.split('/').pop() || name
- return basename === WEBSITE_EVENT_CSV
- })
+ return new Promise((resolve, reject) => {
+ const zipStream = fs.createReadStream(zipPath)
+ const unzipper = new Unzip()
+
+ unzipper.register(UnzipInflate)
+
+ let settled = false
+ let foundEntry = false
+ let fileDescriptor: number | null = null
+ let tempFileCreated = false
+ let extractedBytes = 0
+ let terminateEntry: (() => void) | null = null
+
+ const cleanupTempCsv = () => {
+ if (fileDescriptor !== null) {
+ try {
+ fs.closeSync(fileDescriptor)
+ } catch {
+ //
+ }
+ fileDescriptor = null
+ }
- if (!matchKey) {
- throw new Error(
- `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`,
- )
- }
+ if (tempFileCreated) {
+ try {
+ fs.unlinkSync(tempCsv)
+ } catch {
+ //
+ }
+ }
+ }
- const tempCsv = path.join(os.tmpdir(), `swetrix-umami-${Date.now()}.csv`)
- fs.writeFileSync(tempCsv, Buffer.from(files[matchKey]))
- return tempCsv
+ const fail = (error: Error) => {
+ if (settled) return
+ settled = true
+ terminateEntry?.()
+ zipStream.destroy()
+ cleanupTempCsv()
+ reject(error)
+ }
+
+ const succeed = () => {
+ if (settled) return
+ settled = true
+ if (fileDescriptor !== null) {
+ try {
+ fs.closeSync(fileDescriptor)
+ } catch (error) {
+ cleanupTempCsv()
+ reject(error)
+ return
+ }
+ fileDescriptor = null
+ }
+ zipStream.destroy()
+ resolve(tempCsv)
+ }
+
+ unzipper.onfile = (file) => {
+ if (path.basename(file.name) !== WEBSITE_EVENT_CSV || foundEntry) return
+
+ foundEntry = true
+
+ if (
+ typeof file.originalSize !== 'number' ||
+ !Number.isFinite(file.originalSize)
+ ) {
+ fail(
+ new Error(
+ `Unable to validate ${WEBSITE_EVENT_CSV} size in the ZIP archive.`,
+ ),
+ )
+ return
+ }
+
+ if (file.originalSize > MAX_UMAMI_CSV_BYTES) {
+ fail(
+ new Error(
+ `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,
+ ),
+ )
+ return
+ }
+
+ try {
+ fileDescriptor = fs.openSync(tempCsv, 'wx')
+ tempFileCreated = true
+ } catch (error) {
+ fail(error as Error)
+ return
+ }
+
+ terminateEntry = file.terminate
+ file.ondata = (error, chunk, final) => {
+ if (error) {
+ fail(error)
+ return
+ }
+
+ extractedBytes += chunk.length
+ if (extractedBytes > MAX_UMAMI_CSV_BYTES) {
+ fail(
+ new Error(
+ `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,
+ ),
+ )
+ return
+ }
+
+ try {
+ fs.writeSync(fileDescriptor!, chunk)
+ } catch (writeError) {
+ fail(writeError as Error)
+ return
+ }
+
+ if (final) {
+ succeed()
+ }
+ }
+
+ try {
+ file.start()
+ } catch (error) {
+ fail(error as Error)
+ }
+ }
+
+ zipStream.on('data', (chunk: Buffer) => {
+ if (settled) return
+
+ try {
+ unzipper.push(
+ new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength),
+ )
+ } catch (error) {
+ fail(error as Error)
+ }
+ })
+
+ zipStream.on('end', () => {
+ if (settled) return
+
+ try {
+ unzipper.push(new Uint8Array(0), true)
+ } catch (error) {
+ fail(error as Error)
+ return
+ }
+
+ if (!foundEntry) {
+ fail(
+ new Error(
+ `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`,
+ ),
+ )
+ }
+ })
+
+ zipStream.on('error', (error) => {
+ fail(error)
+ })
+ })
}
async *createRowStream(
@@ -160,7 +309,7 @@ export class UmamiMapper implements ImportMapper {
pid: string,
importID: number,
): AsyncIterable<AnalyticsImportRow> {
- const csvPath = this.extractCsvFromZip(filePath)
+ const csvPath = await this.extractCsvFromZip(filePath)
let cleanedUp = false
const cleanup = () => {
diff --git a/backend/apps/community/src/data-import/mappers/umami.mapper.ts b/backend/apps/community/src/data-import/mappers/umami.mapper.ts
index e7bfb9b93..8e2ea25d4 100644
--- a/backend/apps/community/src/data-import/mappers/umami.mapper.ts
+++ b/backend/apps/community/src/data-import/mappers/umami.mapper.ts
@@ -2,12 +2,13 @@ import * as fs from 'fs'
import * as path from 'path'
import * as os from 'os'
import * as crypto from 'crypto'
-import { unzipSync } from 'fflate'
+import { Unzip, UnzipInflate } from 'fflate'
import { parse } from 'csv-parse'
import { ImportMapper, AnalyticsImportRow } from './mapper.interface'
const WEBSITE_EVENT_CSV = 'website_event.csv'
+const MAX_UMAMI_CSV_BYTES = 512 * 1024 * 1024
const BROWSER_MAP: Record<string, string> = {
chrome: 'Chrome',
@@ -135,24 +136,172 @@ export class UmamiMapper implements ImportMapper {
readonly provider = 'umami'
readonly expectedFileExtension = '.zip'
- private extractCsvFromZip(zipPath: string): string {
- const zipBuffer = fs.readFileSync(zipPath)
- const files = unzipSync(new Uint8Array(zipBuffer))
+ private async extractCsvFromZip(zipPath: string): Promise<string> {
+ const tempCsv = path.join(os.tmpdir(), `swetrix-umami-${Date.now()}.csv`)
- const matchKey = Object.keys(files).find((name) => {
- const basename = name.split('/').pop() || name
- return basename === WEBSITE_EVENT_CSV
- })
+ return new Promise((resolve, reject) => {
+ const zipStream = fs.createReadStream(zipPath)
+ const unzipper = new Unzip()
+
+ unzipper.register(UnzipInflate)
+
+ let settled = false
+ let foundEntry = false
+ let fileDescriptor: number | null = null
+ let tempFileCreated = false
+ let extractedBytes = 0
+ let terminateEntry: (() => void) | null = null
+
+ const cleanupTempCsv = () => {
+ if (fileDescriptor !== null) {
+ try {
+ fs.closeSync(fileDescriptor)
+ } catch {
+ //
+ }
+ fileDescriptor = null
+ }
- if (!matchKey) {
- throw new Error(
- `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`,
- )
- }
+ if (tempFileCreated) {
+ try {
+ fs.unlinkSync(tempCsv)
+ } catch {
+ //
+ }
+ }
+ }
- const tempCsv = path.join(os.tmpdir(), `swetrix-umami-${Date.now()}.csv`)
- fs.writeFileSync(tempCsv, Buffer.from(files[matchKey]))
- return tempCsv
+ const fail = (error: Error) => {
+ if (settled) return
+ settled = true
+ terminateEntry?.()
+ zipStream.destroy()
+ cleanupTempCsv()
+ reject(error)
+ }
+
+ const succeed = () => {
+ if (settled) return
+ settled = true
+ if (fileDescriptor !== null) {
+ try {
+ fs.closeSync(fileDescriptor)
+ } catch (error) {
+ cleanupTempCsv()
+ reject(error)
+ return
+ }
+ fileDescriptor = null
+ }
+ zipStream.destroy()
+ resolve(tempCsv)
+ }
+
+ unzipper.onfile = (file) => {
+ if (path.basename(file.name) !== WEBSITE_EVENT_CSV || foundEntry) return
+
+ foundEntry = true
+
+ if (
+ typeof file.originalSize !== 'number' ||
+ !Number.isFinite(file.originalSize)
+ ) {
+ fail(
+ new Error(
+ `Unable to validate ${WEBSITE_EVENT_CSV} size in the ZIP archive.`,
+ ),
+ )
+ return
+ }
+
+ if (file.originalSize > MAX_UMAMI_CSV_BYTES) {
+ fail(
+ new Error(
+ `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,
+ ),
+ )
+ return
+ }
+
+ try {
+ fileDescriptor = fs.openSync(tempCsv, 'wx')
+ tempFileCreated = true
+ } catch (error) {
+ fail(error as Error)
+ return
+ }
+
+ terminateEntry = file.terminate
+ file.ondata = (error, chunk, final) => {
+ if (error) {
+ fail(error)
+ return
+ }
+
+ extractedBytes += chunk.length
+ if (extractedBytes > MAX_UMAMI_CSV_BYTES) {
+ fail(
+ new Error(
+ `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,
+ ),
+ )
+ return
+ }
+
+ try {
+ fs.writeSync(fileDescriptor!, chunk)
+ } catch (writeError) {
+ fail(writeError as Error)
+ return
+ }
+
+ if (final) {
+ succeed()
+ }
+ }
+
+ try {
+ file.start()
+ } catch (error) {
+ fail(error as Error)
+ }
+ }
+
+ zipStream.on('data', (chunk: Buffer) => {
+ if (settled) return
+
+ try {
+ unzipper.push(
+ new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength),
+ )
+ } catch (error) {
+ fail(error as Error)
+ }
+ })
+
+ zipStream.on('end', () => {
+ if (settled) return
+
+ try {
+ unzipper.push(new Uint8Array(0), true)
+ } catch (error) {
+ fail(error as Error)
+ return
+ }
+
+ if (!foundEntry) {
+ fail(
+ new Error(
+ `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`,
+ ),
+ )
+ }
+ })
+
+ zipStream.on('error', (error) => {
+ fail(error)
+ })
+ })
}
async *createRowStream(
@@ -160,7 +309,7 @@ export class UmamiMapper implements ImportMapper {
pid: string,
importID: number,
): AsyncIterable<AnalyticsImportRow> {
- const csvPath = this.extractCsvFromZip(filePath)
+ const csvPath = await this.extractCsvFromZip(filePath)
let cleanedUp = false
const cleanup = () => {