Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Clean up server-side import code

Edward Cui727398b826f1
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.ts
index 0be9d43a3..c06bedc56 100644
--- a/client/src/api/admin/import.ts
+++ b/client/src/api/admin/import.ts
@@ -12,17 +12,6 @@ interface GetSiteImportsResponse {
fileName: string;
}
-interface ImportSiteDataParams {
- file: File;
- platform: string;
- startDate?: string;
- endDate?: string;
-}
-
-interface ImportSiteDataResponse {
- message: string;
-}
-
interface DeleteImportResponse {
message: string;
}
@@ -42,36 +31,6 @@ export function useGetSiteImports(site: number) {
});
}
-export function useImportSiteData(site: number) {
- const queryClient = useQueryClient();
-
- return useMutation({
- mutationFn: async (params: ImportSiteDataParams) => {
- const formData = new FormData();
-
- formData.append("platform", params.platform);
- if (params.startDate) {
- formData.append("startDate", params.startDate);
- }
- if (params.endDate) {
- formData.append("endDate", params.endDate);
- }
- formData.append("file", params.file);
-
- return await authedFetch<APIResponse<ImportSiteDataResponse>>(`/import-site-data/${site}`, undefined, {
- method: "POST",
- data: formData,
- });
- },
- onSuccess: async () => {
- await queryClient.invalidateQueries({
- queryKey: ["get-site-imports", site],
- });
- },
- retry: false,
- });
-}
-
export function useDeleteSiteImport(site: number) {
const queryClient = useQueryClient();
diff --git a/server/package-lock.json b/server/package-lock.json
index b2694a47d..d781a901b 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -9,12 +9,9 @@
"version": "2.1.0",
"dependencies": {
"@aws-sdk/client-s3": "^3.842.0",
- "@aws-sdk/lib-storage": "^3.917.0",
"@axiomhq/pino": "^1.3.1",
"@clickhouse/client": "1.11.1",
- "@fast-csv/parse": "^5.0.5",
"@fastify/cors": "11.0.1",
- "@fastify/multipart": "^9.3.0",
"@fastify/one-line-logger": "2.0.2",
"@fastify/rate-limit": "^10.3.0",
"@fastify/static": "8.1.1",
@@ -36,7 +33,6 @@
"node-cache": "^5.1.2",
"node-cron": "4.0.6",
"pg": "8.16.0",
- "pg-boss": "^10.3.2",
"postgres": "3.4.5",
"psl": "^1.15.0",
"react": "19.2.0",
@@ -580,37 +576,6 @@
"node": ">=18.0.0"
}
},
- "node_modules/@aws-sdk/lib-storage": {
- "version": "3.917.0",
- "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.917.0.tgz",
- "integrity": "sha512-Z8mRzfP6PgUoybJHx/tH40aop42yeh66cW7wLM8R88egB2UYQ0IgxDoRynqlLi5uceI21wss/CIDkhzO9p1cOg==",
- "license": "Apache-2.0",
- "dependencies": {
- "@smithy/abort-controller": "^4.2.3",
- "@smithy/middleware-endpoint": "^4.3.5",
- "@smithy/smithy-client": "^4.9.1",
- "buffer": "5.6.0",
- "events": "3.3.0",
- "stream-browserify": "3.0.0",
- "tslib": "^2.6.2"
- },
- "engines": {
- "node": ">=18.0.0"
- },
- "peerDependencies": {
- "@aws-sdk/client-s3": "^3.917.0"
- }
- },
- "node_modules/@aws-sdk/lib-storage/node_modules/buffer": {
- "version": "5.6.0",
- "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
- "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
- "license": "MIT",
- "dependencies": {
- "base64-js": "^1.0.2",
- "ieee754": "^1.1.4"
- }
- },
"node_modules/@aws-sdk/middleware-bucket-endpoint": {
"version": "3.914.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.914.0.tgz",
@@ -1008,19 +973,6 @@
"node": ">=16"
}
},
- "node_modules/@axiomhq/js/node_modules/uuid": {
- "version": "11.1.0",
- "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
- "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
- "funding": [
- "https://github.com/sponsors/broofa",
- "https://github.com/sponsors/ctavan"
- ],
- "license": "MIT",
- "bin": {
- "uuid": "dist/esm/bin/uuid"
- }
- },
"node_modules/@axiomhq/pino": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/@axiomhq/pino/-/pino-1.3.1.tgz",
@@ -2115,20 +2067,6 @@
"node": ">=18"
}
},
- "node_modules/@fast-csv/parse": {
- "version": "5.0.5",
- "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.5.tgz",
- "integrity": "sha512-M0IbaXZDbxfOnpVE5Kps/a6FGlILLhtLsvWd9qNH3d2TxNnpbNkFf3KD26OmJX6MHq7PdQAl5htStDwnuwHx6w==",
- "license": "MIT",
- "dependencies": {
- "lodash.escaperegexp": "^4.1.2",
- "lodash.groupby": "^4.6.0",
- "lodash.isfunction": "^3.0.9",
- "lodash.isnil": "^4.0.0",
- "lodash.isundefined": "^3.0.1",
- "lodash.uniq": "^4.5.0"
- }
- },
"node_modules/@fastify/accept-negotiator": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-2.0.1.tgz",
@@ -2166,12 +2104,6 @@
"fast-uri": "^3.0.0"
}
},
- "node_modules/@fastify/busboy": {
- "version": "3.1.1",
- "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-3.1.1.tgz",
- "integrity": "sha512-5DGmA8FTdB2XbDeEwc/5ZXBl6UbBAyBOOLlPuBnZ/N1SwdH9Ii+cOX3tBROlDgcTXxjOYnLMVoKk9+FXAw0CJw==",
- "license": "MIT"
- },
"node_modules/@fastify/cors": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-11.0.1.tgz",
@@ -2192,22 +2124,6 @@
"toad-cache": "^3.7.0"
}
},
- "node_modules/@fastify/deepmerge": {
- "version": "3.1.0",
- "resolved": "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-3.1.0.tgz",
- "integrity": "sha512-lCVONBQINyNhM6LLezB6+2afusgEYR4G8xenMsfe+AT+iZ7Ca6upM5Ha8UkZuYSnuMw3GWl/BiPXnLMi/gSxuQ==",
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/fastify"
- },
- {
- "type": "opencollective",
- "url": "https://opencollective.com/fastify"
- }
- ],
- "license": "MIT"
- },
"node_modules/@fastify/error": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.1.0.tgz",
@@ -2268,29 +2184,6 @@
"dequal": "^2.0.3"
}
},
- "node_modules/@fastify/multipart": {
- "version": "9.3.0",
- "resolved": "https://registry.npmjs.org/@fastify/multipart/-/multipart-9.3.0.tgz",
- "integrity": "sha512-NpeKipTOjjL1dA7SSlRMrOWWtrE8/0yKOmeudkdQoEaz4sVDJw5MVdZIahsWhvpc3YTN7f04f9ep/Y65RKoOWA==",
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/fastify"
- },
- {
- "type": "opencollective",
- "url": "https://opencollective.com/fastify"
- }
- ],
- "license": "MIT",
- "dependencies": {
- "@fastify/busboy": "^3.0.0",
- "@fastify/deepmerge": "^3.0.0",
- "@fastify/error": "^4.0.0",
- "fastify-plugin": "^5.0.0",
- "secure-json-parse": "^4.0.0"
- }
- },
"node_modules/@fastify/one-line-logger": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@fastify/one-line-logger/-/one-line-logger-2.0.2.tgz",
@@ -2394,9 +2287,9 @@
"license": "MIT"
},
"node_modules/@ioredis/commands": {
- "version": "1.2.0",
- "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
- "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz",
+ "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==",
"license": "MIT"
},
"node_modules/@isaacs/cliui": {
@@ -4935,9 +4828,9 @@
"license": "MIT"
},
"node_modules/bullmq": {
- "version": "5.56.5",
- "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.56.5.tgz",
- "integrity": "sha512-nhcVxoE9Y0YUuNYtvaD+N0Bk2kqcU+rXzJwdQIr8i8qC/fxoghwUYb9a+CidTv24pi1eqstLnBoa8xkR/P7Mdw==",
+ "version": "5.63.0",
+ "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.63.0.tgz",
+ "integrity": "sha512-HT1iM3Jt4bZeg3Ru/MxrOy2iIItxcl1Pz5Ync1Vrot70jBpVguMxFEiSaDU57BwYwR4iwnObDnzct2lirKkX5A==",
"license": "MIT",
"dependencies": {
"cron-parser": "^4.9.0",
@@ -4946,7 +4839,7 @@
"node-abort-controller": "^3.1.1",
"semver": "^7.5.4",
"tslib": "^2.0.0",
- "uuid": "^9.0.0"
+ "uuid": "^11.1.0"
}
},
"node_modules/cac": {
@@ -6374,12 +6267,12 @@
"license": "ISC"
},
"node_modules/ioredis": {
- "version": "5.6.1",
- "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",
- "integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",
+ "version": "5.8.2",
+ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz",
+ "integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==",
"license": "MIT",
"dependencies": {
- "@ioredis/commands": "^1.1.1",
+ "@ioredis/commands": "1.4.0",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
@@ -6795,18 +6688,6 @@
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
"license": "MIT"
},
- "node_modules/lodash.escaperegexp": {
- "version": "4.1.2",
- "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz",
- "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==",
- "license": "MIT"
- },
- "node_modules/lodash.groupby": {
- "version": "4.6.0",
- "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
- "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==",
- "license": "MIT"
- },
"node_modules/lodash.includes": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
@@ -6825,24 +6706,12 @@
"integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
"license": "MIT"
},
- "node_modules/lodash.isfunction": {
- "version": "3.0.9",
- "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz",
- "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==",
- "license": "MIT"
- },
"node_modules/lodash.isinteger": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
"integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==",
"license": "MIT"
},
- "node_modules/lodash.isnil": {
- "version": "4.0.0",
- "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz",
- "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==",
- "license": "MIT"
- },
"node_modules/lodash.isnumber": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
@@ -6861,24 +6730,12 @@
"integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
"license": "MIT"
},
- "node_modules/lodash.isundefined": {
- "version": "3.0.1",
- "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz",
- "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==",
- "license": "MIT"
- },
"node_modules/lodash.once": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
"integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
"license": "MIT"
},
- "node_modules/lodash.uniq": {
- "version": "4.5.0",
- "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
- "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==",
- "license": "MIT"
- },
"node_modules/loupe": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.3.tgz",
@@ -7430,20 +7287,6 @@
}
}
},
- "node_modules/pg-boss": {
- "version": "10.3.2",
- "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-10.3.2.tgz",
- "integrity": "sha512-yy96jwKekgUW4DVvve5xjvLZlNqVCRayl1wJvBxQeDNhIrblgEG0YnqbcNth4wsehkYWVJND9TeWbqb+cfxJKQ==",
- "license": "MIT",
- "dependencies": {
- "cron-parser": "^4.9.0",
- "pg": "^8.16.0",
- "serialize-error": "^8.1.0"
- },
- "engines": {
- "node": ">=20"
- }
- },
"node_modules/pg-cloudflare": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.5.tgz",
@@ -8321,21 +8164,6 @@
"node": ">=10"
}
},
- "node_modules/serialize-error": {
- "version": "8.1.0",
- "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
- "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
- "license": "MIT",
- "dependencies": {
- "type-fest": "^0.20.2"
- },
- "engines": {
- "node": ">=10"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/set-cookie-parser": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.1.tgz",
@@ -8599,16 +8427,6 @@
"integrity": "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==",
"dev": true
},
- "node_modules/stream-browserify": {
- "version": "3.0.0",
- "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz",
- "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==",
- "license": "MIT",
- "dependencies": {
- "inherits": "~2.0.4",
- "readable-stream": "^3.5.0"
- }
- },
"node_modules/string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -9065,18 +8883,6 @@
"node": ">= 6"
}
},
- "node_modules/type-fest": {
- "version": "0.20.2",
- "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
- "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
- "license": "(MIT OR CC0-1.0)",
- "engines": {
- "node": ">=10"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/typescript": {
"version": "5.8.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz",
@@ -9171,16 +8977,16 @@
"license": "MIT"
},
"node_modules/uuid": {
- "version": "9.0.1",
- "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
- "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
+ "version": "11.1.0",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
+ "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
- "uuid": "dist/bin/uuid"
+ "uuid": "dist/esm/bin/uuid"
}
},
"node_modules/vite": {
diff --git a/server/package.json b/server/package.json
index 41aa72c84..6d14bd6e1 100644
--- a/server/package.json
+++ b/server/package.json
@@ -24,12 +24,9 @@
},
"dependencies": {
"@aws-sdk/client-s3": "^3.842.0",
- "@aws-sdk/lib-storage": "^3.917.0",
"@axiomhq/pino": "^1.3.1",
"@clickhouse/client": "1.11.1",
- "@fast-csv/parse": "^5.0.5",
"@fastify/cors": "11.0.1",
- "@fastify/multipart": "^9.3.0",
"@fastify/one-line-logger": "2.0.2",
"@fastify/rate-limit": "^10.3.0",
"@fastify/static": "8.1.1",
@@ -51,7 +48,6 @@
"node-cache": "^5.1.2",
"node-cron": "4.0.6",
"pg": "8.16.0",
- "pg-boss": "^10.3.2",
"postgres": "3.4.5",
"psl": "^1.15.0",
"react": "19.2.0",
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
deleted file mode 100644
index 3dfc00309..000000000
--- a/server/src/api/sites/importSiteData.ts
+++ /dev/null
@@ -1,180 +0,0 @@
-import { createWriteStream } from "node:fs";
-import { mkdir } from "node:fs/promises";
-import { dirname } from "node:path";
-import { pipeline } from "node:stream/promises";
-import { FastifyRequest, FastifyReply } from "fastify";
-import { DateTime } from "luxon";
-import { z } from "zod";
-import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
-import { getJobQueue } from "../../services/import/queues/jobQueueFactory.js";
-import { ImportLimiter } from "../../services/import/importLimiter.js";
-import { updateImportStatus } from "../../services/import/importStatusManager.js";
-import { deleteImportFile, getImportStorageLocation } from "../../services/import/utils.js";
-import { CSV_PARSE_QUEUE, CsvParseJob } from "../../services/import/workers/jobs.js";
-import { r2Storage } from "../../services/storage/r2StorageService.js";
-
-const isValidDate = (date: string) => {
- const dt = DateTime.fromFormat(date, "yyyy-MM-dd", { zone: "utc" });
- return dt.isValid;
-};
-
-const parseDate = (date: string) => DateTime.fromFormat(date, "yyyy-MM-dd", { zone: "utc" });
-
-const importDataFieldsSchema = z
- .object({
- fields: z
- .object({
- platform: z.enum(["umami"]),
- startDate: z.string().refine(isValidDate, { message: "Invalid start date format" }).optional(),
- endDate: z.string().refine(isValidDate, { message: "Invalid end date format" }).optional(),
- })
- .refine(
- data => {
- if (!data.startDate || !data.endDate) return true;
- return parseDate(data.startDate) <= parseDate(data.endDate);
- },
- { message: "Start date must be before or equal to end date" }
- )
- .refine(
- data => {
- if (!data.startDate) return true;
- const today = DateTime.utc().startOf("day");
- return parseDate(data.startDate) <= today;
- },
- { message: "Start date cannot be in the future" }
- ),
- })
- .strict();
-
-const importDataRequestSchema = z
- .object({
- params: z.object({
- site: z.string().min(1),
- }),
- })
- .strict();
-
-type ImportDataRequest = {
- Params: z.infer<typeof importDataRequestSchema.shape.params>;
-};
-
-export async function importSiteData(request: FastifyRequest<ImportDataRequest>, reply: FastifyReply) {
- try {
- const jobQueue = getJobQueue();
- if (!jobQueue.isReady()) {
- return reply.status(503).send({ error: "Import system is temporarily unavailable. Please try again later." });
- }
-
- const parsedParams = importDataRequestSchema.safeParse({
- params: request.params,
- });
-
- if (!parsedParams.success) {
- return reply.status(400).send({ error: "Validation error" });
- }
-
- const { site } = parsedParams.data.params;
-
- const userHasAccess = await getUserHasAdminAccessToSite(request, site);
- if (!userHasAccess) {
- return reply.status(403).send({ error: "Forbidden" });
- }
-
- const data = await request.file();
- if (!data) {
- return reply.status(400).send({ error: "No file uploaded." });
- }
-
- if (data.mimetype !== "text/csv" || !data.filename.endsWith(".csv")) {
- return reply.status(400).send({ error: "Invalid file type. Only .csv files are accepted." });
- }
-
- const parsedFields = importDataFieldsSchema.safeParse({
- fields: {
- platform: (data.fields.platform as any)?.value,
- startDate: (data.fields.startDate as any)?.value,
- endDate: (data.fields.endDate as any)?.value,
- },
- });
-
- if (!parsedFields.success) {
- return reply.status(400).send({ error: "Validation error" });
- }
-
- const { platform, startDate, endDate } = parsedFields.data.fields;
- const siteId = Number(site);
-
- // Check organization and get initial limit check
- const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(siteId);
- if (!concurrentImportLimitResult.allowed) {
- return reply.status(429).send({ error: concurrentImportLimitResult.reason });
- }
-
- const organization = concurrentImportLimitResult.organizationId;
-
- // Atomically create import status with concurrency check to prevent race conditions
- const createResult = await ImportLimiter.createImportWithConcurrencyCheck({
- siteId,
- organizationId: organization,
- platform,
- fileName: data.filename,
- });
-
- if (!createResult.success) {
- return reply.status(429).send({ error: createResult.reason });
- }
-
- const importId = createResult.importId;
-
- const storage = getImportStorageLocation(importId, data.filename);
-
- try {
- if (storage.isR2) {
- await r2Storage.storeImportFile(storage.location, data.file);
- console.log(`[Import] File streamed to R2: ${storage.location}`);
- } else {
- const importDir = dirname(storage.location);
- await mkdir(importDir, { recursive: true });
- await pipeline(data.file, createWriteStream(storage.location));
- console.log(`[Import] File stored locally: ${storage.location}`);
- }
-
- if (data.file.truncated) {
- await deleteImportFile(storage.location, storage.isR2);
- await updateImportStatus(importId, "failed", "File too large");
- return reply.status(413).send({ error: "File too large." });
- }
- } catch (fileError) {
- await updateImportStatus(importId, "failed", "Failed to save uploaded file");
- console.error("Failed to save uploaded file:", fileError);
- return reply.status(500).send({ error: "Could not process file upload." });
- }
-
- try {
- await jobQueue.send<CsvParseJob>(CSV_PARSE_QUEUE, {
- site,
- importId,
- platform,
- storageLocation: storage.location,
- isR2Storage: storage.isR2,
- organization,
- startDate,
- endDate,
- });
- } catch (queueError) {
- await updateImportStatus(importId, "failed", "Failed to queue import job");
- await deleteImportFile(storage.location, storage.isR2);
- console.error("Failed to enqueue import job:", queueError);
- return reply.status(500).send({ error: "Failed to initiate import process." });
- }
-
- return reply.status(202).send({
- data: {
- message: "File upload accepted and is now being processed.",
- },
- });
- } catch (error) {
- console.error("Unexpected error during import:", error);
- return reply.status(500).send({ error: "An unexpected error occurred. Please try again later." });
- }
-}
diff --git a/server/src/index.ts b/server/src/index.ts
index 2226f71eb..fcd92ffa6 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -1,6 +1,5 @@
import cors from "@fastify/cors";
import fastifyStatic from "@fastify/static";
-import multipart from "@fastify/multipart";
import { toNodeHandler } from "better-auth/node";
import Fastify from "fastify";
import { dirname, join } from "path";
@@ -87,10 +86,8 @@ import { disconnectGSC } from "./api/gsc/disconnect.js";
import { getGSCData } from "./api/gsc/getData.js";
import { selectGSCProperty } from "./api/gsc/selectProperty.js";
import { getSiteImports } from "./api/sites/getSiteImports.js";
-import { importSiteData } from "./api/sites/importSiteData.js";
import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";
import { batchImportEvents } from "./api/sites/batchImportEvents.js";
-import { getJobQueue } from "./services/import/queues/jobQueueFactory.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -168,13 +165,6 @@ server.register(cors, {
credentials: true,
});
-server.register(multipart, {
- limits: {
- fileSize: IS_CLOUD ? 500 * 1024 * 1024 : 50 * 1024 * 1024 * 1024, // 500 MB for cloud, 50 GB for self-hosted
- files: 1, // only allow 1 file
- },
-});
-
// Serve static files
server.register(fastifyStatic, {
root: join(__dirname, "../public"),
@@ -360,13 +350,9 @@ server.get("/api/session-replay/list/:site", getSessionReplays);
server.get("/api/session-replay/:sessionId/:site", getSessionReplayEvents);
// Imports
-server.post("/api/import-site-data/:site", {
- bodyLimit: IS_CLOUD ? 500 * 1024 * 1024 : 50 * 1024 * 1024 * 1024, // 500 MB for cloud, 50 GB for self-hosted (matches multipart fileSize)
-}, importSiteData);
+server.post("/api/batch-import-events/:site", batchImportEvents);
server.get("/api/get-site-imports/:site", getSiteImports);
server.delete("/api/delete-site-import/:site/:importId", deleteSiteImport);
-// Client-side import endpoint (parses CSV, server transforms)
-server.post("/api/batch-import-events/:site", batchImportEvents);
// Administrative
server.get("/api/config", getConfig);
@@ -472,17 +458,7 @@ const start = async () => {
weeklyReportService.startWeeklyReportCron();
}
- // Initialize and start import job queue system
- const jobQueue = getJobQueue();
- try {
- await jobQueue.start();
- } catch (queueError) {
- server.log.error("Failed to initialize import job queue system:", queueError);
- server.log.warn("Server will continue without import functionality");
- // Don't exit - allow server to run without import features
- }
-
- // Start the server first
+ // Start the server
await server.listen({ port: 3001, host: "0.0.0.0" });
server.log.info("Server is listening on http://0.0.0.0:3001");
@@ -504,8 +480,6 @@ const start = async () => {
// });
// }
} catch (err) {
- const jobQueue = getJobQueue();
- await jobQueue.stop();
server.log.error(err);
process.exit(1);
}
@@ -536,11 +510,6 @@ const shutdown = async (signal: string) => {
await server.close();
server.log.info("Server closed");
- // Shutdown job queue system
- const jobQueue = getJobQueue();
- await jobQueue.stop();
- server.log.info("Job queue shut down");
-
// Shutdown uptime service
// await uptimeService.shutdown();
// server.log.info("Uptime service shut down");
diff --git a/server/src/lib/const.ts b/server/src/lib/const.ts
index 741ea1b15..f2b2ac945 100644
--- a/server/src/lib/const.ts
+++ b/server/src/lib/const.ts
@@ -3,7 +3,6 @@ import dotenv from "dotenv";
dotenv.config();
export const IS_CLOUD = process.env.CLOUD === "true";
-export const IS_PRODUCTION = process.env.NODE_ENV === "production";
export const DISABLE_SIGNUP = process.env.DISABLE_SIGNUP === "true";
export const DISABLE_TELEMETRY = process.env.DISABLE_TELEMETRY === "true";
export const SECRET = process.env.BETTER_AUTH_SECRET;
diff --git a/server/src/services/import/queues/adapters/bullMQAdapter.ts b/server/src/services/import/queues/adapters/bullMQAdapter.ts
deleted file mode 100644
index 8bacbf93d..000000000
--- a/server/src/services/import/queues/adapters/bullMQAdapter.ts
+++ /dev/null
@@ -1,199 +0,0 @@
-import { Job, Queue, QueueEvents, Worker } from "bullmq";
-import { IJobQueue } from "../jobQueue.js";
-import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../workers/jobs.js";
-import { createCsvParseWorker } from "../../workers/csvParseWorker.js";
-import { createDataInsertWorker } from "../../workers/dataInsertWorker.js";
-import { createServiceLogger } from "../../../../lib/logger/logger.js";
-
-const logger = createServiceLogger("import:bullmq");
-
-export class BullMQAdapter implements IJobQueue {
- private readonly connection: { host: string; port: number; password?: string };
- private queues: Map<string, Queue> = new Map();
- private workers: Map<string, Worker> = new Map();
- private queueEvents: Map<string, QueueEvents> = new Map();
- private ready: boolean = false;
-
- constructor() {
- this.connection = {
- host: process.env.REDIS_HOST || "localhost",
- port: parseInt(process.env.REDIS_PORT || "6379", 10),
- ...(process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD }),
- };
- }
-
- async start(): Promise<void> {
- try {
- await this.createQueue(CSV_PARSE_QUEUE);
- await this.createQueue(DATA_INSERT_QUEUE);
-
- await createCsvParseWorker(this);
- await createDataInsertWorker(this);
-
- this.ready = true;
- logger.info("Started successfully");
- } catch (error) {
- this.ready = false;
- logger.error({ error }, "Failed to initialize");
- throw error;
- }
- }
-
- async stop(): Promise<void> {
- logger.info("Stopping...");
- const errors: Error[] = [];
-
- // Close all workers first
- const workerResults = await Promise.allSettled(Array.from(this.workers.values()).map(worker => worker.close()));
- workerResults.forEach((result, index) => {
- if (result.status === "rejected") {
- const workerName = Array.from(this.workers.keys())[index];
- const error = new Error(`Failed to close worker ${workerName}: ${result.reason}`);
- errors.push(error);
- logger.error({ workerName, error: result.reason }, error.message);
- }
- });
- this.workers.clear();
-
- // Close all queue events
- const queueEventsResults = await Promise.allSettled(Array.from(this.queueEvents.values()).map(qe => qe.close()));
- queueEventsResults.forEach((result, index) => {
- if (result.status === "rejected") {
- const queueName = Array.from(this.queueEvents.keys())[index];
- const error = new Error(`Failed to close queue events for ${queueName}: ${result.reason}`);
- errors.push(error);
- logger.error({ queueName, error: result.reason }, error.message);
- }
- });
- this.queueEvents.clear();
-
- // Close all queues (this closes their Redis connections)
- const queueResults = await Promise.allSettled(Array.from(this.queues.values()).map(queue => queue.close()));
- queueResults.forEach((result, index) => {
- if (result.status === "rejected") {
- const queueName = Array.from(this.queues.keys())[index];
- const error = new Error(`Failed to close queue ${queueName}: ${result.reason}`);
- errors.push(error);
- logger.error({ queueName, error: result.reason }, error.message);
- }
- });
- this.queues.clear();
-
- if (errors.length > 0) {
- const aggregatedError = new Error(
- `BullMQ stop encountered ${errors.length} error(s): ${errors.map(e => e.message).join("; ")}`
- );
- logger.error({ errorCount: errors.length }, "Stopped with errors");
- throw aggregatedError;
- }
-
- logger.info("Stopped successfully");
- }
-
- async createQueue(queueName: string): Promise<void> {
- if (!this.queues.has(queueName)) {
- let queue: Queue | undefined;
- let queueEvents: QueueEvents | undefined;
-
- try {
- queue = new Queue(queueName, {
- connection: this.connection,
- defaultJobOptions: {
- attempts: 1,
- removeOnComplete: true,
- removeOnFail: true,
- },
- });
-
- this.queues.set(queueName, queue);
-
- queueEvents = new QueueEvents(queueName, {
- connection: this.connection,
- });
-
- this.queueEvents.set(queueName, queueEvents);
-
- queueEvents.on("completed", ({ jobId }) => {
- logger.debug({ jobId, queueName }, "Job completed");
- });
-
- queueEvents.on("failed", ({ jobId, failedReason }) => {
- logger.error({ jobId, queueName, failedReason }, "Job failed");
- });
- } catch (error) {
- logger.error({ queueName, error }, "Failed to create queue");
-
- const cleanupPromises: Promise<void>[] = [];
-
- if (queueEvents) {
- this.queueEvents.delete(queueName);
- cleanupPromises.push(
- queueEvents.close().catch(closeError => {
- logger.error({ queueName, error: closeError }, "Failed to close queue events during cleanup");
- })
- );
- }
-
- if (queue) {
- this.queues.delete(queueName);
- cleanupPromises.push(
- queue.close().catch(closeError => {
- logger.error({ queueName, error: closeError }, "Failed to close queue during cleanup");
- })
- );
- }
-
- await Promise.all(cleanupPromises);
-
- throw error;
- }
- }
- }
-
- async send<T>(queueName: string, data: T): Promise<void> {
- const queue = this.queues.get(queueName);
- if (!queue) {
- throw new Error(`Queue ${queueName} not found. Call createQueue first.`);
- }
-
- try {
- await queue.add(queueName, data);
- } catch (error) {
- const errorMessage = error instanceof Error ? error.message : String(error);
- logger.error({ queueName, error }, "Failed to enqueue job");
- throw new Error(`Failed to enqueue job to ${queueName}: ${errorMessage}`);
- }
- }
-
- async work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void> {
- try {
- const worker = new Worker(
- queueName,
- async (job: Job<T>) => {
- await handler(job.data);
- },
- {
- connection: this.connection,
- concurrency: 1,
- }
- );
-
- worker.on("error", error => {
- logger.error({ queueName, error }, "Worker error");
- });
-
- worker.on("failed", (job, error) => {
- logger.error({ queueName, jobId: job?.id, error }, "Job failed");
- });
-
- this.workers.set(queueName, worker);
- } catch (error) {
- logger.error({ queueName, error }, "Failed to create worker");
- throw error;
- }
- }
-
- isReady(): boolean {
- return this.ready;
- }
-}
diff --git a/server/src/services/import/queues/adapters/pgBossAdapter.ts b/server/src/services/import/queues/adapters/pgBossAdapter.ts
deleted file mode 100644
index 720f8c60d..000000000
--- a/server/src/services/import/queues/adapters/pgBossAdapter.ts
+++ /dev/null
@@ -1,79 +0,0 @@
-import PgBoss, { Job } from "pg-boss";
-import { IJobQueue } from "../jobQueue.js";
-import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../workers/jobs.js";
-import { createCsvParseWorker } from "../../workers/csvParseWorker.js";
-import { createDataInsertWorker } from "../../workers/dataInsertWorker.js";
-import { createServiceLogger } from "../../../../lib/logger/logger.js";
-
-const logger = createServiceLogger("import:pgboss");
-
-export class PgBossAdapter implements IJobQueue {
- private boss: PgBoss;
- private ready: boolean = false;
-
- constructor() {
- this.boss = new PgBoss({
- host: process.env.POSTGRES_HOST || "postgres",
- port: parseInt(process.env.POSTGRES_PORT || "5432", 10),
- database: process.env.POSTGRES_DB,
- user: process.env.POSTGRES_USER,
- password: process.env.POSTGRES_PASSWORD,
- schema: "pgboss",
- application_name: "data-import-system",
- retryLimit: 0,
- });
-
- this.boss.on("error", error => {
- logger.error({ error }, "PgBoss error");
- });
- }
-
- async start(): Promise<void> {
- try {
- await this.boss.start();
-
- await this.createQueue(CSV_PARSE_QUEUE);
- await this.createQueue(DATA_INSERT_QUEUE);
-
- await createCsvParseWorker(this);
- await createDataInsertWorker(this);
-
- this.ready = true;
- logger.info("Started successfully");
- } catch (error) {
- this.ready = false;
- logger.error({ error }, "Failed to initialize");
- throw error;
- }
- }
-
- async stop(): Promise<void> {
- await this.boss.stop();
- logger.info("Stopped successfully");
- }
-
- async createQueue(queueName: string): Promise<void> {
- await this.boss.createQueue(queueName);
- }
-
- async send<T>(queueName: string, data: T): Promise<void> {
- await this.boss.send(queueName, data as object);
- }
-
- async work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void> {
- await this.boss.work(
- queueName,
- {
- batchSize: 1,
- pollingIntervalSeconds: 3,
- },
- async ([job]: Job<T>[]) => {
- await handler(job.data);
- }
- );
- }
-
- isReady(): boolean {
- return this.ready;
- }
-}
diff --git a/server/src/services/import/queues/jobQueue.ts b/server/src/services/import/queues/jobQueue.ts
deleted file mode 100644
index e83ee669c..000000000
--- a/server/src/services/import/queues/jobQueue.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-export interface IJobQueue {
- start(): Promise<void>;
- stop(): Promise<void>;
- createQueue(queueName: string): Promise<void>;
- send<T>(queueName: string, data: T): Promise<void>;
- work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void>;
- isReady(): boolean;
-}
diff --git a/server/src/services/import/queues/jobQueueFactory.ts b/server/src/services/import/queues/jobQueueFactory.ts
deleted file mode 100644
index bd59d628d..000000000
--- a/server/src/services/import/queues/jobQueueFactory.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import { IS_CLOUD } from "../../../lib/const.js";
-import { IJobQueue } from "./jobQueue.js";
-import { PgBossAdapter } from "./adapters/pgBossAdapter.js";
-import { BullMQAdapter } from "./adapters/bullMQAdapter.js";
-
-let queueInstance: IJobQueue | null = null;
-
-/**
- * Get the singleton job queue instance
- * Returns BullMQ for cloud deployments, pg-boss for self-hosted
- */
-export const getJobQueue = (): IJobQueue => {
- if (!queueInstance) {
- if (IS_CLOUD) {
- queueInstance = new BullMQAdapter();
- } else {
- queueInstance = new PgBossAdapter();
- }
- }
- return queueInstance;
-};
diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.ts
deleted file mode 100644
index 01892bd2d..000000000
--- a/server/src/services/import/utils.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import { unlink } from "node:fs/promises";
-import { IS_CLOUD } from "../../lib/const.js";
-import { r2Storage } from "../storage/r2StorageService.js";
-import { createServiceLogger } from "../../lib/logger/logger.js";
-
-const logger = createServiceLogger("import:utils");
-
-export interface StorageLocationInfo {
- location: string;
- isR2: boolean;
-}
-
-export function getImportStorageLocation(importId: string, filename: string): StorageLocationInfo {
- const isR2 = IS_CLOUD && r2Storage.isEnabled();
- if (isR2) {
- return {
- location: `imports/${importId}/${filename}`,
- isR2: true,
- };
- }
- return {
- location: `/tmp/imports/${importId}/${filename}`,
- isR2: false,
- };
-}
-
-export const deleteImportFile = async (storageLocation: string, isR2Storage: boolean): Promise<void> => {
- if (isR2Storage) {
- await r2Storage.deleteImportFile(storageLocation);
- logger.debug({ storageLocation }, "Deleted R2 file");
- } else {
- await unlink(storageLocation);
- logger.debug({ storageLocation }, "Deleted local file");
- }
-};
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
deleted file mode 100644
index 8dcd50abf..000000000
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ /dev/null
@@ -1,221 +0,0 @@
-import { access, constants } from "node:fs/promises";
-import { createReadStream } from "node:fs";
-import { parse } from "@fast-csv/parse";
-import { DateTime } from "luxon";
-import { IJobQueue } from "../queues/jobQueue.js";
-import { r2Storage } from "../../storage/r2StorageService.js";
-import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE, DataInsertJob } from "./jobs.js";
-import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
-import { updateImportStatus } from "../importStatusManager.js";
-import { deleteImportFile } from "../utils.js";
-import { ImportQuotaTracker } from "../importQuotaChecker.js";
-import { createServiceLogger } from "../../../lib/logger/logger.js";
-
-const logger = createServiceLogger("import:csv-parser");
-
-const getImportDataHeaders = (platform: string) => {
- switch (platform) {
- case "umami":
- return umamiHeaders;
- default:
- throw new Error(`Unsupported platform: ${platform}`);
- }
-};
-
-const safeUpdateStatusToFailed = async (importId: string, errorMessage: string) => {
- try {
- await updateImportStatus(importId, "failed", errorMessage);
- } catch (updateError) {
- logger.error({ importId, error: updateError }, "Could not update status to failed");
- }
-};
-
-const createR2FileStream = async (storageLocation: string, platform: string) => {
- logger.info({ storageLocation }, "Reading from R2");
- const fileStream = await r2Storage.getImportFileStream(storageLocation);
- return fileStream.pipe(
- parse({
- headers: getImportDataHeaders(platform),
- renameHeaders: true,
- ignoreEmpty: true,
- })
- );
-};
-
-const createLocalFileStream = async (storageLocation: string, platform: string) => {
- logger.info({ storageLocation }, "Reading from local disk");
- await access(storageLocation, constants.F_OK | constants.R_OK);
- return createReadStream(storageLocation).pipe(
- parse({
- headers: getImportDataHeaders(platform),
- renameHeaders: true,
- ignoreEmpty: true,
- })
- );
-};
-
-const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {
- const startDate = startDateStr
- ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")
- : null;
- const endDate = endDateStr ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day") : null;
-
- if (startDate && !startDate.isValid) {
- throw new Error(`Invalid start date: ${startDateStr}`);
- }
-
- if (endDate && !endDate.isValid) {
- throw new Error(`Invalid end date: ${endDateStr}`);
- }
-
- return (dateStr: string) => {
- const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
- if (!createdAt.isValid) {
- return false;
- }
-
- if (startDate && createdAt < startDate) {
- return false;
- }
-
- if (endDate && createdAt > endDate) {
- return false;
- }
-
- return true;
- };
-};
-
-export async function createCsvParseWorker(jobQueue: IJobQueue) {
- await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, async job => {
- const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job;
-
- let stream: ReturnType<typeof parse> | null = null;
- let processingTimeout: NodeJS.Timeout | null = null;
-
- try {
- const quotaTracker = await ImportQuotaTracker.create(organization);
-
- const chunkSize = 5000;
- const INACTIVITY_TIMEOUT_MS = 90 * 1000;
-
- let chunk: UmamiEvent[] = [];
- let totalSkippedQuota = 0;
- let totalProcessed = 0;
-
- stream = isR2Storage
- ? await createR2FileStream(storageLocation, platform)
- : await createLocalFileStream(storageLocation, platform);
-
- stream.on("error", error => {
- logger.error({ importId, error }, "Stream error");
- });
-
- await updateImportStatus(importId, "processing");
- logger.info({ importId, platform, organization }, "Started processing CSV import");
-
- const resetTimeout = () => {
- if (processingTimeout) {
- clearTimeout(processingTimeout);
- }
- processingTimeout = setTimeout(() => {
- if (stream) {
- stream.destroy(new Error("Import processing inactivity timeout exceeded"));
- }
- }, INACTIVITY_TIMEOUT_MS);
- };
-
- // Set initial timeout
- resetTimeout();
-
- const isDateInRange = createDateRangeFilter(startDate, endDate);
-
- for await (const data of stream) {
- // Skip rows with missing or invalid dates
- if (!data.created_at) {
- continue;
- }
-
- // Apply user-specified date range filter
- if (!isDateInRange(data.created_at)) {
- continue;
- }
-
- // Check per-month quota (includes historical window check)
- if (!quotaTracker.canImportEvent(data.created_at)) {
- totalSkippedQuota++;
- continue;
- }
-
- chunk.push(data);
- totalProcessed++;
-
- if (chunk.length >= chunkSize) {
- await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
- site,
- importId,
- platform,
- chunk,
- allChunksSent: false,
- });
- chunk = [];
- resetTimeout();
- }
- }
-
- if (chunk.length > 0) {
- await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
- site,
- importId,
- platform,
- chunk,
- allChunksSent: false,
- });
- resetTimeout();
- }
-
- if (totalSkippedQuota > 0) {
- const quotaSummary = quotaTracker.getSummary();
- const errorMessage =
- `${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
- `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +
- `Try importing newer data or upgrade your plan for higher monthly quotas.`;
- await updateImportStatus(importId, "failed", errorMessage);
- await deleteImportFile(storageLocation, isR2Storage);
- return;
- }
-
- logger.info({ importId, totalProcessed, totalSkippedQuota }, "CSV parsing completed successfully");
- await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
- site,
- importId,
- platform,
- chunk: [],
- allChunksSent: true,
- });
- } catch (error) {
- logger.error({ importId, error }, "Error in CSV parse worker");
- await safeUpdateStatusToFailed(importId, "An unexpected error occurred during import processing");
- logger.error({ importId }, "Import failed, worker continuing");
- } finally {
- if (processingTimeout) {
- clearTimeout(processingTimeout);
- }
-
- if (stream) {
- try {
- stream.destroy();
- } catch (streamError) {
- logger.warn({ importId, error: streamError }, "Failed to destroy stream");
- }
- }
-
- try {
- await deleteImportFile(storageLocation, isR2Storage);
- } catch (deleteError) {
- logger.warn({ importId, error: deleteError }, "File cleanup failed, will remain in storage");
- // File will be orphaned but import status is already recorded
- }
- }
- });
-}
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
deleted file mode 100644
index cdab8a131..000000000
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ /dev/null
@@ -1,68 +0,0 @@
-import { IJobQueue } from "../queues/jobQueue.js";
-import { UmamiImportMapper } from "../mappings/umami.js";
-import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { updateImportStatus, updateImportProgress } from "../importStatusManager.js";
-import { createServiceLogger } from "../../../lib/logger/logger.js";
-
-const logger = createServiceLogger("import:data-insert");
-
-const getImportDataMapping = (platform: string) => {
- switch (platform) {
- case "umami":
- return UmamiImportMapper;
- default:
- throw new Error(`Unsupported platform: ${platform}`);
- }
-};
-
-const safeUpdateStatusToFailed = async (importId: string, errorMessage: string) => {
- try {
- await updateImportStatus(importId, "failed", errorMessage);
- } catch (updateError) {
- logger.error({ importId, error: updateError }, "Could not update status to failed");
- }
-};
-
-export async function createDataInsertWorker(jobQueue: IJobQueue) {
- await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {
- const { site, importId, platform, chunk, allChunksSent } = job;
-
- try {
- if (allChunksSent) {
- await updateImportStatus(importId, "completed");
- logger.info({ importId }, "Import completed successfully");
- return;
- }
-
- const dataMapper = getImportDataMapping(platform);
- const transformedRecords = dataMapper.transform(chunk, site, importId);
-
- await clickhouse.insert({
- table: "events",
- values: transformedRecords,
- format: "JSONEachRow",
- });
-
- logger.debug({ importId, recordCount: transformedRecords.length }, "Inserted chunk to ClickHouse");
-
- // Update progress (non-critical - log if fails but don't crash)
- try {
- await updateImportProgress(importId, transformedRecords.length);
- } catch (progressError) {
- logger.warn(
- { importId, error: progressError instanceof Error ? progressError.message : progressError },
- "Progress update failed (data inserted successfully)"
- );
- // Don't throw - data is safely in ClickHouse, progress can be off slightly
- }
- } catch (error) {
- logger.error({ importId, error }, "Worker processing failed");
-
- const errorMessage = allChunksSent ? "Failed to complete import" : "Data insertion failed due to unknown error";
- await safeUpdateStatusToFailed(importId, errorMessage);
-
- logger.error({ importId }, "Import operation failed, worker continuing");
- }
- });
-}
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
deleted file mode 100644
index 0961409e1..000000000
--- a/server/src/services/import/workers/jobs.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-import { UmamiEvent } from "../mappings/umami.js";
-import { IS_PRODUCTION } from "../../../lib/const.js";
-
-export const CSV_PARSE_QUEUE = IS_PRODUCTION ? "csv-parse" : "csv-parse-dev";
-
-export const DATA_INSERT_QUEUE = IS_PRODUCTION ? "data-insert" : "data-insert-dev";
-
-interface ImportJob {
- site: string;
- importId: string;
- platform: "umami";
-}
-
-export interface CsvParseJob extends ImportJob {
- storageLocation: string; // Either local file path or R2 key
- isR2Storage: boolean;
- organization: string;
- startDate?: string;
- endDate?: string;
-}
-
-export interface DataInsertJob extends ImportJob {
- chunk: UmamiEvent[];
- allChunksSent: boolean; // Finalization signal
-}
diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.ts
index 342ca421e..401453f28 100644
--- a/server/src/services/storage/r2StorageService.ts
+++ b/server/src/services/storage/r2StorageService.ts
@@ -3,9 +3,7 @@ import {
PutObjectCommand,
GetObjectCommand,
DeleteObjectCommand,
- ListObjectsV2Command,
} from "@aws-sdk/client-s3";
-import { Upload } from "@aws-sdk/lib-storage";
import { NodeHttpHandler } from "@smithy/node-http-handler";
import { Readable } from "stream";
import { gunzipSync } from "zlib";
@@ -14,11 +12,10 @@ import { IS_CLOUD } from "../../lib/const.js";
import { createServiceLogger } from "../../lib/logger/logger.js";
class R2StorageService {
- private readonly client: S3Client | null = null;
- private readonly bucketName: string = "";
- private readonly enabled: boolean = false;
+ private client: S3Client | null = null;
+ private bucketName: string = "";
+ private enabled: boolean = false;
private logger = createServiceLogger("r2-storage");
- private readonly IMPORT_FILES_PREFIX = "imports/";
constructor() {
// Only initialize R2 in cloud environment
@@ -203,193 +200,6 @@ class R2StorageService {
// Non-critical error, log but don't throw
}
}
-
- /**
- * Store an import CSV file in R2 using streaming upload
- */
- async storeImportFile(key: string, fileStream: Readable): Promise<void> {
- if (!this.enabled || !this.client) {
- throw new Error("R2 storage is not enabled");
- }
-
- if (key.trim().length === 0) {
- throw new Error("Invalid storage key: key cannot be empty");
- }
-
- if (key.includes("..")) {
- throw new Error("Invalid storage key: path traversal not allowed");
- }
-
- try {
- const upload = new Upload({
- client: this.client,
- params: {
- Bucket: this.bucketName,
- Key: key,
- Body: fileStream,
- ContentType: "text/csv",
- },
- queueSize: 4,
- partSize: 5 * 1024 * 1024,
- leavePartsOnError: false,
- });
-
- await upload.done();
- this.logger.info({ key }, "Successfully stored import file");
- } catch (error) {
- this.logger.error({ error, key }, "Failed to store import file");
- throw error;
- }
- }
-
- /**
- * Retrieve an import CSV file from R2 as a stream
- */
- async getImportFileStream(key: string): Promise<Readable> {
- if (!this.enabled || !this.client) {
- throw new Error("R2 storage is not enabled");
- }
-
- if (key.trim().length === 0) {
- throw new Error("Invalid storage key: key cannot be empty");
- }
-
- if (key.includes("..")) {
- throw new Error("Invalid storage key: path traversal not allowed");
- }
-
- try {
- const response = await this.client.send(
- new GetObjectCommand({
- Bucket: this.bucketName,
- Key: key,
- })
- );
-
- if (!response.Body) {
- throw new Error("Empty response body");
- }
-
- this.logger.info({ key }, "Successfully retrieved import file stream");
- return response.Body as Readable;
- } catch (error) {
- this.logger.error({ error, key }, "Failed to retrieve import file stream");
- throw error;
- }
- }
-
- /**
- * Delete an import file from R2 (for cleanup after processing)
- */
- async deleteImportFile(key: string): Promise<void> {
- if (!this.enabled || !this.client) {
- return;
- }
-
- if (key.trim().length === 0) {
- throw new Error("Invalid storage key: key cannot be empty");
- }
-
- if (key.includes("..")) {
- throw new Error("Invalid storage key: path traversal not allowed");
- }
-
- try {
- await this.client.send(
- new DeleteObjectCommand({
- Bucket: this.bucketName,
- Key: key,
- })
- );
- this.logger.info({ key }, "Successfully deleted import file");
- } catch (error) {
- this.logger.error({ error, key }, "Failed to delete import file");
- // Non-critical error, log but don't throw
- }
- }
-
- /**
- * List all import files in R2
- * Returns array of objects with key and lastModified date
- */
- async listImportFiles(): Promise<Array<{ key: string; lastModified: Date }>> {
- if (!this.enabled || !this.client) {
- return [];
- }
-
- try {
- const allFiles: Array<{ key: string; lastModified: Date }> = [];
- let continuationToken: string | undefined;
-
- do {
- const response = await this.client.send(
- new ListObjectsV2Command({
- Bucket: this.bucketName,
- Prefix: this.IMPORT_FILES_PREFIX,
- ContinuationToken: continuationToken,
- })
- );
-
- if (response.Contents) {
- for (const obj of response.Contents) {
- if (obj.Key && obj.LastModified) {
- allFiles.push({
- key: obj.Key,
- lastModified: obj.LastModified,
- });
- }
- }
- }
-
- continuationToken = response.NextContinuationToken;
- } while (continuationToken);
-
- this.logger.info({ count: allFiles.length }, "Listed import files");
- return allFiles;
- } catch (error) {
- this.logger.error({ error }, "Failed to list import files");
- throw error;
- }
- }
-
- /**
- * Delete all import files older than the specified number of days
- * Returns count of deleted files
- */
- async deleteOldImportFiles(daysOld: number = 1): Promise<number> {
- if (!this.enabled || !this.client) {
- return 0;
- }
-
- try {
- const files = await this.listImportFiles();
- const cutoffDate = new Date(Date.now() - daysOld * 24 * 60 * 60 * 1000);
-
- const filesToDelete = files.filter(file => file.lastModified < cutoffDate);
-
- this.logger.info(
- { total: files.length, toDelete: filesToDelete.length, cutoffDate },
- "Deleting old import files"
- );
-
- let deletedCount = 0;
- for (const file of filesToDelete) {
- try {
- await this.deleteImportFile(file.key);
- deletedCount++;
- } catch (error) {
- this.logger.error({ error, key: file.key }, "Failed to delete old import file");
- // Continue with other files even if one fails
- }
- }
-
- this.logger.info({ deletedCount }, "Completed deleting old import files");
- return deletedCount;
- } catch (error) {
- this.logger.error({ error }, "Failed to delete old import files");
- throw error;
- }
- }
}
// Singleton instance