🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Clean up server-side import code
Edward Cui727398b826f1
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.tsindex 0be9d43a3..c06bedc56 100644--- a/client/src/api/admin/import.ts+++ b/client/src/api/admin/import.ts@@ -12,17 +12,6 @@ interface GetSiteImportsResponse {fileName: string;}-interface ImportSiteDataParams {- file: File;- platform: string;- startDate?: string;- endDate?: string;-}--interface ImportSiteDataResponse {- message: string;-}-interface DeleteImportResponse {message: string;}@@ -42,36 +31,6 @@ export function useGetSiteImports(site: number) {});}-export function useImportSiteData(site: number) {- const queryClient = useQueryClient();-- return useMutation({- mutationFn: async (params: ImportSiteDataParams) => {- const formData = new FormData();-- formData.append("platform", params.platform);- if (params.startDate) {- formData.append("startDate", params.startDate);- }- if (params.endDate) {- formData.append("endDate", params.endDate);- }- formData.append("file", params.file);-- return await authedFetch<APIResponse<ImportSiteDataResponse>>(`/import-site-data/${site}`, undefined, {- method: "POST",- data: formData,- });- },- onSuccess: async () => {- await queryClient.invalidateQueries({- queryKey: ["get-site-imports", site],- });- },- retry: false,- });-}-export function useDeleteSiteImport(site: number) {const queryClient = useQueryClient();diff --git a/server/package-lock.json b/server/package-lock.jsonindex b2694a47d..d781a901b 100644--- a/server/package-lock.json+++ b/server/package-lock.json@@ -9,12 +9,9 @@"version": "2.1.0","dependencies": {"@aws-sdk/client-s3": "^3.842.0",- "@aws-sdk/lib-storage": "^3.917.0","@axiomhq/pino": "^1.3.1","@clickhouse/client": "1.11.1",- "@fast-csv/parse": "^5.0.5","@fastify/cors": "11.0.1",- "@fastify/multipart": "^9.3.0","@fastify/one-line-logger": "2.0.2","@fastify/rate-limit": "^10.3.0","@fastify/static": "8.1.1",@@ -36,7 +33,6 @@"node-cache": "^5.1.2","node-cron": "4.0.6","pg": "8.16.0",- "pg-boss": "^10.3.2","postgres": "3.4.5","psl": "^1.15.0","react": "19.2.0",@@ -580,37 +576,6 @@"node": ">=18.0.0"}},- "node_modules/@aws-sdk/lib-storage": {- "version": "3.917.0",- "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.917.0.tgz",- "integrity": "sha512-Z8mRzfP6PgUoybJHx/tH40aop42yeh66cW7wLM8R88egB2UYQ0IgxDoRynqlLi5uceI21wss/CIDkhzO9p1cOg==",- "license": "Apache-2.0",- "dependencies": {- "@smithy/abort-controller": "^4.2.3",- "@smithy/middleware-endpoint": "^4.3.5",- "@smithy/smithy-client": "^4.9.1",- "buffer": "5.6.0",- "events": "3.3.0",- "stream-browserify": "3.0.0",- "tslib": "^2.6.2"- },- "engines": {- "node": ">=18.0.0"- },- "peerDependencies": {- "@aws-sdk/client-s3": "^3.917.0"- }- },- "node_modules/@aws-sdk/lib-storage/node_modules/buffer": {- "version": "5.6.0",- "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",- "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",- "license": "MIT",- "dependencies": {- "base64-js": "^1.0.2",- "ieee754": "^1.1.4"- }- },"node_modules/@aws-sdk/middleware-bucket-endpoint": {"version": "3.914.0","resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.914.0.tgz",@@ -1008,19 +973,6 @@"node": ">=16"}},- "node_modules/@axiomhq/js/node_modules/uuid": {- "version": "11.1.0",- "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",- "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",- "funding": [- "https://github.com/sponsors/broofa",- "https://github.com/sponsors/ctavan"- ],- "license": "MIT",- "bin": {- "uuid": "dist/esm/bin/uuid"- }- },"node_modules/@axiomhq/pino": {"version": "1.3.1","resolved": "https://registry.npmjs.org/@axiomhq/pino/-/pino-1.3.1.tgz",@@ -2115,20 +2067,6 @@"node": ">=18"}},- "node_modules/@fast-csv/parse": {- "version": "5.0.5",- "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.5.tgz",- "integrity": "sha512-M0IbaXZDbxfOnpVE5Kps/a6FGlILLhtLsvWd9qNH3d2TxNnpbNkFf3KD26OmJX6MHq7PdQAl5htStDwnuwHx6w==",- "license": "MIT",- "dependencies": {- "lodash.escaperegexp": "^4.1.2",- "lodash.groupby": "^4.6.0",- "lodash.isfunction": "^3.0.9",- "lodash.isnil": "^4.0.0",- "lodash.isundefined": "^3.0.1",- "lodash.uniq": "^4.5.0"- }- },"node_modules/@fastify/accept-negotiator": {"version": "2.0.1","resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-2.0.1.tgz",@@ -2166,12 +2104,6 @@"fast-uri": "^3.0.0"}},- "node_modules/@fastify/busboy": {- "version": "3.1.1",- "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-3.1.1.tgz",- "integrity": "sha512-5DGmA8FTdB2XbDeEwc/5ZXBl6UbBAyBOOLlPuBnZ/N1SwdH9Ii+cOX3tBROlDgcTXxjOYnLMVoKk9+FXAw0CJw==",- "license": "MIT"- },"node_modules/@fastify/cors": {"version": "11.0.1","resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-11.0.1.tgz",@@ -2192,22 +2124,6 @@"toad-cache": "^3.7.0"}},- "node_modules/@fastify/deepmerge": {- "version": "3.1.0",- "resolved": "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-3.1.0.tgz",- "integrity": "sha512-lCVONBQINyNhM6LLezB6+2afusgEYR4G8xenMsfe+AT+iZ7Ca6upM5Ha8UkZuYSnuMw3GWl/BiPXnLMi/gSxuQ==",- "funding": [- {- "type": "github",- "url": "https://github.com/sponsors/fastify"- },- {- "type": "opencollective",- "url": "https://opencollective.com/fastify"- }- ],- "license": "MIT"- },"node_modules/@fastify/error": {"version": "4.1.0","resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.1.0.tgz",@@ -2268,29 +2184,6 @@"dequal": "^2.0.3"}},- "node_modules/@fastify/multipart": {- "version": "9.3.0",- "resolved": "https://registry.npmjs.org/@fastify/multipart/-/multipart-9.3.0.tgz",- "integrity": "sha512-NpeKipTOjjL1dA7SSlRMrOWWtrE8/0yKOmeudkdQoEaz4sVDJw5MVdZIahsWhvpc3YTN7f04f9ep/Y65RKoOWA==",- "funding": [- {- "type": "github",- "url": "https://github.com/sponsors/fastify"- },- {- "type": "opencollective",- "url": "https://opencollective.com/fastify"- }- ],- "license": "MIT",- "dependencies": {- "@fastify/busboy": "^3.0.0",- "@fastify/deepmerge": "^3.0.0",- "@fastify/error": "^4.0.0",- "fastify-plugin": "^5.0.0",- "secure-json-parse": "^4.0.0"- }- },"node_modules/@fastify/one-line-logger": {"version": "2.0.2","resolved": "https://registry.npmjs.org/@fastify/one-line-logger/-/one-line-logger-2.0.2.tgz",@@ -2394,9 +2287,9 @@"license": "MIT"},"node_modules/@ioredis/commands": {- "version": "1.2.0",- "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",- "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",+ "version": "1.4.0",+ "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz",+ "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==","license": "MIT"},"node_modules/@isaacs/cliui": {@@ -4935,9 +4828,9 @@"license": "MIT"},"node_modules/bullmq": {- "version": "5.56.5",- "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.56.5.tgz",- "integrity": "sha512-nhcVxoE9Y0YUuNYtvaD+N0Bk2kqcU+rXzJwdQIr8i8qC/fxoghwUYb9a+CidTv24pi1eqstLnBoa8xkR/P7Mdw==",+ "version": "5.63.0",+ "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.63.0.tgz",+ "integrity": "sha512-HT1iM3Jt4bZeg3Ru/MxrOy2iIItxcl1Pz5Ync1Vrot70jBpVguMxFEiSaDU57BwYwR4iwnObDnzct2lirKkX5A==","license": "MIT","dependencies": {"cron-parser": "^4.9.0",@@ -4946,7 +4839,7 @@"node-abort-controller": "^3.1.1","semver": "^7.5.4","tslib": "^2.0.0",- "uuid": "^9.0.0"+ "uuid": "^11.1.0"}},"node_modules/cac": {@@ -6374,12 +6267,12 @@"license": "ISC"},"node_modules/ioredis": {- "version": "5.6.1",- "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",- "integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",+ "version": "5.8.2",+ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz",+ "integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==","license": "MIT","dependencies": {- "@ioredis/commands": "^1.1.1",+ "@ioredis/commands": "1.4.0","cluster-key-slot": "^1.1.0","debug": "^4.3.4","denque": "^2.1.0",@@ -6795,18 +6688,6 @@"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==","license": "MIT"},- "node_modules/lodash.escaperegexp": {- "version": "4.1.2",- "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz",- "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==",- "license": "MIT"- },- "node_modules/lodash.groupby": {- "version": "4.6.0",- "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",- "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==",- "license": "MIT"- },"node_modules/lodash.includes": {"version": "4.3.0","resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",@@ -6825,24 +6706,12 @@"integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==","license": "MIT"},- "node_modules/lodash.isfunction": {- "version": "3.0.9",- "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz",- "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==",- "license": "MIT"- },"node_modules/lodash.isinteger": {"version": "4.0.4","resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz","integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==","license": "MIT"},- "node_modules/lodash.isnil": {- "version": "4.0.0",- "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz",- "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==",- "license": "MIT"- },"node_modules/lodash.isnumber": {"version": "3.0.3","resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",@@ -6861,24 +6730,12 @@"integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==","license": "MIT"},- "node_modules/lodash.isundefined": {- "version": "3.0.1",- "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz",- "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==",- "license": "MIT"- },"node_modules/lodash.once": {"version": "4.1.1","resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz","integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==","license": "MIT"},- "node_modules/lodash.uniq": {- "version": "4.5.0",- "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",- "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==",- "license": "MIT"- },"node_modules/loupe": {"version": "3.1.3","resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.3.tgz",@@ -7430,20 +7287,6 @@}}},- "node_modules/pg-boss": {- "version": "10.3.2",- "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-10.3.2.tgz",- "integrity": "sha512-yy96jwKekgUW4DVvve5xjvLZlNqVCRayl1wJvBxQeDNhIrblgEG0YnqbcNth4wsehkYWVJND9TeWbqb+cfxJKQ==",- "license": "MIT",- "dependencies": {- "cron-parser": "^4.9.0",- "pg": "^8.16.0",- "serialize-error": "^8.1.0"- },- "engines": {- "node": ">=20"- }- },"node_modules/pg-cloudflare": {"version": "1.2.5","resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.5.tgz",@@ -8321,21 +8164,6 @@"node": ">=10"}},- "node_modules/serialize-error": {- "version": "8.1.0",- "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",- "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",- "license": "MIT",- "dependencies": {- "type-fest": "^0.20.2"- },- "engines": {- "node": ">=10"- },- "funding": {- "url": "https://github.com/sponsors/sindresorhus"- }- },"node_modules/set-cookie-parser": {"version": "2.7.1","resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.1.tgz",@@ -8599,16 +8427,6 @@"integrity": "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==","dev": true},- "node_modules/stream-browserify": {- "version": "3.0.0",- "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz",- "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==",- "license": "MIT",- "dependencies": {- "inherits": "~2.0.4",- "readable-stream": "^3.5.0"- }- },"node_modules/string_decoder": {"version": "1.3.0","resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",@@ -9065,18 +8883,6 @@"node": ">= 6"}},- "node_modules/type-fest": {- "version": "0.20.2",- "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",- "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",- "license": "(MIT OR CC0-1.0)",- "engines": {- "node": ">=10"- },- "funding": {- "url": "https://github.com/sponsors/sindresorhus"- }- },"node_modules/typescript": {"version": "5.8.3","resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz",@@ -9171,16 +8977,16 @@"license": "MIT"},"node_modules/uuid": {- "version": "9.0.1",- "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",- "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",+ "version": "11.1.0",+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",+ "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==","funding": ["https://github.com/sponsors/broofa","https://github.com/sponsors/ctavan"],"license": "MIT","bin": {- "uuid": "dist/bin/uuid"+ "uuid": "dist/esm/bin/uuid"}},"node_modules/vite": {diff --git a/server/package.json b/server/package.jsonindex 41aa72c84..6d14bd6e1 100644--- a/server/package.json+++ b/server/package.json@@ -24,12 +24,9 @@},"dependencies": {"@aws-sdk/client-s3": "^3.842.0",- "@aws-sdk/lib-storage": "^3.917.0","@axiomhq/pino": "^1.3.1","@clickhouse/client": "1.11.1",- "@fast-csv/parse": "^5.0.5","@fastify/cors": "11.0.1",- "@fastify/multipart": "^9.3.0","@fastify/one-line-logger": "2.0.2","@fastify/rate-limit": "^10.3.0","@fastify/static": "8.1.1",@@ -51,7 +48,6 @@"node-cache": "^5.1.2","node-cron": "4.0.6","pg": "8.16.0",- "pg-boss": "^10.3.2","postgres": "3.4.5","psl": "^1.15.0","react": "19.2.0",diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsdeleted file mode 100644index 3dfc00309..000000000--- a/server/src/api/sites/importSiteData.ts+++ /dev/null@@ -1,180 +0,0 @@-import { createWriteStream } from "node:fs";-import { mkdir } from "node:fs/promises";-import { dirname } from "node:path";-import { pipeline } from "node:stream/promises";-import { FastifyRequest, FastifyReply } from "fastify";-import { DateTime } from "luxon";-import { z } from "zod";-import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";-import { getJobQueue } from "../../services/import/queues/jobQueueFactory.js";-import { ImportLimiter } from "../../services/import/importLimiter.js";-import { updateImportStatus } from "../../services/import/importStatusManager.js";-import { deleteImportFile, getImportStorageLocation } from "../../services/import/utils.js";-import { CSV_PARSE_QUEUE, CsvParseJob } from "../../services/import/workers/jobs.js";-import { r2Storage } from "../../services/storage/r2StorageService.js";--const isValidDate = (date: string) => {- const dt = DateTime.fromFormat(date, "yyyy-MM-dd", { zone: "utc" });- return dt.isValid;-};--const parseDate = (date: string) => DateTime.fromFormat(date, "yyyy-MM-dd", { zone: "utc" });--const importDataFieldsSchema = z- .object({- fields: z- .object({- platform: z.enum(["umami"]),- startDate: z.string().refine(isValidDate, { message: "Invalid start date format" }).optional(),- endDate: z.string().refine(isValidDate, { message: "Invalid end date format" }).optional(),- })- .refine(- data => {- if (!data.startDate || !data.endDate) return true;- return parseDate(data.startDate) <= parseDate(data.endDate);- },- { message: "Start date must be before or equal to end date" }- )- .refine(- data => {- if (!data.startDate) return true;- const today = DateTime.utc().startOf("day");- return parseDate(data.startDate) <= today;- },- { message: "Start date cannot be in the future" }- ),- })- .strict();--const importDataRequestSchema = z- .object({- params: z.object({- site: z.string().min(1),- }),- })- .strict();--type ImportDataRequest = {- Params: z.infer<typeof importDataRequestSchema.shape.params>;-};--export async function importSiteData(request: FastifyRequest<ImportDataRequest>, reply: FastifyReply) {- try {- const jobQueue = getJobQueue();- if (!jobQueue.isReady()) {- return reply.status(503).send({ error: "Import system is temporarily unavailable. Please try again later." });- }-- const parsedParams = importDataRequestSchema.safeParse({- params: request.params,- });-- if (!parsedParams.success) {- return reply.status(400).send({ error: "Validation error" });- }-- const { site } = parsedParams.data.params;-- const userHasAccess = await getUserHasAdminAccessToSite(request, site);- if (!userHasAccess) {- return reply.status(403).send({ error: "Forbidden" });- }-- const data = await request.file();- if (!data) {- return reply.status(400).send({ error: "No file uploaded." });- }-- if (data.mimetype !== "text/csv" || !data.filename.endsWith(".csv")) {- return reply.status(400).send({ error: "Invalid file type. Only .csv files are accepted." });- }-- const parsedFields = importDataFieldsSchema.safeParse({- fields: {- platform: (data.fields.platform as any)?.value,- startDate: (data.fields.startDate as any)?.value,- endDate: (data.fields.endDate as any)?.value,- },- });-- if (!parsedFields.success) {- return reply.status(400).send({ error: "Validation error" });- }-- const { platform, startDate, endDate } = parsedFields.data.fields;- const siteId = Number(site);-- // Check organization and get initial limit check- const concurrentImportLimitResult = await ImportLimiter.checkConcurrentImportLimit(siteId);- if (!concurrentImportLimitResult.allowed) {- return reply.status(429).send({ error: concurrentImportLimitResult.reason });- }-- const organization = concurrentImportLimitResult.organizationId;-- // Atomically create import status with concurrency check to prevent race conditions- const createResult = await ImportLimiter.createImportWithConcurrencyCheck({- siteId,- organizationId: organization,- platform,- fileName: data.filename,- });-- if (!createResult.success) {- return reply.status(429).send({ error: createResult.reason });- }-- const importId = createResult.importId;-- const storage = getImportStorageLocation(importId, data.filename);-- try {- if (storage.isR2) {- await r2Storage.storeImportFile(storage.location, data.file);- console.log(`[Import] File streamed to R2: ${storage.location}`);- } else {- const importDir = dirname(storage.location);- await mkdir(importDir, { recursive: true });- await pipeline(data.file, createWriteStream(storage.location));- console.log(`[Import] File stored locally: ${storage.location}`);- }-- if (data.file.truncated) {- await deleteImportFile(storage.location, storage.isR2);- await updateImportStatus(importId, "failed", "File too large");- return reply.status(413).send({ error: "File too large." });- }- } catch (fileError) {- await updateImportStatus(importId, "failed", "Failed to save uploaded file");- console.error("Failed to save uploaded file:", fileError);- return reply.status(500).send({ error: "Could not process file upload." });- }-- try {- await jobQueue.send<CsvParseJob>(CSV_PARSE_QUEUE, {- site,- importId,- platform,- storageLocation: storage.location,- isR2Storage: storage.isR2,- organization,- startDate,- endDate,- });- } catch (queueError) {- await updateImportStatus(importId, "failed", "Failed to queue import job");- await deleteImportFile(storage.location, storage.isR2);- console.error("Failed to enqueue import job:", queueError);- return reply.status(500).send({ error: "Failed to initiate import process." });- }-- return reply.status(202).send({- data: {- message: "File upload accepted and is now being processed.",- },- });- } catch (error) {- console.error("Unexpected error during import:", error);- return reply.status(500).send({ error: "An unexpected error occurred. Please try again later." });- }-}diff --git a/server/src/index.ts b/server/src/index.tsindex 2226f71eb..fcd92ffa6 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -1,6 +1,5 @@import cors from "@fastify/cors";import fastifyStatic from "@fastify/static";-import multipart from "@fastify/multipart";import { toNodeHandler } from "better-auth/node";import Fastify from "fastify";import { dirname, join } from "path";@@ -87,10 +86,8 @@ import { disconnectGSC } from "./api/gsc/disconnect.js";import { getGSCData } from "./api/gsc/getData.js";import { selectGSCProperty } from "./api/gsc/selectProperty.js";import { getSiteImports } from "./api/sites/getSiteImports.js";-import { importSiteData } from "./api/sites/importSiteData.js";import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";import { batchImportEvents } from "./api/sites/batchImportEvents.js";-import { getJobQueue } from "./services/import/queues/jobQueueFactory.js";const __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);@@ -168,13 +165,6 @@ server.register(cors, {credentials: true,});-server.register(multipart, {- limits: {- fileSize: IS_CLOUD ? 500 * 1024 * 1024 : 50 * 1024 * 1024 * 1024, // 500 MB for cloud, 50 GB for self-hosted- files: 1, // only allow 1 file- },-});-// Serve static filesserver.register(fastifyStatic, {root: join(__dirname, "../public"),@@ -360,13 +350,9 @@ server.get("/api/session-replay/list/:site", getSessionReplays);server.get("/api/session-replay/:sessionId/:site", getSessionReplayEvents);// Imports-server.post("/api/import-site-data/:site", {- bodyLimit: IS_CLOUD ? 500 * 1024 * 1024 : 50 * 1024 * 1024 * 1024, // 500 MB for cloud, 50 GB for self-hosted (matches multipart fileSize)-}, importSiteData);+server.post("/api/batch-import-events/:site", batchImportEvents);server.get("/api/get-site-imports/:site", getSiteImports);server.delete("/api/delete-site-import/:site/:importId", deleteSiteImport);-// Client-side import endpoint (parses CSV, server transforms)-server.post("/api/batch-import-events/:site", batchImportEvents);// Administrativeserver.get("/api/config", getConfig);@@ -472,17 +458,7 @@ const start = async () => {weeklyReportService.startWeeklyReportCron();}- // Initialize and start import job queue system- const jobQueue = getJobQueue();- try {- await jobQueue.start();- } catch (queueError) {- server.log.error("Failed to initialize import job queue system:", queueError);- server.log.warn("Server will continue without import functionality");- // Don't exit - allow server to run without import features- }-- // Start the server first+ // Start the serverawait server.listen({ port: 3001, host: "0.0.0.0" });server.log.info("Server is listening on http://0.0.0.0:3001");@@ -504,8 +480,6 @@ const start = async () => {// });// }} catch (err) {- const jobQueue = getJobQueue();- await jobQueue.stop();server.log.error(err);process.exit(1);}@@ -536,11 +510,6 @@ const shutdown = async (signal: string) => {await server.close();server.log.info("Server closed");- // Shutdown job queue system- const jobQueue = getJobQueue();- await jobQueue.stop();- server.log.info("Job queue shut down");-// Shutdown uptime service// await uptimeService.shutdown();// server.log.info("Uptime service shut down");diff --git a/server/src/lib/const.ts b/server/src/lib/const.tsindex 741ea1b15..f2b2ac945 100644--- a/server/src/lib/const.ts+++ b/server/src/lib/const.ts@@ -3,7 +3,6 @@ import dotenv from "dotenv";dotenv.config();export const IS_CLOUD = process.env.CLOUD === "true";-export const IS_PRODUCTION = process.env.NODE_ENV === "production";export const DISABLE_SIGNUP = process.env.DISABLE_SIGNUP === "true";export const DISABLE_TELEMETRY = process.env.DISABLE_TELEMETRY === "true";export const SECRET = process.env.BETTER_AUTH_SECRET;diff --git a/server/src/services/import/queues/adapters/bullMQAdapter.ts b/server/src/services/import/queues/adapters/bullMQAdapter.tsdeleted file mode 100644index 8bacbf93d..000000000--- a/server/src/services/import/queues/adapters/bullMQAdapter.ts+++ /dev/null@@ -1,199 +0,0 @@-import { Job, Queue, QueueEvents, Worker } from "bullmq";-import { IJobQueue } from "../jobQueue.js";-import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../workers/jobs.js";-import { createCsvParseWorker } from "../../workers/csvParseWorker.js";-import { createDataInsertWorker } from "../../workers/dataInsertWorker.js";-import { createServiceLogger } from "../../../../lib/logger/logger.js";--const logger = createServiceLogger("import:bullmq");--export class BullMQAdapter implements IJobQueue {- private readonly connection: { host: string; port: number; password?: string };- private queues: Map<string, Queue> = new Map();- private workers: Map<string, Worker> = new Map();- private queueEvents: Map<string, QueueEvents> = new Map();- private ready: boolean = false;-- constructor() {- this.connection = {- host: process.env.REDIS_HOST || "localhost",- port: parseInt(process.env.REDIS_PORT || "6379", 10),- ...(process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD }),- };- }-- async start(): Promise<void> {- try {- await this.createQueue(CSV_PARSE_QUEUE);- await this.createQueue(DATA_INSERT_QUEUE);-- await createCsvParseWorker(this);- await createDataInsertWorker(this);-- this.ready = true;- logger.info("Started successfully");- } catch (error) {- this.ready = false;- logger.error({ error }, "Failed to initialize");- throw error;- }- }-- async stop(): Promise<void> {- logger.info("Stopping...");- const errors: Error[] = [];-- // Close all workers first- const workerResults = await Promise.allSettled(Array.from(this.workers.values()).map(worker => worker.close()));- workerResults.forEach((result, index) => {- if (result.status === "rejected") {- const workerName = Array.from(this.workers.keys())[index];- const error = new Error(`Failed to close worker ${workerName}: ${result.reason}`);- errors.push(error);- logger.error({ workerName, error: result.reason }, error.message);- }- });- this.workers.clear();-- // Close all queue events- const queueEventsResults = await Promise.allSettled(Array.from(this.queueEvents.values()).map(qe => qe.close()));- queueEventsResults.forEach((result, index) => {- if (result.status === "rejected") {- const queueName = Array.from(this.queueEvents.keys())[index];- const error = new Error(`Failed to close queue events for ${queueName}: ${result.reason}`);- errors.push(error);- logger.error({ queueName, error: result.reason }, error.message);- }- });- this.queueEvents.clear();-- // Close all queues (this closes their Redis connections)- const queueResults = await Promise.allSettled(Array.from(this.queues.values()).map(queue => queue.close()));- queueResults.forEach((result, index) => {- if (result.status === "rejected") {- const queueName = Array.from(this.queues.keys())[index];- const error = new Error(`Failed to close queue ${queueName}: ${result.reason}`);- errors.push(error);- logger.error({ queueName, error: result.reason }, error.message);- }- });- this.queues.clear();-- if (errors.length > 0) {- const aggregatedError = new Error(- `BullMQ stop encountered ${errors.length} error(s): ${errors.map(e => e.message).join("; ")}`- );- logger.error({ errorCount: errors.length }, "Stopped with errors");- throw aggregatedError;- }-- logger.info("Stopped successfully");- }-- async createQueue(queueName: string): Promise<void> {- if (!this.queues.has(queueName)) {- let queue: Queue | undefined;- let queueEvents: QueueEvents | undefined;-- try {- queue = new Queue(queueName, {- connection: this.connection,- defaultJobOptions: {- attempts: 1,- removeOnComplete: true,- removeOnFail: true,- },- });-- this.queues.set(queueName, queue);-- queueEvents = new QueueEvents(queueName, {- connection: this.connection,- });-- this.queueEvents.set(queueName, queueEvents);-- queueEvents.on("completed", ({ jobId }) => {- logger.debug({ jobId, queueName }, "Job completed");- });-- queueEvents.on("failed", ({ jobId, failedReason }) => {- logger.error({ jobId, queueName, failedReason }, "Job failed");- });- } catch (error) {- logger.error({ queueName, error }, "Failed to create queue");-- const cleanupPromises: Promise<void>[] = [];-- if (queueEvents) {- this.queueEvents.delete(queueName);- cleanupPromises.push(- queueEvents.close().catch(closeError => {- logger.error({ queueName, error: closeError }, "Failed to close queue events during cleanup");- })- );- }-- if (queue) {- this.queues.delete(queueName);- cleanupPromises.push(- queue.close().catch(closeError => {- logger.error({ queueName, error: closeError }, "Failed to close queue during cleanup");- })- );- }-- await Promise.all(cleanupPromises);-- throw error;- }- }- }-- async send<T>(queueName: string, data: T): Promise<void> {- const queue = this.queues.get(queueName);- if (!queue) {- throw new Error(`Queue ${queueName} not found. Call createQueue first.`);- }-- try {- await queue.add(queueName, data);- } catch (error) {- const errorMessage = error instanceof Error ? error.message : String(error);- logger.error({ queueName, error }, "Failed to enqueue job");- throw new Error(`Failed to enqueue job to ${queueName}: ${errorMessage}`);- }- }-- async work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void> {- try {- const worker = new Worker(- queueName,- async (job: Job<T>) => {- await handler(job.data);- },- {- connection: this.connection,- concurrency: 1,- }- );-- worker.on("error", error => {- logger.error({ queueName, error }, "Worker error");- });-- worker.on("failed", (job, error) => {- logger.error({ queueName, jobId: job?.id, error }, "Job failed");- });-- this.workers.set(queueName, worker);- } catch (error) {- logger.error({ queueName, error }, "Failed to create worker");- throw error;- }- }-- isReady(): boolean {- return this.ready;- }-}diff --git a/server/src/services/import/queues/adapters/pgBossAdapter.ts b/server/src/services/import/queues/adapters/pgBossAdapter.tsdeleted file mode 100644index 720f8c60d..000000000--- a/server/src/services/import/queues/adapters/pgBossAdapter.ts+++ /dev/null@@ -1,79 +0,0 @@-import PgBoss, { Job } from "pg-boss";-import { IJobQueue } from "../jobQueue.js";-import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../workers/jobs.js";-import { createCsvParseWorker } from "../../workers/csvParseWorker.js";-import { createDataInsertWorker } from "../../workers/dataInsertWorker.js";-import { createServiceLogger } from "../../../../lib/logger/logger.js";--const logger = createServiceLogger("import:pgboss");--export class PgBossAdapter implements IJobQueue {- private boss: PgBoss;- private ready: boolean = false;-- constructor() {- this.boss = new PgBoss({- host: process.env.POSTGRES_HOST || "postgres",- port: parseInt(process.env.POSTGRES_PORT || "5432", 10),- database: process.env.POSTGRES_DB,- user: process.env.POSTGRES_USER,- password: process.env.POSTGRES_PASSWORD,- schema: "pgboss",- application_name: "data-import-system",- retryLimit: 0,- });-- this.boss.on("error", error => {- logger.error({ error }, "PgBoss error");- });- }-- async start(): Promise<void> {- try {- await this.boss.start();-- await this.createQueue(CSV_PARSE_QUEUE);- await this.createQueue(DATA_INSERT_QUEUE);-- await createCsvParseWorker(this);- await createDataInsertWorker(this);-- this.ready = true;- logger.info("Started successfully");- } catch (error) {- this.ready = false;- logger.error({ error }, "Failed to initialize");- throw error;- }- }-- async stop(): Promise<void> {- await this.boss.stop();- logger.info("Stopped successfully");- }-- async createQueue(queueName: string): Promise<void> {- await this.boss.createQueue(queueName);- }-- async send<T>(queueName: string, data: T): Promise<void> {- await this.boss.send(queueName, data as object);- }-- async work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void> {- await this.boss.work(- queueName,- {- batchSize: 1,- pollingIntervalSeconds: 3,- },- async ([job]: Job<T>[]) => {- await handler(job.data);- }- );- }-- isReady(): boolean {- return this.ready;- }-}diff --git a/server/src/services/import/queues/jobQueue.ts b/server/src/services/import/queues/jobQueue.tsdeleted file mode 100644index e83ee669c..000000000--- a/server/src/services/import/queues/jobQueue.ts+++ /dev/null@@ -1,8 +0,0 @@-export interface IJobQueue {- start(): Promise<void>;- stop(): Promise<void>;- createQueue(queueName: string): Promise<void>;- send<T>(queueName: string, data: T): Promise<void>;- work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void>;- isReady(): boolean;-}diff --git a/server/src/services/import/queues/jobQueueFactory.ts b/server/src/services/import/queues/jobQueueFactory.tsdeleted file mode 100644index bd59d628d..000000000--- a/server/src/services/import/queues/jobQueueFactory.ts+++ /dev/null@@ -1,21 +0,0 @@-import { IS_CLOUD } from "../../../lib/const.js";-import { IJobQueue } from "./jobQueue.js";-import { PgBossAdapter } from "./adapters/pgBossAdapter.js";-import { BullMQAdapter } from "./adapters/bullMQAdapter.js";--let queueInstance: IJobQueue | null = null;--/**- * Get the singleton job queue instance- * Returns BullMQ for cloud deployments, pg-boss for self-hosted- */-export const getJobQueue = (): IJobQueue => {- if (!queueInstance) {- if (IS_CLOUD) {- queueInstance = new BullMQAdapter();- } else {- queueInstance = new PgBossAdapter();- }- }- return queueInstance;-};diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.tsdeleted file mode 100644index 01892bd2d..000000000--- a/server/src/services/import/utils.ts+++ /dev/null@@ -1,35 +0,0 @@-import { unlink } from "node:fs/promises";-import { IS_CLOUD } from "../../lib/const.js";-import { r2Storage } from "../storage/r2StorageService.js";-import { createServiceLogger } from "../../lib/logger/logger.js";--const logger = createServiceLogger("import:utils");--export interface StorageLocationInfo {- location: string;- isR2: boolean;-}--export function getImportStorageLocation(importId: string, filename: string): StorageLocationInfo {- const isR2 = IS_CLOUD && r2Storage.isEnabled();- if (isR2) {- return {- location: `imports/${importId}/${filename}`,- isR2: true,- };- }- return {- location: `/tmp/imports/${importId}/${filename}`,- isR2: false,- };-}--export const deleteImportFile = async (storageLocation: string, isR2Storage: boolean): Promise<void> => {- if (isR2Storage) {- await r2Storage.deleteImportFile(storageLocation);- logger.debug({ storageLocation }, "Deleted R2 file");- } else {- await unlink(storageLocation);- logger.debug({ storageLocation }, "Deleted local file");- }-};diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsdeleted file mode 100644index 8dcd50abf..000000000--- a/server/src/services/import/workers/csvParseWorker.ts+++ /dev/null@@ -1,221 +0,0 @@-import { access, constants } from "node:fs/promises";-import { createReadStream } from "node:fs";-import { parse } from "@fast-csv/parse";-import { DateTime } from "luxon";-import { IJobQueue } from "../queues/jobQueue.js";-import { r2Storage } from "../../storage/r2StorageService.js";-import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE, DataInsertJob } from "./jobs.js";-import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";-import { updateImportStatus } from "../importStatusManager.js";-import { deleteImportFile } from "../utils.js";-import { ImportQuotaTracker } from "../importQuotaChecker.js";-import { createServiceLogger } from "../../../lib/logger/logger.js";--const logger = createServiceLogger("import:csv-parser");--const getImportDataHeaders = (platform: string) => {- switch (platform) {- case "umami":- return umamiHeaders;- default:- throw new Error(`Unsupported platform: ${platform}`);- }-};--const safeUpdateStatusToFailed = async (importId: string, errorMessage: string) => {- try {- await updateImportStatus(importId, "failed", errorMessage);- } catch (updateError) {- logger.error({ importId, error: updateError }, "Could not update status to failed");- }-};--const createR2FileStream = async (storageLocation: string, platform: string) => {- logger.info({ storageLocation }, "Reading from R2");- const fileStream = await r2Storage.getImportFileStream(storageLocation);- return fileStream.pipe(- parse({- headers: getImportDataHeaders(platform),- renameHeaders: true,- ignoreEmpty: true,- })- );-};--const createLocalFileStream = async (storageLocation: string, platform: string) => {- logger.info({ storageLocation }, "Reading from local disk");- await access(storageLocation, constants.F_OK | constants.R_OK);- return createReadStream(storageLocation).pipe(- parse({- headers: getImportDataHeaders(platform),- renameHeaders: true,- ignoreEmpty: true,- })- );-};--const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {- const startDate = startDateStr- ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")- : null;- const endDate = endDateStr ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day") : null;-- if (startDate && !startDate.isValid) {- throw new Error(`Invalid start date: ${startDateStr}`);- }-- if (endDate && !endDate.isValid) {- throw new Error(`Invalid end date: ${endDateStr}`);- }-- return (dateStr: string) => {- const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });- if (!createdAt.isValid) {- return false;- }-- if (startDate && createdAt < startDate) {- return false;- }-- if (endDate && createdAt > endDate) {- return false;- }-- return true;- };-};--export async function createCsvParseWorker(jobQueue: IJobQueue) {- await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, async job => {- const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job;-- let stream: ReturnType<typeof parse> | null = null;- let processingTimeout: NodeJS.Timeout | null = null;-- try {- const quotaTracker = await ImportQuotaTracker.create(organization);-- const chunkSize = 5000;- const INACTIVITY_TIMEOUT_MS = 90 * 1000;-- let chunk: UmamiEvent[] = [];- let totalSkippedQuota = 0;- let totalProcessed = 0;-- stream = isR2Storage- ? await createR2FileStream(storageLocation, platform)- : await createLocalFileStream(storageLocation, platform);-- stream.on("error", error => {- logger.error({ importId, error }, "Stream error");- });-- await updateImportStatus(importId, "processing");- logger.info({ importId, platform, organization }, "Started processing CSV import");-- const resetTimeout = () => {- if (processingTimeout) {- clearTimeout(processingTimeout);- }- processingTimeout = setTimeout(() => {- if (stream) {- stream.destroy(new Error("Import processing inactivity timeout exceeded"));- }- }, INACTIVITY_TIMEOUT_MS);- };-- // Set initial timeout- resetTimeout();-- const isDateInRange = createDateRangeFilter(startDate, endDate);-- for await (const data of stream) {- // Skip rows with missing or invalid dates- if (!data.created_at) {- continue;- }-- // Apply user-specified date range filter- if (!isDateInRange(data.created_at)) {- continue;- }-- // Check per-month quota (includes historical window check)- if (!quotaTracker.canImportEvent(data.created_at)) {- totalSkippedQuota++;- continue;- }-- chunk.push(data);- totalProcessed++;-- if (chunk.length >= chunkSize) {- await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {- site,- importId,- platform,- chunk,- allChunksSent: false,- });- chunk = [];- resetTimeout();- }- }-- if (chunk.length > 0) {- await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {- site,- importId,- platform,- chunk,- allChunksSent: false,- });- resetTimeout();- }-- if (totalSkippedQuota > 0) {- const quotaSummary = quotaTracker.getSummary();- const errorMessage =- `${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +- `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +- `Try importing newer data or upgrade your plan for higher monthly quotas.`;- await updateImportStatus(importId, "failed", errorMessage);- await deleteImportFile(storageLocation, isR2Storage);- return;- }-- logger.info({ importId, totalProcessed, totalSkippedQuota }, "CSV parsing completed successfully");- await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {- site,- importId,- platform,- chunk: [],- allChunksSent: true,- });- } catch (error) {- logger.error({ importId, error }, "Error in CSV parse worker");- await safeUpdateStatusToFailed(importId, "An unexpected error occurred during import processing");- logger.error({ importId }, "Import failed, worker continuing");- } finally {- if (processingTimeout) {- clearTimeout(processingTimeout);- }-- if (stream) {- try {- stream.destroy();- } catch (streamError) {- logger.warn({ importId, error: streamError }, "Failed to destroy stream");- }- }-- try {- await deleteImportFile(storageLocation, isR2Storage);- } catch (deleteError) {- logger.warn({ importId, error: deleteError }, "File cleanup failed, will remain in storage");- // File will be orphaned but import status is already recorded- }- }- });-}diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsdeleted file mode 100644index cdab8a131..000000000--- a/server/src/services/import/workers/dataInsertWorker.ts+++ /dev/null@@ -1,68 +0,0 @@-import { IJobQueue } from "../queues/jobQueue.js";-import { UmamiImportMapper } from "../mappings/umami.js";-import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { updateImportStatus, updateImportProgress } from "../importStatusManager.js";-import { createServiceLogger } from "../../../lib/logger/logger.js";--const logger = createServiceLogger("import:data-insert");--const getImportDataMapping = (platform: string) => {- switch (platform) {- case "umami":- return UmamiImportMapper;- default:- throw new Error(`Unsupported platform: ${platform}`);- }-};--const safeUpdateStatusToFailed = async (importId: string, errorMessage: string) => {- try {- await updateImportStatus(importId, "failed", errorMessage);- } catch (updateError) {- logger.error({ importId, error: updateError }, "Could not update status to failed");- }-};--export async function createDataInsertWorker(jobQueue: IJobQueue) {- await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {- const { site, importId, platform, chunk, allChunksSent } = job;-- try {- if (allChunksSent) {- await updateImportStatus(importId, "completed");- logger.info({ importId }, "Import completed successfully");- return;- }-- const dataMapper = getImportDataMapping(platform);- const transformedRecords = dataMapper.transform(chunk, site, importId);-- await clickhouse.insert({- table: "events",- values: transformedRecords,- format: "JSONEachRow",- });-- logger.debug({ importId, recordCount: transformedRecords.length }, "Inserted chunk to ClickHouse");-- // Update progress (non-critical - log if fails but don't crash)- try {- await updateImportProgress(importId, transformedRecords.length);- } catch (progressError) {- logger.warn(- { importId, error: progressError instanceof Error ? progressError.message : progressError },- "Progress update failed (data inserted successfully)"- );- // Don't throw - data is safely in ClickHouse, progress can be off slightly- }- } catch (error) {- logger.error({ importId, error }, "Worker processing failed");-- const errorMessage = allChunksSent ? "Failed to complete import" : "Data insertion failed due to unknown error";- await safeUpdateStatusToFailed(importId, errorMessage);-- logger.error({ importId }, "Import operation failed, worker continuing");- }- });-}diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.tsdeleted file mode 100644index 0961409e1..000000000--- a/server/src/services/import/workers/jobs.ts+++ /dev/null@@ -1,25 +0,0 @@-import { UmamiEvent } from "../mappings/umami.js";-import { IS_PRODUCTION } from "../../../lib/const.js";--export const CSV_PARSE_QUEUE = IS_PRODUCTION ? "csv-parse" : "csv-parse-dev";--export const DATA_INSERT_QUEUE = IS_PRODUCTION ? "data-insert" : "data-insert-dev";--interface ImportJob {- site: string;- importId: string;- platform: "umami";-}--export interface CsvParseJob extends ImportJob {- storageLocation: string; // Either local file path or R2 key- isR2Storage: boolean;- organization: string;- startDate?: string;- endDate?: string;-}--export interface DataInsertJob extends ImportJob {- chunk: UmamiEvent[];- allChunksSent: boolean; // Finalization signal-}diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.tsindex 342ca421e..401453f28 100644--- a/server/src/services/storage/r2StorageService.ts+++ b/server/src/services/storage/r2StorageService.ts@@ -3,9 +3,7 @@ import {PutObjectCommand,GetObjectCommand,DeleteObjectCommand,- ListObjectsV2Command,} from "@aws-sdk/client-s3";-import { Upload } from "@aws-sdk/lib-storage";import { NodeHttpHandler } from "@smithy/node-http-handler";import { Readable } from "stream";import { gunzipSync } from "zlib";@@ -14,11 +12,10 @@ import { IS_CLOUD } from "../../lib/const.js";import { createServiceLogger } from "../../lib/logger/logger.js";class R2StorageService {- private readonly client: S3Client | null = null;- private readonly bucketName: string = "";- private readonly enabled: boolean = false;+ private client: S3Client | null = null;+ private bucketName: string = "";+ private enabled: boolean = false;private logger = createServiceLogger("r2-storage");- private readonly IMPORT_FILES_PREFIX = "imports/";constructor() {// Only initialize R2 in cloud environment@@ -203,193 +200,6 @@ class R2StorageService {// Non-critical error, log but don't throw}}-- /**- * Store an import CSV file in R2 using streaming upload- */- async storeImportFile(key: string, fileStream: Readable): Promise<void> {- if (!this.enabled || !this.client) {- throw new Error("R2 storage is not enabled");- }-- if (key.trim().length === 0) {- throw new Error("Invalid storage key: key cannot be empty");- }-- if (key.includes("..")) {- throw new Error("Invalid storage key: path traversal not allowed");- }-- try {- const upload = new Upload({- client: this.client,- params: {- Bucket: this.bucketName,- Key: key,- Body: fileStream,- ContentType: "text/csv",- },- queueSize: 4,- partSize: 5 * 1024 * 1024,- leavePartsOnError: false,- });-- await upload.done();- this.logger.info({ key }, "Successfully stored import file");- } catch (error) {- this.logger.error({ error, key }, "Failed to store import file");- throw error;- }- }-- /**- * Retrieve an import CSV file from R2 as a stream- */- async getImportFileStream(key: string): Promise<Readable> {- if (!this.enabled || !this.client) {- throw new Error("R2 storage is not enabled");- }-- if (key.trim().length === 0) {- throw new Error("Invalid storage key: key cannot be empty");- }-- if (key.includes("..")) {- throw new Error("Invalid storage key: path traversal not allowed");- }-- try {- const response = await this.client.send(- new GetObjectCommand({- Bucket: this.bucketName,- Key: key,- })- );-- if (!response.Body) {- throw new Error("Empty response body");- }-- this.logger.info({ key }, "Successfully retrieved import file stream");- return response.Body as Readable;- } catch (error) {- this.logger.error({ error, key }, "Failed to retrieve import file stream");- throw error;- }- }-- /**- * Delete an import file from R2 (for cleanup after processing)- */- async deleteImportFile(key: string): Promise<void> {- if (!this.enabled || !this.client) {- return;- }-- if (key.trim().length === 0) {- throw new Error("Invalid storage key: key cannot be empty");- }-- if (key.includes("..")) {- throw new Error("Invalid storage key: path traversal not allowed");- }-- try {- await this.client.send(- new DeleteObjectCommand({- Bucket: this.bucketName,- Key: key,- })- );- this.logger.info({ key }, "Successfully deleted import file");- } catch (error) {- this.logger.error({ error, key }, "Failed to delete import file");- // Non-critical error, log but don't throw- }- }-- /**- * List all import files in R2- * Returns array of objects with key and lastModified date- */- async listImportFiles(): Promise<Array<{ key: string; lastModified: Date }>> {- if (!this.enabled || !this.client) {- return [];- }-- try {- const allFiles: Array<{ key: string; lastModified: Date }> = [];- let continuationToken: string | undefined;-- do {- const response = await this.client.send(- new ListObjectsV2Command({- Bucket: this.bucketName,- Prefix: this.IMPORT_FILES_PREFIX,- ContinuationToken: continuationToken,- })- );-- if (response.Contents) {- for (const obj of response.Contents) {- if (obj.Key && obj.LastModified) {- allFiles.push({- key: obj.Key,- lastModified: obj.LastModified,- });- }- }- }-- continuationToken = response.NextContinuationToken;- } while (continuationToken);-- this.logger.info({ count: allFiles.length }, "Listed import files");- return allFiles;- } catch (error) {- this.logger.error({ error }, "Failed to list import files");- throw error;- }- }-- /**- * Delete all import files older than the specified number of days- * Returns count of deleted files- */- async deleteOldImportFiles(daysOld: number = 1): Promise<number> {- if (!this.enabled || !this.client) {- return 0;- }-- try {- const files = await this.listImportFiles();- const cutoffDate = new Date(Date.now() - daysOld * 24 * 60 * 60 * 1000);-- const filesToDelete = files.filter(file => file.lastModified < cutoffDate);-- this.logger.info(- { total: files.length, toDelete: filesToDelete.length, cutoffDate },- "Deleting old import files"- );-- let deletedCount = 0;- for (const file of filesToDelete) {- try {- await this.deleteImportFile(file.key);- deletedCount++;- } catch (error) {- this.logger.error({ error, key: file.key }, "Failed to delete old import file");- // Continue with other files even if one fails- }- }-- this.logger.info({ deletedCount }, "Completed deleting old import files");- return deletedCount;- } catch (error) {- this.logger.error({ error }, "Failed to delete old import files");- throw error;- }- }}// Singleton instance