📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
feature: export data from projects
kruzhambus99f50f39a76f
diff --git a/package.json b/package.jsonindex caa86c184..a426cbc64 100644--- a/package.json+++ b/package.json@@ -51,7 +51,9 @@"ioredis": "^5.2.4","ip-range-check": "^0.2.0","isbot": "^3.6.5",+ "json2csv": "^6.0.0-alpha.2","jsonwebtoken": "^9.0.0",+ "jszip": "^3.10.1","lodash": "^4.17.21","mysql2": "^2.3.3","nestjs-telegraf": "^2.6.3",diff --git a/src/marketplace/cdn/cdn.service.ts b/src/marketplace/cdn/cdn.service.tsindex c9f941fc9..d4079b1e6 100644--- a/src/marketplace/cdn/cdn.service.ts+++ b/src/marketplace/cdn/cdn.service.ts@@ -13,16 +13,22 @@ export class CdnService {private readonly configService: ConfigService,) {}- async uploadFile(file: Express.Multer.File): Promise<{ filename: string }> {+ async uploadFile(file: Express.Multer.File | Buffer, originalname?: string): Promise<{ filename: string }> {try {- const filePath = `${tmpdir()}/${file.originalname}`-- await writeFile(filePath, file.buffer)+ let filePath+ if (file instanceof Buffer) {+ filePath = `${tmpdir()}/${originalname}`+ console.log(filePath)+ await writeFile(filePath, file)+ } else {+ filePath = `${tmpdir()}/${file.originalname}`+ await writeFile(filePath, file.buffer)+ }+ console.log('saved')const form = new FormData()form.append('token', this.configService.get('CDN_ACCESS_TOKEN'))form.append('file', createReadStream(filePath))-const { data } = await this.httpService.axiosRef.post('file', form, {headers: {'Content-Type': 'multipart/form-data',@@ -36,6 +42,26 @@ export class CdnService {}}+ async uploadBuffer(buffer: Buffer, originalname: string): Promise<{ filename: string }> {+ try {+ const filePath = `${tmpdir()}/${originalname}`++ const form = new FormData()+ form.append('token', this.configService.get('CDN_ACCESS_TOKEN'))+ form.append('file', buffer, new Date().toISOString() + '.zip')++ const { data } = await this.httpService.axiosRef.post('file', form, {+ headers: {+ 'Content-Type': `multipart/form-data; boundary=${form.getBoundary()}`,+ },+ })++ return data+ } catch (error) {+ console.error(error)+ }+ }+async deleteFile(filename: string): Promise<void> {try {await this.httpService.axiosRef.delete(`file/${filename}`, {diff --git a/src/project/dto/project.dto.ts b/src/project/dto/project.dto.tsindex e80e4e971..ec00067e4 100644--- a/src/project/dto/project.dto.ts+++ b/src/project/dto/project.dto.ts@@ -46,4 +46,10 @@ export class ProjectDTO {"When true, anyone on the internet (including Google) would be able to see the project's Dashboard.",})public: boolean++ @ApiProperty({+ required: false,+ description: 'Export data from the project',+ })+ exportData: boolean}diff --git a/src/project/entity/project.entity.ts b/src/project/entity/project.entity.tsindex 8c6081b63..33f01a4c8 100644--- a/src/project/entity/project.entity.ts+++ b/src/project/entity/project.entity.ts@@ -69,4 +69,7 @@ export class Project {)@JoinTable()extensions: ExtensionToProject[]++ @Column('simple-json', { nullable: true, default: null })+ exportData: any}diff --git a/src/project/project.controller.ts b/src/project/project.controller.tsindex 20e4371b6..d19cea298 100644--- a/src/project/project.controller.ts+++ b/src/project/project.controller.ts@@ -24,6 +24,7 @@ import * as _includes from 'lodash/includes'import * as _omit from 'lodash/omit'import { ProjectService, processProjectUser } from './project.service'+import { CdnService } from 'src/marketplace/cdn/cdn.service'import { UserType, ACCOUNT_PLANS, PlanCode } from '../user/entities/user.entity'import { ActionTokenType } from '../action-tokens/action-token.entity'import { ActionTokensService } from '../action-tokens/action-tokens.service'@@ -56,6 +57,10 @@ import {deleteProjectClickhouse,} from '../common/utils'+import * as JSZip from 'jszip'+import * as dayjs from 'dayjs'+import * as json2csv from 'json2csv'+// const updateProjectRedis = async (id: string, project: Project) => {// const key = getRedisProjectKey(id)@@ -100,6 +105,7 @@ export class ProjectController {private readonly logger: AppLoggerService,private readonly actionTokensService: ActionTokensService,private readonly mailerService: MailerService,+ private readonly cdnService: CdnService,) {}@Get('/')@@ -805,4 +811,79 @@ export class ProjectController {await this.actionTokensService.delete(id)}}++ @Get('/export/:pid')+ @UseGuards(SelfhostedGuard)+ @UseGuards(RolesGuard)+ @Roles(UserType.CUSTOMER, UserType.ADMIN)+ @ApiResponse({ status: 200, type: Project })+ async exportProject(+ @Param('pid') pid: string,+ @Query() date: {+ from: string+ to: string+ },+ @CurrentUserId() uid: string,+ ): Promise<string> {+ this.logger.log({ uid, pid }, 'GET /project/export/:pid')++ let project = await this.projectService.findOne(pid, {+ relations: ['admin', 'share', 'share.user'],+ })++ if (_isEmpty(project)) {+ throw new NotFoundException(`Project with ID ${pid} does not exist`)+ }++ this.projectService.allowedToManage(project, uid, project.admin.roles)++ const fromDateTimestamp = dayjs(date.from).unix()+ const toDateTimestamp = dayjs(date.to).unix()++ const data = await clickhouse.query(+ `SELECT * FROM analytics WHERE created >= '${fromDateTimestamp}' AND created <= '${toDateTimestamp}' AND pid='${pid}' FORMAT CSVWithNames`,+ ).toPromise() as unknown++ const zip = new JSZip()++ Object.keys(data[0]).forEach((key) => {+ const csvDate = json2csv.parse(data, {+ fields: [key],+ })+ zip.file(`${key}.csv`, csvDate)+ console.log(key)+ })++ const zipData = await zip.generateAsync({ type: 'nodebuffer' })++ const { filename } = await this.cdnService.uploadBuffer(zipData, new Date().toLocaleString())++ // const zip = new JSZip();++ // const _data = zip.folder('data')++ // Object.keys(data[0]).forEach((key) => {+ // const csvData = json2csv.parse(data, {+ // fields: [key],+ // })+ // _data.file(`${key}.csv`, csvData)+ // console.log(key);+ // })++ // const zipData = await++ // const { filename } = await this.cdnService.uploadFile(zipData)++ project = {+ ...project,+ exportData: {+ filename,+ createdAt: new Date(),+ }+ }++ await this.projectService.create(project)++ return filename+ }}diff --git a/src/project/project.module.ts b/src/project/project.module.tsindex dba1cb34d..48992840f 100644--- a/src/project/project.module.ts+++ b/src/project/project.module.ts@@ -6,6 +6,7 @@ import { ProjectShare } from './entity/project-share.entity'import { ProjectService } from './project.service'import { ProjectController } from './project.controller'import { UserModule } from '../user/user.module'+import { CdnModule } from 'src/marketplace/cdn/cdn.module'import { ActionTokensModule } from '../action-tokens/action-tokens.module'import { MailerModule } from '../mailer/mailer.module'import { AppLoggerModule } from '../logger/logger.module'@@ -18,6 +19,7 @@ import { AppLoggerModule } from '../logger/logger.module'AppLoggerModule,ActionTokensModule,MailerModule,+ CdnModule,],providers: [ProjectService],exports: [ProjectService],diff --git a/src/task-manager/task-manager.service.ts b/src/task-manager/task-manager.service.tsindex c79d83523..3cdadce47 100644--- a/src/task-manager/task-manager.service.ts+++ b/src/task-manager/task-manager.service.ts@@ -489,4 +489,28 @@ export class TaskManagerService {}}}++ @Cron(CronExpression.EVERY_HOUR)+ async checkProjects(): Promise<void> {+ const projects = await this.projectService.findWhere({+ active: true,+ })++ for (let i = 0; i < _size(projects); ++i) {+ const project = projects[i]++ if (project?.exportData?.filename) {+ const { createdAt } = project.exportData+ const now = new Date()++ if (now.getTime() - createdAt.getTime() > 24 * 60 * 60 * 1000) {+ await this.projectService.update(project.id, {+ ...project,+ exportData: null,+ })+ }+ }+ }+ }+}