Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

feature: export data from projects

kruzhambus99f50f39a76f
diff --git a/package.json b/package.json
index caa86c184..a426cbc64 100644
--- a/package.json
+++ b/package.json
@@ -51,7 +51,9 @@
"ioredis": "^5.2.4",
"ip-range-check": "^0.2.0",
"isbot": "^3.6.5",
+ "json2csv": "^6.0.0-alpha.2",
"jsonwebtoken": "^9.0.0",
+ "jszip": "^3.10.1",
"lodash": "^4.17.21",
"mysql2": "^2.3.3",
"nestjs-telegraf": "^2.6.3",
diff --git a/src/marketplace/cdn/cdn.service.ts b/src/marketplace/cdn/cdn.service.ts
index c9f941fc9..d4079b1e6 100644
--- a/src/marketplace/cdn/cdn.service.ts
+++ b/src/marketplace/cdn/cdn.service.ts
@@ -13,16 +13,22 @@ export class CdnService {
private readonly configService: ConfigService,
) {}
- async uploadFile(file: Express.Multer.File): Promise<{ filename: string }> {
+ async uploadFile(file: Express.Multer.File | Buffer, originalname?: string): Promise<{ filename: string }> {
try {
- const filePath = `${tmpdir()}/${file.originalname}`
-
- await writeFile(filePath, file.buffer)
+ let filePath
+ if (file instanceof Buffer) {
+ filePath = `${tmpdir()}/${originalname}`
+ console.log(filePath)
+ await writeFile(filePath, file)
+ } else {
+ filePath = `${tmpdir()}/${file.originalname}`
+ await writeFile(filePath, file.buffer)
+ }
+ console.log('saved')
const form = new FormData()
form.append('token', this.configService.get('CDN_ACCESS_TOKEN'))
form.append('file', createReadStream(filePath))
-
const { data } = await this.httpService.axiosRef.post('file', form, {
headers: {
'Content-Type': 'multipart/form-data',
@@ -36,6 +42,26 @@ export class CdnService {
}
}
+ async uploadBuffer(buffer: Buffer, originalname: string): Promise<{ filename: string }> {
+ try {
+ const filePath = `${tmpdir()}/${originalname}`
+
+ const form = new FormData()
+ form.append('token', this.configService.get('CDN_ACCESS_TOKEN'))
+ form.append('file', buffer, new Date().toISOString() + '.zip')
+
+ const { data } = await this.httpService.axiosRef.post('file', form, {
+ headers: {
+ 'Content-Type': `multipart/form-data; boundary=${form.getBoundary()}`,
+ },
+ })
+
+ return data
+ } catch (error) {
+ console.error(error)
+ }
+ }
+
async deleteFile(filename: string): Promise<void> {
try {
await this.httpService.axiosRef.delete(`file/${filename}`, {
diff --git a/src/project/dto/project.dto.ts b/src/project/dto/project.dto.ts
index e80e4e971..ec00067e4 100644
--- a/src/project/dto/project.dto.ts
+++ b/src/project/dto/project.dto.ts
@@ -46,4 +46,10 @@ export class ProjectDTO {
"When true, anyone on the internet (including Google) would be able to see the project's Dashboard.",
})
public: boolean
+
+ @ApiProperty({
+ required: false,
+ description: 'Export data from the project',
+ })
+ exportData: boolean
}
diff --git a/src/project/entity/project.entity.ts b/src/project/entity/project.entity.ts
index 8c6081b63..33f01a4c8 100644
--- a/src/project/entity/project.entity.ts
+++ b/src/project/entity/project.entity.ts
@@ -69,4 +69,7 @@ export class Project {
)
@JoinTable()
extensions: ExtensionToProject[]
+
+ @Column('simple-json', { nullable: true, default: null })
+ exportData: any
}
diff --git a/src/project/project.controller.ts b/src/project/project.controller.ts
index 20e4371b6..d19cea298 100644
--- a/src/project/project.controller.ts
+++ b/src/project/project.controller.ts
@@ -24,6 +24,7 @@ import * as _includes from 'lodash/includes'
import * as _omit from 'lodash/omit'
import { ProjectService, processProjectUser } from './project.service'
+import { CdnService } from 'src/marketplace/cdn/cdn.service'
import { UserType, ACCOUNT_PLANS, PlanCode } from '../user/entities/user.entity'
import { ActionTokenType } from '../action-tokens/action-token.entity'
import { ActionTokensService } from '../action-tokens/action-tokens.service'
@@ -56,6 +57,10 @@ import {
deleteProjectClickhouse,
} from '../common/utils'
+import * as JSZip from 'jszip'
+import * as dayjs from 'dayjs'
+import * as json2csv from 'json2csv'
+
// const updateProjectRedis = async (id: string, project: Project) => {
// const key = getRedisProjectKey(id)
@@ -100,6 +105,7 @@ export class ProjectController {
private readonly logger: AppLoggerService,
private readonly actionTokensService: ActionTokensService,
private readonly mailerService: MailerService,
+ private readonly cdnService: CdnService,
) {}
@Get('/')
@@ -805,4 +811,79 @@ export class ProjectController {
await this.actionTokensService.delete(id)
}
}
+
+ @Get('/export/:pid')
+ @UseGuards(SelfhostedGuard)
+ @UseGuards(RolesGuard)
+ @Roles(UserType.CUSTOMER, UserType.ADMIN)
+ @ApiResponse({ status: 200, type: Project })
+ async exportProject(
+ @Param('pid') pid: string,
+ @Query() date: {
+ from: string
+ to: string
+ },
+ @CurrentUserId() uid: string,
+ ): Promise<string> {
+ this.logger.log({ uid, pid }, 'GET /project/export/:pid')
+
+ let project = await this.projectService.findOne(pid, {
+ relations: ['admin', 'share', 'share.user'],
+ })
+
+ if (_isEmpty(project)) {
+ throw new NotFoundException(`Project with ID ${pid} does not exist`)
+ }
+
+ this.projectService.allowedToManage(project, uid, project.admin.roles)
+
+ const fromDateTimestamp = dayjs(date.from).unix()
+ const toDateTimestamp = dayjs(date.to).unix()
+
+ const data = await clickhouse.query(
+ `SELECT * FROM analytics WHERE created >= '${fromDateTimestamp}' AND created <= '${toDateTimestamp}' AND pid='${pid}' FORMAT CSVWithNames`,
+ ).toPromise() as unknown
+
+ const zip = new JSZip()
+
+ Object.keys(data[0]).forEach((key) => {
+ const csvDate = json2csv.parse(data, {
+ fields: [key],
+ })
+ zip.file(`${key}.csv`, csvDate)
+ console.log(key)
+ })
+
+ const zipData = await zip.generateAsync({ type: 'nodebuffer' })
+
+ const { filename } = await this.cdnService.uploadBuffer(zipData, new Date().toLocaleString())
+
+ // const zip = new JSZip();
+
+ // const _data = zip.folder('data')
+
+ // Object.keys(data[0]).forEach((key) => {
+ // const csvData = json2csv.parse(data, {
+ // fields: [key],
+ // })
+ // _data.file(`${key}.csv`, csvData)
+ // console.log(key);
+ // })
+
+ // const zipData = await
+
+ // const { filename } = await this.cdnService.uploadFile(zipData)
+
+ project = {
+ ...project,
+ exportData: {
+ filename,
+ createdAt: new Date(),
+ }
+ }
+
+ await this.projectService.create(project)
+
+ return filename
+ }
}
diff --git a/src/project/project.module.ts b/src/project/project.module.ts
index dba1cb34d..48992840f 100644
--- a/src/project/project.module.ts
+++ b/src/project/project.module.ts
@@ -6,6 +6,7 @@ import { ProjectShare } from './entity/project-share.entity'
import { ProjectService } from './project.service'
import { ProjectController } from './project.controller'
import { UserModule } from '../user/user.module'
+import { CdnModule } from 'src/marketplace/cdn/cdn.module'
import { ActionTokensModule } from '../action-tokens/action-tokens.module'
import { MailerModule } from '../mailer/mailer.module'
import { AppLoggerModule } from '../logger/logger.module'
@@ -18,6 +19,7 @@ import { AppLoggerModule } from '../logger/logger.module'
AppLoggerModule,
ActionTokensModule,
MailerModule,
+ CdnModule,
],
providers: [ProjectService],
exports: [ProjectService],
diff --git a/src/task-manager/task-manager.service.ts b/src/task-manager/task-manager.service.ts
index c79d83523..3cdadce47 100644
--- a/src/task-manager/task-manager.service.ts
+++ b/src/task-manager/task-manager.service.ts
@@ -489,4 +489,28 @@ export class TaskManagerService {
}
}
}
+
+ @Cron(CronExpression.EVERY_HOUR)
+ async checkProjects(): Promise<void> {
+ const projects = await this.projectService.findWhere({
+ active: true,
+ })
+
+ for (let i = 0; i < _size(projects); ++i) {
+ const project = projects[i]
+
+ if (project?.exportData?.filename) {
+ const { createdAt } = project.exportData
+ const now = new Date()
+
+ if (now.getTime() - createdAt.getTime() > 24 * 60 * 60 * 1000) {
+ await this.projectService.update(project.id, {
+ ...project,
+ exportData: null,
+ })
+ }
+ }
+ }
+ }
+
}