📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
clickhouse data deletion: move from lightweight to heavyweight process
Blue Mouse45ad15faab33
diff --git a/backend/apps/cloud/src/project/project.controller.ts b/backend/apps/cloud/src/project/project.controller.tsindex 57b88c6a0..0eaa27813 100644--- a/backend/apps/cloud/src/project/project.controller.ts+++ b/backend/apps/cloud/src/project/project.controller.ts@@ -662,11 +662,12 @@ export class ProjectController {}const queries = [- 'DELETE FROM analytics WHERE pid={pid:FixedString(12)}',- 'DELETE FROM customEV WHERE pid={pid:FixedString(12)}',- 'DELETE FROM performance WHERE pid={pid:FixedString(12)}',- 'DELETE FROM errors WHERE pid={pid:FixedString(12)}',- 'DELETE FROM error_statuses WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}',]try {@@ -680,53 +681,13 @@ export class ProjectController {)await Promise.all(promises)- return 'Project resetted successfully'+ return 'Project reset successfully'} catch (e) {this.logger.error(e)return 'Error while resetting your project'}}- @ApiBearerAuth()- @Delete('/captcha/reset/:id')- @HttpCode(204)- @UseGuards(JwtAccessTokenGuard, RolesGuard)- @Roles(UserType.CUSTOMER, UserType.ADMIN)- @ApiResponse({ status: 204, description: 'Empty body' })- async resetCAPTCHA(- @Param('id') id: string,- @CurrentUserId() uid: string,- ): Promise<any> {- this.logger.log({ uid, id }, 'DELETE /project/captcha/reset/:id')- if (!isValidPID(id)) {- throw new BadRequestException(- 'The provided Project ID (pid) is incorrect',- )- }-- const user = await this.userService.findOne({ where: { id: uid } })- const project = await this.projectService.getFullProject(id)-- if (_isEmpty(project)) {- throw new NotFoundException(`Project with ID ${id} does not exist`)- }-- this.projectService.allowedToManage(project, uid, user.roles)-- try {- await clickhouse.command({- query: 'DELETE FROM captcha WHERE pid={pid:FixedString(12)}',- query_params: {- pid: id,- },- })- return 'CAPTCHA project resetted successfully'- } catch (e) {- this.logger.error(e)- return 'Error while resetting your CAPTCHA project'- }- }-@ApiBearerAuth()@Post('/secret-gen/:pid')@HttpCode(200)@@ -1624,12 +1585,12 @@ export class ProjectController {}const queries = [- 'DELETE FROM analytics WHERE pid={pid:FixedString(12)}',- 'DELETE FROM customEV WHERE pid={pid:FixedString(12)}',- 'DELETE FROM performance WHERE pid={pid:FixedString(12)}',- 'DELETE FROM errors WHERE pid={pid:FixedString(12)}',- 'DELETE FROM error_statuses WHERE pid={pid:FixedString(12)}',- 'DELETE FROM captcha WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}',]try {diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.tsindex 27e6d16af..d425172a2 100644--- a/backend/apps/cloud/src/project/project.service.ts+++ b/backend/apps/cloud/src/project/project.service.ts@@ -710,7 +710,7 @@ export class ProjectService {)}- let query = `DELETE FROM analytics WHERE pid={pid:FixedString(12)} AND (`+ let query = `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)} AND (`const params = {pid,@@ -740,11 +740,11 @@ export class ProjectService {to: string,): Promise<void> {const queries = [- 'DELETE FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'DELETE FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'DELETE FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'DELETE FROM errors WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'DELETE FROM error_statuses WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',]const params = {diff --git a/backend/apps/cloud/src/user/user.controller.ts b/backend/apps/cloud/src/user/user.controller.tsindex b8963b96b..26f614876 100644--- a/backend/apps/cloud/src/user/user.controller.ts+++ b/backend/apps/cloud/src/user/user.controller.ts@@ -343,12 +343,12 @@ export class UserController {)const queries = [- `DELETE FROM analytics WHERE pid IN (${pids})`,- `DELETE FROM customEV WHERE pid IN (${pids})`,- `DELETE FROM performance WHERE pid IN (${pids})`,- `DELETE FROM errors WHERE pid IN (${pids})`,- `DELETE FROM error_statuses WHERE pid IN (${pids})`,- `DELETE FROM captcha WHERE pid IN (${pids})`,+ `ALTER TABLE analytics DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE customEV DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE performance DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE errors DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE error_statuses DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE captcha DELETE WHERE pid IN (${pids})`,]await this.projectService.deleteMultiple(user.projects.map(el => el.id))@@ -409,12 +409,12 @@ export class UserController {',',)const queries = [- `DELETE FROM analytics WHERE pid IN (${pids})`,- `DELETE FROM customEV WHERE pid IN (${pids})`,- `DELETE FROM performance WHERE pid IN (${pids})`,- `DELETE FROM errors WHERE pid IN (${pids})`,- `DELETE FROM error_statuses WHERE pid IN (${pids})`,- `DELETE FROM captcha WHERE pid IN (${pids})`,+ `ALTER TABLE analytics DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE customEV DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE performance DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE errors DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE error_statuses DELETE WHERE pid IN (${pids})`,+ `ALTER TABLE captcha DELETE WHERE pid IN (${pids})`,]await this.projectService.deleteMultiple(user.projects.map(el => el.id))const promises = _map(queries, async query =>diff --git a/backend/apps/community/src/common/utils.ts b/backend/apps/community/src/common/utils.tsindex 77bd38e79..4cb6d5a31 100644--- a/backend/apps/community/src/common/utils.ts+++ b/backend/apps/community/src/common/utils.ts@@ -176,7 +176,7 @@ const updateFunnelClickhouse = async (funnel: any) => {}const deleteFunnelClickhouse = async (id: string) => {- const query = `DELETE FROM funnel WHERE id = {id:String}`+ const query = `ALTER TABLE funnel DELETE WHERE id = {id:String}`await clickhouse.command({query,@@ -317,7 +317,7 @@ const calculateRelativePercentage = (}const deleteProjectClickhouse = async (id: string) => {- const query = `DELETE FROM project WHERE id = {id:FixedString(12)}`+ const query = `ALTER TABLE project DELETE WHERE id = {id:FixedString(12)}`await clickhouse.command({query,@@ -390,7 +390,7 @@ const deleteRefreshTokenClickhouse = async (userId: string,refreshToken: string,) => {- const query = `DELETE FROM refresh_token WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`+ const query = `ALTER TABLE refresh_token DELETE WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`await clickhouse.command({query,@@ -631,7 +631,7 @@ const findProjectViewsClickhouse = async (projectId: string) => {}const deleteProjectViewClickhouse = async (id: string) => {- const query = `DELETE FROM project_views WHERE id = {id:FixedString(36)}`+ const query = `ALTER TABLE project_views DELETE WHERE id = {id:FixedString(36)}`await clickhouse.command({query,@@ -642,7 +642,7 @@ const deleteProjectViewClickhouse = async (id: string) => {}const deleteCustomMetricsClickhouse = async (viewId: string) => {- const query = `DELETE FROM projects_views_custom_events WHERE viewId = {viewId:FixedString(36)}`+ const query = `ALTER TABLE projects_views_custom_events DELETE WHERE viewId = {viewId:FixedString(36)}`await clickhouse.command({query,diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.tsindex c02e3e6ca..f6b7ca06a 100644--- a/backend/apps/community/src/project/project.controller.ts+++ b/backend/apps/community/src/project/project.controller.ts@@ -236,12 +236,12 @@ export class ProjectController {try {await clickhouse.command({- query: `DELETE FROM analytics WHERE pid='${id}'`,+ query: `ALTER TABLE analytics DELETE WHERE pid='${id}'`,})await clickhouse.command({- query: `DELETE FROM customEV WHERE pid='${id}'`,+ query: `ALTER TABLE customEV DELETE WHERE pid='${id}'`,})- return 'Project resetted successfully'+ return 'Project reset successfully'} catch (e) {this.logger.error(e)return 'Error while resetting your project'@@ -467,10 +467,10 @@ export class ProjectController {try {await clickhouse.command({- query: `DELETE FROM analytics WHERE pid='${id}'`,+ query: `ALTER TABLE analytics DELETE WHERE pid='${id}'`,})await clickhouse.command({- query: `DELETE FROM customEV WHERE pid='${id}'`,+ query: `ALTER TABLE customEV DELETE WHERE pid='${id}'`,})return 'Project deleted successfully'} catch (e) {diff --git a/backend/apps/community/src/project/project.service.ts b/backend/apps/community/src/project/project.service.tsindex 785199333..b135016ef 100644--- a/backend/apps/community/src/project/project.service.ts+++ b/backend/apps/community/src/project/project.service.ts@@ -246,11 +246,11 @@ export class ProjectService {to: string,): Promise<void> {const queries = [- 'DELETE FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'DELETE FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'DELETE FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'DELETE FROM errors WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'DELETE FROM error_statuses WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',]const params = {diff --git a/web/app/api/index.ts b/web/app/api/index.tsindex 0b317b279..580399bfc 100644--- a/web/app/api/index.ts+++ b/web/app/api/index.ts@@ -445,14 +445,6 @@ export const resetProject = (id: string) =>throw _isEmpty(error.response.data?.message) ? error.response.data : error.response.data.message})-export const resetCaptchaProject = (id: string) =>- api- .delete(`project/captcha/reset/${id}`)- .then((response) => response.data)- .catch((error) => {- throw _isEmpty(error.response.data?.message) ? error.response.data : error.response.data.message- })-export const getProjectData = (pid: string,tb = 'hour',diff --git a/web/app/pages/Captcha/Settings/CaptchaSettings.tsx b/web/app/pages/Captcha/Settings/CaptchaSettings.tsxindex ca89c5adb..ed4ea451d 100644--- a/web/app/pages/Captcha/Settings/CaptchaSettings.tsx+++ b/web/app/pages/Captcha/Settings/CaptchaSettings.tsx@@ -19,7 +19,7 @@ import {createProject,updateProject,deleteProject,- resetCaptchaProject,+ resetProject,reGenerateCaptchaSecretKey,getProject,} from '~/api'@@ -180,7 +180,7 @@ const CaptchaSettings = ({ isSettings }: CaptchaSettingsProps) => {if (!projectResetting) {setProjectResetting(true)try {- await resetCaptchaProject(id)+ await resetProject(id)toast.success(t('project.settings.resetted'))navigate(routes.dashboard)} catch (reason: any) {