Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

clickhouse data deletion: move from lightweight to heavyweight process

Blue Mouse45ad15faab33
diff --git a/backend/apps/cloud/src/project/project.controller.ts b/backend/apps/cloud/src/project/project.controller.ts
index 57b88c6a0..0eaa27813 100644
--- a/backend/apps/cloud/src/project/project.controller.ts
+++ b/backend/apps/cloud/src/project/project.controller.ts
@@ -662,11 +662,12 @@ export class ProjectController {
}
const queries = [
- 'DELETE FROM analytics WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM customEV WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM performance WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM errors WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM error_statuses WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}',
]
try {
@@ -680,53 +681,13 @@ export class ProjectController {
)
await Promise.all(promises)
- return 'Project resetted successfully'
+ return 'Project reset successfully'
} catch (e) {
this.logger.error(e)
return 'Error while resetting your project'
}
}
- @ApiBearerAuth()
- @Delete('/captcha/reset/:id')
- @HttpCode(204)
- @UseGuards(JwtAccessTokenGuard, RolesGuard)
- @Roles(UserType.CUSTOMER, UserType.ADMIN)
- @ApiResponse({ status: 204, description: 'Empty body' })
- async resetCAPTCHA(
- @Param('id') id: string,
- @CurrentUserId() uid: string,
- ): Promise<any> {
- this.logger.log({ uid, id }, 'DELETE /project/captcha/reset/:id')
- if (!isValidPID(id)) {
- throw new BadRequestException(
- 'The provided Project ID (pid) is incorrect',
- )
- }
-
- const user = await this.userService.findOne({ where: { id: uid } })
- const project = await this.projectService.getFullProject(id)
-
- if (_isEmpty(project)) {
- throw new NotFoundException(`Project with ID ${id} does not exist`)
- }
-
- this.projectService.allowedToManage(project, uid, user.roles)
-
- try {
- await clickhouse.command({
- query: 'DELETE FROM captcha WHERE pid={pid:FixedString(12)}',
- query_params: {
- pid: id,
- },
- })
- return 'CAPTCHA project resetted successfully'
- } catch (e) {
- this.logger.error(e)
- return 'Error while resetting your CAPTCHA project'
- }
- }
-
@ApiBearerAuth()
@Post('/secret-gen/:pid')
@HttpCode(200)
@@ -1624,12 +1585,12 @@ export class ProjectController {
}
const queries = [
- 'DELETE FROM analytics WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM customEV WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM performance WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM errors WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM error_statuses WHERE pid={pid:FixedString(12)}',
- 'DELETE FROM captcha WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}',
]
try {
diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.ts
index 27e6d16af..d425172a2 100644
--- a/backend/apps/cloud/src/project/project.service.ts
+++ b/backend/apps/cloud/src/project/project.service.ts
@@ -710,7 +710,7 @@ export class ProjectService {
)
}
- let query = `DELETE FROM analytics WHERE pid={pid:FixedString(12)} AND (`
+ let query = `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)} AND (`
const params = {
pid,
@@ -740,11 +740,11 @@ export class ProjectService {
to: string,
): Promise<void> {
const queries = [
- 'DELETE FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'DELETE FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'DELETE FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'DELETE FROM errors WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'DELETE FROM error_statuses WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
]
const params = {
diff --git a/backend/apps/cloud/src/user/user.controller.ts b/backend/apps/cloud/src/user/user.controller.ts
index b8963b96b..26f614876 100644
--- a/backend/apps/cloud/src/user/user.controller.ts
+++ b/backend/apps/cloud/src/user/user.controller.ts
@@ -343,12 +343,12 @@ export class UserController {
)
const queries = [
- `DELETE FROM analytics WHERE pid IN (${pids})`,
- `DELETE FROM customEV WHERE pid IN (${pids})`,
- `DELETE FROM performance WHERE pid IN (${pids})`,
- `DELETE FROM errors WHERE pid IN (${pids})`,
- `DELETE FROM error_statuses WHERE pid IN (${pids})`,
- `DELETE FROM captcha WHERE pid IN (${pids})`,
+ `ALTER TABLE analytics DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE customEV DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE performance DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE errors DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE error_statuses DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE captcha DELETE WHERE pid IN (${pids})`,
]
await this.projectService.deleteMultiple(user.projects.map(el => el.id))
@@ -409,12 +409,12 @@ export class UserController {
',',
)
const queries = [
- `DELETE FROM analytics WHERE pid IN (${pids})`,
- `DELETE FROM customEV WHERE pid IN (${pids})`,
- `DELETE FROM performance WHERE pid IN (${pids})`,
- `DELETE FROM errors WHERE pid IN (${pids})`,
- `DELETE FROM error_statuses WHERE pid IN (${pids})`,
- `DELETE FROM captcha WHERE pid IN (${pids})`,
+ `ALTER TABLE analytics DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE customEV DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE performance DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE errors DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE error_statuses DELETE WHERE pid IN (${pids})`,
+ `ALTER TABLE captcha DELETE WHERE pid IN (${pids})`,
]
await this.projectService.deleteMultiple(user.projects.map(el => el.id))
const promises = _map(queries, async query =>
diff --git a/backend/apps/community/src/common/utils.ts b/backend/apps/community/src/common/utils.ts
index 77bd38e79..4cb6d5a31 100644
--- a/backend/apps/community/src/common/utils.ts
+++ b/backend/apps/community/src/common/utils.ts
@@ -176,7 +176,7 @@ const updateFunnelClickhouse = async (funnel: any) => {
}
const deleteFunnelClickhouse = async (id: string) => {
- const query = `DELETE FROM funnel WHERE id = {id:String}`
+ const query = `ALTER TABLE funnel DELETE WHERE id = {id:String}`
await clickhouse.command({
query,
@@ -317,7 +317,7 @@ const calculateRelativePercentage = (
}
const deleteProjectClickhouse = async (id: string) => {
- const query = `DELETE FROM project WHERE id = {id:FixedString(12)}`
+ const query = `ALTER TABLE project DELETE WHERE id = {id:FixedString(12)}`
await clickhouse.command({
query,
@@ -390,7 +390,7 @@ const deleteRefreshTokenClickhouse = async (
userId: string,
refreshToken: string,
) => {
- const query = `DELETE FROM refresh_token WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`
+ const query = `ALTER TABLE refresh_token DELETE WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`
await clickhouse.command({
query,
@@ -631,7 +631,7 @@ const findProjectViewsClickhouse = async (projectId: string) => {
}
const deleteProjectViewClickhouse = async (id: string) => {
- const query = `DELETE FROM project_views WHERE id = {id:FixedString(36)}`
+ const query = `ALTER TABLE project_views DELETE WHERE id = {id:FixedString(36)}`
await clickhouse.command({
query,
@@ -642,7 +642,7 @@ const deleteProjectViewClickhouse = async (id: string) => {
}
const deleteCustomMetricsClickhouse = async (viewId: string) => {
- const query = `DELETE FROM projects_views_custom_events WHERE viewId = {viewId:FixedString(36)}`
+ const query = `ALTER TABLE projects_views_custom_events DELETE WHERE viewId = {viewId:FixedString(36)}`
await clickhouse.command({
query,
diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.ts
index c02e3e6ca..f6b7ca06a 100644
--- a/backend/apps/community/src/project/project.controller.ts
+++ b/backend/apps/community/src/project/project.controller.ts
@@ -236,12 +236,12 @@ export class ProjectController {
try {
await clickhouse.command({
- query: `DELETE FROM analytics WHERE pid='${id}'`,
+ query: `ALTER TABLE analytics DELETE WHERE pid='${id}'`,
})
await clickhouse.command({
- query: `DELETE FROM customEV WHERE pid='${id}'`,
+ query: `ALTER TABLE customEV DELETE WHERE pid='${id}'`,
})
- return 'Project resetted successfully'
+ return 'Project reset successfully'
} catch (e) {
this.logger.error(e)
return 'Error while resetting your project'
@@ -467,10 +467,10 @@ export class ProjectController {
try {
await clickhouse.command({
- query: `DELETE FROM analytics WHERE pid='${id}'`,
+ query: `ALTER TABLE analytics DELETE WHERE pid='${id}'`,
})
await clickhouse.command({
- query: `DELETE FROM customEV WHERE pid='${id}'`,
+ query: `ALTER TABLE customEV DELETE WHERE pid='${id}'`,
})
return 'Project deleted successfully'
} catch (e) {
diff --git a/backend/apps/community/src/project/project.service.ts b/backend/apps/community/src/project/project.service.ts
index 785199333..b135016ef 100644
--- a/backend/apps/community/src/project/project.service.ts
+++ b/backend/apps/community/src/project/project.service.ts
@@ -246,11 +246,11 @@ export class ProjectService {
to: string,
): Promise<void> {
const queries = [
- 'DELETE FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'DELETE FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'DELETE FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'DELETE FROM errors WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'DELETE FROM error_statuses WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
]
const params = {
diff --git a/web/app/api/index.ts b/web/app/api/index.ts
index 0b317b279..580399bfc 100644
--- a/web/app/api/index.ts
+++ b/web/app/api/index.ts
@@ -445,14 +445,6 @@ export const resetProject = (id: string) =>
throw _isEmpty(error.response.data?.message) ? error.response.data : error.response.data.message
})
-export const resetCaptchaProject = (id: string) =>
- api
- .delete(`project/captcha/reset/${id}`)
- .then((response) => response.data)
- .catch((error) => {
- throw _isEmpty(error.response.data?.message) ? error.response.data : error.response.data.message
- })
-
export const getProjectData = (
pid: string,
tb = 'hour',
diff --git a/web/app/pages/Captcha/Settings/CaptchaSettings.tsx b/web/app/pages/Captcha/Settings/CaptchaSettings.tsx
index ca89c5adb..ed4ea451d 100644
--- a/web/app/pages/Captcha/Settings/CaptchaSettings.tsx
+++ b/web/app/pages/Captcha/Settings/CaptchaSettings.tsx
@@ -19,7 +19,7 @@ import {
createProject,
updateProject,
deleteProject,
- resetCaptchaProject,
+ resetProject,
reGenerateCaptchaSecretKey,
getProject,
} from '~/api'
@@ -180,7 +180,7 @@ const CaptchaSettings = ({ isSettings }: CaptchaSettingsProps) => {
if (!projectResetting) {
setProjectResetting(true)
try {
- await resetCaptchaProject(id)
+ await resetProject(id)
toast.success(t('project.settings.resetted'))
navigate(routes.dashboard)
} catch (reason: any) {