📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Chunk processing
Blue Mouse4cb5ffaf4ff9
diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.tsindex addaffa19..d78bc51d1 100644--- a/backend/apps/cloud/src/task-manager/task-manager.service.ts+++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts@@ -1876,48 +1876,84 @@ export class TaskManagerService {}const pids = _map(projects, p => p.id)- const queryParams = {- pids,- nineWeeksAgo,- now,- }// No need to check for performance activity because it's not tracked without tracking analyticsconst queryAnalytics = `SELECT count() FROM analytics WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`const queryCaptcha = `SELECT count() FROM captcha WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`const queryCustomEvents = `SELECT count() FROM customEV WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`- const { data: analyticsResult } = await clickhouse- .query({- query: queryAnalytics,- query_params: queryParams,- })- .then(resultSet => resultSet.json<{ 'count()': number }>())+ // Process project IDs in chunks to avoid ClickHouse field value limit+ let totalAnalytics = 0+ let totalCaptcha = 0+ let totalCustomEvents = 0++ for (let i = 0; i < pids.length; i += CHUNK_SIZE) {+ const pidChunk = pids.slice(i, i + CHUNK_SIZE)+ const queryParams = {+ pids: pidChunk,+ nineWeeksAgo,+ now,+ }- if (analyticsResult[0]['count()'] > 0) {- return+ const { data: analyticsResult } = await clickhouse+ .query({+ query: queryAnalytics,+ query_params: queryParams,+ })+ .then(resultSet => resultSet.json<{ 'count()': number }>())++ totalAnalytics += analyticsResult[0]['count()']++ // Early return if we found activity+ if (totalAnalytics > 0) {+ return+ }}- const { data: captchaResult } = await clickhouse- .query({- query: queryCaptcha,- query_params: queryParams,- })- .then(resultSet => resultSet.json<{ 'count()': number }>())+ for (let i = 0; i < pids.length; i += CHUNK_SIZE) {+ const pidChunk = pids.slice(i, i + CHUNK_SIZE)+ const queryParams = {+ pids: pidChunk,+ nineWeeksAgo,+ now,+ }- if (captchaResult[0]['count()'] > 0) {- return+ const { data: captchaResult } = await clickhouse+ .query({+ query: queryCaptcha,+ query_params: queryParams,+ })+ .then(resultSet => resultSet.json<{ 'count()': number }>())++ totalCaptcha += captchaResult[0]['count()']++ // Early return if we found activity+ if (totalCaptcha > 0) {+ return+ }}- const { data: customEventsResult } = await clickhouse- .query({- query: queryCustomEvents,- query_params: queryParams,- })- .then(resultSet => resultSet.json<{ 'count()': number }>())+ for (let i = 0; i < pids.length; i += CHUNK_SIZE) {+ const pidChunk = pids.slice(i, i + CHUNK_SIZE)+ const queryParams = {+ pids: pidChunk,+ nineWeeksAgo,+ now,+ }- if (customEventsResult[0]['count()'] > 0) {- return+ const { data: customEventsResult } = await clickhouse+ .query({+ query: queryCustomEvents,+ query_params: queryParams,+ })+ .then(resultSet => resultSet.json<{ 'count()': number }>())++ totalCustomEvents += customEventsResult[0]['count()']++ // Early return if we found activity+ if (totalCustomEvents > 0) {+ return+ }}await this.userService.update(id, {