Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Chunk processing

Blue Mouse4cb5ffaf4ff9
diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts
index addaffa19..d78bc51d1 100644
--- a/backend/apps/cloud/src/task-manager/task-manager.service.ts
+++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts
@@ -1876,48 +1876,84 @@ export class TaskManagerService {
}
const pids = _map(projects, p => p.id)
- const queryParams = {
- pids,
- nineWeeksAgo,
- now,
- }
// No need to check for performance activity because it's not tracked without tracking analytics
const queryAnalytics = `SELECT count() FROM analytics WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`
const queryCaptcha = `SELECT count() FROM captcha WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`
const queryCustomEvents = `SELECT count() FROM customEV WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`
- const { data: analyticsResult } = await clickhouse
- .query({
- query: queryAnalytics,
- query_params: queryParams,
- })
- .then(resultSet => resultSet.json<{ 'count()': number }>())
+ // Process project IDs in chunks to avoid ClickHouse field value limit
+ let totalAnalytics = 0
+ let totalCaptcha = 0
+ let totalCustomEvents = 0
+
+ for (let i = 0; i < pids.length; i += CHUNK_SIZE) {
+ const pidChunk = pids.slice(i, i + CHUNK_SIZE)
+ const queryParams = {
+ pids: pidChunk,
+ nineWeeksAgo,
+ now,
+ }
- if (analyticsResult[0]['count()'] > 0) {
- return
+ const { data: analyticsResult } = await clickhouse
+ .query({
+ query: queryAnalytics,
+ query_params: queryParams,
+ })
+ .then(resultSet => resultSet.json<{ 'count()': number }>())
+
+ totalAnalytics += analyticsResult[0]['count()']
+
+ // Early return if we found activity
+ if (totalAnalytics > 0) {
+ return
+ }
}
- const { data: captchaResult } = await clickhouse
- .query({
- query: queryCaptcha,
- query_params: queryParams,
- })
- .then(resultSet => resultSet.json<{ 'count()': number }>())
+ for (let i = 0; i < pids.length; i += CHUNK_SIZE) {
+ const pidChunk = pids.slice(i, i + CHUNK_SIZE)
+ const queryParams = {
+ pids: pidChunk,
+ nineWeeksAgo,
+ now,
+ }
- if (captchaResult[0]['count()'] > 0) {
- return
+ const { data: captchaResult } = await clickhouse
+ .query({
+ query: queryCaptcha,
+ query_params: queryParams,
+ })
+ .then(resultSet => resultSet.json<{ 'count()': number }>())
+
+ totalCaptcha += captchaResult[0]['count()']
+
+ // Early return if we found activity
+ if (totalCaptcha > 0) {
+ return
+ }
}
- const { data: customEventsResult } = await clickhouse
- .query({
- query: queryCustomEvents,
- query_params: queryParams,
- })
- .then(resultSet => resultSet.json<{ 'count()': number }>())
+ for (let i = 0; i < pids.length; i += CHUNK_SIZE) {
+ const pidChunk = pids.slice(i, i + CHUNK_SIZE)
+ const queryParams = {
+ pids: pidChunk,
+ nineWeeksAgo,
+ now,
+ }
- if (customEventsResult[0]['count()'] > 0) {
- return
+ const { data: customEventsResult } = await clickhouse
+ .query({
+ query: queryCustomEvents,
+ query_params: queryParams,
+ })
+ .then(resultSet => resultSet.json<{ 'count()': number }>())
+
+ totalCustomEvents += customEventsResult[0]['count()']
+
+ // Early return if we found activity
+ if (totalCustomEvents > 0) {
+ return
+ }
}
await this.userService.update(id, {