OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(buffer): chunk inserts
Carl-Gerhard Lindesvärd033a6acd4396
diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex 76f2d288d..d9e5346ef 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -222,12 +222,22 @@ export class EventBuffer extends RedisBuffer<BufferType> {};}+ private getChunks(items: BufferType[], size: number) {+ const chunks = [];+ for (let i = 0; i < items.length; i += size) {+ chunks.push(items.slice(i, i + size));+ }+ return chunks;+ }+protected async insertIntoDB(items: BufferType[]): Promise<void> {- await ch.insert({- table: TABLE_NAMES.events,- values: items,- format: 'JSONEachRow',- });+ for (const chunk of this.getChunks(items, 1500)) {+ await ch.insert({+ table: TABLE_NAMES.events,+ values: chunk,+ format: 'JSONEachRow',+ });+ }}public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async (