Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(buffer): chunk inserts

Carl-Gerhard Lindesvärd033a6acd4396
diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.ts
index 76f2d288d..d9e5346ef 100644
--- a/packages/db/src/buffers/event-buffer.ts
+++ b/packages/db/src/buffers/event-buffer.ts
@@ -222,12 +222,22 @@ export class EventBuffer extends RedisBuffer<BufferType> {
};
}
+ private getChunks(items: BufferType[], size: number) {
+ const chunks = [];
+ for (let i = 0; i < items.length; i += size) {
+ chunks.push(items.slice(i, i + size));
+ }
+ return chunks;
+ }
+
protected async insertIntoDB(items: BufferType[]): Promise<void> {
- await ch.insert({
- table: TABLE_NAMES.events,
- values: items,
- format: 'JSONEachRow',
- });
+ for (const chunk of this.getChunks(items, 1500)) {
+ await ch.insert({
+ table: TABLE_NAMES.events,
+ values: chunk,
+ format: 'JSONEachRow',
+ });
+ }
}
public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async (