Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

test(buffer): disable auto flush on profile

Carl-Gerhard Lindesvärdc67412778468
diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts
index a15d1df39..95441ba97 100644
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -175,9 +175,7 @@ async function start() {
{
jobId: 'flushProfiles',
repeat: {
- every: process.env.BATCH_INTERVAL
- ? parseInt(process.env.BATCH_INTERVAL, 10)
- : 1000 * 10,
+ every: 2 * 1000,
},
}
);
@@ -200,8 +198,7 @@ async function start() {
const repeatableJobs = await cronQueue.getRepeatableJobs();
- console.log('Repeatable jobs:');
- console.log(repeatableJobs);
+ logger.info('Repeatable jobs', { repeatableJobs });
await createInitialSalts();
}
diff --git a/packages/db/src/buffers/buffer.ts b/packages/db/src/buffers/buffer.ts
index 80099ee6b..7658de82c 100644
--- a/packages/db/src/buffers/buffer.ts
+++ b/packages/db/src/buffers/buffer.ts
@@ -42,6 +42,7 @@ export abstract class RedisBuffer<T> {
public table: string;
public batchSize?: number;
public logger: ReturnType<typeof createLogger>;
+ public disableAutoFlush?: boolean;
// abstract methods
public abstract onInsert?: OnInsert<T>;
@@ -50,9 +51,14 @@ export abstract class RedisBuffer<T> {
public abstract find: Find<T, unknown>;
public abstract findMany: FindMany<T, unknown>;
- constructor(options: { table: string; batchSize?: number }) {
+ constructor(options: {
+ table: string;
+ batchSize?: number;
+ disableAutoFlush?: boolean;
+ }) {
this.table = options.table;
this.batchSize = options.batchSize;
+ this.disableAutoFlush = options.disableAutoFlush;
this.logger = createLogger({ name: `buffer` }).child({
table: this.table,
});
@@ -75,7 +81,7 @@ export abstract class RedisBuffer<T> {
`Inserted item into buffer ${this.table}. Current length: ${length}`
);
- if (this.batchSize && length >= this.batchSize) {
+ if (!this.disableAutoFlush && this.batchSize && length >= this.batchSize) {
this.logger.info(
`Buffer ${this.table} reached batch size (${this.batchSize}). Flushing...`
);
diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.ts
index c236946a4..4488888da 100644
--- a/packages/db/src/buffers/profile-buffer.ts
+++ b/packages/db/src/buffers/profile-buffer.ts
@@ -24,6 +24,7 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {
super({
table: TABLE_NAMES.profiles,
batchSize: 100,
+ disableAutoFlush: true,
});
}