🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
improve r2 perf (#459)
Bill Yang478745f758b5
diff --git a/server/src/services/replay/sessionReplayQueryService.ts b/server/src/services/replay/sessionReplayQueryService.tsindex e31f3153e..679f93b0c 100644--- a/server/src/services/replay/sessionReplayQueryService.ts+++ b/server/src/services/replay/sessionReplayQueryService.ts@@ -193,34 +193,61 @@ export class SessionReplayQueryService {// Process batches and reconstruct eventsconst events = [];+ // Separate R2 and ClickHouse batches+ const r2Batches: Array<[string, EventRow[]]> = [];+ const clickhouseBatches: Array<[string | null, EventRow[]]> = [];+for (const [batchKey, batchEvents] of eventsByBatch) {if (batchKey && r2Storage.isEnabled()) {- // Fetch event data from R2+ r2Batches.push([batchKey, batchEvents]);+ } else {+ clickhouseBatches.push([batchKey, batchEvents]);+ }+ }++ // Process ClickHouse batches immediately+ for (const [_, batchEvents] of clickhouseBatches) {+ for (const event of batchEvents) {+ events.push({+ timestamp: event.timestamp,+ type: event.type,+ data: JSON.parse(event.data),+ });+ }+ }++ // Fetch R2 batches in parallel (with concurrency limit)+ const PARALLEL_BATCH_SIZE = 20; // Fetch 20 batches at a time+ const r2Results: Array<{ batchKey: string; batchEvents: EventRow[]; data: any[] | null }> = [];++ for (let i = 0; i < r2Batches.length; i += PARALLEL_BATCH_SIZE) {+ const batchSlice = r2Batches.slice(i, i + PARALLEL_BATCH_SIZE);++ const promises = batchSlice.map(async ([batchKey, batchEvents]) => {try {const eventDataArray = await r2Storage.getBatch(batchKey);-- // Map R2 data back to events- for (const event of batchEvents) {- if (event.batch_index !== null && eventDataArray[event.batch_index]) {- events.push({- timestamp: event.timestamp,- type: event.type,- data: eventDataArray[event.batch_index],- });- }- }+ return { batchKey, batchEvents, data: eventDataArray };} catch (error) {console.error(`Failed to fetch R2 batch ${batchKey}:`, error);- // Skip this batch if R2 fetch fails+ return { batchKey, batchEvents, data: null };}- } else {- // Traditional ClickHouse storage+ });++ const results = await Promise.all(promises);+ r2Results.push(...results);+ }++ // Process R2 results+ for (const { batchEvents, data } of r2Results) {+ if (data) {for (const event of batchEvents) {- events.push({- timestamp: event.timestamp,- type: event.type,- data: JSON.parse(event.data),- });+ if (event.batch_index !== null && data[event.batch_index]) {+ events.push({+ timestamp: event.timestamp,+ type: event.type,+ data: data[event.batch_index],+ });+ }}}}