OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M27-004): Replace three serial await loops with bounded-concurrency Promise.all
ralph0dbb83c25898
diff --git a/packages/core/src/modules/chart/chart.service.ts b/packages/core/src/modules/chart/chart.service.tsindex 1e1cf7768..18cb99b8b 100644--- a/packages/core/src/modules/chart/chart.service.ts+++ b/packages/core/src/modules/chart/chart.service.ts@@ -51,6 +51,7 @@ import {getRetentionCohort,} from './retention.service';import { createSankeyService, getSankey } from './sankey.service';+import { mapWithConcurrency } from './src/concurrency';import { formatClickhouseDate } from './src/dates';import { executeAggregateChart, executeChart } from './src/engine/execute';import {@@ -143,6 +144,14 @@ const DEFAULT_EVENT_PROPERTY_VALUE_AUTOCOMPLETE_LIMIT = 500;/** Profile lookups are batched so the `IN (...)` never exceeds max_query_size. */const BUCKET_PROFILES_BATCH_SIZE = 200;++/**+ * In-flight `profiles FINAL` batches. The bucket query is unbounded, so a busy+ * project's chart point is 130+ batches; issuing them all at once would replace+ * a latency problem with a ClickHouse admission-queue one on a 4-core node.+ * Four keeps the node's `max_threads = 4` worth of work busy without queueing.+ */+const BUCKET_PROFILES_FETCH_CONCURRENCY = 4;const FUNNEL_PROFILES_BATCH_SIZE = 500;const FUNNEL_PROFILES_LIMIT = 1000;@@ -178,17 +187,18 @@ async function getProfilesInBatches(projectId: string,batchSize: number): Promise<IServiceProfile[]> {- const profiles: IServiceProfile[] = [];+ const batches: string[][] = [];for (let index = 0; index < ids.length; index += batchSize) {- profiles.push(- ...(await getProfilesCached(- deps,- ids.slice(index, index + batchSize),- projectId- ))- );+ batches.push(ids.slice(index, index + batchSize));}- return profiles;+ // mapWithConcurrency resolves in batch order, so the caller still sees the+ // profiles in the order the bucket query returned their ids.+ const perBatch = await mapWithConcurrency(+ batches,+ BUCKET_PROFILES_FETCH_CONCURRENCY,+ (batch) => getProfilesCached(deps, batch, projectId)+ );+ return perBatch.flat();}// --- shared report input -----------------------------------------------------diff --git a/packages/core/src/modules/chart/src/concurrency.test.ts b/packages/core/src/modules/chart/src/concurrency.test.tsnew file mode 100644index 000000000..795b0d2ea--- /dev/null+++ b/packages/core/src/modules/chart/src/concurrency.test.ts@@ -0,0 +1,59 @@+import { describe, expect, it } from 'bun:test';+import { mapWithConcurrency } from './concurrency';++const settle = (ms: number) =>+ new Promise((resolve) => setTimeout(resolve, ms));++describe('mapWithConcurrency', () => {+ it('returns results in input order, not completion order', async () => {+ const items = [40, 30, 20, 10, 0];+ const results = await mapWithConcurrency(items, 5, async (delay) => {+ await settle(delay);+ return delay;+ });++ expect(results).toEqual(items);+ });++ it('never exceeds the limit and still visits every item once', async () => {+ const items = Array.from({ length: 20 }, (_, index) => index);+ let inFlight = 0;+ let peak = 0;++ const results = await mapWithConcurrency(items, 4, async (index) => {+ inFlight += 1;+ peak = Math.max(peak, inFlight);+ await settle(1);+ inFlight -= 1;+ return index * 2;+ });++ expect(peak).toBe(4);+ expect(results).toEqual(items.map((index) => index * 2));+ });++ it('passes the input index alongside the item', async () => {+ const results = await mapWithConcurrency(+ ['a', 'b', 'c'],+ 2,+ async (item, index) => `${index}:${item}`+ );++ expect(results).toEqual(['0:a', '1:b', '2:c']);+ });++ it('handles an empty input without spawning a worker', async () => {+ expect(await mapWithConcurrency([], 4, async () => 1)).toEqual([]);+ });++ it('rejects when one item rejects', async () => {+ await expect(+ mapWithConcurrency([1, 2, 3], 2, async (item) => {+ if (item === 2) {+ throw new Error('boom');+ }+ return item;+ })+ ).rejects.toThrow('boom');+ });+});diff --git a/packages/core/src/modules/chart/src/concurrency.ts b/packages/core/src/modules/chart/src/concurrency.tsnew file mode 100644index 000000000..9e187e0c7--- /dev/null+++ b/packages/core/src/modules/chart/src/concurrency.ts@@ -0,0 +1,25 @@+/**+ * Fan `items` out over at most `limit` in-flight promises.+ *+ * Results come back in **input order**, never completion order — every caller+ * here assembles an ordered array whose order is part of the response bytes.+ */+export async function mapWithConcurrency<Item, Result>(+ items: Item[],+ limit: number,+ run: (item: Item, index: number) => Promise<Result>+): Promise<Result[]> {+ const results = new Array<Result>(items.length);+ let next = 0;+ const workerCount = Math.max(1, Math.min(limit, items.length));+ await Promise.all(+ Array.from({ length: workerCount }, async () => {+ while (next < items.length) {+ const index = next;+ next += 1;+ results[index] = await run(items[index] as Item, index);+ }+ })+ );+ return results;+}diff --git a/packages/core/src/modules/chart/src/engine/execute.ts b/packages/core/src/modules/chart/src/engine/execute.tsindex e66e4053c..ec0c2871b 100644--- a/packages/core/src/modules/chart/src/engine/execute.ts+++ b/packages/core/src/modules/chart/src/engine/execute.ts@@ -36,25 +36,28 @@ export async function executeChart(): Promise<FinalChart> {const normalized = await normalizeWithinSubscription(deps, input);const executionPlan = plan(normalized);- const computedSeries = compute(- await fetch(deps, executionPlan),- executionPlan.definitions- );+ const previousPlan = input.previous+ ? plan({+ ...normalized,+ ...getChartPrevStartEndDate({+ startDate: normalized.startDate,+ endDate: normalized.endDate,+ }),+ })+ : null;- let previousSeries: ConcreteSeries[] | null = null;- if (input.previous) {- const previousPlan = plan({- ...normalized,- ...getChartPrevStartEndDate({- startDate: normalized.startDate,- endDate: normalized.endDate,- }),- });- previousSeries = compute(- await fetch(deps, previousPlan),- previousPlan.definitions- );- }+ // The two periods share nothing but the plan, so they go out together: a+ // 3-series comparison chart is one round trip's worth of latency, not two.+ const [computedSeries, previousSeries] = await Promise.all([+ fetch(deps, executionPlan).then((series) =>+ compute(series, executionPlan.definitions)+ ),+ previousPlan+ ? fetch(deps, previousPlan).then((series) =>+ compute(series, previousPlan.definitions)+ )+ : Promise.resolve<ConcreteSeries[] | null>(null),+ ]);return format(computedSeries,@@ -76,23 +79,19 @@ export async function executeAggregateChart(startDate: normalized.startDate,endDate: normalized.endDate,};- const computedSeries = compute(- await fetchAggregate(deps, normalized, currentPeriod, timezone),- normalized.series- );-- let previousSeries: ConcreteSeries[] | null = null;- if (input.previous) {- previousSeries = compute(- await fetchAggregate(- deps,- normalized,- getChartPrevStartEndDate(currentPeriod),- timezone- ),- normalized.series- );- }+ const [computedSeries, previousSeries] = await Promise.all([+ fetchAggregate(deps, normalized, currentPeriod, timezone).then((series) =>+ compute(series, normalized.series)+ ),+ input.previous+ ? fetchAggregate(+ deps,+ normalized,+ getChartPrevStartEndDate(currentPeriod),+ timezone+ ).then((series) => compute(series, normalized.series))+ : Promise.resolve<ConcreteSeries[] | null>(null),+ ]);return format(computedSeries,diff --git a/packages/core/src/modules/chart/src/engine/fetch.ts b/packages/core/src/modules/chart/src/engine/fetch.tsindex b48950e84..7693a3148 100644--- a/packages/core/src/modules/chart/src/engine/fetch.ts+++ b/packages/core/src/modules/chart/src/engine/fetch.ts@@ -8,6 +8,7 @@ import type {IGetChartDataInput,} from '../../../report/report.constants';import { alphabetIds } from '../../../report/report.constants';+import { mapWithConcurrency } from '../concurrency';import { runQuery } from '../run-query';import { getAggregateChartSql, getChartSql } from '../statement';import type { NormalizedInput } from './normalize';@@ -15,6 +16,17 @@ import type { ConcreteSeries, Plan, SeriesDefinition } from './types';type EventDefinition = IChartEventItem & { type: 'event' };+/**+ * In-flight ClickHouse statements per period while a chart's series are+ * fetched. ClickHouse runs each of these statements at `max_threads = 4` on a+ * 4-core node, so extra concurrency buys overlap of round-trip and merge wait,+ * not CPU: past a handful the statements queue behind each other and the wall+ * clock stops improving. Four per period — and `executeChart` runs the current+ * and previous periods together — keeps a typical 1-3 series comparison chart+ * at one round trip while capping the fan-out at eight.+ */+const SERIES_FETCH_CONCURRENCY = 4;+function runChartQuery(deps: ServiceDeps,statement: Awaited<ReturnType<typeof getChartSql>>,@@ -124,65 +136,67 @@ export async function fetch(deps: ServiceDeps,plan: Plan): Promise<ConcreteSeries[]> {- const results: ConcreteSeries[] = [];-- for (const [index, definition] of plan.definitions.entries()) {- if (definition.type !== 'event') {- continue;- }- const event = definition as EventDefinition;- const placeholder = plan.concreteSeries.find(- (series) => series.definitionId === definition.id- );- if (!placeholder) {- continue;+ const fetchable = [...plan.definitions.entries()].flatMap(+ ([index, definition]) => {+ if (definition.type !== 'event') {+ return [];+ }+ const placeholder = plan.concreteSeries.find(+ (series) => series.definitionId === definition.id+ );+ return placeholder ? [{ index, definition, placeholder }] : [];}+ );- const queryInput: IGetChartDataInput = {- event: queryEvent(event),- projectId: plan.input.projectId,- startDate: plan.input.startDate,- endDate: plan.input.endDate,- breakdowns: plan.input.breakdowns,- interval: plan.input.interval,- chartType: plan.input.chartType,- metric: plan.input.metric,- previous: plan.input.previous ?? false,- limit: plan.input.limit,- offset: plan.input.offset,- };+ const perDefinition = await mapWithConcurrency(+ fetchable,+ SERIES_FETCH_CONCURRENCY,+ async ({ index, definition, placeholder }) => {+ const event = definition as EventDefinition;+ const queryInput: IGetChartDataInput = {+ event: queryEvent(event),+ projectId: plan.input.projectId,+ startDate: plan.input.startDate,+ endDate: plan.input.endDate,+ breakdowns: plan.input.breakdowns,+ interval: plan.input.interval,+ chartType: plan.input.chartType,+ metric: plan.input.metric,+ previous: plan.input.previous ?? false,+ limit: plan.input.limit,+ offset: plan.input.offset,+ };- let rows = await runChartQuery(- deps,- await getChartSql(deps, { ...queryInput, timezone: plan.timezone }),- plan.timezone- );- // Nothing matched the breakdown: fall back to the plain series.- if (rows.length === 0 && plan.input.breakdowns.length > 0) {- rows = await runChartQuery(+ let rows = await runChartQuery(deps,- await getChartSql(deps, {- ...queryInput,- breakdowns: [],- timezone: plan.timezone,- }),+ await getChartSql(deps, { ...queryInput, timezone: plan.timezone }),plan.timezone);- }+ // Nothing matched the breakdown: fall back to the plain series.+ if (rows.length === 0 && plan.input.breakdowns.length > 0) {+ rows = await runChartQuery(+ deps,+ await getChartSql(deps, {+ ...queryInput,+ breakdowns: [],+ timezone: plan.timezone,+ }),+ plan.timezone+ );+ }- results.push(- ...expandGroupedRows({+ return expandGroupedRows({rows,breakdowns: plan.input.breakdowns,event,definition,definitionIndex: index,seriesId: (nameParts) => `${placeholder.id}-${nameParts.join('-')}`,- })- );- }+ });+ }+ );- return results;+ return perDefinition.flat();}/**@@ -195,41 +209,42 @@ export async function fetchAggregate(period: { startDate: string; endDate: string },timezone: string): Promise<ConcreteSeries[]> {- const results: ConcreteSeries[] = [];+ const fetchable = [...normalized.series.entries()].filter(+ ([, definition]) => definition.type === 'event'+ );- for (const [index, definition] of normalized.series.entries()) {- if (definition.type !== 'event') {- continue;- }- const event = definition as EventDefinition;-- const queryInput = {- event: queryEvent(event),- projectId: normalized.projectId,- startDate: period.startDate,- endDate: period.endDate,- breakdowns: normalized.breakdowns,- limit: normalized.limit,- metric: normalized.metric,- previous: normalized.previous,- timezone,- };+ const perDefinition = await mapWithConcurrency(+ fetchable,+ SERIES_FETCH_CONCURRENCY,+ async ([index, definition]) => {+ const event = definition as EventDefinition;- let rows = await runChartQuery(- deps,- await getAggregateChartSql(deps, queryInput),- timezone- );- if (rows.length === 0 && normalized.breakdowns.length > 0) {- rows = await runChartQuery(+ const queryInput = {+ event: queryEvent(event),+ projectId: normalized.projectId,+ startDate: period.startDate,+ endDate: period.endDate,+ breakdowns: normalized.breakdowns,+ limit: normalized.limit,+ metric: normalized.metric,+ previous: normalized.previous,+ timezone,+ };++ let rows = await runChartQuery(deps,- await getAggregateChartSql(deps, { ...queryInput, breakdowns: [] }),+ await getAggregateChartSql(deps, queryInput),timezone);- }+ if (rows.length === 0 && normalized.breakdowns.length > 0) {+ rows = await runChartQuery(+ deps,+ await getAggregateChartSql(deps, { ...queryInput, breakdowns: [] }),+ timezone+ );+ }- results.push(- ...expandGroupedRows({+ return expandGroupedRows({rows,breakdowns: normalized.breakdowns,event,@@ -237,9 +252,9 @@ export async function fetchAggregate(definitionIndex: index,seriesId: (nameParts) =>`${event.name}-${nameParts.join('-')}-${index}`,- })- );- }+ });+ }+ );- return results;+ return perDefinition.flat();}