Skip to content

Open source alternative to Semrush and Ahrefs

Bound concurrent D1 upserts in saved-keyword metric refresh (#338)

Ben Senescuc993ef493100
diff --git a/src/server/features/keywords/services/research/refresh-metrics.ts b/src/server/features/keywords/services/research/refresh-metrics.ts
index 7214f6ce..3182200f 100644
--- a/src/server/features/keywords/services/research/refresh-metrics.ts
+++ b/src/server/features/keywords/services/research/refresh-metrics.ts
@@ -7,6 +7,11 @@ import {
import type { BillingCustomerContext } from "@/server/billing/subscription";
import type { RefreshSavedKeywordMetricsInput } from "@/types/schemas/keywords";
+// Cap concurrent D1 upserts per group. A project can accumulate thousands of
+// saved keywords in one location/language, and fanning out one promise each
+// would flood D1/Worker resources; write in bounded chunks instead.
+const REFRESH_UPSERT_BATCH_SIZE = 100;
+
export async function refreshSavedKeywordMetrics(
input: RefreshSavedKeywordMetricsInput,
billingCustomer: BillingCustomerContext,
@@ -41,24 +46,27 @@ export async function refreshSavedKeywordMetrics(
metrics.map((metric) => [metric.keyword.toLowerCase(), metric]),
);
- await Promise.all(
- groupRows.map((r) => {
- const metric = byKeyword.get(r.row.keyword.toLowerCase());
- if (!metric) return Promise.resolve();
- return KeywordResearchRepository.upsertKeywordMetric({
- projectId: input.projectId,
- keyword: r.row.keyword,
- locationCode,
- languageCode,
- searchVolume: metric.searchVolume,
- cpc: metric.cpc,
- competition: metric.competition,
- keywordDifficulty: metric.keywordDifficulty,
- intent: normalizeIntent(metric.intent),
- monthlySearchesJson: JSON.stringify(metric.monthlySearches),
- });
- }),
- );
+ for (let i = 0; i < groupRows.length; i += REFRESH_UPSERT_BATCH_SIZE) {
+ const chunk = groupRows.slice(i, i + REFRESH_UPSERT_BATCH_SIZE);
+ await Promise.all(
+ chunk.map((r) => {
+ const metric = byKeyword.get(r.row.keyword.toLowerCase());
+ if (!metric) return Promise.resolve();
+ return KeywordResearchRepository.upsertKeywordMetric({
+ projectId: input.projectId,
+ keyword: r.row.keyword,
+ locationCode,
+ languageCode,
+ searchVolume: metric.searchVolume,
+ cpc: metric.cpc,
+ competition: metric.competition,
+ keywordDifficulty: metric.keywordDifficulty,
+ intent: normalizeIntent(metric.intent),
+ monthlySearchesJson: JSON.stringify(metric.monthlySearches),
+ });
+ }),
+ );
+ }
updated += byKeyword.size;
}