Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Batch click stats updates through a Redis stream cron.

Kiran K5083a67ff459
diff --git a/apps/web/app/(ee)/api/cron/streams/update-click-stats/route.ts b/apps/web/app/(ee)/api/cron/streams/update-click-stats/route.ts
new file mode 100644
index 0000000000..32de415dd0
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/streams/update-click-stats/route.ts
@@ -0,0 +1,389 @@
+import { qstash } from "@/lib/cron";
+import { withCron } from "@/lib/cron/with-cron";
+import { conn } from "@/lib/planetscale";
+import { redis } from "@/lib/upstash/redis";
+import {
+ ClickStatsEvent,
+ clickStatsStream,
+} from "@/lib/upstash/redis-streams/click-stats";
+import { RedisStreamEntry } from "@/lib/upstash/redis-streams/client";
+import { APP_DOMAIN_WITH_NGROK, log } from "@dub/utils";
+import { format } from "date-fns";
+import { NextResponse } from "next/server";
+import { logAndRespond } from "../../utils";
+
+export const dynamic = "force-dynamic";
+
+const BATCH_SIZE = 10_000; // Max stream entries to consume per cron run
+const SUB_BATCH_SIZE = 50; // DB updates to run in parallel within each batch
+const BACKLOG_ALERT_THRESHOLD = 50_000; // Alert when stream length exceeds this
+const BACKLOG_AGE_ALERT_MS = 5 * 60 * 1000; // Alert when oldest pending entry is older than 5 minutes
+const LOCK_KEY = "lock:update-click-stats"; // Prevents concurrent GET/POST from double-counting
+const LOCK_TTL_SECONDS = 600; // ≥ cron maxDuration (600s) so lock outlives a running invocation
+
+type LinkAggregate = {
+ linkId: string;
+ clicks: number;
+ lastClicked: number;
+ entryIds: string[];
+};
+
+type WorkspaceAggregate = {
+ workspaceId: string;
+ clicks: number;
+};
+
+type EnrollmentAggregate = {
+ programId: string;
+ partnerId: string;
+ clicks: number;
+};
+
+const aggregateClickStats = (
+ entries: RedisStreamEntry<ClickStatsEvent>[],
+): {
+ linkUpdates: LinkAggregate[];
+ workspaceUpdates: WorkspaceAggregate[];
+ enrollmentUpdates: EnrollmentAggregate[];
+} => {
+ const links = new Map<string, LinkAggregate>();
+ const workspaces = new Map<string, WorkspaceAggregate>();
+ const enrollments = new Map<string, EnrollmentAggregate>();
+
+ for (const entry of entries) {
+ const { linkId, workspaceId, programId, partnerId, timestamp } = entry.data;
+
+ if (!linkId) {
+ continue;
+ }
+
+ const parsedTimestamp = Date.parse(timestamp);
+ const lastClicked = Number.isFinite(parsedTimestamp)
+ ? parsedTimestamp
+ : Date.now();
+
+ const existingLink = links.get(linkId);
+ if (existingLink) {
+ existingLink.clicks += 1;
+ existingLink.lastClicked = Math.max(
+ existingLink.lastClicked,
+ lastClicked,
+ );
+ existingLink.entryIds.push(entry.id);
+ } else {
+ links.set(linkId, {
+ linkId,
+ clicks: 1,
+ lastClicked,
+ entryIds: [entry.id],
+ });
+ }
+
+ if (workspaceId) {
+ const existingWorkspace = workspaces.get(workspaceId);
+ if (existingWorkspace) {
+ existingWorkspace.clicks += 1;
+ } else {
+ workspaces.set(workspaceId, {
+ workspaceId,
+ clicks: 1,
+ });
+ }
+ }
+
+ if (programId && partnerId) {
+ const key = `${programId}:${partnerId}`;
+ const existingEnrollment = enrollments.get(key);
+ if (existingEnrollment) {
+ existingEnrollment.clicks += 1;
+ } else {
+ enrollments.set(key, {
+ programId,
+ partnerId,
+ clicks: 1,
+ });
+ }
+ }
+ }
+
+ return {
+ linkUpdates: Array.from(links.values()),
+ workspaceUpdates: Array.from(workspaces.values()),
+ enrollmentUpdates: Array.from(enrollments.values()),
+ };
+};
+
+const processInSubBatches = async <T>(
+ items: T[],
+ handler: (item: T) => Promise<{ success: boolean; error?: unknown }>,
+) => {
+ const errors: unknown[] = [];
+ let totalProcessed = 0;
+
+ for (let i = 0; i < items.length; i += SUB_BATCH_SIZE) {
+ const batch = items.slice(i, i + SUB_BATCH_SIZE);
+ const results = await Promise.allSettled(batch.map(handler));
+
+ for (const result of results) {
+ if (result.status === "fulfilled" && result.value.success) {
+ totalProcessed++;
+ } else if (result.status === "fulfilled" && result.value.error) {
+ errors.push(result.value.error);
+ } else if (result.status === "rejected") {
+ errors.push(result.reason);
+ }
+ }
+ }
+
+ return { totalProcessed, errors };
+};
+
+const getStreamEntryAgeMs = (entryId: string | null) => {
+ if (!entryId) {
+ return null;
+ }
+
+ const timestampMs = Number(entryId.split("-")[0]);
+ if (!Number.isFinite(timestampMs)) {
+ return null;
+ }
+
+ return Date.now() - timestampMs;
+};
+
+const processClickStatsStreamBatch = () =>
+ clickStatsStream.processBatch<ClickStatsEvent>(
+ async (entries) => {
+ if (!entries || entries.length === 0) {
+ return {
+ success: true,
+ linkUpdates: [],
+ workspaceUpdates: [],
+ enrollmentUpdates: [],
+ processedEntryIds: [],
+ totalProcessed: 0,
+ errors: [],
+ };
+ }
+
+ console.log(`Aggregating ${entries.length} click stats events`);
+
+ const { linkUpdates, workspaceUpdates, enrollmentUpdates } =
+ aggregateClickStats(entries);
+
+ if (linkUpdates.length === 0) {
+ console.log("No click stats updates to process");
+ return {
+ success: true,
+ linkUpdates: [],
+ workspaceUpdates: [],
+ enrollmentUpdates: [],
+ processedEntryIds: entries.map((entry) => entry.id),
+ totalProcessed: 0,
+ errors: [],
+ };
+ }
+
+ console.log(
+ `Processing ${linkUpdates.length} link, ${workspaceUpdates.length} workspace, ${enrollmentUpdates.length} enrollment click stats updates...`,
+ );
+
+ const processedEntryIds: string[] = [];
+ const errors: unknown[] = [];
+
+ const linkResult = await processInSubBatches(
+ linkUpdates,
+ async (update) => {
+ try {
+ await conn.execute(
+ "UPDATE Link SET clicks = clicks + ?, lastClicked = ? WHERE id = ?",
+ [
+ update.clicks,
+ format(new Date(update.lastClicked), "yyyy-MM-dd HH:mm:ss"),
+ update.linkId,
+ ],
+ );
+ processedEntryIds.push(...update.entryIds);
+ return { success: true };
+ } catch (error) {
+ console.error(`Failed to update link ${update.linkId}:`, error);
+ return {
+ success: false,
+ error: { linkId: update.linkId, error },
+ };
+ }
+ },
+ );
+ errors.push(...linkResult.errors);
+
+ const workspaceResult = await processInSubBatches(
+ workspaceUpdates,
+ async (update) => {
+ try {
+ await conn.execute(
+ "UPDATE Project SET usage = usage + ?, totalClicks = totalClicks + ? WHERE id = ?",
+ [update.clicks, update.clicks, update.workspaceId],
+ );
+ return { success: true };
+ } catch (error) {
+ console.error(
+ `Failed to update workspace ${update.workspaceId}:`,
+ error,
+ );
+ return {
+ success: false,
+ error: { workspaceId: update.workspaceId, error },
+ };
+ }
+ },
+ );
+ errors.push(...workspaceResult.errors);
+
+ const enrollmentResult = await processInSubBatches(
+ enrollmentUpdates,
+ async (update) => {
+ try {
+ await conn.execute(
+ "UPDATE ProgramEnrollment SET totalClicks = totalClicks + ? WHERE programId = ? AND partnerId = ?",
+ [update.clicks, update.programId, update.partnerId],
+ );
+ return { success: true };
+ } catch (error) {
+ console.error(
+ `Failed to update program enrollment ${update.programId}:${update.partnerId}:`,
+ error,
+ );
+ return {
+ success: false,
+ error: {
+ programId: update.programId,
+ partnerId: update.partnerId,
+ error,
+ },
+ };
+ }
+ },
+ );
+ errors.push(...enrollmentResult.errors);
+
+ const totalProcessed =
+ linkResult.totalProcessed +
+ workspaceResult.totalProcessed +
+ enrollmentResult.totalProcessed;
+
+ console.log(
+ `Processed ${linkResult.totalProcessed}/${linkUpdates.length} links, ${workspaceResult.totalProcessed}/${workspaceUpdates.length} workspaces, ${enrollmentResult.totalProcessed}/${enrollmentUpdates.length} enrollments`,
+ );
+
+ if (errors.length > 0) {
+ console.error(
+ `Encountered ${errors.length} errors while processing click stats:`,
+ errors.slice(0, 5),
+ );
+ }
+
+ return {
+ linkUpdates,
+ workspaceUpdates,
+ enrollmentUpdates,
+ errors,
+ totalProcessed,
+ processedEntryIds,
+ entriesProcessed: entries.length,
+ };
+ },
+ {
+ count: BATCH_SIZE,
+ deleteAfterRead: true,
+ },
+ );
+
+const maybeAlertOnBacklog = async (streamInfo: {
+ length: number;
+ firstEntryId: string | null;
+}) => {
+ const ageMs = getStreamEntryAgeMs(streamInfo.firstEntryId);
+ const isBackloggedByLength = streamInfo.length > BACKLOG_ALERT_THRESHOLD;
+ const isBackloggedByAge =
+ ageMs !== null && ageMs > BACKLOG_AGE_ALERT_MS && streamInfo.length > 0;
+
+ if (!isBackloggedByLength && !isBackloggedByAge) {
+ return;
+ }
+
+ await log({
+ message: `Click stats stream backlog alert: length=${streamInfo.length}, oldestAgeMs=${ageMs ?? "unknown"}, firstEntryId=${streamInfo.firstEntryId ?? "none"}`,
+ type: "alerts",
+ });
+};
+
+const executeClickStatsCron = async () => {
+ const {
+ linkUpdates,
+ errors,
+ totalProcessed,
+ entriesProcessed = 0,
+ } = await processClickStatsStreamBatch();
+
+ const streamInfo = await clickStatsStream.getStreamInfo();
+ await maybeAlertOnBacklog(streamInfo);
+
+ const hasMore =
+ streamInfo.length > 0 || (entriesProcessed ?? 0) >= BATCH_SIZE;
+
+ if (hasMore) {
+ await qstash.publishJSON({
+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/streams/update-click-stats`,
+ method: "POST",
+ body: {},
+ });
+ }
+
+ if (!linkUpdates.length) {
+ return NextResponse.json({
+ success: true,
+ message: "No updates to process",
+ processed: 0,
+ streamInfo,
+ hasMore,
+ });
+ }
+
+ const response = {
+ success: true,
+ processed: totalProcessed,
+ errors: errors?.length || 0,
+ streamInfo,
+ hasMore,
+ message: `Successfully processed ${totalProcessed} click stats updates`,
+ };
+
+ console.log(response);
+
+ return NextResponse.json(response);
+};
+
+const runWithLock = async () => {
+ const acquired = await redis.set(LOCK_KEY, "1", {
+ nx: true,
+ ex: LOCK_TTL_SECONDS,
+ });
+
+ if (!acquired) {
+ return logAndRespond(
+ "[update-click-stats] Another run is in progress. Skipping...",
+ );
+ }
+
+ try {
+ return await executeClickStatsCron();
+ } finally {
+ await redis.del(LOCK_KEY);
+ }
+};
+
+// GET /api/cron/streams/update-click-stats
+export const GET = withCron(async () => runWithLock());
+
+// POST /api/cron/streams/update-click-stats (recursively called by QStash)
+export const POST = withCron(async () => runWithLock());
diff --git a/apps/web/lib/tinybird/record-click.ts b/apps/web/lib/tinybird/record-click.ts
index a5deaca012..4080a7d3a5 100644
--- a/apps/web/lib/tinybird/record-click.ts
+++ b/apps/web/lib/tinybird/record-click.ts
@@ -12,11 +12,9 @@ import { recordClickCache } from "../api/links/record-click-cache";
import { detectBot } from "../middleware/utils/detect-bot";
import { detectQr } from "../middleware/utils/detect-qr";
import { getIdentityHash } from "../middleware/utils/get-identity-hash";
-import { conn } from "../planetscale";
import { redis } from "../upstash";
-import { publishPartnerActivityEvent } from "../upstash/redis-streams/partner-activity";
+import { publishClickStatsEvent } from "../upstash/redis-streams/click-stats";
import { publishWorkspaceClickEvent } from "../upstash/redis-streams/workspace-click-events";
-import { publishWorkspaceClicksUsageEvent } from "../upstash/redis-streams/workspace-clicks-usage";
/**
* Recording clicks with geo, ua, referer and timestamp data
@@ -183,44 +181,19 @@ export async function recordClick({
).then((res) => res.json()),
// cache the recorded click for the corresponding IP address in Redis for 1 hour
- recordClickCache.set({ domain, key, identityHash, clickId }),
-
- // increment the click count for the link (based on their ID)
- // we have to use planetscale connection directly (not prismaEdge) because of connection pooling
- conn.execute(
- "UPDATE Link SET clicks = clicks + 1, lastClicked = NOW() WHERE id = ?",
- [linkId],
- ),
- // if the link is associated with a workspace + has a destination URL
- // increment the usage count for the workspace
- workspaceId &&
- url &&
- publishWorkspaceClicksUsageEvent({
- linkId,
- workspaceId,
- timestamp: clickData.timestamp,
- }).catch(() => {
- // Fallback on writing directly to the database
- return conn.execute(
- "UPDATE Project p JOIN Link l ON p.id = l.projectId SET p.usage = p.usage + 1, p.totalClicks = p.totalClicks + 1 WHERE l.id = ?",
- [linkId],
- );
- }),
-
- programId &&
- partnerId &&
- publishPartnerActivityEvent({
- programId,
- partnerId,
- eventType: "click",
- timestamp: new Date().toISOString(),
- }).catch(() => {
- // Fallback on writing directly to the database
- return conn.execute(
- "UPDATE ProgramEnrollment SET totalClicks = totalClicks + 1 WHERE programId = ? AND partnerId = ?",
- [programId, partnerId],
- );
- }),
+ recordClickCache.set({
+ domain,
+ key,
+ identityHash,
+ clickId,
+ }),
+
+ publishClickStatsEvent({
+ linkId,
+ timestamp: clickData.timestamp,
+ ...(workspaceId && url && { workspaceId }),
+ ...(programId && partnerId && { programId, partnerId }),
+ }),
// Publish the click event
publishWorkspaceClickEvent(clickData),
@@ -234,9 +207,7 @@ export async function recordClick({
const operations = [
"Tinybird click event ingestion",
"recordClickCache set",
- "Link clicks increment",
- "Workspace usage increment",
- "Program enrollment totalClicks increment",
+ "Click stats stream publish",
"Workspace click event publish",
];
return {
diff --git a/apps/web/lib/upstash/redis-streams/click-stats.ts b/apps/web/lib/upstash/redis-streams/click-stats.ts
new file mode 100644
index 0000000000..a1557ea38e
--- /dev/null
+++ b/apps/web/lib/upstash/redis-streams/click-stats.ts
@@ -0,0 +1,74 @@
+import { logger, toErrorFields } from "@/lib/axiom/server";
+import { conn } from "@/lib/planetscale";
+import { redis } from "../redis";
+import { RedisStream } from "./client";
+
+// TODO: clean up those legacy streams once remaining publishers
+// are migrated (track-sale → workspace-clicks-usage; lead/sale/commission →
+// partner-activity) or confirmed still needed.
+
+const CLICK_STATS_STREAM_KEY = "click:stats:updates";
+
+export const clickStatsStream = new RedisStream(CLICK_STATS_STREAM_KEY);
+
+export interface ClickStatsEvent {
+ linkId: string;
+ timestamp: string;
+ workspaceId?: string;
+ programId?: string;
+ partnerId?: string;
+}
+
+export const publishClickStatsEvent = async ({
+ linkId,
+ timestamp,
+ workspaceId,
+ programId,
+ partnerId,
+}: ClickStatsEvent) => {
+ const payload = {
+ linkId,
+ timestamp,
+ ...(workspaceId && { workspaceId }),
+ ...(programId && partnerId && { programId, partnerId }),
+ };
+
+ try {
+ return await redis.xadd(CLICK_STATS_STREAM_KEY, "*", payload);
+ } catch (error) {
+ logger.error("stream.publish_failed", {
+ service: "upstash",
+ streamKey: CLICK_STATS_STREAM_KEY,
+ error: toErrorFields(error),
+ correlation: {
+ linkId,
+ workspaceId,
+ programId,
+ partnerId,
+ },
+ });
+
+ return await Promise.allSettled([
+ conn.execute(
+ "UPDATE Link SET clicks = clicks + 1, lastClicked = NOW() WHERE id = ?",
+ [linkId],
+ ),
+
+ workspaceId
+ ? conn.execute(
+ "UPDATE Project SET usage = usage + 1, totalClicks = totalClicks + 1 WHERE id = ?",
+ [workspaceId],
+ )
+ : null,
+
+ programId && partnerId
+ ? conn.execute(
+ "UPDATE ProgramEnrollment SET totalClicks = totalClicks + 1 WHERE programId = ? AND partnerId = ?",
+ [programId, partnerId],
+ )
+ : null,
+
+ logger.flush(),
+ ]);
+ }
+};
diff --git a/apps/web/vercel.json b/apps/web/vercel.json
index 91c88525c6..8e7130cda4 100644
--- a/apps/web/vercel.json
+++ b/apps/web/vercel.json
@@ -24,6 +24,10 @@
"path": "/api/cron/streams/update-workspace-clicks",
"schedule": "* * * * *"
},
+ {
+ "path": "/api/cron/streams/update-click-stats",
+ "schedule": "* * * * *"
+ },
{
"path": "/api/cron/streams/update-workspace-links-usage",
"schedule": "* * * * *"