The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Batch click stats updates through a Redis stream cron.
Kiran K5083a67ff459
diff --git a/apps/web/app/(ee)/api/cron/streams/update-click-stats/route.ts b/apps/web/app/(ee)/api/cron/streams/update-click-stats/route.tsnew file mode 100644index 0000000000..32de415dd0--- /dev/null+++ b/apps/web/app/(ee)/api/cron/streams/update-click-stats/route.ts@@ -0,0 +1,389 @@+import { qstash } from "@/lib/cron";+import { withCron } from "@/lib/cron/with-cron";+import { conn } from "@/lib/planetscale";+import { redis } from "@/lib/upstash/redis";+import {+ ClickStatsEvent,+ clickStatsStream,+} from "@/lib/upstash/redis-streams/click-stats";+import { RedisStreamEntry } from "@/lib/upstash/redis-streams/client";+import { APP_DOMAIN_WITH_NGROK, log } from "@dub/utils";+import { format } from "date-fns";+import { NextResponse } from "next/server";+import { logAndRespond } from "../../utils";++export const dynamic = "force-dynamic";++const BATCH_SIZE = 10_000; // Max stream entries to consume per cron run+const SUB_BATCH_SIZE = 50; // DB updates to run in parallel within each batch+const BACKLOG_ALERT_THRESHOLD = 50_000; // Alert when stream length exceeds this+const BACKLOG_AGE_ALERT_MS = 5 * 60 * 1000; // Alert when oldest pending entry is older than 5 minutes+const LOCK_KEY = "lock:update-click-stats"; // Prevents concurrent GET/POST from double-counting+const LOCK_TTL_SECONDS = 600; // ≥ cron maxDuration (600s) so lock outlives a running invocation++type LinkAggregate = {+ linkId: string;+ clicks: number;+ lastClicked: number;+ entryIds: string[];+};++type WorkspaceAggregate = {+ workspaceId: string;+ clicks: number;+};++type EnrollmentAggregate = {+ programId: string;+ partnerId: string;+ clicks: number;+};++const aggregateClickStats = (+ entries: RedisStreamEntry<ClickStatsEvent>[],+): {+ linkUpdates: LinkAggregate[];+ workspaceUpdates: WorkspaceAggregate[];+ enrollmentUpdates: EnrollmentAggregate[];+} => {+ const links = new Map<string, LinkAggregate>();+ const workspaces = new Map<string, WorkspaceAggregate>();+ const enrollments = new Map<string, EnrollmentAggregate>();++ for (const entry of entries) {+ const { linkId, workspaceId, programId, partnerId, timestamp } = entry.data;++ if (!linkId) {+ continue;+ }++ const parsedTimestamp = Date.parse(timestamp);+ const lastClicked = Number.isFinite(parsedTimestamp)+ ? parsedTimestamp+ : Date.now();++ const existingLink = links.get(linkId);+ if (existingLink) {+ existingLink.clicks += 1;+ existingLink.lastClicked = Math.max(+ existingLink.lastClicked,+ lastClicked,+ );+ existingLink.entryIds.push(entry.id);+ } else {+ links.set(linkId, {+ linkId,+ clicks: 1,+ lastClicked,+ entryIds: [entry.id],+ });+ }++ if (workspaceId) {+ const existingWorkspace = workspaces.get(workspaceId);+ if (existingWorkspace) {+ existingWorkspace.clicks += 1;+ } else {+ workspaces.set(workspaceId, {+ workspaceId,+ clicks: 1,+ });+ }+ }++ if (programId && partnerId) {+ const key = `${programId}:${partnerId}`;+ const existingEnrollment = enrollments.get(key);+ if (existingEnrollment) {+ existingEnrollment.clicks += 1;+ } else {+ enrollments.set(key, {+ programId,+ partnerId,+ clicks: 1,+ });+ }+ }+ }++ return {+ linkUpdates: Array.from(links.values()),+ workspaceUpdates: Array.from(workspaces.values()),+ enrollmentUpdates: Array.from(enrollments.values()),+ };+};++const processInSubBatches = async <T>(+ items: T[],+ handler: (item: T) => Promise<{ success: boolean; error?: unknown }>,+) => {+ const errors: unknown[] = [];+ let totalProcessed = 0;++ for (let i = 0; i < items.length; i += SUB_BATCH_SIZE) {+ const batch = items.slice(i, i + SUB_BATCH_SIZE);+ const results = await Promise.allSettled(batch.map(handler));++ for (const result of results) {+ if (result.status === "fulfilled" && result.value.success) {+ totalProcessed++;+ } else if (result.status === "fulfilled" && result.value.error) {+ errors.push(result.value.error);+ } else if (result.status === "rejected") {+ errors.push(result.reason);+ }+ }+ }++ return { totalProcessed, errors };+};++const getStreamEntryAgeMs = (entryId: string | null) => {+ if (!entryId) {+ return null;+ }++ const timestampMs = Number(entryId.split("-")[0]);+ if (!Number.isFinite(timestampMs)) {+ return null;+ }++ return Date.now() - timestampMs;+};++const processClickStatsStreamBatch = () =>+ clickStatsStream.processBatch<ClickStatsEvent>(+ async (entries) => {+ if (!entries || entries.length === 0) {+ return {+ success: true,+ linkUpdates: [],+ workspaceUpdates: [],+ enrollmentUpdates: [],+ processedEntryIds: [],+ totalProcessed: 0,+ errors: [],+ };+ }++ console.log(`Aggregating ${entries.length} click stats events`);++ const { linkUpdates, workspaceUpdates, enrollmentUpdates } =+ aggregateClickStats(entries);++ if (linkUpdates.length === 0) {+ console.log("No click stats updates to process");+ return {+ success: true,+ linkUpdates: [],+ workspaceUpdates: [],+ enrollmentUpdates: [],+ processedEntryIds: entries.map((entry) => entry.id),+ totalProcessed: 0,+ errors: [],+ };+ }++ console.log(+ `Processing ${linkUpdates.length} link, ${workspaceUpdates.length} workspace, ${enrollmentUpdates.length} enrollment click stats updates...`,+ );++ const processedEntryIds: string[] = [];+ const errors: unknown[] = [];++ const linkResult = await processInSubBatches(+ linkUpdates,+ async (update) => {+ try {+ await conn.execute(+ "UPDATE Link SET clicks = clicks + ?, lastClicked = ? WHERE id = ?",+ [+ update.clicks,+ format(new Date(update.lastClicked), "yyyy-MM-dd HH:mm:ss"),+ update.linkId,+ ],+ );+ processedEntryIds.push(...update.entryIds);+ return { success: true };+ } catch (error) {+ console.error(`Failed to update link ${update.linkId}:`, error);+ return {+ success: false,+ error: { linkId: update.linkId, error },+ };+ }+ },+ );+ errors.push(...linkResult.errors);++ const workspaceResult = await processInSubBatches(+ workspaceUpdates,+ async (update) => {+ try {+ await conn.execute(+ "UPDATE Project SET usage = usage + ?, totalClicks = totalClicks + ? WHERE id = ?",+ [update.clicks, update.clicks, update.workspaceId],+ );+ return { success: true };+ } catch (error) {+ console.error(+ `Failed to update workspace ${update.workspaceId}:`,+ error,+ );+ return {+ success: false,+ error: { workspaceId: update.workspaceId, error },+ };+ }+ },+ );+ errors.push(...workspaceResult.errors);++ const enrollmentResult = await processInSubBatches(+ enrollmentUpdates,+ async (update) => {+ try {+ await conn.execute(+ "UPDATE ProgramEnrollment SET totalClicks = totalClicks + ? WHERE programId = ? AND partnerId = ?",+ [update.clicks, update.programId, update.partnerId],+ );+ return { success: true };+ } catch (error) {+ console.error(+ `Failed to update program enrollment ${update.programId}:${update.partnerId}:`,+ error,+ );+ return {+ success: false,+ error: {+ programId: update.programId,+ partnerId: update.partnerId,+ error,+ },+ };+ }+ },+ );+ errors.push(...enrollmentResult.errors);++ const totalProcessed =+ linkResult.totalProcessed ++ workspaceResult.totalProcessed ++ enrollmentResult.totalProcessed;++ console.log(+ `Processed ${linkResult.totalProcessed}/${linkUpdates.length} links, ${workspaceResult.totalProcessed}/${workspaceUpdates.length} workspaces, ${enrollmentResult.totalProcessed}/${enrollmentUpdates.length} enrollments`,+ );++ if (errors.length > 0) {+ console.error(+ `Encountered ${errors.length} errors while processing click stats:`,+ errors.slice(0, 5),+ );+ }++ return {+ linkUpdates,+ workspaceUpdates,+ enrollmentUpdates,+ errors,+ totalProcessed,+ processedEntryIds,+ entriesProcessed: entries.length,+ };+ },+ {+ count: BATCH_SIZE,+ deleteAfterRead: true,+ },+ );++const maybeAlertOnBacklog = async (streamInfo: {+ length: number;+ firstEntryId: string | null;+}) => {+ const ageMs = getStreamEntryAgeMs(streamInfo.firstEntryId);+ const isBackloggedByLength = streamInfo.length > BACKLOG_ALERT_THRESHOLD;+ const isBackloggedByAge =+ ageMs !== null && ageMs > BACKLOG_AGE_ALERT_MS && streamInfo.length > 0;++ if (!isBackloggedByLength && !isBackloggedByAge) {+ return;+ }++ await log({+ message: `Click stats stream backlog alert: length=${streamInfo.length}, oldestAgeMs=${ageMs ?? "unknown"}, firstEntryId=${streamInfo.firstEntryId ?? "none"}`,+ type: "alerts",+ });+};++const executeClickStatsCron = async () => {+ const {+ linkUpdates,+ errors,+ totalProcessed,+ entriesProcessed = 0,+ } = await processClickStatsStreamBatch();++ const streamInfo = await clickStatsStream.getStreamInfo();+ await maybeAlertOnBacklog(streamInfo);++ const hasMore =+ streamInfo.length > 0 || (entriesProcessed ?? 0) >= BATCH_SIZE;++ if (hasMore) {+ await qstash.publishJSON({+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/streams/update-click-stats`,+ method: "POST",+ body: {},+ });+ }++ if (!linkUpdates.length) {+ return NextResponse.json({+ success: true,+ message: "No updates to process",+ processed: 0,+ streamInfo,+ hasMore,+ });+ }++ const response = {+ success: true,+ processed: totalProcessed,+ errors: errors?.length || 0,+ streamInfo,+ hasMore,+ message: `Successfully processed ${totalProcessed} click stats updates`,+ };++ console.log(response);++ return NextResponse.json(response);+};++const runWithLock = async () => {+ const acquired = await redis.set(LOCK_KEY, "1", {+ nx: true,+ ex: LOCK_TTL_SECONDS,+ });++ if (!acquired) {+ return logAndRespond(+ "[update-click-stats] Another run is in progress. Skipping...",+ );+ }++ try {+ return await executeClickStatsCron();+ } finally {+ await redis.del(LOCK_KEY);+ }+};++// GET /api/cron/streams/update-click-stats+export const GET = withCron(async () => runWithLock());++// POST /api/cron/streams/update-click-stats (recursively called by QStash)+export const POST = withCron(async () => runWithLock());diff --git a/apps/web/lib/tinybird/record-click.ts b/apps/web/lib/tinybird/record-click.tsindex a5deaca012..4080a7d3a5 100644--- a/apps/web/lib/tinybird/record-click.ts+++ b/apps/web/lib/tinybird/record-click.ts@@ -12,11 +12,9 @@ import { recordClickCache } from "../api/links/record-click-cache";import { detectBot } from "../middleware/utils/detect-bot";import { detectQr } from "../middleware/utils/detect-qr";import { getIdentityHash } from "../middleware/utils/get-identity-hash";-import { conn } from "../planetscale";import { redis } from "../upstash";-import { publishPartnerActivityEvent } from "../upstash/redis-streams/partner-activity";+import { publishClickStatsEvent } from "../upstash/redis-streams/click-stats";import { publishWorkspaceClickEvent } from "../upstash/redis-streams/workspace-click-events";-import { publishWorkspaceClicksUsageEvent } from "../upstash/redis-streams/workspace-clicks-usage";/*** Recording clicks with geo, ua, referer and timestamp data@@ -183,44 +181,19 @@ export async function recordClick({).then((res) => res.json()),// cache the recorded click for the corresponding IP address in Redis for 1 hour- recordClickCache.set({ domain, key, identityHash, clickId }),-- // increment the click count for the link (based on their ID)- // we have to use planetscale connection directly (not prismaEdge) because of connection pooling- conn.execute(- "UPDATE Link SET clicks = clicks + 1, lastClicked = NOW() WHERE id = ?",- [linkId],- ),- // if the link is associated with a workspace + has a destination URL- // increment the usage count for the workspace- workspaceId &&- url &&- publishWorkspaceClicksUsageEvent({- linkId,- workspaceId,- timestamp: clickData.timestamp,- }).catch(() => {- // Fallback on writing directly to the database- return conn.execute(- "UPDATE Project p JOIN Link l ON p.id = l.projectId SET p.usage = p.usage + 1, p.totalClicks = p.totalClicks + 1 WHERE l.id = ?",- [linkId],- );- }),-- programId &&- partnerId &&- publishPartnerActivityEvent({- programId,- partnerId,- eventType: "click",- timestamp: new Date().toISOString(),- }).catch(() => {- // Fallback on writing directly to the database- return conn.execute(- "UPDATE ProgramEnrollment SET totalClicks = totalClicks + 1 WHERE programId = ? AND partnerId = ?",- [programId, partnerId],- );- }),+ recordClickCache.set({+ domain,+ key,+ identityHash,+ clickId,+ }),++ publishClickStatsEvent({+ linkId,+ timestamp: clickData.timestamp,+ ...(workspaceId && url && { workspaceId }),+ ...(programId && partnerId && { programId, partnerId }),+ }),// Publish the click eventpublishWorkspaceClickEvent(clickData),@@ -234,9 +207,7 @@ export async function recordClick({const operations = ["Tinybird click event ingestion","recordClickCache set",- "Link clicks increment",- "Workspace usage increment",- "Program enrollment totalClicks increment",+ "Click stats stream publish","Workspace click event publish",];return {diff --git a/apps/web/lib/upstash/redis-streams/click-stats.ts b/apps/web/lib/upstash/redis-streams/click-stats.tsnew file mode 100644index 0000000000..a1557ea38e--- /dev/null+++ b/apps/web/lib/upstash/redis-streams/click-stats.ts@@ -0,0 +1,74 @@+import { logger, toErrorFields } from "@/lib/axiom/server";+import { conn } from "@/lib/planetscale";+import { redis } from "../redis";+import { RedisStream } from "./client";++// TODO: clean up those legacy streams once remaining publishers+// are migrated (track-sale → workspace-clicks-usage; lead/sale/commission →+// partner-activity) or confirmed still needed.++const CLICK_STATS_STREAM_KEY = "click:stats:updates";++export const clickStatsStream = new RedisStream(CLICK_STATS_STREAM_KEY);++export interface ClickStatsEvent {+ linkId: string;+ timestamp: string;+ workspaceId?: string;+ programId?: string;+ partnerId?: string;+}++export const publishClickStatsEvent = async ({+ linkId,+ timestamp,+ workspaceId,+ programId,+ partnerId,+}: ClickStatsEvent) => {+ const payload = {+ linkId,+ timestamp,+ ...(workspaceId && { workspaceId }),+ ...(programId && partnerId && { programId, partnerId }),+ };++ try {+ return await redis.xadd(CLICK_STATS_STREAM_KEY, "*", payload);+ } catch (error) {+ logger.error("stream.publish_failed", {+ service: "upstash",+ streamKey: CLICK_STATS_STREAM_KEY,+ error: toErrorFields(error),+ correlation: {+ linkId,+ workspaceId,+ programId,+ partnerId,+ },+ });++ return await Promise.allSettled([+ conn.execute(+ "UPDATE Link SET clicks = clicks + 1, lastClicked = NOW() WHERE id = ?",+ [linkId],+ ),++ workspaceId+ ? conn.execute(+ "UPDATE Project SET usage = usage + 1, totalClicks = totalClicks + 1 WHERE id = ?",+ [workspaceId],+ )+ : null,++ programId && partnerId+ ? conn.execute(+ "UPDATE ProgramEnrollment SET totalClicks = totalClicks + 1 WHERE programId = ? AND partnerId = ?",+ [programId, partnerId],+ )+ : null,++ logger.flush(),+ ]);+ }+};diff --git a/apps/web/vercel.json b/apps/web/vercel.jsonindex 91c88525c6..8e7130cda4 100644--- a/apps/web/vercel.json+++ b/apps/web/vercel.json@@ -24,6 +24,10 @@"path": "/api/cron/streams/update-workspace-clicks","schedule": "* * * * *"},+ {+ "path": "/api/cron/streams/update-click-stats",+ "schedule": "* * * * *"+ },{"path": "/api/cron/streams/update-workspace-links-usage","schedule": "* * * * *"