The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Rearrange redis streams (#3821)
Steven Tey7eb7b0075947
diff --git a/apps/web/app/(ee)/api/cron/streams/update-partner-stats/route.ts b/apps/web/app/(ee)/api/cron/streams/update-partner-stats/route.tsindex afb0998ec3..552d6378cc 100644--- a/apps/web/app/(ee)/api/cron/streams/update-partner-stats/route.ts+++ b/apps/web/app/(ee)/api/cron/streams/update-partner-stats/route.ts@@ -4,7 +4,7 @@ import { conn } from "@/lib/planetscale";import {PartnerActivityEvent,partnerActivityStream,-} from "@/lib/upstash/redis-streams";+} from "@/lib/upstash/redis-streams/partner-activity";import { prisma } from "@dub/prisma";import { ProgramEnrollment } from "@dub/prisma/client";import { toCentsNumber } from "@dub/utils";diff --git a/apps/web/app/(ee)/api/cron/streams/update-workspace-clicks/route.ts b/apps/web/app/(ee)/api/cron/streams/update-workspace-clicks/route.tsindex cb5b5b8a8e..0ddc6fcf11 100644--- a/apps/web/app/(ee)/api/cron/streams/update-workspace-clicks/route.ts+++ b/apps/web/app/(ee)/api/cron/streams/update-workspace-clicks/route.ts@@ -1,11 +1,11 @@import { handleAndReturnErrorResponse } from "@/lib/api/errors";import { verifyVercelSignature } from "@/lib/cron/verify-vercel";import { conn } from "@/lib/planetscale";+import { RedisStreamEntry } from "@/lib/upstash/redis-streams/client";import {- RedisStreamEntry,WorkspaceClicksUsageEvent,workspaceClicksUsageStream,-} from "@/lib/upstash/redis-streams";+} from "@/lib/upstash/redis-streams/workspace-clicks-usage";import { NextResponse } from "next/server";export const dynamic = "force-dynamic";diff --git a/apps/web/app/(ee)/api/cron/streams/update-workspace-links-usage/route.ts b/apps/web/app/(ee)/api/cron/streams/update-workspace-links-usage/route.tsindex f360a2957e..d1a632d45d 100644--- a/apps/web/app/(ee)/api/cron/streams/update-workspace-links-usage/route.ts+++ b/apps/web/app/(ee)/api/cron/streams/update-workspace-links-usage/route.ts@@ -2,11 +2,11 @@ import { handleAndReturnErrorResponse } from "@/lib/api/errors";import { sendLimitEmail } from "@/lib/cron/send-limit-email";import { verifyVercelSignature } from "@/lib/cron/verify-vercel";import { WorkspaceProps } from "@/lib/types";+import { RedisStreamEntry } from "@/lib/upstash/redis-streams/client";import {- RedisStreamEntry,WorkspaceLinksUsageEvent,workspaceLinksUsageStream,-} from "@/lib/upstash/redis-streams";+} from "@/lib/upstash/redis-streams/workspace-links-usage";import { prisma } from "@dub/prisma";import { log } from "@dub/utils";import { NextResponse } from "next/server";diff --git a/apps/web/app/api/links/sync/route.ts b/apps/web/app/api/links/sync/route.tsindex 73a1a60e81..8a8b3cea24 100644--- a/apps/web/app/api/links/sync/route.ts+++ b/apps/web/app/api/links/sync/route.ts@@ -2,7 +2,7 @@ import { propagateBulkLinkChanges } from "@/lib/api/links/propagate-bulk-link-chimport { withWorkspace } from "@/lib/auth";import { exceededLimitError } from "@/lib/exceeded-limit-error";import { SimpleLinkProps } from "@/lib/types";-import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams";+import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams/workspace-links-usage";import { prisma } from "@dub/prisma";import { NextResponse } from "next/server";diff --git a/apps/web/lib/actions/partners/upload-program-application-image.ts b/apps/web/lib/actions/partners/upload-program-application-image.tsindex dfb7d24394..f967278ec5 100644--- a/apps/web/lib/actions/partners/upload-program-application-image.ts+++ b/apps/web/lib/actions/partners/upload-program-application-image.ts@@ -3,7 +3,7 @@import { getIP } from "@/lib/api/utils/get-ip";import { storage } from "@/lib/storage";import { ratelimit } from "@/lib/upstash";-import { RATE_LIMITS } from "@/lib/upstash/ratelimit-policy";+import { RATELIMIT_POLICIES } from "@/lib/upstash/ratelimit-policies";import { prisma } from "@dub/prisma";import { nanoid, R2_URL } from "@dub/utils";import * as z from "zod/v4";@@ -13,7 +13,7 @@ const inputSchema = z.object({programSlug: z.string().trim().toLowerCase().min(1),});-const rateLimitPolicy = RATE_LIMITS.programImageUpload;+const rateLimitPolicy = RATELIMIT_POLICIES.programImageUpload;export const uploadProgramApplicationImageAction = actionClient.inputSchema(inputSchema)diff --git a/apps/web/lib/api/links/bulk-create-links.ts b/apps/web/lib/api/links/bulk-create-links.tsindex 9838f1b6f2..f923f48011 100644--- a/apps/web/lib/api/links/bulk-create-links.ts+++ b/apps/web/lib/api/links/bulk-create-links.ts@@ -1,5 +1,5 @@import { ProcessedLinkProps } from "@/lib/types";-import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams";+import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams/workspace-links-usage";import { prisma } from "@dub/prisma";import { Prisma } from "@dub/prisma/client";import { getParamsFromURL, linkConstructorSimple, truncate } from "@dub/utils";diff --git a/apps/web/lib/api/links/create-link.ts b/apps/web/lib/api/links/create-link.tsindex 0e062bcd15..f4ea48f268 100644--- a/apps/web/lib/api/links/create-link.ts+++ b/apps/web/lib/api/links/create-link.ts@@ -3,7 +3,7 @@ import { getPartnerEnrollmentInfo } from "@/lib/planetscale/get-partner-enrollmeimport { isNotHostedImage, storage } from "@/lib/storage";import { recordLink } from "@/lib/tinybird";import { ProcessedLinkProps } from "@/lib/types";-import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams";+import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams/workspace-links-usage";import { propagateWebhookTriggerChanges } from "@/lib/webhook/update-webhook";import { prisma } from "@dub/prisma";import { Prisma } from "@dub/prisma/client";diff --git a/apps/web/lib/api/partners/sync-partner-links-stats.ts b/apps/web/lib/api/partners/sync-partner-links-stats.tsindex cb2531b1d0..495565e750 100644--- a/apps/web/lib/api/partners/sync-partner-links-stats.ts+++ b/apps/web/lib/api/partners/sync-partner-links-stats.ts@@ -1,4 +1,4 @@-import { publishPartnerActivityEvent } from "@/lib/upstash/redis-streams";+import { publishPartnerActivityEvent } from "@/lib/upstash/redis-streams/partner-activity";import { prisma } from "@dub/prisma";// syncs the total links stats for a partner in a programdiff --git a/apps/web/lib/api/partners/sync-total-commissions.ts b/apps/web/lib/api/partners/sync-total-commissions.tsindex c842052a3b..d8bc1f20d0 100644--- a/apps/web/lib/api/partners/sync-total-commissions.ts+++ b/apps/web/lib/api/partners/sync-total-commissions.ts@@ -1,4 +1,4 @@-import { publishPartnerActivityEvent } from "@/lib/upstash/redis-streams";+import { publishPartnerActivityEvent } from "@/lib/upstash/redis-streams/partner-activity";import { prisma } from "@dub/prisma";async function aggregateAndUpdateTotalCommissions({diff --git a/apps/web/lib/middleware/utils/crawl-bitly.ts b/apps/web/lib/middleware/utils/crawl-bitly.tsindex d43ca2f624..cd90d6879c 100644--- a/apps/web/lib/middleware/utils/crawl-bitly.ts+++ b/apps/web/lib/middleware/utils/crawl-bitly.ts@@ -2,7 +2,7 @@ import { createId } from "@/lib/api/create-id";import { linkCache } from "@/lib/api/links/cache";import { encodeKeyIfCaseSensitive } from "@/lib/api/links/case-sensitivity";import { recordLink } from "@/lib/tinybird";-import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams";+import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams/workspace-links-usage";import { prisma } from "@dub/prisma";import {DUB_HEADERS,diff --git a/apps/web/lib/tinybird/record-click.ts b/apps/web/lib/tinybird/record-click.tsindex 5fe4099f6d..c1612cc243 100644--- a/apps/web/lib/tinybird/record-click.ts+++ b/apps/web/lib/tinybird/record-click.ts@@ -16,10 +16,8 @@ import { getIdentityHash } from "../middleware/utils/get-identity-hash";import { conn } from "../planetscale";import { WorkspaceProps } from "../types";import { redis } from "../upstash";-import {- publishPartnerActivityEvent,- publishWorkspaceClicksUsageEvent,-} from "../upstash/redis-streams";+import { publishPartnerActivityEvent } from "../upstash/redis-streams/partner-activity";+import { publishWorkspaceClicksUsageEvent } from "../upstash/redis-streams/workspace-clicks-usage";import { webhookCache } from "../webhook/cache";import { sendWebhooks } from "../webhook/qstash";import { transformClickEventData } from "../webhook/transform";diff --git a/apps/web/lib/upstash/ratelimit-policy.ts b/apps/web/lib/upstash/ratelimit-policies.tssimilarity index 83%rename from apps/web/lib/upstash/ratelimit-policy.tsrename to apps/web/lib/upstash/ratelimit-policies.tsindex d3a1382796..f5922189dc 100644--- a/apps/web/lib/upstash/ratelimit-policy.ts+++ b/apps/web/lib/upstash/ratelimit-policies.ts@@ -1,4 +1,4 @@-export const RATE_LIMITS = {+export const RATELIMIT_POLICIES = {programImageUpload: {attempts: 10,window: "24 h",diff --git a/apps/web/lib/upstash/redis-streams.ts b/apps/web/lib/upstash/redis-streams/client.tssimilarity index 60%rename from apps/web/lib/upstash/redis-streams.tsrename to apps/web/lib/upstash/redis-streams/client.tsindex 026b427b37..cbf780e3d0 100644--- a/apps/web/lib/upstash/redis-streams.ts+++ b/apps/web/lib/upstash/redis-streams/client.ts@@ -1,4 +1,4 @@-import { redis } from "./redis";+import { redis } from "../redis";export type RedisStreamEntry<T> = {id: string;@@ -137,88 +137,3 @@ export class RedisStream {}}}--/* Workspace Clicks Usage Stream */-const WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY = "workspace:usage:updates";-export const workspaceClicksUsageStream = new RedisStream(- WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY,-);-export interface WorkspaceClicksUsageEvent {- linkId: string;- workspaceId: string;- timestamp: string;-}-// Publishes a click event to any relevant streams in a single transaction-export const publishWorkspaceClicksUsageEvent = async (- event: WorkspaceClicksUsageEvent,-) => {- const { linkId, workspaceId, timestamp } = event;- try {- return await redis.xadd(WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY, "*", {- linkId,- workspaceId,- timestamp,- });- } catch (error) {- console.error("Failed to publish workspace clicks usage event:", error);- throw error;- }-};--/* Workspace Links Usage Stream */-const WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY = "workspace:linksUsage:updates";-export const workspaceLinksUsageStream = new RedisStream(- WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY,-);--export interface WorkspaceLinksUsageEvent {- workspaceId: string;- linksCount: number;- timestamp: string;-}--export const publishWorkspaceLinksUsageEvent = async (- event: WorkspaceLinksUsageEvent,-) => {- const { workspaceId, linksCount, timestamp } = event;-- try {- return await redis.xadd(WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY, "*", {- workspaceId,- linksCount,- timestamp,- });- } catch (error) {- console.error("Failed to publish workspace links usage event:", error);- throw error;- }-};--/* Partner Stats Stream */-const PARTNER_ACTIVITY_STREAM_KEY = "partner:activity:updates";-export const partnerActivityStream = new RedisStream(- PARTNER_ACTIVITY_STREAM_KEY,-);-export interface PartnerActivityEvent {- programId: string;- partnerId: string;- timestamp: string;- eventType: "click" | "lead" | "sale" | "commission";-}-// Publishes a partner activity event to the stream-export const publishPartnerActivityEvent = async (- event: PartnerActivityEvent,-) => {- const { programId, partnerId, timestamp, eventType } = event;- try {- return await redis.xadd(PARTNER_ACTIVITY_STREAM_KEY, "*", {- programId,- partnerId,- timestamp,- eventType,- });- } catch (error) {- console.error("Failed to publish partner activity event to stream:", error);- throw error;- }-};diff --git a/apps/web/lib/upstash/redis-streams/partner-activity.ts b/apps/web/lib/upstash/redis-streams/partner-activity.tsnew file mode 100644index 0000000000..3fb5761e06--- /dev/null+++ b/apps/web/lib/upstash/redis-streams/partner-activity.ts@@ -0,0 +1,34 @@+import { redis } from "../redis";+import { RedisStream } from "./client";++/* Partner Stats Stream */+const PARTNER_ACTIVITY_STREAM_KEY = "partner:activity:updates";++export const partnerActivityStream = new RedisStream(+ PARTNER_ACTIVITY_STREAM_KEY,+);++export interface PartnerActivityEvent {+ programId: string;+ partnerId: string;+ timestamp: string;+ eventType: "click" | "lead" | "sale" | "commission";+}++// Publishes a partner activity event to the stream+export const publishPartnerActivityEvent = async (+ event: PartnerActivityEvent,+) => {+ const { programId, partnerId, timestamp, eventType } = event;+ try {+ return await redis.xadd(PARTNER_ACTIVITY_STREAM_KEY, "*", {+ programId,+ partnerId,+ timestamp,+ eventType,+ });+ } catch (error) {+ console.error("Failed to publish partner activity event to stream:", error);+ throw error;+ }+};diff --git a/apps/web/lib/upstash/redis-streams/workspace-clicks-usage.ts b/apps/web/lib/upstash/redis-streams/workspace-clicks-usage.tsnew file mode 100644index 0000000000..9f5e930fe6--- /dev/null+++ b/apps/web/lib/upstash/redis-streams/workspace-clicks-usage.ts@@ -0,0 +1,31 @@+import { redis } from "../redis";+import { RedisStream } from "./client";++/* Workspace Clicks Usage Stream */+const WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY = "workspace:usage:updates";++export const workspaceClicksUsageStream = new RedisStream(+ WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY,+);++export interface WorkspaceClicksUsageEvent {+ linkId: string;+ workspaceId: string;+ timestamp: string;+}+// Publishes a click event to any relevant streams in a single transaction+export const publishWorkspaceClicksUsageEvent = async (+ event: WorkspaceClicksUsageEvent,+) => {+ const { linkId, workspaceId, timestamp } = event;+ try {+ return await redis.xadd(WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY, "*", {+ linkId,+ workspaceId,+ timestamp,+ });+ } catch (error) {+ console.error("Failed to publish workspace clicks usage event:", error);+ throw error;+ }+};diff --git a/apps/web/lib/upstash/redis-streams/workspace-links-usage.ts b/apps/web/lib/upstash/redis-streams/workspace-links-usage.tsnew file mode 100644index 0000000000..d7c029d5f8--- /dev/null+++ b/apps/web/lib/upstash/redis-streams/workspace-links-usage.ts@@ -0,0 +1,40 @@+import { conn } from "@/lib/planetscale";+import { redis } from "../redis";+import { RedisStream } from "./client";++/* Workspace Links Usage Stream */+const WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY = "workspace:linksUsage:updates";++export const workspaceLinksUsageStream = new RedisStream(+ WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY,+);++export interface WorkspaceLinksUsageEvent {+ workspaceId: string;+ linksCount: number;+ timestamp: string;+}++export const publishWorkspaceLinksUsageEvent = async (+ event: WorkspaceLinksUsageEvent,+) => {+ const { workspaceId, linksCount, timestamp } = event;++ try {+ return await redis.xadd(WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY, "*", {+ workspaceId,+ linksCount,+ timestamp,+ });+ } catch (error) {+ console.error(+ "Failed to publish workspace links usage event (falling back to database update):",+ error,+ );+ // fallback on writing directly to the database+ return await conn.execute(+ "UPDATE Project SET linksUsage = linksUsage + ?, totalLinks = totalLinks + ? WHERE id = ?",+ [linksCount, linksCount, workspaceId],+ );+ }+};