Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Rearrange redis streams (#3821)

Steven Tey7eb7b0075947
diff --git a/apps/web/app/(ee)/api/cron/streams/update-partner-stats/route.ts b/apps/web/app/(ee)/api/cron/streams/update-partner-stats/route.ts
index afb0998ec3..552d6378cc 100644
--- a/apps/web/app/(ee)/api/cron/streams/update-partner-stats/route.ts
+++ b/apps/web/app/(ee)/api/cron/streams/update-partner-stats/route.ts
@@ -4,7 +4,7 @@ import { conn } from "@/lib/planetscale";
import {
PartnerActivityEvent,
partnerActivityStream,
-} from "@/lib/upstash/redis-streams";
+} from "@/lib/upstash/redis-streams/partner-activity";
import { prisma } from "@dub/prisma";
import { ProgramEnrollment } from "@dub/prisma/client";
import { toCentsNumber } from "@dub/utils";
diff --git a/apps/web/app/(ee)/api/cron/streams/update-workspace-clicks/route.ts b/apps/web/app/(ee)/api/cron/streams/update-workspace-clicks/route.ts
index cb5b5b8a8e..0ddc6fcf11 100644
--- a/apps/web/app/(ee)/api/cron/streams/update-workspace-clicks/route.ts
+++ b/apps/web/app/(ee)/api/cron/streams/update-workspace-clicks/route.ts
@@ -1,11 +1,11 @@
import { handleAndReturnErrorResponse } from "@/lib/api/errors";
import { verifyVercelSignature } from "@/lib/cron/verify-vercel";
import { conn } from "@/lib/planetscale";
+import { RedisStreamEntry } from "@/lib/upstash/redis-streams/client";
import {
- RedisStreamEntry,
WorkspaceClicksUsageEvent,
workspaceClicksUsageStream,
-} from "@/lib/upstash/redis-streams";
+} from "@/lib/upstash/redis-streams/workspace-clicks-usage";
import { NextResponse } from "next/server";
export const dynamic = "force-dynamic";
diff --git a/apps/web/app/(ee)/api/cron/streams/update-workspace-links-usage/route.ts b/apps/web/app/(ee)/api/cron/streams/update-workspace-links-usage/route.ts
index f360a2957e..d1a632d45d 100644
--- a/apps/web/app/(ee)/api/cron/streams/update-workspace-links-usage/route.ts
+++ b/apps/web/app/(ee)/api/cron/streams/update-workspace-links-usage/route.ts
@@ -2,11 +2,11 @@ import { handleAndReturnErrorResponse } from "@/lib/api/errors";
import { sendLimitEmail } from "@/lib/cron/send-limit-email";
import { verifyVercelSignature } from "@/lib/cron/verify-vercel";
import { WorkspaceProps } from "@/lib/types";
+import { RedisStreamEntry } from "@/lib/upstash/redis-streams/client";
import {
- RedisStreamEntry,
WorkspaceLinksUsageEvent,
workspaceLinksUsageStream,
-} from "@/lib/upstash/redis-streams";
+} from "@/lib/upstash/redis-streams/workspace-links-usage";
import { prisma } from "@dub/prisma";
import { log } from "@dub/utils";
import { NextResponse } from "next/server";
diff --git a/apps/web/app/api/links/sync/route.ts b/apps/web/app/api/links/sync/route.ts
index 73a1a60e81..8a8b3cea24 100644
--- a/apps/web/app/api/links/sync/route.ts
+++ b/apps/web/app/api/links/sync/route.ts
@@ -2,7 +2,7 @@ import { propagateBulkLinkChanges } from "@/lib/api/links/propagate-bulk-link-ch
import { withWorkspace } from "@/lib/auth";
import { exceededLimitError } from "@/lib/exceeded-limit-error";
import { SimpleLinkProps } from "@/lib/types";
-import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams";
+import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams/workspace-links-usage";
import { prisma } from "@dub/prisma";
import { NextResponse } from "next/server";
diff --git a/apps/web/lib/actions/partners/upload-program-application-image.ts b/apps/web/lib/actions/partners/upload-program-application-image.ts
index dfb7d24394..f967278ec5 100644
--- a/apps/web/lib/actions/partners/upload-program-application-image.ts
+++ b/apps/web/lib/actions/partners/upload-program-application-image.ts
@@ -3,7 +3,7 @@
import { getIP } from "@/lib/api/utils/get-ip";
import { storage } from "@/lib/storage";
import { ratelimit } from "@/lib/upstash";
-import { RATE_LIMITS } from "@/lib/upstash/ratelimit-policy";
+import { RATELIMIT_POLICIES } from "@/lib/upstash/ratelimit-policies";
import { prisma } from "@dub/prisma";
import { nanoid, R2_URL } from "@dub/utils";
import * as z from "zod/v4";
@@ -13,7 +13,7 @@ const inputSchema = z.object({
programSlug: z.string().trim().toLowerCase().min(1),
});
-const rateLimitPolicy = RATE_LIMITS.programImageUpload;
+const rateLimitPolicy = RATELIMIT_POLICIES.programImageUpload;
export const uploadProgramApplicationImageAction = actionClient
.inputSchema(inputSchema)
diff --git a/apps/web/lib/api/links/bulk-create-links.ts b/apps/web/lib/api/links/bulk-create-links.ts
index 9838f1b6f2..f923f48011 100644
--- a/apps/web/lib/api/links/bulk-create-links.ts
+++ b/apps/web/lib/api/links/bulk-create-links.ts
@@ -1,5 +1,5 @@
import { ProcessedLinkProps } from "@/lib/types";
-import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams";
+import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams/workspace-links-usage";
import { prisma } from "@dub/prisma";
import { Prisma } from "@dub/prisma/client";
import { getParamsFromURL, linkConstructorSimple, truncate } from "@dub/utils";
diff --git a/apps/web/lib/api/links/create-link.ts b/apps/web/lib/api/links/create-link.ts
index 0e062bcd15..f4ea48f268 100644
--- a/apps/web/lib/api/links/create-link.ts
+++ b/apps/web/lib/api/links/create-link.ts
@@ -3,7 +3,7 @@ import { getPartnerEnrollmentInfo } from "@/lib/planetscale/get-partner-enrollme
import { isNotHostedImage, storage } from "@/lib/storage";
import { recordLink } from "@/lib/tinybird";
import { ProcessedLinkProps } from "@/lib/types";
-import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams";
+import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams/workspace-links-usage";
import { propagateWebhookTriggerChanges } from "@/lib/webhook/update-webhook";
import { prisma } from "@dub/prisma";
import { Prisma } from "@dub/prisma/client";
diff --git a/apps/web/lib/api/partners/sync-partner-links-stats.ts b/apps/web/lib/api/partners/sync-partner-links-stats.ts
index cb2531b1d0..495565e750 100644
--- a/apps/web/lib/api/partners/sync-partner-links-stats.ts
+++ b/apps/web/lib/api/partners/sync-partner-links-stats.ts
@@ -1,4 +1,4 @@
-import { publishPartnerActivityEvent } from "@/lib/upstash/redis-streams";
+import { publishPartnerActivityEvent } from "@/lib/upstash/redis-streams/partner-activity";
import { prisma } from "@dub/prisma";
// syncs the total links stats for a partner in a program
diff --git a/apps/web/lib/api/partners/sync-total-commissions.ts b/apps/web/lib/api/partners/sync-total-commissions.ts
index c842052a3b..d8bc1f20d0 100644
--- a/apps/web/lib/api/partners/sync-total-commissions.ts
+++ b/apps/web/lib/api/partners/sync-total-commissions.ts
@@ -1,4 +1,4 @@
-import { publishPartnerActivityEvent } from "@/lib/upstash/redis-streams";
+import { publishPartnerActivityEvent } from "@/lib/upstash/redis-streams/partner-activity";
import { prisma } from "@dub/prisma";
async function aggregateAndUpdateTotalCommissions({
diff --git a/apps/web/lib/middleware/utils/crawl-bitly.ts b/apps/web/lib/middleware/utils/crawl-bitly.ts
index d43ca2f624..cd90d6879c 100644
--- a/apps/web/lib/middleware/utils/crawl-bitly.ts
+++ b/apps/web/lib/middleware/utils/crawl-bitly.ts
@@ -2,7 +2,7 @@ import { createId } from "@/lib/api/create-id";
import { linkCache } from "@/lib/api/links/cache";
import { encodeKeyIfCaseSensitive } from "@/lib/api/links/case-sensitivity";
import { recordLink } from "@/lib/tinybird";
-import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams";
+import { publishWorkspaceLinksUsageEvent } from "@/lib/upstash/redis-streams/workspace-links-usage";
import { prisma } from "@dub/prisma";
import {
DUB_HEADERS,
diff --git a/apps/web/lib/tinybird/record-click.ts b/apps/web/lib/tinybird/record-click.ts
index 5fe4099f6d..c1612cc243 100644
--- a/apps/web/lib/tinybird/record-click.ts
+++ b/apps/web/lib/tinybird/record-click.ts
@@ -16,10 +16,8 @@ import { getIdentityHash } from "../middleware/utils/get-identity-hash";
import { conn } from "../planetscale";
import { WorkspaceProps } from "../types";
import { redis } from "../upstash";
-import {
- publishPartnerActivityEvent,
- publishWorkspaceClicksUsageEvent,
-} from "../upstash/redis-streams";
+import { publishPartnerActivityEvent } from "../upstash/redis-streams/partner-activity";
+import { publishWorkspaceClicksUsageEvent } from "../upstash/redis-streams/workspace-clicks-usage";
import { webhookCache } from "../webhook/cache";
import { sendWebhooks } from "../webhook/qstash";
import { transformClickEventData } from "../webhook/transform";
diff --git a/apps/web/lib/upstash/ratelimit-policy.ts b/apps/web/lib/upstash/ratelimit-policies.ts
similarity index 83%
rename from apps/web/lib/upstash/ratelimit-policy.ts
rename to apps/web/lib/upstash/ratelimit-policies.ts
index d3a1382796..f5922189dc 100644
--- a/apps/web/lib/upstash/ratelimit-policy.ts
+++ b/apps/web/lib/upstash/ratelimit-policies.ts
@@ -1,4 +1,4 @@
-export const RATE_LIMITS = {
+export const RATELIMIT_POLICIES = {
programImageUpload: {
attempts: 10,
window: "24 h",
diff --git a/apps/web/lib/upstash/redis-streams.ts b/apps/web/lib/upstash/redis-streams/client.ts
similarity index 60%
rename from apps/web/lib/upstash/redis-streams.ts
rename to apps/web/lib/upstash/redis-streams/client.ts
index 026b427b37..cbf780e3d0 100644
--- a/apps/web/lib/upstash/redis-streams.ts
+++ b/apps/web/lib/upstash/redis-streams/client.ts
@@ -1,4 +1,4 @@
-import { redis } from "./redis";
+import { redis } from "../redis";
export type RedisStreamEntry<T> = {
id: string;
@@ -137,88 +137,3 @@ export class RedisStream {
}
}
}
-
-/* Workspace Clicks Usage Stream */
-const WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY = "workspace:usage:updates";
-export const workspaceClicksUsageStream = new RedisStream(
- WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY,
-);
-export interface WorkspaceClicksUsageEvent {
- linkId: string;
- workspaceId: string;
- timestamp: string;
-}
-// Publishes a click event to any relevant streams in a single transaction
-export const publishWorkspaceClicksUsageEvent = async (
- event: WorkspaceClicksUsageEvent,
-) => {
- const { linkId, workspaceId, timestamp } = event;
- try {
- return await redis.xadd(WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY, "*", {
- linkId,
- workspaceId,
- timestamp,
- });
- } catch (error) {
- console.error("Failed to publish workspace clicks usage event:", error);
- throw error;
- }
-};
-
-/* Workspace Links Usage Stream */
-const WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY = "workspace:linksUsage:updates";
-export const workspaceLinksUsageStream = new RedisStream(
- WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY,
-);
-
-export interface WorkspaceLinksUsageEvent {
- workspaceId: string;
- linksCount: number;
- timestamp: string;
-}
-
-export const publishWorkspaceLinksUsageEvent = async (
- event: WorkspaceLinksUsageEvent,
-) => {
- const { workspaceId, linksCount, timestamp } = event;
-
- try {
- return await redis.xadd(WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY, "*", {
- workspaceId,
- linksCount,
- timestamp,
- });
- } catch (error) {
- console.error("Failed to publish workspace links usage event:", error);
- throw error;
- }
-};
-
-/* Partner Stats Stream */
-const PARTNER_ACTIVITY_STREAM_KEY = "partner:activity:updates";
-export const partnerActivityStream = new RedisStream(
- PARTNER_ACTIVITY_STREAM_KEY,
-);
-export interface PartnerActivityEvent {
- programId: string;
- partnerId: string;
- timestamp: string;
- eventType: "click" | "lead" | "sale" | "commission";
-}
-// Publishes a partner activity event to the stream
-export const publishPartnerActivityEvent = async (
- event: PartnerActivityEvent,
-) => {
- const { programId, partnerId, timestamp, eventType } = event;
- try {
- return await redis.xadd(PARTNER_ACTIVITY_STREAM_KEY, "*", {
- programId,
- partnerId,
- timestamp,
- eventType,
- });
- } catch (error) {
- console.error("Failed to publish partner activity event to stream:", error);
- throw error;
- }
-};
diff --git a/apps/web/lib/upstash/redis-streams/partner-activity.ts b/apps/web/lib/upstash/redis-streams/partner-activity.ts
new file mode 100644
index 0000000000..3fb5761e06
--- /dev/null
+++ b/apps/web/lib/upstash/redis-streams/partner-activity.ts
@@ -0,0 +1,34 @@
+import { redis } from "../redis";
+import { RedisStream } from "./client";
+
+/* Partner Stats Stream */
+const PARTNER_ACTIVITY_STREAM_KEY = "partner:activity:updates";
+
+export const partnerActivityStream = new RedisStream(
+ PARTNER_ACTIVITY_STREAM_KEY,
+);
+
+export interface PartnerActivityEvent {
+ programId: string;
+ partnerId: string;
+ timestamp: string;
+ eventType: "click" | "lead" | "sale" | "commission";
+}
+
+// Publishes a partner activity event to the stream
+export const publishPartnerActivityEvent = async (
+ event: PartnerActivityEvent,
+) => {
+ const { programId, partnerId, timestamp, eventType } = event;
+ try {
+ return await redis.xadd(PARTNER_ACTIVITY_STREAM_KEY, "*", {
+ programId,
+ partnerId,
+ timestamp,
+ eventType,
+ });
+ } catch (error) {
+ console.error("Failed to publish partner activity event to stream:", error);
+ throw error;
+ }
+};
diff --git a/apps/web/lib/upstash/redis-streams/workspace-clicks-usage.ts b/apps/web/lib/upstash/redis-streams/workspace-clicks-usage.ts
new file mode 100644
index 0000000000..9f5e930fe6
--- /dev/null
+++ b/apps/web/lib/upstash/redis-streams/workspace-clicks-usage.ts
@@ -0,0 +1,31 @@
+import { redis } from "../redis";
+import { RedisStream } from "./client";
+
+/* Workspace Clicks Usage Stream */
+const WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY = "workspace:usage:updates";
+
+export const workspaceClicksUsageStream = new RedisStream(
+ WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY,
+);
+
+export interface WorkspaceClicksUsageEvent {
+ linkId: string;
+ workspaceId: string;
+ timestamp: string;
+}
+// Publishes a click event to any relevant streams in a single transaction
+export const publishWorkspaceClicksUsageEvent = async (
+ event: WorkspaceClicksUsageEvent,
+) => {
+ const { linkId, workspaceId, timestamp } = event;
+ try {
+ return await redis.xadd(WORKSPACE_CLICKS_USAGE_UPDATES_STREAM_KEY, "*", {
+ linkId,
+ workspaceId,
+ timestamp,
+ });
+ } catch (error) {
+ console.error("Failed to publish workspace clicks usage event:", error);
+ throw error;
+ }
+};
diff --git a/apps/web/lib/upstash/redis-streams/workspace-links-usage.ts b/apps/web/lib/upstash/redis-streams/workspace-links-usage.ts
new file mode 100644
index 0000000000..d7c029d5f8
--- /dev/null
+++ b/apps/web/lib/upstash/redis-streams/workspace-links-usage.ts
@@ -0,0 +1,40 @@
+import { conn } from "@/lib/planetscale";
+import { redis } from "../redis";
+import { RedisStream } from "./client";
+
+/* Workspace Links Usage Stream */
+const WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY = "workspace:linksUsage:updates";
+
+export const workspaceLinksUsageStream = new RedisStream(
+ WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY,
+);
+
+export interface WorkspaceLinksUsageEvent {
+ workspaceId: string;
+ linksCount: number;
+ timestamp: string;
+}
+
+export const publishWorkspaceLinksUsageEvent = async (
+ event: WorkspaceLinksUsageEvent,
+) => {
+ const { workspaceId, linksCount, timestamp } = event;
+
+ try {
+ return await redis.xadd(WORKSPACE_LINKS_USAGE_UPDATES_STREAM_KEY, "*", {
+ workspaceId,
+ linksCount,
+ timestamp,
+ });
+ } catch (error) {
+ console.error(
+ "Failed to publish workspace links usage event (falling back to database update):",
+ error,
+ );
+ // fallback on writing directly to the database
+ return await conn.execute(
+ "UPDATE Project SET linksUsage = linksUsage + ?, totalLinks = totalLinks + ? WHERE id = ?",
+ [linksCount, linksCount, workspaceId],
+ );
+ }
+};