Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Move click event validation to stream publish and tune cron throughput

Kiran K52acd1e9e2ba
diff --git a/apps/web/app/(ee)/api/cron/streams/send-link-click-webhooks/route.ts b/apps/web/app/(ee)/api/cron/streams/send-link-click-webhooks/route.ts
index 82dbec848f..4748aff0b5 100644
--- a/apps/web/app/(ee)/api/cron/streams/send-link-click-webhooks/route.ts
+++ b/apps/web/app/(ee)/api/cron/streams/send-link-click-webhooks/route.ts
@@ -13,10 +13,10 @@ import { NextResponse } from "next/server";
export const dynamic = "force-dynamic";
// How many stream entries to drain per run
-const BATCH_SIZE = 1000;
+const BATCH_SIZE = 500;
// How many webhook deliveries to enqueue to QStash in parallel
-const SEND_CONCURRENCY = 50;
+const SEND_CONCURRENCY = 100;
// Drains the workspace:click:events stream and delivers link.clicked webhooks.
// Runs every minute (see vercel.json).
diff --git a/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/route.ts b/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/route.ts
index 7b21b1c7f1..051a75ce86 100644
--- a/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/route.ts
+++ b/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/route.ts
@@ -1,5 +1,5 @@
import { withCron } from "@/lib/cron/with-cron";
-import { rebuildClickWebhookWorkspaces } from "@/lib/webhook/click-webhook-workspaces";
+import { syncClickWebhookWorkspaceSet } from "@/lib/webhook/click-webhook-workspaces";
import { logAndRespond } from "../../utils";
export const dynamic = "force-dynamic";
@@ -8,7 +8,7 @@ export const dynamic = "force-dynamic";
// Rebuild the Redis set of workspaces with active link.clicked webhooks.
// Runs every minute (* * * * *)
export const GET = withCron(async () => {
- const count = await rebuildClickWebhookWorkspaces();
+ const count = await syncClickWebhookWorkspaceSet();
return logAndRespond(
count === 0
diff --git a/apps/web/lib/tinybird/record-click.ts b/apps/web/lib/tinybird/record-click.ts
index e6618167d3..a5deaca012 100644
--- a/apps/web/lib/tinybird/record-click.ts
+++ b/apps/web/lib/tinybird/record-click.ts
@@ -17,7 +17,6 @@ import { redis } from "../upstash";
import { publishPartnerActivityEvent } from "../upstash/redis-streams/partner-activity";
import { publishWorkspaceClickEvent } from "../upstash/redis-streams/workspace-click-events";
import { publishWorkspaceClicksUsageEvent } from "../upstash/redis-streams/workspace-clicks-usage";
-import { clickEventSchemaTB } from "../zod/schemas/clicks";
/**
* Recording clicks with geo, ua, referer and timestamp data
@@ -224,7 +223,7 @@ export async function recordClick({
}),
// Publish the click event
- publishWorkspaceClickEvent(clickEventSchemaTB.parse(clickData)),
+ publishWorkspaceClickEvent(clickData),
]);
// Find the rejected promises and log them
diff --git a/apps/web/lib/upstash/redis-streams/workspace-click-events.ts b/apps/web/lib/upstash/redis-streams/workspace-click-events.ts
index ffbfd1467b..e139f38906 100644
--- a/apps/web/lib/upstash/redis-streams/workspace-click-events.ts
+++ b/apps/web/lib/upstash/redis-streams/workspace-click-events.ts
@@ -1,7 +1,6 @@
import { logger } from "@/lib/axiom/server";
import { clickWebhookWorkspaces } from "@/lib/webhook/click-webhook-workspaces";
import { clickEventSchemaTB } from "@/lib/zod/schemas/clicks";
-import * as z from "zod/v4";
import { redis } from "../redis";
import { RedisStream } from "./client";
@@ -9,19 +8,23 @@ const STREAM_KEY = "workspace:click:events";
export const workspaceClickEventStream = new RedisStream(STREAM_KEY);
-export const publishWorkspaceClickEvent = async (
- event: z.infer<typeof clickEventSchemaTB>,
-) => {
- const { workspace_id: workspaceId } = event;
+export const publishWorkspaceClickEvent = async (event) => {
+ try {
+ const parsedEvent = clickEventSchemaTB.parse({
+ ...event,
+ qr: Number(event.qr),
+ bot: Number(event.bot),
+ });
- const hasWebhooks = await clickWebhookWorkspaces.has(workspaceId);
+ const hasWebhooks = await clickWebhookWorkspaces.has(
+ parsedEvent.workspace_id,
+ );
- if (!hasWebhooks) {
- return;
- }
+ if (!hasWebhooks) {
+ return;
+ }
- try {
- return await redis.xadd(STREAM_KEY, "*", event);
+ return await redis.xadd(STREAM_KEY, "*", parsedEvent);
} catch (error) {
logger.error("stream.publish_failed", {
service: "upstash",
diff --git a/apps/web/lib/webhook/click-webhook-workspaces.ts b/apps/web/lib/webhook/click-webhook-workspaces.ts
index 2a4dc7f65c..c471ef8dde 100644
--- a/apps/web/lib/webhook/click-webhook-workspaces.ts
+++ b/apps/web/lib/webhook/click-webhook-workspaces.ts
@@ -21,7 +21,7 @@ class ClickWebhookWorkspaces {
}
// Rebuild the Redis set of workspaces with active link.clicked webhooks.
-export const rebuildClickWebhookWorkspaces = async () => {
+export const syncClickWebhookWorkspaceSet = async () => {
const webhooks = await prisma.webhook.findMany({
where: {
disabledAt: null,
diff --git a/apps/web/scripts/migrations/backfill-click-webhook-workspaces.ts b/apps/web/scripts/migrations/backfill-click-webhook-workspaces.ts
index d2ff1045f6..38d917f27c 100644
--- a/apps/web/scripts/migrations/backfill-click-webhook-workspaces.ts
+++ b/apps/web/scripts/migrations/backfill-click-webhook-workspaces.ts
@@ -1,8 +1,8 @@
-import { rebuildClickWebhookWorkspaces } from "@/lib/webhook/click-webhook-workspaces";
+import { syncClickWebhookWorkspaceSet } from "@/lib/webhook/click-webhook-workspaces";
import "dotenv-flow/config";
async function main() {
- const count = await rebuildClickWebhookWorkspaces();
+ const count = await syncClickWebhookWorkspaceSet();
console.log(`Rebuilt webhookClickWorkspaces with ${count} workspace(s).`);
}