Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

improve process crons to be more robust & resilient

Steven Tey91ff0dbdd8a0
diff --git a/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts b/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts
index 62a87c6ed1..d119666526 100644
--- a/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts
+++ b/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts
@@ -15,14 +15,14 @@ import { logAndRespond } from "../../utils";
export const dynamic = "force-dynamic";
// How many stream entries to drain per run
-const BATCH_SIZE = 500;
+const BATCH_SIZE = 6000;
// How many webhook deliveries to enqueue to QStash in parallel
const SEND_CONCURRENCY = 100;
// Lock for the cron job
const LOCK_KEY = "lock:send-link-clicked-webhooks";
-const LOCK_TTL_SECONDS = 300;
+const LOCK_TTL_SECONDS = 120;
// Drains the workspace:click:events stream and delivers link.clicked webhooks.
// Runs every minute (see vercel.json).
@@ -280,10 +280,12 @@ const processStreamBatch = (): Promise<{
let failed = 0;
for (const batch of chunk(sendTasks, SEND_CONCURRENCY)) {
- const results = await Promise.all(batch.map(({ task }) => task()));
+ const results = await Promise.allSettled(
+ batch.map(({ task }) => task()),
+ );
batch.forEach(({ entryId }, i) => {
- if (results[i]) {
+ if (results[i].status === "fulfilled") {
processedEntryIds.push(entryId);
sent++;
} else {
diff --git a/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts b/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts
index 29aab520e1..473229d49a 100644
--- a/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts
+++ b/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts
@@ -44,6 +44,9 @@ export const syncClickWebhookWorkspaceSet = async () => {
export const cleanupRedundantLinkWebhookEntries = async () => {
const nonLinkScopeWebhooks = await prisma.webhook.findMany({
where: {
+ triggers: {
+ array_contains: [LINK_CLICK_WEBHOOK_TRIGGER],
+ },
linkScope: {
not: "links",
},
@@ -55,13 +58,20 @@ export const cleanupRedundantLinkWebhookEntries = async () => {
let deletedCount = 0;
while (true) {
- const deleted = await prisma.linkWebhook.deleteMany({
+ const linksToDelete = await prisma.linkWebhook.findMany({
where: {
webhookId: {
in: nonLinkScopeWebhooks.map((webhook) => webhook.id),
},
},
- limit: 250,
+ take: 250,
+ });
+ const deleted = await prisma.linkWebhook.deleteMany({
+ where: {
+ id: {
+ in: linksToDelete.map((link) => link.id),
+ },
+ },
});
deletedCount += deleted.count;
if (deleted.count === 0) {