The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Queue due campaigns from a cron instead of per-campaign QStash schedules
Kiran K7dd253f3432f
diff --git a/apps/web/app/(ee)/api/campaigns/[campaignId]/route.ts b/apps/web/app/(ee)/api/campaigns/[campaignId]/route.tsindex a5ded7839a..eaa8feb6f0 100644--- a/apps/web/app/(ee)/api/campaigns/[campaignId]/route.ts+++ b/apps/web/app/(ee)/api/campaigns/[campaignId]/route.ts@@ -1,8 +1,4 @@import { getCampaignOrThrow } from "@/lib/api/campaigns/get-campaign-or-throw";-import {- deleteCampaignSchedule,- scheduleCampaign,-} from "@/lib/api/campaigns/schedule-campaigns";import {campaignEligibilityIncludes,transformCampaign,@@ -21,7 +17,6 @@ import {} from "@/lib/zod/schemas/campaigns";import { arrayEqual, pluck } from "@dub/utils";import { PartnerGroup } from "@prisma/client";-import { waitUntil } from "@vercel/functions";import { NextResponse } from "next/server";// GET /api/campaigns/[campaignId] - get an email campaign@@ -191,13 +186,6 @@ export const PATCH = withWorkspace(});});- waitUntil(- scheduleCampaign({- campaign,- updatedCampaign,- }),- );-return NextResponse.json(CampaignSchema.parse(transformCampaign(updatedCampaign)),);@@ -217,15 +205,6 @@ export const DELETE = withWorkspace(const campaign = await getCampaignOrThrow({programId,campaignId,- include: {- workflow: {- select: {- id: true,- actions: true,- triggerConditions: true,- },- },- },});await prisma.$transaction(async (tx) => {@@ -244,8 +223,6 @@ export const DELETE = withWorkspace(}});- waitUntil(deleteCampaignSchedule(campaign));-return NextResponse.json({ id: campaignId });},{diff --git a/apps/web/app/(ee)/api/cron/campaigns/broadcast/route.ts b/apps/web/app/(ee)/api/cron/campaigns/broadcast/route.tsindex ced6b7dd48..d4dcfa1949 100644--- a/apps/web/app/(ee)/api/cron/campaigns/broadcast/route.ts+++ b/apps/web/app/(ee)/api/cron/campaigns/broadcast/route.ts@@ -15,7 +15,6 @@ import CampaignEmail from "@dub/email/templates/campaign-email";import { APP_DOMAIN_WITH_NGROK, chunk, log, pluck } from "@dub/utils";import { NotificationEmailType } from "@prisma/client";import { differenceInMinutes } from "date-fns";-import { headers } from "next/headers";import * as z from "zod/v4";import { logAndRespond } from "../../utils";@@ -102,20 +101,6 @@ export async function POST(req: Request) {}}- // This is a safety check to ensure the campaign broadcast is not "initiated" multiple times- const headersList = await headers();- const upstashMessageId = headersList.get("Upstash-Message-Id");-- if (- !startingAfter && // First run- campaign.qstashMessageId &&- upstashMessageId !== campaign.qstashMessageId- ) {- return logAndRespond(- `Campaign ${campaignId} broadcast was skipped because it is not the current message being processed.`,- );- }-const program = campaign.program;// TODO: We should make the from address required. There are existing campaign without from address@@ -126,19 +111,23 @@ export async function POST(req: Request) {});}- // Mark the campaign as sending (if it's in scheduled status)- if (campaign.status === "scheduled") {- try {- await prisma.campaign.update({- where: {- id: campaignId,- },- data: {- status: "sending",- },- });- } catch (error) {- //+ // Claim the first run so leftover delayed messages / scanner retries+ // cannot start a second broadcast.+ if (!startingAfter) {+ const claimed = await prisma.campaign.updateMany({+ where: {+ id: campaignId,+ status: "scheduled",+ },+ data: {+ status: "sending",+ },+ });++ if (claimed.count === 0) {+ return logAndRespond(+ `Campaign ${campaignId} broadcast already initiated. Skipping...`,+ );}}diff --git a/apps/web/app/(ee)/api/cron/campaigns/queue-scheduled/route.ts b/apps/web/app/(ee)/api/cron/campaigns/queue-scheduled/route.tsnew file mode 100644index 0000000000..9008e80021--- /dev/null+++ b/apps/web/app/(ee)/api/cron/campaigns/queue-scheduled/route.ts@@ -0,0 +1,138 @@+import { isScheduledWorkflow } from "@/lib/api/workflows/utils";+import { CRON_BATCH_SIZE, qstash } from "@/lib/cron";+import { withCron } from "@/lib/cron/with-cron";+import { prisma } from "@/lib/prisma";+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";+import { CampaignStatus, CampaignType } from "@prisma/client";+import { logAndRespond } from "../../utils";++export const dynamic = "force-dynamic";+export const maxDuration = 600;++// GET /api/cron/campaigns/queue-scheduled+// Fans out due marketing broadcasts and (on the 12h tick) scheduled transactional workflows.+export const GET = withCron(async () => {+ const now = new Date();++ await Promise.all([+ queueTransactionalCampaigns(now),+ queueMarketingCampaigns(now),+ ]);++ return logAndRespond("Finished the campaigns queueing process.");+});++async function queueTransactionalCampaigns(now: Date) {+ // Matches the 12h enrollment window in executeSendCampaignWorkflow.+ if (now.getUTCMinutes() !== 0 || now.getUTCHours() % 12 !== 0) {+ console.log("[Transactional] Not the 12h tick, skipping campaigns.");+ return;+ }++ let queued = 0;+ let page = 0;++ while (true) {+ const campaigns = await prisma.campaign.findMany({+ where: {+ type: CampaignType.transactional,+ status: CampaignStatus.active,+ workflow: {+ disabledAt: null,+ },+ },+ select: {+ workflow: {+ select: {+ id: true,+ triggerConditions: true,+ actions: true,+ },+ },+ },+ take: CRON_BATCH_SIZE,+ skip: page * CRON_BATCH_SIZE,+ orderBy: {+ id: "asc",+ },+ });++ if (campaigns.length === 0) {+ console.log("[Transactional] No more campaigns to queue.");+ break;+ }++ const scheduledWorkflows = campaigns.flatMap((campaign) =>+ campaign.workflow && isScheduledWorkflow(campaign.workflow)+ ? [campaign.workflow]+ : [],+ );++ if (scheduledWorkflows.length > 0) {+ await qstash.batchJSON(+ scheduledWorkflows.map((workflow) => ({+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workflows/${workflow.id}`,+ deduplicationId: workflow.id,+ flowControl: {+ key: "execute-scheduled-workflow",+ parallelism: 10,+ },+ body: {},+ })),+ );++ queued += scheduledWorkflows.length;+ }++ page++;+ }++ console.log(`[Transactional] Queued ${queued} campaigns.`);+}++async function queueMarketingCampaigns(now: Date) {+ let queued = 0;+ let page = 0;++ while (true) {+ const campaigns = await prisma.campaign.findMany({+ where: {+ type: CampaignType.marketing,+ status: CampaignStatus.scheduled,+ OR: [{ scheduledAt: null }, { scheduledAt: { lte: now } }],+ },+ select: {+ id: true,+ },+ take: CRON_BATCH_SIZE,+ skip: page * CRON_BATCH_SIZE,+ orderBy: {+ id: "asc",+ },+ });++ if (campaigns.length === 0) {+ console.log("[Marketing] No more campaigns to queue.");+ break;+ }++ await qstash.batchJSON(+ campaigns.map((campaign) => ({+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/campaigns/broadcast`,+ deduplicationId: campaign.id,+ flowControl: {+ key: "broadcast-marketing-campaign",+ parallelism: 1,+ },+ body: {+ campaignId: campaign.id,+ },+ })),+ );++ queued += campaigns.length;+ page++;+ }++ console.log(`[Marketing] Queued ${queued} campaigns.`);+}diff --git a/apps/web/lib/api/campaigns/pause-campaigns-on-plan-downgrade.ts b/apps/web/lib/api/campaigns/pause-campaigns-on-plan-downgrade.tsindex e42c8cc74f..89cf02a91e 100644--- a/apps/web/lib/api/campaigns/pause-campaigns-on-plan-downgrade.ts+++ b/apps/web/lib/api/campaigns/pause-campaigns-on-plan-downgrade.ts@@ -1,5 +1,3 @@-import { scheduleTransactionalCampaign } from "@/lib/api/campaigns/schedule-campaigns";-import { qstash } from "@/lib/cron";import { prisma } from "@/lib/prisma";import { CampaignStatus, CampaignType } from "@prisma/client";@@ -8,7 +6,7 @@ export async function pauseOrCancelCampaignsForProgramOnPlanDowngrade({}: {programId: string;}): Promise<void> {- const marketingCampaigns = await prisma.campaign.findMany({+ await prisma.campaign.updateMany({where: {programId,type: CampaignType.marketing,@@ -16,46 +14,22 @@ export async function pauseOrCancelCampaignsForProgramOnPlanDowngrade({in: [CampaignStatus.scheduled, CampaignStatus.sending],},},+ data: {+ status: CampaignStatus.canceled,+ },});- for (const campaign of marketingCampaigns) {- let qstashDeleteSucceeded = !campaign.qstashMessageId;-- if (campaign.qstashMessageId) {- try {- await qstash.messages.cancel(campaign.qstashMessageId);- qstashDeleteSucceeded = true;- } catch (error) {- console.warn(- `Failed to delete QStash message ${campaign.qstashMessageId} for campaign ${campaign.id}:`,- error,- );- }- }-- await prisma.campaign.update({- where: { id: campaign.id },- data: {- status: CampaignStatus.canceled,- ...(qstashDeleteSucceeded ? { qstashMessageId: null } : {}),- },- });- }-const transactionalCampaigns = await prisma.campaign.findMany({where: {programId,type: CampaignType.transactional,status: CampaignStatus.active,},- include: {- workflow: true,- },});for (const campaign of transactionalCampaigns) {try {- const updatedCampaign = await prisma.$transaction(async (tx) => {+ await prisma.$transaction(async (tx) => {if (campaign.workflowId) {await tx.workflow.update({where: { id: campaign.workflowId },@@ -63,17 +37,11 @@ export async function pauseOrCancelCampaignsForProgramOnPlanDowngrade({});}- return tx.campaign.update({+ await tx.campaign.update({where: { id: campaign.id },data: { status: CampaignStatus.paused },- include: { workflow: true },});});-- await scheduleTransactionalCampaign({- campaign,- updatedCampaign,- });} catch (error) {console.warn(`Failed to pause transactional campaign ${campaign.id} on plan downgrade:`,diff --git a/apps/web/lib/api/campaigns/schedule-campaigns.ts b/apps/web/lib/api/campaigns/schedule-campaigns.tsdeleted file mode 100644index a8748beef0..0000000000--- a/apps/web/lib/api/campaigns/schedule-campaigns.ts+++ /dev/null@@ -1,189 +0,0 @@-import { qstash } from "@/lib/cron";-import { prisma } from "@/lib/prisma";-import { PARTNER_ENROLLED_WORKFLOW_CRON } from "@/lib/zod/schemas/workflows";-import { APP_DOMAIN_WITH_NGROK, log } from "@dub/utils";-import { Campaign, CampaignType, Workflow } from "@prisma/client";-import { isScheduledWorkflow } from "../workflows/utils";--type ScheduleCampaignProps = {- campaign: Campaign;- updatedCampaign: Campaign & {- workflow: Workflow | null;- };-};--export const scheduleCampaign = async ({- campaign,- updatedCampaign,-}: ScheduleCampaignProps) => {- if (campaign.type == CampaignType.marketing) {- return scheduleMarketingCampaign({- campaign,- updatedCampaign,- });- }-- if (campaign.type == CampaignType.transactional) {- return scheduleTransactionalCampaign({- campaign,- updatedCampaign,- });- }-};--export const deleteCampaignSchedule = async (- campaign: Pick<Campaign, "type" | "qstashMessageId"> & {- workflow: Pick<Workflow, "id" | "triggerConditions" | "actions"> | null;- },-) => {- if (campaign.type == CampaignType.marketing && campaign.qstashMessageId) {- try {- await qstash.messages.cancel(campaign.qstashMessageId);- } catch (error) {- console.warn(- `Failed to delete QStash message ${campaign.qstashMessageId}:`,- error,- );- }- return;- }-- if (- campaign.type == CampaignType.transactional &&- campaign.workflow &&- isScheduledWorkflow(campaign.workflow)- ) {- try {- await qstash.schedules.delete(campaign.workflow.id);- } catch (error) {- console.warn(- `Failed to delete QStash schedule ${campaign.workflow.id}:`,- error,- );- }- }-};--// Schedule a marketing campaign-const scheduleMarketingCampaign = async ({- campaign,- updatedCampaign,-}: ScheduleCampaignProps) => {- if (updatedCampaign.status === "draft") {- return;- }-- const scheduleChanged =- campaign.scheduledAt?.getTime() !== updatedCampaign.scheduledAt?.getTime();-- const statusChanged =- (campaign.status === "draft" && updatedCampaign.status === "scheduled") ||- (campaign.status === "scheduled" && updatedCampaign.status === "canceled");-- if (!statusChanged && !scheduleChanged) {- return;- }-- let qstashMessageId = updatedCampaign.qstashMessageId;-- // Delete the existing message- if (campaign.qstashMessageId) {- try {- await qstash.messages.cancel(campaign.qstashMessageId);- qstashMessageId = null;- } catch (error) {- console.warn(- `Failed to delete QStash message ${campaign.qstashMessageId}:`,- error,- );- }- }-- // Queue a new message- if (updatedCampaign.status === "scheduled") {- const notBefore = updatedCampaign.scheduledAt- ? Math.floor(updatedCampaign.scheduledAt.getTime() / 1000)- : null;-- try {- const response = await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/campaigns/broadcast`,- method: "POST",- ...(notBefore && { notBefore }),- body: {- campaignId: campaign.id,- },- });-- qstashMessageId = response.messageId;- } catch (error) {- console.warn(- `Failed to queue QStash message for campaign ${campaign.id}:`,- error,- );- }- }-- await prisma.campaign.update({- where: {- id: campaign.id,- },- data: {- qstashMessageId,- },- });-};--// Schedule a transactional campaign-export const scheduleTransactionalCampaign = async ({- campaign,- updatedCampaign,-}: ScheduleCampaignProps) => {- if (- !updatedCampaign.workflow ||- !isScheduledWorkflow(updatedCampaign.workflow)- ) {- console.log(`No workflow found for campaign ${campaign.id}. Skipping...`);- return;- }-- const shouldSchedule =- (campaign.status === "draft" || campaign.status === "paused") &&- updatedCampaign.status === "active";-- if (shouldSchedule) {- try {- return await qstash.schedules.create({- destination: `${APP_DOMAIN_WITH_NGROK}/api/cron/workflows/${updatedCampaign.workflow.id}`,- cron: PARTNER_ENROLLED_WORKFLOW_CRON,- scheduleId: updatedCampaign.workflow.id,- });- } catch (error) {- // should never happen, but just in case- const errorMessage = `Failed to create QStash schedule ${updatedCampaign.workflow.id}: ${error}`;- console.warn(errorMessage);- await log({- type: "errors",- message: errorMessage,- });- }- return;- }-- const shouldDeleteSchedule =- campaign.status === "active" && updatedCampaign.status === "paused";-- if (shouldDeleteSchedule) {- try {- return await qstash.schedules.delete(updatedCampaign.workflow.id);- } catch (error) {- // should never happen, but just in case- const errorMessage = `Failed to delete QStash schedule ${updatedCampaign.workflow.id}: ${error}`;- console.warn(errorMessage);- await log({- type: "errors",- message: errorMessage,- });- }- }-};diff --git a/apps/web/lib/zod/schemas/workflows.ts b/apps/web/lib/zod/schemas/workflows.tsindex c6daae0626..e41a238c3a 100644--- a/apps/web/lib/zod/schemas/workflows.ts+++ b/apps/web/lib/zod/schemas/workflows.ts@@ -5,9 +5,6 @@ import {} from "@/lib/api/workflows/operator-definitions";import * as z from "zod/v4";-// Cron for scheduled workflows that use partnerEnrolledDays conditions-export const PARTNER_ENROLLED_WORKFLOW_CRON = "0 */12 * * *"; // every 12 hours-export enum WORKFLOW_ACTION_TYPES {AwardBounty = "awardBounty",SendCampaign = "sendCampaign",diff --git a/apps/web/playwright/assert-local-database.ts b/apps/web/playwright/assert-local-database.tsindex 897f6279ba..7359c68faa 100644--- a/apps/web/playwright/assert-local-database.ts+++ b/apps/web/playwright/assert-local-database.ts@@ -16,7 +16,7 @@ function assertUrl(if (!value) {if (required) {throw new Error(- `${name} is not set. Playwright tests require a local database. ${LOCAL_URL_HINT}`,+ `${name} is not set. A local database is required. ${LOCAL_URL_HINT}`,);}return;@@ -27,13 +27,13 @@ function assertUrl(hostname = hostnameFromDatabaseUrl(value);} catch {throw new Error(- `${name} is not a valid URL. Playwright tests require a local database. ${LOCAL_URL_HINT}`,+ `${name} is not a valid URL. A local database is required. ${LOCAL_URL_HINT}`,);}if (!LOCAL_HOSTNAMES.has(hostname)) {throw new Error(- `Refusing to run Playwright tests: ${name} host "${hostname}" is not a local database. Only localhost / 127.0.0.1 / ::1 are allowed. ${LOCAL_URL_HINT}`,+ `Refusing to proceed: ${name} host "${hostname}" is not a local database. Only localhost / 127.0.0.1 / ::1 are allowed. ${LOCAL_URL_HINT}`,);}}diff --git a/apps/web/prisma/schema/campaign.prisma b/apps/web/prisma/schema/campaign.prismaindex 7f94a4e060..32192e4535 100644--- a/apps/web/prisma/schema/campaign.prisma+++ b/apps/web/prisma/schema/campaign.prisma@@ -18,28 +18,29 @@ enum CampaignStatus {}model Campaign {- id String @id+ id String @idprogramId String- workflowId String? @unique+ workflowId String? @uniqueuserId String- qstashMessageId String? @unique+ qstashMessageId String? @uniquetype CampaignType- status CampaignStatus @default(draft)+ status CampaignStatus @default(draft)name Stringsubject String- preview String? @db.Text+ preview String? @db.Textfrom String?- bodyJson Json @db.Json+ bodyJson Json @db.JsonscheduledAt DateTime?- createdAt DateTime @default(now())- updatedAt DateTime @updatedAt- program Program @relation(fields: [programId], references: [id])- workflow Workflow? @relation(fields: [workflowId], references: [id], onDelete: Cascade)+ createdAt DateTime @default(now())+ updatedAt DateTime @updatedAt+ program Program @relation(fields: [programId], references: [id])+ workflow Workflow? @relation(fields: [workflowId], references: [id], onDelete: Cascade)groups CampaignGroup[]partnerTags CampaignPartnerTag[]emails NotificationEmail[]@@index(programId)+ @@index([type, status, scheduledAt])}model CampaignGroup {diff --git a/apps/web/scripts/dev/data.json b/apps/web/scripts/dev/data.jsonindex d245191c34..ecf61cdc75 100644--- a/apps/web/scripts/dev/data.json+++ b/apps/web/scripts/dev/data.json@@ -60,6 +60,13 @@"verified": true}],+ "emailDomains": [+ {+ "id": "dom_1KETZ919F83ZJH6A80EMAILDOM",+ "slug": "getacme.link",+ "status": "verified"+ }+ ],"folders": [{"id": "fold_1K2J9DRWPPJ2F1RX53N92TSGB",diff --git a/apps/web/scripts/dev/seed.ts b/apps/web/scripts/dev/seed.tsindex 4683c622ba..9fafe6d266 100644--- a/apps/web/scripts/dev/seed.ts+++ b/apps/web/scripts/dev/seed.ts@@ -3,6 +3,7 @@ import { hashPassword } from "@/lib/auth/password";import { prisma } from "@/lib/prisma";import {Domain,+ EmailDomain,Folder,Integration,Partner,@@ -18,7 +19,7 @@ import {import "dotenv-flow/config";import fs from "fs";import path from "path";-import readline from "readline";+import { assertLocalDatabaseEnv } from "../../playwright/assert-local-database";type Workspace = Pick<Project,@@ -48,6 +49,8 @@ type Workspace = Pick<type DomainSeed = Pick<Domain, "id" | "slug" | "verified">;+type EmailDomainSeed = Pick<EmailDomain, "id" | "slug" | "status">;+type FolderSeed = Pick<Folder, "id" | "name" | "description" | "accessLevel">;type RewardSeed = Pick<@@ -118,6 +121,7 @@ type SeedData = {workspace: Workspace;users: WorkspaceUser[];domains: DomainSeed[];+ emailDomains: EmailDomainSeed[];folders: FolderSeed[];rewards: RewardSeed[];groups: GroupSeed[];@@ -225,6 +229,33 @@ const createDomains = async (data: SeedData) => {console.log(`Created ${count} domains`);};+// Create email domains+const createEmailDomains = async (data: SeedData) => {+ const { emailDomains, workspace, program } = data;++ if (!emailDomains || emailDomains.length === 0) {+ console.log("No email domains to insert");+ return;+ }++ if (!program) {+ console.log("Program is required to create email domains");+ return;+ }++ const { count } = await prisma.emailDomain.createMany({+ data: emailDomains.map((emailDomain) => ({+ id: emailDomain.id,+ slug: emailDomain.slug,+ status: emailDomain.status,+ workspaceId: workspace.id,+ programId: program.id,+ })),+ });++ console.log(`Created ${count} email domains`);+};+// Create foldersconst createFolders = async (data: SeedData) => {const { folders, workspace } = data;@@ -566,44 +597,14 @@ const truncate = async () => {console.log("Database truncated successfully");};-// Ask for confirmation - requires typing "YES DELETE DATA"-const askConfirmation = (question: string): Promise<boolean> => {- const rl = readline.createInterface({- input: process.stdin,- output: process.stdout,- });-- return new Promise((resolve) => {- rl.question(- `${question}\nType "YES DELETE DATA" to confirm: `,- (answer) => {- rl.close();- resolve(answer === "YES DELETE DATA");- },- );- });-};-async function main() {+ assertLocalDatabaseEnv();+// Check for --truncate flag// process.argv[0] = node, process.argv[1] = script path, process.argv[2+] = argumentsconst shouldTruncate = process.argv.slice(2).includes("--truncate");if (shouldTruncate) {- console.log(- "\n⚠️ WARNING: This will delete ALL data from the database.\n",- );- console.log("⚠️ Make sure you are NOT on production database!\n");- const confirmed = await askConfirmation(- "Are you sure you want to delete ALL data from the database?",- );-- if (!confirmed) {- console.log("\nTruncate canceled. Exiting...");- process.exit(0);- }-- console.log("\n");await truncate();console.log("\n");}@@ -615,6 +616,7 @@ async function main() {await createDomains(data);await createFolders(data);await createProgram(data);+ await createEmailDomains(data);await createRewards(data);await createGroups(data);await createPartners(data);diff --git a/apps/web/vercel.json b/apps/web/vercel.jsonindex 35588c8ed5..e59dfb5eda 100644--- a/apps/web/vercel.json+++ b/apps/web/vercel.json@@ -91,6 +91,10 @@{"path": "/api/cron/queue/retry","schedule": "* * * * *"+ },+ {+ "path": "/api/cron/campaigns/queue-scheduled",+ "schedule": "* * * * *"}],"functions": {