The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Process Shopify orders via checkout cache rendezvous and background job
Kiran K47fa290f8554
diff --git a/apps/web/app/(ee)/api/shopify/integration/webhook/orders-paid.ts b/apps/web/app/(ee)/api/shopify/integration/webhook/orders-paid.tsindex 7d11b20dea..5aa28359bb 100644--- a/apps/web/app/(ee)/api/shopify/integration/webhook/orders-paid.ts+++ b/apps/web/app/(ee)/api/shopify/integration/webhook/orders-paid.ts@@ -1,28 +1,27 @@-import { qstash } from "@/lib/cron";-import { createShopifySale } from "@/lib/integrations/shopify/create-sale";import {- attributeViaDiscountCode,- processOrder,-} from "@/lib/integrations/shopify/process-order";-import { orderSchema } from "@/lib/integrations/shopify/schema";+ tryDispatchShopifyOrderJob,+ writeShopifyCheckoutFields,+} from "@/lib/integrations/shopify/checkout-cache";+import { shopifyOrderSchema } from "@/lib/integrations/shopify/schema";+import { processShopifyOrderJob } from "@/lib/jobs/handlers/process-shopify-order-job";import { prisma } from "@/lib/prisma";import { WorkspaceProps } from "@/lib/types";-import { redis } from "@/lib/upstash";-import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";export async function ordersPaid({event,workspace,}: {event: any;- workspace: Pick<WorkspaceProps, "id" | "defaultProgramId" | "webhookEnabled">;+ workspace: Pick<WorkspaceProps, "id" | "defaultProgramId">;}) {+ const order = shopifyOrderSchema.parse(event);const {customer: orderCustomer,checkout_token: checkoutToken,discount_codes: discountCodes,- } = orderSchema.parse(event);+ } = order;+ // If the order has a customer, try to find the customer in our databaseif (orderCustomer) {const { id: externalId } = orderCustomer;@@ -33,17 +32,24 @@ export async function ordersPaid({externalId: externalId.toString(),},},+ select: {+ id: true,+ },});- // customer is found, process the order right awayif (customer) {- await processOrder({- event,- workspaceId: workspace.id,- customerId: customer.id,- });+ await processShopifyOrderJob.dispatch(+ {+ order,+ clickId: null,+ workspaceId: workspace.id,+ },+ {+ deduplicationId: `shopify-order-${checkoutToken}`,+ },+ );- return "[Shopify] Order event processed successfully.";+ return `[Shopify] Existing customer ${customer.id} found. Order queued for processing.`;}}@@ -65,63 +71,39 @@ export async function ordersPaid({});if (programDiscountCodes.length > 0) {- const { customer, leadEvent: leadData } = await attributeViaDiscountCode({- event,- workspace,- link: programDiscountCodes[0].link,- });-- await createShopifySale({- leadData,- event,- workspaceId: workspace.id,- customerId: customer.id,- });+ await processShopifyOrderJob.dispatch(+ {+ order,+ clickId: null,+ workspaceId: workspace.id,+ },+ {+ deduplicationId: `shopify-order-${checkoutToken}`,+ },+ );- return "[Shopify] Order event processed successfully with discount codes.";+ return `[Shopify] Partner discount code ${programDiscountCodes[0].code} found. Order queued for processing.`;}}- // Check the cache to see the pixel event for this checkout token exist before publishing the event to the queue- const clickId = await redis.hget<string>(- `shopify:checkout:${checkoutToken}`,- "clickId",- );-- // clickId is empty, order is not from a Dub link- if (clickId === "") {- await redis.del(`shopify:checkout:${checkoutToken}`);-- return "[Shopify] Order is not from a Dub link. Skipping...";- }-- // clickId is found, process the order for the new customer- else if (clickId) {- await processOrder({- event,+ // At this stage, we know the order has no customer or partner discount code+ // so we need to wait for the pixel event to arrive as this could be a new customer coming via a link+ const checkout = await writeShopifyCheckoutFields({+ checkoutToken,+ fields: {+ order,workspaceId: workspace.id,- clickId,- });+ },+ });- return "[Shopify] Order event processed successfully.";- }-- // clickId is not found, we need to wait for the pixel event to come in so that we can decide if the order is from a Dub link or not- else {- await redis.hset(`shopify:checkout:${checkoutToken}`, {- order: event,- });+ const dispatched = await tryDispatchShopifyOrderJob({+ checkoutToken,+ checkout,+ });- await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/shopify/order-paid`,- body: {- checkoutToken,- workspaceId: workspace.id,- },- retries: 5,- delay: 3,- });-- return "[Shopify] clickId not found, waiting for pixel event to arrive...";+ if (dispatched) {+ return `[Shopify] Click ID ${checkout.clickId} found. Order queued for processing.`;}++ return "[Shopify] Waiting for pixel event to arrive...";}diff --git a/apps/web/app/(ee)/api/shopify/pixel/route.ts b/apps/web/app/(ee)/api/shopify/pixel/route.tsindex 7723bd31fa..b69053866a 100644--- a/apps/web/app/(ee)/api/shopify/pixel/route.ts+++ b/apps/web/app/(ee)/api/shopify/pixel/route.ts@@ -1,14 +1,16 @@import { COMMON_CORS_HEADERS } from "@/lib/api/cors";import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors";import { parseRequestBody } from "@/lib/api/utils";+import {+ tryDispatchShopifyOrderJob,+ writeShopifyCheckoutFields,+} from "@/lib/integrations/shopify/checkout-cache";import { getClickEvent } from "@/lib/tinybird";-import { ratelimit, redis } from "@/lib/upstash";+import { ratelimit } from "@/lib/upstash";import { LOCALHOST_IP } from "@dub/utils";import { ipAddress, waitUntil } from "@vercel/functions";import { NextResponse } from "next/server";-export const runtime = "edge";-// POST /api/shopify/pixel – Handle the Shopify Pixel eventsexport const POST = async (req: Request) => {try {@@ -42,9 +44,19 @@ export const POST = async (req: Request) => {}waitUntil(- redis.hset(`shopify:checkout:${checkoutToken}`, {- clickId: clickId || "",- }),+ (async () => {+ const checkout = await writeShopifyCheckoutFields({+ checkoutToken,+ fields: {+ clickId,+ },+ });++ await tryDispatchShopifyOrderJob({+ checkoutToken,+ checkout,+ });+ })(),);return NextResponse.json("OK", {diff --git a/apps/web/lib/integrations/shopify/checkout-cache.ts b/apps/web/lib/integrations/shopify/checkout-cache.tsnew file mode 100644index 0000000000..e8772fc8cb--- /dev/null+++ b/apps/web/lib/integrations/shopify/checkout-cache.ts@@ -0,0 +1,80 @@+import { processShopifyOrderJob } from "@/lib/jobs/handlers/process-shopify-order-job";+import { redis } from "@/lib/upstash";+import * as z from "zod/v4";+import { shopifyOrderSchema } from "./schema";++const SHOPIFY_CHECKOUT_CACHE_TTL_SECONDS = 60 * 60 * 24; // 24 hours+const SHOPIFY_CHECKOUT_CACHE_KEY_PREFIX = "shopify:checkout:";++const shopifyCheckoutCacheSchema = z.object({+ clickId: z.string().optional(),+ workspaceId: z.string().optional(),+ order: shopifyOrderSchema.optional(),+ dispatched: z.boolean().optional(),+});++type ShopifyCheckoutCache = z.infer<typeof shopifyCheckoutCacheSchema>;++function shopifyCheckoutCacheKey(checkoutToken: string) {+ return `${SHOPIFY_CHECKOUT_CACHE_KEY_PREFIX}${checkoutToken}`;+}++export async function writeShopifyCheckoutFields({+ checkoutToken,+ fields,+}: {+ checkoutToken: string;+ fields: Record<string, unknown>;+}) {+ const key = shopifyCheckoutCacheKey(checkoutToken);++ const pipeline = redis.pipeline();+ pipeline.hset(key, fields);+ pipeline.expire(key, SHOPIFY_CHECKOUT_CACHE_TTL_SECONDS);+ pipeline.hgetall<ShopifyCheckoutCache>(key);++ const results = await pipeline.exec();+ const cache = results[2] as ShopifyCheckoutCache | null;++ return cache ?? {};+}++export async function tryDispatchShopifyOrderJob({+ checkoutToken,+ checkout,+}: {+ checkoutToken: string;+ checkout: ShopifyCheckoutCache;+}) {+ if (!checkout.order || !checkout.workspaceId || !checkout.clickId) {+ return false;+ }++ if (checkout.dispatched) {+ return false;+ }++ // Claim the checkout+ const key = shopifyCheckoutCacheKey(checkoutToken);+ const claim = await redis.hsetnx(key, "dispatched", true);+ const claimed = Boolean(claim);++ if (!claimed) {+ return false;+ }++ await processShopifyOrderJob.dispatch(+ {+ workspaceId: checkout.workspaceId,+ clickId: checkout.clickId,+ order: checkout.order,+ },+ {+ deduplicationId: `shopify-order-${checkoutToken}`,+ },+ );++ await redis.del(key);++ return true;+}diff --git a/apps/web/lib/integrations/shopify/create-lead.ts b/apps/web/lib/integrations/shopify/create-lead.tsindex a361a40ae2..b544aee015 100644--- a/apps/web/lib/integrations/shopify/create-lead.ts+++ b/apps/web/lib/integrations/shopify/create-lead.ts@@ -11,7 +11,7 @@ import { transformLeadEventData } from "@/lib/webhook/transform";import { leadEventSchemaTB } from "@/lib/zod/schemas/leads";import { nanoid } from "@dub/utils";import { waitUntil } from "@vercel/functions";-import { orderSchema } from "./schema";+import { shopifyOrderSchema } from "./schema";export async function createShopifyLead({clickId,@@ -22,7 +22,7 @@ export async function createShopifyLead({workspaceId: string;event: any;}) {- const { customer: orderCustomer } = orderSchema.parse(event);+ const { customer: orderCustomer } = shopifyOrderSchema.parse(event);const customerId = createId({ prefix: "cus_" });/*diff --git a/apps/web/lib/integrations/shopify/create-sale.ts b/apps/web/lib/integrations/shopify/create-sale.tsindex d4b20f2b80..d9f901eff5 100644--- a/apps/web/lib/integrations/shopify/create-sale.ts+++ b/apps/web/lib/integrations/shopify/create-sale.ts@@ -12,7 +12,7 @@ import { sendWorkspaceWebhook } from "@/lib/webhook/publish";import { transformSaleEventData } from "@/lib/webhook/transform";import { nanoid } from "@dub/utils";import { waitUntil } from "@vercel/functions";-import { orderSchema } from "./schema";+import { shopifyOrderSchema } from "./schema";export async function createShopifySale({event,@@ -25,7 +25,7 @@ export async function createShopifySale({workspaceId: string;leadData: LeadEventTB;}) {- const order = orderSchema.parse(event);+ const order = shopifyOrderSchema.parse(event);const {checkout_token: checkoutToken,diff --git a/apps/web/lib/integrations/shopify/process-order.ts b/apps/web/lib/integrations/shopify/process-order.tsindex 1a9f2866d6..9a95571447 100644--- a/apps/web/lib/integrations/shopify/process-order.ts+++ b/apps/web/lib/integrations/shopify/process-order.ts@@ -17,7 +17,7 @@ import { Link } from "@prisma/client";import { waitUntil } from "@vercel/functions";import { createShopifyLead } from "./create-lead";import { createShopifySale } from "./create-sale";-import { orderSchema } from "./schema";+import { orderSchema, ShopifyOrder } from "./schema";// Process the order from Shopify webhookexport async function processOrder({@@ -26,7 +26,7 @@ export async function processOrder({customerId,clickId,}: {- event: unknown;+ event: ShopifyOrder;workspaceId: string;customerId?: string; // ID of the customer in DubclickId?: string; // ID of the click event from Shopify pixeldiff --git a/apps/web/lib/integrations/shopify/schema.ts b/apps/web/lib/integrations/shopify/schema.tsindex cc7c985a5f..726cec6b4b 100644--- a/apps/web/lib/integrations/shopify/schema.ts+++ b/apps/web/lib/integrations/shopify/schema.ts@@ -1,6 +1,6 @@import * as z from "zod/v4";-export const orderSchema = z.object({+export const shopifyOrderSchema = z.object({confirmation_number: z.string(),checkout_token: z.string(),customer: z@@ -39,3 +39,5 @@ export const integrationCredentialsSchema = z.object({.describe("Encrypted access token for the Shopify store."),scope: z.string().nullish().describe("Scope of the Shopify store."),});++export type ShopifyOrder = z.infer<typeof shopifyOrderSchema>;diff --git a/apps/web/lib/jobs/handlers/process-shopify-order-job.ts b/apps/web/lib/jobs/handlers/process-shopify-order-job.tsnew file mode 100644index 0000000000..bac56dfade--- /dev/null+++ b/apps/web/lib/jobs/handlers/process-shopify-order-job.ts@@ -0,0 +1,18 @@+import { orderSchema } from "@/lib/integrations/shopify/schema";+import * as z from "zod/v4";+import { defineJob } from "../index";++const inputSchema = z.object({+ order: orderSchema,+ clickId: z.string().nullable(),+ workspaceId: z.string(),+});++// Process the Shopify order+export const processShopifyOrderJob = defineJob({+ name: "process-shopify-order-job",+ schema: inputSchema,+ async handle(input) {+ const { workspaceId, clickId, order } = input;+ },+});diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.tsindex a93e0a52d1..861a419b3e 100644--- a/apps/web/lib/jobs/registry.ts+++ b/apps/web/lib/jobs/registry.ts@@ -29,6 +29,11 @@ const jobLoaders = {import("./handlers/create-tremendous-campaign-job").then((m) => m.createTremendousCampaignJob,),++ "process-shopify-order-job": () =>+ import("./handlers/process-shopify-order-job").then(+ (m) => m.processShopifyOrderJob,+ ),} as const satisfies Record<string, () => Promise<JobDefinition>>;const jobCache = new Map<string, JobDefinition>();diff --git a/apps/web/playwright/api/setup-test-workspace.ts b/apps/web/playwright/api/setup-test-workspace.tsindex ad01b77312..bc6b4288e4 100644--- a/apps/web/playwright/api/setup-test-workspace.ts+++ b/apps/web/playwright/api/setup-test-workspace.ts@@ -31,6 +31,9 @@ export const TEST_WORKSPACE = {domain: "playwright-api.dub-internal-test.com",url: "https://example.com",},+ shopify: {+ storeId: "playwright-api.myshopify.com",+ },} as const;const authFile = path.join(__dirname, "../.auth/api.json");@@ -72,6 +75,7 @@ export async function setupTestWorkspace() {aiLimit: 1000,partnersLimit: 1000,groupsLimit: 100,+ shopifyStoreId: TEST_WORKSPACE.shopify.storeId,},create: {id: createId({ prefix: "ws_" }),@@ -88,6 +92,7 @@ export async function setupTestWorkspace() {aiLimit: 1000,partnersLimit: 1000,groupsLimit: 100,+ shopifyStoreId: TEST_WORKSPACE.shopify.storeId,},});diff --git a/apps/web/playwright/api/shopify/helpers.ts b/apps/web/playwright/api/shopify/helpers.tsnew file mode 100644index 0000000000..9c1abb2402--- /dev/null+++ b/apps/web/playwright/api/shopify/helpers.ts@@ -0,0 +1,102 @@+import { nanoid } from "@dub/utils";+import { createHmac } from "crypto";+import { PLAYWRIGHT_API_BASE } from "../constants";+import { TEST_WORKSPACE } from "../setup-test-workspace";++export function shopifyCheckoutToken() {+ return `pw_checkout_${nanoid(24)}`;+}++export function shopifyClickId() {+ return `pw_click_${nanoid(16)}`;+}++export function shopifyOrderPayload({+ checkoutToken,+}: {+ checkoutToken: string;+}) {+ return {+ confirmation_number: nanoid(10).toUpperCase(),+ checkout_token: checkoutToken,+ customer: {+ id: Math.floor(Math.random() * 1_000_000_000),+ first_name: "Playwright",+ last_name: "Shopify",+ email: `pw.shopify.${nanoid(8)}@dub-internal-test.com`,+ },+ current_subtotal_price_set: {+ shop_money: {+ amount: "22.50",+ currency_code: "USD",+ },+ },+ discount_codes: [],+ billing_address: {+ province: "California",+ country_code: "US",+ },+ };+}++function shopifyWebhookSignature(body: string) {+ const secret = process.env.SHOPIFY_WEBHOOK_SECRET;+ if (!secret) {+ return undefined;+ }++ return createHmac("sha256", secret).update(body, "utf8").digest("base64");+}++export async function postShopifyPixel({+ clickId,+ checkoutToken,+}: {+ clickId: string | null;+ checkoutToken: string;+}) {+ const response = await fetch(`${PLAYWRIGHT_API_BASE}/api/shopify/pixel`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ clickId,+ checkoutToken,+ }),+ });++ return {+ status: response.status,+ data: await response.json(),+ };+}++export async function postShopifyOrdersPaidWebhook({+ checkoutToken,+}: {+ checkoutToken: string;+}) {+ const payload = shopifyOrderPayload({ checkoutToken });+ const body = JSON.stringify(payload);+ const signature = shopifyWebhookSignature(body);++ const response = await fetch(+ `${PLAYWRIGHT_API_BASE}/api/shopify/integration/webhook`,+ {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "x-shopify-topic": "orders/paid",+ "x-shopify-shop-domain": TEST_WORKSPACE.shopify.storeId,+ ...(signature && { "x-shopify-hmac-sha256": signature }),+ },+ body,+ },+ );++ return {+ status: response.status,+ data: await response.text(),+ };+}diff --git a/apps/web/scripts/dev/simulate-shopify-webhook.ts b/apps/web/scripts/dev/simulate-shopify-webhook.tsdeleted file mode 100644index 85d233beb3..0000000000--- a/apps/web/scripts/dev/simulate-shopify-webhook.ts+++ /dev/null@@ -1,48 +0,0 @@-import "dotenv-flow/config";--import { orderSchema } from "@/lib/integrations/shopify/schema";-import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";--async function main() {- const event = orderSchema.parse({- confirmation_number: "WDQ0YHYU32",- checkout_token: "20585787a5a40274b6b34511d2f637e92",- customer: {- id: 97773681707702,- first_name: "John",- last_name: "Doe",- email: "[email protected]",- },- current_subtotal_price_set: {- shop_money: {- amount: "22.50",- currency_code: "USD",- },- },- discount_codes: [- {- code: "D1",- },- ],- billing_address: {- province: "California",- country_code: "US",- },- });-- const response = await fetch(- `${APP_DOMAIN_WITH_NGROK}/api/shopify/integration/webhook`,- {- method: "POST",- body: JSON.stringify(event),- headers: {- "x-shopify-topic": "orders/paid",- "x-shopify-shop-domain": "dub-conversions.myshopify.com",- },- },- );-- console.log(response.ok);-}--main();