Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Process Shopify orders via checkout cache rendezvous and background job

Kiran K47fa290f8554
diff --git a/apps/web/app/(ee)/api/shopify/integration/webhook/orders-paid.ts b/apps/web/app/(ee)/api/shopify/integration/webhook/orders-paid.ts
index 7d11b20dea..5aa28359bb 100644
--- a/apps/web/app/(ee)/api/shopify/integration/webhook/orders-paid.ts
+++ b/apps/web/app/(ee)/api/shopify/integration/webhook/orders-paid.ts
@@ -1,28 +1,27 @@
-import { qstash } from "@/lib/cron";
-import { createShopifySale } from "@/lib/integrations/shopify/create-sale";
import {
- attributeViaDiscountCode,
- processOrder,
-} from "@/lib/integrations/shopify/process-order";
-import { orderSchema } from "@/lib/integrations/shopify/schema";
+ tryDispatchShopifyOrderJob,
+ writeShopifyCheckoutFields,
+} from "@/lib/integrations/shopify/checkout-cache";
+import { shopifyOrderSchema } from "@/lib/integrations/shopify/schema";
+import { processShopifyOrderJob } from "@/lib/jobs/handlers/process-shopify-order-job";
import { prisma } from "@/lib/prisma";
import { WorkspaceProps } from "@/lib/types";
-import { redis } from "@/lib/upstash";
-import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
export async function ordersPaid({
event,
workspace,
}: {
event: any;
- workspace: Pick<WorkspaceProps, "id" | "defaultProgramId" | "webhookEnabled">;
+ workspace: Pick<WorkspaceProps, "id" | "defaultProgramId">;
}) {
+ const order = shopifyOrderSchema.parse(event);
const {
customer: orderCustomer,
checkout_token: checkoutToken,
discount_codes: discountCodes,
- } = orderSchema.parse(event);
+ } = order;
+ // If the order has a customer, try to find the customer in our database
if (orderCustomer) {
const { id: externalId } = orderCustomer;
@@ -33,17 +32,24 @@ export async function ordersPaid({
externalId: externalId.toString(),
},
},
+ select: {
+ id: true,
+ },
});
- // customer is found, process the order right away
if (customer) {
- await processOrder({
- event,
- workspaceId: workspace.id,
- customerId: customer.id,
- });
+ await processShopifyOrderJob.dispatch(
+ {
+ order,
+ clickId: null,
+ workspaceId: workspace.id,
+ },
+ {
+ deduplicationId: `shopify-order-${checkoutToken}`,
+ },
+ );
- return "[Shopify] Order event processed successfully.";
+ return `[Shopify] Existing customer ${customer.id} found. Order queued for processing.`;
}
}
@@ -65,63 +71,39 @@ export async function ordersPaid({
});
if (programDiscountCodes.length > 0) {
- const { customer, leadEvent: leadData } = await attributeViaDiscountCode({
- event,
- workspace,
- link: programDiscountCodes[0].link,
- });
-
- await createShopifySale({
- leadData,
- event,
- workspaceId: workspace.id,
- customerId: customer.id,
- });
+ await processShopifyOrderJob.dispatch(
+ {
+ order,
+ clickId: null,
+ workspaceId: workspace.id,
+ },
+ {
+ deduplicationId: `shopify-order-${checkoutToken}`,
+ },
+ );
- return "[Shopify] Order event processed successfully with discount codes.";
+ return `[Shopify] Partner discount code ${programDiscountCodes[0].code} found. Order queued for processing.`;
}
}
- // Check the cache to see the pixel event for this checkout token exist before publishing the event to the queue
- const clickId = await redis.hget<string>(
- `shopify:checkout:${checkoutToken}`,
- "clickId",
- );
-
- // clickId is empty, order is not from a Dub link
- if (clickId === "") {
- await redis.del(`shopify:checkout:${checkoutToken}`);
-
- return "[Shopify] Order is not from a Dub link. Skipping...";
- }
-
- // clickId is found, process the order for the new customer
- else if (clickId) {
- await processOrder({
- event,
+ // At this stage, we know the order has no customer or partner discount code
+ // so we need to wait for the pixel event to arrive as this could be a new customer coming via a link
+ const checkout = await writeShopifyCheckoutFields({
+ checkoutToken,
+ fields: {
+ order,
workspaceId: workspace.id,
- clickId,
- });
+ },
+ });
- return "[Shopify] Order event processed successfully.";
- }
-
- // clickId is not found, we need to wait for the pixel event to come in so that we can decide if the order is from a Dub link or not
- else {
- await redis.hset(`shopify:checkout:${checkoutToken}`, {
- order: event,
- });
+ const dispatched = await tryDispatchShopifyOrderJob({
+ checkoutToken,
+ checkout,
+ });
- await qstash.publishJSON({
- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/shopify/order-paid`,
- body: {
- checkoutToken,
- workspaceId: workspace.id,
- },
- retries: 5,
- delay: 3,
- });
-
- return "[Shopify] clickId not found, waiting for pixel event to arrive...";
+ if (dispatched) {
+ return `[Shopify] Click ID ${checkout.clickId} found. Order queued for processing.`;
}
+
+ return "[Shopify] Waiting for pixel event to arrive...";
}
diff --git a/apps/web/app/(ee)/api/shopify/pixel/route.ts b/apps/web/app/(ee)/api/shopify/pixel/route.ts
index 7723bd31fa..b69053866a 100644
--- a/apps/web/app/(ee)/api/shopify/pixel/route.ts
+++ b/apps/web/app/(ee)/api/shopify/pixel/route.ts
@@ -1,14 +1,16 @@
import { COMMON_CORS_HEADERS } from "@/lib/api/cors";
import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors";
import { parseRequestBody } from "@/lib/api/utils";
+import {
+ tryDispatchShopifyOrderJob,
+ writeShopifyCheckoutFields,
+} from "@/lib/integrations/shopify/checkout-cache";
import { getClickEvent } from "@/lib/tinybird";
-import { ratelimit, redis } from "@/lib/upstash";
+import { ratelimit } from "@/lib/upstash";
import { LOCALHOST_IP } from "@dub/utils";
import { ipAddress, waitUntil } from "@vercel/functions";
import { NextResponse } from "next/server";
-export const runtime = "edge";
-
// POST /api/shopify/pixel – Handle the Shopify Pixel events
export const POST = async (req: Request) => {
try {
@@ -42,9 +44,19 @@ export const POST = async (req: Request) => {
}
waitUntil(
- redis.hset(`shopify:checkout:${checkoutToken}`, {
- clickId: clickId || "",
- }),
+ (async () => {
+ const checkout = await writeShopifyCheckoutFields({
+ checkoutToken,
+ fields: {
+ clickId,
+ },
+ });
+
+ await tryDispatchShopifyOrderJob({
+ checkoutToken,
+ checkout,
+ });
+ })(),
);
return NextResponse.json("OK", {
diff --git a/apps/web/lib/integrations/shopify/checkout-cache.ts b/apps/web/lib/integrations/shopify/checkout-cache.ts
new file mode 100644
index 0000000000..e8772fc8cb
--- /dev/null
+++ b/apps/web/lib/integrations/shopify/checkout-cache.ts
@@ -0,0 +1,80 @@
+import { processShopifyOrderJob } from "@/lib/jobs/handlers/process-shopify-order-job";
+import { redis } from "@/lib/upstash";
+import * as z from "zod/v4";
+import { shopifyOrderSchema } from "./schema";
+
+const SHOPIFY_CHECKOUT_CACHE_TTL_SECONDS = 60 * 60 * 24; // 24 hours
+const SHOPIFY_CHECKOUT_CACHE_KEY_PREFIX = "shopify:checkout:";
+
+const shopifyCheckoutCacheSchema = z.object({
+ clickId: z.string().optional(),
+ workspaceId: z.string().optional(),
+ order: shopifyOrderSchema.optional(),
+ dispatched: z.boolean().optional(),
+});
+
+type ShopifyCheckoutCache = z.infer<typeof shopifyCheckoutCacheSchema>;
+
+function shopifyCheckoutCacheKey(checkoutToken: string) {
+ return `${SHOPIFY_CHECKOUT_CACHE_KEY_PREFIX}${checkoutToken}`;
+}
+
+export async function writeShopifyCheckoutFields({
+ checkoutToken,
+ fields,
+}: {
+ checkoutToken: string;
+ fields: Record<string, unknown>;
+}) {
+ const key = shopifyCheckoutCacheKey(checkoutToken);
+
+ const pipeline = redis.pipeline();
+ pipeline.hset(key, fields);
+ pipeline.expire(key, SHOPIFY_CHECKOUT_CACHE_TTL_SECONDS);
+ pipeline.hgetall<ShopifyCheckoutCache>(key);
+
+ const results = await pipeline.exec();
+ const cache = results[2] as ShopifyCheckoutCache | null;
+
+ return cache ?? {};
+}
+
+export async function tryDispatchShopifyOrderJob({
+ checkoutToken,
+ checkout,
+}: {
+ checkoutToken: string;
+ checkout: ShopifyCheckoutCache;
+}) {
+ if (!checkout.order || !checkout.workspaceId || !checkout.clickId) {
+ return false;
+ }
+
+ if (checkout.dispatched) {
+ return false;
+ }
+
+ // Claim the checkout
+ const key = shopifyCheckoutCacheKey(checkoutToken);
+ const claim = await redis.hsetnx(key, "dispatched", true);
+ const claimed = Boolean(claim);
+
+ if (!claimed) {
+ return false;
+ }
+
+ await processShopifyOrderJob.dispatch(
+ {
+ workspaceId: checkout.workspaceId,
+ clickId: checkout.clickId,
+ order: checkout.order,
+ },
+ {
+ deduplicationId: `shopify-order-${checkoutToken}`,
+ },
+ );
+
+ await redis.del(key);
+
+ return true;
+}
diff --git a/apps/web/lib/integrations/shopify/create-lead.ts b/apps/web/lib/integrations/shopify/create-lead.ts
index a361a40ae2..b544aee015 100644
--- a/apps/web/lib/integrations/shopify/create-lead.ts
+++ b/apps/web/lib/integrations/shopify/create-lead.ts
@@ -11,7 +11,7 @@ import { transformLeadEventData } from "@/lib/webhook/transform";
import { leadEventSchemaTB } from "@/lib/zod/schemas/leads";
import { nanoid } from "@dub/utils";
import { waitUntil } from "@vercel/functions";
-import { orderSchema } from "./schema";
+import { shopifyOrderSchema } from "./schema";
export async function createShopifyLead({
clickId,
@@ -22,7 +22,7 @@ export async function createShopifyLead({
workspaceId: string;
event: any;
}) {
- const { customer: orderCustomer } = orderSchema.parse(event);
+ const { customer: orderCustomer } = shopifyOrderSchema.parse(event);
const customerId = createId({ prefix: "cus_" });
/*
diff --git a/apps/web/lib/integrations/shopify/create-sale.ts b/apps/web/lib/integrations/shopify/create-sale.ts
index d4b20f2b80..d9f901eff5 100644
--- a/apps/web/lib/integrations/shopify/create-sale.ts
+++ b/apps/web/lib/integrations/shopify/create-sale.ts
@@ -12,7 +12,7 @@ import { sendWorkspaceWebhook } from "@/lib/webhook/publish";
import { transformSaleEventData } from "@/lib/webhook/transform";
import { nanoid } from "@dub/utils";
import { waitUntil } from "@vercel/functions";
-import { orderSchema } from "./schema";
+import { shopifyOrderSchema } from "./schema";
export async function createShopifySale({
event,
@@ -25,7 +25,7 @@ export async function createShopifySale({
workspaceId: string;
leadData: LeadEventTB;
}) {
- const order = orderSchema.parse(event);
+ const order = shopifyOrderSchema.parse(event);
const {
checkout_token: checkoutToken,
diff --git a/apps/web/lib/integrations/shopify/process-order.ts b/apps/web/lib/integrations/shopify/process-order.ts
index 1a9f2866d6..9a95571447 100644
--- a/apps/web/lib/integrations/shopify/process-order.ts
+++ b/apps/web/lib/integrations/shopify/process-order.ts
@@ -17,7 +17,7 @@ import { Link } from "@prisma/client";
import { waitUntil } from "@vercel/functions";
import { createShopifyLead } from "./create-lead";
import { createShopifySale } from "./create-sale";
-import { orderSchema } from "./schema";
+import { orderSchema, ShopifyOrder } from "./schema";
// Process the order from Shopify webhook
export async function processOrder({
@@ -26,7 +26,7 @@ export async function processOrder({
customerId,
clickId,
}: {
- event: unknown;
+ event: ShopifyOrder;
workspaceId: string;
customerId?: string; // ID of the customer in Dub
clickId?: string; // ID of the click event from Shopify pixel
diff --git a/apps/web/lib/integrations/shopify/schema.ts b/apps/web/lib/integrations/shopify/schema.ts
index cc7c985a5f..726cec6b4b 100644
--- a/apps/web/lib/integrations/shopify/schema.ts
+++ b/apps/web/lib/integrations/shopify/schema.ts
@@ -1,6 +1,6 @@
import * as z from "zod/v4";
-export const orderSchema = z.object({
+export const shopifyOrderSchema = z.object({
confirmation_number: z.string(),
checkout_token: z.string(),
customer: z
@@ -39,3 +39,5 @@ export const integrationCredentialsSchema = z.object({
.describe("Encrypted access token for the Shopify store."),
scope: z.string().nullish().describe("Scope of the Shopify store."),
});
+
+export type ShopifyOrder = z.infer<typeof shopifyOrderSchema>;
diff --git a/apps/web/lib/jobs/handlers/process-shopify-order-job.ts b/apps/web/lib/jobs/handlers/process-shopify-order-job.ts
new file mode 100644
index 0000000000..bac56dfade
--- /dev/null
+++ b/apps/web/lib/jobs/handlers/process-shopify-order-job.ts
@@ -0,0 +1,18 @@
+import { orderSchema } from "@/lib/integrations/shopify/schema";
+import * as z from "zod/v4";
+import { defineJob } from "../index";
+
+const inputSchema = z.object({
+ order: orderSchema,
+ clickId: z.string().nullable(),
+ workspaceId: z.string(),
+});
+
+// Process the Shopify order
+export const processShopifyOrderJob = defineJob({
+ name: "process-shopify-order-job",
+ schema: inputSchema,
+ async handle(input) {
+ const { workspaceId, clickId, order } = input;
+ },
+});
diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.ts
index a93e0a52d1..861a419b3e 100644
--- a/apps/web/lib/jobs/registry.ts
+++ b/apps/web/lib/jobs/registry.ts
@@ -29,6 +29,11 @@ const jobLoaders = {
import("./handlers/create-tremendous-campaign-job").then(
(m) => m.createTremendousCampaignJob,
),
+
+ "process-shopify-order-job": () =>
+ import("./handlers/process-shopify-order-job").then(
+ (m) => m.processShopifyOrderJob,
+ ),
} as const satisfies Record<string, () => Promise<JobDefinition>>;
const jobCache = new Map<string, JobDefinition>();
diff --git a/apps/web/playwright/api/setup-test-workspace.ts b/apps/web/playwright/api/setup-test-workspace.ts
index ad01b77312..bc6b4288e4 100644
--- a/apps/web/playwright/api/setup-test-workspace.ts
+++ b/apps/web/playwright/api/setup-test-workspace.ts
@@ -31,6 +31,9 @@ export const TEST_WORKSPACE = {
domain: "playwright-api.dub-internal-test.com",
url: "https://example.com",
},
+ shopify: {
+ storeId: "playwright-api.myshopify.com",
+ },
} as const;
const authFile = path.join(__dirname, "../.auth/api.json");
@@ -72,6 +75,7 @@ export async function setupTestWorkspace() {
aiLimit: 1000,
partnersLimit: 1000,
groupsLimit: 100,
+ shopifyStoreId: TEST_WORKSPACE.shopify.storeId,
},
create: {
id: createId({ prefix: "ws_" }),
@@ -88,6 +92,7 @@ export async function setupTestWorkspace() {
aiLimit: 1000,
partnersLimit: 1000,
groupsLimit: 100,
+ shopifyStoreId: TEST_WORKSPACE.shopify.storeId,
},
});
diff --git a/apps/web/playwright/api/shopify/helpers.ts b/apps/web/playwright/api/shopify/helpers.ts
new file mode 100644
index 0000000000..9c1abb2402
--- /dev/null
+++ b/apps/web/playwright/api/shopify/helpers.ts
@@ -0,0 +1,102 @@
+import { nanoid } from "@dub/utils";
+import { createHmac } from "crypto";
+import { PLAYWRIGHT_API_BASE } from "../constants";
+import { TEST_WORKSPACE } from "../setup-test-workspace";
+
+export function shopifyCheckoutToken() {
+ return `pw_checkout_${nanoid(24)}`;
+}
+
+export function shopifyClickId() {
+ return `pw_click_${nanoid(16)}`;
+}
+
+export function shopifyOrderPayload({
+ checkoutToken,
+}: {
+ checkoutToken: string;
+}) {
+ return {
+ confirmation_number: nanoid(10).toUpperCase(),
+ checkout_token: checkoutToken,
+ customer: {
+ id: Math.floor(Math.random() * 1_000_000_000),
+ first_name: "Playwright",
+ last_name: "Shopify",
+ email: `pw.shopify.${nanoid(8)}@dub-internal-test.com`,
+ },
+ current_subtotal_price_set: {
+ shop_money: {
+ amount: "22.50",
+ currency_code: "USD",
+ },
+ },
+ discount_codes: [],
+ billing_address: {
+ province: "California",
+ country_code: "US",
+ },
+ };
+}
+
+function shopifyWebhookSignature(body: string) {
+ const secret = process.env.SHOPIFY_WEBHOOK_SECRET;
+ if (!secret) {
+ return undefined;
+ }
+
+ return createHmac("sha256", secret).update(body, "utf8").digest("base64");
+}
+
+export async function postShopifyPixel({
+ clickId,
+ checkoutToken,
+}: {
+ clickId: string | null;
+ checkoutToken: string;
+}) {
+ const response = await fetch(`${PLAYWRIGHT_API_BASE}/api/shopify/pixel`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ clickId,
+ checkoutToken,
+ }),
+ });
+
+ return {
+ status: response.status,
+ data: await response.json(),
+ };
+}
+
+export async function postShopifyOrdersPaidWebhook({
+ checkoutToken,
+}: {
+ checkoutToken: string;
+}) {
+ const payload = shopifyOrderPayload({ checkoutToken });
+ const body = JSON.stringify(payload);
+ const signature = shopifyWebhookSignature(body);
+
+ const response = await fetch(
+ `${PLAYWRIGHT_API_BASE}/api/shopify/integration/webhook`,
+ {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ "x-shopify-topic": "orders/paid",
+ "x-shopify-shop-domain": TEST_WORKSPACE.shopify.storeId,
+ ...(signature && { "x-shopify-hmac-sha256": signature }),
+ },
+ body,
+ },
+ );
+
+ return {
+ status: response.status,
+ data: await response.text(),
+ };
+}
diff --git a/apps/web/scripts/dev/simulate-shopify-webhook.ts b/apps/web/scripts/dev/simulate-shopify-webhook.ts
deleted file mode 100644
index 85d233beb3..0000000000
--- a/apps/web/scripts/dev/simulate-shopify-webhook.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-import "dotenv-flow/config";
-
-import { orderSchema } from "@/lib/integrations/shopify/schema";
-import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
-
-async function main() {
- const event = orderSchema.parse({
- confirmation_number: "WDQ0YHYU32",
- checkout_token: "20585787a5a40274b6b34511d2f637e92",
- customer: {
- id: 97773681707702,
- first_name: "John",
- last_name: "Doe",
- email: "[email protected]",
- },
- current_subtotal_price_set: {
- shop_money: {
- amount: "22.50",
- currency_code: "USD",
- },
- },
- discount_codes: [
- {
- code: "D1",
- },
- ],
- billing_address: {
- province: "California",
- country_code: "US",
- },
- });
-
- const response = await fetch(
- `${APP_DOMAIN_WITH_NGROK}/api/shopify/integration/webhook`,
- {
- method: "POST",
- body: JSON.stringify(event),
- headers: {
- "x-shopify-topic": "orders/paid",
- "x-shopify-shop-domain": "dub-conversions.myshopify.com",
- },
- },
- );
-
- console.log(response.ok);
-}
-
-main();