The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Keep the sync delays inside QStash's deduplication window
Rafia1a73f9288e1
diff --git a/apps/web/lib/api/partners/queue-partner-search-sync.ts b/apps/web/lib/api/partners/queue-partner-search-sync.tsindex 114a98ca70..710e7d1f2e 100644--- a/apps/web/lib/api/partners/queue-partner-search-sync.ts+++ b/apps/web/lib/api/partners/queue-partner-search-sync.ts@@ -5,6 +5,17 @@ import {PARTNER_SEARCH_SYNC_BATCH_SIZE,} from "./search";+/**+ * How long QStash remembers a deduplication ID.+ *+ * A delayed message is only collapsed while it is still pending *and* still+ * inside this window, so any delay at or above it gets no deduplication at all.+ * That makes this an upper bound on every delay below, not a trivium.+ *+ * https://upstash.com/docs/qstash/features/deduplication+ */+export const QSTASH_DEDUPLICATION_WINDOW_SECONDS = 600;+/*** How long a sync waits before it runs. The delay is what gives deduplication* something to collapse: an admin editing one partner three times in a row@@ -14,16 +25,21 @@ export const PARTNER_SEARCH_SYNC_DELAY_SECONDS = 5;/*** Links change far more often than the other document sources, and a link edit- * only moves `shortLinks` and `destinationUrls`, so they are worth batching- * harder. Nothing is lost by waiting: the handler re-reads the whole document,- * so a link sync that runs ten minutes late still picks up every other change- * that landed in the meantime.+ * only moves `shortLinks` and `destinationUrls`, so they are worth deferring.+ * Nothing is lost by waiting: the handler re-reads the whole document, so a+ * late link sync still picks up every other change that landed meanwhile.+ *+ * Half the deduplication window rather than all of it. Deduplication only ever+ * collapses repeat edits to the *same* partner, so the delay is not what keeps+ * a bulk link change cheap — chunking and flow control do that. Buying a little+ * more collapse by sitting at the window boundary would trade a real margin for+ * a rare case.*- * Link *creation* should use the default delay instead — a partner's short link+ * Link *creation* should use the default delay instead: a partner's short link* is how people search for them, so a newly enrolled partner being unfindable- * by link for ten minutes is a worse trade than the extra write.+ * by link is a worse trade than the extra write.*/-export const PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS = 600;+export const PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS = 300;type PartnerSearchSyncPayload =| { type: "enrollments"; enrollmentIds: string[] }@@ -60,13 +76,19 @@ function unique(values: string[] | undefined): string[] {* queued behind it, which would drag an interactive edit out to the link delay.* The cost is at most one redundant write, and the job is idempotent.*- * If the delay ever exceeds QStash's deduplication retention the collapse stops- * happening and every edit writes, which is wasteful but still correct.+ * A delay at or past the deduplication window gets no key at all. QStash would+ * have forgotten the ID before the first message fired, so every edit would+ * write anyway; sending a key regardless would just make the caller believe in+ * a collapse that is not happening. Failing openly beats failing quietly.*/function buildDeduplicationId(payload: PartnerSearchSyncPayload,delay: number,): string | undefined {+ if (delay >= QSTASH_DEDUPLICATION_WINDOW_SECONDS) {+ return undefined;+ }+if (payload.type === "enrollments") {return payload.enrollmentIds.length === 1? `enrollment:${payload.enrollmentIds[0]}:${delay}`diff --git a/apps/web/tests/partners/queue-partner-search-sync.test.ts b/apps/web/tests/partners/queue-partner-search-sync.test.tsindex 316a82eab1..59d047b51c 100644--- a/apps/web/tests/partners/queue-partner-search-sync.test.ts+++ b/apps/web/tests/partners/queue-partner-search-sync.test.ts@@ -1,6 +1,7 @@import {PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS,PARTNER_SEARCH_SYNC_DELAY_SECONDS,+ QSTASH_DEDUPLICATION_WINDOW_SECONDS,queuePartnerSearchSync,} from "@/lib/api/partners/queue-partner-search-sync";import { beforeEach, describe, expect, it, vi } from "vitest";@@ -145,6 +146,31 @@ describe("queuePartnerSearchSync", () => {]);});+ // The delays are only collapsed while QStash still remembers the key, so this+ // is the invariant that keeps the batching real rather than assumed. If the+ // documented window ever shrinks below a delay, this fails here rather than+ // silently costing writes in production.+ it("keeps every delay it deduplicates on inside the deduplication window", () => {+ expect(PARTNER_SEARCH_SYNC_DELAY_SECONDS).toBeLessThan(+ QSTASH_DEDUPLICATION_WINDOW_SECONDS,+ );+ expect(PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS).toBeLessThan(+ QSTASH_DEDUPLICATION_WINDOW_SECONDS,+ );+ });++ it("drops the dedup key past the window rather than claiming a collapse", async () => {+ await queuePartnerSearchSync({+ enrollmentIds: ["pge_1"],+ delay: QSTASH_DEDUPLICATION_WINDOW_SECONDS,+ });++ const [{ options }] = dispatchedWithOptions();++ expect(options.delay).toBe(QSTASH_DEDUPLICATION_WINDOW_SECONDS);+ expect(options.deduplicationId).toBeUndefined();+ });+it("swallows a dispatch failure so it cannot break the mutation that queued it", async () => {mocks.dispatchBatch.mockRejectedValue(new Error("qstash unreachable"));