Rallly is an open-source scheduling and collaboration tool designed to make organizing events and meetings easier.
TypeScript i18nextnext-authnextjspostgresqlprismareact-emailt3-stacktailwindcsstrpcturborepotypescriptzod
✨ Deliver webhooks right after the write, in-process (#3344)
Luke Vella32ddc8c1bf20
diff --git a/apps/web/src/app/api/house-keeping/[...method]/route.ts b/apps/web/src/app/api/house-keeping/[...method]/route.tsindex ec75081dd..3901b54cc 100644--- a/apps/web/src/app/api/house-keeping/[...method]/route.ts+++ b/apps/web/src/app/api/house-keeping/[...method]/route.ts@@ -209,6 +209,8 @@ app.get("/delete-orphaned-anonymous-users", async (c) => {});app.get("/deliver-webhooks", async (c) => {+ // The scheduled run over every space. A write schedules its own scoped+ // run in-process (scheduleWebhookDispatch); this one is the guarantee.// A DatabaseError rejects here and Hono answers 500, as an uncaught throw// did before.const summary = await runtime.runPromise(diff --git a/apps/web/src/features/poll/activity/data.ts b/apps/web/src/features/activity/data.tssimilarity index 100%rename from apps/web/src/features/poll/activity/data.tsrename to apps/web/src/features/activity/data.tsdiff --git a/apps/web/src/features/poll/activity/mutations.ts b/apps/web/src/features/activity/mutations.tssimilarity index 85%rename from apps/web/src/features/poll/activity/mutations.tsrename to apps/web/src/features/activity/mutations.tsindex 349ac0b1d..c022ba79a 100644--- a/apps/web/src/features/poll/activity/mutations.ts+++ b/apps/web/src/features/activity/mutations.ts@@ -12,6 +12,10 @@ export type PollActivityWrite = { pollId: string } & PollActivityEvent;* records: activity cannot be backfilled, so a mutation committing without* its event is a permanent hole in the poll's history, and an event without* its mutation records something that never happened.+ *+ * The log is also the webhook outbox. Delivery is not triggered here: this+ * feature is below the webhook feature in the graph, so the mutation that+ * owns the transaction schedules the dispatch once it commits.*/export async function recordPollActivities(tx: Prisma.TransactionClient,diff --git a/apps/web/src/features/poll/activity/schema.test.ts b/apps/web/src/features/activity/schema.test.tssimilarity index 100%rename from apps/web/src/features/poll/activity/schema.test.tsrename to apps/web/src/features/activity/schema.test.tsdiff --git a/apps/web/src/features/poll/activity/schema.ts b/apps/web/src/features/activity/schema.tssimilarity index 95%rename from apps/web/src/features/poll/activity/schema.tsrename to apps/web/src/features/activity/schema.tsindex a9ea03627..49687a105 100644--- a/apps/web/src/features/poll/activity/schema.ts+++ b/apps/web/src/features/activity/schema.ts@@ -1,5 +1,9 @@import * as z from "zod";-import { pollClosedReasonSchema } from "@/features/poll/schema";++/** Why a poll closed: the organizer, or every option having passed. */+export const pollClosedReasonSchema = z.enum(["auto", "manual"]);++export type PollClosedReason = z.infer<typeof pollClosedReasonSchema>;/*** Poll activity vocabulary v1. The `type` column is a plain string in thediff --git a/apps/web/src/features/poll/invite/mutations.ts b/apps/web/src/features/poll/invite/mutations.tsindex ff868c0ab..c5d59403b 100644--- a/apps/web/src/features/poll/invite/mutations.ts+++ b/apps/web/src/features/poll/invite/mutations.ts@@ -5,8 +5,8 @@ import { sendPollInviteEmail } from "@rallly/emails/templates/poll-invite";import { createLogger } from "@rallly/logger";import { shortUrl } from "@rallly/utils/absolute-url";import { getInstanceBranding, getSpaceBranding } from "@/emails/branding";+import { recordPollActivities } from "@/features/activity/mutations";import { resolveSpaceTier } from "@/features/billing/utils";-import { recordPollActivities } from "@/features/poll/activity/mutations";import { MAX_POLL_INVITES_PER_DAY } from "@/features/poll/invite/constants";import { getPollInvitePath } from "@/features/poll/invite/utils";import { generateAccessToken } from "@/features/poll/utils";diff --git a/apps/web/src/features/poll/mutations.test.ts b/apps/web/src/features/poll/mutations.test.tsindex d712138d9..20b2d2e05 100644--- a/apps/web/src/features/poll/mutations.test.ts+++ b/apps/web/src/features/poll/mutations.test.ts@@ -1,5 +1,10 @@import { beforeEach, describe, expect, it, vi } from "vitest";import type { AuthorizedSpaceId } from "@/features/space/types";++// The mutations schedule webhook dispatch, whose module validates the whole+// environment on import. Stub the env with nothing set.+vi.mock("@/env", () => ({ env: {} }));+import { closePoll, deleteInactivePolls, setPollMuted } from "./mutations";const { mockUpdateMany, mockFindFirst, mockUpdate, mockActivityCreateMany } =diff --git a/apps/web/src/features/poll/mutations.ts b/apps/web/src/features/poll/mutations.tsindex 4774fdb8c..e9e601e10 100644--- a/apps/web/src/features/poll/mutations.ts+++ b/apps/web/src/features/poll/mutations.ts@@ -4,8 +4,9 @@ import type { Prisma } from "@rallly/database";import { prisma } from "@rallly/database";import { nanoid } from "@rallly/utils/nanoid";import { revalidatePath } from "next/cache";-import { recordPollActivities } from "@/features/poll/activity/mutations";+import { recordPollActivities } from "@/features/activity/mutations";import type { AuthorizedSpaceId } from "@/features/space/types";+import { scheduleWebhookDispatch } from "@/features/webhook/mutations";export type PollOption = {startTime: Date;@@ -119,6 +120,7 @@ export const createPoll = async ({payload: { title },},]);+ scheduleWebhookDispatch({ pollId: poll.id });return poll;});@@ -176,6 +178,7 @@ export const closePoll = async ({payload: { reason: "manual" },},]);+ scheduleWebhookDispatch({ pollId: pollId });return toPollResponse(closedPoll);});@@ -229,6 +232,7 @@ export const deletePoll = async (await recordPollActivities(tx, [{ pollId, type: "poll_deleted", userId: null, payload: {} },]);+ scheduleWebhookDispatch({ pollId: pollId });return { id: pollId };});diff --git a/apps/web/src/features/poll/participant/mutations.ts b/apps/web/src/features/poll/participant/mutations.tsindex 709bf4539..a58d2f9da 100644--- a/apps/web/src/features/poll/participant/mutations.ts+++ b/apps/web/src/features/poll/participant/mutations.ts@@ -2,7 +2,7 @@ import "server-only";import type { Prisma, VoteType } from "@rallly/database";import { prisma } from "@rallly/database";-import { recordPollActivities } from "@/features/poll/activity/mutations";+import { recordPollActivities } from "@/features/activity/mutations";import { MAX_PARTICIPANTS } from "@/features/poll/constants";import {attachParticipantToInvite,@@ -11,6 +11,7 @@ import {import { revalidatePollPages } from "@/features/poll/mutations";import { generateAccessToken } from "@/features/poll/utils";import type { SpaceTier } from "@/features/space/schema";+import { scheduleWebhookDispatch } from "@/features/webhook/mutations";type WriteRefusal = "notFound" | "closed" | "full" | "tentativeVotesNotAllowed";@@ -239,6 +240,7 @@ export async function addParticipant({payload: { name: participant.name },},]);+ scheduleWebhookDispatch({ pollId: pollId });if (invite) {await attachParticipantToInvite(tx, {@@ -320,6 +322,7 @@ export async function updateParticipantVotes({payload: { name: participant.name },},]);+ scheduleWebhookDispatch({ pollId: pollId });});} catch (error) {return refusal(error);@@ -361,6 +364,7 @@ export async function renameParticipant({payload: { name },},]);+ scheduleWebhookDispatch({ pollId: pollId });});} catch (error) {return refusal(error);@@ -427,6 +431,7 @@ export async function deleteParticipant({},},]);+ scheduleWebhookDispatch({ pollId: pollId });});} catch (error) {return refusal(error);diff --git a/apps/web/src/features/poll/schema.ts b/apps/web/src/features/poll/schema.tsindex da8239b2b..d340d43c6 100644--- a/apps/web/src/features/poll/schema.ts+++ b/apps/web/src/features/poll/schema.ts@@ -28,9 +28,12 @@ export const pollStatusSchema = z.enum([export type PollStatus = z.infer<typeof pollStatusSchema>;-export const pollClosedReasonSchema = z.enum(["auto", "manual"]);--export type PollClosedReason = z.infer<typeof pollClosedReasonSchema>;+// Defined with the activity vocabulary, which records the reason; kept+// reachable from the poll schema for its other readers.+export {+ type PollClosedReason,+ pollClosedReasonSchema,+} from "@/features/activity/schema";export const setPollMutedSchema = z.object({pollId: z.string(),diff --git a/apps/web/src/features/webhook/constants.ts b/apps/web/src/features/webhook/constants.tsindex dcc4b1435..cf213fbd8 100644--- a/apps/web/src/features/webhook/constants.ts+++ b/apps/web/src/features/webhook/constants.ts@@ -27,7 +27,9 @@ export const MAX_DELIVERY_ATTEMPTS = RETRY_DELAYS_MS.length + 1;* Fan-out reads activities up to now minus this lag. cuid ids are not* monotonic, so the cursor is `createdAt`, which is the start of the* transaction that wrote the row, not its commit. The lag keeps the common- * case from ever seeing an uncommitted row.+ * case from ever seeing an uncommitted row. A run triggered by a write+ * skips it: its own row has committed, and the overlap below catches any+ * other that has not.*/export const FAN_OUT_LAG_MS = 10_000;diff --git a/apps/web/src/features/webhook/data.ts b/apps/web/src/features/webhook/data.tsindex da68be781..38a900a07 100644--- a/apps/web/src/features/webhook/data.ts+++ b/apps/web/src/features/webhook/data.ts@@ -73,9 +73,13 @@ export function countSpaceWebhooks({ spaceId }: { spaceId: string }) {* created — not by a session, so they take plain ids.*/-export async function listEnabledWebhooks() {+export async function listEnabledWebhooks({+ spaceId,+}: {+ spaceId?: string;+} = {}) {return prisma.spaceWebhook.findMany({- where: { enabled: true },+ where: { enabled: true, ...(spaceId ? { spaceId } : {}) },select: {id: true,spaceId: true,@@ -140,15 +144,17 @@ export async function listWebhookActivities({export async function listDueDeliveryIds({now,limit,+ spaceId,}: {now: Date;limit: number;+ spaceId?: string;}) {const rows = await prisma.webhookDelivery.findMany({where: {status: { in: ["pending", "failed"] },nextAttemptAt: { lte: now },- webhook: { enabled: true },+ webhook: { enabled: true, ...(spaceId ? { spaceId } : {}) },},select: { id: true },orderBy: { nextAttemptAt: "asc" },@@ -156,3 +162,12 @@ export async function listDueDeliveryIds({});return rows.map((row) => row.id);}++/** The space a poll belongs to, for scoping an immediate dispatcher run. */+export async function findPollSpaceId({ pollId }: { pollId: string }) {+ const poll = await prisma.poll.findUnique({+ where: { id: pollId },+ select: { spaceId: true },+ });+ return poll?.spaceId ?? null;+}diff --git a/apps/web/src/features/webhook/mutations.ts b/apps/web/src/features/webhook/mutations.tsindex 77ed3fbb3..e79b1f4ff 100644--- a/apps/web/src/features/webhook/mutations.ts+++ b/apps/web/src/features/webhook/mutations.ts@@ -4,13 +4,17 @@ import { randomBytes } from "node:crypto";import { subject } from "@casl/ability";import type { Prisma } from "@rallly/database";import { prisma } from "@rallly/database";+import { createLogger } from "@rallly/logger";import { decrypt, encrypt } from "@rallly/utils/encryption";import { Clock, Data, Effect } from "effect";+import { after } from "next/server";import { env } from "@/env";import { resolveSpaceTier } from "@/features/billing/utils";import type { DatabaseError } from "@/lib/effect/db";import { fromPrisma } from "@/lib/effect/db";+import { runtime } from "@/lib/effect/runtime";import { AppError } from "@/lib/errors/app-error";+import { isFeatureEnabled } from "@/lib/feature-flags/server";import { defineAbilityForWebhooks } from "./ability";import {DELIVERY_CLAIM_BATCH_SIZE,@@ -24,6 +28,7 @@ import {WEBHOOK_VERSION,} from "./constants";import {+ findPollSpaceId,listDueDeliveryIds,listEnabledWebhooks,listWebhookActivities,@@ -37,6 +42,9 @@ import {toWebhookEventType,} from "./utils";+/** Limits a dispatcher run to one space's endpoints. */+export type DispatchScope = { spaceId: string };+/*** Turns new activity into delivery rows, one per (webhook, activity). The* unique constraint makes re-reading harmless, so every run re-reads an@@ -55,11 +63,17 @@ import {* behind the cursor can still deliver events from the last few minutes.*/export const fanOutWebhookEvents = Effect.fn("webhook.fanOutWebhookEvents")(- function* ({ now }: { now: Date }) {- const until = new Date(now.getTime() - FAN_OUT_LAG_MS);+ function* ({ now, scope }: { now: Date; scope?: DispatchScope }) {+ // A run triggered right after a write has nothing uncommitted to fear+ // from its own row, so it reads up to now; the overlap window still+ // covers any other row that commits late.+ const lag = scope ? 0 : FAN_OUT_LAG_MS;+ const until = new Date(now.getTime() - lag);let created = 0;- for (const webhook of yield* fromPrisma(listEnabledWebhooks)) {+ for (const webhook of yield* fromPrisma(() =>+ listEnabledWebhooks({ spaceId: scope?.spaceId }),+ )) {if (webhook.cursor >= until) {continue;}@@ -182,10 +196,18 @@ export const reclaimStaleDeliveries = Effect.fn(* completion means a run that dies after sending still used an attempt.*/export const claimDueDeliveries = Effect.fn("webhook.claimDueDeliveries")(- function* ({ now, limit }: { now: Date; limit: number }) {+ function* ({+ now,+ limit,+ scope,+ }: {+ now: Date;+ limit: number;+ scope?: DispatchScope;+ }) {const claimed: string[] = [];for (const id of yield* fromPrisma(() =>- listDueDeliveryIds({ now, limit }),+ listDueDeliveryIds({ now, limit, spaceId: scope?.spaceId }),)) {const { count } = yield* fromPrisma(() =>prisma.webhookDelivery.updateMany({@@ -387,17 +409,25 @@ export type DeliverWebhooksSummary = {* One dispatcher run: recover orphans, fan out new activity, then send what* is due. Sends run a few at a time so a slow endpoint cannot hold the whole* batch to its timeout, and every outcome is recorded before the run ends.+ *+ * The scheduled run takes no scope and covers everything. A run triggered+ * by a write is scoped to that write's space and reads without the fan-out+ * lag, so the event it was triggered for goes out at once; the scheduled+ * run remains the guarantee for anything it misses.*/export const deliverWebhooks = Effect.fn("webhook.deliverWebhooks")(function* ({now,+ scope,}: {now: Date;+ scope?: DispatchScope;}): Effect.fn.Return<DeliverWebhooksSummary, DatabaseError, WebhookSender> {- const reclaimed = yield* reclaimStaleDeliveries({ now });- const fannedOut = yield* fanOutWebhookEvents({ now });+ const reclaimed = scope ? 0 : yield* reclaimStaleDeliveries({ now });+ const fannedOut = yield* fanOutWebhookEvents({ now, scope });const deliveries = yield* claimDueDeliveries({now,limit: DELIVERY_CLAIM_BATCH_SIZE,+ scope,});const outcomes = yield* Effect.forEach(@@ -434,6 +464,51 @@ export const deliverWebhooks = Effect.fn("webhook.deliverWebhooks")(function* ({return summary;});+const logger = createLogger("webhook/dispatch");++/**+ * Runs the dispatcher for a poll's space once the current response is sent,+ * so an event goes out seconds after the write instead of on the next+ * scheduled minute. Called by the mutation that owns the transaction, after+ * it commits. Needs a request scope for `after`; from a system context the+ * call is a no-op and the scheduled run covers it. A space with no enabled+ * endpoint costs one read and nothing else.+ */+export function scheduleWebhookDispatch({ pollId }: { pollId: string }) {+ if (!isFeatureEnabled("webhooks")) {+ return;+ }++ try {+ after(async () => {+ try {+ const spaceId = await findPollSpaceId({ pollId });+ if (!spaceId) {+ return;+ }+ const summary = await runtime.runPromise(+ deliverWebhooks({ now: new Date(), scope: { spaceId } }).pipe(+ Effect.provide(WebhookSender.layer),+ ),+ );+ if (summary.fannedOut > 0 || summary.attempted > 0) {+ logger.info(+ { spaceId, ...summary },+ "Dispatched webhook deliveries after a write",+ );+ }+ } catch (error) {+ logger.error(+ { pollId, error },+ "Webhook dispatch after a write failed",+ );+ }+ });+ } catch {+ // No request scope; the scheduled run delivers it.+ }+}+/*** Endpoint management for the settings page. Scope is proven by the caller's* action gate: the spaceId passed here is the actor's own space, and CASLdiff --git a/apps/web/src/features/webhook/schema.ts b/apps/web/src/features/webhook/schema.tsindex d258acc17..3ec4326ed 100644--- a/apps/web/src/features/webhook/schema.ts+++ b/apps/web/src/features/webhook/schema.ts@@ -1,5 +1,5 @@import * as z from "zod";-import { pollClosedReasonSchema } from "@/features/poll/schema";+import { pollClosedReasonSchema } from "@/features/activity/schema";import { WEBHOOK_VERSION } from "./constants";import { getWebhookUrlRejection } from "./utils";diff --git a/apps/web/src/features/webhook/utils.ts b/apps/web/src/features/webhook/utils.tsindex 9b7ce761b..89d7e2ba1 100644--- a/apps/web/src/features/webhook/utils.ts+++ b/apps/web/src/features/webhook/utils.ts@@ -1,4 +1,4 @@-import { pollActivitySchema } from "@/features/poll/activity/schema";+import { pollActivitySchema } from "@/features/activity/schema";import { RETRY_DELAYS_MS, WEBHOOK_VERSION } from "./constants";import type { WebhookEvent, WebhookEventType } from "./schema";diff --git a/apps/web/src/trpc/routers/polls.ts b/apps/web/src/trpc/routers/polls.tsindex 22ef41775..5292cb5fd 100644--- a/apps/web/src/trpc/routers/polls.ts+++ b/apps/web/src/trpc/routers/polls.ts@@ -8,9 +8,9 @@ import { TRPCError } from "@trpc/server";import { after } from "next/server";import * as z from "zod";import { getInstanceBranding, getSpaceBranding } from "@/emails/branding";+import { recordPollActivities } from "@/features/activity/mutations";import { getInstancePolicy } from "@/features/instance-policy/data";import { moderateContent } from "@/features/moderation/mutations";-import { recordPollActivities } from "@/features/poll/activity/mutations";import {canUserManagePoll,getPolls,@@ -23,6 +23,7 @@ import {isSpaceAttributionHidden,isSpaceBrandingActive,} from "@/features/space/utils";+import { scheduleWebhookDispatch } from "@/features/webhook/mutations";import { dayjs } from "@/lib/dayjs";import { identifyGroup, track } from "@/lib/posthog";import { createIcsEvent } from "@/lib/utils/ics";@@ -221,6 +222,7 @@ export const polls = router({payload: { title },},]);+ scheduleWebhookDispatch({ pollId: pollId });return poll;});@@ -579,6 +581,7 @@ export const polls = router({]: []),]);+ scheduleWebhookDispatch({ pollId: pollId });});// Get updated poll data for group update@@ -726,6 +729,7 @@ export const polls = router({payload: {},},]);+ scheduleWebhookDispatch({ pollId: pollId });});// Track poll deletion analytics@@ -1104,6 +1108,7 @@ export const polls = router({},},]);+ scheduleWebhookDispatch({ pollId: poll.id });return event;});@@ -1303,6 +1308,7 @@ export const polls = router({payload: {},},]);+ scheduleWebhookDispatch({ pollId: input.pollId });});track(ctx.user, {@@ -1358,6 +1364,7 @@ export const polls = router({payload: { reason: "manual" },},]);+ scheduleWebhookDispatch({ pollId: input.pollId });});track(ctx.user, {@@ -1445,6 +1452,7 @@ export const polls = router({payload: { title: input.newTitle },},]);+ scheduleWebhookDispatch({ pollId: newPoll.id });return newPoll;});diff --git a/apps/web/tests/webhooks.spec.ts b/apps/web/tests/webhooks.spec.tsindex 92bce7e7c..10e0f9e61 100644--- a/apps/web/tests/webhooks.spec.ts+++ b/apps/web/tests/webhooks.spec.ts@@ -8,6 +8,7 @@ import { prisma } from "@rallly/database";import { encrypt } from "@rallly/utils/encryption";import { WEBHOOK_VERSION } from "@/features/webhook/constants";import { WEBHOOK_EVENT_TYPES } from "@/features/webhook/schema";+import { InvitePage } from "./invite-page";import { createUserInDb, upgradeSpaceToPro } from "./test-utils";/**@@ -141,6 +142,15 @@ test.describe("Webhook delivery", () => {timeZone: "Europe/London",userId: owner.id,spaceId,+ // Three options so the invite page's vote flow has something to+ // answer; the immediate delivery test responds through it.+ options: {+ create: [+ { startTime: new Date("2030-01-01T09:00:00Z"), duration: 30 },+ { startTime: new Date("2030-01-02T09:00:00Z"), duration: 30 },+ { startTime: new Date("2030-01-03T09:00:00Z"), duration: 30 },+ ],+ },},});pollId = poll.id;@@ -460,6 +470,24 @@ test.describe("Webhook delivery", () => {expect(again.fannedOut).toBe(0);});+ test("delivers a response as soon as it is saved, without waiting for the cron", async ({+ page,+ }) => {+ await createWebhook({ events: ["poll.participant.created"] });++ await page.goto(`/invite/${pollId}`);+ await new InvitePage(page).addParticipant("Jessie", "[email protected]");++ // The write's own trigger delivers it; no cron run in this test.+ await expect+ .poll(() => receiver.requests.length, { timeout: 15_000 })+ .toBe(1);+ const body = JSON.parse(receiver.requests[0]?.body ?? "{}");+ expect(body.type).toBe("poll.participant.created");+ expect(body.data.poll.id).toBe(pollId);+ expect(body.data.participant.id).toEqual(expect.any(String));+ });+test("leaves activities inside the fan-out lag for the next run", async ({request,}) => {