Rallly is an open-source scheduling and collaboration tool designed to make organizing events and meetings easier.
TypeScript i18nextnext-authnextjspostgresqlprismareact-emailt3-stacktailwindcsstrpcturborepotypescriptzod
🔒️ Document reason for flagged/suspicious content (#2198)
Luke Vellacb3dc5e9b782
diff --git a/apps/web/src/features/moderation/index.ts b/apps/web/src/features/moderation/index.tsindex 37acb8633..dcbc6ac5b 100644--- a/apps/web/src/features/moderation/index.ts+++ b/apps/web/src/features/moderation/index.ts@@ -2,13 +2,18 @@ import { createLogger } from "@rallly/logger";import { waitUntil } from "@vercel/functions";import { env } from "@/env";import { getEmailClient } from "@/utils/emails";-import type { ModerationVerdict } from "./libs/ai-moderation";+import type { ModerationResult, ModerationVerdict } from "./libs/ai-moderation";import { moderateContentWithAI } from "./libs/ai-moderation";import { containsSuspiciousPatterns } from "./libs/pattern-moderation";const logger = createLogger("moderation");-export type { ModerationVerdict };+export type { ModerationResult, ModerationVerdict };++const safeResult: ModerationResult = {+ verdict: "safe",+ reason: "",+};/*** Moderates content to detect spam, inappropriate content, or abuse@@ -16,18 +21,18 @@ export type { ModerationVerdict };* 1. Pattern-based detection for common spam patterns* 2. AI-based moderation for more sophisticated content analysis*- * @returns "flagged" (block), "suspicious" (allow but notify), or "safe" (allow)+ * @returns Moderation result with verdict and explanation*/export async function moderateContent({userId,content,}: {userId: string;- content: Array<string | undefined>;-}): Promise<ModerationVerdict> {+ content: Record<string, string>;+}): Promise<ModerationResult> {// Skip moderation if the feature is disabled in environmentif (env.MODERATION_ENABLED !== "true") {- return "safe";+ return safeResult;}// Check if OpenAI API key is available@@ -35,10 +40,13 @@ export async function moderateContent({logger.warn("Content moderation is enabled but OPENAI_API_KEY is not set. AI-based moderation will be skipped.",);- return "safe";+ return safeResult;}- const textToModerate = content.filter(Boolean).join("\n");+ const textToModerate = Object.entries(content)+ .filter(([_, value]) => value.trim() !== "")+ .map(([key, value]) => `${key}:\n${value}`)+ .join("\n\n");// First check for suspicious patterns (faster)const hasSuspiciousPatterns = containsSuspiciousPatterns(textToModerate);@@ -47,18 +55,22 @@ export async function moderateContent({if (hasSuspiciousPatterns) {logger.info("Suspicious patterns detected, performing AI moderation check");try {- const verdict = await moderateContentWithAI(textToModerate);+ const result = await moderateContentWithAI(textToModerate);- if (verdict === "flagged" || verdict === "suspicious") {- logger.warn({ userId, verdict }, `Content ${verdict} by AI moderation`);+ if (result.verdict === "flagged" || result.verdict === "suspicious") {+ logger.warn(+ { userId, verdict: result.verdict, reason: result.reason },+ `Content ${result.verdict} by AI moderation`,+ );const emailClient = await getEmailClient();waitUntil(emailClient.sendEmail({to: env.SUPPORT_EMAIL,- subject: `Content ${verdict} by moderation`,+ subject: `Content ${result.verdict} by moderation`,text: [`User ID: ${userId}`,- `Verdict: ${verdict}`,+ `Verdict: ${result.verdict}`,+ `Reason: ${result.reason}`,"Content:",textToModerate,].join("\n\n"),@@ -66,14 +78,14 @@ export async function moderateContent({);}- return verdict;+ return result;} catch (error) {logger.error({ error }, "Error during AI content moderation");- return "safe";+ return safeResult;}}- return "safe";+ return safeResult;}/**diff --git a/apps/web/src/features/moderation/libs/ai-moderation.ts b/apps/web/src/features/moderation/libs/ai-moderation.tsindex 4fe895802..b97fada72 100644--- a/apps/web/src/features/moderation/libs/ai-moderation.ts+++ b/apps/web/src/features/moderation/libs/ai-moderation.ts@@ -6,14 +6,19 @@ const logger = createLogger("moderation/ai");export type ModerationVerdict = "flagged" | "suspicious" | "safe";+export type ModerationResult = {+ verdict: ModerationVerdict;+ reason: string;+};+/*** Moderates content using AI to detect inappropriate content* @param text The text to moderate- * @returns The moderation verdict: "flagged", "suspicious", or "safe"+ * @returns The moderation result with verdict and explanation*/export async function moderateContentWithAI(text: string,-): Promise<ModerationVerdict> {+): Promise<ModerationResult> {try {const result = await generateText({model: openai("gpt-4.1-mini"),@@ -30,34 +35,42 @@ Respond with one of three verdicts:- Malicious links: content whose primary purpose is to trick readers into clicking an external link unrelated to the poll- Spam relay: content that is not a genuine poll but a mass marketing message, advertisement, or message blast using the platform as a delivery mechanism-- SUSPICIOUS: Content is ambiguous — it could be legitimate but has characteristics that warrant human review. Use this when content doesn't clearly fit the FLAGGED categories but seems unusual for a polling app.+- SUSPICIOUS: Content shows some indicators of harmful intent (e.g. partial scam language, obfuscated links) but not enough to conclusively block. Use this for borderline cases that warrant human review.-- SAFE: Content is clearly legitimate.+- SAFE: Content is not harmful.-Do NOT flag or mark as suspicious content because it is in a non-English language, uses uppercase letters, is about a non-scheduling topic (e.g. votes, approvals, group decisions), or includes instructions for participants.+Do NOT flag or mark as suspicious content because:+- It is in a non-English language+- It is short, brief, or lacks detailWhen in doubt between FLAGGED and SUSPICIOUS, choose SUSPICIOUS.When in doubt between SUSPICIOUS and SAFE, choose SAFE.-Respond with only 'FLAGGED', 'SUSPICIOUS', or 'SAFE'.`,+Respond in exactly this format:+VERDICT+Brief explanation of why this verdict was chosen.`,},{ role: "user", content: text },],});- const response = result.text.trim().toUpperCase();-- if (response.includes("FLAGGED")) {- return "flagged";- }+ const lines = result.text.trim().split("\n");+ const verdictLine = (lines[0] ?? "").trim().toUpperCase();+ const reason = lines.slice(1).join("\n").trim() || "No reason provided";- if (response.includes("SUSPICIOUS")) {- return "suspicious";+ let verdict: ModerationVerdict = "safe";+ if (verdictLine.includes("FLAGGED")) {+ verdict = "flagged";+ } else if (verdictLine.includes("SUSPICIOUS")) {+ verdict = "suspicious";}- return "safe";+ return { verdict, reason };} catch (err) {logger.error({ error: err }, "AI moderation failed");- return "safe";+ return {+ verdict: "safe",+ reason: "AI moderation failed, defaulting to safe",+ };}}diff --git a/apps/web/src/trpc/routers/polls.ts b/apps/web/src/trpc/routers/polls.tsindex ff1812a1d..c112915d9 100644--- a/apps/web/src/trpc/routers/polls.ts+++ b/apps/web/src/trpc/routers/polls.ts@@ -187,19 +187,28 @@ export const polls = router({.use(requireUserMiddleware).use(createRateLimitMiddleware("create_poll", 20, "1 h")).mutation(async ({ ctx, input }) => {- const moderationVerdict = await moderateContent({+ const moderation = await moderateContent({userId: ctx.user.id,- content: [input.title, input.description, input.location],+ content: {+ Title: input.title,+ Description: input.description || "",+ Location: input.location || "",+ },});- if (moderationVerdict === "flagged") {+ if (moderation.verdict !== "safe") {ctx.posthog?.capture({distinctId: ctx.user.id,event: "flagged_content",properties: {action: "create_poll",+ verdict: moderation.verdict,+ reason: moderation.reason,},});+ }++ if (moderation.verdict === "flagged") {return {ok: false as const,error: { code: "INAPPROPRIATE_CONTENT" as const },@@ -370,19 +379,28 @@ export const polls = router({});}- const moderationVerdict = await moderateContent({+ const moderation = await moderateContent({userId: ctx.user.id,- content: [input.title, input.description, input.location],+ content: {+ Title: input.title || "",+ Description: input.description || "",+ Location: input.location || "",+ },});- if (moderationVerdict === "flagged") {+ if (moderation.verdict !== "safe") {ctx.posthog?.capture({distinctId: ctx.user.id,event: "flagged_content",properties: {action: "update_poll",+ verdict: moderation.verdict,+ reason: moderation.reason,},});+ }++ if (moderation.verdict === "flagged") {return {ok: false as const,error: { code: "INAPPROPRIATE_CONTENT" as const },@@ -562,19 +580,28 @@ export const polls = router({.use(requireUserMiddleware).use(createRateLimitMiddleware("create_poll", 20, "1 h")).use(async ({ ctx, input, next }) => {- const moderationVerdict = await moderateContent({+ const moderation = await moderateContent({userId: ctx.user.id,- content: [input.title, input.description, input.location],+ content: {+ Title: input.title || "",+ Description: input.description || "",+ Location: input.location || "",+ },});- if (moderationVerdict === "flagged") {+ if (moderation.verdict !== "safe") {ctx.posthog?.capture({distinctId: ctx.user.id,event: "flagged_content",properties: {action: "create_poll",+ verdict: moderation.verdict,+ reason: moderation.reason,},});+ }++ if (moderation.verdict === "flagged") {throw new TRPCError({code: "BAD_REQUEST",message: "Inappropriate content",@@ -740,19 +767,28 @@ export const polls = router({.use(requireUserMiddleware).use(createRateLimitMiddleware("update_poll", 10, "1 m")).use(async ({ ctx, input, next }) => {- const moderationVerdict = await moderateContent({+ const moderation = await moderateContent({userId: ctx.user.id,- content: [input.title, input.description, input.location],+ content: {+ Title: input.title || "",+ Description: input.description || "",+ Location: input.location || "",+ },});- if (moderationVerdict === "flagged") {+ if (moderation.verdict !== "safe") {ctx.posthog?.capture({distinctId: ctx.user.id,event: "flagged_content",properties: {action: "update_poll",+ verdict: moderation.verdict,+ reason: moderation.reason,},});+ }++ if (moderation.verdict === "flagged") {throw new TRPCError({code: "BAD_REQUEST",message: "Inappropriate content",