Skip to content

Rallly is an open-source scheduling and collaboration tool designed to make organizing events and meetings easier.

🔒️ Document reason for flagged/suspicious content (#2198)

Luke Vellacb3dc5e9b782
diff --git a/apps/web/src/features/moderation/index.ts b/apps/web/src/features/moderation/index.ts
index 37acb8633..dcbc6ac5b 100644
--- a/apps/web/src/features/moderation/index.ts
+++ b/apps/web/src/features/moderation/index.ts
@@ -2,13 +2,18 @@ import { createLogger } from "@rallly/logger";
import { waitUntil } from "@vercel/functions";
import { env } from "@/env";
import { getEmailClient } from "@/utils/emails";
-import type { ModerationVerdict } from "./libs/ai-moderation";
+import type { ModerationResult, ModerationVerdict } from "./libs/ai-moderation";
import { moderateContentWithAI } from "./libs/ai-moderation";
import { containsSuspiciousPatterns } from "./libs/pattern-moderation";
const logger = createLogger("moderation");
-export type { ModerationVerdict };
+export type { ModerationResult, ModerationVerdict };
+
+const safeResult: ModerationResult = {
+ verdict: "safe",
+ reason: "",
+};
/**
* Moderates content to detect spam, inappropriate content, or abuse
@@ -16,18 +21,18 @@ export type { ModerationVerdict };
* 1. Pattern-based detection for common spam patterns
* 2. AI-based moderation for more sophisticated content analysis
*
- * @returns "flagged" (block), "suspicious" (allow but notify), or "safe" (allow)
+ * @returns Moderation result with verdict and explanation
*/
export async function moderateContent({
userId,
content,
}: {
userId: string;
- content: Array<string | undefined>;
-}): Promise<ModerationVerdict> {
+ content: Record<string, string>;
+}): Promise<ModerationResult> {
// Skip moderation if the feature is disabled in environment
if (env.MODERATION_ENABLED !== "true") {
- return "safe";
+ return safeResult;
}
// Check if OpenAI API key is available
@@ -35,10 +40,13 @@ export async function moderateContent({
logger.warn(
"Content moderation is enabled but OPENAI_API_KEY is not set. AI-based moderation will be skipped.",
);
- return "safe";
+ return safeResult;
}
- const textToModerate = content.filter(Boolean).join("\n");
+ const textToModerate = Object.entries(content)
+ .filter(([_, value]) => value.trim() !== "")
+ .map(([key, value]) => `${key}:\n${value}`)
+ .join("\n\n");
// First check for suspicious patterns (faster)
const hasSuspiciousPatterns = containsSuspiciousPatterns(textToModerate);
@@ -47,18 +55,22 @@ export async function moderateContent({
if (hasSuspiciousPatterns) {
logger.info("Suspicious patterns detected, performing AI moderation check");
try {
- const verdict = await moderateContentWithAI(textToModerate);
+ const result = await moderateContentWithAI(textToModerate);
- if (verdict === "flagged" || verdict === "suspicious") {
- logger.warn({ userId, verdict }, `Content ${verdict} by AI moderation`);
+ if (result.verdict === "flagged" || result.verdict === "suspicious") {
+ logger.warn(
+ { userId, verdict: result.verdict, reason: result.reason },
+ `Content ${result.verdict} by AI moderation`,
+ );
const emailClient = await getEmailClient();
waitUntil(
emailClient.sendEmail({
to: env.SUPPORT_EMAIL,
- subject: `Content ${verdict} by moderation`,
+ subject: `Content ${result.verdict} by moderation`,
text: [
`User ID: ${userId}`,
- `Verdict: ${verdict}`,
+ `Verdict: ${result.verdict}`,
+ `Reason: ${result.reason}`,
"Content:",
textToModerate,
].join("\n\n"),
@@ -66,14 +78,14 @@ export async function moderateContent({
);
}
- return verdict;
+ return result;
} catch (error) {
logger.error({ error }, "Error during AI content moderation");
- return "safe";
+ return safeResult;
}
}
- return "safe";
+ return safeResult;
}
/**
diff --git a/apps/web/src/features/moderation/libs/ai-moderation.ts b/apps/web/src/features/moderation/libs/ai-moderation.ts
index 4fe895802..b97fada72 100644
--- a/apps/web/src/features/moderation/libs/ai-moderation.ts
+++ b/apps/web/src/features/moderation/libs/ai-moderation.ts
@@ -6,14 +6,19 @@ const logger = createLogger("moderation/ai");
export type ModerationVerdict = "flagged" | "suspicious" | "safe";
+export type ModerationResult = {
+ verdict: ModerationVerdict;
+ reason: string;
+};
+
/**
* Moderates content using AI to detect inappropriate content
* @param text The text to moderate
- * @returns The moderation verdict: "flagged", "suspicious", or "safe"
+ * @returns The moderation result with verdict and explanation
*/
export async function moderateContentWithAI(
text: string,
-): Promise<ModerationVerdict> {
+): Promise<ModerationResult> {
try {
const result = await generateText({
model: openai("gpt-4.1-mini"),
@@ -30,34 +35,42 @@ Respond with one of three verdicts:
- Malicious links: content whose primary purpose is to trick readers into clicking an external link unrelated to the poll
- Spam relay: content that is not a genuine poll but a mass marketing message, advertisement, or message blast using the platform as a delivery mechanism
-- SUSPICIOUS: Content is ambiguous — it could be legitimate but has characteristics that warrant human review. Use this when content doesn't clearly fit the FLAGGED categories but seems unusual for a polling app.
+- SUSPICIOUS: Content shows some indicators of harmful intent (e.g. partial scam language, obfuscated links) but not enough to conclusively block. Use this for borderline cases that warrant human review.
-- SAFE: Content is clearly legitimate.
+- SAFE: Content is not harmful.
-Do NOT flag or mark as suspicious content because it is in a non-English language, uses uppercase letters, is about a non-scheduling topic (e.g. votes, approvals, group decisions), or includes instructions for participants.
+Do NOT flag or mark as suspicious content because:
+- It is in a non-English language
+- It is short, brief, or lacks detail
When in doubt between FLAGGED and SUSPICIOUS, choose SUSPICIOUS.
When in doubt between SUSPICIOUS and SAFE, choose SAFE.
-Respond with only 'FLAGGED', 'SUSPICIOUS', or 'SAFE'.`,
+Respond in exactly this format:
+VERDICT
+Brief explanation of why this verdict was chosen.`,
},
{ role: "user", content: text },
],
});
- const response = result.text.trim().toUpperCase();
-
- if (response.includes("FLAGGED")) {
- return "flagged";
- }
+ const lines = result.text.trim().split("\n");
+ const verdictLine = (lines[0] ?? "").trim().toUpperCase();
+ const reason = lines.slice(1).join("\n").trim() || "No reason provided";
- if (response.includes("SUSPICIOUS")) {
- return "suspicious";
+ let verdict: ModerationVerdict = "safe";
+ if (verdictLine.includes("FLAGGED")) {
+ verdict = "flagged";
+ } else if (verdictLine.includes("SUSPICIOUS")) {
+ verdict = "suspicious";
}
- return "safe";
+ return { verdict, reason };
} catch (err) {
logger.error({ error: err }, "AI moderation failed");
- return "safe";
+ return {
+ verdict: "safe",
+ reason: "AI moderation failed, defaulting to safe",
+ };
}
}
diff --git a/apps/web/src/trpc/routers/polls.ts b/apps/web/src/trpc/routers/polls.ts
index ff1812a1d..c112915d9 100644
--- a/apps/web/src/trpc/routers/polls.ts
+++ b/apps/web/src/trpc/routers/polls.ts
@@ -187,19 +187,28 @@ export const polls = router({
.use(requireUserMiddleware)
.use(createRateLimitMiddleware("create_poll", 20, "1 h"))
.mutation(async ({ ctx, input }) => {
- const moderationVerdict = await moderateContent({
+ const moderation = await moderateContent({
userId: ctx.user.id,
- content: [input.title, input.description, input.location],
+ content: {
+ Title: input.title,
+ Description: input.description || "",
+ Location: input.location || "",
+ },
});
- if (moderationVerdict === "flagged") {
+ if (moderation.verdict !== "safe") {
ctx.posthog?.capture({
distinctId: ctx.user.id,
event: "flagged_content",
properties: {
action: "create_poll",
+ verdict: moderation.verdict,
+ reason: moderation.reason,
},
});
+ }
+
+ if (moderation.verdict === "flagged") {
return {
ok: false as const,
error: { code: "INAPPROPRIATE_CONTENT" as const },
@@ -370,19 +379,28 @@ export const polls = router({
});
}
- const moderationVerdict = await moderateContent({
+ const moderation = await moderateContent({
userId: ctx.user.id,
- content: [input.title, input.description, input.location],
+ content: {
+ Title: input.title || "",
+ Description: input.description || "",
+ Location: input.location || "",
+ },
});
- if (moderationVerdict === "flagged") {
+ if (moderation.verdict !== "safe") {
ctx.posthog?.capture({
distinctId: ctx.user.id,
event: "flagged_content",
properties: {
action: "update_poll",
+ verdict: moderation.verdict,
+ reason: moderation.reason,
},
});
+ }
+
+ if (moderation.verdict === "flagged") {
return {
ok: false as const,
error: { code: "INAPPROPRIATE_CONTENT" as const },
@@ -562,19 +580,28 @@ export const polls = router({
.use(requireUserMiddleware)
.use(createRateLimitMiddleware("create_poll", 20, "1 h"))
.use(async ({ ctx, input, next }) => {
- const moderationVerdict = await moderateContent({
+ const moderation = await moderateContent({
userId: ctx.user.id,
- content: [input.title, input.description, input.location],
+ content: {
+ Title: input.title || "",
+ Description: input.description || "",
+ Location: input.location || "",
+ },
});
- if (moderationVerdict === "flagged") {
+ if (moderation.verdict !== "safe") {
ctx.posthog?.capture({
distinctId: ctx.user.id,
event: "flagged_content",
properties: {
action: "create_poll",
+ verdict: moderation.verdict,
+ reason: moderation.reason,
},
});
+ }
+
+ if (moderation.verdict === "flagged") {
throw new TRPCError({
code: "BAD_REQUEST",
message: "Inappropriate content",
@@ -740,19 +767,28 @@ export const polls = router({
.use(requireUserMiddleware)
.use(createRateLimitMiddleware("update_poll", 10, "1 m"))
.use(async ({ ctx, input, next }) => {
- const moderationVerdict = await moderateContent({
+ const moderation = await moderateContent({
userId: ctx.user.id,
- content: [input.title, input.description, input.location],
+ content: {
+ Title: input.title || "",
+ Description: input.description || "",
+ Location: input.location || "",
+ },
});
- if (moderationVerdict === "flagged") {
+ if (moderation.verdict !== "safe") {
ctx.posthog?.capture({
distinctId: ctx.user.id,
event: "flagged_content",
properties: {
action: "update_poll",
+ verdict: moderation.verdict,
+ reason: moderation.reason,
},
});
+ }
+
+ if (moderation.verdict === "flagged") {
throw new TRPCError({
code: "BAD_REQUEST",
message: "Inappropriate content",