Skip to content

A platform for community discussion. Free, open, simple.

FIX: Match SES bounces by original message ID (#44995)

Samdef5934727ee
diff --git a/app/jobs/regular/process_sns_notification.rb b/app/jobs/regular/process_sns_notification.rb
index 2785887da9c..73784c52138 100644
--- a/app/jobs/regular/process_sns_notification.rb
+++ b/app/jobs/regular/process_sns_notification.rb
@@ -17,7 +17,7 @@ module Jobs
end
return unless message && message["notificationType"] == "Bounce"
- return unless message_id = message.dig("mail", "messageId").presence
+ return unless message_id = original_message_id(message["mail"])
return unless bounce_type = message.dig("bounce", "bounceType").presence
return if !Email::Sns.allowed_topic_arn?(json["TopicArn"])
@@ -39,5 +39,15 @@ module Jobs
end
end
end
+
+ private
+
+ def original_message_id(mail)
+ header =
+ Array(mail&.dig("headers")).find { |entry| entry["name"].to_s.casecmp?("Message-ID") }
+ message_id = header&.dig("value").presence || mail&.dig("commonHeaders", "messageId").presence
+ # Provider-assigned IDs do not identify the original message stored in EmailLog.
+ Email::MessageIdService.message_id_clean(message_id.strip) if message_id
+ end
end
end
diff --git a/docs/INSTALL-email.md b/docs/INSTALL-email.md
index b2abfbd3e28..7f38e3fe745 100644
--- a/docs/INSTALL-email.md
+++ b/docs/INSTALL-email.md
@@ -59,3 +59,11 @@ NOTE: By default, Elastic Email will add an additional UNSUBSCRIBE link at the b
### Bounce Handling
When using a third party email service, you will need to enable VERP, or activate their **webhooks** in order to handle bouncing emails. [Full details here.](https://meta.discourse.org/t/handling-bouncing-e-mails/45343)
+
+For Amazon SES bounce notifications delivered through SNS:
+
+- Subscribe `https://<your-discourse-host>/webhooks/aws` to the bounce notification topic.
+- Add the exact topic ARN to the `aws_sns_topic_arn_allowlist` site setting.
+- Enable **Include original email headers** for bounce notifications on the SES sending identity. Discourse uses the original `Message-ID` and bounced recipient to match a sent email. The SES-assigned message ID alone cannot identify the email in Discourse, so notifications without the original Message-ID are ignored.
+
+See the [AWS notification contents documentation](https://docs.aws.amazon.com/ses/latest/dg/notification-contents.html) for the distinction between the original and SES-assigned message IDs.
diff --git a/spec/requests/webhooks_controller_spec.rb b/spec/requests/webhooks_controller_spec.rb
index 56527a5f3bd..4c3ae3f56e9 100644
--- a/spec/requests/webhooks_controller_spec.rb
+++ b/spec/requests/webhooks_controller_spec.rb
@@ -793,64 +793,68 @@ RSpec.describe WebhooksController do
describe "#aws" do
let(:topic_arn) { "arn:aws:sns:us-east-1:123456789012:discourse-bounces" }
let(:other_topic_arn) { "arn:aws:sns:us-east-1:999999999999:attacker-topic" }
+ let(:ses_message_id) { "000001378603177f-7a5433e7-8edb-42ae-af10-f0181f34d6ee-000000" }
let(:bounce_status) { "5.1.1" }
let(:payload) do
{
"Type" => "Notification",
"TopicArn" => topic_arn,
- "Message" => {
- "notificationType" => "Bounce",
- :bounce => {
- "bounceType" => "Permanent",
- "reportingMTA" => "dns; email.example.com",
- :bouncedRecipients => [
- {
- "emailAddress" => email,
- "status" => bounce_status,
- "action" => "failed",
- "diagnosticCode" => "smtp; 550 5.1.1 <#{email}>... User",
- },
- ],
- "bounceSubType" => "General",
- "timestamp" => "2016-01-27T14:59:38.237Z",
- "feedbackId" => "00000138111222aa-33322211-cccc-cccc-cccc-ddddaaaa068a-000000",
- "remoteMtaIp" => "127.0.2.0",
- },
- :mail => {
- "timestamp" => "2016-01-27T14:59:38.237Z",
- "source" => "[email protected]",
- "sourceArn" => "arn:aws:ses:us-east-1:888888888888:identity/example.com",
- "sourceIp" => "127.0.3.0",
- "sendingAccountId" => "123456789012",
- "callerIdentity" => "IAM_user_or_role_name",
- "messageId" => message_id,
- "destination" => [email, "[email protected]", "[email protected]", "[email protected]"],
- "headersTruncated" => false,
- "headers" => [
- { "name" => "From", "value" => "\"John Doe\" <[email protected]>" },
- {
- "name" => "To",
- "value" =>
- "\"Test\" <#{email}>, \"Jane Doe\" <[email protected]>, \"Mary Doe\" <[email protected]>, \"Richard Doe\" <[email protected]>",
- },
- { "name" => "Message-ID", "value" => message_id },
- { "name" => "Subject", "value" => "Hello" },
- { "name" => "Content-Type", "value" => "text/plain; charset=\"UTF-8\"" },
- { "name" => "Content-Transfer-Encoding", "value" => "base64" },
- { "name" => "Date", "value" => "Wed, 27 Jan 2016 14:05:45 +0000" },
- ],
- "commonHeaders" => {
- "from" => ["John Doe <[email protected]>"],
- "date" => "Wed, 27 Jan 2016 14:05:45 +0000",
- "to" => [
- "\"Test\" <#{email}>, Jane Doe <[email protected]>, Mary Doe <[email protected]>, Richard Doe <[email protected]>",
- ],
- "messageId" => message_id,
- "subject" => "Hello",
+ "Message" => notification.to_json,
+ }.to_json
+ end
+ let(:notification) do
+ {
+ "notificationType" => "Bounce",
+ :bounce => {
+ "bounceType" => "Permanent",
+ "reportingMTA" => "dns; email.example.com",
+ :bouncedRecipients => [
+ {
+ "emailAddress" => email,
+ "status" => bounce_status,
+ "action" => "failed",
+ "diagnosticCode" => "smtp; 550 5.1.1 <#{email}>... User",
},
+ ],
+ "bounceSubType" => "General",
+ "timestamp" => "2016-01-27T14:59:38.237Z",
+ "feedbackId" => "00000138111222aa-33322211-cccc-cccc-cccc-ddddaaaa068a-000000",
+ "remoteMtaIp" => "127.0.2.0",
+ },
+ :mail => {
+ "timestamp" => "2016-01-27T14:59:38.237Z",
+ "source" => "[email protected]",
+ "sourceArn" => "arn:aws:ses:us-east-1:888888888888:identity/example.com",
+ "sourceIp" => "127.0.3.0",
+ "sendingAccountId" => "123456789012",
+ "callerIdentity" => "IAM_user_or_role_name",
+ "messageId" => ses_message_id,
+ "destination" => [email, "[email protected]", "[email protected]", "[email protected]"],
+ "headersTruncated" => false,
+ "headers" => [
+ { "name" => "From", "value" => "\"John Doe\" <[email protected]>" },
+ {
+ "name" => "To",
+ "value" =>
+ "\"Test\" <#{email}>, \"Jane Doe\" <[email protected]>, \"Mary Doe\" <[email protected]>, \"Richard Doe\" <[email protected]>",
+ },
+ { "name" => "Message-ID", "value" => " <#{message_id}> " },
+ { "name" => "Subject", "value" => "Hello" },
+ { "name" => "Content-Type", "value" => "text/plain; charset=\"UTF-8\"" },
+ { "name" => "Content-Transfer-Encoding", "value" => "base64" },
+ { "name" => "Date", "value" => "Wed, 27 Jan 2016 14:05:45 +0000" },
+ ],
+ "commonHeaders" => {
+ "from" => ["John Doe <[email protected]>"],
+ "date" => "Wed, 27 Jan 2016 14:05:45 +0000",
+ "to" => [
+ "\"Test\" <#{email}>, Jane Doe <[email protected]>, Mary Doe <[email protected]>, Richard Doe <[email protected]>",
+ ],
+ "messageId" => " <#{message_id}> ",
+ "subject" => "Hello",
},
- }.to_json,
- }.to_json
+ },
+ }
end
let(:subscription_confirmation_payload) do
{
@@ -870,7 +874,7 @@ RSpec.describe WebhooksController do
SiteSetting.aws_sns_topic_arn_allowlist = topic_arn
end
- it "hard bounces" do
+ it "hard bounces using the original message ID" do
user = Fabricate(:user, email: email)
email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)
@@ -883,7 +887,7 @@ RSpec.describe WebhooksController do
expect(email_log.user.user_stat.bounce_score).to eq(SiteSetting.hard_bounce_score)
end
- it "does not bounce an email log with a different SES message id" do
+ it "does not bounce an email log with a different original message ID" do
user = Fabricate(:user, email: email)
email_log =
Fabricate(:email_log, user: user, message_id: "other-message-id", to_address: email)
@@ -895,6 +899,82 @@ RSpec.describe WebhooksController do
expect(email_log.user.user_stat.bounce_score).to eq(0)
end
+ it "matches the original message ID from raw headers without common headers" do
+ notification[:mail].delete("commonHeaders")
+ header = notification[:mail]["headers"].find { |entry| entry["name"] == "Message-ID" }
+ header["name"] = "mEsSaGe-Id"
+ user = Fabricate(:user, email: email)
+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)
+
+ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }
+
+ expect(response.status).to eq(200)
+ expect(email_log.reload.bounced).to eq(true)
+ expect(user.user_stat.bounce_score).to eq(SiteSetting.hard_bounce_score)
+ end
+
+ it "matches the original message ID from common headers without raw headers" do
+ notification[:mail].delete("headers")
+ user = Fabricate(:user, email: email)
+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)
+
+ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }
+
+ expect(response.status).to eq(200)
+ expect(email_log.reload.bounced).to eq(true)
+ expect(user.user_stat.bounce_score).to eq(SiteSetting.hard_bounce_score)
+ end
+
+ it "uses common headers when the raw message ID is blank" do
+ header = notification[:mail]["headers"].find { |entry| entry["name"] == "Message-ID" }
+ header["value"] = " "
+ user = Fabricate(:user, email: email)
+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)
+
+ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }
+
+ expect(response.status).to eq(200)
+ expect(email_log.reload.bounced).to eq(true)
+ end
+
+ it "prefers the raw original message ID over the common headers message ID" do
+ notification[:mail]["commonHeaders"]["messageId"] = ses_message_id
+ user = Fabricate(:user, email: email)
+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)
+ other_email_log =
+ Fabricate(:email_log, user: user, message_id: ses_message_id, to_address: email)
+
+ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }
+
+ expect(response.status).to eq(200)
+ expect(email_log.reload.bounced).to eq(true)
+ expect(other_email_log.reload.bounced).to eq(false)
+ end
+
+ it "ignores notifications without an original message ID even if the SES ID matches a log" do
+ notification[:mail].delete("headers")
+ notification[:mail].delete("commonHeaders")
+ user = Fabricate(:user, email: email)
+ email_log = Fabricate(:email_log, user: user, message_id: ses_message_id, to_address: email)
+
+ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }
+
+ expect(response.status).to eq(200)
+ expect(email_log.reload.bounced).to eq(false)
+ expect(user.user_stat.bounce_score).to eq(0)
+ end
+
+ it "ignores a matching message ID sent to a different recipient" do
+ user = Fabricate(:user)
+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: user.email)
+
+ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }
+
+ expect(response.status).to eq(200)
+ expect(email_log.reload.bounced).to eq(false)
+ expect(user.user_stat.bounce_score).to eq(0)
+ end
+
it "does not increase the bounce score for duplicate notifications" do
user = Fabricate(:user, email: email)
email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)