A platform for community discussion. Free, open, simple.
FIX: Match SES bounces by original message ID (#44995)
Samdef5934727ee
diff --git a/app/jobs/regular/process_sns_notification.rb b/app/jobs/regular/process_sns_notification.rbindex 2785887da9c..73784c52138 100644--- a/app/jobs/regular/process_sns_notification.rb+++ b/app/jobs/regular/process_sns_notification.rb@@ -17,7 +17,7 @@ module Jobsendreturn unless message && message["notificationType"] == "Bounce"- return unless message_id = message.dig("mail", "messageId").presence+ return unless message_id = original_message_id(message["mail"])return unless bounce_type = message.dig("bounce", "bounceType").presencereturn if !Email::Sns.allowed_topic_arn?(json["TopicArn"])@@ -39,5 +39,15 @@ module Jobsendendend++ private++ def original_message_id(mail)+ header =+ Array(mail&.dig("headers")).find { |entry| entry["name"].to_s.casecmp?("Message-ID") }+ message_id = header&.dig("value").presence || mail&.dig("commonHeaders", "messageId").presence+ # Provider-assigned IDs do not identify the original message stored in EmailLog.+ Email::MessageIdService.message_id_clean(message_id.strip) if message_id+ endendenddiff --git a/docs/INSTALL-email.md b/docs/INSTALL-email.mdindex b2abfbd3e28..7f38e3fe745 100644--- a/docs/INSTALL-email.md+++ b/docs/INSTALL-email.md@@ -59,3 +59,11 @@ NOTE: By default, Elastic Email will add an additional UNSUBSCRIBE link at the b### Bounce HandlingWhen using a third party email service, you will need to enable VERP, or activate their **webhooks** in order to handle bouncing emails. [Full details here.](https://meta.discourse.org/t/handling-bouncing-e-mails/45343)++For Amazon SES bounce notifications delivered through SNS:++- Subscribe `https://<your-discourse-host>/webhooks/aws` to the bounce notification topic.+- Add the exact topic ARN to the `aws_sns_topic_arn_allowlist` site setting.+- Enable **Include original email headers** for bounce notifications on the SES sending identity. Discourse uses the original `Message-ID` and bounced recipient to match a sent email. The SES-assigned message ID alone cannot identify the email in Discourse, so notifications without the original Message-ID are ignored.++See the [AWS notification contents documentation](https://docs.aws.amazon.com/ses/latest/dg/notification-contents.html) for the distinction between the original and SES-assigned message IDs.diff --git a/spec/requests/webhooks_controller_spec.rb b/spec/requests/webhooks_controller_spec.rbindex 56527a5f3bd..4c3ae3f56e9 100644--- a/spec/requests/webhooks_controller_spec.rb+++ b/spec/requests/webhooks_controller_spec.rb@@ -793,64 +793,68 @@ RSpec.describe WebhooksController dodescribe "#aws" dolet(:topic_arn) { "arn:aws:sns:us-east-1:123456789012:discourse-bounces" }let(:other_topic_arn) { "arn:aws:sns:us-east-1:999999999999:attacker-topic" }+ let(:ses_message_id) { "000001378603177f-7a5433e7-8edb-42ae-af10-f0181f34d6ee-000000" }let(:bounce_status) { "5.1.1" }let(:payload) do{"Type" => "Notification","TopicArn" => topic_arn,- "Message" => {- "notificationType" => "Bounce",- :bounce => {- "bounceType" => "Permanent",- "reportingMTA" => "dns; email.example.com",- :bouncedRecipients => [- {- "emailAddress" => email,- "status" => bounce_status,- "action" => "failed",- "diagnosticCode" => "smtp; 550 5.1.1 <#{email}>... User",- },- ],- "bounceSubType" => "General",- "timestamp" => "2016-01-27T14:59:38.237Z",- "feedbackId" => "00000138111222aa-33322211-cccc-cccc-cccc-ddddaaaa068a-000000",- "remoteMtaIp" => "127.0.2.0",- },- :mail => {- "timestamp" => "2016-01-27T14:59:38.237Z",- "source" => "[email protected]",- "sourceArn" => "arn:aws:ses:us-east-1:888888888888:identity/example.com",- "sourceIp" => "127.0.3.0",- "sendingAccountId" => "123456789012",- "callerIdentity" => "IAM_user_or_role_name",- "messageId" => message_id,- "headersTruncated" => false,- "headers" => [- { "name" => "From", "value" => "\"John Doe\" <[email protected]>" },- {- "name" => "To",- "value" =>- "\"Test\" <#{email}>, \"Jane Doe\" <[email protected]>, \"Mary Doe\" <[email protected]>, \"Richard Doe\" <[email protected]>",- },- { "name" => "Message-ID", "value" => message_id },- { "name" => "Subject", "value" => "Hello" },- { "name" => "Content-Type", "value" => "text/plain; charset=\"UTF-8\"" },- { "name" => "Content-Transfer-Encoding", "value" => "base64" },- { "name" => "Date", "value" => "Wed, 27 Jan 2016 14:05:45 +0000" },- ],- "commonHeaders" => {- "from" => ["John Doe <[email protected]>"],- "date" => "Wed, 27 Jan 2016 14:05:45 +0000",- "to" => [- "\"Test\" <#{email}>, Jane Doe <[email protected]>, Mary Doe <[email protected]>, Richard Doe <[email protected]>",- ],- "messageId" => message_id,- "subject" => "Hello",+ "Message" => notification.to_json,+ }.to_json+ end+ let(:notification) do+ {+ "notificationType" => "Bounce",+ :bounce => {+ "bounceType" => "Permanent",+ "reportingMTA" => "dns; email.example.com",+ :bouncedRecipients => [+ {+ "emailAddress" => email,+ "status" => bounce_status,+ "action" => "failed",+ "diagnosticCode" => "smtp; 550 5.1.1 <#{email}>... User",},+ ],+ "bounceSubType" => "General",+ "timestamp" => "2016-01-27T14:59:38.237Z",+ "feedbackId" => "00000138111222aa-33322211-cccc-cccc-cccc-ddddaaaa068a-000000",+ "remoteMtaIp" => "127.0.2.0",+ },+ :mail => {+ "timestamp" => "2016-01-27T14:59:38.237Z",+ "source" => "[email protected]",+ "sourceArn" => "arn:aws:ses:us-east-1:888888888888:identity/example.com",+ "sourceIp" => "127.0.3.0",+ "sendingAccountId" => "123456789012",+ "callerIdentity" => "IAM_user_or_role_name",+ "messageId" => ses_message_id,+ "headersTruncated" => false,+ "headers" => [+ { "name" => "From", "value" => "\"John Doe\" <[email protected]>" },+ {+ "name" => "To",+ "value" =>+ "\"Test\" <#{email}>, \"Jane Doe\" <[email protected]>, \"Mary Doe\" <[email protected]>, \"Richard Doe\" <[email protected]>",+ },+ { "name" => "Message-ID", "value" => " <#{message_id}> " },+ { "name" => "Subject", "value" => "Hello" },+ { "name" => "Content-Type", "value" => "text/plain; charset=\"UTF-8\"" },+ { "name" => "Content-Transfer-Encoding", "value" => "base64" },+ { "name" => "Date", "value" => "Wed, 27 Jan 2016 14:05:45 +0000" },+ ],+ "commonHeaders" => {+ "from" => ["John Doe <[email protected]>"],+ "date" => "Wed, 27 Jan 2016 14:05:45 +0000",+ "to" => [+ "\"Test\" <#{email}>, Jane Doe <[email protected]>, Mary Doe <[email protected]>, Richard Doe <[email protected]>",+ ],+ "messageId" => " <#{message_id}> ",+ "subject" => "Hello",},- }.to_json,- }.to_json+ },+ }endlet(:subscription_confirmation_payload) do{@@ -870,7 +874,7 @@ RSpec.describe WebhooksController doSiteSetting.aws_sns_topic_arn_allowlist = topic_arnend- it "hard bounces" do+ it "hard bounces using the original message ID" douser = Fabricate(:user, email: email)email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)@@ -883,7 +887,7 @@ RSpec.describe WebhooksController doexpect(email_log.user.user_stat.bounce_score).to eq(SiteSetting.hard_bounce_score)end- it "does not bounce an email log with a different SES message id" do+ it "does not bounce an email log with a different original message ID" douser = Fabricate(:user, email: email)email_log =Fabricate(:email_log, user: user, message_id: "other-message-id", to_address: email)@@ -895,6 +899,82 @@ RSpec.describe WebhooksController doexpect(email_log.user.user_stat.bounce_score).to eq(0)end+ it "matches the original message ID from raw headers without common headers" do+ notification[:mail].delete("commonHeaders")+ header = notification[:mail]["headers"].find { |entry| entry["name"] == "Message-ID" }+ header["name"] = "mEsSaGe-Id"+ user = Fabricate(:user, email: email)+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)++ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }++ expect(response.status).to eq(200)+ expect(email_log.reload.bounced).to eq(true)+ expect(user.user_stat.bounce_score).to eq(SiteSetting.hard_bounce_score)+ end++ it "matches the original message ID from common headers without raw headers" do+ notification[:mail].delete("headers")+ user = Fabricate(:user, email: email)+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)++ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }++ expect(response.status).to eq(200)+ expect(email_log.reload.bounced).to eq(true)+ expect(user.user_stat.bounce_score).to eq(SiteSetting.hard_bounce_score)+ end++ it "uses common headers when the raw message ID is blank" do+ header = notification[:mail]["headers"].find { |entry| entry["name"] == "Message-ID" }+ header["value"] = " "+ user = Fabricate(:user, email: email)+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)++ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }++ expect(response.status).to eq(200)+ expect(email_log.reload.bounced).to eq(true)+ end++ it "prefers the raw original message ID over the common headers message ID" do+ notification[:mail]["commonHeaders"]["messageId"] = ses_message_id+ user = Fabricate(:user, email: email)+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)+ other_email_log =+ Fabricate(:email_log, user: user, message_id: ses_message_id, to_address: email)++ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }++ expect(response.status).to eq(200)+ expect(email_log.reload.bounced).to eq(true)+ expect(other_email_log.reload.bounced).to eq(false)+ end++ it "ignores notifications without an original message ID even if the SES ID matches a log" do+ notification[:mail].delete("headers")+ notification[:mail].delete("commonHeaders")+ user = Fabricate(:user, email: email)+ email_log = Fabricate(:email_log, user: user, message_id: ses_message_id, to_address: email)++ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }++ expect(response.status).to eq(200)+ expect(email_log.reload.bounced).to eq(false)+ expect(user.user_stat.bounce_score).to eq(0)+ end++ it "ignores a matching message ID sent to a different recipient" do+ user = Fabricate(:user)+ email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: user.email)++ post "/webhooks/aws.json", headers: { "RAW_POST_DATA" => payload }++ expect(response.status).to eq(200)+ expect(email_log.reload.bounced).to eq(false)+ expect(user.user_stat.bounce_score).to eq(0)+ end+it "does not increase the bounce score for duplicate notifications" douser = Fabricate(:user, email: email)email_log = Fabricate(:email_log, user: user, message_id: message_id, to_address: email)