Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Handle Postmark webhooks to suppress addresses (#6685)
Adam0f362bd84eaa
diff --git a/config/.env.test b/config/.env.testindex 354c0bffdb..909c9bd18f 100644--- a/config/.env.test+++ b/config/.env.test@@ -19,6 +19,8 @@ HELP_SCOUT_APP_ID=fake_app_idHELP_SCOUT_APP_SECRET=fake_app_secretHELP_SCOUT_SIGNATURE_KEY=fake_signature_keyHELP_SCOUT_VAULT_KEY=ym9ZQg0KPNGCH3C2eD5y6KpL0tFzUqAhwxQO6uEv/ZM=+POSTMARK_WEBHOOK_USERNAME=fake_webhook_username+POSTMARK_WEBHOOK_PASSWORD=fake_webhook_passwordS3_DISABLED=falseS3_ACCESS_KEY_ID=minioadmindiff --git a/config/runtime.exs b/config/runtime.exsindex a723cc361c..c080d61015 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -257,6 +257,8 @@ paddle_vendor_id = get_var_from_path_or_env(config_dir, "PADDLE_VENDOR_ID")google_cid = get_var_from_path_or_env(config_dir, "GOOGLE_CLIENT_ID")google_secret = get_var_from_path_or_env(config_dir, "GOOGLE_CLIENT_SECRET")postmark_api_key = get_var_from_path_or_env(config_dir, "POSTMARK_API_KEY")+postmark_webhook_username = get_var_from_path_or_env(config_dir, "POSTMARK_WEBHOOK_USERNAME")+postmark_webhook_password = get_var_from_path_or_env(config_dir, "POSTMARK_WEBHOOK_PASSWORD")help_scout_app_id = get_var_from_path_or_env(config_dir, "HELP_SCOUT_APP_ID")help_scout_app_secret = get_var_from_path_or_env(config_dir, "HELP_SCOUT_APP_SECRET")help_scout_signature_key = get_var_from_path_or_env(config_dir, "HELP_SCOUT_SIGNATURE_KEY")@@ -621,6 +623,11 @@ config :plausible, Plausible.HelpScout,signature_key: help_scout_signature_key,vault_key: help_scout_vault_key+config :plausible, Plausible.Postmark,+ api_key: postmark_api_key,+ webhook_username: postmark_webhook_username,+ webhook_password: postmark_webhook_password+config :plausible, :imported,max_buffer_size: get_int_from_path_or_env(config_dir, "IMPORTED_MAX_BUFFER_SIZE", 10_000)diff --git a/lib/plausible/email_suppression.ex b/lib/plausible/email_suppression.exindex 338a336a74..4d3675f1be 100644--- a/lib/plausible/email_suppression.ex+++ b/lib/plausible/email_suppression.ex@@ -10,7 +10,14 @@ defmodule Plausible.EmailSuppression do@type t() :: %__MODULE__{}- @reasons [:hard_bounce, :bad_email_address, :blocked, :spam_complaint, :manual]+ @reasons [+ :hard_bounce,+ :bad_email_address,+ :blocked,+ :spam_complaint,+ :spam_notification,+ :manual+ ]@sources [:webhook, :backfill, :manual]schema "email_suppressions" dodiff --git a/lib/plausible_web/controllers/api/postmark_controller.ex b/lib/plausible_web/controllers/api/postmark_controller.exnew file mode 100644index 0000000000..9ef0dc777e--- /dev/null+++ b/lib/plausible_web/controllers/api/postmark_controller.ex@@ -0,0 +1,95 @@+defmodule PlausibleWeb.Api.PostmarkController do+ @moduledoc """+ Receives Postmark's Bounce and SpamComplaint webhooks and records+ addresses in `Plausible.EmailSuppressions`.++ See https://postmarkapp.com/developer/webhooks/webhooks-overview+ """++ use PlausibleWeb, :controller++ plug :verify_basic_auth++ # https://postmarkapp.com/developer/api/bounce-api#bounce-types+ @suppressing_bounce_reasons %{+ "HardBounce" => :hard_bounce,+ "BadEmailAddress" => :bad_email_address,+ "Blocked" => :blocked,+ # SpamNotification is an automated rejection at delivery time+ # Postmark classifies it as a Bounce (not a SpamComplaint),+ # confirmed with Postmark support+ "SpamNotification" => :spam_notification+ }++ def webhook(conn, %{"RecordType" => "Bounce"} = params) do+ case Map.fetch(@suppressing_bounce_reasons, params["Type"]) do+ {:ok, reason} ->+ params+ |> suppression_attrs()+ |> Map.put(:reason, reason)+ |> Plausible.EmailSuppressions.create_from_bounce()+ |> log_on_error(params)++ :error ->+ :ignored+ end++ ok(conn)+ end++ def webhook(conn, %{"RecordType" => "SpamComplaint"} = params) do+ params+ |> suppression_attrs()+ |> Plausible.EmailSuppressions.create_from_spam_complaint()+ |> log_on_error(params)++ ok(conn)+ end++ def webhook(conn, params) do+ Sentry.capture_message("Received unexpected Postmark webhook record type",+ extra: %{record_type: params["RecordType"], params: params}+ )++ ok(conn)+ end++ defp suppression_attrs(params) do+ %{+ email: params["Email"],+ source: :webhook,+ postmark_bounce_id: params["ID"],+ postmark_inactive: params["Inactive"] || false,+ can_activate: params["CanActivate"] || false,+ details: params["Details"]+ }+ end++ defp log_on_error({:ok, _suppression}, _params), do: :ok++ defp log_on_error({:error, changeset}, params) do+ Sentry.capture_message("Failed to record Postmark suppression",+ extra: %{+ email: params["Email"],+ record_type: params["RecordType"],+ errors: inspect(changeset.errors)+ }+ )+ end++ defp ok(conn), do: json(conn, %{})++ defp verify_basic_auth(conn, _opts) do+ config = Application.get_env(:plausible, Plausible.Postmark, [])++ username =+ Keyword.get(config, :webhook_username) ||+ raise "POSTMARK_WEBHOOK_USERNAME is not configured"++ password =+ Keyword.get(config, :webhook_password) ||+ raise "POSTMARK_WEBHOOK_PASSWORD is not configured"++ Plug.BasicAuth.basic_auth(conn, username: username, password: password)+ end+enddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex d34baef1cf..ee746d6951 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -421,6 +421,11 @@ defmodule PlausibleWeb.Router doscope [] dopipe_through :api++ on_ee do+ post "/postmark/webhook", Api.PostmarkController, :webhook+ end+post "/paddle/webhook", Api.PaddleController, :webhookget "/paddle/currency", Api.PaddleController, :currencydiff --git a/test/plausible_web/controllers/api/postmark_controller_test.exs b/test/plausible_web/controllers/api/postmark_controller_test.exsnew file mode 100644index 0000000000..89fb89f094--- /dev/null+++ b/test/plausible_web/controllers/api/postmark_controller_test.exs@@ -0,0 +1,172 @@+defmodule PlausibleWeb.Api.PostmarkControllerTest do+ use PlausibleWeb.ConnCase, async: true++ on_ee do+ alias Plausible.EmailSuppressions++ # see config/.env.test+ @webhook_username "fake_webhook_username"+ @webhook_password "fake_webhook_password"++ setup %{conn: conn} do+ conn =+ Plug.Conn.put_req_header(+ conn,+ "authorization",+ Plug.BasicAuth.encode_basic_auth(@webhook_username, @webhook_password)+ )++ {:ok, conn: conn}+ end++ @bounce_payload %{+ "RecordType" => "Bounce",+ "ID" => 692_560_173,+ "Type" => "HardBounce",+ "TypeCode" => 1,+ "Email" => "[email protected]",+ "Details" => "Unknown user",+ "Inactive" => true,+ "CanActivate" => true+ }++ @spam_complaint_payload %{+ "RecordType" => "SpamComplaint",+ "ID" => 692_560_174,+ "Type" => "SpamComplaint",+ "TypeCode" => 100_001,+ "Email" => "[email protected]",+ "Details" => "Test spam complaint details",+ "Inactive" => true,+ "CanActivate" => false+ }++ describe "authentication" do+ test "rejects requests without valid basic auth", %{conn: conn} do+ conn =+ conn+ |> Plug.Conn.put_req_header(+ "authorization",+ Plug.BasicAuth.encode_basic_auth("wrong", "creds")+ )+ |> post(~p"/api/postmark/webhook", @bounce_payload)++ assert conn.status == 401+ refute EmailSuppressions.suppressed?("[email protected]")+ end++ test "rejects requests with no authorization header at all", %{conn: conn} do+ conn =+ conn+ |> Plug.Conn.delete_req_header("authorization")+ |> post(~p"/api/postmark/webhook", @bounce_payload)++ assert conn.status == 401+ end+ end++ describe "Bounce webhook" do+ test "suppresses on a hard bounce", %{conn: conn} do+ conn = post(conn, ~p"/api/postmark/webhook", @bounce_payload)++ assert json_response(conn, 200) == %{}+ assert EmailSuppressions.suppressed?("[email protected]")+ end++ test "suppresses on a bad email address", %{conn: conn} do+ payload = %{@bounce_payload | "Type" => "BadEmailAddress"}+ conn = post(conn, ~p"/api/postmark/webhook", payload)++ assert json_response(conn, 200) == %{}+ assert EmailSuppressions.suppressed?("[email protected]")+ end++ test "suppresses on an ISP block", %{conn: conn} do+ payload = %{@bounce_payload | "Type" => "Blocked"}+ conn = post(conn, ~p"/api/postmark/webhook", payload)++ assert json_response(conn, 200) == %{}+ assert EmailSuppressions.suppressed?("[email protected]")+ end++ test "suppresses on a gateway spam-filter rejection (SpamNotification)", %{conn: conn} do+ payload = %{@bounce_payload | "Type" => "SpamNotification"}+ conn = post(conn, ~p"/api/postmark/webhook", payload)++ assert json_response(conn, 200) == %{}+ assert EmailSuppressions.suppressed?("[email protected]")++ suppression =+ Plausible.Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")++ assert suppression.reason == :spam_notification+ end++ test "ignores a transient/soft bounce", %{conn: conn} do+ payload = %{@bounce_payload | "Type" => "Transient"}+ conn = post(conn, ~p"/api/postmark/webhook", payload)++ assert json_response(conn, 200) == %{}+ refute EmailSuppressions.suppressed?("[email protected]")+ end++ test "records the Postmark bounce details", %{conn: conn} do+ post(conn, ~p"/api/postmark/webhook", @bounce_payload)++ suppression =+ Plausible.Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")++ assert suppression.reason == :hard_bounce+ assert suppression.source == :webhook+ assert suppression.postmark_bounce_id == 692_560_173+ assert suppression.postmark_inactive == true+ assert suppression.can_activate == true+ assert suppression.details == "Unknown user"+ end++ test "still acknowledges the webhook when the payload can't be persisted", %{conn: conn} do+ payload = Map.delete(@bounce_payload, "Email")+ conn = post(conn, ~p"/api/postmark/webhook", payload)++ assert json_response(conn, 200) == %{}+ assert Plausible.Repo.aggregate(Plausible.EmailSuppression, :count) == 0+ end+ end++ describe "SpamComplaint webhook" do+ test "suppresses the complaining address", %{conn: conn} do+ conn = post(conn, ~p"/api/postmark/webhook", @spam_complaint_payload)++ assert json_response(conn, 200) == %{}+ assert EmailSuppressions.suppressed?("[email protected]")++ suppression =+ Plausible.Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")++ assert suppression.reason == :spam_complaint+ end+ end++ describe "other webhook types" do+ setup %{test_pid: test_pid} do+ Plausible.Test.Support.Sentry.setup(test_pid)+ end++ test "acknowledges but ignores unhandled record types, reporting to Sentry", %{conn: conn} do+ conn =+ post(conn, ~p"/api/postmark/webhook", %{+ "RecordType" => "Delivery",+ "Email" => "[email protected]"+ })++ assert json_response(conn, 200) == %{}+ refute EmailSuppressions.suppressed?("[email protected]")++ assert [report] = Sentry.Test.pop_sentry_reports()+ assert report.message.formatted == "Received unexpected Postmark webhook record type"+ assert report.extra.record_type == "Delivery"+ assert report.extra.params["Email"] == "[email protected]"+ end+ end+ end+end