Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Implement email suppressions backfill via Postmark API (#6695)
Adam562b2239de19
diff --git a/config/test.exs b/config/test.exsindex 17f97b1398..c6d1369eee 100644--- a/config/test.exs+++ b/config/test.exs@@ -52,6 +52,12 @@ config :plausible, Plausible.HelpScout,plug: {Req.Test, Plausible.HelpScout}]+config :plausible, Plausible.Postmark,+ req_opts: [+ plug: {Req.Test, Plausible.Postmark},+ retry: false+ ]+config :plausible, Plausible.InstallationSupport.Checks.Detection,req_opts: [plug: {Req.Test, Plausible.InstallationSupport.Checks.Detection}diff --git a/lib/plausible/postmark.ex b/lib/plausible/postmark.exnew file mode 100644index 0000000000..fe4baefad4--- /dev/null+++ b/lib/plausible/postmark.ex@@ -0,0 +1,193 @@+defmodule Plausible.Postmark do+ @moduledoc """+ Minimal client for the parts of Postmark's HTTP API++ See: https://postmarkapp.com/developer/api/bounce-api+ """++ require Logger++ alias Plausible.EmailSuppressions++ @base_api_url "https://api.postmarkapp.com"++ # https://postmarkapp.com/developer/api/bounce-api#bounce-types+ @suppressing_bounce_types %{+ "HardBounce" => :hard_bounce,+ "BadEmailAddress" => :bad_email_address,+ "Blocked" => :blocked,+ "SpamNotification" => :spam_notification+ }++ @spec suppressing_reason(String.t()) :: {:ok, atom()} | :error+ def suppressing_reason(type), do: Map.fetch(@suppressing_bounce_types, type)++ @spec suppressing_bounce_types() :: [String.t()]+ def suppressing_bounce_types(), do: Map.keys(@suppressing_bounce_types)++ @bounces_page_size 500+ @bounces_max_total 10_000++ @doc """+ https://postmarkapp.com/developer/api/bounce-api#get-bounces+ """+ @spec list_bounces(map()) :: {:ok, [map()]} | {:error, term()}+ def list_bounces(params \\ %{}) do+ list_bounces(params, 0, [])+ end++ defp list_bounces(params, offset, acc) do+ query = Map.merge(params, %{count: @bounces_page_size, offset: offset})++ case get("/bounces", query) do+ {:ok, %{"Bounces" => bounces, "TotalCount" => total_count}} ->+ acc = acc ++ bounces+ next_offset = offset + @bounces_page_size++ if bounces == [] or length(acc) >= total_count or next_offset >= @bounces_max_total do+ {:ok, acc}+ else+ list_bounces(params, next_offset, acc)+ end++ {:error, _} = error ->+ error+ end+ end++ @doc """+ One-off backfill of e-mail suppressions from Postmark's Bounce API.++ Postmark returns one record per bounce *event*, not per address, so the+ same address can show up more than once (e.g. hard-bounced once and blocked separately).+ Reported `processed` (raw events fetched) and `distinct_addresses` (addresses+ actually upserted, one row each) can therefore differ.++ Postmark doesn't document a sort order for these, so we employ+ our own winner picking mechanism (see `pick_winner/1`).+ """+ @spec backfill_suppressions() :: %{+ processed: non_neg_integer(),+ distinct_addresses: non_neg_integer(),+ upsert_errors: non_neg_integer(),+ fetch_errors: non_neg_integer()+ }+ def backfill_suppressions() do+ types = suppressing_bounce_types() ++ ["SpamComplaint"]++ {fetch_errors, events} =+ Enum.reduce(types, {0, []}, fn type, {fetch_errors, events} ->+ case fetch_type(type) do+ {:error, _reason} -> {fetch_errors + 1, events}+ {:ok, type_events} -> {fetch_errors, events ++ type_events}+ end+ end)++ winners =+ events+ |> Enum.group_by(fn {_type, bounce} -> String.downcase(bounce["Email"]) end)+ |> Map.values()+ |> Enum.map(&pick_winner/1)++ {:ok, upsert_results} =+ Plausible.Repo.transact(fn ->+ {:ok, Enum.map(winners, fn {type, bounce} -> upsert_suppression(bounce, type) end)}+ end)++ %{+ processed: length(events),+ distinct_addresses: length(winners),+ upsert_errors: Enum.count(upsert_results, &match?({:error, _}, &1)),+ fetch_errors: fetch_errors+ }+ end++ defp fetch_type(type) do+ case list_bounces(%{type: type}) do+ {:ok, bounces} ->+ {:ok, Enum.map(bounces, &{type, &1})}++ {:error, reason} ->+ Logger.error(+ "Failed to fetch Postmark bounces for backfill, type=#{type}: #{inspect(reason)}"+ )++ {:error, reason}+ end+ end++ # Spam complaint always wins regardless of timing.+ # Otherwise, the most recent event wins, since it best reflects+ # the address' current state (Postmark's `Inactive`/`CanActivate` flags+ # can change between events for the same address).+ defp pick_winner(events) do+ {spam_complaints, bounces} =+ Enum.split_with(events, fn {type, _bounce} -> type == "SpamComplaint" end)++ candidates = if spam_complaints != [], do: spam_complaints, else: bounces++ Enum.max_by(candidates, fn {_type, bounce} -> bounced_at(bounce) end)+ end++ defp bounced_at(bounce) do+ with bounced_at when is_binary(bounced_at) <- bounce["BouncedAt"],+ {:ok, datetime, _utc_offset} <- DateTime.from_iso8601(bounced_at) do+ datetime+ else+ # missing/invalid = oldest+ _ -> DateTime.from_unix!(0)+ end+ end++ defp upsert_suppression(bounce, "SpamComplaint") do+ bounce+ |> suppression_attrs()+ |> EmailSuppressions.create_from_spam_complaint()+ end++ defp upsert_suppression(bounce, type) do+ {:ok, reason} = suppressing_reason(type)++ bounce+ |> suppression_attrs()+ |> Map.put(:reason, reason)+ |> EmailSuppressions.create_from_bounce()+ end++ defp suppression_attrs(bounce) do+ %{+ email: bounce["Email"],+ source: :backfill,+ postmark_bounce_id: bounce["ID"],+ postmark_inactive: bounce["Inactive"] || false,+ can_activate: bounce["CanActivate"] || false,+ details: bounce["Details"]+ }+ end++ defp get(path, params) do+ extra_opts = Application.get_env(:plausible, __MODULE__)[:req_opts] || []++ opts =+ [+ params: params,+ headers: [{"accept", "application/json"}, {"x-postmark-server-token", api_key()}]+ ]+ |> Keyword.merge(extra_opts)++ case Req.get(@base_api_url <> path, opts) do+ {:ok, %{status: 200, body: body}} ->+ {:ok, body}++ {:ok, %{status: status, body: body}} ->+ {:error, {:unexpected_status, status, body}}++ {:error, reason} ->+ {:error, reason}+ end+ end++ defp api_key(), do: Keyword.fetch!(config(), :api_key)++ defp config(), do: Application.fetch_env!(:plausible, __MODULE__)+enddiff --git a/lib/plausible_web/controllers/api/postmark_controller.ex b/lib/plausible_web/controllers/api/postmark_controller.exindex 9ef0dc777e..786264d6f5 100644--- a/lib/plausible_web/controllers/api/postmark_controller.ex+++ b/lib/plausible_web/controllers/api/postmark_controller.ex@@ -10,19 +10,8 @@ defmodule PlausibleWeb.Api.PostmarkController doplug :verify_basic_auth- # https://postmarkapp.com/developer/api/bounce-api#bounce-types- @suppressing_bounce_reasons %{- "HardBounce" => :hard_bounce,- "BadEmailAddress" => :bad_email_address,- "Blocked" => :blocked,- # SpamNotification is an automated rejection at delivery time- # Postmark classifies it as a Bounce (not a SpamComplaint),- # confirmed with Postmark support- "SpamNotification" => :spam_notification- }-def webhook(conn, %{"RecordType" => "Bounce"} = params) do- case Map.fetch(@suppressing_bounce_reasons, params["Type"]) do+ case Plausible.Postmark.suppressing_reason(params["Type"]) do{:ok, reason} ->params|> suppression_attrs()diff --git a/test/plausible/mailer_test.exs b/test/plausible/mailer_test.exsindex 1d4d615261..b276004601 100644--- a/test/plausible/mailer_test.exs+++ b/test/plausible/mailer_test.exs@@ -33,6 +33,7 @@ defmodule Plausible.MailerTest dodescribe "suppression" do@describetag :ee_only+ @describetag :capture_logtest "refuses to send to a suppressed address" douser = insert(:user, email: "[email protected]")@@ -50,6 +51,22 @@ defmodule Plausible.MailerTest dorefute_delivered_email(email)end+ test "refuses to send even when the outgoing address's case differs from how it's stored" do+ {:ok, _} =+ Plausible.EmailSuppressions.create_from_bounce(%{+ email: "[email protected]",+ reason: :hard_bounce,+ source: :webhook+ })++ user = insert(:user, email: "[email protected]")++ email = PlausibleWeb.Email.welcome_email(user)+ assert {:error, :suppressed} = Plausible.Mailer.send(email)++ refute_delivered_email(email)+ end+test "sends normally once the address is no longer suppressed" douser = insert(:user, email: "[email protected]")reviewer = insert(:user)diff --git a/test/plausible/postmark_test.exs b/test/plausible/postmark_test.exsnew file mode 100644index 0000000000..9625d7d953--- /dev/null+++ b/test/plausible/postmark_test.exs@@ -0,0 +1,323 @@+defmodule Plausible.PostmarkTest do+ use Plausible.DataCase, async: true+ @moduletag :ee_only+ @moduletag :capture_log++ alias Plausible.EmailSuppressions+ alias Plausible.Postmark++ describe "suppressing_reason/1" do+ test "maps known permanently-bad bounce types" do+ assert Postmark.suppressing_reason("HardBounce") == {:ok, :hard_bounce}+ assert Postmark.suppressing_reason("BadEmailAddress") == {:ok, :bad_email_address}+ assert Postmark.suppressing_reason("Blocked") == {:ok, :blocked}+ assert Postmark.suppressing_reason("SpamNotification") == {:ok, :spam_notification}+ end++ test "does not map transient bounce types" do+ assert Postmark.suppressing_reason("Transient") == :error+ assert Postmark.suppressing_reason("SpamComplaint") == :error+ assert Postmark.suppressing_reason("SomethingUnknown") == :error+ end+ end++ describe "suppressing_bounce_types/0" do+ test "returns the bounce type strings suppressing_reason/1 maps" do+ assert Enum.sort(Postmark.suppressing_bounce_types()) ==+ Enum.sort(["HardBounce", "BadEmailAddress", "Blocked", "SpamNotification"])+ end+ end++ describe "list_bounces/1" do+ test "returns bounces from a single page" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)+ assert conn.query_params["type"] == "HardBounce"+ assert conn.query_params["offset"] == "0"++ Req.Test.json(conn, %{+ "TotalCount" => 2,+ "Bounces" => [%{"Email" => "[email protected]"}, %{"Email" => "[email protected]"}]+ })+ end)++ assert {:ok, bounces} = Postmark.list_bounces(%{type: "HardBounce"})+ assert Enum.map(bounces, & &1["Email"]) == ["[email protected]", "[email protected]"]+ end++ test "pages until every bounce has been fetched" do+ page_size = 500++ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)+ {offset, ""} = Integer.parse(conn.query_params["offset"])++ bounces =+ if offset == 0 do+ List.duplicate(%{"Email" => "[email protected]"}, page_size)+ else+ [%{"Email" => "[email protected]"}]+ end++ Req.Test.json(conn, %{"TotalCount" => page_size + 1, "Bounces" => bounces})+ end)++ assert {:ok, bounces} = Postmark.list_bounces(%{type: "HardBounce"})+ assert length(bounces) == page_size + 1+ assert List.last(bounces)["Email"] == "[email protected]"+ end++ test "returns an error on a non-200 response" do+ Req.Test.stub(Postmark, fn conn ->+ conn+ |> Plug.Conn.put_status(422)+ |> Req.Test.json(%{"Message" => "Invalid type"})+ end)++ assert {:error, {:unexpected_status, 422, _body}} = Postmark.list_bounces(%{type: "Bogus"})+ end+ end++ describe "backfill_suppressions/0" do+ test "upserts a suppression for every bounce and complaint found" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)++ bounces =+ case conn.query_params["type"] do+ "HardBounce" -> [%{"Email" => "[email protected]", "ID" => 1}]+ "BadEmailAddress" -> [%{"Email" => "[email protected]", "ID" => 2}]+ "Blocked" -> [%{"Email" => "[email protected]", "ID" => 3}]+ "SpamComplaint" -> [%{"Email" => "[email protected]", "ID" => 4}]+ "SpamNotification" -> [%{"Email" => "[email protected]", "ID" => 5}]+ end++ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ end)++ assert %{processed: 5, distinct_addresses: 5, upsert_errors: 0, fetch_errors: 0} =+ Postmark.backfill_suppressions()++ assert EmailSuppressions.suppressed?("[email protected]")+ assert EmailSuppressions.suppressed?("[email protected]")+ assert EmailSuppressions.suppressed?("[email protected]")+ assert EmailSuppressions.suppressed?("[email protected]")+ assert EmailSuppressions.suppressed?("[email protected]")++ hard = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert hard.reason == :hard_bounce+ assert hard.source == :backfill+ assert hard.postmark_bounce_id == 1++ complainer = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert complainer.reason == :spam_complaint+ assert complainer.source == :backfill++ spam_notified = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert spam_notified.reason == :spam_notification+ assert spam_notified.source == :backfill+ end++ test "counts fetch failures separately" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)++ case conn.query_params["type"] do+ "HardBounce" ->+ conn |> Plug.Conn.put_status(500) |> Req.Test.json(%{"Message" => "boom"})++ _other ->+ Req.Test.json(conn, %{"TotalCount" => 0, "Bounces" => []})+ end+ end)++ assert %{processed: 0, distinct_addresses: 0, upsert_errors: 0, fetch_errors: 1} =+ Postmark.backfill_suppressions()+ end++ test "counts distinct_addresses" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)++ bounces =+ case conn.query_params["type"] do+ "HardBounce" ->+ [+ %{"Email" => "[email protected]", "ID" => 1},+ %{"Email" => "[email protected]", "ID" => 2}+ ]++ "Blocked" ->+ [%{"Email" => "[email protected]", "ID" => 3}]++ _other ->+ []+ end++ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ end)++ assert %{processed: 3, distinct_addresses: 1, upsert_errors: 0, fetch_errors: 0} =+ Postmark.backfill_suppressions()++ assert EmailSuppressions.suppressed?("[email protected]")+ assert Repo.aggregate(Plausible.EmailSuppression, :count) == 1+ end++ test "counts email citext variants" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)++ bounces =+ case conn.query_params["type"] do+ "HardBounce" ->+ [%{"Email" => "[email protected]", "ID" => 1}]++ "Blocked" ->+ [%{"Email" => "[email protected]", "ID" => 2}]++ _other ->+ []+ end++ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ end)++ assert %{processed: 2, distinct_addresses: 1, upsert_errors: 0, fetch_errors: 0} =+ Postmark.backfill_suppressions()++ assert Repo.aggregate(Plausible.EmailSuppression, :count) == 1+ end++ test "for multiple bounce events, the most recent one wins" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)++ bounces =+ case conn.query_params["type"] do+ "HardBounce" ->+ [+ %{+ "Email" => "[email protected]",+ "ID" => 1,+ "BouncedAt" => "2026-01-01T00:00:00Z",+ "Details" => "older hard bounce"+ }+ ]++ "Blocked" ->+ [+ %{+ "Email" => "[email protected]",+ "ID" => 2,+ "BouncedAt" => "2026-06-01T00:00:00Z",+ "Details" => "newer block"+ }+ ]++ _other ->+ []+ end++ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ end)++ assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()++ suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert suppression.reason == :blocked+ assert suppression.postmark_bounce_id == 2+ assert suppression.details == "newer block"+ end++ test "spam complaint always wins over a bounce" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)++ bounces =+ case conn.query_params["type"] do+ "HardBounce" ->+ [+ %{+ "Email" => "[email protected]",+ "ID" => 1,+ "BouncedAt" => "2026-06-01T00:00:00Z"+ }+ ]++ "SpamComplaint" ->+ [+ %{+ "Email" => "[email protected]",+ "ID" => 2,+ "BouncedAt" => "2026-01-01T00:00:00Z"+ }+ ]++ _other ->+ []+ end++ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ end)++ assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()++ suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert suppression.reason == :spam_complaint+ assert suppression.postmark_bounce_id == 2+ end++ test "the most recent spam complaint wins when there are several" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)++ bounces =+ case conn.query_params["type"] do+ "SpamComplaint" ->+ [+ %{+ "Email" => "[email protected]",+ "ID" => 1,+ "BouncedAt" => "2026-01-01T00:00:00Z"+ },+ %{+ "Email" => "[email protected]",+ "ID" => 2,+ "BouncedAt" => "2026-06-01T00:00:00Z"+ }+ ]++ _other ->+ []+ end++ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ end)++ assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()++ suppression =+ Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")++ assert suppression.postmark_bounce_id == 2+ end++ test "a missing BouncedAt does not crash the backfill" do+ Req.Test.stub(Postmark, fn conn ->+ conn = Plug.Conn.fetch_query_params(conn)++ bounces =+ case conn.query_params["type"] do+ "HardBounce" -> [%{"Email" => "[email protected]", "ID" => 1}]+ _other -> []+ end++ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ end)++ assert %{distinct_addresses: 1, upsert_errors: 0} = Postmark.backfill_suppressions()+ assert EmailSuppressions.suppressed?("[email protected]")+ end+ end+end