Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Implement email suppressions backfill via Postmark API (#6695)

Adam562b2239de19
diff --git a/config/test.exs b/config/test.exs
index 17f97b1398..c6d1369eee 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -52,6 +52,12 @@ config :plausible, Plausible.HelpScout,
plug: {Req.Test, Plausible.HelpScout}
]
+config :plausible, Plausible.Postmark,
+ req_opts: [
+ plug: {Req.Test, Plausible.Postmark},
+ retry: false
+ ]
+
config :plausible, Plausible.InstallationSupport.Checks.Detection,
req_opts: [
plug: {Req.Test, Plausible.InstallationSupport.Checks.Detection}
diff --git a/lib/plausible/postmark.ex b/lib/plausible/postmark.ex
new file mode 100644
index 0000000000..fe4baefad4
--- /dev/null
+++ b/lib/plausible/postmark.ex
@@ -0,0 +1,193 @@
+defmodule Plausible.Postmark do
+ @moduledoc """
+ Minimal client for the parts of Postmark's HTTP API
+
+ See: https://postmarkapp.com/developer/api/bounce-api
+ """
+
+ require Logger
+
+ alias Plausible.EmailSuppressions
+
+ @base_api_url "https://api.postmarkapp.com"
+
+ # https://postmarkapp.com/developer/api/bounce-api#bounce-types
+ @suppressing_bounce_types %{
+ "HardBounce" => :hard_bounce,
+ "BadEmailAddress" => :bad_email_address,
+ "Blocked" => :blocked,
+ "SpamNotification" => :spam_notification
+ }
+
+ @spec suppressing_reason(String.t()) :: {:ok, atom()} | :error
+ def suppressing_reason(type), do: Map.fetch(@suppressing_bounce_types, type)
+
+ @spec suppressing_bounce_types() :: [String.t()]
+ def suppressing_bounce_types(), do: Map.keys(@suppressing_bounce_types)
+
+ @bounces_page_size 500
+ @bounces_max_total 10_000
+
+ @doc """
+ https://postmarkapp.com/developer/api/bounce-api#get-bounces
+ """
+ @spec list_bounces(map()) :: {:ok, [map()]} | {:error, term()}
+ def list_bounces(params \\ %{}) do
+ list_bounces(params, 0, [])
+ end
+
+ defp list_bounces(params, offset, acc) do
+ query = Map.merge(params, %{count: @bounces_page_size, offset: offset})
+
+ case get("/bounces", query) do
+ {:ok, %{"Bounces" => bounces, "TotalCount" => total_count}} ->
+ acc = acc ++ bounces
+ next_offset = offset + @bounces_page_size
+
+ if bounces == [] or length(acc) >= total_count or next_offset >= @bounces_max_total do
+ {:ok, acc}
+ else
+ list_bounces(params, next_offset, acc)
+ end
+
+ {:error, _} = error ->
+ error
+ end
+ end
+
+ @doc """
+ One-off backfill of e-mail suppressions from Postmark's Bounce API.
+
+ Postmark returns one record per bounce *event*, not per address, so the
+ same address can show up more than once (e.g. hard-bounced once and blocked separately).
+ Reported `processed` (raw events fetched) and `distinct_addresses` (addresses
+ actually upserted, one row each) can therefore differ.
+
+ Postmark doesn't document a sort order for these, so we employ
+ our own winner picking mechanism (see `pick_winner/1`).
+ """
+ @spec backfill_suppressions() :: %{
+ processed: non_neg_integer(),
+ distinct_addresses: non_neg_integer(),
+ upsert_errors: non_neg_integer(),
+ fetch_errors: non_neg_integer()
+ }
+ def backfill_suppressions() do
+ types = suppressing_bounce_types() ++ ["SpamComplaint"]
+
+ {fetch_errors, events} =
+ Enum.reduce(types, {0, []}, fn type, {fetch_errors, events} ->
+ case fetch_type(type) do
+ {:error, _reason} -> {fetch_errors + 1, events}
+ {:ok, type_events} -> {fetch_errors, events ++ type_events}
+ end
+ end)
+
+ winners =
+ events
+ |> Enum.group_by(fn {_type, bounce} -> String.downcase(bounce["Email"]) end)
+ |> Map.values()
+ |> Enum.map(&pick_winner/1)
+
+ {:ok, upsert_results} =
+ Plausible.Repo.transact(fn ->
+ {:ok, Enum.map(winners, fn {type, bounce} -> upsert_suppression(bounce, type) end)}
+ end)
+
+ %{
+ processed: length(events),
+ distinct_addresses: length(winners),
+ upsert_errors: Enum.count(upsert_results, &match?({:error, _}, &1)),
+ fetch_errors: fetch_errors
+ }
+ end
+
+ defp fetch_type(type) do
+ case list_bounces(%{type: type}) do
+ {:ok, bounces} ->
+ {:ok, Enum.map(bounces, &{type, &1})}
+
+ {:error, reason} ->
+ Logger.error(
+ "Failed to fetch Postmark bounces for backfill, type=#{type}: #{inspect(reason)}"
+ )
+
+ {:error, reason}
+ end
+ end
+
+ # Spam complaint always wins regardless of timing.
+ # Otherwise, the most recent event wins, since it best reflects
+ # the address' current state (Postmark's `Inactive`/`CanActivate` flags
+ # can change between events for the same address).
+ defp pick_winner(events) do
+ {spam_complaints, bounces} =
+ Enum.split_with(events, fn {type, _bounce} -> type == "SpamComplaint" end)
+
+ candidates = if spam_complaints != [], do: spam_complaints, else: bounces
+
+ Enum.max_by(candidates, fn {_type, bounce} -> bounced_at(bounce) end)
+ end
+
+ defp bounced_at(bounce) do
+ with bounced_at when is_binary(bounced_at) <- bounce["BouncedAt"],
+ {:ok, datetime, _utc_offset} <- DateTime.from_iso8601(bounced_at) do
+ datetime
+ else
+ # missing/invalid = oldest
+ _ -> DateTime.from_unix!(0)
+ end
+ end
+
+ defp upsert_suppression(bounce, "SpamComplaint") do
+ bounce
+ |> suppression_attrs()
+ |> EmailSuppressions.create_from_spam_complaint()
+ end
+
+ defp upsert_suppression(bounce, type) do
+ {:ok, reason} = suppressing_reason(type)
+
+ bounce
+ |> suppression_attrs()
+ |> Map.put(:reason, reason)
+ |> EmailSuppressions.create_from_bounce()
+ end
+
+ defp suppression_attrs(bounce) do
+ %{
+ email: bounce["Email"],
+ source: :backfill,
+ postmark_bounce_id: bounce["ID"],
+ postmark_inactive: bounce["Inactive"] || false,
+ can_activate: bounce["CanActivate"] || false,
+ details: bounce["Details"]
+ }
+ end
+
+ defp get(path, params) do
+ extra_opts = Application.get_env(:plausible, __MODULE__)[:req_opts] || []
+
+ opts =
+ [
+ params: params,
+ headers: [{"accept", "application/json"}, {"x-postmark-server-token", api_key()}]
+ ]
+ |> Keyword.merge(extra_opts)
+
+ case Req.get(@base_api_url <> path, opts) do
+ {:ok, %{status: 200, body: body}} ->
+ {:ok, body}
+
+ {:ok, %{status: status, body: body}} ->
+ {:error, {:unexpected_status, status, body}}
+
+ {:error, reason} ->
+ {:error, reason}
+ end
+ end
+
+ defp api_key(), do: Keyword.fetch!(config(), :api_key)
+
+ defp config(), do: Application.fetch_env!(:plausible, __MODULE__)
+end
diff --git a/lib/plausible_web/controllers/api/postmark_controller.ex b/lib/plausible_web/controllers/api/postmark_controller.ex
index 9ef0dc777e..786264d6f5 100644
--- a/lib/plausible_web/controllers/api/postmark_controller.ex
+++ b/lib/plausible_web/controllers/api/postmark_controller.ex
@@ -10,19 +10,8 @@ defmodule PlausibleWeb.Api.PostmarkController do
plug :verify_basic_auth
- # https://postmarkapp.com/developer/api/bounce-api#bounce-types
- @suppressing_bounce_reasons %{
- "HardBounce" => :hard_bounce,
- "BadEmailAddress" => :bad_email_address,
- "Blocked" => :blocked,
- # SpamNotification is an automated rejection at delivery time
- # Postmark classifies it as a Bounce (not a SpamComplaint),
- # confirmed with Postmark support
- "SpamNotification" => :spam_notification
- }
-
def webhook(conn, %{"RecordType" => "Bounce"} = params) do
- case Map.fetch(@suppressing_bounce_reasons, params["Type"]) do
+ case Plausible.Postmark.suppressing_reason(params["Type"]) do
{:ok, reason} ->
params
|> suppression_attrs()
diff --git a/test/plausible/mailer_test.exs b/test/plausible/mailer_test.exs
index 1d4d615261..b276004601 100644
--- a/test/plausible/mailer_test.exs
+++ b/test/plausible/mailer_test.exs
@@ -33,6 +33,7 @@ defmodule Plausible.MailerTest do
describe "suppression" do
@describetag :ee_only
+ @describetag :capture_log
test "refuses to send to a suppressed address" do
user = insert(:user, email: "[email protected]")
@@ -50,6 +51,22 @@ defmodule Plausible.MailerTest do
refute_delivered_email(email)
end
+ test "refuses to send even when the outgoing address's case differs from how it's stored" do
+ {:ok, _} =
+ Plausible.EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :webhook
+ })
+
+ user = insert(:user, email: "[email protected]")
+
+ email = PlausibleWeb.Email.welcome_email(user)
+ assert {:error, :suppressed} = Plausible.Mailer.send(email)
+
+ refute_delivered_email(email)
+ end
+
test "sends normally once the address is no longer suppressed" do
user = insert(:user, email: "[email protected]")
reviewer = insert(:user)
diff --git a/test/plausible/postmark_test.exs b/test/plausible/postmark_test.exs
new file mode 100644
index 0000000000..9625d7d953
--- /dev/null
+++ b/test/plausible/postmark_test.exs
@@ -0,0 +1,323 @@
+defmodule Plausible.PostmarkTest do
+ use Plausible.DataCase, async: true
+ @moduletag :ee_only
+ @moduletag :capture_log
+
+ alias Plausible.EmailSuppressions
+ alias Plausible.Postmark
+
+ describe "suppressing_reason/1" do
+ test "maps known permanently-bad bounce types" do
+ assert Postmark.suppressing_reason("HardBounce") == {:ok, :hard_bounce}
+ assert Postmark.suppressing_reason("BadEmailAddress") == {:ok, :bad_email_address}
+ assert Postmark.suppressing_reason("Blocked") == {:ok, :blocked}
+ assert Postmark.suppressing_reason("SpamNotification") == {:ok, :spam_notification}
+ end
+
+ test "does not map transient bounce types" do
+ assert Postmark.suppressing_reason("Transient") == :error
+ assert Postmark.suppressing_reason("SpamComplaint") == :error
+ assert Postmark.suppressing_reason("SomethingUnknown") == :error
+ end
+ end
+
+ describe "suppressing_bounce_types/0" do
+ test "returns the bounce type strings suppressing_reason/1 maps" do
+ assert Enum.sort(Postmark.suppressing_bounce_types()) ==
+ Enum.sort(["HardBounce", "BadEmailAddress", "Blocked", "SpamNotification"])
+ end
+ end
+
+ describe "list_bounces/1" do
+ test "returns bounces from a single page" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+ assert conn.query_params["type"] == "HardBounce"
+ assert conn.query_params["offset"] == "0"
+
+ Req.Test.json(conn, %{
+ "TotalCount" => 2,
+ "Bounces" => [%{"Email" => "[email protected]"}, %{"Email" => "[email protected]"}]
+ })
+ end)
+
+ assert {:ok, bounces} = Postmark.list_bounces(%{type: "HardBounce"})
+ assert Enum.map(bounces, & &1["Email"]) == ["[email protected]", "[email protected]"]
+ end
+
+ test "pages until every bounce has been fetched" do
+ page_size = 500
+
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+ {offset, ""} = Integer.parse(conn.query_params["offset"])
+
+ bounces =
+ if offset == 0 do
+ List.duplicate(%{"Email" => "[email protected]"}, page_size)
+ else
+ [%{"Email" => "[email protected]"}]
+ end
+
+ Req.Test.json(conn, %{"TotalCount" => page_size + 1, "Bounces" => bounces})
+ end)
+
+ assert {:ok, bounces} = Postmark.list_bounces(%{type: "HardBounce"})
+ assert length(bounces) == page_size + 1
+ assert List.last(bounces)["Email"] == "[email protected]"
+ end
+
+ test "returns an error on a non-200 response" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn
+ |> Plug.Conn.put_status(422)
+ |> Req.Test.json(%{"Message" => "Invalid type"})
+ end)
+
+ assert {:error, {:unexpected_status, 422, _body}} = Postmark.list_bounces(%{type: "Bogus"})
+ end
+ end
+
+ describe "backfill_suppressions/0" do
+ test "upserts a suppression for every bounce and complaint found" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+
+ bounces =
+ case conn.query_params["type"] do
+ "HardBounce" -> [%{"Email" => "[email protected]", "ID" => 1}]
+ "BadEmailAddress" -> [%{"Email" => "[email protected]", "ID" => 2}]
+ "Blocked" -> [%{"Email" => "[email protected]", "ID" => 3}]
+ "SpamComplaint" -> [%{"Email" => "[email protected]", "ID" => 4}]
+ "SpamNotification" -> [%{"Email" => "[email protected]", "ID" => 5}]
+ end
+
+ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})
+ end)
+
+ assert %{processed: 5, distinct_addresses: 5, upsert_errors: 0, fetch_errors: 0} =
+ Postmark.backfill_suppressions()
+
+ assert EmailSuppressions.suppressed?("[email protected]")
+ assert EmailSuppressions.suppressed?("[email protected]")
+ assert EmailSuppressions.suppressed?("[email protected]")
+ assert EmailSuppressions.suppressed?("[email protected]")
+ assert EmailSuppressions.suppressed?("[email protected]")
+
+ hard = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")
+ assert hard.reason == :hard_bounce
+ assert hard.source == :backfill
+ assert hard.postmark_bounce_id == 1
+
+ complainer = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")
+ assert complainer.reason == :spam_complaint
+ assert complainer.source == :backfill
+
+ spam_notified = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")
+ assert spam_notified.reason == :spam_notification
+ assert spam_notified.source == :backfill
+ end
+
+ test "counts fetch failures separately" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+
+ case conn.query_params["type"] do
+ "HardBounce" ->
+ conn |> Plug.Conn.put_status(500) |> Req.Test.json(%{"Message" => "boom"})
+
+ _other ->
+ Req.Test.json(conn, %{"TotalCount" => 0, "Bounces" => []})
+ end
+ end)
+
+ assert %{processed: 0, distinct_addresses: 0, upsert_errors: 0, fetch_errors: 1} =
+ Postmark.backfill_suppressions()
+ end
+
+ test "counts distinct_addresses" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+
+ bounces =
+ case conn.query_params["type"] do
+ "HardBounce" ->
+ [
+ %{"Email" => "[email protected]", "ID" => 1},
+ %{"Email" => "[email protected]", "ID" => 2}
+ ]
+
+ "Blocked" ->
+ [%{"Email" => "[email protected]", "ID" => 3}]
+
+ _other ->
+ []
+ end
+
+ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})
+ end)
+
+ assert %{processed: 3, distinct_addresses: 1, upsert_errors: 0, fetch_errors: 0} =
+ Postmark.backfill_suppressions()
+
+ assert EmailSuppressions.suppressed?("[email protected]")
+ assert Repo.aggregate(Plausible.EmailSuppression, :count) == 1
+ end
+
+ test "counts email citext variants" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+
+ bounces =
+ case conn.query_params["type"] do
+ "HardBounce" ->
+ [%{"Email" => "[email protected]", "ID" => 1}]
+
+ "Blocked" ->
+ [%{"Email" => "[email protected]", "ID" => 2}]
+
+ _other ->
+ []
+ end
+
+ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})
+ end)
+
+ assert %{processed: 2, distinct_addresses: 1, upsert_errors: 0, fetch_errors: 0} =
+ Postmark.backfill_suppressions()
+
+ assert Repo.aggregate(Plausible.EmailSuppression, :count) == 1
+ end
+
+ test "for multiple bounce events, the most recent one wins" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+
+ bounces =
+ case conn.query_params["type"] do
+ "HardBounce" ->
+ [
+ %{
+ "Email" => "[email protected]",
+ "ID" => 1,
+ "BouncedAt" => "2026-01-01T00:00:00Z",
+ "Details" => "older hard bounce"
+ }
+ ]
+
+ "Blocked" ->
+ [
+ %{
+ "Email" => "[email protected]",
+ "ID" => 2,
+ "BouncedAt" => "2026-06-01T00:00:00Z",
+ "Details" => "newer block"
+ }
+ ]
+
+ _other ->
+ []
+ end
+
+ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})
+ end)
+
+ assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()
+
+ suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")
+ assert suppression.reason == :blocked
+ assert suppression.postmark_bounce_id == 2
+ assert suppression.details == "newer block"
+ end
+
+ test "spam complaint always wins over a bounce" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+
+ bounces =
+ case conn.query_params["type"] do
+ "HardBounce" ->
+ [
+ %{
+ "Email" => "[email protected]",
+ "ID" => 1,
+ "BouncedAt" => "2026-06-01T00:00:00Z"
+ }
+ ]
+
+ "SpamComplaint" ->
+ [
+ %{
+ "Email" => "[email protected]",
+ "ID" => 2,
+ "BouncedAt" => "2026-01-01T00:00:00Z"
+ }
+ ]
+
+ _other ->
+ []
+ end
+
+ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})
+ end)
+
+ assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()
+
+ suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")
+ assert suppression.reason == :spam_complaint
+ assert suppression.postmark_bounce_id == 2
+ end
+
+ test "the most recent spam complaint wins when there are several" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+
+ bounces =
+ case conn.query_params["type"] do
+ "SpamComplaint" ->
+ [
+ %{
+ "Email" => "[email protected]",
+ "ID" => 1,
+ "BouncedAt" => "2026-01-01T00:00:00Z"
+ },
+ %{
+ "Email" => "[email protected]",
+ "ID" => 2,
+ "BouncedAt" => "2026-06-01T00:00:00Z"
+ }
+ ]
+
+ _other ->
+ []
+ end
+
+ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})
+ end)
+
+ assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()
+
+ suppression =
+ Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")
+
+ assert suppression.postmark_bounce_id == 2
+ end
+
+ test "a missing BouncedAt does not crash the backfill" do
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+
+ bounces =
+ case conn.query_params["type"] do
+ "HardBounce" -> [%{"Email" => "[email protected]", "ID" => 1}]
+ _other -> []
+ end
+
+ Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})
+ end)
+
+ assert %{distinct_addresses: 1, upsert_errors: 0} = Postmark.backfill_suppressions()
+ assert EmailSuppressions.suppressed?("[email protected]")
+ end
+ end
+end