Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Postmark: use suppression API to backfill email_suppresions (#6703)
Adam3dfc2720024f
diff --git a/lib/plausible/email_suppression.ex b/lib/plausible/email_suppression.exindex 4d3675f1be..74db0a2edc 100644--- a/lib/plausible/email_suppression.ex+++ b/lib/plausible/email_suppression.ex@@ -16,6 +16,7 @@ defmodule Plausible.EmailSuppression do:blocked,:spam_complaint,:spam_notification,+ :unsubscribe,:manual]@sources [:webhook, :backfill, :manual]diff --git a/lib/plausible/postmark.ex b/lib/plausible/postmark.exindex fe4baefad4..dd3cd7a179 100644--- a/lib/plausible/postmark.ex+++ b/lib/plausible/postmark.ex@@ -2,7 +2,8 @@ defmodule Plausible.Postmark do@moduledoc """Minimal client for the parts of Postmark's HTTP API- See: https://postmarkapp.com/developer/api/bounce-api+ See: https://postmarkapp.com/developer/api/bounce-api and+ https://postmarkapp.com/developer/api/suppressions-api"""require Logger@@ -22,49 +23,19 @@ defmodule Plausible.Postmark do@spec suppressing_reason(String.t()) :: {:ok, atom()} | :errordef suppressing_reason(type), do: Map.fetch(@suppressing_bounce_types, type)- @spec suppressing_bounce_types() :: [String.t()]- def suppressing_bounce_types(), do: Map.keys(@suppressing_bounce_types)+ @streams ["outbound", "priority"]- @bounces_page_size 500- @bounces_max_total 10_000-- @doc """- https://postmarkapp.com/developer/api/bounce-api#get-bounces- """- @spec list_bounces(map()) :: {:ok, [map()]} | {:error, term()}- def list_bounces(params \\ %{}) do- list_bounces(params, 0, [])- end-- defp list_bounces(params, offset, acc) do- query = Map.merge(params, %{count: @bounces_page_size, offset: offset})-- case get("/bounces", query) do- {:ok, %{"Bounces" => bounces, "TotalCount" => total_count}} ->- acc = acc ++ bounces- next_offset = offset + @bounces_page_size-- if bounces == [] or length(acc) >= total_count or next_offset >= @bounces_max_total do- {:ok, acc}- else- list_bounces(params, next_offset, acc)- end-- {:error, _} = error ->- error- end- end+ @suppression_reasons %{+ "HardBounce" => :hard_bounce,+ "SpamComplaint" => :spam_complaint+ }@doc """- One-off backfill of e-mail suppressions from Postmark's Bounce API.+ One-off backfill of e-mail suppressions from Postmark's Suppressions API,+ across every message stream we send from.- Postmark returns one record per bounce *event*, not per address, so the- same address can show up more than once (e.g. hard-bounced once and blocked separately).- Reported `processed` (raw events fetched) and `distinct_addresses` (addresses- actually upserted, one row each) can therefore differ.-- Postmark doesn't document a sort order for these, so we employ- our own winner picking mechanism (see `pick_winner/1`).+ If an address suppressed on more than one stream is reported once,+ we pick the winner ourselves - see: `pick_winner/1`"""@spec backfill_suppressions() :: %{processed: non_neg_integer(),@@ -73,65 +44,61 @@ defmodule Plausible.Postmark dofetch_errors: non_neg_integer()}def backfill_suppressions() do- types = suppressing_bounce_types() ++ ["SpamComplaint"]-- {fetch_errors, events} =- Enum.reduce(types, {0, []}, fn type, {fetch_errors, events} ->- case fetch_type(type) do- {:error, _reason} -> {fetch_errors + 1, events}- {:ok, type_events} -> {fetch_errors, events ++ type_events}+ {fetch_errors, suppressions} =+ Enum.reduce(@streams, {0, []}, fn stream, {fetch_errors, suppressions} ->+ case fetch_stream(stream) do+ {:error, _reason} -> {fetch_errors + 1, suppressions}+ {:ok, stream_suppressions} -> {fetch_errors, suppressions ++ stream_suppressions}endend)winners =- events- |> Enum.group_by(fn {_type, bounce} -> String.downcase(bounce["Email"]) end)+ suppressions+ |> Enum.group_by(&String.downcase(&1["EmailAddress"]))|> Map.values()|> Enum.map(&pick_winner/1){:ok, upsert_results} =Plausible.Repo.transact(fn ->- {:ok, Enum.map(winners, fn {type, bounce} -> upsert_suppression(bounce, type) end)}+ {:ok, Enum.map(winners, &upsert_suppression/1)}end)%{- processed: length(events),+ processed: length(suppressions),distinct_addresses: length(winners),upsert_errors: Enum.count(upsert_results, &match?({:error, _}, &1)),fetch_errors: fetch_errors}end- defp fetch_type(type) do- case list_bounces(%{type: type}) do- {:ok, bounces} ->- {:ok, Enum.map(bounces, &{type, &1})}+ defp fetch_stream(stream) do+ case get("/message-streams/#{stream}/suppressions/dump", %{}) do+ {:ok, %{"Suppressions" => suppressions}} ->+ {:ok, suppressions}{:error, reason} ->Logger.error(- "Failed to fetch Postmark bounces for backfill, type=#{type}: #{inspect(reason)}"+ "Failed to fetch Postmark suppressions for backfill, stream=#{stream}: #{inspect(reason)}"){:error, reason}endend- # Spam complaint always wins regardless of timing.- # Otherwise, the most recent event wins, since it best reflects- # the address' current state (Postmark's `Inactive`/`CanActivate` flags- # can change between events for the same address).- defp pick_winner(events) do- {spam_complaints, bounces} =- Enum.split_with(events, fn {type, _bounce} -> type == "SpamComplaint" end)+ # Spam complaint always wins regardless of timing. Otherwise, the most+ # recent one wins, since it best reflects the address' current state.+ defp pick_winner(entries) do+ {spam_complaints, others} =+ Enum.split_with(entries, &(&1["SuppressionReason"] == "SpamComplaint"))- candidates = if spam_complaints != [], do: spam_complaints, else: bounces+ candidates = if spam_complaints != [], do: spam_complaints, else: others- Enum.max_by(candidates, fn {_type, bounce} -> bounced_at(bounce) end)+ Enum.max_by(candidates, &created_at/1)end- defp bounced_at(bounce) do- with bounced_at when is_binary(bounced_at) <- bounce["BouncedAt"],- {:ok, datetime, _utc_offset} <- DateTime.from_iso8601(bounced_at) do+ defp created_at(entry) do+ with created_at when is_binary(created_at) <- entry["CreatedAt"],+ {:ok, datetime, _utc_offset} <- DateTime.from_iso8601(created_at) dodatetimeelse# missing/invalid = oldest@@ -139,30 +106,28 @@ defmodule Plausible.Postmark doendend- defp upsert_suppression(bounce, "SpamComplaint") do- bounce- |> suppression_attrs()- |> EmailSuppressions.create_from_spam_complaint()+ defp upsert_suppression(entry) do+ %{+ email: entry["EmailAddress"],+ source: :backfill,+ reason: suppression_reason(entry),+ details: "Postmark suppression (origin: #{entry["Origin"]})"+ }+ |> EmailSuppressions.create_from_bounce()end- defp upsert_suppression(bounce, type) do- {:ok, reason} = suppressing_reason(type)-- bounce- |> suppression_attrs()- |> Map.put(:reason, reason)- |> EmailSuppressions.create_from_bounce()+ # Postmark's API has no distinct "Unsubscribe" value - the dashboard shows+ # that label for a ManualSuppression whose Origin is the recipient themself.+ defp suppression_reason(%{"SuppressionReason" => "ManualSuppression", "Origin" => "Recipient"}) do+ :unsubscribeend- defp suppression_attrs(bounce) do- %{- email: bounce["Email"],- source: :backfill,- postmark_bounce_id: bounce["ID"],- postmark_inactive: bounce["Inactive"] || false,- can_activate: bounce["CanActivate"] || false,- details: bounce["Details"]- }+ defp suppression_reason(%{"SuppressionReason" => "ManualSuppression"}), do: :manual++ defp suppression_reason(%{"SuppressionReason" => reason}) do+ # :manual as a fallback: Postmark only documents HardBounce/SpamComplaint/+ # ManualSuppression, but an unrecognized reason shouldn't crash the backfill+ Map.get(@suppression_reasons, reason, :manual)enddefp get(path, params) dodiff --git a/test/plausible/postmark_test.exs b/test/plausible/postmark_test.exsindex 9625d7d953..862544ddd7 100644--- a/test/plausible/postmark_test.exs+++ b/test/plausible/postmark_test.exs@@ -21,113 +21,119 @@ defmodule Plausible.PostmarkTest doendend- describe "suppressing_bounce_types/0" do- test "returns the bounce type strings suppressing_reason/1 maps" do- assert Enum.sort(Postmark.suppressing_bounce_types()) ==- Enum.sort(["HardBounce", "BadEmailAddress", "Blocked", "SpamNotification"])- end- end-- describe "list_bounces/1" do- test "returns bounces from a single page" do+ describe "backfill_suppressions/0" do+ test "upserts a suppression for every entry found, across both streams" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)- assert conn.query_params["type"] == "HardBounce"- assert conn.query_params["offset"] == "0"-- Req.Test.json(conn, %{- "TotalCount" => 2,- "Bounces" => [%{"Email" => "[email protected]"}, %{"Email" => "[email protected]"}]- })+ entries =+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->+ [+ %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "HardBounce",+ "Origin" => "Recipient"+ },+ %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "SpamComplaint",+ "Origin" => "Recipient"+ }+ ]++ "/message-streams/priority/suppressions/dump" ->+ [+ %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "ManualSuppression",+ "Origin" => "Admin"+ }+ ]+ end++ Req.Test.json(conn, %{"Suppressions" => entries})end)- assert {:ok, bounces} = Postmark.list_bounces(%{type: "HardBounce"})- assert Enum.map(bounces, & &1["Email"]) == ["[email protected]", "[email protected]"]- end+ assert %{processed: 3, distinct_addresses: 3, upsert_errors: 0, fetch_errors: 0} =+ Postmark.backfill_suppressions()- test "pages until every bounce has been fetched" do- page_size = 500+ assert EmailSuppressions.suppressed?("[email protected]")+ assert EmailSuppressions.suppressed?("[email protected]")+ assert EmailSuppressions.suppressed?("[email protected]")- Req.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)- {offset, ""} = Integer.parse(conn.query_params["offset"])-- bounces =- if offset == 0 do- List.duplicate(%{"Email" => "[email protected]"}, page_size)- else- [%{"Email" => "[email protected]"}]- end+ hard = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert hard.reason == :hard_bounce+ assert hard.source == :backfill+ assert hard.details == "Postmark suppression (origin: Recipient)"- Req.Test.json(conn, %{"TotalCount" => page_size + 1, "Bounces" => bounces})- end)+ complainer = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert complainer.reason == :spam_complaint- assert {:ok, bounces} = Postmark.list_bounces(%{type: "HardBounce"})- assert length(bounces) == page_size + 1- assert List.last(bounces)["Email"] == "[email protected]"+ manual = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert manual.reason == :manualend- test "returns an error on a non-200 response" do+ test "maps a recipient-originated ManualSuppression to :unsubscribe" doReq.Test.stub(Postmark, fn conn ->- conn- |> Plug.Conn.put_status(422)- |> Req.Test.json(%{"Message" => "Invalid type"})+ entries =+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->+ [+ %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "ManualSuppression",+ "Origin" => "Recipient"+ }+ ]++ "/message-streams/priority/suppressions/dump" ->+ []+ end++ Req.Test.json(conn, %{"Suppressions" => entries})end)- assert {:error, {:unexpected_status, 422, _body}} = Postmark.list_bounces(%{type: "Bogus"})+ assert %{upsert_errors: 0} = Postmark.backfill_suppressions()++ suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert suppression.reason == :unsubscribeend- end- describe "backfill_suppressions/0" do- test "upserts a suppression for every bounce and complaint found" do+ test "maps a customer/admin-originated ManualSuppression to :manual" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)-- bounces =- case conn.query_params["type"] do- "HardBounce" -> [%{"Email" => "[email protected]", "ID" => 1}]- "BadEmailAddress" -> [%{"Email" => "[email protected]", "ID" => 2}]- "Blocked" -> [%{"Email" => "[email protected]", "ID" => 3}]- "SpamComplaint" -> [%{"Email" => "[email protected]", "ID" => 4}]- "SpamNotification" -> [%{"Email" => "[email protected]", "ID" => 5}]+ entries =+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->+ [+ %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "ManualSuppression",+ "Origin" => "Customer"+ }+ ]++ "/message-streams/priority/suppressions/dump" ->+ []end- Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ Req.Test.json(conn, %{"Suppressions" => entries})end)- assert %{processed: 5, distinct_addresses: 5, upsert_errors: 0, fetch_errors: 0} =- Postmark.backfill_suppressions()-- assert EmailSuppressions.suppressed?("[email protected]")- assert EmailSuppressions.suppressed?("[email protected]")- assert EmailSuppressions.suppressed?("[email protected]")- assert EmailSuppressions.suppressed?("[email protected]")- assert EmailSuppressions.suppressed?("[email protected]")-- hard = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")- assert hard.reason == :hard_bounce- assert hard.source == :backfill- assert hard.postmark_bounce_id == 1+ assert %{upsert_errors: 0} = Postmark.backfill_suppressions()- complainer = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")- assert complainer.reason == :spam_complaint- assert complainer.source == :backfill+ suppression =+ Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")- spam_notified = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")- assert spam_notified.reason == :spam_notification- assert spam_notified.source == :backfill+ assert suppression.reason == :manualend- test "counts fetch failures separately" do+ test "counts fetch failures separately, per stream" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)-- case conn.query_params["type"] do- "HardBounce" ->+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->conn |> Plug.Conn.put_status(500) |> Req.Test.json(%{"Message" => "boom"})- _other ->- Req.Test.json(conn, %{"TotalCount" => 0, "Bounces" => []})+ "/message-streams/priority/suppressions/dump" ->+ Req.Test.json(conn, %{"Suppressions" => []})endend)@@ -135,52 +141,48 @@ defmodule Plausible.PostmarkTest doPostmark.backfill_suppressions()end- test "counts distinct_addresses" do+ test "counts distinct_addresses, deduping the same address across streams" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)-- bounces =- case conn.query_params["type"] do- "HardBounce" ->- [- %{"Email" => "[email protected]", "ID" => 1},- %{"Email" => "[email protected]", "ID" => 2}- ]-- "Blocked" ->- [%{"Email" => "[email protected]", "ID" => 3}]+ entry = %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "HardBounce",+ "Origin" => "Recipient"+ }- _other ->- []- end-- Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ Req.Test.json(conn, %{"Suppressions" => [entry]})end)- assert %{processed: 3, distinct_addresses: 1, upsert_errors: 0, fetch_errors: 0} =+ assert %{processed: 2, distinct_addresses: 1, upsert_errors: 0, fetch_errors: 0} =Postmark.backfill_suppressions()assert EmailSuppressions.suppressed?("[email protected]")assert Repo.aggregate(Plausible.EmailSuppression, :count) == 1end- test "counts email citext variants" do+ test "counts email citext variants as the same address" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)-- bounces =- case conn.query_params["type"] do- "HardBounce" ->- [%{"Email" => "[email protected]", "ID" => 1}]-- "Blocked" ->- [%{"Email" => "[email protected]", "ID" => 2}]+ entries =+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->+ [+ %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "HardBounce",+ "Origin" => "Recipient"+ }+ ]- _other ->- []+ "/message-streams/priority/suppressions/dump" ->+ [+ %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "HardBounce",+ "Origin" => "Recipient"+ }+ ]end- Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ Req.Test.json(conn, %{"Suppressions" => entries})end)assert %{processed: 2, distinct_addresses: 1, upsert_errors: 0, fetch_errors: 0} =@@ -189,135 +191,123 @@ defmodule Plausible.PostmarkTest doassert Repo.aggregate(Plausible.EmailSuppression, :count) == 1end- test "for multiple bounce events, the most recent one wins" do+ test "for multiple entries, the most recent one wins" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)-- bounces =- case conn.query_params["type"] do- "HardBounce" ->+ entries =+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->[%{- "Email" => "[email protected]",- "ID" => 1,- "BouncedAt" => "2026-01-01T00:00:00Z",- "Details" => "older hard bounce"+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "HardBounce",+ "Origin" => "Recipient",+ "CreatedAt" => "2026-01-01T00:00:00Z"}]- "Blocked" ->+ "/message-streams/priority/suppressions/dump" ->[%{- "Email" => "[email protected]",- "ID" => 2,- "BouncedAt" => "2026-06-01T00:00:00Z",- "Details" => "newer block"+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "ManualSuppression",+ "Origin" => "Admin",+ "CreatedAt" => "2026-06-01T00:00:00Z"}]-- _other ->- []end- Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ Req.Test.json(conn, %{"Suppressions" => entries})end)assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")- assert suppression.reason == :blocked- assert suppression.postmark_bounce_id == 2- assert suppression.details == "newer block"+ assert suppression.reason == :manual+ assert suppression.details == "Postmark suppression (origin: Admin)"end- test "spam complaint always wins over a bounce" do+ test "spam complaint always wins over another reason" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)-- bounces =- case conn.query_params["type"] do- "HardBounce" ->+ entries =+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->[%{- "Email" => "[email protected]",- "ID" => 1,- "BouncedAt" => "2026-06-01T00:00:00Z"+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "HardBounce",+ "Origin" => "Recipient",+ "CreatedAt" => "2026-06-01T00:00:00Z"}]- "SpamComplaint" ->+ "/message-streams/priority/suppressions/dump" ->[%{- "Email" => "[email protected]",- "ID" => 2,- "BouncedAt" => "2026-01-01T00:00:00Z"+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "SpamComplaint",+ "Origin" => "Recipient",+ "CreatedAt" => "2026-01-01T00:00:00Z"}]-- _other ->- []end- Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ Req.Test.json(conn, %{"Suppressions" => entries})end)assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")assert suppression.reason == :spam_complaint- assert suppression.postmark_bounce_id == 2end- test "the most recent spam complaint wins when there are several" do+ test "a missing CreatedAt does not crash the backfill" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)-- bounces =- case conn.query_params["type"] do- "SpamComplaint" ->+ entries =+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->[%{- "Email" => "[email protected]",- "ID" => 1,- "BouncedAt" => "2026-01-01T00:00:00Z"- },- %{- "Email" => "[email protected]",- "ID" => 2,- "BouncedAt" => "2026-06-01T00:00:00Z"+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "HardBounce",+ "Origin" => "Recipient"}]- _other ->+ "/message-streams/priority/suppressions/dump" ->[]end- Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ Req.Test.json(conn, %{"Suppressions" => entries})end)- assert %{distinct_addresses: 1} = Postmark.backfill_suppressions()-- suppression =- Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")-- assert suppression.postmark_bounce_id == 2+ assert %{distinct_addresses: 1, upsert_errors: 0} = Postmark.backfill_suppressions()+ assert EmailSuppressions.suppressed?("[email protected]")end- test "a missing BouncedAt does not crash the backfill" do+ test "falls back to :manual for an unrecognized SuppressionReason" doReq.Test.stub(Postmark, fn conn ->- conn = Plug.Conn.fetch_query_params(conn)+ entries =+ case conn.request_path do+ "/message-streams/outbound/suppressions/dump" ->+ [+ %{+ "EmailAddress" => "[email protected]",+ "SuppressionReason" => "SomethingNew",+ "Origin" => "Recipient"+ }+ ]- bounces =- case conn.query_params["type"] do- "HardBounce" -> [%{"Email" => "[email protected]", "ID" => 1}]- _other -> []+ "/message-streams/priority/suppressions/dump" ->+ []end- Req.Test.json(conn, %{"TotalCount" => length(bounces), "Bounces" => bounces})+ Req.Test.json(conn, %{"Suppressions" => entries})end)- assert %{distinct_addresses: 1, upsert_errors: 0} = Postmark.backfill_suppressions()- assert EmailSuppressions.suppressed?("[email protected]")+ assert %{upsert_errors: 0} = Postmark.backfill_suppressions()++ suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")+ assert suppression.reason == :manualendendend