Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

E-mail suppressions background cleanup (#6733)

Adamd21298d9015e
diff --git a/config/runtime.exs b/config/runtime.exs
index 7d837e9461..d1c0c63b60 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -868,7 +868,9 @@ cloud_cron = [
# Daily at 5:00 UTC
{"0 5 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff},
# Daily at 2:00 UTC
- {"0 2 * * *", Plausible.Workers.ScoreTrialProspects}
+ {"0 2 * * *", Plausible.Workers.ScoreTrialProspects},
+ # Daily at 23:00 UTC
+ {"0 23 * * *", Plausible.Workers.CleanEmailSuppressions}
]
crontab = if(is_selfhost, do: base_cron, else: base_cron ++ cloud_cron)
@@ -888,7 +890,8 @@ base_queues = [
domain_change_transition: 1,
check_accept_traffic_until: 1,
clickhouse_clean_sites: 1,
- locations_sync: 1
+ locations_sync: 1,
+ clean_email_suppressions: 1
]
cloud_queues = [
diff --git a/extra/lib/plausible_web/live/customer_support/email_suppressions.ex b/extra/lib/plausible_web/live/customer_support/email_suppressions.ex
index 094e48c9cc..798076b5e0 100644
--- a/extra/lib/plausible_web/live/customer_support/email_suppressions.ex
+++ b/extra/lib/plausible_web/live/customer_support/email_suppressions.ex
@@ -223,11 +223,13 @@ defmodule PlausibleWeb.Live.CustomerSupport.EmailSuppressions do
defp reason_label(:spam_notification), do: "Spam notification"
defp reason_label(:unsubscribe), do: "Unsubscribed"
defp reason_label(:manual), do: "Manual"
+ defp reason_label(:recipient_rejected), do: "Recipient rejected"
defp reason_color(:spam_complaint), do: :red
defp reason_color(:spam_notification), do: :red
defp reason_color(:hard_bounce), do: :red
defp reason_color(:blocked), do: :red
+ defp reason_color(:recipient_rejected), do: :red
defp reason_color(:bad_email_address), do: :yellow
defp reason_color(:unsubscribe), do: :yellow
defp reason_color(:manual), do: :gray
diff --git a/lib/plausible/email_suppression.ex b/lib/plausible/email_suppression.ex
index 74db0a2edc..144fee7dbe 100644
--- a/lib/plausible/email_suppression.ex
+++ b/lib/plausible/email_suppression.ex
@@ -17,9 +17,10 @@ defmodule Plausible.EmailSuppression do
:spam_complaint,
:spam_notification,
:unsubscribe,
- :manual
+ :manual,
+ :recipient_rejected
]
- @sources [:webhook, :backfill, :manual]
+ @sources [:webhook, :backfill, :manual, :rejected]
schema "email_suppressions" do
field :email, :string
diff --git a/lib/plausible/email_suppressions.ex b/lib/plausible/email_suppressions.ex
index 442ef10188..1f49734a72 100644
--- a/lib/plausible/email_suppressions.ex
+++ b/lib/plausible/email_suppressions.ex
@@ -110,6 +110,74 @@ defmodule Plausible.EmailSuppressions do
where(query, [s], ilike(s.email, ^"%#{search}%"))
end
+ @doc """
+ Finds suppressions we no longer have any records for: not a user, a
+ pending team invitation, a pending site transfer, or a recipient on a
+ weekly report, monthly report, or traffic change (spike/drop)
+ notification.
+ """
+ @spec list_orphaned(Duration.t(), pos_integer()) :: [EmailSuppression.t()]
+ def list_orphaned(stale_after, limit) do
+ cutoff = NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(stale_after)
+
+ from(s in EmailSuppression, as: :suppression)
+ |> where([s], s.updated_at < ^cutoff)
+ |> where(
+ [s],
+ not exists(from(u in Plausible.Auth.User, where: u.email == parent_as(:suppression).email))
+ )
+ |> where(
+ [s],
+ not exists(
+ from(i in Plausible.Teams.Invitation, where: i.email == parent_as(:suppression).email)
+ )
+ )
+ |> where(
+ [s],
+ not exists(
+ from(t in Plausible.Teams.SiteTransfer, where: t.email == parent_as(:suppression).email)
+ )
+ )
+ |> where(
+ [s],
+ not exists(
+ from(r in Plausible.Site.WeeklyReport,
+ where: parent_as(:suppression).email in r.recipients
+ )
+ )
+ )
+ |> where(
+ [s],
+ not exists(
+ from(r in Plausible.Site.MonthlyReport,
+ where: parent_as(:suppression).email in r.recipients
+ )
+ )
+ )
+ |> where(
+ [s],
+ not exists(
+ from(r in Plausible.Site.TrafficChangeNotification,
+ where: parent_as(:suppression).email in r.recipients
+ )
+ )
+ )
+ |> order_by([s], asc: s.updated_at, asc: s.id)
+ |> limit(^limit)
+ |> Repo.all()
+ end
+
+ @doc """
+ Deletes the given suppressions locally
+ """
+ @spec delete_all([EmailSuppression.t()]) :: non_neg_integer()
+ def delete_all(suppressions) do
+ ids = Enum.map(suppressions, & &1.id)
+
+ {count, _} = Repo.delete_all(from(s in EmailSuppression, where: s.id in ^ids))
+ count
+ end
+
@doc """
Records (or refreshes) a suppression originating from Postmark's
Subscription Change webhook (an address unsubscribing).
@@ -123,6 +191,21 @@ defmodule Plausible.EmailSuppressions do
|> upsert()
end
+ @doc """
+ Records (or refreshes) a suppression detected after send attempt
+ was rejected by Postmark (406) - for when Postmark suppression exists still.
+ See: `Plausible.Mailer`.
+ """
+ @spec create_from_rejected_send(map()) ::
+ {:ok, EmailSuppression.t()} | {:error, Ecto.Changeset.t()}
+ def create_from_rejected_send(attrs) do
+ attrs
+ |> Map.take([:email, :details])
+ |> Map.put(:reason, :recipient_rejected)
+ |> Map.put(:source, :rejected)
+ |> upsert()
+ end
+
@doc """
Lifts a suppression after manual review, recording who did it.
Encapsulates suppression remote deletion at Postmark - can't be done for spam complaints.
diff --git a/lib/plausible/mailer.ex b/lib/plausible/mailer.ex
index 46f28faea6..c5ea987b39 100644
--- a/lib/plausible/mailer.ex
+++ b/lib/plausible/mailer.ex
@@ -23,6 +23,8 @@ defmodule Plausible.Mailer do
deliver_now!(email)
rescue
e ->
+ maybe_record_rejected_send(e)
+
# this message is ignored by Sentry, only appears in logs
log = "Failed to send e-mail:\n\n " <> Exception.format(:error, e, __STACKTRACE__)
# Sentry report is built entirely from crash_reason
@@ -35,6 +37,27 @@ defmodule Plausible.Mailer do
end
end
+ defp maybe_record_rejected_send(exception)
+
+ on_ce do
+ defp maybe_record_rejected_send(_exception), do: :ok
+ end
+
+ on_ee do
+ defp maybe_record_rejected_send(%Bamboo.PostmarkAdapter.Error{} = e) do
+ if Bamboo.PostmarkAdapter.Error.is_hard_bounce(e) do
+ Plausible.EmailSuppressions.create_from_rejected_send(%{
+ email: sole_recipient(e.email),
+ details: "Rejected by Postmark on send: #{inspect(e.reason)}"
+ })
+ end
+
+ :ok
+ end
+
+ defp maybe_record_rejected_send(_exception), do: :ok
+ end
+
defp suppressed_recipients(email)
on_ce do
diff --git a/lib/plausible/postmark.ex b/lib/plausible/postmark.ex
index 84931f4898..3d3306a22b 100644
--- a/lib/plausible/postmark.ex
+++ b/lib/plausible/postmark.ex
@@ -59,6 +59,56 @@ defmodule Plausible.Postmark do
end
end
+ # Potsmark limit
+ @delete_chunk_size 50
+
+ @doc """
+ Best-effort delete many suppressions at once, across every stream.
+ """
+ @spec delete_suppressions([String.t()]) :: [String.t()]
+ def delete_suppressions([]), do: []
+
+ def delete_suppressions(emails) do
+ emails
+ |> Enum.chunk_every(@delete_chunk_size)
+ |> Enum.flat_map(&delete_suppressions_chunk/1)
+ end
+
+ defp delete_suppressions_chunk(chunk) do
+ if @streams |> Enum.map(&delete_suppressions_chunk(&1, chunk)) |> Enum.all?() do
+ chunk
+ else
+ []
+ end
+ end
+
+ defp delete_suppressions_chunk(stream, emails) do
+ # https://postmarkapp.com/developer/api/suppressions-api#delete-a-suppression
+ case post("/message-streams/#{stream}/suppressions/delete", %{
+ "Suppressions" => Enum.map(emails, &%{"EmailAddress" => &1})
+ }) do
+ {:ok, %{"Suppressions" => results}} ->
+ case Enum.filter(results, &(&1["Status"] == "Failed")) do
+ [] ->
+ :ok
+
+ failed ->
+ Logger.warning(
+ "Failed to delete some Postmark suppressions, stream=#{stream}: #{inspect(failed)}"
+ )
+ end
+
+ true
+
+ {:error, reason} ->
+ Logger.error(
+ "Failed to delete a batch of #{length(emails)} Postmark suppressions, stream=#{stream}: #{inspect(reason)}"
+ )
+
+ false
+ end
+ end
+
@doc """
One-off backfill of e-mail suppressions from Postmark's Suppressions API,
across every message stream we send from.
diff --git a/lib/workers/clean_email_suppressions.ex b/lib/workers/clean_email_suppressions.ex
new file mode 100644
index 0000000000..7c36512f60
--- /dev/null
+++ b/lib/workers/clean_email_suppressions.ex
@@ -0,0 +1,38 @@
+defmodule Plausible.Workers.CleanEmailSuppressions do
+ @moduledoc """
+ Daily job garbage-collecting e-mail suppressions
+ """
+
+ use Oban.Worker, queue: :clean_email_suppressions
+ use Plausible
+
+ @impl Oban.Worker
+ def perform(_job)
+
+ on_ce do
+ def perform(_job), do: :ok
+ end
+
+ on_ee do
+ alias Plausible.EmailSuppressions
+
+ @stale_after Duration.new!(day: -14)
+ @batch_size 1_000
+
+ def perform(_job) do
+ candidates = EmailSuppressions.list_orphaned(@stale_after, @batch_size)
+
+ confirmed =
+ candidates
+ |> Enum.map(& &1.email)
+ |> Plausible.Postmark.delete_suppressions()
+ |> MapSet.new()
+
+ candidates
+ |> Enum.filter(&MapSet.member?(confirmed, &1.email))
+ |> EmailSuppressions.delete_all()
+
+ :ok
+ end
+ end
+end
diff --git a/test/plausible/email_suppressions_test.exs b/test/plausible/email_suppressions_test.exs
index f2e9b19122..db6a97cbd2 100644
--- a/test/plausible/email_suppressions_test.exs
+++ b/test/plausible/email_suppressions_test.exs
@@ -20,6 +20,12 @@ defmodule Plausible.EmailSuppressionsTest do
:ok
end
+ defp touch_updated_at(suppression, naive_datetime) do
+ suppression
+ |> Ecto.Changeset.change(updated_at: naive_datetime)
+ |> Repo.update!()
+ end
+
describe "suppressed?/1" do
test "false when no record exists" do
refute EmailSuppressions.suppressed?("[email protected]")
@@ -277,6 +283,269 @@ defmodule Plausible.EmailSuppressionsTest do
end
end
+ describe "create_from_rejected_send/1" do
+ test "sets reason to :recipient_rejected and source to :rejected" do
+ {:ok, suppression} =
+ EmailSuppressions.create_from_rejected_send(%{
+ email: "[email protected]",
+ details: "Rejected by Postmark on send: some reason"
+ })
+
+ assert suppression.reason == :recipient_rejected
+ assert suppression.source == :rejected
+ assert suppression.details == "Rejected by Postmark on send: some reason"
+ assert EmailSuppressions.suppressed?("[email protected]")
+ end
+
+ test "a rejected send clears a reactivation left by an earlier bounce" do
+ user = insert(:user)
+
+ {:ok, _} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :webhook
+ })
+
+ {:ok, _} = EmailSuppressions.reactivate("[email protected]", user)
+ refute EmailSuppressions.suppressed?("[email protected]")
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_rejected_send(%{email: "[email protected]"})
+
+ assert EmailSuppressions.suppressed?("[email protected]")
+ assert suppression.reason == :recipient_rejected
+ assert is_nil(suppression.reactivated_at)
+ assert is_nil(suppression.reactivated_by_user_id)
+ end
+ end
+
+ describe "list_orphaned/2" do
+ test "returns a suppression with no matching user, invitation or transfer, stale enough" do
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+
+ assert [%{email: "[email protected]"}] =
+ EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100)
+ end
+
+ test "excludes a suppression that isn't stale enough yet" do
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -13)
+ )
+
+ assert EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100) == []
+ end
+
+ test "excludes a suppression matching an existing user's email" do
+ user = insert(:user, email: "[email protected]")
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: user.email,
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+
+ assert EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100) == []
+ end
+
+ test "excludes a suppression matching a pending team invitation's email" do
+ owner = insert(:user)
+ {:ok, team} = Plausible.Teams.get_or_create(owner)
+
+ insert(:team_invitation,
+ team: team,
+ inviter: owner,
+ email: "[email protected]",
+ role: :guest
+ )
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+
+ assert EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100) == []
+ end
+
+ test "excludes a suppression matching a pending site transfer's email" do
+ owner = new_user()
+ site = new_site(owner: owner)
+
+ insert(:site_transfer, email: "[email protected]", site: site, initiator: owner)
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+
+ assert EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100) == []
+ end
+
+ test "excludes a suppression matching a weekly report recipient" do
+ owner = new_user()
+ site = new_site(owner: owner)
+
+ insert(:weekly_report, site: site, recipients: ["[email protected]"])
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+
+ assert EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100) == []
+ end
+
+ test "excludes a suppression matching a monthly report recipient" do
+ owner = new_user()
+ site = new_site(owner: owner)
+
+ insert(:monthly_report, site: site, recipients: ["[email protected]"])
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+
+ assert EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100) == []
+ end
+
+ test "excludes a suppression matching a traffic change (spike/drop) notification recipient" do
+ owner = new_user()
+ site = new_site(owner: owner)
+
+ insert(:spike_notification, site: site, recipients: ["[email protected]"])
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+
+ assert EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100) == []
+ end
+
+ test "matches case-insensitively, since all relevant email columns are citext" do
+ insert(:user, email: "[email protected]")
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(
+ suppression,
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+
+ assert EmailSuppressions.list_orphaned(Duration.new!(day: -14), 100) == []
+ end
+
+ test "returns the oldest candidates first, bounded by limit" do
+ now = NaiveDateTime.utc_now(:second)
+
+ for n <- 1..3 do
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "orphan-#{n}@example.com",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ touch_updated_at(suppression, NaiveDateTime.shift(now, day: -20 + n))
+ end
+
+ assert [%{email: "[email protected]"}] =
+ EmailSuppressions.list_orphaned(Duration.new!(day: -14), 1)
+ end
+ end
+
+ describe "delete_all/1" do
+ test "removes exactly the given suppressions" do
+ {:ok, keep} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ {:ok, remove} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ assert EmailSuppressions.delete_all([remove]) == 1
+
+ assert Repo.get(Plausible.EmailSuppression, keep.id)
+ refute Repo.get(Plausible.EmailSuppression, remove.id)
+ end
+
+ test "returns 0 for an empty list" do
+ assert EmailSuppressions.delete_all([]) == 0
+ end
+ end
+
describe "reactivate/2" do
test "returns :not_found when there is no suppression for the address" do
user = insert(:user)
diff --git a/test/plausible/mailer_test.exs b/test/plausible/mailer_test.exs
index 25dfd4fc29..3a3e7b92f2 100644
--- a/test/plausible/mailer_test.exs
+++ b/test/plausible/mailer_test.exs
@@ -148,4 +148,64 @@ defmodule Plausible.MailerTest do
end
end
end
+
+ describe "recording a suppression from a rejected send (Postmark ErrorCode 406)" do
+ @describetag :ee_only
+ @describetag :capture_log
+
+ setup do
+ bypass = Bypass.open()
+
+ original_base_uri = Application.get_env(:bamboo, :postmark_base_uri)
+ Application.put_env(:bamboo, :postmark_base_uri, "http://localhost:#{bypass.port}")
+ on_exit(fn -> Application.put_env(:bamboo, :postmark_base_uri, original_base_uri) end)
+
+ patch_env(Plausible.Mailer, adapter: Bamboo.PostmarkAdapter, api_key: "test-key")
+
+ %{bypass: bypass}
+ end
+
+ test "records a suppression when Postmark actually rejects the send as an inactive recipient",
+ %{bypass: bypass} do
+ user = insert(:user, email: "[email protected]")
+
+ Bypass.expect_once(bypass, "POST", "/email", fn conn ->
+ Plug.Conn.resp(
+ conn,
+ 422,
+ Jason.encode!(%{
+ "ErrorCode" => 406,
+ "Message" => "Rejected outright!"
+ })
+ )
+ end)
+
+ email = PlausibleWeb.Email.welcome_email(user)
+ assert {:error, :unknown_error} = Plausible.Mailer.send(email)
+
+ assert Plausible.EmailSuppressions.suppressed?("[email protected]")
+
+ suppression = Repo.get_by!(Plausible.EmailSuppression, email: "[email protected]")
+ assert suppression.reason == :recipient_rejected
+ assert suppression.source == :rejected
+ assert suppression.details =~ "Rejected outright!"
+ end
+
+ test "does not record a suppression for an unrelated Postmark error", %{bypass: bypass} do
+ user = insert(:user, email: "[email protected]")
+
+ Bypass.expect_once(bypass, "POST", "/email", fn conn ->
+ Plug.Conn.resp(
+ conn,
+ 422,
+ Jason.encode!(%{"ErrorCode" => 300, "Message" => "Invalid email request"})
+ )
+ end)
+
+ email = PlausibleWeb.Email.welcome_email(user)
+ assert {:error, :unknown_error} = Plausible.Mailer.send(email)
+
+ refute Plausible.EmailSuppressions.suppressed?("[email protected]")
+ end
+ end
end
diff --git a/test/plausible/postmark_test.exs b/test/plausible/postmark_test.exs
index 182e3c6f78..879af4e90f 100644
--- a/test/plausible/postmark_test.exs
+++ b/test/plausible/postmark_test.exs
@@ -93,6 +93,139 @@ defmodule Plausible.PostmarkTest do
end
end
+ describe "delete_suppressions/1" do
+ test "no-ops without making any request for an empty list" do
+ Req.Test.stub(Postmark, fn _conn -> flunk("Postmark should not have been called") end)
+
+ assert [] = Postmark.delete_suppressions([])
+ end
+
+ test "deletes every address, on every stream, in a single request each, and confirms all" do
+ test_pid = self()
+
+ Req.Test.stub(Postmark, fn conn ->
+ assert conn.request_path in [
+ "/message-streams/outbound/suppressions/delete",
+ "/message-streams/priority/suppressions/delete"
+ ]
+
+ conn = Plug.Conn.fetch_query_params(conn)
+ {:ok, body, conn} = Plug.Conn.read_body(conn)
+ send(test_pid, {:request, conn.request_path, Jason.decode!(body)})
+
+ Req.Test.json(conn, %{
+ "Suppressions" => [
+ %{"EmailAddress" => "[email protected]", "Status" => "Deleted"},
+ %{"EmailAddress" => "[email protected]", "Status" => "Deleted"}
+ ]
+ })
+ end)
+
+ assert Postmark.delete_suppressions(["[email protected]", "[email protected]"]) == [
+ ]
+
+ for stream <- ["outbound", "priority"] do
+ path = "/message-streams/#{stream}/suppressions/delete"
+ assert_received {:request, ^path, body}
+
+ assert body == %{
+ "Suppressions" => [
+ %{"EmailAddress" => "[email protected]"},
+ %{"EmailAddress" => "[email protected]"}
+ ]
+ }
+ end
+ end
+
+ test "splits a large batch into multiple requests per stream, respecting Postmark's max of 50" do
+ test_pid = self()
+ emails = for n <- 1..51, do: "user-#{n}@example.com"
+
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+ {:ok, body, conn} = Plug.Conn.read_body(conn)
+ %{"Suppressions" => suppressions} = Jason.decode!(body)
+ send(test_pid, {:chunk_size, length(suppressions)})
+
+ Req.Test.json(
+ conn,
+ %{"Suppressions" => Enum.map(suppressions, &Map.put(&1, "Status", "Deleted"))}
+ )
+ end)
+
+ assert Postmark.delete_suppressions(emails) |> length() == 51
+
+ # 51 emails, chunked at 50, on each of the 2 streams = 4 requests total
+ assert_received {:chunk_size, 50}
+ assert_received {:chunk_size, 50}
+ assert_received {:chunk_size, 1}
+ assert_received {:chunk_size, 1}
+ end
+
+ test "still confirms addresses Postmark refused (e.g. a spam complaint), only logging it" do
+ Req.Test.stub(Postmark, fn conn ->
+ Req.Test.json(conn, %{
+ "Suppressions" => [
+ %{"EmailAddress" => "[email protected]", "Status" => "Deleted"},
+ %{
+ "EmailAddress" => "[email protected]",
+ "Status" => "Failed",
+ "Message" => "SpamComplaint suppressions cannot be deleted"
+ }
+ ]
+ })
+ end)
+
+ assert Postmark.delete_suppressions(["[email protected]", "[email protected]"]) == [
+ ]
+ end
+
+ test "does not confirm a chunk whose call errored outright on any stream" do
+ Req.Test.stub(Postmark, fn conn ->
+ case conn.request_path do
+ "/message-streams/outbound/suppressions/delete" ->
+ Req.Test.json(conn, %{
+ "Suppressions" => [%{"EmailAddress" => "[email protected]", "Status" => "Deleted"}]
+ })
+
+ "/message-streams/priority/suppressions/delete" ->
+ conn |> Plug.Conn.put_status(500) |> Req.Test.json(%{"Message" => "boom"})
+ end
+ end)
+
+ assert Postmark.delete_suppressions(["[email protected]"]) == []
+ end
+
+ test "only leaves out the chunk that failed, not unrelated chunks" do
+ ok_emails = for n <- 1..50, do: "ok-#{n}@example.com"
+ failing_email = "[email protected]"
+
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+ {:ok, body, conn} = Plug.Conn.read_body(conn)
+ %{"Suppressions" => suppressions} = Jason.decode!(body)
+
+ if Enum.any?(suppressions, &(&1["EmailAddress"] == failing_email)) do
+ conn |> Plug.Conn.put_status(500) |> Req.Test.json(%{"Message" => "boom"})
+ else
+ Req.Test.json(
+ conn,
+ %{"Suppressions" => Enum.map(suppressions, &Map.put(&1, "Status", "Deleted"))}
+ )
+ end
+ end)
+
+ confirmed = Postmark.delete_suppressions(ok_emails ++ [failing_email])
+
+ assert Enum.sort(confirmed) == Enum.sort(ok_emails)
+ refute failing_email in confirmed
+ end
+ end
+
describe "backfill_suppressions/0" do
test "upserts a suppression for every entry found, across both streams" do
Req.Test.stub(Postmark, fn conn ->
diff --git a/test/plausible_web/live/customer_support/email_suppressions_test.exs b/test/plausible_web/live/customer_support/email_suppressions_test.exs
index 9a28c753c6..f8bc36af53 100644
--- a/test/plausible_web/live/customer_support/email_suppressions_test.exs
+++ b/test/plausible_web/live/customer_support/email_suppressions_test.exs
@@ -70,6 +70,27 @@ defmodule PlausibleWeb.Live.CustomerSupport.EmailSuppressionsTest do
assert text_of_element(html, "tbody tr:first-child") =~ "[email protected]"
end
+ test "renders a suppression recorded from a rejected send", %{conn: conn} do
+ {:ok, _} =
+ EmailSuppressions.create_from_rejected_send(%{
+ email: "[email protected]",
+ details: "Rejected by Postmark on send: some reason"
+ })
+
+ {:ok, _lv, html} = live(conn, open_suppressions())
+ text = text(html)
+
+ assert text =~ "[email protected]"
+ assert text =~ "Recipient rejected"
+ assert text =~ "rejected"
+
+ row = text_of_element(html, "tbody tr:first-child")
+ assert row =~ "Details"
+ assert row =~ "Reactivate"
+
+ assert html =~ "Rejected by Postmark on send: some reason"
+ end
+
test "filters by reason", %{conn: conn} do
{:ok, _} =
EmailSuppressions.create_from_bounce(%{
diff --git a/test/workers/clean_email_suppressions_test.exs b/test/workers/clean_email_suppressions_test.exs
new file mode 100644
index 0000000000..ce4f0668e1
--- /dev/null
+++ b/test/workers/clean_email_suppressions_test.exs
@@ -0,0 +1,189 @@
+defmodule Plausible.Workers.CleanEmailSuppressionsTest do
+ use Plausible.DataCase
+ @moduletag :ee_only
+ @moduletag :capture_log
+
+ import ExUnit.CaptureLog
+
+ alias Plausible.EmailSuppressions
+ alias Plausible.Postmark
+ alias Plausible.Workers.CleanEmailSuppressions
+
+ defp create_stale_suppression(email, opts \\ []) do
+ source = Keyword.get(opts, :source, :backfill)
+
+ {:ok, suppression} =
+ EmailSuppressions.create_from_bounce(%{email: email, reason: :hard_bounce, source: source})
+
+ suppression
+ |> Ecto.Changeset.change(
+ updated_at: NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+ |> Repo.update!()
+ end
+
+ test "removes a stale suppression with no matching user, invitation or transfer" do
+ create_stale_suppression("[email protected]")
+
+ Req.Test.stub(Postmark, fn conn ->
+ Req.Test.json(conn, %{
+ "Suppressions" => [%{"EmailAddress" => "[email protected]", "Status" => "Deleted"}]
+ })
+ end)
+
+ assert :ok = CleanEmailSuppressions.perform(nil)
+
+ refute Repo.exists?(Plausible.EmailSuppression)
+ end
+
+ test "keeps a stale suppression that still matches a user, regardless of Postmark" do
+ insert(:user, email: "[email protected]")
+ create_stale_suppression("[email protected]")
+
+ Req.Test.stub(Postmark, fn _conn -> flunk("Postmark should not have been called") end)
+
+ assert :ok = CleanEmailSuppressions.perform(nil)
+
+ assert Repo.exists?(Plausible.EmailSuppression)
+ end
+
+ test "keeps a suppression that isn't stale enough yet" do
+ {:ok, _suppression} =
+ EmailSuppressions.create_from_bounce(%{
+ email: "[email protected]",
+ reason: :hard_bounce,
+ source: :backfill
+ })
+
+ Req.Test.stub(Postmark, fn _conn -> flunk("Postmark should not have been called") end)
+
+ assert :ok = CleanEmailSuppressions.perform(nil)
+
+ assert Repo.exists?(Plausible.EmailSuppression)
+ end
+
+ test "still removes the local row when Postmark refuses (e.g. a spam complaint)" do
+ {:ok, suppression} =
+ EmailSuppressions.create_from_spam_complaint(%{
+ email: "[email protected]",
+ source: :backfill
+ })
+
+ suppression
+ |> Ecto.Changeset.change(
+ updated_at: NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+ |> Repo.update!()
+
+ Req.Test.stub(Postmark, fn conn ->
+ Req.Test.json(conn, %{
+ "Suppressions" => [
+ %{
+ "EmailAddress" => "[email protected]",
+ "Status" => "Failed",
+ "Message" => "SpamComplaint suppressions cannot be deleted"
+ }
+ ]
+ })
+ end)
+
+ log = capture_log(fn -> assert :ok = CleanEmailSuppressions.perform(nil) end)
+
+ assert log =~ "Failed to delete some Postmark suppressions"
+ assert log =~ "[email protected]"
+ assert log =~ "SpamComplaint suppressions cannot be deleted"
+
+ refute Repo.exists?(Plausible.EmailSuppression)
+ end
+
+ test "keeps the local row when Postmark call errors out (to retry later)" do
+ create_stale_suppression("[email protected]")
+ create_stale_suppression("[email protected]")
+
+ Req.Test.stub(Postmark, fn conn ->
+ conn |> Plug.Conn.put_status(500) |> Req.Test.json(%{"Message" => "boom"})
+ end)
+
+ log = capture_log(fn -> assert :ok = CleanEmailSuppressions.perform(nil) end)
+
+ assert log =~ "Failed to delete a batch of 2 Postmark suppressions"
+ assert log =~ "unexpected_status, 500"
+
+ assert Repo.exists?(Plausible.EmailSuppression)
+ end
+
+ test "removes every row in a chunk whose call succeeded, even one Postmark refused" do
+ create_stale_suppression("[email protected]")
+
+ {:ok, complaint} =
+ EmailSuppressions.create_from_spam_complaint(%{
+ email: "[email protected]",
+ source: :backfill
+ })
+
+ complaint
+ |> Ecto.Changeset.change(
+ updated_at: NaiveDateTime.utc_now(:second) |> NaiveDateTime.shift(day: -15)
+ )
+ |> Repo.update!()
+
+ Req.Test.stub(Postmark, fn conn ->
+ Req.Test.json(conn, %{
+ "Suppressions" => [
+ %{"EmailAddress" => "[email protected]", "Status" => "Deleted"},
+ %{
+ "EmailAddress" => "[email protected]",
+ "Status" => "Failed",
+ "Message" => "SpamComplaint suppressions cannot be deleted"
+ }
+ ]
+ })
+ end)
+
+ log = capture_log(fn -> assert :ok = CleanEmailSuppressions.perform(nil) end)
+
+ assert log =~ "Failed to delete some Postmark suppressions"
+ assert log =~ "[email protected]"
+
+ # The HTTP call itself completed, so the whole chunk is confirmed,
+ # even if Postmark refused to delete some suppressions
+ refute Repo.exists?(Plausible.EmailSuppression)
+ end
+
+ test "removes only the chunk Postmark confirmed, keeping the rest for next time" do
+ for n <- 1..50 do
+ create_stale_suppression("confirmed-#{n}@example.com")
+ end
+
+ create_stale_suppression("[email protected]")
+
+ Req.Test.stub(Postmark, fn conn ->
+ conn = Plug.Conn.fetch_query_params(conn)
+ {:ok, body, conn} = Plug.Conn.read_body(conn)
+ %{"Suppressions" => suppressions} = Jason.decode!(body)
+
+ if Enum.any?(suppressions, &(&1["EmailAddress"] == "[email protected]")) do
+ conn |> Plug.Conn.put_status(500) |> Req.Test.json(%{"Message" => "boom"})
+ else
+ Req.Test.json(
+ conn,
+ %{"Suppressions" => Enum.map(suppressions, &Map.put(&1, "Status", "Deleted"))}
+ )
+ end
+ end)
+
+ log = capture_log(fn -> assert :ok = CleanEmailSuppressions.perform(nil) end)
+
+ assert log =~ "Failed to delete a batch of 1 Postmark suppressions"
+ assert log =~ "unexpected_status, 500"
+
+ refute Repo.get_by(Plausible.EmailSuppression, email: "[email protected]")
+ assert Repo.get_by(Plausible.EmailSuppression, email: "[email protected]")
+ end
+
+ test "does nothing when there are no candidates" do
+ Req.Test.stub(Postmark, fn _conn -> flunk("Postmark should not have been called") end)
+
+ assert :ok = CleanEmailSuppressions.perform(nil)
+ end
+end