Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Retry failed cache warm-ups instead of waiting for the next interval (#6799)
Raphael Fakhri24ea392a0f9d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 4989d898e7..79b0c3a718 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -50,6 +50,7 @@ All notable changes to this project will be documented in this file.### Fixed+- Cache warmer retries a failed warm-up of critical caches- Stats API `/api/v1/stats/breakdown` now returns a `400` error for an invalid `page` parameter instead of crashing with a `500`- Fixed "Create Team" going through when the team name was rejected, creating the team under a name the user never entered- Improve team member removal/team role changediff --git a/extra/lib/plausible/consolidated_view/cache.ex b/extra/lib/plausible/consolidated_view/cache.exindex 970d2d6d62..61b8af893d 100644--- a/extra/lib/plausible/consolidated_view/cache.ex+++ b/extra/lib/plausible/consolidated_view/cache.ex@@ -42,7 +42,7 @@ defmodule Plausible.ConsolidatedView.Cache do}end- @spec refresh_updated_recently(Keyword.t()) :: :ok+ @spec refresh_updated_recently(Keyword.t()) :: :ok | {:error, any()}def refresh_updated_recently(opts) dorecently_updated_site_ids =from sc in ConsolidatedView.sites(),diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex f8daa8b72e..0b374b858a 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -73,7 +73,8 @@ defmodule Plausible.Application dowarmers: [refresh_all:{Plausible.Site.Cache.All,- interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10))},+ interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10)),+ retry_interval: :timer.seconds(30)},refresh_updated_recently:{Plausible.Site.Cache.RecentlyUpdated, interval: :timer.seconds(30)}]@@ -103,7 +104,8 @@ defmodule Plausible.Application dowarmers: [refresh_all:{Plausible.Shield.IPRuleCache.All,- interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},+ interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),+ retry_interval: :timer.seconds(30)},refresh_updated_recently:{Plausible.Shield.IPRuleCache.RecentlyUpdated, interval: :timer.seconds(35)}]@@ -174,7 +176,9 @@ defmodule Plausible.Application dowarmers: [refresh_all:{Plausible.Site.TrackerScriptIdCache.All,- interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10))},+ interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10)),+ retry_interval: :timer.seconds(30),+ max_retries: 8},refresh_updated_recently:{Plausible.Site.TrackerScriptIdCache.RecentlyUpdated,interval: :timer.seconds(30)}@@ -190,7 +194,9 @@ defmodule Plausible.Application dowarmers: [refresh_all:{Plausible.Site.TrackerScriptCache.All,- interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10))},+ interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10)),+ retry_interval: :timer.seconds(30),+ max_retries: 8},refresh_updated_recently:{Plausible.Site.TrackerScriptCache.RecentlyUpdated, interval: :timer.seconds(30)}]diff --git a/lib/plausible/cache.ex b/lib/plausible/cache.exindex 2bbefbcc93..de6f28d22b 100644--- a/lib/plausible/cache.ex+++ b/lib/plausible/cache.ex@@ -62,7 +62,6 @@ defmodule Plausible.Cache do@modes [:all, :updated_recently]alias Plausible.Cache.Adapter- require Logger@spec put(any(), any(), Keyword.t()) :: any()def put(key, value, opts \\ []) do@@ -131,7 +130,7 @@ defmodule Plausible.Cache dodef repo(), do: Plausible.Repodefoverridable repo: 0- @spec refresh_all(Keyword.t()) :: :ok+ @spec refresh_all(Keyword.t()) :: :ok | {:error, any()}def refresh_all(opts \\ []) dorefresh(:all,@@ -140,7 +139,7 @@ defmodule Plausible.Cache do)end- @spec refresh_updated_recently(Keyword.t()) :: :ok+ @spec refresh_updated_recently(Keyword.t()) :: :ok | {:error, any()}def refresh_updated_recently(opts \\ [])def refresh_updated_recently(opts) do@@ -229,13 +228,10 @@ defmodule Plausible.Cache doitems = repo().all(query):ok = merge_items(items, opts)catch- _, e ->- Logger.error("Error refreshing '#{cache_name}' - #{inspect(e)}")+ _, e -> {:error, e}endend)-- :okendendenddiff --git a/lib/plausible/cache/warmer.ex b/lib/plausible/cache/warmer.exindex 3114f50b15..12735056dd 100644--- a/lib/plausible/cache/warmer.ex+++ b/lib/plausible/cache/warmer.ex@@ -6,6 +6,13 @@ defmodule Plausible.Cache.Warmer do* `cache_impl` - module expected to implement `Plausible.Cache` behaviour* `interval` - the number of milliseconds for each warm-up cycle+ * `retry_interval` - optional. The number of milliseconds to wait before+ the warmer retries a warm-up that returned `{:error, reason}`. The wait+ doubles after each failed retry, plus up to a third of it at random, so+ warmers that fail together do not retry together. Without this option,+ the warmer does not retry and waits for the next cycle.+ * `max_retries` - defaults to 3. The number of retries before the warmer+ gives up and waits for the next cycle.* `cache_name` - defaults to cache_impl.name() but can be overridden for testing* `force_start?` - enforcess process startup for testing, even if it's barredby `Plausible.Cache.enabled?`. This is useful for avoiding issues with DB ownership@@ -19,6 +26,8 @@ defmodule Plausible.Cache.Warmer do@behaviour :gen_cycle+ @default_max_retries 3+require Logger@spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore@@ -54,30 +63,62 @@ defmodule Plausible.Cache.Warmer do"#{__MODULE__} initializing #{inspect(warmer_fn)} #{cache_name} with interval #{interval}...")- {:ok,- {interval,+ opts =opts|> Keyword.put(:cache_name, cache_name)- |> Keyword.put(:warmer_fn, warmer_fn)}}+ |> Keyword.put(:warmer_fn, warmer_fn)++ {:ok, {interval, {opts, nil}}}else:ignoreendend@impl true- def handle_cycle(opts) do+ def handle_cycle({opts, _retry}) do+ {:continue_hibernated, warm(opts, 0)}+ end++ @impl true+ def handle_info({:retry, ref}, {opts, {ref, retries}}) do+ {:continue_hibernated, warm(opts, retries)}+ end++ def handle_info(_msg, state) do+ {:continue, state}+ end++ defp warm(opts, retries) docache_name = Keyword.fetch!(opts, :cache_name)warmer_fn = Keyword.fetch!(opts, :warmer_fn)Logger.notice("#{__MODULE__} running #{inspect(warmer_fn)} on #{cache_name}...")- warmer_fn.(opts)-- {:continue_hibernated, opts}+ case warmer_fn.(opts) do+ {:error, reason} -> {opts, schedule_retry(opts, retries + 1, reason)}+ _ -> {opts, nil}+ endend- @impl true- def handle_info(_msg, state) do- {:continue, state}+ defp schedule_retry(opts, retry, reason) do+ retry_interval = Keyword.get(opts, :retry_interval)+ max_retries = Keyword.get(opts, :max_retries, @default_max_retries)+ failure = "#{__MODULE__} #{inspect(opts[:warmer_fn])} on #{opts[:cache_name]} failed"++ if retry_interval && retry <= max_retries do+ backoff = retry_interval * Integer.pow(2, retry - 1)+ delay = backoff + Enum.random(0..div(backoff, 3))++ Logger.warning(+ "#{failure}, retrying in #{delay}ms (#{retry}/#{max_retries}) - #{inspect(reason)}"+ )++ ref = make_ref()+ Process.send_after(self(), {:retry, ref}, delay)+ {ref, retry}+ else+ Logger.error("#{failure} - #{inspect(reason)}")+ nil+ endendenddiff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.exindex ea8a44787a..0c8b8e7a6a 100644--- a/lib/plausible/site/cache.ex+++ b/lib/plausible/site/cache.ex@@ -15,8 +15,6 @@ defmodule Plausible.Site.Cache doAlso see tests for more comprehensive examples."""- require Logger-import Ecto.Queryalias Plausible.Sitediff --git a/test/plausible/cache_test.exs b/test/plausible/cache_test.exsindex d23dc37233..2de172db4d 100644--- a/test/plausible/cache_test.exs+++ b/test/plausible/cache_test.exs@@ -24,6 +24,23 @@ defmodule Plausible.CacheTest doendend+ defmodule UnavailableRepo do+ def all(_query) do+ raise DBConnection.ConnectionError, "connection not available and request was dropped"+ end+ end++ defmodule UnavailableCache do+ use Plausible.Cache++ def name(), do: __MODULE__+ def child_id(), do: __MODULE__+ def count_all(), do: 100+ def repo(), do: UnavailableRepo+ def base_db_query(), do: %Ecto.Query{}+ def get_from_source(key), do: repo().all(key)+ end+describe "public cache interface" dotest "cache process is started, but falls back to the database if cache is disabled", %{test: test@@ -56,6 +73,11 @@ defmodule Plausible.CacheTest dotest "cache is not ready when it doesn't exist", %{test: test} dorefute ExampleCache.ready?(test)end++ test "refresh_all returns an error when the database is unavailable", %{test: test} do+ assert {:error, %DBConnection.ConnectionError{}} =+ UnavailableCache.refresh_all(cache_name: test)+ endenddescribe "merging cache items" do@@ -193,12 +215,159 @@ defmodule Plausible.CacheTest doassert at1 < at2assert at3 > at2end++ test "cache warmer backs off retries until it succeeds, then stops", %{test: test} do+ opts = [+ cache_impl: ExampleCache,+ force_start?: true,+ warmer_fn: fail_first(self(), 3),+ cache_name: test,+ interval: :timer.hours(1),+ retry_interval: 60,+ max_retries: 4+ ]++ capture_log(fn ->+ {:ok, _} = start_test_warmer(opts)++ warmed_at =+ for _ <- 1..4 do+ assert_receive {:cache_warmed, %{at: at}}, 1000+ at+ end++ scheduling_tolerance = 100++ for {[previous_at, at], backoff} <-+ Enum.zip(Enum.chunk_every(warmed_at, 2, 1, :discard), [60, 120, 240]) do+ elapsed = System.convert_time_unit(at - previous_at, :native, :millisecond)++ assert elapsed >= backoff+ assert elapsed <= backoff + div(backoff, 3) + scheduling_tolerance+ end++ refute_receive {:cache_warmed, _}, 800+ end)+ end++ test "cache warmer stops retrying after a few failed retries", %{test: test} do+ opts = [+ cache_impl: ExampleCache,+ force_start?: true,+ warmer_fn: report_back(self(), {:error, :db_unavailable}),+ cache_name: test,+ interval: :timer.hours(1),+ retry_interval: 10+ ]++ errors =+ capture_log([level: :error], fn ->+ {:ok, _} = start_test_warmer(opts)++ for _ <- 1..4, do: assert_receive({:cache_warmed, _}, 1000)+ refute_receive {:cache_warmed, _}, 300+ end)++ assert [_] = Regex.scan(~r/failed - :db_unavailable/, errors)+ end++ for max_retries <- [0, 1, 4] do+ test "cache warmer stops after #{max_retries} configured retries", %{test: test} do+ max_retries = unquote(max_retries)++ opts = [+ cache_impl: ExampleCache,+ force_start?: true,+ warmer_fn: report_back(self(), {:error, :db_unavailable}),+ cache_name: test,+ interval: :timer.hours(1),+ retry_interval: 10,+ max_retries: max_retries+ ]++ capture_log(fn ->+ {:ok, _} = start_test_warmer(opts)++ for _ <- 0..max_retries, do: assert_receive({:cache_warmed, _}, 1000)+ refute_receive {:cache_warmed, _}, 300+ end)+ end+ end++ test "cache warmer ignores a pending retry after a regular cycle", %{test: test} do+ opts = [+ cache_impl: ExampleCache,+ force_start?: true,+ warmer_fn: fail_first(self(), 1),+ cache_name: test,+ interval: :timer.hours(1),+ retry_interval: 300+ ]++ log =+ capture_log(fn ->+ {:ok, warmer} = start_test_warmer(opts)+ assert_receive {:cache_warmed, _}, 1000++ send(warmer, :"$cycle")+ assert_receive {:cache_warmed, _}, 1000++ refute_receive {:cache_warmed, _}, 600+ end)++ assert log =~ "retrying in"+ end++ test "cache warmer does not retry a failing run without a retry interval", %{test: test} do+ opts = [+ cache_impl: ExampleCache,+ force_start?: true,+ warmer_fn: report_back(self(), {:error, :db_unavailable}),+ cache_name: test,+ interval: :timer.hours(1)+ ]++ capture_log(fn ->+ {:ok, _} = start_test_warmer(opts)++ assert_receive {:cache_warmed, _}, 1000+ refute_receive {:cache_warmed, _}, 200+ end)+ end++ test "cache warmer does not retry a successful run", %{test: test} do+ test_pid = self()++ opts = [+ cache_impl: ExampleCache,+ force_start?: true,+ warmer_fn: report_back(test_pid),+ cache_name: test,+ interval: :timer.hours(1),+ retry_interval: 30+ ]++ {:ok, _} = start_test_warmer(opts)++ assert_receive {:cache_warmed, _}, 1000+ refute_receive {:cache_warmed, _}, 150+ endend- defp report_back(test_pid) do+ defp report_back(test_pid, result \\ :ok) dofn opts ->send(test_pid, {:cache_warmed, %{at: System.monotonic_time(), opts: opts}})- :ok+ result+ end+ end++ defp fail_first(test_pid, times) do+ runs = :counters.new(1, [])++ fn opts ->+ :counters.add(runs, 1, 1)+ result = if :counters.get(runs, 1) <= times, do: {:error, :db_unavailable}, else: :ok+ report_back(test_pid, result).(opts)endend