Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Retry failed cache warm-ups instead of waiting for the next interval (#6799)

Raphael Fakhri24ea392a0f9d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4989d898e7..79b0c3a718 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -50,6 +50,7 @@ All notable changes to this project will be documented in this file.
### Fixed
+- Cache warmer retries a failed warm-up of critical caches
- Stats API `/api/v1/stats/breakdown` now returns a `400` error for an invalid `page` parameter instead of crashing with a `500`
- Fixed "Create Team" going through when the team name was rejected, creating the team under a name the user never entered
- Improve team member removal/team role change
diff --git a/extra/lib/plausible/consolidated_view/cache.ex b/extra/lib/plausible/consolidated_view/cache.ex
index 970d2d6d62..61b8af893d 100644
--- a/extra/lib/plausible/consolidated_view/cache.ex
+++ b/extra/lib/plausible/consolidated_view/cache.ex
@@ -42,7 +42,7 @@ defmodule Plausible.ConsolidatedView.Cache do
}
end
- @spec refresh_updated_recently(Keyword.t()) :: :ok
+ @spec refresh_updated_recently(Keyword.t()) :: :ok | {:error, any()}
def refresh_updated_recently(opts) do
recently_updated_site_ids =
from sc in ConsolidatedView.sites(),
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index f8daa8b72e..0b374b858a 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -73,7 +73,8 @@ defmodule Plausible.Application do
warmers: [
refresh_all:
{Plausible.Site.Cache.All,
- interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10))},
+ interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10)),
+ retry_interval: :timer.seconds(30)},
refresh_updated_recently:
{Plausible.Site.Cache.RecentlyUpdated, interval: :timer.seconds(30)}
]
@@ -103,7 +104,8 @@ defmodule Plausible.Application do
warmers: [
refresh_all:
{Plausible.Shield.IPRuleCache.All,
- interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
+ interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),
+ retry_interval: :timer.seconds(30)},
refresh_updated_recently:
{Plausible.Shield.IPRuleCache.RecentlyUpdated, interval: :timer.seconds(35)}
]
@@ -174,7 +176,9 @@ defmodule Plausible.Application do
warmers: [
refresh_all:
{Plausible.Site.TrackerScriptIdCache.All,
- interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10))},
+ interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10)),
+ retry_interval: :timer.seconds(30),
+ max_retries: 8},
refresh_updated_recently:
{Plausible.Site.TrackerScriptIdCache.RecentlyUpdated,
interval: :timer.seconds(30)}
@@ -190,7 +194,9 @@ defmodule Plausible.Application do
warmers: [
refresh_all:
{Plausible.Site.TrackerScriptCache.All,
- interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10))},
+ interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10)),
+ retry_interval: :timer.seconds(30),
+ max_retries: 8},
refresh_updated_recently:
{Plausible.Site.TrackerScriptCache.RecentlyUpdated, interval: :timer.seconds(30)}
]
diff --git a/lib/plausible/cache.ex b/lib/plausible/cache.ex
index 2bbefbcc93..de6f28d22b 100644
--- a/lib/plausible/cache.ex
+++ b/lib/plausible/cache.ex
@@ -62,7 +62,6 @@ defmodule Plausible.Cache do
@modes [:all, :updated_recently]
alias Plausible.Cache.Adapter
- require Logger
@spec put(any(), any(), Keyword.t()) :: any()
def put(key, value, opts \\ []) do
@@ -131,7 +130,7 @@ defmodule Plausible.Cache do
def repo(), do: Plausible.Repo
defoverridable repo: 0
- @spec refresh_all(Keyword.t()) :: :ok
+ @spec refresh_all(Keyword.t()) :: :ok | {:error, any()}
def refresh_all(opts \\ []) do
refresh(
:all,
@@ -140,7 +139,7 @@ defmodule Plausible.Cache do
)
end
- @spec refresh_updated_recently(Keyword.t()) :: :ok
+ @spec refresh_updated_recently(Keyword.t()) :: :ok | {:error, any()}
def refresh_updated_recently(opts \\ [])
def refresh_updated_recently(opts) do
@@ -229,13 +228,10 @@ defmodule Plausible.Cache do
items = repo().all(query)
:ok = merge_items(items, opts)
catch
- _, e ->
- Logger.error("Error refreshing '#{cache_name}' - #{inspect(e)}")
+ _, e -> {:error, e}
end
end
)
-
- :ok
end
end
end
diff --git a/lib/plausible/cache/warmer.ex b/lib/plausible/cache/warmer.ex
index 3114f50b15..12735056dd 100644
--- a/lib/plausible/cache/warmer.ex
+++ b/lib/plausible/cache/warmer.ex
@@ -6,6 +6,13 @@ defmodule Plausible.Cache.Warmer do
* `cache_impl` - module expected to implement `Plausible.Cache` behaviour
* `interval` - the number of milliseconds for each warm-up cycle
+ * `retry_interval` - optional. The number of milliseconds to wait before
+ the warmer retries a warm-up that returned `{:error, reason}`. The wait
+ doubles after each failed retry, plus up to a third of it at random, so
+ warmers that fail together do not retry together. Without this option,
+ the warmer does not retry and waits for the next cycle.
+ * `max_retries` - defaults to 3. The number of retries before the warmer
+ gives up and waits for the next cycle.
* `cache_name` - defaults to cache_impl.name() but can be overridden for testing
* `force_start?` - enforcess process startup for testing, even if it's barred
by `Plausible.Cache.enabled?`. This is useful for avoiding issues with DB ownership
@@ -19,6 +26,8 @@ defmodule Plausible.Cache.Warmer do
@behaviour :gen_cycle
+ @default_max_retries 3
+
require Logger
@spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore
@@ -54,30 +63,62 @@ defmodule Plausible.Cache.Warmer do
"#{__MODULE__} initializing #{inspect(warmer_fn)} #{cache_name} with interval #{interval}..."
)
- {:ok,
- {interval,
+ opts =
opts
|> Keyword.put(:cache_name, cache_name)
- |> Keyword.put(:warmer_fn, warmer_fn)}}
+ |> Keyword.put(:warmer_fn, warmer_fn)
+
+ {:ok, {interval, {opts, nil}}}
else
:ignore
end
end
@impl true
- def handle_cycle(opts) do
+ def handle_cycle({opts, _retry}) do
+ {:continue_hibernated, warm(opts, 0)}
+ end
+
+ @impl true
+ def handle_info({:retry, ref}, {opts, {ref, retries}}) do
+ {:continue_hibernated, warm(opts, retries)}
+ end
+
+ def handle_info(_msg, state) do
+ {:continue, state}
+ end
+
+ defp warm(opts, retries) do
cache_name = Keyword.fetch!(opts, :cache_name)
warmer_fn = Keyword.fetch!(opts, :warmer_fn)
Logger.notice("#{__MODULE__} running #{inspect(warmer_fn)} on #{cache_name}...")
- warmer_fn.(opts)
-
- {:continue_hibernated, opts}
+ case warmer_fn.(opts) do
+ {:error, reason} -> {opts, schedule_retry(opts, retries + 1, reason)}
+ _ -> {opts, nil}
+ end
end
- @impl true
- def handle_info(_msg, state) do
- {:continue, state}
+ defp schedule_retry(opts, retry, reason) do
+ retry_interval = Keyword.get(opts, :retry_interval)
+ max_retries = Keyword.get(opts, :max_retries, @default_max_retries)
+ failure = "#{__MODULE__} #{inspect(opts[:warmer_fn])} on #{opts[:cache_name]} failed"
+
+ if retry_interval && retry <= max_retries do
+ backoff = retry_interval * Integer.pow(2, retry - 1)
+ delay = backoff + Enum.random(0..div(backoff, 3))
+
+ Logger.warning(
+ "#{failure}, retrying in #{delay}ms (#{retry}/#{max_retries}) - #{inspect(reason)}"
+ )
+
+ ref = make_ref()
+ Process.send_after(self(), {:retry, ref}, delay)
+ {ref, retry}
+ else
+ Logger.error("#{failure} - #{inspect(reason)}")
+ nil
+ end
end
end
diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex
index ea8a44787a..0c8b8e7a6a 100644
--- a/lib/plausible/site/cache.ex
+++ b/lib/plausible/site/cache.ex
@@ -15,8 +15,6 @@ defmodule Plausible.Site.Cache do
Also see tests for more comprehensive examples.
"""
- require Logger
-
import Ecto.Query
alias Plausible.Site
diff --git a/test/plausible/cache_test.exs b/test/plausible/cache_test.exs
index d23dc37233..2de172db4d 100644
--- a/test/plausible/cache_test.exs
+++ b/test/plausible/cache_test.exs
@@ -24,6 +24,23 @@ defmodule Plausible.CacheTest do
end
end
+ defmodule UnavailableRepo do
+ def all(_query) do
+ raise DBConnection.ConnectionError, "connection not available and request was dropped"
+ end
+ end
+
+ defmodule UnavailableCache do
+ use Plausible.Cache
+
+ def name(), do: __MODULE__
+ def child_id(), do: __MODULE__
+ def count_all(), do: 100
+ def repo(), do: UnavailableRepo
+ def base_db_query(), do: %Ecto.Query{}
+ def get_from_source(key), do: repo().all(key)
+ end
+
describe "public cache interface" do
test "cache process is started, but falls back to the database if cache is disabled", %{
test: test
@@ -56,6 +73,11 @@ defmodule Plausible.CacheTest do
test "cache is not ready when it doesn't exist", %{test: test} do
refute ExampleCache.ready?(test)
end
+
+ test "refresh_all returns an error when the database is unavailable", %{test: test} do
+ assert {:error, %DBConnection.ConnectionError{}} =
+ UnavailableCache.refresh_all(cache_name: test)
+ end
end
describe "merging cache items" do
@@ -193,12 +215,159 @@ defmodule Plausible.CacheTest do
assert at1 < at2
assert at3 > at2
end
+
+ test "cache warmer backs off retries until it succeeds, then stops", %{test: test} do
+ opts = [
+ cache_impl: ExampleCache,
+ force_start?: true,
+ warmer_fn: fail_first(self(), 3),
+ cache_name: test,
+ interval: :timer.hours(1),
+ retry_interval: 60,
+ max_retries: 4
+ ]
+
+ capture_log(fn ->
+ {:ok, _} = start_test_warmer(opts)
+
+ warmed_at =
+ for _ <- 1..4 do
+ assert_receive {:cache_warmed, %{at: at}}, 1000
+ at
+ end
+
+ scheduling_tolerance = 100
+
+ for {[previous_at, at], backoff} <-
+ Enum.zip(Enum.chunk_every(warmed_at, 2, 1, :discard), [60, 120, 240]) do
+ elapsed = System.convert_time_unit(at - previous_at, :native, :millisecond)
+
+ assert elapsed >= backoff
+ assert elapsed <= backoff + div(backoff, 3) + scheduling_tolerance
+ end
+
+ refute_receive {:cache_warmed, _}, 800
+ end)
+ end
+
+ test "cache warmer stops retrying after a few failed retries", %{test: test} do
+ opts = [
+ cache_impl: ExampleCache,
+ force_start?: true,
+ warmer_fn: report_back(self(), {:error, :db_unavailable}),
+ cache_name: test,
+ interval: :timer.hours(1),
+ retry_interval: 10
+ ]
+
+ errors =
+ capture_log([level: :error], fn ->
+ {:ok, _} = start_test_warmer(opts)
+
+ for _ <- 1..4, do: assert_receive({:cache_warmed, _}, 1000)
+ refute_receive {:cache_warmed, _}, 300
+ end)
+
+ assert [_] = Regex.scan(~r/failed - :db_unavailable/, errors)
+ end
+
+ for max_retries <- [0, 1, 4] do
+ test "cache warmer stops after #{max_retries} configured retries", %{test: test} do
+ max_retries = unquote(max_retries)
+
+ opts = [
+ cache_impl: ExampleCache,
+ force_start?: true,
+ warmer_fn: report_back(self(), {:error, :db_unavailable}),
+ cache_name: test,
+ interval: :timer.hours(1),
+ retry_interval: 10,
+ max_retries: max_retries
+ ]
+
+ capture_log(fn ->
+ {:ok, _} = start_test_warmer(opts)
+
+ for _ <- 0..max_retries, do: assert_receive({:cache_warmed, _}, 1000)
+ refute_receive {:cache_warmed, _}, 300
+ end)
+ end
+ end
+
+ test "cache warmer ignores a pending retry after a regular cycle", %{test: test} do
+ opts = [
+ cache_impl: ExampleCache,
+ force_start?: true,
+ warmer_fn: fail_first(self(), 1),
+ cache_name: test,
+ interval: :timer.hours(1),
+ retry_interval: 300
+ ]
+
+ log =
+ capture_log(fn ->
+ {:ok, warmer} = start_test_warmer(opts)
+ assert_receive {:cache_warmed, _}, 1000
+
+ send(warmer, :"$cycle")
+ assert_receive {:cache_warmed, _}, 1000
+
+ refute_receive {:cache_warmed, _}, 600
+ end)
+
+ assert log =~ "retrying in"
+ end
+
+ test "cache warmer does not retry a failing run without a retry interval", %{test: test} do
+ opts = [
+ cache_impl: ExampleCache,
+ force_start?: true,
+ warmer_fn: report_back(self(), {:error, :db_unavailable}),
+ cache_name: test,
+ interval: :timer.hours(1)
+ ]
+
+ capture_log(fn ->
+ {:ok, _} = start_test_warmer(opts)
+
+ assert_receive {:cache_warmed, _}, 1000
+ refute_receive {:cache_warmed, _}, 200
+ end)
+ end
+
+ test "cache warmer does not retry a successful run", %{test: test} do
+ test_pid = self()
+
+ opts = [
+ cache_impl: ExampleCache,
+ force_start?: true,
+ warmer_fn: report_back(test_pid),
+ cache_name: test,
+ interval: :timer.hours(1),
+ retry_interval: 30
+ ]
+
+ {:ok, _} = start_test_warmer(opts)
+
+ assert_receive {:cache_warmed, _}, 1000
+ refute_receive {:cache_warmed, _}, 150
+ end
end
- defp report_back(test_pid) do
+ defp report_back(test_pid, result \\ :ok) do
fn opts ->
send(test_pid, {:cache_warmed, %{at: System.monotonic_time(), opts: opts}})
- :ok
+ result
+ end
+ end
+
+ defp fail_first(test_pid, times) do
+ runs = :counters.new(1, [])
+
+ fn opts ->
+ :counters.add(runs, 1, 1)
+ result = if :counters.get(runs, 1) <= times, do: {:error, :db_unavailable}, else: :ok
+ report_back(test_pid, result).(opts)
end
end