Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
rm Hammer (#3571)
ruslandoga0eedf9aa984c
diff --git a/config/runtime.exs b/config/runtime.exsindex 67dd3858e7..27f1b54137 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -570,9 +570,6 @@ config :ref_inspector,config :ua_inspector,init: {Plausible.Release, :configure_ua_inspector}-config :hammer,- backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]}-if config_env() in [:dev, :staging, :prod] doconfig :kaffy,otp_app: :plausible,diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex b5a91296fb..b9d58c3e5c 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -12,6 +12,7 @@ defmodule Plausible.Application doPlausible.IngestRepo,Plausible.AsyncInsertRepo,Plausible.ImportDeletionRepo,+ {Plausible.RateLimit, clean_period: :timer.minutes(10)},Plausible.Ingestion.Counters,{Finch, name: Plausible.Finch, pools: finch_pool_config()},{Phoenix.PubSub, name: Plausible.PubSub},diff --git a/lib/plausible/rate_limit.ex b/lib/plausible/rate_limit.exnew file mode 100644index 0000000000..7e1d01d73b--- /dev/null+++ b/lib/plausible/rate_limit.ex@@ -0,0 +1,82 @@+defmodule Plausible.RateLimit do+ @moduledoc """+ Thin wrapper around `:ets.update_counter/4` and a+ clean-up process to act as a rate limiter.+ """++ use GenServer++ @doc """+ Starts the process that creates and cleans the ETS table.++ Accepts the following options:+ - `GenServer.options()`+ - `:table` for the ETS table name, defaults to `#{__MODULE__}`+ - `:clean_period` for how often to perform garbage collection+ """+ @spec start_link([GenServer.option() | {:table, atom} | {:clean_period, pos_integer}]) ::+ GenServer.on_start()+ def start_link(opts) do+ {gen_opts, opts} =+ Keyword.split(opts, [:debug, :name, :timeout, :spawn_opt, :hibernate_after])++ GenServer.start_link(__MODULE__, opts, gen_opts)+ end++ @doc """+ Checks the rate-limit for a key.+ """+ @spec check_rate(:ets.table(), key, scale, limit, increment) :: {:allow, count} | {:deny, limit}+ when key: term,+ scale: pos_integer,+ limit: pos_integer,+ increment: pos_integer,+ count: pos_integer+ def check_rate(table \\ __MODULE__, key, scale, limit, increment \\ 1) do+ bucket = div(now(), scale)+ full_key = {key, bucket}+ expires_at = (bucket + 1) * scale+ count = :ets.update_counter(table, full_key, increment, {full_key, 0, expires_at})+ if count <= limit, do: {:allow, count}, else: {:deny, limit}+ end++ @impl true+ def init(opts) do+ clean_period = Keyword.fetch!(opts, :clean_period)+ table = Keyword.get(opts, :table, __MODULE__)++ ^table =+ :ets.new(table, [+ :named_table,+ :set,+ :public,+ {:read_concurrency, true},+ {:write_concurrency, true},+ {:decentralized_counters, true}+ ])++ schedule(clean_period)+ {:ok, %{table: table, clean_period: clean_period}}+ end++ @impl true+ def handle_info(:clean, state) do+ clean(state.table)+ schedule(state.clean_period)+ {:noreply, state}+ end++ defp schedule(clean_period) do+ Process.send_after(self(), :clean, clean_period)+ end++ defp clean(table) do+ ms = [{{{:_, :_}, :_, :"$1"}, [], [{:<, :"$1", {:const, now()}}]}]+ :ets.select_delete(table, ms)+ end++ @compile inline: [now: 0]+ defp now do+ System.system_time(:millisecond)+ end+enddiff --git a/lib/plausible/site/gate_keeper.ex b/lib/plausible/site/gate_keeper.exindex 8b064f8e25..e543dfcbc2 100644--- a/lib/plausible/site/gate_keeper.ex+++ b/lib/plausible/site/gate_keeper.ex@@ -5,7 +5,7 @@ defmodule Plausible.Site.GateKeeper do@type t() :: {:allow, Plausible.Site.t()} | {:deny, policy()}@moduledoc """- Thin wrapper around Hammer for gate keeping domain-specific events+ Thin wrapper around `Plausible.RateLimit` for gate keeping domain-specific eventsduring the ingestion phase. When the site is allowed, gate keepingcheck returns `:allow`, otherwise a `:deny` tagged tuple is returnedwith one of the following policy markers:@@ -24,7 +24,7 @@ defmodule Plausible.Site.GateKeeper do* when the underlying rate limiting mechanism returnsan internal error: :allow"""- alias Plausible.Site+ alias Plausible.{Site, RateLimit}alias Plausible.Site.Cacherequire Logger@@ -68,19 +68,9 @@ defmodule Plausible.Site.GateKeeper dokey = Keyword.get(opts, :key, key(site.domain))scale_ms = site.ingest_rate_limit_scale_seconds * 1_000- case Hammer.check_rate(key, scale_ms, threshold) do- {:deny, _} ->- :throttle-- {:allow, _} ->- {:allow, site}-- {:error, reason} ->- Logger.error(- "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: :allow"- )-- {:allow, site}+ case RateLimit.check_rate(key, scale_ms, threshold) do+ {:deny, _} -> :throttle+ {:allow, _} -> {:allow, site}endendenddiff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.exindex 23f4d6e851..439db16187 100644--- a/lib/plausible_web/controllers/auth_controller.ex+++ b/lib/plausible_web/controllers/auth_controller.ex@@ -2,7 +2,7 @@ defmodule PlausibleWeb.AuthController douse PlausibleWeb, :controlleruse Plausible.Repo- alias Plausible.Auth+ alias Plausible.{Auth, RateLimit}alias Plausible.Billing.Quotaalias PlausibleWeb.TwoFactor@@ -304,14 +304,14 @@ defmodule PlausibleWeb.AuthController dodefp check_ip_rate_limit(conn) doip_address = PlausibleWeb.RemoteIp.get(conn)- case Hammer.check_rate("login:ip:#{ip_address}", @login_interval, @login_limit) do+ case RateLimit.check_rate("login:ip:#{ip_address}", @login_interval, @login_limit) do{:allow, _} -> :ok{:deny, _} -> {:rate_limit, :ip_address}endenddefp check_user_rate_limit(user) do- case Hammer.check_rate("login:user:#{user.id}", @login_interval, @login_limit) do+ case RateLimit.check_rate("login:user:#{user.id}", @login_interval, @login_limit) do{:allow, _} -> :ok{:deny, _} -> {:rate_limit, :user}enddiff --git a/lib/plausible_web/plugs/authorize_stats_api.ex b/lib/plausible_web/plugs/authorize_stats_api.exindex f30a143625..c099191879 100644--- a/lib/plausible_web/plugs/authorize_stats_api.ex+++ b/lib/plausible_web/plugs/authorize_stats_api.ex@@ -3,6 +3,7 @@ defmodule PlausibleWeb.AuthorizeStatsApiPlug douse Plausible.Repoalias Plausible.Authalias Plausible.Sites+ alias Plausible.RateLimitalias PlausibleWeb.Api.Helpers, as: Hdef init(options) do@@ -101,7 +102,11 @@ defmodule PlausibleWeb.AuthorizeStatsApiPlug do@one_hour 60 * 60 * 1000defp check_api_key_rate_limit(api_key) do- case Hammer.check_rate("api_request:#{api_key.id}", @one_hour, api_key.hourly_request_limit) do+ case RateLimit.check_rate(+ "api_request:#{api_key.id}",+ @one_hour,+ api_key.hourly_request_limit+ ) do{:allow, _} -> :ok{:deny, _} -> {:error, :rate_limit, api_key.hourly_request_limit}enddiff --git a/mix.exs b/mix.exsindex 05a1348003..604142cb0c 100644--- a/mix.exs+++ b/mix.exs@@ -90,7 +90,6 @@ defmodule Plausible.MixProject do{:locus, "~> 2.3"},{:gen_cycle, "~> 1.0.4"},{:hackney, "~> 1.8"},- {:hammer, "~> 6.0"},{:httpoison, "~> 1.4"},{:jason, "~> 1.3"},{:kaffy, "~> 0.10.2", only: [:dev, :test, :staging, :prod]},diff --git a/mix.lock b/mix.lockindex bea604e548..ed08d8590f 100644--- a/mix.lock+++ b/mix.lock@@ -61,7 +61,6 @@"gproc": {:hex, :gproc, "0.8.0", "cea02c578589c61e5341fce149ea36ccef236cc2ecac8691fba408e7ea77ec2f", [:rebar3], [], "hexpm", "580adafa56463b75263ef5a5df4c86af321f68694e7786cb057fd805d1e2a7de"},"grpcbox": {:hex, :grpcbox, "0.16.0", "b83f37c62d6eeca347b77f9b1ec7e9f62231690cdfeb3a31be07cd4002ba9c82", [:rebar3], [{:acceptor_pool, "~>1.0.0", [hex: :acceptor_pool, repo: "hexpm", optional: false]}, {:chatterbox, "~>0.13.0", [hex: :ts_chatterbox, repo: "hexpm", optional: false]}, {:ctx, "~>0.6.0", [hex: :ctx, repo: "hexpm", optional: false]}, {:gproc, "~>0.8.0", [hex: :gproc, repo: "hexpm", optional: false]}], "hexpm", "294df743ae20a7e030889f00644001370a4f7ce0121f3bbdaf13cf3169c62913"},"hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},- "hammer": {:hex, :hammer, "6.1.0", "f263e3c3e9946bd410ea0336b2abe0cb6260af4afb3a221e1027540706e76c55", [:make, :mix], [{:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "b47e415a562a6d072392deabcd58090d8a41182cf9044cdd6b0d0faaaf68ba57"},"heroicons": {:hex, :heroicons, "0.5.3", "ee8ae8335303df3b18f2cc07f46e1cb6e761ba4cf2c901623fbe9a28c0bc51dd", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:phoenix_live_view, ">= 0.18.2", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "a210037e8a09ac17e2a0a0779d729e89c821c944434c3baa7edfc1f5b32f3502"},"hpack": {:hex, :hpack_erl, "0.2.3", "17670f83ff984ae6cd74b1c456edde906d27ff013740ee4d9efaa4f1bf999633", [:rebar3], [], "hexpm", "06f580167c4b8b8a6429040df36cc93bba6d571faeaec1b28816523379cbb23a"},"hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},@@ -118,7 +117,6 @@"plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"},"plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"},"plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},- "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},"postgrex": {:hex, :postgrex, "0.17.1", "01c29fd1205940ee55f7addb8f1dc25618ca63a8817e56fac4f6846fc2cddcbe", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "14b057b488e73be2beee508fb1955d8db90d6485c6466428fe9ccf1d6692a555"},"prom_ex": {:hex, :prom_ex, "1.9.0", "63e6dda6c05cdeec1f26c48443dcc38ffd2118b3665ae8d2bd0e5b79f2aea03e", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "01f3d4f69ec93068219e686cc65e58a29c42bea5429a8ff4e2121f19db178ee6"},"public_suffix": {:git, "https://github.com/axelson/publicsuffix-elixir", "fa40c243d4b5d8598b90cff268bc4e33f3bb63f1", []},diff --git a/test/plausible/rate_limit_test.exs b/test/plausible/rate_limit_test.exsnew file mode 100644index 0000000000..ba57787da2--- /dev/null+++ b/test/plausible/rate_limit_test.exs@@ -0,0 +1,67 @@+defmodule Plausible.RateLimitTest do+ use ExUnit.Case, async: true+ alias Plausible.RateLimit++ @table __MODULE__++ defp key, do: "key:#{System.unique_integer([:positive])}"++ @tag :slow+ test "garbage collection" do+ start_supervised!({RateLimit, clean_period: _100_ms = 100, table: @table})++ key = key()+ scale = _50_ms = 50+ limit = 10++ for _ <- 1..3 do+ assert {:allow, 1} = RateLimit.check_rate(@table, key, scale, limit)+ assert [{{^key, _bucket}, _count = 1, expires_at}] = :ets.tab2list(@table)++ assert expires_at > System.system_time(:millisecond)+ assert expires_at <= System.system_time(:millisecond) + 50++ :timer.sleep(150)++ assert :ets.tab2list(@table) == []+ end+ end++ describe "check_rate/3" do+ setup do+ start_supervised!({RateLimit, clean_period: :timer.minutes(1), table: @table})+ :ok+ end++ test "increments" do+ key = key()+ scale = :timer.seconds(10)+ limit = 10++ assert {:allow, 1} = RateLimit.check_rate(@table, key, scale, limit)+ assert {:allow, 2} = RateLimit.check_rate(@table, key, scale, limit)+ assert {:allow, 3} = RateLimit.check_rate(@table, key, scale, limit)+ end++ test "resets" do+ key = key()+ scale = 10+ limit = 10++ assert {:allow, 1} = RateLimit.check_rate(@table, key, scale, limit)+ :timer.sleep(scale * 2 + 1)+ assert {:allow, 1} = RateLimit.check_rate(@table, key, scale, limit)+ end++ test "denies" do+ key = key()+ scale = :timer.seconds(10)+ limit = 3++ assert {:allow, 1} = RateLimit.check_rate(@table, key, scale, limit)+ assert {:allow, 2} = RateLimit.check_rate(@table, key, scale, limit)+ assert {:allow, 3} = RateLimit.check_rate(@table, key, scale, limit)+ assert {:deny, 3} = RateLimit.check_rate(@table, key, scale, limit)+ end+ end+enddiff --git a/test/plausible/site/gate_keeper_test.exs b/test/plausible/site/gate_keeper_test.exsindex 2b0c4f0c81..c31285e711 100644--- a/test/plausible/site/gate_keeper_test.exs+++ b/test/plausible/site/gate_keeper_test.exs@@ -4,8 +4,6 @@ defmodule Plausible.Site.GateKeeperTest doalias Plausible.Site.Cachealias Plausible.Site.GateKeeper- import ExUnit.CaptureLog-setup %{test: test} do{:ok, _} = start_test_cache(test)opts = [cache_opts: [cache_name: test, force?: true]]@@ -87,55 +85,6 @@ defmodule Plausible.Site.GateKeeperTest doassert {:deny, :not_found} = GateKeeper.check(domain, opts)end- test "rate limiter policy switches to allow when RL backend errors bubble-up", %{- test: test,- opts: opts- } do- domain = "causingerrors.example.com"-- site =- add_site_and_refresh_cache(test,- domain: domain,- ingest_rate_limit_threshold: 1,- ingest_rate_limit_scale_seconds: 600- )-- site_id = site.id-- assert {:allow, %Plausible.Site{id: ^site_id, from_cache?: true}} =- GateKeeper.check(domain, opts)-- assert {:deny, :throttle} = GateKeeper.check(domain, opts)-- {:ok, :broken} = break_hammer(site)-- log =- capture_log(fn ->- assert {:allow, %Plausible.Site{id: ^site_id, from_cache?: true}} =- GateKeeper.check(domain, opts)- end)-- assert log =~ "Error checking rate limit for 'ingest:site:causingerrors.example.com'"- assert log =~ "Falling back to: :allow"- end-- # We need a way to force Hammer to error-out on Hammer.check_rate/3.- # This is tricky because we don't configure multiple backends,- # so the easiest (and naive) way to do it, without mocking, is to- # plant a hand-crafted ets entry that makes it throw an exception- # when it gets to it. This should not affect any shared state tests- # because the rogue entry is only stored for a specific key.- # The drawback of doing this, the test will break if we- # ever change the underlying Rate Limiting backend/implementation.- defp break_hammer(site) do- scale_ms = site.ingest_rate_limit_scale_seconds * 1_000- rogue_key = site.domain- our_key = GateKeeper.key(rogue_key)- {_, key} = Hammer.Utils.stamp_key(our_key, scale_ms)- true = :ets.insert(:hammer_ets_buckets, {key, 1, "TOTALLY-WRONG", "ABSOLUTELY-BREAKING"})- {:ok, :broken}- end-defp start_test_cache(cache_name) do%{start: {m, f, a}} = Cache.child_spec(cache_name: cache_name)apply(m, f, a)