Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Ingest throughput fixes (#5378)

hq1aa4a8339cb34
diff --git a/Makefile b/Makefile
index d755b1a1c3..811c303343 100644
--- a/Makefile
+++ b/Makefile
@@ -56,3 +56,12 @@ minio: ## Start a transient container with a recent version of minio (s3)
minio-stop: ## Stop and remove the minio container
docker stop plausible_minio
+
+loadtest-server:
+ @echo "Ensure your OTP installation is built with --enable-lock-counter"
+ MIX_ENV=dev LOADTEST=1 ERL_FLAGS="-emu_type lcnt +Mdai max" iex -S mix phx.server
+
+loadtest-client:
+ @echo "Set your limits for file descriptors/ephermeal ports high... Test begins shortly"
+ @sleep 5
+ k6 run test/load/script.js
diff --git a/config/dev.exs b/config/dev.exs
index 564dcb86a0..ce5a34f632 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -3,29 +3,8 @@ import Config
config :plausible, PlausibleWeb.Endpoint,
server: true,
debug_errors: true,
- code_reloader: true,
- check_origin: false,
- watchers: [
- esbuild: {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]},
- tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]},
- storybook_tailwind: {Tailwind, :install_and_run, [:storybook, ~w(--watch)]},
- npm: ["--prefix", "assets", "run", "typecheck", "--", "--watch", "--preserveWatchOutput"],
- npm: [
- "run",
- "deploy",
- cd: Path.expand("../tracker", __DIR__)
- ]
- ],
- live_reload: [
- dirs: [
- "extra"
- ],
- patterns: [
- ~r{priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$},
- ~r"lib/plausible_web/(controllers|live|components|templates|views|plugs)/.*(ex|heex)$",
- ~r"storybook/.*(exs)$"
- ]
- ]
+ code_reloader: false,
+ check_origin: false
config :plausible, paddle_api: Plausible.Billing.DevPaddleApiMock
diff --git a/config/runtime.exs b/config/runtime.exs
index 3052d4f3e2..756690d923 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -331,6 +331,41 @@ config :plausible, PlausibleWeb.Endpoint,
websocket_url: websocket_url,
secure_cookie: secure_cookie
+if config_env() in [:dev, :ce_dev] do
+ loadtest_mode = get_var_from_path_or_env(config_dir, "LOADTEST")
+
+ if loadtest_mode do
+ config :plausible, PlausibleWeb.Endpoint,
+ debug_errors: false,
+ code_reloader: false
+ else
+ config :plausible, PlausibleWeb.Endpoint,
+ debug_errors: true,
+ code_reloader: true,
+ watchers: [
+ esbuild: {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]},
+ tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]},
+ storybook_tailwind: {Tailwind, :install_and_run, [:storybook, ~w(--watch)]},
+ npm: ["--prefix", "assets", "run", "typecheck", "--", "--watch", "--preserveWatchOutput"],
+ npm: [
+ "run",
+ "deploy",
+ cd: Path.expand("../tracker", __DIR__)
+ ]
+ ],
+ live_reload: [
+ dirs: [
+ "extra"
+ ],
+ patterns: [
+ ~r{priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$},
+ ~r"lib/plausible_web/(controllers|live|components|templates|views|plugs)/.*(ex|heex)$",
+ ~r"storybook/.*(exs)$"
+ ]
+ ]
+ end
+end
+
# maybe enable HTTPS in CE
if config_env() in [:ce, :ce_dev, :ce_test] do
if https_port do
diff --git a/lib/plausible/ingestion/counters/telemetry_handler.ex b/lib/plausible/ingestion/counters/telemetry_handler.ex
index 34523a1e8c..8caa37681e 100644
--- a/lib/plausible/ingestion/counters/telemetry_handler.ex
+++ b/lib/plausible/ingestion/counters/telemetry_handler.ex
@@ -1,7 +1,7 @@
defmodule Plausible.Ingestion.Counters.TelemetryHandler do
@moduledoc """
Subscribes to telemetry events emitted by `Plausible.Ingestion.Event`.
- Every time a request derived event is dropped,
+ Every time a request derived event is either dispatched to clickhouse or dropped,
a telemetry event is emitted respectively. That event is captured here,
its metadata is extracted and sent for internal stats aggregation via
`Counters.Buffer` interface.
@@ -10,8 +10,9 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandler do
alias Plausible.Ingestion.Event
@event_dropped Event.telemetry_event_dropped()
+ @event_buffered Event.telemetry_event_buffered()
- @telemetry_events [@event_dropped]
+ @telemetry_events [@event_dropped, @event_buffered]
@telemetry_handler &__MODULE__.handle_event/4
@spec install(Counters.Buffer.t()) :: :ok
@@ -47,4 +48,18 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandler do
:ok
end
+
+ def handle_event(
+ @event_buffered,
+ _measurements,
+ %{
+ domain: domain,
+ request_timestamp: timestamp,
+ tracker_script_version: tracker_script_version
+ },
+ buffer
+ ) do
+ Counters.Buffer.aggregate(buffer, "buffered", domain, timestamp, tracker_script_version)
+ :ok
+ end
end
diff --git a/lib/plausible/rate_limit.ex b/lib/plausible/rate_limit.ex
index 7e1d01d73b..a92f579842 100644
--- a/lib/plausible/rate_limit.ex
+++ b/lib/plausible/rate_limit.ex
@@ -10,16 +10,14 @@ defmodule Plausible.RateLimit do
Starts the process that creates and cleans the ETS table.
Accepts the following options:
- - `GenServer.options()`
+ - `GenServer.option()`
- `:table` for the ETS table name, defaults to `#{__MODULE__}`
- `:clean_period` for how often to perform garbage collection
"""
@spec start_link([GenServer.option() | {:table, atom} | {:clean_period, pos_integer}]) ::
GenServer.on_start()
def start_link(opts) do
- {gen_opts, opts} =
- Keyword.split(opts, [:debug, :name, :timeout, :spawn_opt, :hibernate_after])
-
+ {gen_opts, opts} = Keyword.split(opts, [:debug, :name, :spawn_opt, :hibernate_after])
GenServer.start_link(__MODULE__, opts, gen_opts)
end
@@ -36,7 +34,25 @@ defmodule Plausible.RateLimit do
bucket = div(now(), scale)
full_key = {key, bucket}
expires_at = (bucket + 1) * scale
- count = :ets.update_counter(table, full_key, increment, {full_key, 0, expires_at})
+
+ count =
+ case :ets.lookup(table, full_key) do
+ [{_, counter, _expires_at}] ->
+ :atomics.add_get(counter, 1, increment)
+
+ [] ->
+ counter = :atomics.new(1, signed: false)
+
+ case :ets.insert_new(table, {full_key, counter, expires_at}) do
+ true ->
+ :atomics.add_get(counter, 1, increment)
+
+ false ->
+ [{_, counter, _expires_at}] = :ets.lookup(table, full_key)
+ :atomics.add_get(counter, 1, increment)
+ end
+ end
+
if count <= limit, do: {:allow, count}, else: {:deny, limit}
end
@@ -51,7 +67,7 @@ defmodule Plausible.RateLimit do
:set,
:public,
{:read_concurrency, true},
- {:write_concurrency, true},
+ {:write_concurrency, :auto},
{:decentralized_counters, true}
])
diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex
index ec784b0758..49c6dce3c8 100644
--- a/lib/plausible/telemetry/plausible_metrics.ex
+++ b/lib/plausible/telemetry/plausible_metrics.ex
@@ -82,12 +82,14 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
counter(
metric_prefix ++ [:plausible_cache, :hit],
event_name: ConCache.Operations.telemetry_hit(),
- tags: [:name]
+ tags: [:name],
+ tag_values: &%{name: &1.cache.name}
),
counter(
metric_prefix ++ [:plausible_cache, :miss],
event_name: ConCache.Operations.telemetry_miss(),
- tags: [:name]
+ tags: [:name],
+ tag_values: &%{name: &1.cache.name}
),
distribution(
metric_prefix ++ [:sessions, :transfer, :duration],
diff --git a/rel/vm.args.eex b/rel/vm.args.eex
index 957d983203..c608ce8ea3 100644
--- a/rel/vm.args.eex
+++ b/rel/vm.args.eex
@@ -9,3 +9,4 @@
## Tweak GC to run more often
##-env ERL_FULLSWEEP_AFTER 10
++Mdai max
diff --git a/test/load/script.js b/test/load/script.js
index 7be00a7233..6936375417 100644
--- a/test/load/script.js
+++ b/test/load/script.js
@@ -1,54 +1,44 @@
import http from "k6/http";
const PAYLOAD = JSON.stringify({
- "n":"pageview",
- "u":"http://loadtest.site/some-page",
- "d":"loadtest.site",
- "r":null,
- "w":1666
+ n: "pageview",
+ u: "http://dummy.site/some-page",
+ d: "dummy.site",
+ r: null,
+ w: 1666,
});
function newParams() {
- const ip = (Math.floor(Math.random() * 255) + 1)+"."+(Math.floor(Math.random() * 255))+"."+(Math.floor(Math.random() * 255))+"."+(Math.floor(Math.random() * 255));
+ const ip =
+ Math.floor(Math.random() * 255) +
+ 1 +
+ "." +
+ Math.floor(Math.random() * 255) +
+ "." +
+ Math.floor(Math.random() * 255) +
+ "." +
+ Math.floor(Math.random() * 255);
return {
headers: {
- 'Content-Type': 'application/json',
- 'X-Forwarded-For': ip,
- 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36 OPR/71.0.3770.284'
- }
- }
+ "Content-Type": "application/json",
+ "X-Forwarded-For": ip,
+ "User-Agent": `${Math.random() > 0.5 ? "Mozilla/5.0" : "Mozilla/4.0"} (${Math.random() > 0.5 ? "Macintosh" : "Windows"}; ${Math.random() > 0.5 ? "Intel Mac OS X 10_15_6" : "Windows NT 10.0"}) AppleWebKit/${Math.floor(Math.random() * 1000) + 500}.36 (KHTML, like Gecko) Chrome/${Math.floor(Math.random() * 100) + 50}.0.${Math.floor(Math.random() * 5000) + 1000}.${Math.floor(Math.random() * 500)} Safari/${Math.floor(Math.random() * 1000) + 500}.${Math.floor(Math.random() * 100)} OPR/${Math.floor(Math.random() * 100)}.0.${Math.floor(Math.random() * 5000) + 1000}.${Math.floor(Math.random() * 500)}`,
+ },
+ };
}
export const options = {
- scenarios: {
- wave1: {
- executor: 'constant-vus',
- startTime: '0s',
- vus: 200,
- duration: '60s'
- },
- wave2: {
- executor: 'constant-vus',
- startTime: '60s',
- vus: 200,
- duration: '60s'
- },
- wave3: {
- executor: 'constant-vus',
- startTime: '120s',
- vus: 200,
- duration: '60s'
- },
- wave4: {
- executor: 'constant-vus',
- startTime: '180s',
- vus: 200,
- duration: '60s'
- },
- },
+ stages: [
+ { target: 10000, duration: "10s" },
+ { target: 15000, duration: "20s" },
+ { target: 20000, duration: "30s" },
+ { target: 20000, duration: "30s" },
+ { target: 30000, duration: "30s" },
+ { target: 30000, duration: "90s" },
+ ],
};
-export default function() {
+export default function () {
http.post("http://localhost:8000/api/event", PAYLOAD, newParams());
-};
+}
diff --git a/test/plausible/ingestion/counters/telemetry_handler_test.exs b/test/plausible/ingestion/counters/telemetry_handler_test.exs
index 8a668f9c62..084fb31ab7 100644
--- a/test/plausible/ingestion/counters/telemetry_handler_test.exs
+++ b/test/plausible/ingestion/counters/telemetry_handler_test.exs
@@ -19,6 +19,11 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest do
handler.config == buffer and
handler.event_name == Event.telemetry_event_dropped()
end)
+
+ assert Enum.find(all_handlers, fn handler ->
+ handler.config == buffer and
+ handler.event_name == Event.telemetry_event_buffered()
+ end)
end
test "handles ingest events by aggregating the counts", %{test: test} do
@@ -39,8 +44,14 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest do
request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}
}
+ e3 = %{
+ domain: "c.example.com",
+ request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}
+ }
+
:ok = Event.emit_telemetry_dropped(e1, :invalid)
:ok = Event.emit_telemetry_dropped(e2, :not_found)
+ :ok = Event.emit_telemetry_buffered(e3)
:ok = Event.emit_telemetry_dropped(e2, :not_found)
future = DateTime.utc_now() |> DateTime.add(120, :second)
@@ -49,5 +60,6 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest do
assert Enum.find(aggregates, &match?({_, "dropped_invalid", "a.example.com", 137, 1}, &1))
assert Enum.find(aggregates, &match?({_, "dropped_not_found", "b.example.com", 137, 2}, &1))
+ assert Enum.find(aggregates, &match?({_, "buffered", "c.example.com", 137, 1}, &1))
end
end
diff --git a/test/plausible/ingestion/counters_test.exs b/test/plausible/ingestion/counters_test.exs
index 06ec462942..e60d091a25 100644
--- a/test/plausible/ingestion/counters_test.exs
+++ b/test/plausible/ingestion/counters_test.exs
@@ -38,8 +38,12 @@ defmodule Plausible.Ingestion.CountersTest do
{:ok, dropped} = emit_dropped_request(at: now)
{:ok, _dropped} = emit_dropped_request(domain: dropped.domain, at: now)
+ {:ok, buffered} = emit_buffered_request(at: now)
verify_record_written(dropped.domain, "dropped_not_found", 2)
+
+ site_id = Plausible.Sites.get_by_domain(buffered.domain).id
+ verify_record_written(buffered.domain, "buffered", 1, site_id)
end
test "the database eventually sums the records within 1-minute buckets", %{test: test} do
@@ -122,6 +126,25 @@ defmodule Plausible.Ingestion.CountersTest do
{:ok, dropped}
end
+ defp emit_buffered_request(opts) do
+ domain = Keyword.get(opts, :domain, random_domain())
+ at = Keyword.get(opts, :at, @ts)
+
+ site = new_site(domain: domain)
+
+ payload = %{
+ name: "pageview",
+ url: "http://#{site.domain}",
+ v: 137
+ }
+
+ conn = build_conn(:post, "/api/event", payload)
+ assert {:ok, request} = Request.build(conn, at)
+ assert {:ok, %{buffered: [buffered]}} = Event.build_and_buffer(request)
+
+ {:ok, buffered}
+ end
+
defp start_counters(opts) do
opts = Keyword.put(opts, :force_start?, true)
%{start: {m, f, a}} = Counters.child_spec(opts)
diff --git a/test/plausible/rate_limit_test.exs b/test/plausible/rate_limit_test.exs
index 6c6230b75f..a9c2e35778 100644
--- a/test/plausible/rate_limit_test.exs
+++ b/test/plausible/rate_limit_test.exs
@@ -16,7 +16,8 @@ defmodule Plausible.RateLimitTest do
for _ <- 1..3 do
assert {:allow, 1} = RateLimit.check_rate(@table, key, scale, limit)
- assert [{{^key, _bucket}, _count = 1, expires_at}] = :ets.tab2list(@table)
+ assert [{{^key, _bucket}, counter, expires_at}] = :ets.tab2list(@table)
+ assert :atomics.get(counter, 1) == 1
assert expires_at >= System.system_time(:millisecond)
assert expires_at <= System.system_time(:millisecond) + 50