Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Ingest throughput fixes (#5378)
hq1aa4a8339cb34
diff --git a/Makefile b/Makefileindex d755b1a1c3..811c303343 100644--- a/Makefile+++ b/Makefile@@ -56,3 +56,12 @@ minio: ## Start a transient container with a recent version of minio (s3)minio-stop: ## Stop and remove the minio containerdocker stop plausible_minio++loadtest-server:+ @echo "Ensure your OTP installation is built with --enable-lock-counter"+ MIX_ENV=dev LOADTEST=1 ERL_FLAGS="-emu_type lcnt +Mdai max" iex -S mix phx.server++loadtest-client:+ @echo "Set your limits for file descriptors/ephermeal ports high... Test begins shortly"+ @sleep 5+ k6 run test/load/script.jsdiff --git a/config/dev.exs b/config/dev.exsindex 564dcb86a0..ce5a34f632 100644--- a/config/dev.exs+++ b/config/dev.exs@@ -3,29 +3,8 @@ import Configconfig :plausible, PlausibleWeb.Endpoint,server: true,debug_errors: true,- code_reloader: true,- check_origin: false,- watchers: [- esbuild: {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]},- tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]},- storybook_tailwind: {Tailwind, :install_and_run, [:storybook, ~w(--watch)]},- npm: ["--prefix", "assets", "run", "typecheck", "--", "--watch", "--preserveWatchOutput"],- npm: [- "run",- "deploy",- cd: Path.expand("../tracker", __DIR__)- ]- ],- live_reload: [- dirs: [- "extra"- ],- patterns: [- ~r{priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$},- ~r"lib/plausible_web/(controllers|live|components|templates|views|plugs)/.*(ex|heex)$",- ~r"storybook/.*(exs)$"- ]- ]+ code_reloader: false,+ check_origin: falseconfig :plausible, paddle_api: Plausible.Billing.DevPaddleApiMockdiff --git a/config/runtime.exs b/config/runtime.exsindex 3052d4f3e2..756690d923 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -331,6 +331,41 @@ config :plausible, PlausibleWeb.Endpoint,websocket_url: websocket_url,secure_cookie: secure_cookie+if config_env() in [:dev, :ce_dev] do+ loadtest_mode = get_var_from_path_or_env(config_dir, "LOADTEST")++ if loadtest_mode do+ config :plausible, PlausibleWeb.Endpoint,+ debug_errors: false,+ code_reloader: false+ else+ config :plausible, PlausibleWeb.Endpoint,+ debug_errors: true,+ code_reloader: true,+ watchers: [+ esbuild: {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]},+ tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]},+ storybook_tailwind: {Tailwind, :install_and_run, [:storybook, ~w(--watch)]},+ npm: ["--prefix", "assets", "run", "typecheck", "--", "--watch", "--preserveWatchOutput"],+ npm: [+ "run",+ "deploy",+ cd: Path.expand("../tracker", __DIR__)+ ]+ ],+ live_reload: [+ dirs: [+ "extra"+ ],+ patterns: [+ ~r{priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$},+ ~r"lib/plausible_web/(controllers|live|components|templates|views|plugs)/.*(ex|heex)$",+ ~r"storybook/.*(exs)$"+ ]+ ]+ end+end+# maybe enable HTTPS in CEif config_env() in [:ce, :ce_dev, :ce_test] doif https_port dodiff --git a/lib/plausible/ingestion/counters/telemetry_handler.ex b/lib/plausible/ingestion/counters/telemetry_handler.exindex 34523a1e8c..8caa37681e 100644--- a/lib/plausible/ingestion/counters/telemetry_handler.ex+++ b/lib/plausible/ingestion/counters/telemetry_handler.ex@@ -1,7 +1,7 @@defmodule Plausible.Ingestion.Counters.TelemetryHandler do@moduledoc """Subscribes to telemetry events emitted by `Plausible.Ingestion.Event`.- Every time a request derived event is dropped,+ Every time a request derived event is either dispatched to clickhouse or dropped,a telemetry event is emitted respectively. That event is captured here,its metadata is extracted and sent for internal stats aggregation via`Counters.Buffer` interface.@@ -10,8 +10,9 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandler doalias Plausible.Ingestion.Event@event_dropped Event.telemetry_event_dropped()+ @event_buffered Event.telemetry_event_buffered()- @telemetry_events [@event_dropped]+ @telemetry_events [@event_dropped, @event_buffered]@telemetry_handler &__MODULE__.handle_event/4@spec install(Counters.Buffer.t()) :: :ok@@ -47,4 +48,18 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandler do:okend++ def handle_event(+ @event_buffered,+ _measurements,+ %{+ domain: domain,+ request_timestamp: timestamp,+ tracker_script_version: tracker_script_version+ },+ buffer+ ) do+ Counters.Buffer.aggregate(buffer, "buffered", domain, timestamp, tracker_script_version)+ :ok+ endenddiff --git a/lib/plausible/rate_limit.ex b/lib/plausible/rate_limit.exindex 7e1d01d73b..a92f579842 100644--- a/lib/plausible/rate_limit.ex+++ b/lib/plausible/rate_limit.ex@@ -10,16 +10,14 @@ defmodule Plausible.RateLimit doStarts the process that creates and cleans the ETS table.Accepts the following options:- - `GenServer.options()`+ - `GenServer.option()`- `:table` for the ETS table name, defaults to `#{__MODULE__}`- `:clean_period` for how often to perform garbage collection"""@spec start_link([GenServer.option() | {:table, atom} | {:clean_period, pos_integer}]) ::GenServer.on_start()def start_link(opts) do- {gen_opts, opts} =- Keyword.split(opts, [:debug, :name, :timeout, :spawn_opt, :hibernate_after])-+ {gen_opts, opts} = Keyword.split(opts, [:debug, :name, :spawn_opt, :hibernate_after])GenServer.start_link(__MODULE__, opts, gen_opts)end@@ -36,7 +34,25 @@ defmodule Plausible.RateLimit dobucket = div(now(), scale)full_key = {key, bucket}expires_at = (bucket + 1) * scale- count = :ets.update_counter(table, full_key, increment, {full_key, 0, expires_at})++ count =+ case :ets.lookup(table, full_key) do+ [{_, counter, _expires_at}] ->+ :atomics.add_get(counter, 1, increment)++ [] ->+ counter = :atomics.new(1, signed: false)++ case :ets.insert_new(table, {full_key, counter, expires_at}) do+ true ->+ :atomics.add_get(counter, 1, increment)++ false ->+ [{_, counter, _expires_at}] = :ets.lookup(table, full_key)+ :atomics.add_get(counter, 1, increment)+ end+ end+if count <= limit, do: {:allow, count}, else: {:deny, limit}end@@ -51,7 +67,7 @@ defmodule Plausible.RateLimit do:set,:public,{:read_concurrency, true},- {:write_concurrency, true},+ {:write_concurrency, :auto},{:decentralized_counters, true}])diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.exindex ec784b0758..49c6dce3c8 100644--- a/lib/plausible/telemetry/plausible_metrics.ex+++ b/lib/plausible/telemetry/plausible_metrics.ex@@ -82,12 +82,14 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics docounter(metric_prefix ++ [:plausible_cache, :hit],event_name: ConCache.Operations.telemetry_hit(),- tags: [:name]+ tags: [:name],+ tag_values: &%{name: &1.cache.name}),counter(metric_prefix ++ [:plausible_cache, :miss],event_name: ConCache.Operations.telemetry_miss(),- tags: [:name]+ tags: [:name],+ tag_values: &%{name: &1.cache.name}),distribution(metric_prefix ++ [:sessions, :transfer, :duration],diff --git a/rel/vm.args.eex b/rel/vm.args.eexindex 957d983203..c608ce8ea3 100644--- a/rel/vm.args.eex+++ b/rel/vm.args.eex@@ -9,3 +9,4 @@## Tweak GC to run more often##-env ERL_FULLSWEEP_AFTER 10++Mdai maxdiff --git a/test/load/script.js b/test/load/script.jsindex 7be00a7233..6936375417 100644--- a/test/load/script.js+++ b/test/load/script.js@@ -1,54 +1,44 @@import http from "k6/http";const PAYLOAD = JSON.stringify({- "n":"pageview",- "u":"http://loadtest.site/some-page",- "d":"loadtest.site",- "r":null,- "w":1666+ n: "pageview",+ u: "http://dummy.site/some-page",+ d: "dummy.site",+ r: null,+ w: 1666,});function newParams() {- const ip = (Math.floor(Math.random() * 255) + 1)+"."+(Math.floor(Math.random() * 255))+"."+(Math.floor(Math.random() * 255))+"."+(Math.floor(Math.random() * 255));+ const ip =+ Math.floor(Math.random() * 255) ++ 1 ++ "." ++ Math.floor(Math.random() * 255) ++ "." ++ Math.floor(Math.random() * 255) ++ "." ++ Math.floor(Math.random() * 255);return {headers: {- 'Content-Type': 'application/json',- 'X-Forwarded-For': ip,- 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36 OPR/71.0.3770.284'- }- }+ "Content-Type": "application/json",+ "X-Forwarded-For": ip,+ "User-Agent": `${Math.random() > 0.5 ? "Mozilla/5.0" : "Mozilla/4.0"} (${Math.random() > 0.5 ? "Macintosh" : "Windows"}; ${Math.random() > 0.5 ? "Intel Mac OS X 10_15_6" : "Windows NT 10.0"}) AppleWebKit/${Math.floor(Math.random() * 1000) + 500}.36 (KHTML, like Gecko) Chrome/${Math.floor(Math.random() * 100) + 50}.0.${Math.floor(Math.random() * 5000) + 1000}.${Math.floor(Math.random() * 500)} Safari/${Math.floor(Math.random() * 1000) + 500}.${Math.floor(Math.random() * 100)} OPR/${Math.floor(Math.random() * 100)}.0.${Math.floor(Math.random() * 5000) + 1000}.${Math.floor(Math.random() * 500)}`,+ },+ };}export const options = {- scenarios: {- wave1: {- executor: 'constant-vus',- startTime: '0s',- vus: 200,- duration: '60s'- },- wave2: {- executor: 'constant-vus',- startTime: '60s',- vus: 200,- duration: '60s'- },- wave3: {- executor: 'constant-vus',- startTime: '120s',- vus: 200,- duration: '60s'- },- wave4: {- executor: 'constant-vus',- startTime: '180s',- vus: 200,- duration: '60s'- },- },+ stages: [+ { target: 10000, duration: "10s" },+ { target: 15000, duration: "20s" },+ { target: 20000, duration: "30s" },+ { target: 20000, duration: "30s" },+ { target: 30000, duration: "30s" },+ { target: 30000, duration: "90s" },+ ],};-export default function() {+export default function () {http.post("http://localhost:8000/api/event", PAYLOAD, newParams());-};+}diff --git a/test/plausible/ingestion/counters/telemetry_handler_test.exs b/test/plausible/ingestion/counters/telemetry_handler_test.exsindex 8a668f9c62..084fb31ab7 100644--- a/test/plausible/ingestion/counters/telemetry_handler_test.exs+++ b/test/plausible/ingestion/counters/telemetry_handler_test.exs@@ -19,6 +19,11 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest dohandler.config == buffer andhandler.event_name == Event.telemetry_event_dropped()end)++ assert Enum.find(all_handlers, fn handler ->+ handler.config == buffer and+ handler.event_name == Event.telemetry_event_buffered()+ end)endtest "handles ingest events by aggregating the counts", %{test: test} do@@ -39,8 +44,14 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest dorequest: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}}+ e3 = %{+ domain: "c.example.com",+ request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}+ }+:ok = Event.emit_telemetry_dropped(e1, :invalid):ok = Event.emit_telemetry_dropped(e2, :not_found)+ :ok = Event.emit_telemetry_buffered(e3):ok = Event.emit_telemetry_dropped(e2, :not_found)future = DateTime.utc_now() |> DateTime.add(120, :second)@@ -49,5 +60,6 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest doassert Enum.find(aggregates, &match?({_, "dropped_invalid", "a.example.com", 137, 1}, &1))assert Enum.find(aggregates, &match?({_, "dropped_not_found", "b.example.com", 137, 2}, &1))+ assert Enum.find(aggregates, &match?({_, "buffered", "c.example.com", 137, 1}, &1))endenddiff --git a/test/plausible/ingestion/counters_test.exs b/test/plausible/ingestion/counters_test.exsindex 06ec462942..e60d091a25 100644--- a/test/plausible/ingestion/counters_test.exs+++ b/test/plausible/ingestion/counters_test.exs@@ -38,8 +38,12 @@ defmodule Plausible.Ingestion.CountersTest do{:ok, dropped} = emit_dropped_request(at: now){:ok, _dropped} = emit_dropped_request(domain: dropped.domain, at: now)+ {:ok, buffered} = emit_buffered_request(at: now)verify_record_written(dropped.domain, "dropped_not_found", 2)++ site_id = Plausible.Sites.get_by_domain(buffered.domain).id+ verify_record_written(buffered.domain, "buffered", 1, site_id)endtest "the database eventually sums the records within 1-minute buckets", %{test: test} do@@ -122,6 +126,25 @@ defmodule Plausible.Ingestion.CountersTest do{:ok, dropped}end+ defp emit_buffered_request(opts) do+ domain = Keyword.get(opts, :domain, random_domain())+ at = Keyword.get(opts, :at, @ts)++ site = new_site(domain: domain)++ payload = %{+ name: "pageview",+ url: "http://#{site.domain}",+ v: 137+ }++ conn = build_conn(:post, "/api/event", payload)+ assert {:ok, request} = Request.build(conn, at)+ assert {:ok, %{buffered: [buffered]}} = Event.build_and_buffer(request)++ {:ok, buffered}+ end+defp start_counters(opts) doopts = Keyword.put(opts, :force_start?, true)%{start: {m, f, a}} = Counters.child_spec(opts)diff --git a/test/plausible/rate_limit_test.exs b/test/plausible/rate_limit_test.exsindex 6c6230b75f..a9c2e35778 100644--- a/test/plausible/rate_limit_test.exs+++ b/test/plausible/rate_limit_test.exs@@ -16,7 +16,8 @@ defmodule Plausible.RateLimitTest dofor _ <- 1..3 doassert {:allow, 1} = RateLimit.check_rate(@table, key, scale, limit)- assert [{{^key, _bucket}, _count = 1, expires_at}] = :ets.tab2list(@table)+ assert [{{^key, _bucket}, counter, expires_at}] = :ets.tab2list(@table)+ assert :atomics.get(counter, 1) == 1assert expires_at >= System.system_time(:millisecond)assert expires_at <= System.system_time(:millisecond) + 50