Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Revert "experiment in improving throughput of write buffers (#3511)" (#3614)
hq17b5c20db1764
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a48e8fd44a..ad7e438779 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -27,7 +27,6 @@ All notable changes to this project will be documented in this file.- Show Outbound Links / File Downloads / 404 Pages / Cloaked Links instead of Goal Conversions when filtering by the corresponding goal- Require custom properties to be explicitly added from Site Settings > Custom Properties in order for them to show up on the dashboard- GA/SC sections moved to new settings: Integrations-- `CLICKHOUSE_MAX_BUFFER_SIZE` now represents bytes and defaults to `100000` (100KB)### Fixed- Only return `(none)` values in custom property breakdown for the first page (pagination) of resultsdiff --git a/config/runtime.exs b/config/runtime.exsindex 986a8b09da..27f1b54137 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -142,7 +142,7 @@ ch_db_url ={ch_max_buffer_size, ""} =config_dir- |> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "100000")+ |> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")|> Integer.parse()# Can be generated with `Base.encode64(:crypto.strong_rand_bytes(32))` fromdiff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex 886993d12b..b9d58c3e5c 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -17,8 +17,8 @@ defmodule Plausible.Application do{Finch, name: Plausible.Finch, pools: finch_pool_config()},{Phoenix.PubSub, name: Plausible.PubSub},Plausible.Session.Salts,- Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer),- Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer),+ Plausible.Event.WriteBuffer,+ Plausible.Session.WriteBuffer,ReferrerBlocklist,Supervisor.child_spec({Cachex, name: :user_agents, limit: 10_000, stats: true},id: :cachex_user_agentsdiff --git a/lib/plausible/event/write_buffer.ex b/lib/plausible/event/write_buffer.exindex 3966b5a3f7..37a9cd39b0 100644--- a/lib/plausible/event/write_buffer.ex+++ b/lib/plausible/event/write_buffer.ex@@ -1,38 +1,79 @@defmodule Plausible.Event.WriteBuffer do- @moduledoc false+ use GenServer+ require Logger- %{- header: header,- insert_sql: insert_sql,- insert_opts: insert_opts,- fields: fields,- encoding_types: encoding_types- } =- Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseEventV2)+ alias Plausible.IngestRepo- def child_spec(opts) do- opts =- Keyword.merge(opts,- name: __MODULE__,- header: unquote(header),- insert_sql: unquote(insert_sql),- insert_opts: unquote(insert_opts)- )+ def start_link(_opts) do+ GenServer.start_link(__MODULE__, [], name: __MODULE__)+ end- Plausible.Ingestion.WriteBuffer.child_spec(opts)+ def init(buffer) do+ Process.flag(:trap_exit, true)+ timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:ok, %{buffer: buffer, timer: timer}}enddef insert(event) do- row_binary =- [Enum.map(unquote(fields), fn field -> Map.fetch!(event, field) end)]- |> Ch.RowBinary._encode_rows(unquote(encoding_types))- |> IO.iodata_to_binary()-- :ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary)+ GenServer.cast(__MODULE__, {:insert, event}){:ok, event}end- def flush do- Plausible.Ingestion.WriteBuffer.flush(__MODULE__)+ def flush() do+ GenServer.call(__MODULE__, :flush, :infinity)+ :ok+ end++ def handle_cast({:insert, event}, %{buffer: buffer} = state) do+ new_buffer = [event | buffer]++ if length(new_buffer) >= max_buffer_size() do+ Logger.info("Buffer full, flushing to disk")+ Process.cancel_timer(state[:timer])+ do_flush(new_buffer)+ new_timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:noreply, %{buffer: [], timer: new_timer}}+ else+ {:noreply, %{state | buffer: new_buffer}}+ end+ end++ def handle_info(:tick, %{buffer: buffer}) do+ do_flush(buffer)+ timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:noreply, %{buffer: [], timer: timer}}+ end++ def handle_call(:flush, _from, %{buffer: buffer} = state) do+ Process.cancel_timer(state[:timer])+ do_flush(buffer)+ new_timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:reply, nil, %{buffer: [], timer: new_timer}}+ end++ def terminate(_reason, %{buffer: buffer}) do+ Logger.info("Flushing event buffer before shutdown...")+ do_flush(buffer)+ end++ defp do_flush(buffer) do+ case buffer do+ [] ->+ nil++ events ->+ Logger.info("Flushing #{length(events)} events")+ events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__)))++ IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)+ end+ end++ defp flush_interval_ms() do+ Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)+ end++ defp max_buffer_size() do+ Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)endenddiff --git a/lib/plausible/ingestion/write_buffer.ex b/lib/plausible/ingestion/write_buffer.exdeleted file mode 100644index 7854357fb5..0000000000--- a/lib/plausible/ingestion/write_buffer.ex+++ /dev/null@@ -1,150 +0,0 @@-defmodule Plausible.Ingestion.WriteBuffer do- @moduledoc false- use GenServer- require Logger-- alias Plausible.IngestRepo-- def start_link(opts) do- GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name))- end-- def insert(server, row_binary) do- GenServer.cast(server, {:insert, row_binary})- end-- def flush(server) do- GenServer.call(server, :flush, :infinity)- end-- @impl true- def init(opts) do- buffer = opts[:buffer] || []- max_buffer_size = opts[:max_buffer_size] || default_max_buffer_size()- flush_interval_ms = opts[:flush_interval_ms] || default_flush_interval_ms()-- Process.flag(:trap_exit, true)- timer = Process.send_after(self(), :tick, flush_interval_ms)-- {:ok,- %{- buffer: buffer,- timer: timer,- name: Keyword.fetch!(opts, :name),- insert_sql: Keyword.fetch!(opts, :insert_sql),- insert_opts: Keyword.fetch!(opts, :insert_opts),- header: Keyword.fetch!(opts, :header),- buffer_size: IO.iodata_length(buffer),- max_buffer_size: max_buffer_size,- flush_interval_ms: flush_interval_ms- }}- end-- @impl true-- def handle_cast({:insert, row_binary}, state) do- state = %{- state- | buffer: [state.buffer | row_binary],- buffer_size: state.buffer_size + IO.iodata_length(row_binary)- }-- if state.buffer_size >= state.max_buffer_size do- Logger.info("#{state.name} buffer full, flushing to ClickHouse")- Process.cancel_timer(state.timer)- do_flush(state)- new_timer = Process.send_after(self(), :tick, state.flush_interval_ms)- {:noreply, %{state | buffer: [], timer: new_timer, buffer_size: 0}}- else- {:noreply, state}- end- end-- @impl true- def handle_info(:tick, state) do- do_flush(state)- timer = Process.send_after(self(), :tick, state.flush_interval_ms)- {:noreply, %{state | buffer: [], buffer_size: 0, timer: timer}}- end-- @impl true- def handle_call(:flush, _from, state) do- %{timer: timer, flush_interval_ms: flush_interval_ms} = state- Process.cancel_timer(timer)- do_flush(state)- new_timer = Process.send_after(self(), :tick, flush_interval_ms)- {:reply, :ok, %{state | buffer: [], buffer_size: 0, timer: new_timer}}- end-- @impl true- def terminate(_reason, %{name: name} = state) do- Logger.info("Flushing #{name} buffer before shutdown...")- do_flush(state)- end-- defp do_flush(state) do- %{- buffer: buffer,- buffer_size: buffer_size,- insert_opts: insert_opts,- insert_sql: insert_sql,- header: header,- name: name- } = state-- case buffer do- [] ->- nil-- _not_empty ->- Logger.info("Flushing #{buffer_size} byte(s) RowBinary from #{name}")- IngestRepo.query!(insert_sql, [header | buffer], insert_opts)- end- end-- defp default_flush_interval_ms do- Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)- end-- defp default_max_buffer_size do- Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)- end-- @doc false- def compile_time_prepare(schema) do- fields = schema.__schema__(:fields)-- types =- Enum.map(fields, fn field ->- type = schema.__schema__(:type, field) || raise "missing type for #{field}"-- type- |> Ecto.Type.type()- |> Ecto.Adapters.ClickHouse.Schema.remap_type(schema, field)- end)-- encoding_types = Ch.RowBinary.encoding_types(types)-- header =- fields- |> Enum.map(&to_string/1)- |> Ch.RowBinary.encode_names_and_types(types)- |> IO.iodata_to_binary()-- insert_sql = "INSERT INTO #{schema.__schema__(:source)} FORMAT RowBinaryWithNamesAndTypes"-- %{- fields: fields,- types: types,- encoding_types: encoding_types,- header: header,- insert_sql: insert_sql,- insert_opts: [- command: :insert,- encode: false,- source: schema.__schema__(:source),- cast_params: []- ]- }- end-enddiff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.exindex 1adcbbb67d..770fc965c1 100644--- a/lib/plausible/session/write_buffer.ex+++ b/lib/plausible/session/write_buffer.ex@@ -1,43 +1,83 @@defmodule Plausible.Session.WriteBuffer do- @moduledoc false+ use GenServer+ require Logger- %{- header: header,- insert_sql: insert_sql,- insert_opts: insert_opts,- fields: fields,- encoding_types: encoding_types- } =- Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseSessionV2)+ alias Plausible.IngestRepo- def child_spec(opts) do- opts =- Keyword.merge(opts,- name: __MODULE__,- header: unquote(header),- insert_sql: unquote(insert_sql),- insert_opts: unquote(insert_opts)- )+ def start_link(_opts) do+ GenServer.start_link(__MODULE__, [], name: __MODULE__)+ end- Plausible.Ingestion.WriteBuffer.child_spec(opts)+ def init(buffer) do+ Process.flag(:trap_exit, true)+ timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:ok, %{buffer: buffer, timer: timer}}enddef insert(sessions) do- row_binary =- sessions- |> Enum.map(fn %{is_bounce: is_bounce} = session ->- {:ok, is_bounce} = Plausible.ClickhouseSessionV2.BoolUInt8.dump(is_bounce)- session = %{session | is_bounce: is_bounce}- Enum.map(unquote(fields), fn field -> Map.fetch!(session, field) end)- end)- |> Ch.RowBinary._encode_rows(unquote(encoding_types))- |> IO.iodata_to_binary()-- :ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary)+ GenServer.cast(__MODULE__, {:insert, sessions}){:ok, sessions}end- def flush do- Plausible.Ingestion.WriteBuffer.flush(__MODULE__)+ def flush() do+ GenServer.call(__MODULE__, :flush, :infinity)+ :ok+ end++ def handle_cast({:insert, sessions}, %{buffer: buffer} = state) do+ new_buffer = sessions ++ buffer++ if length(new_buffer) >= max_buffer_size() do+ Logger.info("Buffer full, flushing to disk")+ Process.cancel_timer(state[:timer])+ do_flush(new_buffer)+ new_timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:noreply, %{buffer: [], timer: new_timer}}+ else+ {:noreply, %{state | buffer: new_buffer}}+ end+ end++ def handle_info(:tick, %{buffer: buffer}) do+ do_flush(buffer)+ timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:noreply, %{buffer: [], timer: timer}}+ end++ def handle_call(:flush, _from, %{buffer: buffer} = state) do+ Process.cancel_timer(state[:timer])+ do_flush(buffer)+ new_timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:reply, nil, %{buffer: [], timer: new_timer}}+ end++ def terminate(_reason, %{buffer: buffer}) do+ Logger.info("Flushing session buffer before shutdown...")+ do_flush(buffer)+ end++ defp do_flush(buffer) do+ case buffer do+ [] ->+ nil++ sessions ->+ Logger.info("Flushing #{length(sessions)} sessions")++ sessions =+ sessions+ |> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__)))+ |> Enum.reverse()++ IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)+ end+ end++ defp flush_interval_ms() do+ Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)+ end++ defp max_buffer_size() do+ Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)endend