Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Store `tracker_script_version` under `ingest_counters` (#5203)
Karl-Aksel Puulmann7439529fdd01
diff --git a/lib/plausible/ingestion/counters.ex b/lib/plausible/ingestion/counters.exindex ad15327661..2799fe5491 100644--- a/lib/plausible/ingestion/counters.ex+++ b/lib/plausible/ingestion/counters.ex@@ -70,12 +70,13 @@ defmodule Plausible.Ingestion.Counters dorecords ->records =- Enum.map(records, fn {bucket, metric, domain, value} ->+ Enum.map(records, fn {bucket, metric, domain, tracker_script_version, value} ->%{event_timebucket: to_0_minute_datetime(bucket),metric: metric,site_id: Plausible.Site.Cache.get_site_id(domain),domain: domain,+ tracker_script_version: tracker_script_version,value: value}end)diff --git a/lib/plausible/ingestion/counters/buffer.ex b/lib/plausible/ingestion/counters/buffer.exindex 2519149a06..ea8ffffa1b 100644--- a/lib/plausible/ingestion/counters/buffer.ex+++ b/lib/plausible/ingestion/counters/buffer.ex@@ -5,7 +5,7 @@ defmodule Plausible.Ingestion.Counters.Buffer doSee `Plausible.Ingestion.Counters` for integration.Flushing is by default possible only once the 10s bucket is complete- (its window has moved). This is to avoid race conditions+ (its window has moved). This is to avoid race conditionswhen clearing up the buffer on dequeue - because there is no atomic "get and delete",and items are buffered concurrently, there is a gap between get and deletein which items written may disappear otherwise.@@ -45,21 +45,28 @@ defmodule Plausible.Ingestion.Counters.Buffer do}end- @spec aggregate(t(), binary(), binary(), timestamp :: NaiveDateTime.t()) :: t()+ @spec aggregate(+ t(),+ binary(),+ binary(),+ timestamp :: NaiveDateTime.t(),+ tracker_script_version :: integer()+ ) :: t()def aggregate(%__MODULE__{buffer_name: buffer_name, aggregate_bucket_fn: bucket_fn} = buffer,metric,domain,- timestamp+ timestamp,+ tracker_script_version) dobucket =bucket_fn.(timestamp):ets.update_counter(buffer_name,- {bucket, metric, domain},+ {bucket, metric, domain, tracker_script_version},{2, 1},- {{bucket, metric, domain}, 0}+ {{bucket, metric, domain, tracker_script_version}, 0})buffer@@ -72,9 +79,9 @@ defmodule Plausible.Ingestion.Counters.Buffer do) doboundary = flush_boundary_fn.(now)- match = {{:"$1", :"$2", :"$3"}, :"$4"}+ match = {{:"$1", :"$2", :"$3", :"$4"}, :"$5"}guard = {:"=<", :"$1", boundary}- select = {{:"$1", :"$2", :"$3", :"$4"}}+ select = {{:"$1", :"$2", :"$3", :"$4", :"$5"}}match_specs_read = [{match, [guard], [select]}]match_specs_delete = [{match, [guard], [true]}]diff --git a/lib/plausible/ingestion/counters/record.ex b/lib/plausible/ingestion/counters/record.exindex 8b79d721ca..ae8ce399b3 100644--- a/lib/plausible/ingestion/counters/record.ex+++ b/lib/plausible/ingestion/counters/record.ex@@ -13,5 +13,6 @@ defmodule Plausible.Ingestion.Counters.Record dofield :domain, Ch, type: "LowCardinality(String)"field :metric, Ch, type: "LowCardinality(String)"field :value, Ch, type: "UInt64"+ field :tracker_script_version, Ch, type: "UInt16"endenddiff --git a/lib/plausible/ingestion/counters/telemetry_handler.ex b/lib/plausible/ingestion/counters/telemetry_handler.exindex 1abfa83ca3..8caa37681e 100644--- a/lib/plausible/ingestion/counters/telemetry_handler.ex+++ b/lib/plausible/ingestion/counters/telemetry_handler.ex@@ -30,20 +30,36 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandler dodef handle_event(@event_dropped,_measurements,- %{domain: domain, reason: reason, request_timestamp: timestamp},+ %{+ domain: domain,+ reason: reason,+ request_timestamp: timestamp,+ tracker_script_version: tracker_script_version+ },buffer) do- Counters.Buffer.aggregate(buffer, "dropped_#{reason}", domain, timestamp)+ Counters.Buffer.aggregate(+ buffer,+ "dropped_#{reason}",+ domain,+ timestamp,+ tracker_script_version+ )+:okenddef handle_event(@event_buffered,_measurements,- %{domain: domain, request_timestamp: timestamp},+ %{+ domain: domain,+ request_timestamp: timestamp,+ tracker_script_version: tracker_script_version+ },buffer) do- Counters.Buffer.aggregate(buffer, "buffered", domain, timestamp)+ Counters.Buffer.aggregate(buffer, "buffered", domain, timestamp, tracker_script_version):okendenddiff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex acc3a73eb3..da4b7f9525 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -96,7 +96,8 @@ defmodule Plausible.Ingestion.Event dodef emit_telemetry_buffered(event) do:telemetry.execute(telemetry_event_buffered(), %{}, %{domain: event.domain,- request_timestamp: event.request.timestamp+ request_timestamp: event.request.timestamp,+ tracker_script_version: event.request.tracker_script_version})end@@ -108,7 +109,8 @@ defmodule Plausible.Ingestion.Event do%{domain: event.domain,reason: reason,- request_timestamp: event.request.timestamp+ request_timestamp: event.request.timestamp,+ tracker_script_version: event.request.tracker_script_version})enddiff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex 4548ef5a3b..19ec409a6e 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -43,6 +43,7 @@ defmodule Plausible.Ingestion.Request dofield :props, :mapfield :scroll_depth, :integerfield :engagement_time, :integer+ field :tracker_script_version, :integer, default: 0on_ee dofield :revenue_source, :map@@ -84,6 +85,7 @@ defmodule Plausible.Ingestion.Request do|> put_pathname()|> put_query_params()|> put_revenue_source(request_body)+ |> put_tracker_script_version(request_body)|> map_domains(request_body)|> Changeset.validate_required([:event_name,@@ -270,6 +272,16 @@ defmodule Plausible.Ingestion.Request doendend+ defp put_tracker_script_version(changeset, %{} = request_body) do+ case request_body["v"] do+ version when is_integer(version) ->+ Changeset.put_change(changeset, :tracker_script_version, version)++ _ ->+ changeset+ end+ end+defp put_query_params(changeset) docase Changeset.get_field(changeset, :uri) do%{query: query} when is_binary(query) ->diff --git a/test/plausible/ingestion/counters/buffer_test.exs b/test/plausible/ingestion/counters/buffer_test.exsindex e67a23e234..5d0d98523d 100644--- a/test/plausible/ingestion/counters/buffer_test.exs+++ b/test/plausible/ingestion/counters/buffer_test.exs@@ -4,7 +4,7 @@ defmodule Plausible.Ingestion.Counters.BufferTest dotest "10s buckets are created from input datetime" do# time (...) :58 :59 :00 :01 :02 :03 :04 :05 :06 :07 :08 :09 :10 :11 :12 :13 (...)- # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10+ # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10test_input = [%{input: ~N[2023-02-14 01:00:00], bucket: ~U[2023-02-14 01:00:00Z]},@@ -25,9 +25,9 @@ defmodule Plausible.Ingestion.Counters.BufferTest dotest "aggregates metrics every 10 seconds", %{test: test} do# time (...) :58 :59 :00 :01 :02 :03 :04 :05 :06 :07 :08 :09 :10 :11 :12 :13 (...)- # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10- # metric x x x x- # value 1 1 2 3+ # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10+ # metric x x x x+ # value 1 1 2 3timestamps = [~N[2023-02-14 01:00:59],@@ -39,12 +39,12 @@ defmodule Plausible.Ingestion.Counters.BufferTest dobuffer = Buffer.new(test)for ts <- timestamps do- Buffer.aggregate(buffer, "metric", "example.com", ts)+ Buffer.aggregate(buffer, "metric", "example.com", ts, 0)endassert [- {bucket1, "metric", "example.com", 1},- {bucket2, "metric", "example.com", 3}+ {bucket1, "metric", "example.com", 0, 1},+ {bucket2, "metric", "example.com", 0, 3}] = Buffer.flush(buffer)assert bucket2 - bucket1 == 10@@ -52,9 +52,9 @@ defmodule Plausible.Ingestion.Counters.BufferTest dotest "allows flushing only complete buckets", %{test: test} do# time (...) :58 :59 :00 :01 :02 :03 :04 :05 :06 :07 :08 :09 :10 :11 :12 :13 (...)- # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10- # metric x x x x- # aggregate 1 0 1 1 2 2 2 2 2 2 3 3 0+ # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10+ # metric x x x x+ # aggregate 1 0 1 1 2 2 2 2 2 2 3 3 0# flush attempt x x x x# flushed count 0 1 0 3@@ -68,12 +68,19 @@ defmodule Plausible.Ingestion.Counters.BufferTest dobuffer = Buffer.new(test)for ts <- timestamps do- Buffer.aggregate(buffer, "metric", "example.com", ts)+ Buffer.aggregate(buffer, "metric", "example.com", ts, 0)endassert [] = Buffer.flush(buffer, ~U[2023-02-14 01:00:59.999999Z])- assert [{_, _, _, 1}] = Buffer.flush(buffer, ~U[2023-02-14 01:01:00.999999Z])+ assert [{_, _, _, 0, 1}] = Buffer.flush(buffer, ~U[2023-02-14 01:01:00.999999Z])assert [] = Buffer.flush(buffer, ~U[2023-02-14 01:01:05.999999Z])- assert [{_, _, _, 3}] = Buffer.flush(buffer, ~U[2023-02-14 01:01:11.999999Z])+ assert [{_, _, _, 0, 3}] = Buffer.flush(buffer, ~U[2023-02-14 01:01:11.999999Z])+ end++ test "allows setting tracker script version", %{test: test} do+ buffer = Buffer.new(test)++ Buffer.aggregate(buffer, "metric", "example.com", ~N[2023-02-14 01:00:59], 137)+ assert [{_, "metric", "example.com", 137, 1}] = Buffer.flush(buffer)endenddiff --git a/test/plausible/ingestion/counters/telemetry_handler_test.exs b/test/plausible/ingestion/counters/telemetry_handler_test.exsindex cd13699f95..084fb31ab7 100644--- a/test/plausible/ingestion/counters/telemetry_handler_test.exs+++ b/test/plausible/ingestion/counters/telemetry_handler_test.exs@@ -34,9 +34,20 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest dobuffer = Buffer.new(test)assert :ok = TelemetryHandler.install(buffer)- e1 = %{domain: "a.example.com", request: %{timestamp: NaiveDateTime.utc_now()}}- e2 = %{domain: "b.example.com", request: %{timestamp: NaiveDateTime.utc_now()}}- e3 = %{domain: "c.example.com", request: %{timestamp: NaiveDateTime.utc_now()}}+ e1 = %{+ domain: "a.example.com",+ request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}+ }++ e2 = %{+ domain: "b.example.com",+ request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}+ }++ e3 = %{+ domain: "c.example.com",+ request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}+ }:ok = Event.emit_telemetry_dropped(e1, :invalid):ok = Event.emit_telemetry_dropped(e2, :not_found)@@ -47,8 +58,8 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest doassert aggregates = Buffer.flush(buffer, future)- assert Enum.find(aggregates, &match?({_, "dropped_invalid", "a.example.com", 1}, &1))- assert Enum.find(aggregates, &match?({_, "dropped_not_found", "b.example.com", 2}, &1))- assert Enum.find(aggregates, &match?({_, "buffered", "c.example.com", 1}, &1))+ assert Enum.find(aggregates, &match?({_, "dropped_invalid", "a.example.com", 137, 1}, &1))+ assert Enum.find(aggregates, &match?({_, "dropped_not_found", "b.example.com", 137, 2}, &1))+ assert Enum.find(aggregates, &match?({_, "buffered", "c.example.com", 137, 1}, &1))endenddiff --git a/test/plausible/ingestion/counters_test.exs b/test/plausible/ingestion/counters_test.exsindex d7ce3b75a6..e60d091a25 100644--- a/test/plausible/ingestion/counters_test.exs+++ b/test/plausible/ingestion/counters_test.exs@@ -116,7 +116,8 @@ defmodule Plausible.Ingestion.CountersTest dopayload = %{name: "pageview",- url: "http://#{site.domain}"+ url: "http://#{site.domain}",+ v: 137}conn = build_conn(:post, "/api/event", payload)@@ -133,7 +134,8 @@ defmodule Plausible.Ingestion.CountersTest dopayload = %{name: "pageview",- url: "http://#{site.domain}"+ url: "http://#{site.domain}",+ v: 137}conn = build_conn(:post, "/api/event", payload)@@ -152,8 +154,8 @@ defmodule Plausible.Ingestion.CountersTest dodefp verify_record_written(domain, metric, value, site_id \\ nil) doquery =from(r in Record,- group_by: [:site_id, :domain, :metric, :event_timebucket],- where: r.domain == ^domain and r.metric == ^metric,+ group_by: [:site_id, :domain, :metric, :event_timebucket, :tracker_script_version],+ where: r.domain == ^domain and r.metric == ^metric and r.tracker_script_version == 137,select: sum(r.value))diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exsindex ac102a42a8..6a80c5df44 100644--- a/test/plausible/ingestion/request_test.exs+++ b/test/plausible/ingestion/request_test.exs@@ -159,7 +159,8 @@ defmodule Plausible.Ingestion.RequestTest doprops: %{"custom1" => "property1","custom2" => "property2"- }+ },+ v: 137}conn = build_conn(:post, "/api/events", payload)@@ -169,6 +170,7 @@ defmodule Plausible.Ingestion.RequestTest doassert request.hash_mode == 1assert request.props["custom1"] == "property1"assert request.props["custom2"] == "property2"+ assert request.tracker_script_version == 137end@tag :ee_only@@ -491,7 +493,8 @@ defmodule Plausible.Ingestion.RequestTest do"user_agent" => "Mozilla","ip_classification" => nil,"scroll_depth" => nil,- "engagement_time" => nil+ "engagement_time" => nil,+ "tracker_script_version" => 0}assert %NaiveDateTime{} = NaiveDateTime.from_iso8601!(request["timestamp"])