Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Store `tracker_script_version` under `ingest_counters` (#5203)

Karl-Aksel Puulmann7439529fdd01
diff --git a/lib/plausible/ingestion/counters.ex b/lib/plausible/ingestion/counters.ex
index ad15327661..2799fe5491 100644
--- a/lib/plausible/ingestion/counters.ex
+++ b/lib/plausible/ingestion/counters.ex
@@ -70,12 +70,13 @@ defmodule Plausible.Ingestion.Counters do
records ->
records =
- Enum.map(records, fn {bucket, metric, domain, value} ->
+ Enum.map(records, fn {bucket, metric, domain, tracker_script_version, value} ->
%{
event_timebucket: to_0_minute_datetime(bucket),
metric: metric,
site_id: Plausible.Site.Cache.get_site_id(domain),
domain: domain,
+ tracker_script_version: tracker_script_version,
value: value
}
end)
diff --git a/lib/plausible/ingestion/counters/buffer.ex b/lib/plausible/ingestion/counters/buffer.ex
index 2519149a06..ea8ffffa1b 100644
--- a/lib/plausible/ingestion/counters/buffer.ex
+++ b/lib/plausible/ingestion/counters/buffer.ex
@@ -5,7 +5,7 @@ defmodule Plausible.Ingestion.Counters.Buffer do
See `Plausible.Ingestion.Counters` for integration.
Flushing is by default possible only once the 10s bucket is complete
- (its window has moved). This is to avoid race conditions
+ (its window has moved). This is to avoid race conditions
when clearing up the buffer on dequeue - because there is no atomic "get and delete",
and items are buffered concurrently, there is a gap between get and delete
in which items written may disappear otherwise.
@@ -45,21 +45,28 @@ defmodule Plausible.Ingestion.Counters.Buffer do
}
end
- @spec aggregate(t(), binary(), binary(), timestamp :: NaiveDateTime.t()) :: t()
+ @spec aggregate(
+ t(),
+ binary(),
+ binary(),
+ timestamp :: NaiveDateTime.t(),
+ tracker_script_version :: integer()
+ ) :: t()
def aggregate(
%__MODULE__{buffer_name: buffer_name, aggregate_bucket_fn: bucket_fn} = buffer,
metric,
domain,
- timestamp
+ timestamp,
+ tracker_script_version
) do
bucket =
bucket_fn.(timestamp)
:ets.update_counter(
buffer_name,
- {bucket, metric, domain},
+ {bucket, metric, domain, tracker_script_version},
{2, 1},
- {{bucket, metric, domain}, 0}
+ {{bucket, metric, domain, tracker_script_version}, 0}
)
buffer
@@ -72,9 +79,9 @@ defmodule Plausible.Ingestion.Counters.Buffer do
) do
boundary = flush_boundary_fn.(now)
- match = {{:"$1", :"$2", :"$3"}, :"$4"}
+ match = {{:"$1", :"$2", :"$3", :"$4"}, :"$5"}
guard = {:"=<", :"$1", boundary}
- select = {{:"$1", :"$2", :"$3", :"$4"}}
+ select = {{:"$1", :"$2", :"$3", :"$4", :"$5"}}
match_specs_read = [{match, [guard], [select]}]
match_specs_delete = [{match, [guard], [true]}]
diff --git a/lib/plausible/ingestion/counters/record.ex b/lib/plausible/ingestion/counters/record.ex
index 8b79d721ca..ae8ce399b3 100644
--- a/lib/plausible/ingestion/counters/record.ex
+++ b/lib/plausible/ingestion/counters/record.ex
@@ -13,5 +13,6 @@ defmodule Plausible.Ingestion.Counters.Record do
field :domain, Ch, type: "LowCardinality(String)"
field :metric, Ch, type: "LowCardinality(String)"
field :value, Ch, type: "UInt64"
+ field :tracker_script_version, Ch, type: "UInt16"
end
end
diff --git a/lib/plausible/ingestion/counters/telemetry_handler.ex b/lib/plausible/ingestion/counters/telemetry_handler.ex
index 1abfa83ca3..8caa37681e 100644
--- a/lib/plausible/ingestion/counters/telemetry_handler.ex
+++ b/lib/plausible/ingestion/counters/telemetry_handler.ex
@@ -30,20 +30,36 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandler do
def handle_event(
@event_dropped,
_measurements,
- %{domain: domain, reason: reason, request_timestamp: timestamp},
+ %{
+ domain: domain,
+ reason: reason,
+ request_timestamp: timestamp,
+ tracker_script_version: tracker_script_version
+ },
buffer
) do
- Counters.Buffer.aggregate(buffer, "dropped_#{reason}", domain, timestamp)
+ Counters.Buffer.aggregate(
+ buffer,
+ "dropped_#{reason}",
+ domain,
+ timestamp,
+ tracker_script_version
+ )
+
:ok
end
def handle_event(
@event_buffered,
_measurements,
- %{domain: domain, request_timestamp: timestamp},
+ %{
+ domain: domain,
+ request_timestamp: timestamp,
+ tracker_script_version: tracker_script_version
+ },
buffer
) do
- Counters.Buffer.aggregate(buffer, "buffered", domain, timestamp)
+ Counters.Buffer.aggregate(buffer, "buffered", domain, timestamp, tracker_script_version)
:ok
end
end
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
index acc3a73eb3..da4b7f9525 100644
--- a/lib/plausible/ingestion/event.ex
+++ b/lib/plausible/ingestion/event.ex
@@ -96,7 +96,8 @@ defmodule Plausible.Ingestion.Event do
def emit_telemetry_buffered(event) do
:telemetry.execute(telemetry_event_buffered(), %{}, %{
domain: event.domain,
- request_timestamp: event.request.timestamp
+ request_timestamp: event.request.timestamp,
+ tracker_script_version: event.request.tracker_script_version
})
end
@@ -108,7 +109,8 @@ defmodule Plausible.Ingestion.Event do
%{
domain: event.domain,
reason: reason,
- request_timestamp: event.request.timestamp
+ request_timestamp: event.request.timestamp,
+ tracker_script_version: event.request.tracker_script_version
}
)
end
diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex
index 4548ef5a3b..19ec409a6e 100644
--- a/lib/plausible/ingestion/request.ex
+++ b/lib/plausible/ingestion/request.ex
@@ -43,6 +43,7 @@ defmodule Plausible.Ingestion.Request do
field :props, :map
field :scroll_depth, :integer
field :engagement_time, :integer
+ field :tracker_script_version, :integer, default: 0
on_ee do
field :revenue_source, :map
@@ -84,6 +85,7 @@ defmodule Plausible.Ingestion.Request do
|> put_pathname()
|> put_query_params()
|> put_revenue_source(request_body)
+ |> put_tracker_script_version(request_body)
|> map_domains(request_body)
|> Changeset.validate_required([
:event_name,
@@ -270,6 +272,16 @@ defmodule Plausible.Ingestion.Request do
end
end
+ defp put_tracker_script_version(changeset, %{} = request_body) do
+ case request_body["v"] do
+ version when is_integer(version) ->
+ Changeset.put_change(changeset, :tracker_script_version, version)
+
+ _ ->
+ changeset
+ end
+ end
+
defp put_query_params(changeset) do
case Changeset.get_field(changeset, :uri) do
%{query: query} when is_binary(query) ->
diff --git a/test/plausible/ingestion/counters/buffer_test.exs b/test/plausible/ingestion/counters/buffer_test.exs
index e67a23e234..5d0d98523d 100644
--- a/test/plausible/ingestion/counters/buffer_test.exs
+++ b/test/plausible/ingestion/counters/buffer_test.exs
@@ -4,7 +4,7 @@ defmodule Plausible.Ingestion.Counters.BufferTest do
test "10s buckets are created from input datetime" do
# time (...) :58 :59 :00 :01 :02 :03 :04 :05 :06 :07 :08 :09 :10 :11 :12 :13 (...)
- # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10
+ # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10
test_input = [
%{input: ~N[2023-02-14 01:00:00], bucket: ~U[2023-02-14 01:00:00Z]},
@@ -25,9 +25,9 @@ defmodule Plausible.Ingestion.Counters.BufferTest do
test "aggregates metrics every 10 seconds", %{test: test} do
# time (...) :58 :59 :00 :01 :02 :03 :04 :05 :06 :07 :08 :09 :10 :11 :12 :13 (...)
- # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10
- # metric x x x x
- # value 1 1 2 3
+ # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10
+ # metric x x x x
+ # value 1 1 2 3
timestamps = [
~N[2023-02-14 01:00:59],
@@ -39,12 +39,12 @@ defmodule Plausible.Ingestion.Counters.BufferTest do
buffer = Buffer.new(test)
for ts <- timestamps do
- Buffer.aggregate(buffer, "metric", "example.com", ts)
+ Buffer.aggregate(buffer, "metric", "example.com", ts, 0)
end
assert [
- {bucket1, "metric", "example.com", 1},
- {bucket2, "metric", "example.com", 3}
+ {bucket1, "metric", "example.com", 0, 1},
+ {bucket2, "metric", "example.com", 0, 3}
] = Buffer.flush(buffer)
assert bucket2 - bucket1 == 10
@@ -52,9 +52,9 @@ defmodule Plausible.Ingestion.Counters.BufferTest do
test "allows flushing only complete buckets", %{test: test} do
# time (...) :58 :59 :00 :01 :02 :03 :04 :05 :06 :07 :08 :09 :10 :11 :12 :13 (...)
- # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10
- # metric x x x x
- # aggregate 1 0 1 1 2 2 2 2 2 2 3 3 0
+ # bucket 50 50 00 00 00 00 00 00 00 00 00 00 10 10 10 10
+ # metric x x x x
+ # aggregate 1 0 1 1 2 2 2 2 2 2 3 3 0
# flush attempt x x x x
# flushed count 0 1 0 3
@@ -68,12 +68,19 @@ defmodule Plausible.Ingestion.Counters.BufferTest do
buffer = Buffer.new(test)
for ts <- timestamps do
- Buffer.aggregate(buffer, "metric", "example.com", ts)
+ Buffer.aggregate(buffer, "metric", "example.com", ts, 0)
end
assert [] = Buffer.flush(buffer, ~U[2023-02-14 01:00:59.999999Z])
- assert [{_, _, _, 1}] = Buffer.flush(buffer, ~U[2023-02-14 01:01:00.999999Z])
+ assert [{_, _, _, 0, 1}] = Buffer.flush(buffer, ~U[2023-02-14 01:01:00.999999Z])
assert [] = Buffer.flush(buffer, ~U[2023-02-14 01:01:05.999999Z])
- assert [{_, _, _, 3}] = Buffer.flush(buffer, ~U[2023-02-14 01:01:11.999999Z])
+ assert [{_, _, _, 0, 3}] = Buffer.flush(buffer, ~U[2023-02-14 01:01:11.999999Z])
+ end
+
+ test "allows setting tracker script version", %{test: test} do
+ buffer = Buffer.new(test)
+
+ Buffer.aggregate(buffer, "metric", "example.com", ~N[2023-02-14 01:00:59], 137)
+ assert [{_, "metric", "example.com", 137, 1}] = Buffer.flush(buffer)
end
end
diff --git a/test/plausible/ingestion/counters/telemetry_handler_test.exs b/test/plausible/ingestion/counters/telemetry_handler_test.exs
index cd13699f95..084fb31ab7 100644
--- a/test/plausible/ingestion/counters/telemetry_handler_test.exs
+++ b/test/plausible/ingestion/counters/telemetry_handler_test.exs
@@ -34,9 +34,20 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest do
buffer = Buffer.new(test)
assert :ok = TelemetryHandler.install(buffer)
- e1 = %{domain: "a.example.com", request: %{timestamp: NaiveDateTime.utc_now()}}
- e2 = %{domain: "b.example.com", request: %{timestamp: NaiveDateTime.utc_now()}}
- e3 = %{domain: "c.example.com", request: %{timestamp: NaiveDateTime.utc_now()}}
+ e1 = %{
+ domain: "a.example.com",
+ request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}
+ }
+
+ e2 = %{
+ domain: "b.example.com",
+ request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}
+ }
+
+ e3 = %{
+ domain: "c.example.com",
+ request: %{timestamp: NaiveDateTime.utc_now(), tracker_script_version: 137}
+ }
:ok = Event.emit_telemetry_dropped(e1, :invalid)
:ok = Event.emit_telemetry_dropped(e2, :not_found)
@@ -47,8 +58,8 @@ defmodule Plausible.Ingestion.Counters.TelemetryHandlerTest do
assert aggregates = Buffer.flush(buffer, future)
- assert Enum.find(aggregates, &match?({_, "dropped_invalid", "a.example.com", 1}, &1))
- assert Enum.find(aggregates, &match?({_, "dropped_not_found", "b.example.com", 2}, &1))
- assert Enum.find(aggregates, &match?({_, "buffered", "c.example.com", 1}, &1))
+ assert Enum.find(aggregates, &match?({_, "dropped_invalid", "a.example.com", 137, 1}, &1))
+ assert Enum.find(aggregates, &match?({_, "dropped_not_found", "b.example.com", 137, 2}, &1))
+ assert Enum.find(aggregates, &match?({_, "buffered", "c.example.com", 137, 1}, &1))
end
end
diff --git a/test/plausible/ingestion/counters_test.exs b/test/plausible/ingestion/counters_test.exs
index d7ce3b75a6..e60d091a25 100644
--- a/test/plausible/ingestion/counters_test.exs
+++ b/test/plausible/ingestion/counters_test.exs
@@ -116,7 +116,8 @@ defmodule Plausible.Ingestion.CountersTest do
payload = %{
name: "pageview",
- url: "http://#{site.domain}"
+ url: "http://#{site.domain}",
+ v: 137
}
conn = build_conn(:post, "/api/event", payload)
@@ -133,7 +134,8 @@ defmodule Plausible.Ingestion.CountersTest do
payload = %{
name: "pageview",
- url: "http://#{site.domain}"
+ url: "http://#{site.domain}",
+ v: 137
}
conn = build_conn(:post, "/api/event", payload)
@@ -152,8 +154,8 @@ defmodule Plausible.Ingestion.CountersTest do
defp verify_record_written(domain, metric, value, site_id \\ nil) do
query =
from(r in Record,
- group_by: [:site_id, :domain, :metric, :event_timebucket],
- where: r.domain == ^domain and r.metric == ^metric,
+ group_by: [:site_id, :domain, :metric, :event_timebucket, :tracker_script_version],
+ where: r.domain == ^domain and r.metric == ^metric and r.tracker_script_version == 137,
select: sum(r.value)
)
diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs
index ac102a42a8..6a80c5df44 100644
--- a/test/plausible/ingestion/request_test.exs
+++ b/test/plausible/ingestion/request_test.exs
@@ -159,7 +159,8 @@ defmodule Plausible.Ingestion.RequestTest do
props: %{
"custom1" => "property1",
"custom2" => "property2"
- }
+ },
+ v: 137
}
conn = build_conn(:post, "/api/events", payload)
@@ -169,6 +170,7 @@ defmodule Plausible.Ingestion.RequestTest do
assert request.hash_mode == 1
assert request.props["custom1"] == "property1"
assert request.props["custom2"] == "property2"
+ assert request.tracker_script_version == 137
end
@tag :ee_only
@@ -491,7 +493,8 @@ defmodule Plausible.Ingestion.RequestTest do
"user_agent" => "Mozilla",
"ip_classification" => nil,
"scroll_depth" => nil,
- "engagement_time" => nil
+ "engagement_time" => nil,
+ "tracker_script_version" => 0
}
assert %NaiveDateTime{} = NaiveDateTime.from_iso8601!(request["timestamp"])