Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Write event table session columns (#3865)

Karl-Aksel Puulmannc60a2faee4c7
diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex
index 3071f25395..8792099df8 100644
--- a/lib/plausible/clickhouse_event_v2.ex
+++ b/lib/plausible/clickhouse_event_v2.ex
@@ -23,26 +23,25 @@ defmodule Plausible.ClickhouseEventV2 do
field :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))"
field :revenue_reporting_currency, Ch, type: "FixedString(3)"
- # Fields which are in the schema but not managed by us anymore.
- # field :referrer, :string
- # field :referrer_source, :string
- # field :utm_medium, :string
- # field :utm_source, :string
- # field :utm_campaign, :string
- # field :utm_content, :string
- # field :utm_term, :string
+ # Session attributes
+ field :referrer, :string
+ field :referrer_source, :string
+ field :utm_medium, :string
+ field :utm_source, :string
+ field :utm_campaign, :string
+ field :utm_content, :string
+ field :utm_term, :string
- # field :country_code, Ch, type: "FixedString(2)"
- # field :subdivision1_code, Ch, type: "LowCardinality(String)"
- # field :subdivision2_code, Ch, type: "LowCardinality(String)"
- # field :city_geoname_id, Ch, type: "UInt32"
+ field :country_code, Ch, type: "FixedString(2)"
+ field :subdivision1_code, Ch, type: "LowCardinality(String)"
+ field :subdivision2_code, Ch, type: "LowCardinality(String)"
+ field :city_geoname_id, Ch, type: "UInt32"
- # field :screen_size, Ch, type: "LowCardinality(String)"
- # field :operating_system, Ch, type: "LowCardinality(String)"
- # field :operating_system_version, Ch, type: "LowCardinality(String)"
- # field :browser, Ch, type: "LowCardinality(String)"
- # field :browser_version, Ch, type: "LowCardinality(String)"
- # field :transferred_from, :string
+ field :screen_size, Ch, type: "LowCardinality(String)"
+ field :operating_system, Ch, type: "LowCardinality(String)"
+ field :operating_system_version, Ch, type: "LowCardinality(String)"
+ field :browser, Ch, type: "LowCardinality(String)"
+ field :browser_version, Ch, type: "LowCardinality(String)"
end
def new(attrs) do
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
index 59c0765c53..f11c435ac7 100644
--- a/lib/plausible/ingestion/event.ex
+++ b/lib/plausible/ingestion/event.ex
@@ -43,6 +43,26 @@ defmodule Plausible.Ingestion.Event do
changeset: %Ecto.Changeset{}
}
+ @session_properties [
+ :session_id,
+ :referrer,
+ :referrer_source,
+ :utm_medium,
+ :utm_source,
+ :utm_campaign,
+ :utm_content,
+ :utm_term,
+ :country_code,
+ :subdivision1_code,
+ :subdivision2_code,
+ :city_geoname_id,
+ :screen_size,
+ :operating_system,
+ :operating_system_version,
+ :browser,
+ :browser_version
+ ]
+
@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}
def build_and_buffer(%Request{domains: domains} = request) do
processed_events =
@@ -315,15 +335,18 @@ defmodule Plausible.Ingestion.Event do
event.salts.previous
)
- session_id =
+ session =
Plausible.Session.CacheStore.on_event(
event.clickhouse_event,
event.clickhouse_session_attrs,
previous_user_id
)
- clickhouse_event = Map.put(event.clickhouse_event, :session_id, session_id)
- %{event | clickhouse_event: clickhouse_event}
+ %{
+ event
+ | clickhouse_event:
+ Map.merge(event.clickhouse_event, Map.take(session, @session_properties))
+ }
end
defp write_to_buffer(%__MODULE__{clickhouse_event: clickhouse_event} = event) do
diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex
index 565698b8c7..6c29407d57 100644
--- a/lib/plausible/session/cache_store.ex
+++ b/lib/plausible/session/cache_store.ex
@@ -5,18 +5,15 @@ defmodule Plausible.Session.CacheStore do
def on_event(event, session_attributes, prev_user_id, buffer \\ WriteBuffer) do
found_session = find_session(event, event.user_id) || find_session(event, prev_user_id)
- session =
- if found_session do
- updated_session = update_session(found_session, event)
- buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}])
- persist_session(updated_session)
- else
- new_session = new_session_from_event(event, session_attributes)
- buffer.insert([new_session])
- persist_session(new_session)
- end
-
- session.session_id
+ if found_session do
+ updated_session = update_session(found_session, event)
+ buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}])
+ persist_session(updated_session)
+ else
+ new_session = new_session_from_event(event, session_attributes)
+ buffer.insert([new_session])
+ persist_session(new_session)
+ end
end
defp find_session(_domain, nil), do: nil
diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs
index e2162dfcb0..b6627d8bc1 100644
--- a/test/plausible/ingestion/event_test.exs
+++ b/test/plausible/ingestion/event_test.exs
@@ -226,7 +226,7 @@ defmodule Plausible.Ingestion.EventTest do
assert {:ok, request} = Request.build(conn)
assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)
- assert event.clickhouse_event_attrs.hostname == "192.168.0.1"
+ assert event.clickhouse_event.hostname == "192.168.0.1"
end
test "Hostname is stored with public suffix processing" do
@@ -241,6 +241,6 @@ defmodule Plausible.Ingestion.EventTest do
assert {:ok, request} = Request.build(conn)
assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)
- assert event.clickhouse_event_attrs.hostname == "foo.netlify.app"
+ assert event.clickhouse_event.hostname == "foo.netlify.app"
end
end
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index 3583e3272a..e37ec9e94d 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -12,7 +12,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
{:ok, site: site}
end
- test "records the event", %{conn: conn, site: site} do
+ test "records the event and session", %{conn: conn, site: site} do
params = %{
domain: site.domain,
name: "pageview",
@@ -26,6 +26,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
pageview = get_event(site)
+ session = get_created_session(site)
assert response(conn, 202) == "ok"
assert pageview.hostname == "example.com"
@@ -33,6 +34,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.site_id == site.id
assert pageview.pathname == "/"
+ assert pageview.session_id == session.session_id
end
test "works with Content-Type: text/plain", %{conn: conn, site: site} do
@@ -224,12 +226,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
session = get_created_session(site)
+ event = get_event(site)
assert response(conn, 202) == "ok"
assert session.operating_system == "Mac"
assert session.operating_system_version == "10.13"
assert session.browser == "Chrome"
assert session.browser_version == "70.0"
+ assert event.operating_system == "Mac"
+ assert event.operating_system_version == "10.13"
+ assert event.browser == "Chrome"
+ assert event.browser_version == "70.0"
end
test "parses referrer", %{conn: conn, site: site} do
@@ -265,10 +272,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
session = get_created_session(site)
+ event = get_event(site)
assert response(conn, 202) == "ok"
assert session.referrer == "facebook.com/page"
assert session.referrer_source == "Facebook"
+ assert event.referrer == session.referrer
+ assert event.referrer_source == session.referrer_source
end
test "ignores event when referrer is a spammer", %{conn: conn, site: site} do
@@ -811,11 +821,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
session = get_created_session(site)
+ event = get_event(site)
assert session.country_code == "GB"
assert session.subdivision1_code == "GB-ENG"
assert session.subdivision2_code == "GB-WBK"
assert session.city_geoname_id == 2_655_045
+ assert event.country_code == session.country_code
+ assert event.subdivision1_code == session.subdivision1_code
+ assert event.subdivision2_code == session.subdivision2_code
+ assert event.city_geoname_id == session.city_geoname_id
end
test "ignores unknown country code ZZ", %{conn: conn, site: site} do
@@ -1108,6 +1123,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.hostname == "test.com"
assert pageview.pathname == "/ﺝﻭﺎﺋﺯ-ﻮﻤﺳﺎﺒﻗﺎﺗ"
assert session.utm_source == "%balle%"
+ assert pageview.utm_source == session.utm_source
end
test "can use double quotes in query params", %{conn: conn, site: site} do
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 090ca55d33..6ab32a7501 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -89,14 +89,13 @@ defmodule Plausible.Factory do
def event_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
- %{
+ %Plausible.ClickhouseEventV2{
hostname: hostname,
site_id: Enum.random(1000..10_000),
pathname: "/",
timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
- session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
- _factory_event: true
+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
}
end
diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex
index 9847315267..91db1b99b3 100644
--- a/test/support/test_utils.ex
+++ b/test/support/test_utils.ex
@@ -82,8 +82,9 @@ defmodule Plausible.TestUtils do
|> Map.put(:site_id, pageview.site.id)
Factory.build(:pageview, pageview)
+ |> Map.from_struct()
+ |> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
- |> Map.delete(:_factory_event)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)
@@ -93,8 +94,9 @@ defmodule Plausible.TestUtils do
events =
Enum.map(events, fn event ->
Factory.build(:event, event)
+ |> Map.from_struct()
+ |> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
- |> Map.delete(:_factory_event)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
@@ -148,7 +150,7 @@ defmodule Plausible.TestUtils do
event = Map.put(event, :site_id, site.id)
case event do
- %{_factory_event: true} ->
+ %Plausible.ClickhouseEventV2{} ->
event
imported_event ->
@@ -179,7 +181,7 @@ defmodule Plausible.TestUtils do
end)
|> Enum.split_with(fn event ->
case event do
- %{_factory_event: true} ->
+ %Plausible.ClickhouseEventV2{} ->
true
_ ->
@@ -192,20 +194,13 @@ defmodule Plausible.TestUtils do
end
defp populate_native_stats(events) do
- sessions =
- Enum.reduce(events, %{}, fn event, sessions ->
- session_id = Plausible.Session.CacheStore.on_event(event, event, nil)
- Map.put(sessions, {event.site_id, event.user_id}, session_id)
- end)
-
- Enum.each(events, fn event ->
- clickhouse_event =
- %Plausible.ClickhouseEventV2{}
- |> Map.merge(event)
- |> Map.put(:session_id, sessions[{event.site_id, event.user_id}])
+ for event_params <- events do
+ session = Plausible.Session.CacheStore.on_event(event_params, event_params, nil)
- Plausible.Event.WriteBuffer.insert(clickhouse_event)
- end)
+ event_params
+ |> Map.merge(session)
+ |> Plausible.Event.WriteBuffer.insert()
+ end
Plausible.Session.WriteBuffer.flush()
Plausible.Event.WriteBuffer.flush()