Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Write event table session columns (#3865)
Karl-Aksel Puulmannc60a2faee4c7
diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.exindex 3071f25395..8792099df8 100644--- a/lib/plausible/clickhouse_event_v2.ex+++ b/lib/plausible/clickhouse_event_v2.ex@@ -23,26 +23,25 @@ defmodule Plausible.ClickhouseEventV2 dofield :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))"field :revenue_reporting_currency, Ch, type: "FixedString(3)"- # Fields which are in the schema but not managed by us anymore.- # field :referrer, :string- # field :referrer_source, :string- # field :utm_medium, :string- # field :utm_source, :string- # field :utm_campaign, :string- # field :utm_content, :string- # field :utm_term, :string+ # Session attributes+ field :referrer, :string+ field :referrer_source, :string+ field :utm_medium, :string+ field :utm_source, :string+ field :utm_campaign, :string+ field :utm_content, :string+ field :utm_term, :string- # field :country_code, Ch, type: "FixedString(2)"- # field :subdivision1_code, Ch, type: "LowCardinality(String)"- # field :subdivision2_code, Ch, type: "LowCardinality(String)"- # field :city_geoname_id, Ch, type: "UInt32"+ field :country_code, Ch, type: "FixedString(2)"+ field :subdivision1_code, Ch, type: "LowCardinality(String)"+ field :subdivision2_code, Ch, type: "LowCardinality(String)"+ field :city_geoname_id, Ch, type: "UInt32"- # field :screen_size, Ch, type: "LowCardinality(String)"- # field :operating_system, Ch, type: "LowCardinality(String)"- # field :operating_system_version, Ch, type: "LowCardinality(String)"- # field :browser, Ch, type: "LowCardinality(String)"- # field :browser_version, Ch, type: "LowCardinality(String)"- # field :transferred_from, :string+ field :screen_size, Ch, type: "LowCardinality(String)"+ field :operating_system, Ch, type: "LowCardinality(String)"+ field :operating_system_version, Ch, type: "LowCardinality(String)"+ field :browser, Ch, type: "LowCardinality(String)"+ field :browser_version, Ch, type: "LowCardinality(String)"enddef new(attrs) dodiff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex 59c0765c53..f11c435ac7 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -43,6 +43,26 @@ defmodule Plausible.Ingestion.Event dochangeset: %Ecto.Changeset{}}+ @session_properties [+ :session_id,+ :referrer,+ :referrer_source,+ :utm_medium,+ :utm_source,+ :utm_campaign,+ :utm_content,+ :utm_term,+ :country_code,+ :subdivision1_code,+ :subdivision2_code,+ :city_geoname_id,+ :screen_size,+ :operating_system,+ :operating_system_version,+ :browser,+ :browser_version+ ]+@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}def build_and_buffer(%Request{domains: domains} = request) doprocessed_events =@@ -315,15 +335,18 @@ defmodule Plausible.Ingestion.Event doevent.salts.previous)- session_id =+ session =Plausible.Session.CacheStore.on_event(event.clickhouse_event,event.clickhouse_session_attrs,previous_user_id)- clickhouse_event = Map.put(event.clickhouse_event, :session_id, session_id)- %{event | clickhouse_event: clickhouse_event}+ %{+ event+ | clickhouse_event:+ Map.merge(event.clickhouse_event, Map.take(session, @session_properties))+ }enddefp write_to_buffer(%__MODULE__{clickhouse_event: clickhouse_event} = event) dodiff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.exindex 565698b8c7..6c29407d57 100644--- a/lib/plausible/session/cache_store.ex+++ b/lib/plausible/session/cache_store.ex@@ -5,18 +5,15 @@ defmodule Plausible.Session.CacheStore dodef on_event(event, session_attributes, prev_user_id, buffer \\ WriteBuffer) dofound_session = find_session(event, event.user_id) || find_session(event, prev_user_id)- session =- if found_session do- updated_session = update_session(found_session, event)- buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}])- persist_session(updated_session)- else- new_session = new_session_from_event(event, session_attributes)- buffer.insert([new_session])- persist_session(new_session)- end-- session.session_id+ if found_session do+ updated_session = update_session(found_session, event)+ buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}])+ persist_session(updated_session)+ else+ new_session = new_session_from_event(event, session_attributes)+ buffer.insert([new_session])+ persist_session(new_session)+ endenddefp find_session(_domain, nil), do: nildiff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exsindex e2162dfcb0..b6627d8bc1 100644--- a/test/plausible/ingestion/event_test.exs+++ b/test/plausible/ingestion/event_test.exs@@ -226,7 +226,7 @@ defmodule Plausible.Ingestion.EventTest doassert {:ok, request} = Request.build(conn)assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)- assert event.clickhouse_event_attrs.hostname == "192.168.0.1"+ assert event.clickhouse_event.hostname == "192.168.0.1"endtest "Hostname is stored with public suffix processing" do@@ -241,6 +241,6 @@ defmodule Plausible.Ingestion.EventTest doassert {:ok, request} = Request.build(conn)assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)- assert event.clickhouse_event_attrs.hostname == "foo.netlify.app"+ assert event.clickhouse_event.hostname == "foo.netlify.app"endenddiff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 3583e3272a..e37ec9e94d 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -12,7 +12,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do{:ok, site: site}end- test "records the event", %{conn: conn, site: site} do+ test "records the event and session", %{conn: conn, site: site} doparams = %{domain: site.domain,name: "pageview",@@ -26,6 +26,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> post("/api/event", params)pageview = get_event(site)+ session = get_created_session(site)assert response(conn, 202) == "ok"assert pageview.hostname == "example.com"@@ -33,6 +34,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert pageview.site_id == site.idassert pageview.pathname == "/"+ assert pageview.session_id == session.session_idendtest "works with Content-Type: text/plain", %{conn: conn, site: site} do@@ -224,12 +226,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> post("/api/event", params)session = get_created_session(site)+ event = get_event(site)assert response(conn, 202) == "ok"assert session.operating_system == "Mac"assert session.operating_system_version == "10.13"assert session.browser == "Chrome"assert session.browser_version == "70.0"+ assert event.operating_system == "Mac"+ assert event.operating_system_version == "10.13"+ assert event.browser == "Chrome"+ assert event.browser_version == "70.0"endtest "parses referrer", %{conn: conn, site: site} do@@ -265,10 +272,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> post("/api/event", params)session = get_created_session(site)+ event = get_event(site)assert response(conn, 202) == "ok"assert session.referrer == "facebook.com/page"assert session.referrer_source == "Facebook"+ assert event.referrer == session.referrer+ assert event.referrer_source == session.referrer_sourceendtest "ignores event when referrer is a spammer", %{conn: conn, site: site} do@@ -811,11 +821,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> post("/api/event", params)session = get_created_session(site)+ event = get_event(site)assert session.country_code == "GB"assert session.subdivision1_code == "GB-ENG"assert session.subdivision2_code == "GB-WBK"assert session.city_geoname_id == 2_655_045+ assert event.country_code == session.country_code+ assert event.subdivision1_code == session.subdivision1_code+ assert event.subdivision2_code == session.subdivision2_code+ assert event.city_geoname_id == session.city_geoname_idendtest "ignores unknown country code ZZ", %{conn: conn, site: site} do@@ -1108,6 +1123,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert pageview.hostname == "test.com"assert pageview.pathname == "/ﺝﻭﺎﺋﺯ-ﻮﻤﺳﺎﺒﻗﺎﺗ"assert session.utm_source == "%balle%"+ assert pageview.utm_source == session.utm_sourceendtest "can use double quotes in query params", %{conn: conn, site: site} dodiff --git a/test/support/factory.ex b/test/support/factory.exindex 090ca55d33..6ab32a7501 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -89,14 +89,13 @@ defmodule Plausible.Factory dodef event_factory dohostname = sequence(:domain, &"example-#{&1}.com")- %{+ %Plausible.ClickhouseEventV2{hostname: hostname,site_id: Enum.random(1000..10_000),pathname: "/",timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),- session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),- _factory_event: true+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())}enddiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex 9847315267..91db1b99b3 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -82,8 +82,9 @@ defmodule Plausible.TestUtils do|> Map.put(:site_id, pageview.site.id)Factory.build(:pageview, pageview)+ |> Map.from_struct()+ |> Map.delete(:__meta__)|> update_in([:timestamp], &to_naive_truncate/1)- |> Map.delete(:_factory_event)end)Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)@@ -93,8 +94,9 @@ defmodule Plausible.TestUtils doevents =Enum.map(events, fn event ->Factory.build(:event, event)+ |> Map.from_struct()+ |> Map.delete(:__meta__)|> update_in([:timestamp], &to_naive_truncate/1)- |> Map.delete(:_factory_event)end)Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)@@ -148,7 +150,7 @@ defmodule Plausible.TestUtils doevent = Map.put(event, :site_id, site.id)case event do- %{_factory_event: true} ->+ %Plausible.ClickhouseEventV2{} ->eventimported_event ->@@ -179,7 +181,7 @@ defmodule Plausible.TestUtils doend)|> Enum.split_with(fn event ->case event do- %{_factory_event: true} ->+ %Plausible.ClickhouseEventV2{} ->true_ ->@@ -192,20 +194,13 @@ defmodule Plausible.TestUtils doenddefp populate_native_stats(events) do- sessions =- Enum.reduce(events, %{}, fn event, sessions ->- session_id = Plausible.Session.CacheStore.on_event(event, event, nil)- Map.put(sessions, {event.site_id, event.user_id}, session_id)- end)-- Enum.each(events, fn event ->- clickhouse_event =- %Plausible.ClickhouseEventV2{}- |> Map.merge(event)- |> Map.put(:session_id, sessions[{event.site_id, event.user_id}])+ for event_params <- events do+ session = Plausible.Session.CacheStore.on_event(event_params, event_params, nil)- Plausible.Event.WriteBuffer.insert(clickhouse_event)- end)+ event_params+ |> Map.merge(session)+ |> Plausible.Event.WriteBuffer.insert()+ endPlausible.Session.WriteBuffer.flush()Plausible.Event.WriteBuffer.flush()