Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add custom props to full export (#5666)
Adrian Gruntkowski70c9a55bf89d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9854dcf5a0..ffca5be0ee 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.- Custom events can now be marked as non-interactive in events API and tracker script. Events marked as non-interactive are not counted towards bounce rate.- Ability to leave team via Team Settings > Leave Team- Stats APIv2 now supports `include.trim_relative_date_range`. This option allows trimming empty values after current time for `day`, `month` and `year` date_range values.+- Properties are now included in full site exports done via Site Settings > Imports & Exports### Removeddiff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex f5d7439baa..a701b1a024 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -208,13 +208,13 @@ defmodule Plausible.Exports doBuilds Ecto queries to export data from `events_v2` and `sessions_v2`tables into the format of `imported_*` tables for a website."""- @spec export_queries(pos_integer,+ @spec export_queries(Plausible.Site.t(),extname: String.t(),date_range: Date.Range.t(),timezone: String.t()) ::%{String.t() => Ecto.Query.t()}- def export_queries(site_id, opts \\ []) do+ def export_queries(site, opts \\ []) doextname = opts[:extname] || ".csv"date_range = opts[:date_range]timezone = opts[:timezone] || "UTC"@@ -231,18 +231,18 @@ defmodule Plausible.Exports dofilename = fn name -> name <> suffix end%{- filename.("imported_visitors") => export_visitors_q(site_id, timezone, date_range),- filename.("imported_sources") => export_sources_q(site_id, timezone, date_range),- filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),- filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),- filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),- filename.("imported_custom_events") =>- export_custom_events_q(site_id, timezone, date_range),- filename.("imported_locations") => export_locations_q(site_id, timezone, date_range),- filename.("imported_devices") => export_devices_q(site_id, timezone, date_range),- filename.("imported_browsers") => export_browsers_q(site_id, timezone, date_range),+ filename.("imported_visitors") => export_visitors_q(site, timezone, date_range),+ filename.("imported_sources") => export_sources_q(site, timezone, date_range),+ filename.("imported_pages") => export_pages_q(site, timezone, date_range),+ filename.("imported_entry_pages") => export_entry_pages_q(site, timezone, date_range),+ filename.("imported_exit_pages") => export_exit_pages_q(site, timezone, date_range),+ filename.("imported_custom_events") => export_custom_events_q(site, timezone, date_range),+ filename.("imported_locations") => export_locations_q(site, timezone, date_range),+ filename.("imported_devices") => export_devices_q(site, timezone, date_range),+ filename.("imported_browsers") => export_browsers_q(site, timezone, date_range),filename.("imported_operating_systems") =>- export_operating_systems_q(site_id, timezone, date_range)+ export_operating_systems_q(site, timezone, date_range),+ filename.("imported_custom_props") => export_custom_props_q(site, timezone, date_range)}end@@ -336,10 +336,10 @@ defmodule Plausible.Exports doendend- defp export_visitors_q(site_id, timezone, date_range) do+ defp export_visitors_q(site, timezone, date_range) dovisitors_sessions_q =from s in sampled("sessions_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),group_by: selected_as(:date),select: %{date: date(s.timestamp, ^timezone),@@ -351,7 +351,7 @@ defmodule Plausible.Exports dovisitors_events_q =from e in sampled("events_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),group_by: selected_as(:date),select: %{date: date(e.timestamp, ^timezone),@@ -381,9 +381,9 @@ defmodule Plausible.Exports do]end- defp export_sources_q(site_id, timezone, date_range) do+ defp export_sources_q(site, timezone, date_range) dofrom s in sampled("sessions_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),group_by: [selected_as(:date),selected_as(:source),@@ -412,10 +412,10 @@ defmodule Plausible.Exports do]end- defp export_pages_q(site_id, timezone, date_range) do+ defp export_pages_q(site, timezone, date_range) dobase_q =from(e in sampled("events_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),where: [name: "pageview"],group_by: [selected_as(:date), selected_as(:page)],order_by: selected_as(:date)@@ -423,7 +423,7 @@ defmodule Plausible.Exports domax_scroll_depth_per_session_q =from(e in "events_v2",- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),where: e.name == "engagement" and e.scroll_depth <= 100,select: %{date: date(e.timestamp, ^timezone),@@ -465,7 +465,7 @@ defmodule Plausible.Exports doselected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)})- |> add_time_on_page_columns(site_id, timezone, date_range)+ |> add_time_on_page_columns(site.id, timezone, date_range)enddefp add_time_on_page_columns(q, site_id, timezone, date_range) do@@ -508,9 +508,9 @@ defmodule Plausible.Exports doendend- defp export_entry_pages_q(site_id, timezone, date_range) do+ defp export_entry_pages_q(site, timezone, date_range) dofrom s in sampled("sessions_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),group_by: [selected_as(:date), s.entry_page],order_by: selected_as(:date),select: [@@ -527,9 +527,9 @@ defmodule Plausible.Exports do]end- defp export_exit_pages_q(site_id, timezone, date_range) do+ defp export_exit_pages_q(site, timezone, date_range) dofrom s in sampled("sessions_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),group_by: [selected_as(:date), s.exit_page],order_by: selected_as(:date),select: [@@ -546,9 +546,9 @@ defmodule Plausible.Exports do]end- defp export_custom_events_q(site_id, timezone, date_range) do+ defp export_custom_events_q(site, timezone, date_range) dofrom e in sampled("events_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),where: e.name != "pageview",group_by: [selected_as(:date),@@ -583,9 +583,37 @@ defmodule Plausible.Exports do]end- defp export_locations_q(site_id, timezone, date_range) do+ defp export_custom_props_q(site, timezone, date_range) do+ query =+ from e in sampled("events_v2"),+ join: pv in fragment("arrayZip(`meta.key`, `meta.value`)"),+ on: true,+ hints: "ARRAY",+ where: ^export_filter(site.id, date_range),+ group_by: [+ selected_as(:date),+ selected_as(:property),+ selected_as(:value)+ ],+ order_by: selected_as(:date),+ select: [+ date(e.timestamp, ^timezone),+ selected_as(fragment("tupleElement(?, 1)", pv), :property),+ selected_as(fragment("tupleElement(?, 2)", pv), :value),+ visitors(e),+ selected_as(scale_sample(fragment("count()")), :events)+ ]++ if Plausible.Billing.Feature.Props.enabled?(site) do+ query+ else+ where(query, [], selected_as(:property) in ^Plausible.Props.internal_keys())+ end+ end++ defp export_locations_q(site, timezone, date_range) dofrom s in sampled("sessions_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),where: s.country_code != "\0\0" and s.country_code != "ZZ",group_by: [selected_as(:date), s.country_code, s.subdivision1_code, s.city_geoname_id],order_by: selected_as(:date),@@ -602,9 +630,9 @@ defmodule Plausible.Exports do]end- defp export_devices_q(site_id, timezone, date_range) do+ defp export_devices_q(site, timezone, date_range) dofrom s in sampled("sessions_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),group_by: [selected_as(:date), s.screen_size],order_by: selected_as(:date),select: [@@ -618,9 +646,9 @@ defmodule Plausible.Exports do]end- defp export_browsers_q(site_id, timezone, date_range) do+ defp export_browsers_q(site, timezone, date_range) dofrom s in sampled("sessions_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),group_by: [selected_as(:date), s.browser, s.browser_version],order_by: selected_as(:date),select: [@@ -635,9 +663,9 @@ defmodule Plausible.Exports do]end- defp export_operating_systems_q(site_id, timezone, date_range) do+ defp export_operating_systems_q(site, timezone, date_range) dofrom s in sampled("sessions_v2"),- where: ^export_filter(site_id, date_range),+ where: ^export_filter(site.id, date_range),group_by: [selected_as(:date), s.operating_system, s.operating_system_version],order_by: selected_as(:date),select: [@@ -661,7 +689,7 @@ defmodule Plausible.Exports doDBConnection.run(pool, fn conn ->conn- |> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")+ |> stream_archive(export_queries(site), format: "CSVWithNames")|> Stream.into(File.stream!("export.zip"))|> Stream.run()end)diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exindex 3ec1061286..ab51268b8b 100644--- a/lib/plausible/imported/csv_importer.ex+++ b/lib/plausible/imported/csv_importer.ex@@ -18,6 +18,8 @@ defmodule Plausible.Imported.CSVImporter do@impl truedef parse_args(%{"uploads" => uploads, "storage" => storage}) do+ uploads = Enum.reject(uploads, &String.starts_with?(&1["filename"], "imported_custom_props_"))+[uploads: uploads, storage: storage]end@@ -208,6 +210,14 @@ defmodule Plausible.Imported.CSVImporter dodef date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)def date_range([]), do: nil+ defp date_range(+ [%{"filename" => "imported_custom_props_" <> _} | uploads],+ prev_start_date,+ prev_end_date+ ) do+ date_range(uploads, prev_start_date, prev_end_date)+ end+defp date_range([upload | uploads], prev_start_date, prev_end_date) dofilename =case upload dodiff --git a/lib/workers/export_analytics.ex b/lib/workers/export_analytics.exindex c7e7221d43..15678fb601 100644--- a/lib/workers/export_analytics.ex+++ b/lib/workers/export_analytics.ex@@ -33,7 +33,7 @@ defmodule Plausible.Workers.ExportAnalytics do%Date.Range{} = date_range = Exports.date_range(site.id, site.timezone)queries =- Exports.export_queries(site_id,+ Exports.export_queries(site,date_range: date_range,timezone: site.timezone,extname: ".csv"diff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exsindex d6016916fc..896d09cfd3 100644--- a/test/plausible/exports_test.exs+++ b/test/plausible/exports_test.exs@@ -9,12 +9,13 @@ defmodule Plausible.ExportsTest dosetup [:create_user, :create_site]test "returns named ecto queries", %{site: site} do- queries = Plausible.Exports.export_queries(site.id)+ queries = Plausible.Exports.export_queries(site)assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))assert Map.keys(queries) == ["imported_browsers.csv","imported_custom_events.csv",+ "imported_custom_props.csv","imported_devices.csv","imported_entry_pages.csv","imported_exit_pages.csv",@@ -28,13 +29,14 @@ defmodule Plausible.ExportsTest dotest "with date range", %{site: site} doqueries =- Plausible.Exports.export_queries(site.id,+ Plausible.Exports.export_queries(site,date_range: Date.range(~D[2023-01-01], ~D[2024-03-12]))assert Map.keys(queries) == ["imported_browsers_20230101_20240312.csv","imported_custom_events_20230101_20240312.csv",+ "imported_custom_props_20230101_20240312.csv","imported_devices_20230101_20240312.csv","imported_entry_pages_20230101_20240312.csv","imported_exit_pages_20230101_20240312.csv",@@ -47,11 +49,12 @@ defmodule Plausible.ExportsTest doendtest "with custom extension", %{site: site} do- queries = Plausible.Exports.export_queries(site.id, extname: ".ch")+ queries = Plausible.Exports.export_queries(site, extname: ".ch")assert Map.keys(queries) == ["imported_browsers.ch","imported_custom_events.ch",+ "imported_custom_props.ch","imported_devices.ch","imported_entry_pages.ch","imported_exit_pages.ch",diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex 23fdd6bb75..5c67fa46ee 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -608,7 +608,7 @@ defmodule Plausible.Imported.CSVImporterTest doimported_site: imported_site}- %{site_import: site_import} =+ %{site_import: site_import, exported_files: exported_files} =initial_context|> export_archive()|> assert_email_notification()@@ -617,6 +617,16 @@ defmodule Plausible.Imported.CSVImporterTest do|> upload_csvs()|> run_import()+ assert custom_props_export =+ exported_files+ |> Enum.find(&String.contains?(&1, "imported_custom_props_"))+ |> File.read!()+ |> String.split("\n")++ assert ~s|"date","property","value","visitors","events"| in custom_props_export+ assert ~s|"2024-04-01","author","Marko Saric",43,57| in custom_props_export+ assert ~s|"2024-04-01","category","Posts",43,56| in custom_props_export+assert %SiteImport{start_date: ~D[2024-04-01],end_date: ~D[2024-04-30],