Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Add custom props to full export (#5666)

Adrian Gruntkowski70c9a55bf89d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9854dcf5a0..ffca5be0ee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
- Custom events can now be marked as non-interactive in events API and tracker script. Events marked as non-interactive are not counted towards bounce rate.
- Ability to leave team via Team Settings > Leave Team
- Stats APIv2 now supports `include.trim_relative_date_range`. This option allows trimming empty values after current time for `day`, `month` and `year` date_range values.
+- Properties are now included in full site exports done via Site Settings > Imports & Exports
### Removed
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex
index f5d7439baa..a701b1a024 100644
--- a/lib/plausible/exports.ex
+++ b/lib/plausible/exports.ex
@@ -208,13 +208,13 @@ defmodule Plausible.Exports do
Builds Ecto queries to export data from `events_v2` and `sessions_v2`
tables into the format of `imported_*` tables for a website.
"""
- @spec export_queries(pos_integer,
+ @spec export_queries(Plausible.Site.t(),
extname: String.t(),
date_range: Date.Range.t(),
timezone: String.t()
) ::
%{String.t() => Ecto.Query.t()}
- def export_queries(site_id, opts \\ []) do
+ def export_queries(site, opts \\ []) do
extname = opts[:extname] || ".csv"
date_range = opts[:date_range]
timezone = opts[:timezone] || "UTC"
@@ -231,18 +231,18 @@ defmodule Plausible.Exports do
filename = fn name -> name <> suffix end
%{
- filename.("imported_visitors") => export_visitors_q(site_id, timezone, date_range),
- filename.("imported_sources") => export_sources_q(site_id, timezone, date_range),
- filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),
- filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),
- filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),
- filename.("imported_custom_events") =>
- export_custom_events_q(site_id, timezone, date_range),
- filename.("imported_locations") => export_locations_q(site_id, timezone, date_range),
- filename.("imported_devices") => export_devices_q(site_id, timezone, date_range),
- filename.("imported_browsers") => export_browsers_q(site_id, timezone, date_range),
+ filename.("imported_visitors") => export_visitors_q(site, timezone, date_range),
+ filename.("imported_sources") => export_sources_q(site, timezone, date_range),
+ filename.("imported_pages") => export_pages_q(site, timezone, date_range),
+ filename.("imported_entry_pages") => export_entry_pages_q(site, timezone, date_range),
+ filename.("imported_exit_pages") => export_exit_pages_q(site, timezone, date_range),
+ filename.("imported_custom_events") => export_custom_events_q(site, timezone, date_range),
+ filename.("imported_locations") => export_locations_q(site, timezone, date_range),
+ filename.("imported_devices") => export_devices_q(site, timezone, date_range),
+ filename.("imported_browsers") => export_browsers_q(site, timezone, date_range),
filename.("imported_operating_systems") =>
- export_operating_systems_q(site_id, timezone, date_range)
+ export_operating_systems_q(site, timezone, date_range),
+ filename.("imported_custom_props") => export_custom_props_q(site, timezone, date_range)
}
end
@@ -336,10 +336,10 @@ defmodule Plausible.Exports do
end
end
- defp export_visitors_q(site_id, timezone, date_range) do
+ defp export_visitors_q(site, timezone, date_range) do
visitors_sessions_q =
from s in sampled("sessions_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
group_by: selected_as(:date),
select: %{
date: date(s.timestamp, ^timezone),
@@ -351,7 +351,7 @@ defmodule Plausible.Exports do
visitors_events_q =
from e in sampled("events_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
group_by: selected_as(:date),
select: %{
date: date(e.timestamp, ^timezone),
@@ -381,9 +381,9 @@ defmodule Plausible.Exports do
]
end
- defp export_sources_q(site_id, timezone, date_range) do
+ defp export_sources_q(site, timezone, date_range) do
from s in sampled("sessions_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
group_by: [
selected_as(:date),
selected_as(:source),
@@ -412,10 +412,10 @@ defmodule Plausible.Exports do
]
end
- defp export_pages_q(site_id, timezone, date_range) do
+ defp export_pages_q(site, timezone, date_range) do
base_q =
from(e in sampled("events_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
where: [name: "pageview"],
group_by: [selected_as(:date), selected_as(:page)],
order_by: selected_as(:date)
@@ -423,7 +423,7 @@ defmodule Plausible.Exports do
max_scroll_depth_per_session_q =
from(e in "events_v2",
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
where: e.name == "engagement" and e.scroll_depth <= 100,
select: %{
date: date(e.timestamp, ^timezone),
@@ -465,7 +465,7 @@ defmodule Plausible.Exports do
selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)
}
)
- |> add_time_on_page_columns(site_id, timezone, date_range)
+ |> add_time_on_page_columns(site.id, timezone, date_range)
end
defp add_time_on_page_columns(q, site_id, timezone, date_range) do
@@ -508,9 +508,9 @@ defmodule Plausible.Exports do
end
end
- defp export_entry_pages_q(site_id, timezone, date_range) do
+ defp export_entry_pages_q(site, timezone, date_range) do
from s in sampled("sessions_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
group_by: [selected_as(:date), s.entry_page],
order_by: selected_as(:date),
select: [
@@ -527,9 +527,9 @@ defmodule Plausible.Exports do
]
end
- defp export_exit_pages_q(site_id, timezone, date_range) do
+ defp export_exit_pages_q(site, timezone, date_range) do
from s in sampled("sessions_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
group_by: [selected_as(:date), s.exit_page],
order_by: selected_as(:date),
select: [
@@ -546,9 +546,9 @@ defmodule Plausible.Exports do
]
end
- defp export_custom_events_q(site_id, timezone, date_range) do
+ defp export_custom_events_q(site, timezone, date_range) do
from e in sampled("events_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
where: e.name != "pageview",
group_by: [
selected_as(:date),
@@ -583,9 +583,37 @@ defmodule Plausible.Exports do
]
end
- defp export_locations_q(site_id, timezone, date_range) do
+ defp export_custom_props_q(site, timezone, date_range) do
+ query =
+ from e in sampled("events_v2"),
+ join: pv in fragment("arrayZip(`meta.key`, `meta.value`)"),
+ on: true,
+ hints: "ARRAY",
+ where: ^export_filter(site.id, date_range),
+ group_by: [
+ selected_as(:date),
+ selected_as(:property),
+ selected_as(:value)
+ ],
+ order_by: selected_as(:date),
+ select: [
+ date(e.timestamp, ^timezone),
+ selected_as(fragment("tupleElement(?, 1)", pv), :property),
+ selected_as(fragment("tupleElement(?, 2)", pv), :value),
+ visitors(e),
+ selected_as(scale_sample(fragment("count()")), :events)
+ ]
+
+ if Plausible.Billing.Feature.Props.enabled?(site) do
+ query
+ else
+ where(query, [], selected_as(:property) in ^Plausible.Props.internal_keys())
+ end
+ end
+
+ defp export_locations_q(site, timezone, date_range) do
from s in sampled("sessions_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
where: s.country_code != "\0\0" and s.country_code != "ZZ",
group_by: [selected_as(:date), s.country_code, s.subdivision1_code, s.city_geoname_id],
order_by: selected_as(:date),
@@ -602,9 +630,9 @@ defmodule Plausible.Exports do
]
end
- defp export_devices_q(site_id, timezone, date_range) do
+ defp export_devices_q(site, timezone, date_range) do
from s in sampled("sessions_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
group_by: [selected_as(:date), s.screen_size],
order_by: selected_as(:date),
select: [
@@ -618,9 +646,9 @@ defmodule Plausible.Exports do
]
end
- defp export_browsers_q(site_id, timezone, date_range) do
+ defp export_browsers_q(site, timezone, date_range) do
from s in sampled("sessions_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
group_by: [selected_as(:date), s.browser, s.browser_version],
order_by: selected_as(:date),
select: [
@@ -635,9 +663,9 @@ defmodule Plausible.Exports do
]
end
- defp export_operating_systems_q(site_id, timezone, date_range) do
+ defp export_operating_systems_q(site, timezone, date_range) do
from s in sampled("sessions_v2"),
- where: ^export_filter(site_id, date_range),
+ where: ^export_filter(site.id, date_range),
group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
order_by: selected_as(:date),
select: [
@@ -661,7 +689,7 @@ defmodule Plausible.Exports do
DBConnection.run(pool, fn conn ->
conn
- |> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")
+ |> stream_archive(export_queries(site), format: "CSVWithNames")
|> Stream.into(File.stream!("export.zip"))
|> Stream.run()
end)
diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex
index 3ec1061286..ab51268b8b 100644
--- a/lib/plausible/imported/csv_importer.ex
+++ b/lib/plausible/imported/csv_importer.ex
@@ -18,6 +18,8 @@ defmodule Plausible.Imported.CSVImporter do
@impl true
def parse_args(%{"uploads" => uploads, "storage" => storage}) do
+ uploads = Enum.reject(uploads, &String.starts_with?(&1["filename"], "imported_custom_props_"))
+
[uploads: uploads, storage: storage]
end
@@ -208,6 +210,14 @@ defmodule Plausible.Imported.CSVImporter do
def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
def date_range([]), do: nil
+ defp date_range(
+ [%{"filename" => "imported_custom_props_" <> _} | uploads],
+ prev_start_date,
+ prev_end_date
+ ) do
+ date_range(uploads, prev_start_date, prev_end_date)
+ end
+
defp date_range([upload | uploads], prev_start_date, prev_end_date) do
filename =
case upload do
diff --git a/lib/workers/export_analytics.ex b/lib/workers/export_analytics.ex
index c7e7221d43..15678fb601 100644
--- a/lib/workers/export_analytics.ex
+++ b/lib/workers/export_analytics.ex
@@ -33,7 +33,7 @@ defmodule Plausible.Workers.ExportAnalytics do
%Date.Range{} = date_range = Exports.date_range(site.id, site.timezone)
queries =
- Exports.export_queries(site_id,
+ Exports.export_queries(site,
date_range: date_range,
timezone: site.timezone,
extname: ".csv"
diff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exs
index d6016916fc..896d09cfd3 100644
--- a/test/plausible/exports_test.exs
+++ b/test/plausible/exports_test.exs
@@ -9,12 +9,13 @@ defmodule Plausible.ExportsTest do
setup [:create_user, :create_site]
test "returns named ecto queries", %{site: site} do
- queries = Plausible.Exports.export_queries(site.id)
+ queries = Plausible.Exports.export_queries(site)
assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))
assert Map.keys(queries) == [
"imported_browsers.csv",
"imported_custom_events.csv",
+ "imported_custom_props.csv",
"imported_devices.csv",
"imported_entry_pages.csv",
"imported_exit_pages.csv",
@@ -28,13 +29,14 @@ defmodule Plausible.ExportsTest do
test "with date range", %{site: site} do
queries =
- Plausible.Exports.export_queries(site.id,
+ Plausible.Exports.export_queries(site,
date_range: Date.range(~D[2023-01-01], ~D[2024-03-12])
)
assert Map.keys(queries) == [
"imported_browsers_20230101_20240312.csv",
"imported_custom_events_20230101_20240312.csv",
+ "imported_custom_props_20230101_20240312.csv",
"imported_devices_20230101_20240312.csv",
"imported_entry_pages_20230101_20240312.csv",
"imported_exit_pages_20230101_20240312.csv",
@@ -47,11 +49,12 @@ defmodule Plausible.ExportsTest do
end
test "with custom extension", %{site: site} do
- queries = Plausible.Exports.export_queries(site.id, extname: ".ch")
+ queries = Plausible.Exports.export_queries(site, extname: ".ch")
assert Map.keys(queries) == [
"imported_browsers.ch",
"imported_custom_events.ch",
+ "imported_custom_props.ch",
"imported_devices.ch",
"imported_entry_pages.ch",
"imported_exit_pages.ch",
diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
index 23fdd6bb75..5c67fa46ee 100644
--- a/test/plausible/imported/csv_importer_test.exs
+++ b/test/plausible/imported/csv_importer_test.exs
@@ -608,7 +608,7 @@ defmodule Plausible.Imported.CSVImporterTest do
imported_site: imported_site
}
- %{site_import: site_import} =
+ %{site_import: site_import, exported_files: exported_files} =
initial_context
|> export_archive()
|> assert_email_notification()
@@ -617,6 +617,16 @@ defmodule Plausible.Imported.CSVImporterTest do
|> upload_csvs()
|> run_import()
+ assert custom_props_export =
+ exported_files
+ |> Enum.find(&String.contains?(&1, "imported_custom_props_"))
+ |> File.read!()
+ |> String.split("\n")
+
+ assert ~s|"date","property","value","visitors","events"| in custom_props_export
+ assert ~s|"2024-04-01","author","Marko Saric",43,57| in custom_props_export
+ assert ~s|"2024-04-01","category","Posts",43,56| in custom_props_export
+
assert %SiteImport{
start_date: ~D[2024-04-01],
end_date: ~D[2024-04-30],