Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Dashboard csv export v2 continued (#6455)
RobertJoonase20b0c2c0a51
diff --git a/assets/js/dashboard/stats/csv-export/csv-export-body.ts b/assets/js/dashboard/stats/csv-export/csv-export-body.tsindex 860ee609e0..b03731702b 100644--- a/assets/js/dashboard/stats/csv-export/csv-export-body.ts+++ b/assets/js/dashboard/stats/csv-export/csv-export-body.ts@@ -19,7 +19,8 @@ import { Metric } from '../metrics'import {BREAKDOWN_REPORTS,BreakdownReportConfig,- BreakdownReportKey+ BreakdownReportKey,+ getCustomPropsMetrics} from '../reports/reports-config'const BREAKDOWN_CSV_REPORTS = {@@ -41,9 +42,13 @@ const BREAKDOWN_CSV_REPORTS = {'utm_terms.csv': BreakdownReportKey.utmTerms,'countries.csv': BreakdownReportKey.countries,'regions.csv': BreakdownReportKey.regions,- 'cities.csv': BreakdownReportKey.cities+ 'cities.csv': BreakdownReportKey.cities,+ 'conversions.csv': BreakdownReportKey.goals}-type CsvFilename = 'visitors.csv' | keyof typeof BREAKDOWN_CSV_REPORTS+type CsvFilename =+ | 'visitors.csv'+ | 'custom_props.csv'+ | keyof typeof BREAKDOWN_CSV_REPORTStype CsvReportParams = {dimensions: [TimeDimension] | BreakdownReportConfig['dimensions']@@ -114,6 +119,13 @@ export function createCsvExportRequestBody(? PAGE_FILTERED_VISITORS_CSV_METRICS: DEFAULT_VISITORS_CSV_METRICS},+ 'custom_props.csv': {+ dimensions: ['event:props:*'],+ metrics: getCustomPropsMetrics({+ hasConversionGoalFilter: isGoalFilter,+ isRevenueAvailable: false+ })+ },...Object.entries(BREAKDOWN_CSV_REPORTS).reduce((acc, [filename, reportKey]) => {const config = BREAKDOWN_REPORTS[reportKey]@@ -128,7 +140,7 @@ export function createCsvExportRequestBody(}return acc},- {} as Omit<CsvReportsConfig, 'visitors.csv'>+ {} as Omit<CsvReportsConfig, 'visitors.csv' | 'custom_props.csv'>)}diff --git a/assets/js/dashboard/stats/reports/reports-config.ts b/assets/js/dashboard/stats/reports/reports-config.tsindex 7ef2be6c27..8f0fbb823b 100644--- a/assets/js/dashboard/stats/reports/reports-config.ts+++ b/assets/js/dashboard/stats/reports/reports-config.ts@@ -408,6 +408,9 @@ export const BREAKDOWN_REPORTS: Record<[BreakdownReportKey.goals]: {dimensions: ['event:goal'],getMetrics: (ctx: MetricContext) => {+ if (ctx.isCsv) {+ return ['visitors', 'events']+ }if (ctx.isRevenueAvailable) {return ['visitors',@@ -430,23 +433,25 @@ export function customPropsReportConfig(): BreakdownReportConfig {return {dimensions: [`event:props:${propKey}` as NonTimeDimension],- getMetrics: (ctx: MetricContext) => {- if (ctx.hasConversionGoalFilter && ctx.isRevenueAvailable) {- return [- 'visitors',- 'events',- 'conversion_rate',- 'total_revenue',- 'average_revenue'- ]- }- if (ctx.hasConversionGoalFilter) {- return ['visitors', 'events', 'conversion_rate']- }- return ['visitors', 'events', 'percentage']- },+ getMetrics: getCustomPropsMetrics,detailsTitle: 'Custom property breakdown',detailsPath: `custom-prop-values/${propKey}`,dimensionLabel: propKey}}++export function getCustomPropsMetrics(ctx: MetricContext): Metric[] {+ if (ctx.hasConversionGoalFilter && ctx.isRevenueAvailable) {+ return [+ 'visitors',+ 'events',+ 'conversion_rate',+ 'total_revenue',+ 'average_revenue'+ ]+ }+ if (ctx.hasConversionGoalFilter) {+ return ['visitors', 'events', 'conversion_rate']+ }+ return ['visitors', 'events', 'percentage']+}diff --git a/e2e/tests/dashboard/csv-export.spec.ts b/e2e/tests/dashboard/csv-export.spec.tsindex dc87e89344..a7d742acdc 100644--- a/e2e/tests/dashboard/csv-export.spec.ts+++ b/e2e/tests/dashboard/csv-export.spec.ts@@ -57,7 +57,21 @@ const EXPECTED_HEADERS = {commonGoalFilteredBreakdown: ['name', 'conversions', 'conversion_rate'],nameAndVisitors: ['name', 'visitors'],nameVersionAndVisitors: ['name', 'version', 'visitors'],- nameVersionGoalFiltered: ['name', 'version', 'conversions', 'conversion_rate']+ nameVersionGoalFiltered: [+ 'name',+ 'version',+ 'conversions',+ 'conversion_rate'+ ],+ customPropsDefault: ['property', 'value', 'visitors', 'events', 'percentage'],+ customPropsGoalFiltered: [+ 'property',+ 'value',+ 'visitors',+ 'events',+ 'conversion_rate'+ ],+ conversionsBreakdown: ['name', 'unique_conversions', 'total_conversions']}const UTM_AND_SOURCE_REPORTS = [@@ -119,11 +133,14 @@ test('csv export column headers match expected metrics for each report', async (await populateStats({request,domain,- events: [{ name: 'pageview', timestamp: '2021-01-01 12:00:00' }]+ events: [+ { name: 'pageview', 'meta.key': ['author'], 'meta.value': ['john'] },+ { name: 'Signup', 'meta.key': ['author'], 'meta.value': ['john'] }+ ]})await test.step('without any filters', async () => {- await page.goto(`/${domain}`, {+ await page.goto(`/${domain}?period=all`, {waitUntil: 'commit'})@@ -157,13 +174,23 @@ test('csv export column headers match expected metrics for each report', async (EXPECTED_HEADERS.nameVersionAndVisitors)}+ expect(getCsv(csvs, 'custom_props.csv')[0]).toEqual(+ EXPECTED_HEADERS.customPropsDefault+ )+ expect(getCsv(csvs, 'conversions.csv')[0]).toEqual(+ EXPECTED_HEADERS.conversionsBreakdown+ )})await test.step('with a page filter', async () => {- await page.goto(`/${domain}?f=is,page,/blog`, {+ await page.goto(`/${domain}?period=all&f=is,page,/`, {waitUntil: 'commit'})+ await expect(+ page.getByRole('button', { name: 'Remove filter: Page is /' })+ ).toBeVisible()+const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(@@ -194,15 +221,25 @@ test('csv export column headers match expected metrics for each report', async (EXPECTED_HEADERS.nameVersionAndVisitors)}+ expect(getCsv(csvs, 'custom_props.csv')[0]).toEqual(+ EXPECTED_HEADERS.customPropsDefault+ )+ expect(getCsv(csvs, 'conversions.csv')[0]).toEqual(+ EXPECTED_HEADERS.conversionsBreakdown+ )})await test.step('with a goal filter', async () => {await addGoal({ request, domain, params: { event_name: 'Signup' } })- await page.goto(`/${domain}?f=is,goal,Signup`, {+ await page.goto(`/${domain}?period=all&f=is,goal,Signup`, {waitUntil: 'commit'})+ await expect(+ page.getByRole('button', { name: 'Remove filter: Goal is Signup' })+ ).toBeVisible()+const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(@@ -225,13 +262,25 @@ test('csv export column headers match expected metrics for each report', async (EXPECTED_HEADERS.nameVersionGoalFiltered)}+ expect(getCsv(csvs, 'custom_props.csv')[0]).toEqual(+ EXPECTED_HEADERS.customPropsGoalFiltered+ )+ expect(getCsv(csvs, 'conversions.csv')[0]).toEqual(+ EXPECTED_HEADERS.conversionsBreakdown+ )})await test.step('with a custom prop filter', async () => {- await page.goto(`/${domain}?f=is,props:author,john`, {+ await page.goto(`/${domain}?period=all&f=is,props:author,john`, {waitUntil: 'commit'})+ await expect(+ page.getByRole('button', {+ name: 'Remove filter: Property author is john'+ })+ ).toBeVisible()+const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(@@ -262,6 +311,12 @@ test('csv export column headers match expected metrics for each report', async (EXPECTED_HEADERS.nameVersionAndVisitors)}+ expect(getCsv(csvs, 'custom_props.csv')[0]).toEqual(+ EXPECTED_HEADERS.customPropsDefault+ )+ expect(getCsv(csvs, 'conversions.csv')[0]).toEqual(+ EXPECTED_HEADERS.conversionsBreakdown+ )})})@@ -309,7 +364,7 @@ test('filters out empty visit:* dimension values', async ({]})- await page.goto(`/${domain}?period=day&date=2021-01-01`, {+ await page.goto(`/${domain}?period=all`, {waitUntil: 'commit'})diff --git a/lib/plausible/stats/api_query_parser.ex b/lib/plausible/stats/api_query_parser.exindex 35837f1eba..c8d16cc1f7 100644--- a/lib/plausible/stats/api_query_parser.ex+++ b/lib/plausible/stats/api_query_parser.ex@@ -250,8 +250,8 @@ defmodule Plausible.Stats.ApiQueryParser dodefp parse_dimension_entry(key, error_message) docase {- parse_time(key),- parse_filter_dimension_string(key)+ parse_time_dimension(key),+ parse_non_time_dimension(key)} do{{:ok, time}, _} -> {:ok, time}{_, {:ok, dimension}} -> {:ok, dimension}@@ -261,18 +261,18 @@ defmodule Plausible.Stats.ApiQueryParser dodefp parse_metric_or_dimension([value, _] = entry) docase {- parse_time(value),+ parse_time_dimension(value),parse_metric(value),- parse_filter_dimension_string(value)+ parse_non_time_dimension(value)} do- {{:ok, time}, _, _} ->- {:ok, time}+ {{:ok, time_dimension}, _, _} ->+ {:ok, time_dimension}{_, {:ok, metric}, _} ->{:ok, metric}- {_, _, {:ok, dimension}} ->- {:ok, dimension}+ {_, _, {:ok, non_time_dimension}} ->+ {:ok, non_time_dimension}_ ->{:error,@@ -280,13 +280,23 @@ defmodule Plausible.Stats.ApiQueryParser doendend- defp parse_time("time"), do: {:ok, "time"}- defp parse_time("time:minute"), do: {:ok, "time:minute"}- defp parse_time("time:hour"), do: {:ok, "time:hour"}- defp parse_time("time:day"), do: {:ok, "time:day"}- defp parse_time("time:week"), do: {:ok, "time:week"}- defp parse_time("time:month"), do: {:ok, "time:month"}- defp parse_time(_), do: :error+ defp parse_time_dimension("time"), do: {:ok, "time"}+ defp parse_time_dimension("time:minute"), do: {:ok, "time:minute"}+ defp parse_time_dimension("time:hour"), do: {:ok, "time:hour"}+ defp parse_time_dimension("time:day"), do: {:ok, "time:day"}+ defp parse_time_dimension("time:week"), do: {:ok, "time:week"}+ defp parse_time_dimension("time:month"), do: {:ok, "time:month"}+ defp parse_time_dimension(_), do: :error++ @breakdown_only_non_time_dimensions ["event:prop_key"]++ defp parse_non_time_dimension(dimension) do+ if dimension in @breakdown_only_non_time_dimensions do+ {:ok, dimension}+ else+ parse_filter_dimension_string(dimension)+ end+ enddefp parse_order_direction([_, "asc"]), do: {:ok, :asc}defp parse_order_direction([_, "desc"]), do: {:ok, :desc}diff --git a/lib/plausible/stats/dashboard/csv_export.ex b/lib/plausible/stats/dashboard/csv_export.exindex af78a18e6d..869b6e8cf1 100644--- a/lib/plausible/stats/dashboard/csv_export.ex+++ b/lib/plausible/stats/dashboard/csv_export.ex@@ -24,39 +24,55 @@ defmodule Plausible.Stats.Dashboard.CsvExport do"utm_terms.csv","countries.csv","regions.csv",- "cities.csv"+ "cities.csv",+ "custom_props.csv",+ "conversions.csv"]def get_csvs(site, params, debug_metadata) do- with {:ok, queries_by_filename} <- create_queries_by_filename(site, params, debug_metadata) do+ with {:ok, tasks_by_filename} <- create_tasks_by_filename(site, params, debug_metadata) docsv_values =- queries_by_filename- |> Enum.map(fn {filename, query} ->- fn -> run_query_into_csv(site, filename, query) end- end)+ tasks_by_filename+ |> Enum.map(fn {_filename, task} -> task end)|> Plausible.ClickhouseRepo.parallel_tasks()csvs =- queries_by_filename- |> Enum.map(fn {filename, _query} -> filename end)+ tasks_by_filename+ |> Enum.map(fn {filename, _task} -> filename end)|> Enum.zip(csv_values)+ |> Enum.reject(fn {_filename, csv_values} -> is_nil(csv_values) end)|> Enum.map(fn {k, v} -> {String.to_charlist(k), IO.iodata_to_binary(v)} end){:ok, csvs}endend- defp create_queries_by_filename(site, params, debug_metadata) do+ defp create_tasks_by_filename(site, params, debug_metadata) dorequested = Map.keys(params["reports"] || %{}) |> Enum.filter(&(&1 in @csv_filenames))Enum.reduce_while(requested, {:ok, []}, fn filename, acc ->- case parse_csv_query(site, filename, params, debug_metadata) do- {:ok, query} -> {:cont, {:ok, elem(acc, 1) ++ [{filename, query}]}}+ case create_task(site, filename, params, debug_metadata) do+ {:ok, task} -> {:cont, {:ok, elem(acc, 1) ++ [{filename, task}]}}error_tuple -> {:halt, error_tuple}endend)end+ defp create_task(site, "custom_props.csv", params, debug_metadata) do+ with {:ok, prop_keys_query} <- get_custom_prop_key_query(site, params, debug_metadata),+ prop_keys = get_custom_prop_keys(site, prop_keys_query),+ {:ok, prop_value_queries_by_prop_key} <-+ get_custom_prop_queries(site, params, debug_metadata, prop_keys) do+ {:ok, fn -> get_custom_props_csv(site, prop_value_queries_by_prop_key) end}+ end+ end++ defp create_task(site, filename, params, debug_metadata) do+ with {:ok, query} <- parse_csv_query(site, filename, params, debug_metadata) do+ {:ok, fn -> run_query_into_csv(site, filename, query) end}+ end+ end+defp parse_csv_query(site, filename, params, debug_metadata) doparams = construct_raw_query_params(filename, params)@@ -100,11 +116,10 @@ defmodule Plausible.Stats.Dashboard.CsvExport do# data like time on page or bounce_rate for pages in a separate query using# the IN filter, it causes the requests to balloon in payload size.limit = if filename in ["pages.csv", "exit_pages.csv"], do: 100, else: 300- order_by = [{:visitors, :desc} | Enum.map(query.dimensions, &{&1, :asc})]query =query- |> Query.set(order_by: order_by)+ |> Query.set(order_by: get_order_by(query.dimensions))|> Query.set(pagination: %{limit: limit, offset: 0})QueryRunner.run(site, query) |> query_result_to_csv()@@ -136,6 +151,12 @@ defmodule Plausible.Stats.Dashboard.CsvExport do|> Map.get(timelabel)end+ defp get_order_by(["event:goal"]), do: [{:visitors, :desc}]++ defp get_order_by(dimensions) do+ [{:visitors, :desc} | Enum.map(dimensions, &{&1, :asc})]+ end+defp csv_first_row(query) docsv_dimension_labels(query[:dimensions]) ++ csv_metric_labels(query)end@@ -165,6 +186,12 @@ defmodule Plausible.Stats.Dashboard.CsvExport dogroup_conversion_rate: :conversion_rate}+ ["event:goal"] ->+ %{+ visitors: :unique_conversions,+ events: :total_conversions+ }+_ ->%{visitors: :conversions,@@ -181,6 +208,10 @@ defmodule Plausible.Stats.Dashboard.CsvExport do["visit:exit_page"] => %{visitors: :unique_exits,visits: :total_exits+ },+ ["event:goal"] => %{+ visitors: :unique_conversions,+ events: :total_conversions}}@@ -211,4 +242,88 @@ defmodule Plausible.Stats.Dashboard.CsvExport dodefp csv_dimension_values([_dimension], row) dorow[:dimensions]end++ defp get_custom_prop_key_query(site, params, debug_metadata) do+ raw_params =+ construct_raw_query_params("custom_props.csv", params)+ |> Map.merge(%{+ "dimensions" => ["event:prop_key"],+ "metrics" => ["visitors"],+ "pagination" => %{"limit" => 25}+ })++ with {:ok, parsed} <- QueryParser.parse(raw_params),+ {:ok, query} <- QueryBuilder.build(site, parsed, debug_metadata) do+ {:ok, Query.set(query, order_by: [{:visitors, :desc}, {"event:prop_key", :asc}])}+ end+ end++ defp get_custom_prop_keys(site, query) do+ case Plausible.Stats.Filters.get_toplevel_filter(query, "event:props:") do+ [_op, "event:props:" <> key | _rest] ->+ [key]++ _ ->+ %QueryResult{results: results} = QueryRunner.run(site, query)++ Enum.map(results, fn r -> hd(r.dimensions) end)+ |> maybe_allowed_props_only(site)+ end+ end++ defp get_custom_prop_queries(site, params, debug_metadata, prop_keys) do+ Enum.reduce_while(prop_keys, {:ok, []}, fn prop_key, acc ->+ dimension = "event:props:#{prop_key}"++ raw_params =+ construct_raw_query_params("custom_props.csv", params)+ |> Map.put("dimensions", [dimension])++ with {:ok, parsed} <- QueryParser.parse(raw_params),+ {:ok, query} <- QueryBuilder.build(site, parsed, debug_metadata) do+ query =+ query+ |> Query.set(order_by: [{:visitors, :desc}, {dimension, :asc}])+ |> Query.set(pagination: %{limit: 300, offset: 0})++ {:cont, {:ok, elem(acc, 1) ++ [{prop_key, query}]}}+ else+ error_tuple -> {:halt, error_tuple}+ end+ end)+ end++ defp get_custom_props_csv(_site, []), do: nil++ defp get_custom_props_csv(site, prop_value_queries_by_prop_key) do+ [{_prop_key, %Query{metrics: metrics}} | _] = prop_value_queries_by_prop_key+ header_row = format_custom_props_header_row(metrics)+ data_rows = custom_prop_queries_into_data_rows(site, prop_value_queries_by_prop_key)++ NimbleCSV.RFC4180.dump_to_iodata([header_row] ++ data_rows)+ end++ defp custom_prop_queries_into_data_rows(site, prop_value_queries_by_prop_key) do+ prop_value_queries_by_prop_key+ |> Enum.map(fn {prop_key, query} ->+ fn ->+ %QueryResult{results: results} = QueryRunner.run(site, query)+ Enum.map(results, &format_custom_props_data_row(prop_key, &1))+ end+ end)+ |> Plausible.ClickhouseRepo.parallel_tasks()+ |> Enum.concat()+ end++ defp format_custom_props_header_row(metrics), do: [:property, :value] ++ metrics++ defp format_custom_props_data_row(prop_key, row),+ do: [prop_key | row.dimensions] ++ row.metrics++ defp maybe_allowed_props_only(prop_keys, site) do+ case Plausible.Props.allowed_for(site) do+ :all -> prop_keys+ allowed -> Enum.filter(prop_keys, &(&1 in allowed))+ end+ endenddiff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex 91caed4c96..8e5b4132c4 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -170,6 +170,16 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> group_by([], selected_as(^key))end+ defp dimension_group_by(q, :events, query, "event:prop_key" = dimension) do+ from(e in q,+ join: meta in fragment("meta"),+ hints: "ARRAY",+ on: true,+ group_by: meta.key+ )+ |> select_merge_as([_e, meta], %{shortname(query, dimension) => meta.key})+ end+defp dimension_group_by(q, :events, query, "event:goal" = dimension) dogoal_join_data = Plausible.Stats.Goals.goal_join_data(query)diff --git a/test/plausible/stats/dashboard/query_parser_test.exs b/test/plausible/stats/dashboard/query_parser_test.exsindex 32b08eaa8d..8e855caad2 100644--- a/test/plausible/stats/dashboard/query_parser_test.exs+++ b/test/plausible/stats/dashboard/query_parser_test.exs@@ -193,6 +193,27 @@ defmodule Plausible.Stats.Dashboard.QueryParserTest doendend+ describe "dimensions" do+ test "parses breakdown-only dimensions" do+ params = Map.merge(@base_params, %{"dimensions" => ["event:prop_key"]})+ {:ok, parsed} = parse(params)+ assert parsed.dimensions == ["event:prop_key"]+ end+ end++ describe "order_by" do+ test "breakdown-only dimensions can be used in order_by" do+ params =+ Map.merge(@base_params, %{+ "dimensions" => ["event:prop_key"],+ "order_by" => [["event:prop_key", "asc"]]+ })++ {:ok, parsed} = parse(params)+ assert parsed.order_by == [{"event:prop_key", :asc}]+ end+ end+describe "filters" dotest "parses valid filters" doparams =@@ -215,6 +236,15 @@ defmodule Plausible.Stats.Dashboard.QueryParserTest do]end+ test "cannot filter by breakdown-only dimensions" do+ params =+ Map.merge(@base_params, %{+ "filters" => [["is", "event:prop_key", ["author"]]]+ })++ {:error, %QueryError{code: :invalid_filters}} = parse(params)+ end+test "parses city filter with multiple clauses" doparams =Map.merge(@base_params, %{diff --git a/test/plausible/stats/query/query_test.exs b/test/plausible/stats/query/query_test.exsindex 4b513f4bfb..1616571d52 100644--- a/test/plausible/stats/query/query_test.exs+++ b/test/plausible/stats/query/query_test.exs@@ -601,4 +601,73 @@ defmodule Plausible.Stats.QueryTest doassert is_nil(meta[:empty_metrics])endend++ describe "event:prop_key dimension" do+ test "returns distinct prop keys with visitor counts", %{site: site} do+ populate_stats(site, [+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"]),+ build(:pageview)+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors],+ input_date_range: :all,+ dimensions: ["event:prop_key"]+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert [+ %{dimensions: ["author"], metrics: [2]},+ %{dimensions: ["logged_in"], metrics: [1]}+ ] = Enum.sort_by(results, &hd(&1.dimensions))+ end++ test "supports order_by", %{site: site} do+ populate_stats(site, [+ build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),+ build(:pageview, "meta.key": ["author"], "meta.value": ["b"]),+ build(:pageview, "meta.key": ["author"], "meta.value": ["b"]),+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors],+ input_date_range: :all,+ dimensions: ["event:prop_key"],+ order_by: [{"event:prop_key", :desc}]+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["logged_in"], metrics: [1]},+ %{dimensions: ["author"], metrics: [3]}+ ]+ end++ test "works combined with a filter", %{site: site} do+ populate_stats(site, [+ build(:event, name: "purchase", "meta.key": ["author"], "meta.value": ["uku"]),+ build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors],+ input_date_range: :all,+ dimensions: ["event:prop_key"],+ filters: [[:is, "event:name", ["purchase"]]]+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [%{dimensions: ["author"], metrics: [1]}]+ end+ endenddiff --git a/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs b/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exsindex ecc13363e7..fee06be0c2 100644--- a/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs@@ -99,6 +99,14 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest dodimensions: ["visit:city_name"],always_on_filters: [["is_not", "visit:city", [0]]],metrics: ["visitors"]+ },+ "custom_props.csv" => %{+ dimensions: ["event:props:*"],+ metrics: ["visitors", "events", "percentage"]+ },+ "conversions.csv" => %{+ dimensions: ["event:goal"],+ metrics: ["visitors", "events"]}}@@ -114,12 +122,24 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do["exit_pages.csv", :metrics],get_in(@default_reports, ["exit_pages.csv", :metrics]) -- ["exit_rate"])+ @prop_filtered_reports put_in(+ @default_reports,+ ["exit_pages.csv", :metrics],+ get_in(@default_reports, ["exit_pages.csv", :metrics]) -- ["exit_rate"]+ )@goal_filtered_reports Map.new(@default_reports, fn{"visitors.csv" = filename, params} ->{filename,%{params | metrics: ["visitors", "events", "group_conversion_rate"]}}+ {"custom_props.csv" = filename, params} ->+ {filename,+ %{params | metrics: ["visitors", "events", "conversion_rate"]}}++ {"conversions.csv" = filename, params} ->+ {filename, params}+{filename, params} ->{filename,%{params | metrics: ["visitors", "group_conversion_rate"]}}@@ -150,7 +170,7 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest doassert ~c"browsers.csv" in zipassert ~c"browser_versions.csv" in zipassert ~c"cities.csv" in zip- # assert ~c"conversions.csv" in zip+ assert ~c"conversions.csv" in zipassert ~c"countries.csv" in zipassert ~c"devices.csv" in zipassert ~c"entry_pages.csv" in zip@@ -185,6 +205,33 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest doassert length(exit_pages) == 102end+ test "limits custom_props.csv to 25 prop keys", %{conn: conn, site: site} do+ keys = for i <- 1..26, do: "key_#{i}"+ {:ok, site} = Plausible.Props.allow(site, keys)++ populate_stats(site, [+ build(:pageview,+ "meta.key": keys,+ "meta.value": List.duplicate("v", 26)+ )+ ])++ custom_props =+ conn+ |> do_export(site, %{@base_params | date_range: "day"})+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")++ prop_keys =+ custom_props+ |> Enum.drop(1)+ |> Enum.reject(&(&1 == [""]))+ |> Enum.map(&hd/1)+ |> Enum.uniq()++ assert length(prop_keys) == 25+ end+test "limits other breakdown reports to 300 rows", %{conn: conn, site: site} doevents = for i <- 1..301, do: build(:pageview, referrer_source: "Source #{i}")populate_stats(site, events)@@ -268,57 +315,57 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do]end- # test "exports only internally used props in custom_props.csv for a growth plan", %{- # conn: conn,- # site: site- # } do- # {:ok, site} = Plausible.Props.allow(site, ["author"])-- # [owner | _] = Repo.preload(site, :owners).owners- # subscribe_to_growth_plan(owner)-- # populate_stats(site, [- # build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),- # build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])- # ])-- # result =- # conn- # |> do_export(site, %{@base_params | date_range: "day"})- # |> response(200)- # |> unzip_and_parse_csv(~c"custom_props.csv")-- # assert result == [- # ["property", "value", "visitors", "events", "percentage"],- # ["url", "(none)", "1", "1", "50.0"],- # ["url", "b", "1", "1", "50.0"],- # [""]- # ]- # end-- # test "does not include custom_props.csv for a growth plan if no internal props used", %{- # conn: conn,- # site: site- # } do- # {:ok, site} = Plausible.Props.allow(site, ["author"])-- # [owner | _] = Repo.preload(site, :owners).owners- # subscribe_to_growth_plan(owner)-- # populate_stats(site, [- # build(:pageview, "meta.key": ["author"], "meta.value": ["a"])- # ])-- # {:ok, zip} =- # conn- # |> do_export(site, %{@base_params | date_range: "day"})- # |> response(200)- # |> :zip.unzip([:memory])-- # files = Map.new(zip)-- # refute Map.has_key?(files, ~c"custom_props.csv")- # end+ test "exports only internally used props in custom_props.csv for a growth plan", %{+ conn: conn,+ site: site+ } do+ {:ok, site} = Plausible.Props.allow(site, ["author"])++ [owner | _] = Repo.preload(site, :owners).owners+ subscribe_to_growth_plan(owner)++ populate_stats(site, [+ build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),+ build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])+ ])++ result =+ conn+ |> do_export(site, %{@base_params | date_range: "day"})+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")++ assert result == [+ ["property", "value", "visitors", "events", "percentage"],+ ["url", "(none)", "1", "1", "50.0"],+ ["url", "b", "1", "1", "50.0"],+ [""]+ ]+ end++ test "does not include custom_props.csv for a growth plan if no internal props used", %{+ conn: conn,+ site: site+ } do+ {:ok, site} = Plausible.Props.allow(site, ["author"])++ [owner | _] = Repo.preload(site, :owners).owners+ subscribe_to_growth_plan(owner)++ populate_stats(site, [+ build(:pageview, "meta.key": ["author"], "meta.value": ["a"])+ ])++ {:ok, zip} =+ conn+ |> do_export(site, %{@base_params | date_range: "day"})+ |> response(200)+ |> :zip.unzip([:memory])++ files = Map.new(zip)++ refute Map.has_key?(files, ~c"custom_props.csv")+ endtest "exports data in zipped csvs", %{conn: conn, site: site} dopopulate_exported_stats(site)@@ -329,35 +376,35 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest doassert_zip(conn, "30d")end- # test "exports allowed event props for a trial account", %{conn: conn, site: site} do- # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])-- # populate_stats(site, [- # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),- # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),- # build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),- # build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),- # build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),- # build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),- # build(:pageview)- # ])-- # result =- # conn- # |> do_export(site, %{@base_params | date_range: "day"})- # |> response(200)- # |> unzip_and_parse_csv(~c"custom_props.csv")-- # assert result == [- # ["property", "value", "visitors", "events", "percentage"],- # ["author", "(none)", "3", "4", "50.0"],- # ["author", "uku", "2", "2", "33.33"],- # ["author", "marko", "1", "1", "16.67"],- # ["logged_in", "(none)", "5", "5", "83.33"],- # ["logged_in", "true", "1", "2", "16.67"],- # [""]- # ]- # end+ test "exports allowed event props for a trial account", %{conn: conn, site: site} do+ {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])++ populate_stats(site, [+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),+ build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),+ build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),+ build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),+ build(:pageview)+ ])++ result =+ conn+ |> do_export(site, %{@base_params | date_range: "day"})+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")++ assert result == [+ ["property", "value", "visitors", "events", "percentage"],+ ["author", "(none)", "3", "4", "50.0"],+ ["author", "uku", "2", "2", "33.33"],+ ["author", "marko", "1", "1", "16.67"],+ ["logged_in", "(none)", "5", "5", "83.33"],+ ["logged_in", "true", "1", "2", "16.67"],+ [""]+ ]+ endtest "exports data grouped by interval", %{conn: conn, site: site} dopopulate_exported_stats(site)@@ -479,7 +526,7 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do"operating_systems.csv","operating_system_versions.csv","devices.csv",- # "conversions.csv",+ "conversions.csv","referrers.csv"]@@ -630,6 +677,9 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do["Direct / None", "1", "0.0", "10.0"],[""]]++ {~c"custom_props.csv", _data} ->+ :okend)endend@@ -752,38 +802,39 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest doendend- # describe "POST /api/stats/:domain/export - with a custom prop filter" do- # setup [:create_user, :create_site, :log_in]-- # test "custom-props.csv only returns the prop and its value in filter", %{- # conn: conn,- # site: site- # } do- # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])-- # populate_stats(site, [- # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),- # build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),- # build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])- # ])-- # result =- # conn- # |> do_export(site, %{- # @base_params- # | date_range: "day",- # filters: [["is", "event:props:author", ["marko"]]]- # })- # |> response(200)- # |> unzip_and_parse_csv(~c"custom_props.csv")-- # assert result == [- # ["property", "value", "visitors", "events", "percentage"],- # ["author", "marko", "1", "1", "100.0"],- # [""]- # ]- # end- # end+ describe "POST /api/stats/:domain/export - with a custom prop filter" do+ setup [:create_user, :create_site, :log_in]++ test "custom-props.csv only returns the prop and its value in filter", %{+ conn: conn,+ site: site+ } do+ {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])++ populate_stats(site, [+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])+ ])++ result =+ conn+ |> do_export(site, %{+ @base_params+ | date_range: "day",+ filters: [["is", "event:props:author", ["marko"]]],+ reports: @prop_filtered_reports+ })+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")++ assert result == [+ ["property", "value", "visitors", "events", "percentage"],+ ["author", "marko", "1", "1", "100.0"],+ [""]+ ]+ end+ enddefp unzip_and_parse_csv(archive, filename) do{:ok, zip} = :zip.unzip(archive, [:memory])@@ -998,39 +1049,39 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest doassert_zip(conn, "30d-filter-goal")end- # test "custom-props.csv only returns the prop names for the goal in filter", %{- # conn: conn,- # site: site- # } do- # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])-- # populate_stats(site, [- # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),- # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),- # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),- # build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])- # ])-- # insert(:goal, site: site, event_name: "Newsletter Signup")-- # result =- # conn- # |> do_export(site, %{- # @base_params- # | date_range: "day",- # filters: [["is", "event:goal", ["Newsletter Signup"]]],- # reports: @goal_filtered_reports- # })- # |> response(200)- # |> unzip_and_parse_csv(~c"custom_props.csv")-- # assert result == [- # ["property", "value", "visitors", "events", "conversion_rate"],- # ["author", "marko", "2", "2", "50.0"],- # ["author", "uku", "1", "1", "25.0"],- # [""]- # ]- # end+ test "custom-props.csv only returns the prop names for the goal in filter", %{+ conn: conn,+ site: site+ } do+ {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])++ populate_stats(site, [+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])+ ])++ insert(:goal, site: site, event_name: "Newsletter Signup")++ result =+ conn+ |> do_export(site, %{+ @base_params+ | date_range: "day",+ filters: [["is", "event:goal", ["Newsletter Signup"]]],+ reports: @goal_filtered_reports+ })+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")++ assert result == [+ ["property", "value", "visitors", "events", "conversion_rate"],+ ["author", "marko", "2", "2", "50.0"],+ ["author", "uku", "1", "1", "25.0"],+ [""]+ ]+ endtest "exports conversions and conversion rate for operating system versions", %{conn: conn,