Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Scroll depth dashboard warnings (imported data) (#5051)
RobertJoonas269430281da6
diff --git a/assets/js/dashboard/components/sort-button.tsx b/assets/js/dashboard/components/sort-button.tsxindex 94aedb5ba8..21cd21e80c 100644--- a/assets/js/dashboard/components/sort-button.tsx+++ b/assets/js/dashboard/components/sort-button.tsx@@ -17,11 +17,11 @@ export const SortButton = ({return (<buttononClick={toggleSort}- title={next.hint}className={classNames('group', 'hover:underline', 'relative')}>{children}<span+ title={next.hint}className={classNames('absolute','rounded inline-block h-4 w-4',diff --git a/assets/js/dashboard/components/table.tsx b/assets/js/dashboard/components/table.tsxindex 53976ec023..d8486bae6a 100644--- a/assets/js/dashboard/components/table.tsx+++ b/assets/js/dashboard/components/table.tsx@@ -4,6 +4,7 @@ import classNames from 'classnames'import React, { ReactNode } from 'react'import { SortDirection } from '../hooks/use-order-by'import { SortButton } from './sort-button'+import { Tooltip } from '../util/tooltip'export type ColumnConfiguraton<T extends Record<string, unknown>> = {/** Unique column ID, used for sorting purposes and to get the value of the cell using rowItem[key] */@@ -17,6 +18,8 @@ export type ColumnConfiguraton<T extends Record<string, unknown>> = {width: string/** Aligns column content. */align?: 'left' | 'right'+ /** A warning to be rendered as a tooltip for the column header */+ metricWarning?: string/*** Function used to transform the value found at item[key] for the cell. Superseded by renderItem if present. @example 1120 => "1.1k"*/@@ -100,6 +103,29 @@ export const Table = <T extends Record<string, string | number | ReactNode>>({columns: ColumnConfiguraton<T>[]data: T[] | { pages: T[][] }}) => {+ const renderColumnLabel = (column: ColumnConfiguraton<T>) => {+ if (column.metricWarning) {+ return (+ <Tooltip+ info={warningSpan(column.metricWarning)}+ className="inline-block"+ >+ {column.label + ' *'}+ </Tooltip>+ )+ } else {+ return column.label+ }+ }++ const warningSpan = (warning: string) => {+ return (+ <span className="text-xs font-normal whitespace-nowrap">+ {'*' + warning}+ </span>+ )+ }+return (<table className="w-max overflow-x-auto md:w-full table-striped table-fixed"><thead>@@ -115,10 +141,10 @@ export const Table = <T extends Record<string, string | number | ReactNode>>({toggleSort={column.onSort}sortDirection={column.sortDirection ?? null}>- {column.label}+ {renderColumnLabel(column)}</SortButton>) : (- column.label+ renderColumnLabel(column))}</TableHeaderCell>))}diff --git a/assets/js/dashboard/query.ts b/assets/js/dashboard/query.tsindex 424fac127a..68859c482a 100644--- a/assets/js/dashboard/query.ts+++ b/assets/js/dashboard/query.ts@@ -52,6 +52,7 @@ export type DashboardQuery = typeof queryDefaultValueexport type BreakdownResultMeta = {date_range_label: stringcomparison_date_range_label?: string+ metric_warnings: Record<string, Record<string, string>> | undefined}export function addFilter(diff --git a/assets/js/dashboard/stats/graph/top-stats.js b/assets/js/dashboard/stats/graph/top-stats.jsindex 24b9d02e04..45b9ca3576 100644--- a/assets/js/dashboard/stats/graph/top-stats.js+++ b/assets/js/dashboard/stats/graph/top-stats.js@@ -64,6 +64,14 @@ export default function TopStats({ data, onMetricUpdate, tooltipBoundary }) {<SecondsSinceLastLoad lastLoadTimestamp={lastLoadTimestamp} />s ago</p>)}++ {stat.name === 'Scroll depth' &&+ data.meta.metric_warnings?.scroll_depth?.code ===+ 'no_imported_scroll_depth' && (+ <p className="font-normal text-xs whitespace-nowrap">+ * Does not include imported data+ </p>+ )}</div>)}@@ -115,6 +123,7 @@ export default function TopStats({ data, onMetricUpdate, tooltipBoundary }) {{statExtraName && (<span className="hidden sm:inline-block ml-1">{statExtraName}</span>)}+ {stat.warning_code && <span className="inline-block ml-1">*</span>}</div>)}diff --git a/assets/js/dashboard/stats/modals/breakdown-modal.tsx b/assets/js/dashboard/stats/modals/breakdown-modal.tsxindex 35b50644f4..946d14db52 100644--- a/assets/js/dashboard/stats/modals/breakdown-modal.tsx+++ b/assets/js/dashboard/stats/modals/breakdown-modal.tsx@@ -152,6 +152,7 @@ export default function BreakdownModal<TListItem extends { name: string }>({key: m.key,width: m.width,align: 'right',+ metricWarning: getMetricWarning(m, meta),renderValue: (item) => m.renderValue(item, meta),onSort: m.sortable ? () => toggleSortByMetric(m) : undefined,sortDirection: orderByDictionary[m.key]@@ -233,3 +234,15 @@ const ExternalLinkIcon = ({ url }: { url?: string }) =></svg></a>) : null++const getMetricWarning = (metric: Metric, meta: BreakdownResultMeta | null) => {+ const warnings = meta?.metric_warnings++ if (warnings) {+ const code = warnings[metric.key]?.code++ if (code == 'no_imported_scroll_depth') {+ return 'Does not include imported data'+ }+ }+}diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.exindex 749fbeaa2c..cf806edd5e 100644--- a/lib/plausible/data_migration/site_imports.ex+++ b/lib/plausible/data_migration/site_imports.ex@@ -9,20 +9,51 @@ defmodule Plausible.DataMigration.SiteImports doimport Ecto.Query- alias Plausible.ClickhouseRepo- alias Plausible.Imported- alias Plausible.Imported.SiteImport- alias Plausible.Repo- alias Plausible.Site+ alias Plausible.{Repo, ClickhouseRepo, Site}- require Plausible.Imported.SiteImport+ defmodule SiteImportSnapshot do+ @moduledoc """+ A snapshot of the Plausible.Imported.SiteImport schema from April 2024.+ """++ use Ecto.Schema++ schema "site_imports" do+ field :start_date, :date+ field :end_date, :date+ field :label, :string+ field :source, Ecto.Enum, values: [:universal_analytics, :google_analytics_4, :csv, :noop]+ field :status, Ecto.Enum, values: [:pending, :importing, :completed, :failed]+ field :legacy, :boolean, default: false++ belongs_to :site, Plausible.Site+ belongs_to :imported_by, Plausible.Auth.User++ timestamps()+ end+ end++ @imported_tables_april_2024 [+ "imported_visitors",+ "imported_sources",+ "imported_pages",+ "imported_entry_pages",+ "imported_exit_pages",+ "imported_custom_events",+ "imported_locations",+ "imported_devices",+ "imported_browsers",+ "imported_operating_systems"+ ]++ def imported_tables_april_2024(), do: @imported_tables_april_2024def run(opts \\ []) dodry_run? = Keyword.get(opts, :dry_run?, true)site_import_query =- from(i in Imported.SiteImport,- where: i.site_id == parent_as(:site).id and i.status == ^SiteImport.completed(),+ from(i in SiteImportSnapshot,+ where: i.site_id == parent_as(:site).id and i.status == ^:completed,select: 1)@@ -37,7 +68,7 @@ defmodule Plausible.DataMigration.SiteImports do|> Repo.all(log: false)site_imports =- from(i in Imported.SiteImport, where: i.status == ^SiteImport.completed())+ from(i in SiteImportSnapshot, where: i.status == ^:completed)|> Repo.all(log: false)legacy_site_imports = backfill_legacy_site_imports(sites_with_only_legacy_import, dry_run?)@@ -64,7 +95,7 @@ defmodule Plausible.DataMigration.SiteImports do|> Map.put(:site_id, site.id)|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])- %Imported.SiteImport{}+ %SiteImportSnapshot{}|> Ecto.Changeset.change(params)|> insert!(dry_run?)end@@ -146,11 +177,11 @@ defmodule Plausible.DataMigration.SiteImports do# Exposed for testing purposes@doc falsedef imported_stats_end_date(site_id, import_ids) do- [first_schema | schemas] = Imported.schemas()+ [first_table | tables] = @imported_tables_april_2024query =- Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->- from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))+ Enum.reduce(tables, max_date_query(first_table, site_id, import_ids), fn table, query ->+ from(s in subquery(union_all(query, ^max_date_query(table, site_id, import_ids))))end)dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)@@ -211,8 +242,8 @@ defmodule Plausible.DataMigration.SiteImports doentityend- defp max_date_query(schema, site_id, import_ids) do- from(q in schema,+ defp max_date_query(table, site_id, import_ids) do+ from(q in table,where: q.site_id == ^site_id,where: q.import_id in ^import_ids,select: %{max_date: fragment("max(?)", q.date)}@@ -222,12 +253,12 @@ defmodule Plausible.DataMigration.SiteImports dodefp from_legacy(%Site.ImportedData{} = data) dostatus =case data.status do- "ok" -> SiteImport.completed()- "error" -> SiteImport.failed()- _ -> SiteImport.importing()+ "ok" -> :completed+ "error" -> :failed+ _ -> :importingend- %SiteImport{+ %SiteImportSnapshot{id: 0,legacy: true,start_date: data.start_date,diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exindex 2f4674030b..d2b49f4467 100644--- a/lib/plausible/imported.ex+++ b/lib/plausible/imported.ex@@ -9,10 +9,9 @@ defmodule Plausible.Imported doimport Ecto.Query- alias Plausible.Imported+ alias Plausible.{Site, Repo, Imported}alias Plausible.Imported.SiteImport- alias Plausible.Repo- alias Plausible.Site+ alias Plausible.Stats.Queryrequire Plausible.Imported.SiteImport@@ -97,6 +96,17 @@ defmodule Plausible.Imported doendend+ @spec completed_imports_in_query_range(Site.t(), Query.t()) :: [SiteImport.t()]+ def completed_imports_in_query_range(%Site{} = site, %Query{} = query) do+ date_range = Query.date_range(query)++ site+ |> get_completed_imports()+ |> Enum.filter(fn site_import ->+ site_import.start_date <= date_range.last && site_import.end_date >= date_range.first+ end)+ end+@spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nildef get_import(site, import_id) doRepo.get_by(SiteImport, id: import_id, site_id: site.id)diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exindex 277a9dd8da..6a62d01c79 100644--- a/lib/plausible/imported/csv_importer.ex+++ b/lib/plausible/imported/csv_importer.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.CSVImporter do"""use Plausible.Imported.Importer+ import Ecto.Query, only: [from: 2]@impl truedef name(), do: :csv@@ -54,6 +55,26 @@ defmodule Plausible.Imported.CSVImporter do{:error, Exception.message(e)}end+ def on_success(site_import, _extra_data) do+ has_scroll_depth? =+ Plausible.ClickhouseRepo.exists?(+ from(i in "imported_pages",+ where: i.site_id == ^site_import.site_id,+ where: i.import_id == ^site_import.id,+ where: not is_nil(i.scroll_depth),+ select: 1+ )+ )++ if has_scroll_depth? do+ site_import+ |> Ecto.Changeset.change(%{has_scroll_depth: true})+ |> Plausible.Repo.update!()+ end++ :ok+ end+defp import_s3(ch, site_import, uploads) do%{id: import_id,diff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.exindex effcaae239..48fae49254 100644--- a/lib/plausible/imported/site_import.ex+++ b/lib/plausible/imported/site_import.ex@@ -22,6 +22,7 @@ defmodule Plausible.Imported.SiteImport dofield :source, Ecto.Enum, values: ImportSources.names()field :status, Ecto.Enum, values: @statusesfield :legacy, :boolean, default: false+ field :has_scroll_depth, :boolean, default: falsebelongs_to :site, Sitebelongs_to :imported_by, Userdiff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex bc3bc7393c..7e884c7d68 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -9,6 +9,7 @@ defmodule Plausible.Stats.QueryResult douse Plausiblealias Plausible.Stats.{DateTimeRange, Query, QueryRunner}+ alias Plausible.Importeddefstruct results: [],meta: %{},@@ -89,8 +90,8 @@ defmodule Plausible.Stats.QueryResult doendend- defp add_metric_warnings_meta(meta, %QueryRunner{main_query: query}) do- warnings = metric_warnings(query)+ defp add_metric_warnings_meta(meta, runner) do+ warnings = metric_warnings(meta, runner)if map_size(warnings) > 0 doMap.put(meta, :metric_warnings, warnings)@@ -129,9 +130,9 @@ defmodule Plausible.Stats.QueryResult doendend- defp metric_warnings(query) do+ defp metric_warnings(meta, %QueryRunner{main_query: query} = runner) doEnum.reduce(query.metrics, %{}, fn metric, acc ->- case metric_warning(metric, query) do+ case metric_warning(metric, meta, runner) donil -> acc%{} = warning -> Map.put(acc, metric, warning)end@@ -150,7 +151,8 @@ defmodule Plausible.Stats.QueryResult do"Revenue metrics are null as there are no matching revenue goals."}- defp metric_warning(metric, query) when metric in @revenue_metrics do+ defp metric_warning(metric, _meta, %QueryRunner{main_query: query})+ when metric in @revenue_metrics doif query.revenue_warning do%{code: query.revenue_warning,@@ -162,7 +164,30 @@ defmodule Plausible.Stats.QueryResult doendend- defp metric_warning(_metric, _query), do: nil+ @no_imported_scroll_depth_metric_warning %{+ code: :no_imported_scroll_depth,+ warning: "No imports with scroll depth data were found"+ }++ defp metric_warning(:scroll_depth, %{imports_included: true} = _meta, %QueryRunner{} = runner) do+ %QueryRunner{site: site, main_query: main_query, comparison_query: comparison_query} = runner++ imports_in_main_range = Imported.completed_imports_in_query_range(site, main_query)++ imports_in_comparison_range =+ case comparison_query do+ nil -> []+ %Query{} = query -> Imported.completed_imports_in_query_range(site, query)+ end++ imports_in_range = imports_in_main_range ++ imports_in_comparison_range++ if not Enum.any?(imports_in_range, & &1.has_scroll_depth) do+ @no_imported_scroll_depth_metric_warning+ end+ end++ defp metric_warning(_metric, _meta, _query), do: nildefp to_iso8601(datetime, timezone) dodatetimediff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex e9c2c3e6e2..1456c86c76 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -207,6 +207,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{top_stats: top_stats,+ meta: meta,interval: query.interval,sample_percent: sample_percent,with_imported_switch: with_imported_switch_info(meta),@@ -890,7 +891,7 @@ defmodule PlausibleWeb.Api.StatsController doelsejson(conn, %{results: pages,- meta: Stats.Breakdown.formatted_date_ranges(query),+ meta: Map.merge(meta, Stats.Breakdown.formatted_date_ranges(query)),skip_imported_reason: meta[:imports_skip_reason]})enddiff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exsindex 397a5684c5..ee7cbe3357 100644--- a/test/plausible/data_migration/site_imports_test.exs+++ b/test/plausible/data_migration/site_imports_test.exs@@ -253,10 +253,12 @@ defmodule Plausible.DataMigration.SiteImportsTest do~D[2021-02-11]end- test "considers all imported tables" do+ test "considers all imported tables (existing April 2024)" dodate = ~D[2021-01-11]- for {table, idx} <- Enum.with_index(Imported.tables()) do+ all_tables = SiteImports.imported_tables_april_2024()++ for {table, idx} <- Enum.with_index(all_tables) dosite_import = insert(:site_import)end_date = Date.add(date, idx)diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex 8d28464954..d6a6920e56 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -1088,6 +1088,7 @@ defmodule Plausible.Imported.CSVImporterTest dostart_date: start_date,end_date: end_date,source: :csv,+ has_scroll_depth: true,status: :completed} = site_import@@ -1166,11 +1167,15 @@ defmodule Plausible.Imported.CSVImporterTest doimported_site: new_site(owner: user)}- %{exported_files: exported_files} =+ %{exported_files: exported_files, site_import: site_import} =context|> export_archive()|> download_archive()|> unzip_archive()+ |> upload_csvs()+ |> run_import()++ assert %SiteImport{has_scroll_depth: false} = site_importimported_pages_content =exported_filesdiff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exsindex be396e14d2..01669acb79 100644--- a/test/plausible/imported_test.exs+++ b/test/plausible/imported_test.exs@@ -3,6 +3,7 @@ defmodule Plausible.ImportedTest douse Plausiblealias Plausible.Imported+ alias Plausible.Stats.{Query, DateTimeRange}describe "list_all_imports/1" dotest "returns empty when there are no imports" do@@ -116,6 +117,67 @@ defmodule Plausible.ImportedTest doendend+ describe "completed_imports_in_query_range/2" do+ setup do+ site = insert(:site)++ site_import_feb =+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])++ site_import_apr =+ insert(:site_import, site: site, start_date: ~D[2021-04-10], end_date: ~D[2021-04-20])++ {:ok, %{site: site, site_import_feb: site_import_feb, site_import_apr: site_import_apr}}+ end++ test "returns empty list if no imports exist" do+ site = insert(:site)+ tz = "Etc/UTC"++ query = %Query{+ utc_time_range: DateTimeRange.new!(~D[2021-01-01], ~D[2021-12-31], tz),+ timezone: tz+ }++ assert Imported.completed_imports_in_query_range(site, query) == []+ end++ test "returns imports in query range", %{+ site: site,+ site_import_feb: site_import_feb,+ site_import_apr: site_import_apr+ } do+ tz = "Etc/UTC"++ query = %Query{+ utc_time_range: DateTimeRange.new!(~D[2021-01-01], ~D[2021-12-31], tz),+ timezone: tz+ }++ imports_in_range = Imported.completed_imports_in_query_range(site, query)++ assert Enum.find(imports_in_range, &(&1.id == site_import_feb.id))+ assert Enum.find(imports_in_range, &(&1.id == site_import_apr.id))+ end++ test "returns imports in a non-utc timezone query range", %{+ site: site,+ site_import_feb: site_import_feb+ } do+ datetime_from = DateTime.new!(~D[2021-03-01], ~T[03:00:00], "Etc/UTC")+ datetime_to = DateTime.new!(~D[2021-04-10], ~T[03:00:00], "Etc/UTC")++ query = %Query{+ utc_time_range: DateTimeRange.new!(datetime_from, datetime_to),+ timezone: "America/Chicago"+ }++ [site_import] = Imported.completed_imports_in_query_range(site, query)++ assert site_import.id == site_import_feb.id+ end+ end+describe "clamp_dates/3" dotest "crops dates from both ends when overlapping with existing import and native stats" dosite = insert(:site)diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsindex 7c30e50009..f3e09454e9 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -6,6 +6,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do@unsupported_interval_warning "Imported stats are not included because the time dimension (i.e. the interval) is too short."@unsupported_query_warning "Imported stats are not included in the results because query parameters are not supported. For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"+ @no_imported_scroll_depth_metric_warning %{+ "code" => "no_imported_scroll_depth",+ "warning" => "No imports with scroll depth data were found"+ }+setup [:create_user, :create_site, :create_api_key, :use_api_key]describe "aggregation with imported data" do@@ -1362,6 +1367,210 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest doendend+ describe "scroll depth metric warnings" do+ test "returns warning when a site_import without scroll depth is in queried range", %{+ conn: conn,+ site: site+ } do+ site_import =+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),+ build(:engagement,+ user_id: 123,+ pathname: "/",+ timestamp: ~N[2021-02-01 00:01:00],+ scroll_depth: 70+ ),+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),+ build(:imported_pages, page: "/", date: ~D[2021-02-28])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "scroll_depth"],+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/"]]],+ "include" => %{"imports" => true}+ })++ assert %{"results" => results, "meta" => meta} = json_response(conn, 200)++ assert results == [%{"dimensions" => [], "metrics" => [3, 70]}]++ assert meta["imports_included"] == true++ assert meta["metric_warnings"] == %{+ "scroll_depth" => @no_imported_scroll_depth_metric_warning+ }+ end++ test "returns warning when import without scroll depth in comparison range", %{+ conn: conn,+ site: site+ } do+ site_import =+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),+ build(:engagement,+ user_id: 123,+ pathname: "/",+ timestamp: ~N[2021-02-01 00:01:00],+ scroll_depth: 70+ ),+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),+ build(:imported_pages, page: "/", date: ~D[2021-02-28])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "scroll_depth"],+ "date_range" => ["2022-01-01", "2022-12-31"],+ "filters" => [["is", "event:page", ["/"]]],+ "include" => %{"imports" => true, "comparisons" => %{"mode" => "previous_period"}}+ })++ assert %{"results" => results, "meta" => meta} = json_response(conn, 200)++ assert results == [+ %{+ "dimensions" => [],+ "metrics" => [0, nil],+ "comparison" => %{+ "change" => [-100, nil],+ "dimensions" => [],+ "metrics" => [3, 70]+ }+ }+ ]++ assert meta["imports_included"] == true++ assert meta["metric_warnings"] == %{+ "scroll_depth" => @no_imported_scroll_depth_metric_warning+ }+ end++ test "does not return warning when imports requested but rejected", %{+ conn: conn,+ site: site+ } do+ site_import =+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),+ build(:engagement,+ user_id: 123,+ pathname: "/",+ timestamp: ~N[2021-02-01 00:01:00],+ scroll_depth: 70+ ),+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),+ build(:imported_pages, page: "/", date: ~D[2021-02-28])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "scroll_depth"],+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/"]], ["is_not", "visit:browser", ["abc"]]],+ "include" => %{"imports" => true}+ })++ assert %{"results" => results, "meta" => meta} = json_response(conn, 200)++ assert results == [%{"dimensions" => [], "metrics" => [1, 70]}]++ refute meta["imports_included"]+ refute meta["metric_warnings"]["scroll_depth"]+ end++ test "does not return warning when imports without scroll depth exist but outside the queried range",+ %{+ conn: conn,+ site: site+ } do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2021-02-01],+ end_date: ~D[2021-02-28]+ )++ populate_stats(site, site_import.id, [+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),+ build(:imported_pages, page: "/", date: ~D[2021-02-28])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "scroll_depth"],+ "date_range" => ["2022-01-01", "2022-01-31"],+ "dimensions" => ["event:page"],+ "include" => %{"imports" => true}+ })++ assert %{"results" => [], "meta" => meta} = json_response(conn, 200)++ refute meta["imports_included"]+ refute meta["metric_warnings"]["scroll_depth"]+ end++ test "does not return warning when imported scroll depth exists", %{+ conn: conn,+ site: site+ } do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2021-02-01],+ end_date: ~D[2021-02-28],+ has_scroll_depth: true+ )++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),+ build(:engagement,+ user_id: 123,+ pathname: "/",+ timestamp: ~N[2021-02-01 00:01:00],+ scroll_depth: 70+ ),+ build(:imported_pages,+ page: "/",+ date: ~D[2021-02-01],+ scroll_depth: 80,+ pageleave_visitors: 1+ ),+ build(:imported_pages, page: "/", date: ~D[2021-02-28])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "scroll_depth"],+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/"]]],+ "include" => %{"imports" => true}+ })++ assert %{"results" => results, "meta" => meta} = json_response(conn, 200)++ assert results == [%{"dimensions" => [], "metrics" => [3, 75]}]++ assert meta["imports_included"]+ refute meta["metric_warnings"]["scroll_depth"]+ end+ end+describe "page scroll goals" dosetup :create_site_importdiff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 87cc63ef16..05b9d9b21a 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -1137,6 +1137,18 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end+ test "returns scroll depth warning code", %{conn: conn, site: site} do+ Plausible.Sites.set_scroll_depth_visible_at(site)++ conn =+ get(conn, "/api/stats/#{site.domain}/pages?period=day&detailed=true&with_imported=true")++ response = json_response(conn, 200)++ assert response["meta"]["metric_warnings"]["scroll_depth"]["code"] ==+ "no_imported_scroll_depth"+ end+test "returns imported pages with a pageview goal filter", %{conn: conn, site: site} doinsert(:goal, site: site, page_path: "/blog**")diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex a179205395..46c251daf8 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -486,6 +486,26 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do:set_scroll_depth_visible_at]+ test "puts scroll depth warning code", %{conn: conn, site: site} do+ filters = Jason.encode!([[:is, "event:page", ["/"]]])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}&with_imported=true"+ )++ %{"top_stats" => top_stats, "meta" => meta} = json_response(conn, 200)++ assert meta["metric_warnings"]["scroll_depth"]["code"] == "no_imported_scroll_depth"++ assert %{+ "graph_metric" => "scroll_depth",+ "name" => "Scroll depth",+ "value" => nil+ } in top_stats+ end+test "returns divisible metrics as 0 when no stats exist", %{site: site,conn: conn@@ -613,7 +633,10 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doconn: conn,site: site} do- populate_stats(site, [+ site_import =+ insert(:site_import, site: site, start_date: ~D[2021-01-01], has_scroll_depth: true)++ populate_stats(site, site_import.id, [build(:pageview,pathname: "/",user_id: @user_id,@@ -996,7 +1019,8 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doendtest "returns scroll_depth with a page filter with imported data", %{conn: conn, site: site} do- site_import = insert(:site_import, site: site)+ site_import =+ insert(:site_import, site: site, start_date: ~D[2021-01-01], has_scroll_depth: true)populate_stats(site, site_import.id, [build(:pageview, user_id: 123, timestamp: ~N[2021-01-01 00:00:00]),@@ -1023,11 +1047,17 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do"/api/stats/#{site.domain}/top-stats?period=7d&date=2021-01-07&filters=#{filters}&with_imported=true")- res = json_response(conn, 200)+ scroll_depth_stat =+ conn+ |> json_response(200)+ |> Map.get("top_stats")+ |> Enum.find(&(&1["name"] == "Scroll depth"))- assert %{"name" => "Scroll depth", "value" => 55, "graph_metric" => "scroll_depth"} in res[- "top_stats"- ]+ assert scroll_depth_stat == %{+ "name" => "Scroll depth",+ "value" => 55,+ "graph_metric" => "scroll_depth"+ }endtest "contains filter", %{conn: conn, site: site} do