Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Scroll depth dashboard warnings (imported data) (#5051)

RobertJoonas269430281da6
diff --git a/assets/js/dashboard/components/sort-button.tsx b/assets/js/dashboard/components/sort-button.tsx
index 94aedb5ba8..21cd21e80c 100644
--- a/assets/js/dashboard/components/sort-button.tsx
+++ b/assets/js/dashboard/components/sort-button.tsx
@@ -17,11 +17,11 @@ export const SortButton = ({
return (
<button
onClick={toggleSort}
- title={next.hint}
className={classNames('group', 'hover:underline', 'relative')}
>
{children}
<span
+ title={next.hint}
className={classNames(
'absolute',
'rounded inline-block h-4 w-4',
diff --git a/assets/js/dashboard/components/table.tsx b/assets/js/dashboard/components/table.tsx
index 53976ec023..d8486bae6a 100644
--- a/assets/js/dashboard/components/table.tsx
+++ b/assets/js/dashboard/components/table.tsx
@@ -4,6 +4,7 @@ import classNames from 'classnames'
import React, { ReactNode } from 'react'
import { SortDirection } from '../hooks/use-order-by'
import { SortButton } from './sort-button'
+import { Tooltip } from '../util/tooltip'
export type ColumnConfiguraton<T extends Record<string, unknown>> = {
/** Unique column ID, used for sorting purposes and to get the value of the cell using rowItem[key] */
@@ -17,6 +18,8 @@ export type ColumnConfiguraton<T extends Record<string, unknown>> = {
width: string
/** Aligns column content. */
align?: 'left' | 'right'
+ /** A warning to be rendered as a tooltip for the column header */
+ metricWarning?: string
/**
* Function used to transform the value found at item[key] for the cell. Superseded by renderItem if present. @example 1120 => "1.1k"
*/
@@ -100,6 +103,29 @@ export const Table = <T extends Record<string, string | number | ReactNode>>({
columns: ColumnConfiguraton<T>[]
data: T[] | { pages: T[][] }
}) => {
+ const renderColumnLabel = (column: ColumnConfiguraton<T>) => {
+ if (column.metricWarning) {
+ return (
+ <Tooltip
+ info={warningSpan(column.metricWarning)}
+ className="inline-block"
+ >
+ {column.label + ' *'}
+ </Tooltip>
+ )
+ } else {
+ return column.label
+ }
+ }
+
+ const warningSpan = (warning: string) => {
+ return (
+ <span className="text-xs font-normal whitespace-nowrap">
+ {'*' + warning}
+ </span>
+ )
+ }
+
return (
<table className="w-max overflow-x-auto md:w-full table-striped table-fixed">
<thead>
@@ -115,10 +141,10 @@ export const Table = <T extends Record<string, string | number | ReactNode>>({
toggleSort={column.onSort}
sortDirection={column.sortDirection ?? null}
>
- {column.label}
+ {renderColumnLabel(column)}
</SortButton>
) : (
- column.label
+ renderColumnLabel(column)
)}
</TableHeaderCell>
))}
diff --git a/assets/js/dashboard/query.ts b/assets/js/dashboard/query.ts
index 424fac127a..68859c482a 100644
--- a/assets/js/dashboard/query.ts
+++ b/assets/js/dashboard/query.ts
@@ -52,6 +52,7 @@ export type DashboardQuery = typeof queryDefaultValue
export type BreakdownResultMeta = {
date_range_label: string
comparison_date_range_label?: string
+ metric_warnings: Record<string, Record<string, string>> | undefined
}
export function addFilter(
diff --git a/assets/js/dashboard/stats/graph/top-stats.js b/assets/js/dashboard/stats/graph/top-stats.js
index 24b9d02e04..45b9ca3576 100644
--- a/assets/js/dashboard/stats/graph/top-stats.js
+++ b/assets/js/dashboard/stats/graph/top-stats.js
@@ -64,6 +64,14 @@ export default function TopStats({ data, onMetricUpdate, tooltipBoundary }) {
<SecondsSinceLastLoad lastLoadTimestamp={lastLoadTimestamp} />s ago
</p>
)}
+
+ {stat.name === 'Scroll depth' &&
+ data.meta.metric_warnings?.scroll_depth?.code ===
+ 'no_imported_scroll_depth' && (
+ <p className="font-normal text-xs whitespace-nowrap">
+ * Does not include imported data
+ </p>
+ )}
</div>
)
}
@@ -115,6 +123,7 @@ export default function TopStats({ data, onMetricUpdate, tooltipBoundary }) {
{statExtraName && (
<span className="hidden sm:inline-block ml-1">{statExtraName}</span>
)}
+ {stat.warning_code && <span className="inline-block ml-1">*</span>}
</div>
)
}
diff --git a/assets/js/dashboard/stats/modals/breakdown-modal.tsx b/assets/js/dashboard/stats/modals/breakdown-modal.tsx
index 35b50644f4..946d14db52 100644
--- a/assets/js/dashboard/stats/modals/breakdown-modal.tsx
+++ b/assets/js/dashboard/stats/modals/breakdown-modal.tsx
@@ -152,6 +152,7 @@ export default function BreakdownModal<TListItem extends { name: string }>({
key: m.key,
width: m.width,
align: 'right',
+ metricWarning: getMetricWarning(m, meta),
renderValue: (item) => m.renderValue(item, meta),
onSort: m.sortable ? () => toggleSortByMetric(m) : undefined,
sortDirection: orderByDictionary[m.key]
@@ -233,3 +234,15 @@ const ExternalLinkIcon = ({ url }: { url?: string }) =>
</svg>
</a>
) : null
+
+const getMetricWarning = (metric: Metric, meta: BreakdownResultMeta | null) => {
+ const warnings = meta?.metric_warnings
+
+ if (warnings) {
+ const code = warnings[metric.key]?.code
+
+ if (code == 'no_imported_scroll_depth') {
+ return 'Does not include imported data'
+ }
+ }
+}
diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.ex
index 749fbeaa2c..cf806edd5e 100644
--- a/lib/plausible/data_migration/site_imports.ex
+++ b/lib/plausible/data_migration/site_imports.ex
@@ -9,20 +9,51 @@ defmodule Plausible.DataMigration.SiteImports do
import Ecto.Query
- alias Plausible.ClickhouseRepo
- alias Plausible.Imported
- alias Plausible.Imported.SiteImport
- alias Plausible.Repo
- alias Plausible.Site
+ alias Plausible.{Repo, ClickhouseRepo, Site}
- require Plausible.Imported.SiteImport
+ defmodule SiteImportSnapshot do
+ @moduledoc """
+ A snapshot of the Plausible.Imported.SiteImport schema from April 2024.
+ """
+
+ use Ecto.Schema
+
+ schema "site_imports" do
+ field :start_date, :date
+ field :end_date, :date
+ field :label, :string
+ field :source, Ecto.Enum, values: [:universal_analytics, :google_analytics_4, :csv, :noop]
+ field :status, Ecto.Enum, values: [:pending, :importing, :completed, :failed]
+ field :legacy, :boolean, default: false
+
+ belongs_to :site, Plausible.Site
+ belongs_to :imported_by, Plausible.Auth.User
+
+ timestamps()
+ end
+ end
+
+ @imported_tables_april_2024 [
+ "imported_visitors",
+ "imported_sources",
+ "imported_pages",
+ "imported_entry_pages",
+ "imported_exit_pages",
+ "imported_custom_events",
+ "imported_locations",
+ "imported_devices",
+ "imported_browsers",
+ "imported_operating_systems"
+ ]
+
+ def imported_tables_april_2024(), do: @imported_tables_april_2024
def run(opts \\ []) do
dry_run? = Keyword.get(opts, :dry_run?, true)
site_import_query =
- from(i in Imported.SiteImport,
- where: i.site_id == parent_as(:site).id and i.status == ^SiteImport.completed(),
+ from(i in SiteImportSnapshot,
+ where: i.site_id == parent_as(:site).id and i.status == ^:completed,
select: 1
)
@@ -37,7 +68,7 @@ defmodule Plausible.DataMigration.SiteImports do
|> Repo.all(log: false)
site_imports =
- from(i in Imported.SiteImport, where: i.status == ^SiteImport.completed())
+ from(i in SiteImportSnapshot, where: i.status == ^:completed)
|> Repo.all(log: false)
legacy_site_imports = backfill_legacy_site_imports(sites_with_only_legacy_import, dry_run?)
@@ -64,7 +95,7 @@ defmodule Plausible.DataMigration.SiteImports do
|> Map.put(:site_id, site.id)
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
- %Imported.SiteImport{}
+ %SiteImportSnapshot{}
|> Ecto.Changeset.change(params)
|> insert!(dry_run?)
end
@@ -146,11 +177,11 @@ defmodule Plausible.DataMigration.SiteImports do
# Exposed for testing purposes
@doc false
def imported_stats_end_date(site_id, import_ids) do
- [first_schema | schemas] = Imported.schemas()
+ [first_table | tables] = @imported_tables_april_2024
query =
- Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
- from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
+ Enum.reduce(tables, max_date_query(first_table, site_id, import_ids), fn table, query ->
+ from(s in subquery(union_all(query, ^max_date_query(table, site_id, import_ids))))
end)
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
@@ -211,8 +242,8 @@ defmodule Plausible.DataMigration.SiteImports do
entity
end
- defp max_date_query(schema, site_id, import_ids) do
- from(q in schema,
+ defp max_date_query(table, site_id, import_ids) do
+ from(q in table,
where: q.site_id == ^site_id,
where: q.import_id in ^import_ids,
select: %{max_date: fragment("max(?)", q.date)}
@@ -222,12 +253,12 @@ defmodule Plausible.DataMigration.SiteImports do
defp from_legacy(%Site.ImportedData{} = data) do
status =
case data.status do
- "ok" -> SiteImport.completed()
- "error" -> SiteImport.failed()
- _ -> SiteImport.importing()
+ "ok" -> :completed
+ "error" -> :failed
+ _ -> :importing
end
- %SiteImport{
+ %SiteImportSnapshot{
id: 0,
legacy: true,
start_date: data.start_date,
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
index 2f4674030b..d2b49f4467 100644
--- a/lib/plausible/imported.ex
+++ b/lib/plausible/imported.ex
@@ -9,10 +9,9 @@ defmodule Plausible.Imported do
import Ecto.Query
- alias Plausible.Imported
+ alias Plausible.{Site, Repo, Imported}
alias Plausible.Imported.SiteImport
- alias Plausible.Repo
- alias Plausible.Site
+ alias Plausible.Stats.Query
require Plausible.Imported.SiteImport
@@ -97,6 +96,17 @@ defmodule Plausible.Imported do
end
end
+ @spec completed_imports_in_query_range(Site.t(), Query.t()) :: [SiteImport.t()]
+ def completed_imports_in_query_range(%Site{} = site, %Query{} = query) do
+ date_range = Query.date_range(query)
+
+ site
+ |> get_completed_imports()
+ |> Enum.filter(fn site_import ->
+ site_import.start_date <= date_range.last && site_import.end_date >= date_range.first
+ end)
+ end
+
@spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil
def get_import(site, import_id) do
Repo.get_by(SiteImport, id: import_id, site_id: site.id)
diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex
index 277a9dd8da..6a62d01c79 100644
--- a/lib/plausible/imported/csv_importer.ex
+++ b/lib/plausible/imported/csv_importer.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.CSVImporter do
"""
use Plausible.Imported.Importer
+ import Ecto.Query, only: [from: 2]
@impl true
def name(), do: :csv
@@ -54,6 +55,26 @@ defmodule Plausible.Imported.CSVImporter do
{:error, Exception.message(e)}
end
+ def on_success(site_import, _extra_data) do
+ has_scroll_depth? =
+ Plausible.ClickhouseRepo.exists?(
+ from(i in "imported_pages",
+ where: i.site_id == ^site_import.site_id,
+ where: i.import_id == ^site_import.id,
+ where: not is_nil(i.scroll_depth),
+ select: 1
+ )
+ )
+
+ if has_scroll_depth? do
+ site_import
+ |> Ecto.Changeset.change(%{has_scroll_depth: true})
+ |> Plausible.Repo.update!()
+ end
+
+ :ok
+ end
+
defp import_s3(ch, site_import, uploads) do
%{
id: import_id,
diff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.ex
index effcaae239..48fae49254 100644
--- a/lib/plausible/imported/site_import.ex
+++ b/lib/plausible/imported/site_import.ex
@@ -22,6 +22,7 @@ defmodule Plausible.Imported.SiteImport do
field :source, Ecto.Enum, values: ImportSources.names()
field :status, Ecto.Enum, values: @statuses
field :legacy, :boolean, default: false
+ field :has_scroll_depth, :boolean, default: false
belongs_to :site, Site
belongs_to :imported_by, User
diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex
index bc3bc7393c..7e884c7d68 100644
--- a/lib/plausible/stats/query_result.ex
+++ b/lib/plausible/stats/query_result.ex
@@ -9,6 +9,7 @@ defmodule Plausible.Stats.QueryResult do
use Plausible
alias Plausible.Stats.{DateTimeRange, Query, QueryRunner}
+ alias Plausible.Imported
defstruct results: [],
meta: %{},
@@ -89,8 +90,8 @@ defmodule Plausible.Stats.QueryResult do
end
end
- defp add_metric_warnings_meta(meta, %QueryRunner{main_query: query}) do
- warnings = metric_warnings(query)
+ defp add_metric_warnings_meta(meta, runner) do
+ warnings = metric_warnings(meta, runner)
if map_size(warnings) > 0 do
Map.put(meta, :metric_warnings, warnings)
@@ -129,9 +130,9 @@ defmodule Plausible.Stats.QueryResult do
end
end
- defp metric_warnings(query) do
+ defp metric_warnings(meta, %QueryRunner{main_query: query} = runner) do
Enum.reduce(query.metrics, %{}, fn metric, acc ->
- case metric_warning(metric, query) do
+ case metric_warning(metric, meta, runner) do
nil -> acc
%{} = warning -> Map.put(acc, metric, warning)
end
@@ -150,7 +151,8 @@ defmodule Plausible.Stats.QueryResult do
"Revenue metrics are null as there are no matching revenue goals."
}
- defp metric_warning(metric, query) when metric in @revenue_metrics do
+ defp metric_warning(metric, _meta, %QueryRunner{main_query: query})
+ when metric in @revenue_metrics do
if query.revenue_warning do
%{
code: query.revenue_warning,
@@ -162,7 +164,30 @@ defmodule Plausible.Stats.QueryResult do
end
end
- defp metric_warning(_metric, _query), do: nil
+ @no_imported_scroll_depth_metric_warning %{
+ code: :no_imported_scroll_depth,
+ warning: "No imports with scroll depth data were found"
+ }
+
+ defp metric_warning(:scroll_depth, %{imports_included: true} = _meta, %QueryRunner{} = runner) do
+ %QueryRunner{site: site, main_query: main_query, comparison_query: comparison_query} = runner
+
+ imports_in_main_range = Imported.completed_imports_in_query_range(site, main_query)
+
+ imports_in_comparison_range =
+ case comparison_query do
+ nil -> []
+ %Query{} = query -> Imported.completed_imports_in_query_range(site, query)
+ end
+
+ imports_in_range = imports_in_main_range ++ imports_in_comparison_range
+
+ if not Enum.any?(imports_in_range, & &1.has_scroll_depth) do
+ @no_imported_scroll_depth_metric_warning
+ end
+ end
+
+ defp metric_warning(_metric, _meta, _query), do: nil
defp to_iso8601(datetime, timezone) do
datetime
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index e9c2c3e6e2..1456c86c76 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -207,6 +207,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
top_stats: top_stats,
+ meta: meta,
interval: query.interval,
sample_percent: sample_percent,
with_imported_switch: with_imported_switch_info(meta),
@@ -890,7 +891,7 @@ defmodule PlausibleWeb.Api.StatsController do
else
json(conn, %{
results: pages,
- meta: Stats.Breakdown.formatted_date_ranges(query),
+ meta: Map.merge(meta, Stats.Breakdown.formatted_date_ranges(query)),
skip_imported_reason: meta[:imports_skip_reason]
})
end
diff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exs
index 397a5684c5..ee7cbe3357 100644
--- a/test/plausible/data_migration/site_imports_test.exs
+++ b/test/plausible/data_migration/site_imports_test.exs
@@ -253,10 +253,12 @@ defmodule Plausible.DataMigration.SiteImportsTest do
~D[2021-02-11]
end
- test "considers all imported tables" do
+ test "considers all imported tables (existing April 2024)" do
date = ~D[2021-01-11]
- for {table, idx} <- Enum.with_index(Imported.tables()) do
+ all_tables = SiteImports.imported_tables_april_2024()
+
+ for {table, idx} <- Enum.with_index(all_tables) do
site_import = insert(:site_import)
end_date = Date.add(date, idx)
diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
index 8d28464954..d6a6920e56 100644
--- a/test/plausible/imported/csv_importer_test.exs
+++ b/test/plausible/imported/csv_importer_test.exs
@@ -1088,6 +1088,7 @@ defmodule Plausible.Imported.CSVImporterTest do
start_date: start_date,
end_date: end_date,
source: :csv,
+ has_scroll_depth: true,
status: :completed
} = site_import
@@ -1166,11 +1167,15 @@ defmodule Plausible.Imported.CSVImporterTest do
imported_site: new_site(owner: user)
}
- %{exported_files: exported_files} =
+ %{exported_files: exported_files, site_import: site_import} =
context
|> export_archive()
|> download_archive()
|> unzip_archive()
+ |> upload_csvs()
+ |> run_import()
+
+ assert %SiteImport{has_scroll_depth: false} = site_import
imported_pages_content =
exported_files
diff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exs
index be396e14d2..01669acb79 100644
--- a/test/plausible/imported_test.exs
+++ b/test/plausible/imported_test.exs
@@ -3,6 +3,7 @@ defmodule Plausible.ImportedTest do
use Plausible
alias Plausible.Imported
+ alias Plausible.Stats.{Query, DateTimeRange}
describe "list_all_imports/1" do
test "returns empty when there are no imports" do
@@ -116,6 +117,67 @@ defmodule Plausible.ImportedTest do
end
end
+ describe "completed_imports_in_query_range/2" do
+ setup do
+ site = insert(:site)
+
+ site_import_feb =
+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])
+
+ site_import_apr =
+ insert(:site_import, site: site, start_date: ~D[2021-04-10], end_date: ~D[2021-04-20])
+
+ {:ok, %{site: site, site_import_feb: site_import_feb, site_import_apr: site_import_apr}}
+ end
+
+ test "returns empty list if no imports exist" do
+ site = insert(:site)
+ tz = "Etc/UTC"
+
+ query = %Query{
+ utc_time_range: DateTimeRange.new!(~D[2021-01-01], ~D[2021-12-31], tz),
+ timezone: tz
+ }
+
+ assert Imported.completed_imports_in_query_range(site, query) == []
+ end
+
+ test "returns imports in query range", %{
+ site: site,
+ site_import_feb: site_import_feb,
+ site_import_apr: site_import_apr
+ } do
+ tz = "Etc/UTC"
+
+ query = %Query{
+ utc_time_range: DateTimeRange.new!(~D[2021-01-01], ~D[2021-12-31], tz),
+ timezone: tz
+ }
+
+ imports_in_range = Imported.completed_imports_in_query_range(site, query)
+
+ assert Enum.find(imports_in_range, &(&1.id == site_import_feb.id))
+ assert Enum.find(imports_in_range, &(&1.id == site_import_apr.id))
+ end
+
+ test "returns imports in a non-utc timezone query range", %{
+ site: site,
+ site_import_feb: site_import_feb
+ } do
+ datetime_from = DateTime.new!(~D[2021-03-01], ~T[03:00:00], "Etc/UTC")
+ datetime_to = DateTime.new!(~D[2021-04-10], ~T[03:00:00], "Etc/UTC")
+
+ query = %Query{
+ utc_time_range: DateTimeRange.new!(datetime_from, datetime_to),
+ timezone: "America/Chicago"
+ }
+
+ [site_import] = Imported.completed_imports_in_query_range(site, query)
+
+ assert site_import.id == site_import_feb.id
+ end
+ end
+
describe "clamp_dates/3" do
test "crops dates from both ends when overlapping with existing import and native stats" do
site = insert(:site)
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
index 7c30e50009..f3e09454e9 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
@@ -6,6 +6,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
@unsupported_interval_warning "Imported stats are not included because the time dimension (i.e. the interval) is too short."
@unsupported_query_warning "Imported stats are not included in the results because query parameters are not supported. For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"
+ @no_imported_scroll_depth_metric_warning %{
+ "code" => "no_imported_scroll_depth",
+ "warning" => "No imports with scroll depth data were found"
+ }
+
setup [:create_user, :create_site, :create_api_key, :use_api_key]
describe "aggregation with imported data" do
@@ -1362,6 +1367,210 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
end
end
+ describe "scroll depth metric warnings" do
+ test "returns warning when a site_import without scroll depth is in queried range", %{
+ conn: conn,
+ site: site
+ } do
+ site_import =
+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),
+ build(:engagement,
+ user_id: 123,
+ pathname: "/",
+ timestamp: ~N[2021-02-01 00:01:00],
+ scroll_depth: 70
+ ),
+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),
+ build(:imported_pages, page: "/", date: ~D[2021-02-28])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "scroll_depth"],
+ "date_range" => "all",
+ "filters" => [["is", "event:page", ["/"]]],
+ "include" => %{"imports" => true}
+ })
+
+ assert %{"results" => results, "meta" => meta} = json_response(conn, 200)
+
+ assert results == [%{"dimensions" => [], "metrics" => [3, 70]}]
+
+ assert meta["imports_included"] == true
+
+ assert meta["metric_warnings"] == %{
+ "scroll_depth" => @no_imported_scroll_depth_metric_warning
+ }
+ end
+
+ test "returns warning when import without scroll depth in comparison range", %{
+ conn: conn,
+ site: site
+ } do
+ site_import =
+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),
+ build(:engagement,
+ user_id: 123,
+ pathname: "/",
+ timestamp: ~N[2021-02-01 00:01:00],
+ scroll_depth: 70
+ ),
+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),
+ build(:imported_pages, page: "/", date: ~D[2021-02-28])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "scroll_depth"],
+ "date_range" => ["2022-01-01", "2022-12-31"],
+ "filters" => [["is", "event:page", ["/"]]],
+ "include" => %{"imports" => true, "comparisons" => %{"mode" => "previous_period"}}
+ })
+
+ assert %{"results" => results, "meta" => meta} = json_response(conn, 200)
+
+ assert results == [
+ %{
+ "dimensions" => [],
+ "metrics" => [0, nil],
+ "comparison" => %{
+ "change" => [-100, nil],
+ "dimensions" => [],
+ "metrics" => [3, 70]
+ }
+ }
+ ]
+
+ assert meta["imports_included"] == true
+
+ assert meta["metric_warnings"] == %{
+ "scroll_depth" => @no_imported_scroll_depth_metric_warning
+ }
+ end
+
+ test "does not return warning when imports requested but rejected", %{
+ conn: conn,
+ site: site
+ } do
+ site_import =
+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),
+ build(:engagement,
+ user_id: 123,
+ pathname: "/",
+ timestamp: ~N[2021-02-01 00:01:00],
+ scroll_depth: 70
+ ),
+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),
+ build(:imported_pages, page: "/", date: ~D[2021-02-28])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "scroll_depth"],
+ "date_range" => "all",
+ "filters" => [["is", "event:page", ["/"]], ["is_not", "visit:browser", ["abc"]]],
+ "include" => %{"imports" => true}
+ })
+
+ assert %{"results" => results, "meta" => meta} = json_response(conn, 200)
+
+ assert results == [%{"dimensions" => [], "metrics" => [1, 70]}]
+
+ refute meta["imports_included"]
+ refute meta["metric_warnings"]["scroll_depth"]
+ end
+
+ test "does not return warning when imports without scroll depth exist but outside the queried range",
+ %{
+ conn: conn,
+ site: site
+ } do
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2021-02-01],
+ end_date: ~D[2021-02-28]
+ )
+
+ populate_stats(site, site_import.id, [
+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),
+ build(:imported_pages, page: "/", date: ~D[2021-02-28])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "scroll_depth"],
+ "date_range" => ["2022-01-01", "2022-01-31"],
+ "dimensions" => ["event:page"],
+ "include" => %{"imports" => true}
+ })
+
+ assert %{"results" => [], "meta" => meta} = json_response(conn, 200)
+
+ refute meta["imports_included"]
+ refute meta["metric_warnings"]["scroll_depth"]
+ end
+
+ test "does not return warning when imported scroll depth exists", %{
+ conn: conn,
+ site: site
+ } do
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2021-02-01],
+ end_date: ~D[2021-02-28],
+ has_scroll_depth: true
+ )
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),
+ build(:engagement,
+ user_id: 123,
+ pathname: "/",
+ timestamp: ~N[2021-02-01 00:01:00],
+ scroll_depth: 70
+ ),
+ build(:imported_pages,
+ page: "/",
+ date: ~D[2021-02-01],
+ scroll_depth: 80,
+ pageleave_visitors: 1
+ ),
+ build(:imported_pages, page: "/", date: ~D[2021-02-28])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "scroll_depth"],
+ "date_range" => "all",
+ "filters" => [["is", "event:page", ["/"]]],
+ "include" => %{"imports" => true}
+ })
+
+ assert %{"results" => results, "meta" => meta} = json_response(conn, 200)
+
+ assert results == [%{"dimensions" => [], "metrics" => [3, 75]}]
+
+ assert meta["imports_included"]
+ refute meta["metric_warnings"]["scroll_depth"]
+ end
+ end
+
describe "page scroll goals" do
setup :create_site_import
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index 87cc63ef16..05b9d9b21a 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -1137,6 +1137,18 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
+ test "returns scroll depth warning code", %{conn: conn, site: site} do
+ Plausible.Sites.set_scroll_depth_visible_at(site)
+
+ conn =
+ get(conn, "/api/stats/#{site.domain}/pages?period=day&detailed=true&with_imported=true")
+
+ response = json_response(conn, 200)
+
+ assert response["meta"]["metric_warnings"]["scroll_depth"]["code"] ==
+ "no_imported_scroll_depth"
+ end
+
test "returns imported pages with a pageview goal filter", %{conn: conn, site: site} do
insert(:goal, site: site, page_path: "/blog**")
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index a179205395..46c251daf8 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -486,6 +486,26 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
:set_scroll_depth_visible_at
]
+ test "puts scroll depth warning code", %{conn: conn, site: site} do
+ filters = Jason.encode!([[:is, "event:page", ["/"]]])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}&with_imported=true"
+ )
+
+ %{"top_stats" => top_stats, "meta" => meta} = json_response(conn, 200)
+
+ assert meta["metric_warnings"]["scroll_depth"]["code"] == "no_imported_scroll_depth"
+
+ assert %{
+ "graph_metric" => "scroll_depth",
+ "name" => "Scroll depth",
+ "value" => nil
+ } in top_stats
+ end
+
test "returns divisible metrics as 0 when no stats exist", %{
site: site,
conn: conn
@@ -613,7 +633,10 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
conn: conn,
site: site
} do
- populate_stats(site, [
+ site_import =
+ insert(:site_import, site: site, start_date: ~D[2021-01-01], has_scroll_depth: true)
+
+ populate_stats(site, site_import.id, [
build(:pageview,
pathname: "/",
user_id: @user_id,
@@ -996,7 +1019,8 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
end
test "returns scroll_depth with a page filter with imported data", %{conn: conn, site: site} do
- site_import = insert(:site_import, site: site)
+ site_import =
+ insert(:site_import, site: site, start_date: ~D[2021-01-01], has_scroll_depth: true)
populate_stats(site, site_import.id, [
build(:pageview, user_id: 123, timestamp: ~N[2021-01-01 00:00:00]),
@@ -1023,11 +1047,17 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
"/api/stats/#{site.domain}/top-stats?period=7d&date=2021-01-07&filters=#{filters}&with_imported=true"
)
- res = json_response(conn, 200)
+ scroll_depth_stat =
+ conn
+ |> json_response(200)
+ |> Map.get("top_stats")
+ |> Enum.find(&(&1["name"] == "Scroll depth"))
- assert %{"name" => "Scroll depth", "value" => 55, "graph_metric" => "scroll_depth"} in res[
- "top_stats"
- ]
+ assert scroll_depth_stat == %{
+ "name" => "Scroll depth",
+ "value" => 55,
+ "graph_metric" => "scroll_depth"
+ }
end
test "contains filter", %{conn: conn, site: site} do