Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Improve import stats toggle and `with_imported` flag computation (#3960)

Adrian Gruntkowski23a3699dd77a
diff --git a/assets/js/dashboard/stats/graph/visitor-graph.js b/assets/js/dashboard/stats/graph/visitor-graph.js
index e925649c00..966d31d140 100644
--- a/assets/js/dashboard/stats/graph/visitor-graph.js
+++ b/assets/js/dashboard/stats/graph/visitor-graph.js
@@ -15,6 +15,7 @@ import * as url from '../../util/url'
import classNames from 'classnames';
import { monthsBetweenDates, parseNaiveDate, isBefore } from '../../util/date'
import { isComparisonEnabled } from '../../comparison-input'
+import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
const calculateMaximumY = function(dataset) {
const yAxisValues = dataset
@@ -286,7 +287,7 @@ class LineGraph extends React.Component {
}
importedNotice() {
- if (!this.props.topStatData?.imported_source) return
+ if (!this.props.topStatData?.imports_exist) return
const isBeforeNativeStats = (date) => {
if (!date) return false
@@ -301,18 +302,15 @@ class LineGraph extends React.Component {
const isComparingImportedPeriod = isBeforeNativeStats(this.props.topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
- const source = this.props.topStatData.imported_source
const withImported = this.props.topStatData.with_imported;
- const strike = withImported ? "" : " line-through"
+ const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
- <div tooltip={`Stats ${tip}include data imported from ${source}.`} className="cursor-pointer w-4 h-4">
- <svg className="absolute dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
- <text x="4" y="18" fontSize="24" fill="currentColor" className={"text-gray-700 dark:text-gray-300" + strike}>{source[0].toUpperCase()}</text>
- </svg>
+ <div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
+ <BarsArrowUpIcon className={"absolute " + toggleColor} />
</div>
</Link>
)
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
index 7c6700683e..742886a7ba 100644
--- a/lib/plausible/imported.ex
+++ b/lib/plausible/imported.ex
@@ -51,13 +51,13 @@ defmodule Plausible.Imported do
def load_import_data(site) do
complete_import_ids = list_complete_import_ids(site)
- earliest_import = get_earliest_import(site) || %{}
+ dates = get_imports_date_range(site)
%{
site
| import_data_loaded: true,
- earliest_import_start_date: Map.get(earliest_import, :start_date),
- earliest_import_end_date: Map.get(earliest_import, :end_date),
+ earliest_import_start_date: dates.start_date,
+ latest_import_end_date: dates.end_date,
complete_import_ids: complete_import_ids
}
end
@@ -110,24 +110,35 @@ defmodule Plausible.Imported do
end
end
- @spec get_earliest_import(Site.t()) :: SiteImport.t() | nil
- def get_earliest_import(site) do
- first_import =
+ @spec get_imports_date_range(Site.t()) :: %{
+ start_date: Date.t() | nil,
+ end_date: Date.t() | nil
+ }
+ def get_imports_date_range(site) do
+ dates =
from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
- order_by: i.start_date,
- limit: 1
+ select: %{start_date: min(i.start_date), end_date: max(i.end_date)}
)
|> Repo.one()
- legacy_import =
- if site.imported_data && site.imported_data.status == "ok" do
- site.imported_data
- end
+ dates = dates || %{start_date: nil, end_date: nil}
- [legacy_import, first_import]
- |> Enum.reject(&is_nil/1)
- |> Enum.min_by(& &1.start_date, Date, fn -> nil end)
+ if site.imported_data && site.imported_data.status == "ok" do
+ start_date =
+ [dates.start_date, site.imported_data.start_date]
+ |> Enum.reject(&is_nil/1)
+ |> Enum.min(Date, fn -> nil end)
+
+ end_date =
+ [dates.end_date, site.imported_data.end_date]
+ |> Enum.reject(&is_nil/1)
+ |> Enum.max(Date, fn -> nil end)
+
+ %{start_date: start_date, end_date: end_date}
+ else
+ dates
+ end
end
@spec delete_imports_for_site(Site.t()) :: :ok
diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex
index 5a4de9f01d..6085bb5d08 100644
--- a/lib/plausible/site.ex
+++ b/lib/plausible/site.ex
@@ -59,7 +59,7 @@ defmodule Plausible.Site do
# `site` in `assigns`.
field :import_data_loaded, :boolean, default: false, virtual: true
field :earliest_import_start_date, :date, virtual: true
- field :earliest_import_end_date, :date, virtual: true
+ field :latest_import_end_date, :date, virtual: true
field :complete_import_ids, {:array, :integer}, default: [], virtual: true
timestamps()
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 6af57aca8b..6a0e3bb351 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -261,8 +261,8 @@ defmodule Plausible.Stats.Query do
@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()
def include_imported?(query, site, requested?) do
cond do
- is_nil(site.earliest_import_end_date) -> false
- Date.after?(query.date_range.first, site.earliest_import_end_date) -> false
+ is_nil(site.latest_import_end_date) -> false
+ Date.after?(query.date_range.first, site.latest_import_end_date) -> false
Enum.any?(query.filters) -> false
query.period == "realtime" -> false
true -> requested?
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 6b9798d101..c6900f82d6 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -4,7 +4,6 @@ defmodule PlausibleWeb.Api.StatsController do
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
- alias Plausible.Imported.SiteImport
alias Plausible.Stats
alias Plausible.Stats.{Query, Comparisons}
alias PlausibleWeb.Api.Helpers, as: H
@@ -68,8 +67,8 @@ defmodule PlausibleWeb.Api.StatsController do
* `with_imported` - boolean indicating whether the Google Analytics data
was queried or not.
- * `imported_source` - the source of the imported data, when applicable.
- Currently only Google Analytics is supported.
+ * `imports_exist` - boolean indicating whether there are any completed
+ imports for a given site or not.
* `full_intervals` - map of dates indicating whether the interval has been
cut off by the requested date range or not. For example, if looking at a
@@ -87,7 +86,7 @@ defmodule PlausibleWeb.Api.StatsController do
"2021-11-01" => true,
"2021-12-01" => false
},
- "imported_source" => nil,
+ "imports_exist" => false,
"interval" => "month",
"labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],
"plot" => [0, 0, 0, 0],
@@ -129,8 +128,6 @@ defmodule PlausibleWeb.Api.StatsController do
present_index = present_index_for(site, query, labels)
full_intervals = build_full_intervals(query, labels)
- site_import = Plausible.Imported.get_earliest_import(site)
-
json(conn, %{
plot: plot_timeseries(timeseries_result, metric),
labels: labels,
@@ -139,7 +136,7 @@ defmodule PlausibleWeb.Api.StatsController do
present_index: present_index,
interval: query.interval,
with_imported: with_imported?(query, comparison_query),
- imported_source: site_import && SiteImport.label(site_import),
+ imports_exist: site.complete_import_ids != [],
full_intervals: full_intervals
})
else
@@ -214,14 +211,12 @@ defmodule PlausibleWeb.Api.StatsController do
{top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query)
- site_import = Plausible.Imported.get_earliest_import(site)
-
json(conn, %{
top_stats: top_stats,
interval: query.interval,
sample_percent: sample_percent,
with_imported: with_imported?(query, comparison_query),
- imported_source: site_import && SiteImport.label(site_import),
+ imports_exist: site.complete_import_ids != [],
comparing_from: comparison_query && comparison_query.date_range.first,
comparing_to: comparison_query && comparison_query.date_range.last,
from: query.date_range.first,
diff --git a/lib/plausible_web/templates/email/google_analytics_import.html.heex b/lib/plausible_web/templates/email/google_analytics_import.html.heex
index a9879165c8..c0369fd65b 100644
--- a/lib/plausible_web/templates/email/google_analytics_import.html.heex
+++ b/lib/plausible_web/templates/email/google_analytics_import.html.heex
@@ -6,22 +6,8 @@
<%= link("Click here", to: @link) %> to view your dashboard.
<% else %>
Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that!
- <br /> <br />
- This may be due to the user metric setting in your Google Analytics account. We found that changing the user metric setting is necessary to get the correct data out of the Google API.
- <br /> <br />
- Please log into your Google Analytics account and go to Admin -> Property Settings -> User Analysis. Then make sure that "Enable User Metric in Reporting" is set to OFF. Then try importing again. You can switch the setting back after the import.
- <br /> <br /> If you've already changed that setting, then it is something else: <br />
-
- <ul>
- <li>
- Do you have any stats in the Google Analytics property you tried to import? We only import the stats from your first Google Analytics visitor until your first Plausible Analytics visitor. The import fails if there's no returned data from Google from before you counted your first Plausible visitor.
- </li>
- <li>
- Did you set strict data retention limits in your Google Analytics account? You can check the data retention limit you have set for your property in your Google Analytics admin settings by visiting the "Tracking Info" section and clicking on "Data Retention".
- </li>
- </ul>
- <br />
- If all of the above is fine, please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need.
+ <br /><br />
+ Please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need.
<%= if full_build?() do %>
<br /> <br />
Please reply to this email to let us know if you're still experiencing issues with the import.
diff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exs
index e5a6d0ba11..df81f5a416 100644
--- a/test/plausible/imported_test.exs
+++ b/test/plausible/imported_test.exs
@@ -60,6 +60,65 @@ defmodule Plausible.ImportedTest do
end
end
+ describe "get_imports_date_range/1" do
+ test "returns empty when there are no imports" do
+ site = insert(:site)
+
+ assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)
+ end
+
+ test "returns empty when only incomplete or failed imports are present" do
+ site =
+ insert(:site)
+ |> Plausible.Site.start_import(
+ ~D[2020-04-01],
+ ~D[2022-06-22],
+ "Google Analytics",
+ "error"
+ )
+ |> Repo.update!()
+
+ _import1 = insert(:site_import, site: site, status: :pending)
+ _import2 = insert(:site_import, site: site, status: :importing)
+ _import3 = insert(:site_import, site: site, status: :failed)
+ _rogue_import = insert(:site_import, site: build(:site), status: :completed)
+
+ assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)
+ end
+
+ test "returns start and end dates considering all imports" do
+ site =
+ insert(:site)
+ |> Plausible.Site.start_import(
+ ~D[2020-04-01],
+ ~D[2022-06-22],
+ "Google Analytics",
+ "ok"
+ )
+ |> Repo.update!()
+
+ _import1 =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2020-04-02],
+ end_date: ~D[2022-06-22],
+ status: :completed,
+ legacy: true
+ )
+
+ _import2 =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2022-06-22],
+ end_date: ~D[2024-01-08],
+ status: :completed
+ )
+
+ assert %{start_date: ~D[2020-04-01], end_date: ~D[2024-01-08]} =
+ Imported.get_imports_date_range(site)
+ end
+ end
+
describe "check_dates/3" do
test "crops dates from both ends when overlapping with existing import and native stats" do
site = insert(:site)
diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
index e7efc4f9a9..e2f5c93c33 100644
--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
@@ -93,7 +93,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
- assert %{"plot" => plot, "imported_source" => "Google Analytics"} =
+ assert %{"plot" => plot, "imports_exist" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@@ -142,7 +142,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"
)
- assert %{"plot" => plot, "imported_source" => "Google Analytics"} =
+ assert %{"plot" => plot, "imports_exist" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 5
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index bb74bf5315..deeff48857 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -70,7 +70,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true"
)
- assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
+ assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert plot == [2] ++ List.duplicate(0, 23)
@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
- assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
+ assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@@ -158,7 +158,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
- assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
+ assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@@ -1027,7 +1027,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => plot,
"comparison_plot" => comparison_plot,
- "imported_source" => "Google Analytics",
+ "imports_exist" => true,
"with_imported" => true
} = json_response(conn, 200)
@@ -1084,7 +1084,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => plot,
"comparison_plot" => comparison_plot,
- "imported_source" => "Google Analytics",
+ "imports_exist" => true,
"with_imported" => false
} = json_response(conn, 200)
@@ -1114,7 +1114,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => this_week_plot,
"comparison_plot" => last_week_plot,
- "imported_source" => "Google Analytics",
+ "imports_exist" => true,
"with_imported" => false
} = json_response(conn, 200)