Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Ga import improvements (#1784)
Uku Taht203f87520b04
diff --git a/assets/js/dashboard/datepicker.js b/assets/js/dashboard/datepicker.jsindex d83f88667a..0f3d6ad19e 100644--- a/assets/js/dashboard/datepicker.js+++ b/assets/js/dashboard/datepicker.js@@ -22,7 +22,7 @@ import {import { navigateToQuery, QueryLink, QueryButton } from "./query";function renderArrow(query, site, period, prevDate, nextDate) {- const insertionDate = parseUTCDate(site.insertedAt);+ const insertionDate = parseUTCDate(site.statsBegin);const disabledLeft = isBefore(parseUTCDate(prevDate),insertionDate,@@ -136,7 +136,7 @@ class DatePicker extends React.Component {date: false,};- const insertionDate = parseUTCDate(this.props.site.insertedAt);+ const insertionDate = parseUTCDate(this.props.site.statsBegin);if (e.key === "ArrowLeft") {const prevDate = formatISO(shiftDays(query.date, -1));@@ -358,7 +358,7 @@ class DatePicker extends React.Component {</div>);} if (this.state.mode === "calendar") {- const insertionDate = new Date(this.props.site.insertedAt);+ const insertionDate = new Date(this.props.site.statsBegin);const dayBeforeCreation = insertionDate - 86400000;return (<div className="h-0">diff --git a/assets/js/dashboard/mount.js b/assets/js/dashboard/mount.jsindex 635d2e5c40..38cc914a3a 100644--- a/assets/js/dashboard/mount.js+++ b/assets/js/dashboard/mount.js@@ -13,7 +13,7 @@ if (container) {domain: container.dataset.domain,offset: container.dataset.offset,hasGoals: container.dataset.hasGoals === 'true',- insertedAt: container.dataset.insertedAt,+ statsBegin: container.dataset.statsBegin,embedded: container.dataset.embedded,background: container.dataset.background,selfhosted: container.dataset.selfhosted === 'true'diff --git a/lib/mix/tasks/call_google.ex b/lib/mix/tasks/call_google.exdeleted file mode 100644index 32fdad2033..0000000000--- a/lib/mix/tasks/call_google.ex+++ /dev/null@@ -1,48 +0,0 @@-defmodule Mix.Tasks.CallGoogle do- use Mix.Task- use Plausible.Repo- require Logger-- @token "ya29.A0ARrdaM-GC-C38oUNmoUlica3v2IGT1eACtvreAFcM4zCYrpQxHdTFfhpmWQfiYAblnKPVRkt_fW-jTTwJ_ScXtkaqq6P_XiBbPIa3g9yJ_F6bP2tEOeHgq07aPlPwHvqXaK8I8gmwtO86VvJLvmhXolHtlXN0g"- @view_id "204615196"- @dimensions ["ga:date"]- @metrics ["ga:users", "ga:pageviews", "ga:bounces", "ga:sessions", "ga:sessionDuration"]-- def run(_) do- Mix.Task.run("app.start")-- report = %{- viewId: @view_id,- dateRanges: [- %{- # The earliest valid date- startDate: "2005-01-01",- endDate: "2019-12-05"- }- ],- dimensions: Enum.map(@dimensions, &%{name: &1, histogramBuckets: []}),- metrics: Enum.map(@metrics, &%{expression: &1}),- hideTotals: true,- hideValueRanges: true,- orderBys: [- %{- fieldName: "ga:date",- sortOrder: "DESCENDING"- }- ],- pageSize: 100_00,- pageToken: ""- }-- Logger.debug(report)-- res =- HTTPoison.post!(- "https://analyticsreporting.googleapis.com/v4/reports:batchGet",- Jason.encode!(%{reportRequests: [report]}),- Authorization: "Bearer #{@token}"- )-- Logger.debug(res.body)- end-enddiff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 942ed1b308..91f9c28ad1 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -6,6 +6,7 @@ defmodule Plausible.Google.Api do@scope URI.encode_www_form("https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly")+ @import_scope URI.encode_www_form("email https://www.googleapis.com/auth/analytics.readonly")@verified_permission_levels ["siteOwner", "siteFullUser", "siteRestrictedUser"]def authorize_url(site_id, redirect_to) do@@ -17,6 +18,15 @@ defmodule Plausible.Google.Api doendend+ def import_authorize_url(site_id, redirect_to) do+ if Application.get_env(:plausible, :environment) == "test" do+ ""+ else+ "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>+ Jason.encode!([site_id, redirect_to])+ end+ end+def fetch_access_token(code) dores =HTTPoison.post!(@@ -120,17 +130,11 @@ defmodule Plausible.Google.Api doendend- def get_analytics_view_ids(site) do- with {:ok, auth} <- refresh_if_needed(site.google_auth) do- do_get_analytics_view_ids(auth)- end- end-- def do_get_analytics_view_ids(auth) do+ def get_analytics_view_ids(token) dores =HTTPoison.get!("https://www.googleapis.com/analytics/v3/management/accounts/~all/webproperties/~all/profiles",- Authorization: "Bearer #{auth.access_token}"+ Authorization: "Bearer #{token}")case res.status_code do@@ -153,11 +157,60 @@ defmodule Plausible.Google.Api doendend- def import_analytics(site, view_id, end_date) do- with {:ok, auth} <- refresh_if_needed(site.google_auth) do- do_import_analytics(site, auth, view_id, end_date)- end- end+ # Each element is: {dataset, dimensions, metrics}+ @request_data [+ {+ "imported_visitors",+ ["ga:date"],+ [+ "ga:users",+ "ga:pageviews",+ "ga:bounces",+ "ga:sessions",+ "ga:sessionDuration"+ ]+ },+ {+ "imported_sources",+ ["ga:date", "ga:source", "ga:medium", "ga:campaign", "ga:adContent", "ga:keyword"],+ ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]+ },+ {+ "imported_pages",+ ["ga:date", "ga:hostname", "ga:pagePath"],+ ["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"]+ },+ {+ "imported_entry_pages",+ ["ga:date", "ga:landingPagePath"],+ ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"]+ },+ {+ "imported_exit_pages",+ ["ga:date", "ga:exitPagePath"],+ ["ga:users", "ga:exits"]+ },+ {+ "imported_locations",+ ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"],+ ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]+ },+ {+ "imported_devices",+ ["ga:date", "ga:deviceCategory"],+ ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]+ },+ {+ "imported_browsers",+ ["ga:date", "ga:browser"],+ ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]+ },+ {+ "imported_operating_systems",+ ["ga:date", "ga:operatingSystem"],+ ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]+ }+ ]@doc """API reference:@@ -165,71 +218,17 @@ defmodule Plausible.Google.Api doDimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer"""- def do_import_analytics(site, auth, view_id, end_date) do+ def import_analytics(site, view_id, start_date, end_date, access_token) dorequest = %{- auth: auth,- profile: view_id,+ access_token: access_token,+ view_id: view_id,+ start_date: start_date,end_date: end_date}- # Each element is: {dataset, dimensions, metrics}- request_data = [- {- "imported_visitors",- ["ga:date"],- [- "ga:users",- "ga:pageviews",- "ga:bounces",- "ga:sessions",- "ga:sessionDuration"- ]- },- {- "imported_sources",- ["ga:date", "ga:source", "ga:medium", "ga:campaign", "ga:adContent", "ga:keyword"],- ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]- },- {- "imported_pages",- ["ga:date", "ga:hostname", "ga:pagePath"],- ["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"]- },- {- "imported_entry_pages",- ["ga:date", "ga:landingPagePath"],- ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"]- },- {- "imported_exit_pages",- ["ga:date", "ga:exitPagePath"],- ["ga:users", "ga:exits"]- },- {- "imported_locations",- ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"],- ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]- },- {- "imported_devices",- ["ga:date", "ga:deviceCategory"],- ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]- },- {- "imported_browsers",- ["ga:date", "ga:browser"],- ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]- },- {- "imported_operating_systems",- ["ga:date", "ga:operatingSystem"],- ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]- }- ]-responses =Enum.map(- request_data,+ @request_data,fn {dataset, dimensions, metrics} ->fetch_analytic_reports(dataset, dimensions, metrics, request)end@@ -272,11 +271,11 @@ defmodule Plausible.Google.Api dodefp fetch_analytic_reports(dataset, dimensions, metrics, request, page_token \\ "") doreport = %{- viewId: request.profile,+ viewId: request.view_id,dateRanges: [%{# The earliest valid date- startDate: "2005-01-01",+ startDate: request.start_date,endDate: request.end_date}],@@ -298,7 +297,7 @@ defmodule Plausible.Google.Api doHTTPoison.post!("https://analyticsreporting.googleapis.com/v4/reports:batchGet",Jason.encode!(%{reportRequests: [report]}),- [Authorization: "Bearer #{request.auth.access_token}"],+ [Authorization: "Bearer #{request.access_token}"],timeout: 30_000,recv_timeout: 30_000)diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.exindex 8dee788c0b..127dffc01c 100644--- a/lib/plausible/site/schema.ex+++ b/lib/plausible/site/schema.ex@@ -2,6 +2,7 @@ defmodule Plausible.Site.ImportedData douse Ecto.Schemaembedded_schema do+ field :start_date, :datefield :end_date, :datefield :source, :stringfield :status, :string@@ -62,10 +63,11 @@ defmodule Plausible.Site dochange(site, has_stats: has_stats_val)end- def start_import(site, imported_source, status \\ "importing") do+ def start_import(site, start_date, end_date, imported_source, status \\ "importing") dochange(site,- imported_data: %Plausible.Site.ImportedData{- end_date: Timex.today(),+ imported_data: %{+ start_date: start_date,+ end_date: end_date,source: imported_source,status: status}diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex 92c783bfc0..25bba9c846 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -55,6 +55,14 @@ defmodule Plausible.Sites doendend+ def stats_begin(site) do+ if site.imported_data do+ site.imported_data.start_date+ else+ site.inserted_at+ end+ end+def create_shared_link(site, name, password \\ nil) dochanges =SharedLink.changeset(diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 545f66ce44..143053aa0a 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -5,6 +5,24 @@ defmodule Plausible.Stats.Clickhouse douse Plausible.Stats.Fragments@no_ref "Direct / None"+ def pageview_start_date_local(site) do+ date =+ ClickhouseRepo.one(+ from e in "events",+ select: fragment("toDate(min(?))", e.timestamp),+ where: e.domain == ^site.domain+ )++ case date do+ # no stats for this domain yet+ ~D[1970-01-01] ->+ Timex.today(site.timezone)++ date ->+ Timex.Timezone.convert(date, site.timezone)+ end+ end+def usage_breakdown(domains) dorange =Date.range(diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.exindex 917e0201d4..d1c22ac5a9 100644--- a/lib/plausible_web/controllers/auth_controller.ex+++ b/lib/plausible_web/controllers/auth_controller.ex@@ -538,24 +538,36 @@ defmodule PlausibleWeb.AuthController dodef google_auth_callback(conn, %{"code" => code, "state" => state}) dores = Plausible.Google.Api.fetch_access_token(code)- id_token = res["id_token"]- [_, body, _] = String.split(id_token, ".")- id = body |> Base.decode64!(padding: false) |> Jason.decode!()-[site_id, redirect_to] = Jason.decode!(state)+ site = Repo.get(Plausible.Site, site_id)++ case redirect_to do+ "import" ->+ redirect(conn,+ to:+ Routes.site_path(conn, :import_from_google_form, site.domain,+ access_token: res["access_token"]+ )+ )- Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{- email: id["email"],- refresh_token: res["refresh_token"],- access_token: res["access_token"],- expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]),- user_id: conn.assigns[:current_user].id,- site_id: site_id- })- |> Repo.insert!()+ _ ->+ id_token = res["id_token"]+ [_, body, _] = String.split(id_token, ".")+ id = body |> Base.decode64!(padding: false) |> Jason.decode!()- site = Repo.get(Plausible.Site, site_id)+ Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{+ email: id["email"],+ refresh_token: res["refresh_token"],+ access_token: res["access_token"],+ expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]),+ user_id: conn.assigns[:current_user].id,+ site_id: site_id+ })+ |> Repo.insert!()- redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings/#{redirect_to}")+ site = Repo.get(Plausible.Site, site_id)++ redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings/#{redirect_to}")+ endendenddiff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex c53cabb489..cb90523255 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -167,28 +167,15 @@ defmodule PlausibleWeb.SiteController doredirect(conn, to: Routes.site_path(conn, :settings_general, website))end- defp can_trigger_import(site) do- no_import = is_nil(site.imported_data) || site.imported_data.status == "error"-- no_import && site.google_auth- end-def settings_general(conn, _params) dosite =conn.assigns[:site]|> Repo.preload([:custom_domain, :google_auth])- google_profiles =- if can_trigger_import(site) do- Plausible.Google.Api.get_analytics_view_ids(site)- end-conn|> assign(:skip_plausible_tracking, true)|> render("settings_general.html",site: site,- google_profiles: google_profiles,- imported_data: site.imported_data,changeset: Plausible.Site.changeset(site, %{}),layout: {PlausibleWeb.LayoutView, "site_settings.html"})@@ -643,48 +630,81 @@ defmodule PlausibleWeb.SiteController do|> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings/general")end- def import_from_google_form(conn, _params) do- site = conn.assigns[:site] |> Repo.preload(:google_auth)+ def import_from_google_form(conn, %{"access_token" => access_token}) do+ site = conn.assigns[:site]- view_ids = Plausible.Google.Api.get_analytics_view_ids(site)+ view_ids = Plausible.Google.Api.get_analytics_view_ids(access_token)conn|> assign(:skip_plausible_tracking, true)|> render("import_from_google.html",+ access_token: access_token,site: site,view_ids: view_ids,layout: {PlausibleWeb.LayoutView, "focus.html"})end- def import_from_google(conn, %{"view_id" => view_id, "end_date" => end_date}) do- site =- conn.assigns[:site]- |> Repo.preload(:google_auth)+ def import_from_google_view_id(conn, %{"view_id" => view_id, "access_token" => access_token}) do+ site = conn.assigns[:site]- cond do- site.imported_data ->- conn- |> put_flash(:error, "Data already imported from: #{site.imported_data.source}")- |> redirect(to: Routes.site_path(conn, :settings_general, site.domain))+ redirect(conn,+ to:+ Routes.site_path(conn, :import_from_google_confirm, site.domain,+ view_id: view_id,+ access_token: access_token+ )+ )+ end- true ->- job =- Plausible.Workers.ImportGoogleAnalytics.new(%{- "site_id" => site.id,- "view_id" => view_id,- "end_date" => end_date- })-- Ecto.Multi.new()- |> Ecto.Multi.update(:update_site, Plausible.Site.start_import(site, "Google Analytics"))- |> Oban.insert(:oban_job, job)- |> Repo.transaction()+ def import_from_google_confirm(conn, %{"access_token" => access_token, "view_id" => view_id}) do+ site = conn.assigns[:site]- conn- |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")- |> redirect(to: Routes.site_path(conn, :settings_general, site.domain))- end+ # TODO: Plausible.Google.Api.get_analytics_start_date(access_token)+ start_date = {:ok, ~D[2019-01-02]}+ end_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site)++ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("import_from_google_confirm.html",+ access_token: access_token,+ site: site,+ # TODO: Also send the name here+ view_id: view_id,+ start_date: start_date,+ end_date: end_date,+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )+ end++ def import_from_google(conn, %{+ "view_id" => view_id,+ "start_date" => start_date,+ "end_date" => end_date,+ "access_token" => access_token+ }) do+ site = conn.assigns[:site]++ job =+ Plausible.Workers.ImportGoogleAnalytics.new(%{+ "site_id" => site.id,+ "view_id" => view_id,+ "start_date" => start_date,+ "end_date" => end_date,+ "access_token" => access_token+ })++ Ecto.Multi.new()+ |> Ecto.Multi.update(+ :update_site,+ Plausible.Site.start_import(site, start_date, end_date, "Google Analytics")+ )+ |> Oban.insert(:oban_job, job)+ |> Repo.transaction()++ conn+ |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")+ |> redirect(to: Routes.site_path(conn, :settings_general, site.domain))enddef forget_imported(conn, _params) dodiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex e925904e6a..0da123cb21 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -22,6 +22,7 @@ defmodule PlausibleWeb.StatsController do|> render("stats.html",site: site,has_goals: Plausible.Sites.has_goals?(site),+ stats_begin: Plausible.Sites.stats_begin(site),title: "Plausible · " <> site.domain,offer_email_report: offer_email_report,demo: demo@@ -177,6 +178,7 @@ defmodule PlausibleWeb.StatsController do|> render("stats.html",site: shared_link.site,has_goals: Plausible.Sites.has_goals?(shared_link.site),+ stats_begin: Plausible.Sites.stats_begin(shared_link.site),title: "Plausible · " <> shared_link.site.domain,offer_email_report: false,demo: false,diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex 1ef1e9b252..39f2b77b92 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -239,6 +239,8 @@ defmodule PlausibleWeb.Router dodelete "/:website/stats", SiteController, :reset_statsget "/:website/import/google-analytics", SiteController, :import_from_google_form+ post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id+ get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirmpost "/:website/settings/google-import", SiteController, :import_from_googledelete "/:website/settings/forget-imported", SiteController, :forget_importeddiff --git a/lib/plausible_web/templates/site/import_from_google.html.eex b/lib/plausible_web/templates/site/import_from_google.html.eexindex 441670302f..04297e9554 100644--- a/lib/plausible_web/templates/site/import_from_google.html.eex+++ b/lib/plausible_web/templates/site/import_from_google.html.eex@@ -1,27 +1,22 @@-<%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>+<%= form_for @conn, Routes.site_path(@conn, :import_from_google_view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %><h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>+ <%= hidden_input(f, :access_token, value: @access_token) %>+<%= case @view_ids do %><% {:ok, view_ids} -> %><div class="mt-6 text-sm text-gray-500">- You can choose the view ID that will be imported to the <%= @site.domain %> dashboard.- </div>-- <div class="my-6">- <%= label f, :view_id, "Google Analytics view id", class: "block text-sm font-medium text-gray-700" %>- <%= select f, :view_id, view_ids, prompt: "(Choose view id)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>+ Choose the view in your Google Analytics account that will be imported to the <%= @site.domain %> dashboard.</div>- <div class="my-6">- <%= label f, :end_date, "Import end date", class: "block text-sm font-medium text-gray-700" %>- <div class="mt-1">- <%= date_input f, :end_date, value: Timex.today() |> Date.to_iso8601(), required: "true", class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %>- </div>+ <div class="mt-3">+ <%= label f, :view_id, "Google Analytics view", class: "block text-sm font-medium text-gray-700" %>+ <%= select f, :view_id, view_ids, prompt: "(Choose view)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %></div><% {:error, error} -> %><p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics view ids.</p><p class="text-red-700 font-medium mt-3"><%= error %></p><% end %>- <%= submit "Import", class: "button" %>+ <%= submit "Continue ->", class: "button mt-6" %><% end %>diff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/site/import_from_google_confirm.html.eexnew file mode 100644index 0000000000..2298054b59--- /dev/null+++ b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex@@ -0,0 +1,38 @@+<%= form_for @conn, Routes.site_path(@conn, :import_from_google, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>+ <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>++ <%= hidden_input(f, :access_token, value: @access_token) %>++ <%= case @start_date do %>+ <% {:ok, start_date} -> %>++ <div class="mt-6 text-sm text-gray-500">+ Choose the view in your Google Analytics account that will be imported to the <%= @site.domain %> dashboard.+ </div>++ <div class="mt-3">+ <%= label f, :view_id, "Google Analytics view", class: "block text-sm font-medium text-gray-700" %>+ <%= text_input f, :view_id, value: @view_id, readonly: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>+ </div>+ <div class="mt-6 text-sm text-gray-500">+ We will import historical data from your first Google Analytics visitor to your first Plausible visitor. If you'd+ like to import data for a different time period, please contact support.+ </div>+ <div class="flex justify-between mt-3">+ <div class="w-36">+ <%= label f, :start_date, "From", class: "block text-sm font-medium text-gray-700" %>+ <%= date_input f, :start_date, value: start_date, readonly: "true", class: "mt-1 block w-full px-3 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>+ </div>+ <div class="align-middle pt-8">→</div>+ <div class="w-36">+ <%= label f, :end_date, "To", class: "block text-sm font-medium text-gray-700" %>+ <%= date_input f, :end_date, value: @end_date, readonly: "true", class: "mt-1 block w-full px-3 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>+ </div>+ </div>+ <% {:error, error} -> %>+ <p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics data.</p>+ <p class="text-red-700 font-medium mt-3"><%= error %></p>+ <% end %>++ <%= submit "Confirm import", class: "button mt-6" %>+<% end %>diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eexindex 51b24d3c3c..83fee31669 100644--- a/lib/plausible_web/templates/site/settings_general.html.eex+++ b/lib/plausible_web/templates/site/settings_general.html.eex@@ -58,73 +58,42 @@<%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %><%= cond do %>- <% @imported_data && @imported_data.status == "importing" -> %>+ <% @site.imported_data && @site.imported_data.status == "importing" -> %><div class="py-2"></div>- <div class="text-sm">We are importing data from <%= @imported_data.source %> in the background... You will receive an email when it's completed</div>+ <div class="text-sm">We are importing data from <%= @site.imported_data.source %> in the background... You will receive an email when it's completed</div>- <% @imported_data && @imported_data.status == "ok" -> %>+ <% @site.imported_data && @site.imported_data.status == "ok" -> %><li class="py-4 flex items-center justify-between space-x-4"><div class="flex flex-col"><p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">Forget Imported Data</p><p class="text-sm leading-5 text-gray-500 dark:text-gray-200">- Removes all data imported from <%= @imported_data.source %>+ Removes all data imported from <%= @site.imported_data.source %></p></div><%= link("Forget imported stats", to: "/#{URI.encode_www_form(@site.domain)}/settings/forget-imported", method: :delete, class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150") %></li>- <%= if @site.google_auth do %>- <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-import", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %>- <% end %>-- <% @site.google_auth -> %>+ <% @site.imported_data && @site.imported_data.status == "error" -> %><div class="py-2"></div>- <span class="text-gray-700 dark:text-gray-300">Linked Google account: <b><%= @site.google_auth.email %></b></span>-- <%= if @imported_data && @imported_data.status == "error" do %>- <div class="mt-1 text-red-600">- Your last GA import resulted in an error. You can try again below.- </div>- <% end %>-- <%= case @google_profiles do %>- <% {:ok, profiles} -> %>- <p class="text-gray-700 dark:text-gray-300 mt-1">- Select the Google Analytics view id you would like to import data from.- </p>-- <%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-xs"], fn f -> %>- <div class="my-6">- <%= label f, :view_id, "Google Analytics view id", class: "block text-sm font-medium text-gray-700" %>- <%= select f, :view_id, profiles, prompt: "(Choose view id)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>- </div>-- <div class="my-6">- <%= label f, :end_date, "Import end date", class: "block text-sm font-medium text-gray-700" %>- <div class="mt-1">- <%= date_input f, :end_date, value: Timex.today() |> Date.to_iso8601(), required: "true", class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %>- </div>- </div>+ <div class="text-sm">Your latest import has failed. You can try importing again by clicking the button below. If you try multiple times and the import keeps failing, please contact support.</div>- <%= submit "Import", class: "button" %>- <% end %>-- <% {:error, error} -> %>- <p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics profiles.</p>- <p class="text-red-700 font-medium mt-3"><%= error %></p>- <% end %>-- <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-import", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %>+ <div class="flex">+ <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "import"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %>+ <%= google_logo() %>+ <span style="font-family: Roboto, system-ui" class="text-sm font-medium text-gray-600">Continue with Google<span>+ <% end %>+ </div><% true -> %><div class="flex">- <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "general"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %>+ <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "import"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %><%= google_logo() %><span style="font-family: Roboto, system-ui" class="text-sm font-medium text-gray-600">Continue with Google<span><% end %></div>+<% end %><% else %><div class="my-8 text-center text-lg">diff --git a/lib/plausible_web/templates/stats/stats.html.eex b/lib/plausible_web/templates/stats/stats.html.eexindex a47aac1505..f6838654bb 100644--- a/lib/plausible_web/templates/stats/stats.html.eex+++ b/lib/plausible_web/templates/stats/stats.html.eex@@ -12,7 +12,7 @@<% end %><div class="pt-6"></div>- <div id="stats-react-container" style="overflow-anchor: none;" data-domain="<%= @site.domain %>" data-offset="<%= Timex.Timezone.total_offset(Timex.Timezone.get(@site.timezone)) %>" data-has-goals="<%= @has_goals %>" data-logged-in="<%= [email protected][:current_user] %>" data-inserted-at="<%= @site.inserted_at %>" data-shared-link-auth="<%= assigns[:shared_link_auth] %>" data-embedded="<%= @conn.assigns[:embedded] %>" data-background="<%= @conn.assigns[:background] %>" data-selfhosted="<%= Application.get_env(:plausible, :is_selfhost) %>" data-current-user-role="<%= @conn.assigns[:current_user_role] %>"></div>+ <div id="stats-react-container" style="overflow-anchor: none;" data-domain="<%= @site.domain %>" data-offset="<%= Timex.Timezone.total_offset(Timex.Timezone.get(@site.timezone)) %>" data-has-goals="<%= @has_goals %>" data-logged-in="<%= [email protected][:current_user] %>" data-stats-begin="<%= @stats_begin %>" data-shared-link-auth="<%= assigns[:shared_link_auth] %>" data-embedded="<%= @conn.assigns[:embedded] %>" data-background="<%= @conn.assigns[:background] %>" data-selfhosted="<%= Application.get_env(:plausible, :is_selfhost) %>" data-current-user-role="<%= @conn.assigns[:current_user_role] %>"></div><div id="modal_root"></div><%= if [email protected][:current_user] && @conn.assigns[:demo] do %><div class="bg-gray-50 dark:bg-gray-850">diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.exindex b313891e10..6c26990ff4 100644--- a/lib/workers/import_google_analytics.ex+++ b/lib/workers/import_google_analytics.ex@@ -8,14 +8,20 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do@impl Oban.Workerdef perform(- %Oban.Job{args: %{"site_id" => site_id, "view_id" => view_id, "end_date" => end_date}},+ %Oban.Job{+ args: %{+ "site_id" => site_id,+ "view_id" => view_id,+ "start_date" => start_date,+ "end_date" => end_date,+ "access_token" => access_token+ }+ },google_api \\ Plausible.Google.Api) do- site =- Repo.get(Plausible.Site, site_id)- |> Repo.preload([:google_auth, [memberships: :user]])+ site = Repo.get(Plausible.Site, site_id) |> Repo.preload([[memberships: :user]])- case google_api.import_analytics(site, view_id, end_date) do+ case google_api.import_analytics(site, view_id, start_date, end_date, access_token) do{:ok, _} ->Plausible.Site.import_success(site)|> Repo.update!()diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex dc3b475e5b..8389435589 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -727,26 +727,36 @@ defmodule PlausibleWeb.SiteControllerTest dotest "adds in-progress imported tag to site", %{conn: conn, site: site} dopost(conn, "/#{site.domain}/settings/google-import", %{"view_id" => "123",- "end_date" => "2022-03-01"+ "start_date" => "2018-03-01",+ "end_date" => "2022-03-01",+ "access_token" => "token"})imported_data = Repo.reload(site).imported_dataassert imported_dataassert imported_data.source == "Google Analytics"- assert imported_data.end_date == Timex.today()+ assert imported_data.end_date == ~D[2022-03-01]assert imported_data.status == "importing"endtest "schedules an import job in Oban", %{conn: conn, site: site} dopost(conn, "/#{site.domain}/settings/google-import", %{"view_id" => "123",- "end_date" => "2022-03-01"+ "start_date" => "2018-03-01",+ "end_date" => "2022-03-01",+ "access_token" => "token"})assert_enqueued(worker: Plausible.Workers.ImportGoogleAnalytics,- args: %{"site_id" => site.id, "view_id" => "123", "end_date" => "2022-03-01"}+ args: %{+ "site_id" => site.id,+ "view_id" => "123",+ "start_date" => "2018-03-01",+ "end_date" => "2022-03-01",+ "access_token" => "token"+ })endenddiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex e0fdeb0946..bb9076597f 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -14,7 +14,7 @@ defmodule Plausible.TestUtils dodef add_imported_data(%{site: site}) dosite =site- |> Plausible.Site.start_import("Google Analytics", "ok")+ |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")|> Repo.update!(){:ok, site: site}diff --git a/test/workers/import_google_analytics_test.exs b/test/workers/import_google_analytics_test.exsindex 63a06c47f8..a26a5c58cc 100644--- a/test/workers/import_google_analytics_test.exs+++ b/test/workers/import_google_analytics_test.exs@@ -15,13 +15,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest dosite = insert(:site, members: [user], imported_data: @imported_data)api_stub =- stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date ->+ stub(Plausible.Google.Api, :import_analytics, fn _site,+ _view_id,+ _start_date,+ _end_date,+ _access_token ->{:ok, nil}end)ImportGoogleAnalytics.perform(%Oban.Job{- args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"}+ args: %{+ "site_id" => site.id,+ "view_id" => "view_id",+ "start_date" => "2020-01-01",+ "end_date" => "2022-01-01",+ "access_token" => "token"+ }},api_stub)@@ -34,13 +44,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest dosite = insert(:site, members: [user], imported_data: @imported_data)api_stub =- stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date ->+ stub(Plausible.Google.Api, :import_analytics, fn _site,+ _view_id,+ _start_date,+ _end_date,+ _access_token ->{:ok, nil}end)ImportGoogleAnalytics.perform(%Oban.Job{- args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"}+ args: %{+ "site_id" => site.id,+ "view_id" => "view_id",+ "start_date" => "2020-01-01",+ "end_date" => "2022-01-01",+ "access_token" => "token"+ }},api_stub)@@ -56,13 +76,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest dosite = insert(:site, members: [user], imported_data: @imported_data)api_stub =- stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date ->+ stub(Plausible.Google.Api, :import_analytics, fn _site,+ _view_id,+ _start_date,+ _end_date,+ _access_token ->{:error, "Something went wrong"}end)ImportGoogleAnalytics.perform(%Oban.Job{- args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"}+ args: %{+ "site_id" => site.id,+ "view_id" => "view_id",+ "start_date" => "2020-01-01",+ "end_date" => "2022-01-01",+ "access_token" => "token"+ }},api_stub)@@ -75,13 +105,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest dosite = insert(:site, members: [user], imported_data: @imported_data)api_stub =- stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date ->+ stub(Plausible.Google.Api, :import_analytics, fn _site,+ _view_id,+ _start_date,+ _end_date,+ _access_token ->{:error, "Something went wrong"}end)ImportGoogleAnalytics.perform(%Oban.Job{- args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"}+ args: %{+ "site_id" => site.id,+ "view_id" => "view_id",+ "start_date" => "2020-01-01",+ "end_date" => "2022-01-01",+ "access_token" => "token"+ }},api_stub)