Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Prevent robots from looking too much into public dashboards (#3118)
hq1fea9bb32ee3b
diff --git a/lib/plausible_web/controllers/api/internal_controller.ex b/lib/plausible_web/controllers/api/internal_controller.exindex b721bc9d08..c1e2707513 100644--- a/lib/plausible_web/controllers/api/internal_controller.ex+++ b/lib/plausible_web/controllers/api/internal_controller.ex@@ -6,12 +6,14 @@ defmodule PlausibleWeb.Api.InternalController doalias Plausible.Auth.Userdef domain_status(conn, %{"domain" => domain}) do- site = Sites.get_by_domain(domain)-- if Stats.has_pageviews?(site) do+ with %User{id: user_id} <- conn.assigns[:current_user],+ %Site{} = site <- Sites.get_by_domain(domain),+ true <- Sites.has_admin_access?(user_id, site) || Auth.is_super_admin?(user_id),+ true <- Stats.has_pageviews?(site) dojson(conn, "READY")else- json(conn, "WAITING")+ _ ->+ json(conn, "WAITING")endenddiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 72b6ba9a4b..991a62c463 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -67,7 +67,7 @@ defmodule PlausibleWeb.StatsController dofunnels: Plausible.Funnels.list(site),stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),- title: "Plausible · " <> site.domain,+ title: title(conn, site),offer_email_report: offer_email_report,demo: demo,flags: get_flags(conn.assigns[:current_user]),@@ -295,7 +295,7 @@ defmodule PlausibleWeb.StatsController dofunnels: Plausible.Funnels.list(shared_link.site),stats_start_date: shared_link.site.stats_start_date,native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),- title: "Plausible · " <> shared_link.site.domain,+ title: title(conn, shared_link.site),offer_email_report: false,demo: false,skip_plausible_tracking: true,@@ -343,4 +343,12 @@ defmodule PlausibleWeb.StatsController do!!is_or_nilend++ defp title(%{path_info: ["plausible.io"]}, _) do+ "Plausible Analytics: Live Demo"+ end++ defp title(_conn, site) do+ "Plausible · " <> site.domain+ endenddiff --git a/lib/plausible_web/plugs/no_robots.ex b/lib/plausible_web/plugs/no_robots.exnew file mode 100644index 0000000000..86f178fbd8--- /dev/null+++ b/lib/plausible_web/plugs/no_robots.ex@@ -0,0 +1,29 @@+defmodule PlausibleWeb.Plugs.NoRobots do+ @moduledoc """+ Rejects bot requests by any means available.++ We're adding `x-robots-tag` to the response header and annotate the conn+ with "noindex, nofollow" under `private.robots` key.++ The only exception is, if the request is trying to access our live demo+ at plausible.io/plausible.io - in which case we'll allow indexing, but deny+ following links and skip the bot detection, in kind robots we trust.+ """+ @behaviour Plug+ import Plug.Conn++ @impl true+ def init(opts), do: opts++ @impl true+ def call(conn, _opts \\ nil) do+ conn =+ if conn.path_info == ["plausible.io"] do+ put_private(conn, :robots, "index, nofollow")+ else+ put_private(conn, :robots, "noindex, nofollow")+ end++ put_resp_header(conn, "x-robots-tag", conn.private.robots)+ end+enddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex 018ab227fa..8c26db6f9a 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -8,6 +8,7 @@ defmodule PlausibleWeb.Router doplug :fetch_sessionplug :fetch_live_flashplug :put_secure_browser_headers+ plug PlausibleWeb.Plugs.NoRobotsplug PlausibleWeb.FirstLaunchPlug, redirect_to: "/register"plug PlausibleWeb.SessionTimeoutPlug, timeout_after_seconds: @two_weeks_in_secondsplug PlausibleWeb.AuthPlug@@ -17,6 +18,7 @@ defmodule PlausibleWeb.Router dopipeline :shared_link doplug :accepts, ["html"]plug :put_secure_browser_headers+ plug PlausibleWeb.Plugs.NoRobotsendpipeline :csrf do@@ -33,6 +35,7 @@ defmodule PlausibleWeb.Router doplug :accepts, ["json"]plug :fetch_sessionplug PlausibleWeb.AuthorizeSiteAccess+ plug PlausibleWeb.Plugs.NoRobotsendpipeline :public_api do@@ -42,6 +45,7 @@ defmodule PlausibleWeb.Router dopipeline :flags doplug :accepts, ["html"]plug :put_secure_browser_headers+ plug PlausibleWeb.Plugs.NoRobotsplug :fetch_sessionplug PlausibleWeb.CRMAuthPlugend@@ -50,7 +54,9 @@ defmodule PlausibleWeb.Router doforward "/sent-emails", Bamboo.SentEmailViewerPlugend- use Kaffy.Routes, scope: "/crm", pipe_through: [PlausibleWeb.CRMAuthPlug]+ use Kaffy.Routes,+ scope: "/crm",+ pipe_through: [PlausibleWeb.Plugs.NoRobots, PlausibleWeb.CRMAuthPlug]scope path: "/flags" dopipe_through :flags@@ -59,7 +65,7 @@ defmodule PlausibleWeb.Router doscope "/api/stats", PlausibleWeb.Api dopipe_through :internal_stats_api- get "/:domain//funnels/:id", StatsController, :funnel+ get "/:domain/funnels/:id", StatsController, :funnelget "/:domain/current-visitors", StatsController, :current_visitorsget "/:domain/main-graph", StatsController, :main_graphget "/:domain/top-stats", StatsController, :top_statsdiff --git a/lib/plausible_web/templates/layout/app.html.eex b/lib/plausible_web/templates/layout/app.html.eexindex a9d79c8ee0..b58a7a2951 100644--- a/lib/plausible_web/templates/layout/app.html.eex+++ b/lib/plausible_web/templates/layout/app.html.eex@@ -8,6 +8,7 @@<%= if Plausible.Funnels.enabled_for?(@conn.assigns[:current_user]) do %><meta name="csrf-token" content="<%= Plug.CSRFProtection.get_csrf_token() %>" /><meta name="websocket-url" content="<%= websocket_url() %>" />+ <meta name="robots" content="<%= @conn.private.robots %>" /><% end %><link rel="icon" type="image/png" sizes="32x32" href="<%= PlausibleWeb.Router.Helpers.static_path(@conn, "/images/icon/plausible_favicon.png") %>"><link rel="apple-touch-icon" href="/images/icon/apple-touch-icon.png">diff --git a/lib/plausible_web/templates/layout/focus.html.eex b/lib/plausible_web/templates/layout/focus.html.eexindex 3cfa049fa6..64f20ad5e7 100644--- a/lib/plausible_web/templates/layout/focus.html.eex+++ b/lib/plausible_web/templates/layout/focus.html.eex@@ -5,6 +5,7 @@<meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta name="description" content="A lightweight, non-intrusive alternative to Google Analytics."/>+ <meta name="robots" content="<%= @conn.private.robots %>" /><link rel="icon" type="image/png" sizes="32x32" href="<%= PlausibleWeb.Router.Helpers.static_path(@conn, "/images/icon/plausible_favicon.png") %>"><title><%= assigns[:title] || "Plausible · Web analytics" %></title><link rel="stylesheet" href="<%= Routes.static_path(@conn, "/css/app.css") %>"/>diff --git a/test/plausible_web/controllers/api/internal_controller_test.exs b/test/plausible_web/controllers/api/internal_controller_test.exsindex c7ce0800ec..2a2e5cd489 100644--- a/test/plausible_web/controllers/api/internal_controller_test.exs+++ b/test/plausible_web/controllers/api/internal_controller_test.exs@@ -20,6 +20,21 @@ defmodule PlausibleWeb.Api.InternalControllerTest doassert json_response(conn, 200) == "READY"end++ test "is WAITING when unauthenticated", %{user: user} do+ site = insert(:site, members: [user])+ Plausible.TestUtils.create_pageviews([%{site: site}])++ conn = get(build_conn(), "/api/#{site.domain}/status")++ assert json_response(conn, 200) == "WAITING"+ end++ test "is WAITING when non-existing site", %{conn: conn} do+ conn = get(conn, "/api/example.com/status")++ assert json_response(conn, 200) == "WAITING"+ endenddescribe "GET /api/sites" dodiff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 7529d6367e..048e2d6313 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -1,6 +1,7 @@defmodule PlausibleWeb.StatsControllerTest douse PlausibleWeb.ConnCase, async: trueuse Plausible.Repo+ import Plausible.Test.Support.HTMLdescribe "GET /:website - anonymous user" dotest "public site - shows site stats", %{conn: conn} do@@ -8,7 +9,31 @@ defmodule PlausibleWeb.StatsControllerTest dopopulate_stats(site, [build(:pageview)])conn = get(conn, "/#{site.domain}")- assert html_response(conn, 200) =~ "stats-react-container"+ resp = html_response(conn, 200)+ assert element_exists?(resp, "div#stats-react-container")++ assert ["noindex, nofollow"] ==+ resp+ |> find("meta[name=robots]")+ |> Floki.attribute("content")++ assert text_of_element(resp, "title") == "Plausible · #{site.domain}"+ end++ test "plausible.io live demo - shows site stats", %{conn: conn} do+ site = insert(:site, domain: "plausible.io", public: true)+ populate_stats(site, [build(:pageview)])++ conn = get(conn, "/#{site.domain}")+ resp = html_response(conn, 200)+ assert element_exists?(resp, "div#stats-react-container")++ assert ["index, nofollow"] ==+ resp+ |> find("meta[name=robots]")+ |> Floki.attribute("content")++ assert text_of_element(resp, "title") == "Plausible Analytics: Live Demo"endtest "public site - shows waiting for first pageview", %{conn: conn} dodiff --git a/test/plausible_web/plugs/no_robots_test.exs b/test/plausible_web/plugs/no_robots_test.exsnew file mode 100644index 0000000000..1897b4c870--- /dev/null+++ b/test/plausible_web/plugs/no_robots_test.exs@@ -0,0 +1,36 @@+defmodule PlausibleWeb.Plugs.NoRobotsTest do+ use Plausible.DataCase, async: true+ use Plug.Test++ alias PlausibleWeb.Plugs.NoRobots++ test "non-bots pass - when no user agent is supplied" do+ conn = :get |> conn("/") |> NoRobots.call()+ assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]+ assert get_resp_header(conn, "x-plausible-forbidden-reason") == []+ assert conn.private.robots == "noindex, nofollow"++ refute conn.halted+ refute conn.status+ end++ test "non-bots pass - when user agent is supplied" do+ conn = :get |> conn("/") |> NoRobots.call()+ assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]+ assert get_resp_header(conn, "x-plausible-forbidden-reason") == []+ assert conn.private.robots == "noindex, nofollow"++ refute conn.halted+ refute conn.status+ end++ test "writes index, nofollow for plausible.io live demo" do+ conn = :get |> conn("/plausible.io") |> NoRobots.call()++ assert get_resp_header(conn, "x-robots-tag") == ["index, nofollow"]+ assert get_resp_header(conn, "x-plausible-forbidden-reason") == []++ refute conn.halted+ refute conn.status+ end+end