Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
ScriptV2: TrackerScriptCache on ee (#5648)
Karl-Aksel Puulmanncf423dbf9951
diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex a0ab741377..ffc144a01a 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -128,22 +128,20 @@ defmodule Plausible.Application do])end,- on_ce do- warmed_cache(PlausibleWeb.TrackerScriptCache,- adapter_opts: [- n_lock_partitions: 1,- ttl_check_interval: false,- ets_options: [:bag, read_concurrency: true]- ],- warmers: [- refresh_all:- {PlausibleWeb.TrackerScriptCache.All,- interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10))},- refresh_updated_recently:- {PlausibleWeb.TrackerScriptCache.RecentlyUpdated, interval: :timer.seconds(120)}- ]- )- end,+ warmed_cache(PlausibleWeb.TrackerScriptCache,+ adapter_opts: [+ n_lock_partitions: 1,+ ttl_check_interval: false,+ ets_options: [:bag, read_concurrency: true]+ ],+ warmers: [+ refresh_all:+ {PlausibleWeb.TrackerScriptCache.All,+ interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10))},+ refresh_updated_recently:+ {PlausibleWeb.TrackerScriptCache.RecentlyUpdated, interval: :timer.seconds(30)}+ ]+ ),Plausible.Ingestion.Counters,Plausible.Session.Salts,Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer),diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.exindex 1ba4a7d9b6..dd8d4ab25e 100644--- a/lib/plausible/telemetry/plausible_metrics.ex+++ b/lib/plausible/telemetry/plausible_metrics.ex@@ -47,6 +47,24 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics dounit: {:native, :millisecond},measurement: :duration),+ distribution(+ metric_prefix ++ [:cache_warmer, :tracker_script, :refresh, :all],+ event_name: PlausibleWeb.TrackerScriptCache.telemetry_event_refresh(:all),+ reporter_options: [+ buckets: [500, 1000, 2000, 5000, 10_000]+ ],+ unit: {:native, :millisecond},+ measurement: :duration+ ),+ distribution(+ metric_prefix ++ [:cache_warmer, :tracker_script, :refresh, :updated_recently],+ event_name: PlausibleWeb.TrackerScriptCache.telemetry_event_refresh(:updated_recently),+ reporter_options: [+ buckets: [500, 1000, 2000, 5000, 10_000]+ ],+ unit: {:native, :millisecond},+ measurement: :duration+ ),distribution(metric_prefix ++ [:ingest, :events, :pipeline, :steps],event_name: Ingestion.Event.telemetry_pipeline_step_duration(),diff --git a/lib/plausible_web/controllers/api/system_controller.ex b/lib/plausible_web/controllers/api/system_controller.exindex 95482e0d52..3fdd993fa9 100644--- a/lib/plausible_web/controllers/api/system_controller.ex+++ b/lib/plausible_web/controllers/api/system_controller.ex@@ -22,6 +22,11 @@ defmodule PlausibleWeb.Api.SystemController doend@task_timeout 15_000+ @critical_caches [+ Plausible.Site.Cache,+ Plausible.Shield.IPRuleCache,+ PlausibleWeb.TrackerScriptCache+ ]def readiness(conn, _params) dopostgres_health_task =Task.async(fn ->@@ -54,8 +59,7 @@ defmodule PlausibleWeb.Api.SystemController doendcache_health =- if postgres_health == "ok" and Plausible.Site.Cache.ready?() and- Plausible.Shield.IPRuleCache.ready?() do+ if postgres_health == "ok" and Enum.all?(@critical_caches, & &1.ready?()) do"ok"enddiff --git a/lib/plausible_web/plugs/tracker_plug.ex b/lib/plausible_web/plugs/tracker_plug.exindex 9e464fdccb..2f067ffcd0 100644--- a/lib/plausible_web/plugs/tracker_plug.ex+++ b/lib/plausible_web/plugs/tracker_plug.ex@@ -62,8 +62,8 @@ defmodule PlausibleWeb.TrackerPlug dodef telemetry_event(name), do: [:plausible, :tracker_script, :request, name]defp request_tracker_script(id, conn) do- case get_plausible_web_script(id) do- {:ok, script_tag} ->+ case PlausibleWeb.Tracker.get_plausible_main_script(id) do+ script_tag when is_binary(script_tag) ->:telemetry.execute(telemetry_event(:v2),%{},@@ -82,7 +82,7 @@ defmodule PlausibleWeb.TrackerPlug do|> send_resp(200, script_tag)|> halt()- {:error, :not_found} ->+ nil ->:telemetry.execute(telemetry_event(:v2),%{},@@ -95,23 +95,6 @@ defmodule PlausibleWeb.TrackerPlug doendend- defp get_plausible_web_script(id) do- script =- on_ee do- # On cloud, we generate the script always on the fly relying on CDN caching- PlausibleWeb.TrackerScriptCache.get_from_source(id)- else- # On self-hosted, we have a pre-warmed cache for the script- PlausibleWeb.TrackerScriptCache.get(id)- end-- if script do- {:ok, script}- else- {:error, :not_found}- end- end-defp legacy_request_file(filename, files_available, conn) doif filename && MapSet.member?(files_available, filename) dolocation = Application.app_dir(:plausible, "priv/tracker/js/" <> filename)diff --git a/lib/plausible_web/tracker.ex b/lib/plausible_web/tracker.exindex 6a79af3dd9..8a98b42d3a 100644--- a/lib/plausible_web/tracker.ex+++ b/lib/plausible_web/tracker.ex@@ -14,7 +14,36 @@ defmodule PlausibleWeb.Tracker do@plausible_main_script File.read!(path)@external_resource "priv/tracker/js/plausible-web.js"- def plausible_main_script_tag(tracker_script_configuration) do+ @spec get_plausible_main_script(String.t(), Keyword.t()) :: String.t() | nil+ def get_plausible_main_script(id, cache_opts \\ []) do+ on_ee do+ # On cloud:+ # 1. Check if tracker script ID is in the cache+ # 2. If it is, generate the script on the fly+ #+ # Note that EE is relying on CDN caching the script+ if PlausibleWeb.TrackerScriptCache.get(id, cache_opts) do+ get_tracker_script_configuration(id)+ |> build_script()+ end+ else+ # On self-hosted, we have a pre-warmed cache for the script+ PlausibleWeb.TrackerScriptCache.get(id, cache_opts)+ end+ end++ # Exposed for testing+ def plausible_main_config(tracker_script_configuration) do+ %{+ domain: tracker_script_configuration.site.domain,+ endpoint: tracker_ingestion_endpoint(),+ outboundLinks: tracker_script_configuration.outbound_links,+ fileDownloads: tracker_script_configuration.file_downloads,+ formSubmissions: tracker_script_configuration.form_submissions+ }+ end++ def build_script(%TrackerScriptConfiguration{} = tracker_script_configuration) doconfig_js_content =tracker_script_configuration|> plausible_main_config()@@ -32,15 +61,7 @@ defmodule PlausibleWeb.Tracker do|> String.replace("\"<%= @config_js %>\"", "{#{config_js_content}}")end- def plausible_main_config(tracker_script_configuration) do- %{- domain: tracker_script_configuration.site.domain,- endpoint: tracker_ingestion_endpoint(),- outboundLinks: tracker_script_configuration.outbound_links,- fileDownloads: tracker_script_configuration.file_downloads,- formSubmissions: tracker_script_configuration.form_submissions- }- end+ def build_script(nil), do: nildef update_script_configuration(site, config_update, changeset_type) doRepo.transact(fn ->@@ -108,6 +129,19 @@ defmodule PlausibleWeb.Tracker doconfigend+ on_ee do+ import Ecto.Query++ defp get_tracker_script_configuration(id) do+ from(t in TrackerScriptConfiguration,+ where: t.id == ^id,+ join: s in assoc(t, :site),+ preload: [site: s]+ )+ |> Plausible.Repo.one()+ end+ end+# Sync plausible goals with the updated script configdefp sync_goals(site, original_config, updated_config) do[:track_404_pages, :outbound_links, :file_downloads, :form_submissions]diff --git a/lib/plausible_web/tracker_script_cache.ex b/lib/plausible_web/tracker_script_cache.exindex 74ec817ba4..031aebbc57 100644--- a/lib/plausible_web/tracker_script_cache.ex+++ b/lib/plausible_web/tracker_script_cache.ex@@ -1,10 +1,14 @@defmodule PlausibleWeb.TrackerScriptCache do@moduledoc """- Cache for tracker script(s) for self-hosted Plausible instances.+ Cache for tracker script.++ On self-hosted instances, we cache the entire tracker script.+ On EE instances, we cache valid tracker script ids to avoid database lookups."""alias Plausible.Site.TrackerScriptConfigurationimport Ecto.Query+ use Plausibleuse Plausible.Cache@cache_name :tracker_script_cache@@ -37,7 +41,7 @@ defmodule PlausibleWeb.TrackerScriptCache docase Plausible.Repo.one(query) do%TrackerScriptConfiguration{} = tracker_script_configuration ->- PlausibleWeb.Tracker.plausible_main_script_tag(tracker_script_configuration)+ cache_content(tracker_script_configuration)_ ->nil@@ -49,10 +53,17 @@ defmodule PlausibleWeb.TrackerScriptCache doEnum.reduce(items, [], fntracker_script_configuration, acc ->[- {tracker_script_configuration.id,- PlausibleWeb.Tracker.plausible_main_script_tag(tracker_script_configuration)}+ {tracker_script_configuration.id, cache_content(tracker_script_configuration)}| acc]end)end++ defp cache_content(tracker_script_configuration) do+ if ee?() do+ true+ else+ PlausibleWeb.Tracker.build_script(tracker_script_configuration)+ end+ endenddiff --git a/test/plausible_web/tracker_script_cache_test.exs b/test/plausible_web/tracker_script_cache_test.exsindex 7a8a9b4e7d..31f0c8714b 100644--- a/test/plausible_web/tracker_script_cache_test.exs+++ b/test/plausible_web/tracker_script_cache_test.exs@@ -1,4 +1,5 @@defmodule PlausibleWeb.TrackerScriptCacheTest do+ use Plausibleuse Plausible.DataCase, async: trueuse Plausible.Teams.Test@@ -6,7 +7,7 @@ defmodule PlausibleWeb.TrackerScriptCacheTest doalias PlausibleWeb.TrackerScriptCachedescribe "public cache interface" do- test "cache caches tracker script configurations", %{test: test} do+ test "cache caches tracker scripts by id", %{test: test} do{:ok, _} =Supervisor.start_link([{TrackerScriptCache, [cache_name: test, child_id: :test_cache_tracker_script]}],@@ -23,8 +24,13 @@ defmodule PlausibleWeb.TrackerScriptCacheTest doassert TrackerScriptCache.size(test) == 1- assert script_tag = TrackerScriptCache.get(config.id, force?: true, cache_name: test)- assert is_binary(script_tag)+ assert result = TrackerScriptCache.get(config.id, force?: true, cache_name: test)++ on_ee do+ assert result == true+ else+ assert is_binary(result)+ endrefute TrackerScriptCache.get("nonexistent", cache_name: test, force?: true)enddiff --git a/test/plausible_web/tracker_test.exs b/test/plausible_web/tracker_test.exsindex d727d610f3..78b6bb0530 100644--- a/test/plausible_web/tracker_test.exs+++ b/test/plausible_web/tracker_test.exs@@ -1,38 +1,39 @@defmodule PlausibleWeb.TrackerTest do- use Plausible.DataCase, async: true+ use Plausible.DataCaseuse Oban.Testing, repo: Plausible.Repouse Plausibleuse Plausible.Teams.Testalias Plausible.Site.TrackerScriptConfigurationalias PlausibleWeb.Tracker+ alias PlausibleWeb.TrackerScriptCache++ @example_config %{+ installation_type: :manual,+ track_404_pages: true,+ hash_based_routing: true,+ file_downloads: false,+ outbound_links: false,+ pageview_props: false,+ tagged_events: true,+ revenue_tracking: false,+ form_submissions: true+ }++ test "plausible_main_config/1" do+ site = new_site()+ tracker_script_configuration = create_config(site)++ assert Tracker.plausible_main_config(tracker_script_configuration) == %{+ domain: site.domain,+ endpoint: "#{PlausibleWeb.Endpoint.url()}/api/event",+ outboundLinks: false,+ fileDownloads: false,+ formSubmissions: true+ }+ end- describe "plausible-web.js" do- @example_config %{- installation_type: :manual,- track_404_pages: true,- hash_based_routing: true,- file_downloads: false,- outbound_links: false,- pageview_props: false,- tagged_events: true,- revenue_tracking: false,- form_submissions: true- }-- test "can calculate config" do- site = new_site()- tracker_script_configuration = create_config(site)-- assert PlausibleWeb.Tracker.plausible_main_config(tracker_script_configuration) == %{- domain: site.domain,- endpoint: "#{PlausibleWeb.Endpoint.url()}/api/event",- outboundLinks: false,- fileDownloads: false,- formSubmissions: true- }- end-+ describe "get_or_create_tracker_script_configuration!/1 and update_script_configuration!/3" dotest "can create config with params" dosite = new_site()@@ -116,12 +117,14 @@ defmodule PlausibleWeb.TrackerTest do)endend+ end+ describe "build_script/1" dotest "can turn config into a script tag" dosite = new_site()tracker_script_configuration = create_config(site)- script_tag = PlausibleWeb.Tracker.plausible_main_script_tag(tracker_script_configuration)+ script_tag = PlausibleWeb.Tracker.build_script(tracker_script_configuration)assert script_tag =~~s(={endpoint:"#{PlausibleWeb.Endpoint.url()}/api/event",domain:"#{site.domain}",formSubmissions:!0})@@ -130,21 +133,84 @@ defmodule PlausibleWeb.TrackerTest dotest "script tag escapes problematic characters as expected" dosite = new_site(domain: "naughty domain &<> \"\'\nfoo ")tracker_script_configuration = create_config(site)- script_tag = PlausibleWeb.Tracker.plausible_main_script_tag(tracker_script_configuration)+ script_tag = PlausibleWeb.Tracker.build_script(tracker_script_configuration)assert script_tag =~ ~s(domain:"naughty domain &<> \\"'\\nfoo ")end+ end- def create_config(site) do- Tracker.update_script_configuration!(- site,- Map.put(@example_config, :site_id, site.id),- :installation- )+ describe "get_plausible_main_script/1" do+ setup %{test: test} do+ site = new_site()+ tracker_script_configuration = create_config(site)- Plausible.Repo.one(- from(c in TrackerScriptConfiguration, where: c.site_id == ^site.id, preload: [:site])- )+ opts = start_test_cache(test)++ {:ok, %{opts: opts, site: site, tracker_script_configuration: tracker_script_configuration}}end++ test "does 0 database queries on non-existing config", %{opts: opts} do+ query_count = attach_query_counter()+ script = Tracker.get_plausible_main_script("non-existing-id", opts)+ assert is_nil(script)+ assert query_count.() == 0+ end++ test "does minimal database queries on existing config", %{+ opts: opts,+ tracker_script_configuration: tracker_script_configuration+ } do+ query_count = attach_query_counter()+ script = Tracker.get_plausible_main_script(tracker_script_configuration.id, opts)+ assert is_binary(script)++ on_ee do+ assert query_count.() == 1+ else+ # On self-hosted, we have a pre-warmed cache for the script+ assert query_count.() == 0+ end+ end+ end++ def create_config(site) do+ Tracker.update_script_configuration!(+ site,+ Map.put(@example_config, :site_id, site.id),+ :installation+ )++ Plausible.Repo.one(+ from(c in TrackerScriptConfiguration, where: c.site_id == ^site.id, preload: [:site])+ )+ end++ defp start_test_cache(cache_name) do+ opts = [cache_name: cache_name, force?: true]++ %{start: {m, f, a}} = TrackerScriptCache.child_spec(cache_name: cache_name)+ {:ok, _} = apply(m, f, a)+ TrackerScriptCache.refresh_all(opts)++ opts+ end++ defp attach_query_counter() do+ query_count = :counters.new(1, [])++ :telemetry.attach(+ "test-query-counter",+ [:plausible, :repo, :query],+ fn _event, _measurements, _metadata, _config ->+ :counters.add(query_count, 1, 1)+ end,+ %{}+ )++ on_exit(fn ->+ :telemetry.detach("test-query-counter")+ end)++ fn -> :counters.get(query_count, 1) endendend