Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Fix: application startup, event timestamps (#2479)
Adam Rutkowski467317f24e68
diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex d9d015a84e..1a6c36d5d7 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -21,12 +21,12 @@ defmodule Plausible.Application doSupervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true},id: :cachex_sessions),- Plausible.PromEx,{Plausible.Site.Cache, []},{Plausible.Site.Cache.Warmer.All, []},{Plausible.Site.Cache.Warmer.RecentlyUpdated, []},PlausibleWeb.Endpoint,- {Oban, Application.get_env(:plausible, Oban)}+ {Oban, Application.get_env(:plausible, Oban)},+ Plausible.PromEx]opts = [strategy: :one_for_one, name: Plausible.Supervisor]diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex 3264affdde..695a1d2e1d 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -22,8 +22,7 @@ defmodule Plausible.Ingestion.Request dofield :props, :mapfield :query_params, :map- field :timestamp, :naive_datetime,- default: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)+ field :timestamp, :naive_datetimeend@type t() :: %__MODULE__{}@@ -33,7 +32,13 @@ defmodule Plausible.Ingestion.Request doBuilds and initially validates %Plausible.Ingestion.Request{} struct from %Plug.Conn{}."""def build(%Plug.Conn{} = conn) do- changeset = Changeset.change(%__MODULE__{})+ changeset =+ %__MODULE__{}+ |> Changeset.change()+ |> Changeset.put_change(+ :timestamp,+ NaiveDateTime.utc_now()+ )case parse_body(conn) do{:ok, request_body} ->diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.exindex 8069d655f1..f85a780d4f 100644--- a/lib/plausible/site/cache.ex+++ b/lib/plausible/site/cache.ex@@ -55,6 +55,7 @@ defmodule Plausible.Site.Cache do@type t() :: Site.t()+ @spec name() :: atom()def name(), do: @cache_name@spec child_spec(Keyword.t()) :: Supervisor.child_spec()@@ -68,6 +69,22 @@ defmodule Plausible.Site.Cache do)end+ @doc """+ Ensures the cache has non-zero size unless no sites exist.+ Useful for orchestrating app startup to prevent the service+ going up asynchronously with an empty cache.+ """+ @spec ready?(atom()) :: boolean+ def ready?(cache_name \\ @cache_name) do+ case size(cache_name) do+ n when n > 0 ->+ true++ 0 ->+ Plausible.Repo.aggregate(Site, :count) == 0+ end+ end+@spec refresh_all(Keyword.t()) :: :okdef refresh_all(opts \\ []) dosites_by_domain_query =diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.exindex f81b4944a2..29e5dd9063 100644--- a/lib/plausible_web/controllers/api/external_controller.ex+++ b/lib/plausible_web/controllers/api/external_controller.ex@@ -60,16 +60,22 @@ defmodule PlausibleWeb.Api.ExternalController doe -> "error: #{inspect(e)}"end+ sites_cache_health =+ if postgres_health == "ok" and Plausible.Site.Cache.ready?() do+ "ok"+ end+status =- case {postgres_health, clickhouse_health} do- {"ok", "ok"} -> 200+ case {postgres_health, clickhouse_health, sites_cache_health} do+ {"ok", "ok", "ok"} -> 200_ -> 500endput_status(conn, status)|> json(%{postgres: postgres_health,- clickhouse: clickhouse_health+ clickhouse: clickhouse_health,+ sites_cache: sites_cache_health})enddiff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exsindex 10d529ac2c..320f5aa8d7 100644--- a/test/plausible/ingestion/request_test.exs+++ b/test/plausible/ingestion/request_test.exs@@ -43,6 +43,24 @@ defmodule Plausible.Ingestion.RequestTest doassert request.props == %{}end+ test "requests include moving timestamp" do+ payload = %{+ name: "pageview",+ url: "http://dummy.site"+ }++ conn = build_conn(:post, "/api/events", payload)++ assert {:ok, request1} = Request.build(conn)+ :timer.sleep(1500)+ assert {:ok, request2} = Request.build(conn)++ ts1 = request1.timestamp+ ts2 = request2.timestamp++ assert NaiveDateTime.compare(ts1, ts2) == :lt+ end+test "request can be built with domain" dopayload = %{name: "pageview",diff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exsindex a6a34ca13a..8db8c00f19 100644--- a/test/plausible/site/cache_test.exs+++ b/test/plausible/site/cache_test.exs@@ -53,6 +53,24 @@ defmodule Plausible.Site.CacheTest dorefute Cache.get("site3.example.com", cache_name: test, force?: true)end+ test "cache is ready when no sites exist in the db", %{test: test} do+ {:ok, _} = start_test_cache(test)+ assert Cache.ready?(test)+ end++ test "cache is not ready when sites exist in the db but cache needs refresh", %{test: test} do+ {:ok, _} = start_test_cache(test)+ insert(:site)+ refute Cache.ready?(test)+ end++ test "cache is ready when refreshed", %{test: test} do+ {:ok, _} = start_test_cache(test)+ insert(:site)+ :ok = Cache.refresh_all(cache_name: test)+ assert Cache.ready?(test)+ end+test "cache exposes hit rate", %{test: test} do{:ok, _} = start_test_cache(test)diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex eae881be33..cab75db5c2 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -62,7 +62,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert conn.status == 400end- test "can send to multiple dashboards by listing multiple domains", %{+ test "can send to multiple dashboards by listing multiple domains - same timestamp", %{conn: conn,domain: domain1} do@@ -82,8 +82,34 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> post("/api/event", params)assert response(conn, 202) == "ok"- assert get_event(domain1)- assert get_event(domain2)+ assert e1 = get_event(domain1)+ assert e2 = get_event(domain2)++ assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :eq+ end++ test "timestamps differ when two events sent in a row", %{conn: conn, domain: domain} do+ params = %{+ domain: domain,+ name: "pageview",+ url: "http://gigride.live/",+ referrer: "http://m.facebook.com/",+ screen_width: 1440+ }++ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ :timer.sleep(1500)++ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ [e1, e2] = get_events(domain)++ assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :gtendtest "www. is stripped from domain", %{conn: conn, domain: domain} do@@ -1120,7 +1146,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dotest "returns 200 OK", %{conn: conn} doconn = get(conn, "/api/health")- assert conn.status == 200+ assert payload = json_response(conn, 200)+ assert payload["postgres"] == "ok"+ assert payload["clickhouse"] == "ok"+ assert payload["sites_cache"] == "ok"endend