Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

19.2 KB
Raw Blame
defmodule Plausible.Ingestion.EventTest do
  use Plausible.DataCase, async: false

  import Phoenix.ConnTest

  alias Plausible.Ingestion.Request
  alias Plausible.Ingestion.Event

  test "processes a request into an event" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://#{site.domain}"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request)
  end

  @regressive_user_agents [
    ~s|Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/537.3666 (KHTML, like Gecko) Chrome/110.0.0.0.0 Safari/537.3666|,
    ~s|Mozilla/5.0 (Linux; arm_64; Android 10; Mi Note 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.5765.05 Mobile Safari/537.36|
  ]

  for {user_agent, idx} <- Enum.with_index(@regressive_user_agents) do
    test "processes user agents known to cause problems parsing in the past (case #{idx})" do
      site = new_site()

      payload = %{
        name: "pageview",
        url: "http://#{site.domain}"
      }

      conn =
        build_conn(:post, "/api/events", payload)
        |> Plug.Conn.put_req_header("user-agent", unquote(user_agent))

      assert {:ok, request, _conn} = Request.build(conn)

      assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request)
    end
  end

  test "times out parsing user agent", %{test: test} do
    on_exit(:detach, fn ->
      :telemetry.detach("ua-timeout-#{test}")
    end)

    test_pid = self()
    event = Event.telemetry_ua_parse_timeout()

    :telemetry.attach(
      "ua-timeout-#{test}",
      event,
      fn ^event, _, _, _ ->
        send(test_pid, :telemetry_handled)
      end,
      %{}
    )

    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://#{site.domain}"
    }

    conn =
      :post
      |> build_conn("/api/events", payload)
      |> Plug.Conn.put_req_header("user-agent", :binary.copy("a", 1024 * 32))

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request)
    assert_receive :telemetry_handled
  end

  on_ee do
    test "drops installation support user agent" do
      site = new_site()

      payload = %{
        name: "pageview",
        url: "http://#{site.domain}"
      }

      conn =
        build_conn(:post, "/api/events", payload)
        |> Plug.Conn.put_req_header("user-agent", Plausible.InstallationSupport.user_agent())

      assert {:ok, request, _conn} = Request.build(conn)

      assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
      assert dropped.drop_reason == :verification_agent
    end
  end

  test "drops a request when site does not exists" do
    payload = %{
      name: "pageview",
      url: "http://dummy.site"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :not_found
  end

  test "drops a request when referrer is spam" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      referrer: "https://www.1-best-seo.com",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :spam_referrer
  end

  test "drops a request when referrer is spam for multiple domains" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      referrer: "https://www.1-best-seo.com",
      d: "#{site.domain},#{site.domain}"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{dropped: [dropped, dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :spam_referrer
  end

  test "selectively drops an event for multiple domains" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      d: "#{site.domain},thisdoesnotexist.com"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [_], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :not_found
  end

  test "selectively drops an event when rate-limited" do
    site = new_site(ingest_rate_limit_threshold: 1)

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      d: "#{site.domain}"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request)
    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :throttle
  end

  test "drops a request when header x-plausible-ip-type is dc_ip" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)
    conn = Plug.Conn.put_req_header(conn, "x-plausible-ip-type", "dc_ip")
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :dc_ip
  end

  test "drops a request when header x-plausible-ip-type is threat_ip" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)
    conn = Plug.Conn.put_req_header(conn, "x-plausible-ip-type", "threat_ip")
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :threat_ip
  end

  test "drops a request when ip is on blocklist" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)
    conn = %{conn | remote_ip: {127, 7, 7, 7}}

    {:ok, _} = Plausible.Shields.add_ip_rule(site, %{"inet" => "127.7.7.7"})

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :site_ip_blocklist
  end

  test "drops a request when country is on blocklist" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)
    conn = %{conn | remote_ip: {216, 160, 83, 56}}

    %{country_code: cc} = Plausible.Ingestion.Geolocation.lookup("216.160.83.56")
    {:ok, _} = Plausible.Shields.add_country_rule(site, %{"country_code" => cc})

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :site_country_blocklist
  end

  test "drops a request when country is unknown and Unknown (ZZ) is on blocklist" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)
    conn = %{conn | remote_ip: {127, 0, 0, 1}}

    {:ok, _} = Plausible.Shields.add_country_rule(site, %{"country_code" => "ZZ"})

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :site_country_blocklist
  end

  test "allows a request when country is unknown and Unknown (ZZ) is not on blocklist" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)
    conn = %{conn | remote_ip: {127, 0, 0, 1}}

    {:ok, _} = Plausible.Shields.add_country_rule(site, %{"country_code" => "EE"})

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request)
  end

  test "drops a request when page is on blocklist" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site/blocked/page",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)

    {:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/blocked/**"})

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :site_page_blocklist
  end

  test "drops a request when hostname allowlist is defined and hostname is not on the list" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)

    {:ok, _} = Plausible.Shields.add_hostname_rule(site, %{"hostname" => "subdomain.dummy.site"})

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :site_hostname_allowlist
  end

  test "passes a request when hostname allowlist is defined and hostname is on the list" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://subdomain.dummy.site",
      domain: site.domain
    }

    conn = build_conn(:post, "/api/events", payload)

    {:ok, _} = Plausible.Shields.add_hostname_rule(site, %{"hostname" => "subdomain.dummy.site"})

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request)
  end

  test "drops events for site with accept_traffic_until in the past" do
    yesterday = Date.add(Date.utc_today(), -1)

    owner = new_user(team: [accept_traffic_until: yesterday])
    site = new_site(ingest_rate_limit_threshold: 1, owner: owner)

    payload = %{
      name: "pageview",
      url: "http://dummy.site",
      d: "#{site.domain}"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :payment_required
  end

  @tag :slow
  test "drops events on session lock timeout" do
    site = new_site()

    test = self()

    very_slow_buffer = fn _sessions ->
      send(test, :slow_buffer_insert_started)
      Process.sleep(800)
    end

    first_conn =
      build_conn(:post, "/api/events", %{
        name: "pageview",
        url: "http://dummy.site",
        d: "#{site.domain}"
      })

    assert {:ok, first_request, _conn} = Request.build(first_conn)

    second_conn =
      build_conn(:post, "/api/events", %{
        name: "page_scrolled",
        url: "http://dummy.site",
        d: "#{site.domain}"
      })

    assert {:ok, second_request, _conn} = Request.build(second_conn)

    Task.start(fn ->
      assert {:ok, %{buffered: [_event], dropped: []}} =
               Event.build_and_buffer(first_request,
                 persistor_opts: [session_write_buffer_insert: very_slow_buffer]
               )
    end)

    receive do
      :slow_buffer_insert_started ->
        assert {:ok, %{buffered: [], dropped: [dropped]}} =
                 Event.build_and_buffer(second_request,
                   persistor_opts: [session_write_buffer_insert: very_slow_buffer]
                 )

        assert dropped.drop_reason == :lock_timeout
    end
  end

  test "drops engagement event when no session found from cache" do
    site = new_site()

    payload = %{
      name: "engagement",
      url: "https://#{site.domain}/123",
      d: "#{site.domain}",
      sd: 25,
      et: 1000
    }

    conn = build_conn(:post, "/api/events", payload)

    assert {:ok, request, _conn} = Request.build(conn)
    assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
    assert dropped.drop_reason == :no_session_for_engagement
  end

  for {input, expected} <- [
        {0, true},
        {nil, true},
        {"invalid", true},
        {true, true},
        {false, false}
      ] do
    test "parses events interactive value #{inspect(input)} to #{inspect(expected)}" do
      site = new_site()

      payload = %{
        name: "ping",
        url: "http://#{site.domain}",
        interactive: unquote(input)
      }

      conn = build_conn(:post, "/api/events", payload)
      assert {:ok, request, _conn} = Request.build(conn)

      assert {:ok, %{buffered: [event], dropped: []}} = Event.build_and_buffer(request)
      assert event.clickhouse_event.interactive? == unquote(expected)
    end
  end

  @tag :ee_only
  test "saves revenue amount" do
    site = new_site()
    _goal = insert(:goal, event_name: "checkout", currency: "USD", site: site)

    payload = %{
      name: "checkout",
      url: "http://#{site.domain}",
      revenue: %{amount: 10.2, currency: "USD"}
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [event], dropped: []}} = Event.build_and_buffer(request)
    assert Decimal.eq?(event.clickhouse_event.revenue_source_amount, Decimal.new("10.2"))
  end

  @tag :ee_only
  test "saves replay session id when passed in headers" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://#{site.domain}"
    }

    conn =
      build_conn(:post, "/api/events", payload)
      |> Plug.Conn.put_req_header("x-replay-event-id", "123")
      |> Plug.Conn.put_req_header("x-replay-session-id", "456")
      |> Plug.Conn.put_req_header("x-replay-time", "2026-06-02 12:43:00")

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [event], dropped: []}} = Event.build_and_buffer(request)
    assert event.clickhouse_event.replay_session_id == 456
    assert event.clickhouse_event.timestamp == ~N[2026-06-02 12:43:00]
  end

  @tag :ee_only
  test "leaves replay session id empty when not passed in headers" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://#{site.domain}"
    }

    conn =
      build_conn(:post, "/api/events", payload)

    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [event], dropped: []}} = Event.build_and_buffer(request)
    assert is_nil(event.clickhouse_event.replay_session_id)
    assert %NaiveDateTime{} = event.clickhouse_event.timestamp
  end

  @tag :ee_only
  test "replayed events have a distinct user id due to reply session-dependent salt" do
    site = new_site()

    payload = %{
      name: "pageview",
      url: "http://#{site.domain}"
    }

    conn1 =
      build_conn(:post, "/api/events", payload)
      |> Plug.Conn.put_req_header("user-agent", "Mozilla")
      |> Plug.Conn.put_req_header("x-plausible-ip", "1.2.3.4")

    assert {:ok, request1, _conn} = Request.build(conn1)
    assert {:ok, %{buffered: [event1], dropped: []}} = Event.build_and_buffer(request1)

    conn2 =
      build_conn(:post, "/api/events", payload)
      |> Plug.Conn.put_req_header("user-agent", "Mozilla")
      |> Plug.Conn.put_req_header("x-plausible-ip", "1.2.3.4")
      |> Plug.Conn.put_req_header("x-replay-event-id", "123")
      |> Plug.Conn.put_req_header("x-replay-session-id", "456")
      |> Plug.Conn.put_req_header("x-replay-time", "2026-06-02 12:43:00")

    assert {:ok, request2, _conn} = Request.build(conn2)
    assert {:ok, %{buffered: [event2], dropped: []}} = Event.build_and_buffer(request2)

    conn3 =
      build_conn(:post, "/api/events", payload)
      |> Plug.Conn.put_req_header("user-agent", "Mozilla")
      |> Plug.Conn.put_req_header("x-plausible-ip", "1.2.3.4")
      |> Plug.Conn.put_req_header("x-replay-event-id", "124")
      |> Plug.Conn.put_req_header("x-replay-session-id", "456")
      |> Plug.Conn.put_req_header("x-replay-time", "2026-06-02 12:44:01")

    assert {:ok, request3, _conn} = Request.build(conn3)
    assert {:ok, %{buffered: [event3], dropped: []}} = Event.build_and_buffer(request3)

    conn4 =
      build_conn(:post, "/api/events", payload)
      |> Plug.Conn.put_req_header("user-agent", "Mozilla")
      |> Plug.Conn.put_req_header("x-plausible-ip", "1.2.3.4")
      |> Plug.Conn.put_req_header("x-replay-event-id", "125")
      |> Plug.Conn.put_req_header("x-replay-session-id", "789")
      |> Plug.Conn.put_req_header("x-replay-time", "2026-06-02 12:44:01")

    assert {:ok, request4, _conn} = Request.build(conn4)
    assert {:ok, %{buffered: [event4], dropped: []}} = Event.build_and_buffer(request4)

    # non-replayed event user id different from replayed event user despite
    # identical fingerprint
    assert event1.clickhouse_event.user_id != event2.clickhouse_event.user_id
    # replayed events with the same session for the same fingerprint have the
    # same user id
    assert event2.clickhouse_event.user_id == event3.clickhouse_event.user_id
    # replayed event from different sessions for the same fingerprint
    # have different users
    assert event3.clickhouse_event.user_id != event4.clickhouse_event.user_id
  end

  test "does not save revenue amount when there is no revenue goal" do
    site = new_site()

    payload = %{
      name: "checkout",
      url: "http://#{site.domain}",
      revenue: %{amount: 10.2, currency: "USD"}
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [event], dropped: []}} = Event.build_and_buffer(request)
    assert event.clickhouse_event.revenue_source_amount == nil
  end

  test "IPv4 hostname is stored without public suffix processing" do
    _site = new_site(domain: "192.168.0.1")

    payload = %{
      name: "checkout",
      url: "http://192.168.0.1"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)
    assert event.clickhouse_event.hostname == "192.168.0.1"
  end

  test "Hostname is stored with public suffix processing" do
    _site = new_site(domain: "foo.netlify.app")

    payload = %{
      name: "checkout",
      url: "http://foo.netlify.app"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)
    assert event.clickhouse_event.hostname == "foo.netlify.app"
  end

  test "hostname is (none) when no hostname can be derived from the url" do
    site = new_site(domain: "foo.example.com")

    payload = %{
      domain: site.domain,
      name: "pageview",
      url: "/no/hostname"
    }

    conn = build_conn(:post, "/api/events", payload)
    assert {:ok, request, _conn} = Request.build(conn)

    assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)
    assert event.clickhouse_event.hostname == "(none)"
  end
end