Skip to content

A platform for community discussion. Free, open, simple.

17.8 KB
Raw Blame
# frozen_string_literal: true

RSpec.describe SearchLog, type: :model do
  after { SearchLog.clear_debounce_cache! }

  describe ".log" do
    context "with invalid arguments" do
      it "no search type returns error" do
        status, _ =
          SearchLog.log(term: "bounty hunter", search_type: :missing, ip_address: "127.0.0.1")

        expect(status).to eq(:error)
      end

      it "no IP returns error" do
        status, _ = SearchLog.log(term: "bounty hunter", search_type: :header, ip_address: nil)

        expect(status).to eq(:error)
      end

      it "truncates the `user_agent` attribute if it exceeds #{described_class::MAXIMUM_USER_AGENT_LENGTH} characters" do
        user_agent = "a" * (described_class::MAXIMUM_USER_AGENT_LENGTH + 1)

        status, _ =
          SearchLog.log(
            term: "bounty hunter",
            search_type: :header,
            user_agent:,
            ip_address: "127.0.0.1",
          )

        expect(status).to eq(:created)
        expect(SearchLog.last.user_agent).to eq("a" * described_class::MAXIMUM_USER_AGENT_LENGTH)
      end
    end

    context "when anonymous" do
      it "logs and updates the search" do
        freeze_time
        action, log_id =
          SearchLog.log(
            term: "jabba",
            search_type: :header,
            ip_address: "192.168.0.33",
            user_agent: "Mozilla",
          )
        expect(action).to eq(:created)
        log = SearchLog.find(log_id)
        expect(log.term).to eq("jabba")
        expect(log.search_type).to eq(SearchLog.search_types[:header])
        expect(log.ip_address).to eq("192.168.0.33")
        expect(log.user_agent).to eq("Mozilla")

        action, updated_log_id =
          SearchLog.log(term: "jabba the hut", search_type: :header, ip_address: "192.168.0.33")
        expect(action).to eq(:updated)
        expect(updated_log_id).to eq(log_id)
      end

      it "creates a new search with a different prefix" do
        freeze_time
        action, _ = SearchLog.log(term: "darth", search_type: :header, ip_address: "127.0.0.1")
        expect(action).to eq(:created)

        action, _ = SearchLog.log(term: "anakin", search_type: :header, ip_address: "127.0.0.1")
        expect(action).to eq(:created)
      end

      it "creates a new search with a different ip" do
        freeze_time
        action, _ = SearchLog.log(term: "darth", search_type: :header, ip_address: "127.0.0.1")
        expect(action).to eq(:created)

        action, _ = SearchLog.log(term: "darth", search_type: :header, ip_address: "127.0.0.2")
        expect(action).to eq(:created)
      end
    end

    context "when logged in" do
      fab!(:user)
      let!(:plugin) { Plugin::Instance.new }
      let!(:modifier) { :search_log_can_log }
      let!(:deny_block) { Proc.new { false } }
      let!(:allow_block) { Proc.new { true } }

      it "logs and updates the search" do
        freeze_time
        action, log_id =
          SearchLog.log(
            term: "hello",
            search_type: :full_page,
            ip_address: "192.168.0.1",
            user_agent: "Mozilla",
            user_id: user.id,
          )
        expect(action).to eq(:created)
        log = SearchLog.find(log_id)
        expect(log.term).to eq("hello")
        expect(log.search_type).to eq(SearchLog.search_types[:full_page])
        expect(log.ip_address).to eq(nil)
        expect(log.user_agent).to eq("Mozilla")
        expect(log.user_id).to eq(user.id)

        action, updated_log_id =
          SearchLog.log(
            term: "hello dolly",
            search_type: :header,
            ip_address: "192.168.0.33",
            user_id: user.id,
          )
        expect(action).to eq(:updated)
        expect(updated_log_id).to eq(log_id)
      end

      it "logs again if time has passed" do
        freeze_time(10.minutes.ago)

        action, _ =
          SearchLog.log(
            term: "hello",
            search_type: :full_page,
            ip_address: "192.168.0.1",
            user_id: user.id,
          )
        expect(action).to eq(:created)

        freeze_time(10.minutes.from_now)
        Discourse.redis.del(SearchLog.redis_key(ip_address: "192.168.0.1", user_id: user.id))

        action, _ =
          SearchLog.log(
            term: "hello",
            search_type: :full_page,
            ip_address: "192.168.0.1",
            user_id: user.id,
          )

        expect(action).to eq(:created)
      end

      it "logs again with a different user" do
        freeze_time

        action, _ =
          SearchLog.log(
            term: "hello",
            search_type: :full_page,
            ip_address: "192.168.0.1",
            user_id: user.id,
          )
        expect(action).to eq(:created)

        action, _ =
          SearchLog.log(
            term: "hello dolly",
            search_type: :full_page,
            ip_address: "192.168.0.1",
            user_id: Fabricate(:user).id,
          )
        expect(action).to eq(:created)
      end

      it "allows plugins to control logging" do
        DiscoursePluginRegistry.register_modifier(plugin, modifier, &deny_block)
        action, _ =
          SearchLog.log(
            term: "hello dolly",
            search_type: :full_page,
            ip_address: "192.168.0.1",
            user_id: Fabricate(:user).id,
          )
        expect(action).to_not eq(:created)

        DiscoursePluginRegistry.register_modifier(plugin, modifier, &allow_block)
        action, _ =
          SearchLog.log(
            term: "hello dolly",
            search_type: :full_page,
            ip_address: "192.168.0.1",
            user_id: Fabricate(:user).id,
          )
        expect(action).to eq(:created)
      ensure
        DiscoursePluginRegistry.unregister_modifier(plugin, modifier, &deny_block)
        DiscoursePluginRegistry.unregister_modifier(plugin, modifier, &allow_block)
      end
    end
  end

  describe ".log crawler detection" do
    it "flags an anonymous search from a known crawler user agent" do
      _status, id =
        SearchLog.log(
          term: "ruby",
          search_type: :header,
          ip_address: "127.0.0.1",
          user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",
        )

      expect(SearchLog.find(id).crawler).to eq(true)
    end

    it "flags a logged-in search from a crawler user agent" do
      user = Fabricate(:user)
      _status, id =
        SearchLog.log(
          term: "ruby",
          search_type: :header,
          ip_address: "127.0.0.1",
          user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",
          user_id: user.id,
        )

      expect(SearchLog.find(id).crawler).to eq(true)
    end

    it "flags a search from a scripted client that is not a browser" do
      _status, id =
        SearchLog.log(
          term: "ruby",
          search_type: :header,
          ip_address: "127.0.0.1",
          user_agent: "python-requests/2.31.0",
        )

      expect(SearchLog.find(id).crawler).to eq(true)
    end

    it "records the pageview session the search was made from" do
      _status, id =
        SearchLog.log(
          term: "ruby",
          search_type: :header,
          ip_address: "127.0.0.1",
          session_id: "a" * (SearchLog::MAXIMUM_SESSION_ID_LENGTH + 10),
        )

      expect(SearchLog.find(id).session_id).to eq("a" * SearchLog::MAXIMUM_SESSION_ID_LENGTH)
    end

    it "does not flag an anonymous search that arrived without a user agent" do
      _status, id = SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.1")

      expect(SearchLog.find(id).crawler).to eq(false)
    end

    it "does not flag an anonymous search from a browser user agent" do
      _status, id =
        SearchLog.log(
          term: "ruby",
          search_type: :header,
          ip_address: "127.0.0.1",
          user_agent:
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
        )

      expect(SearchLog.find(id).crawler).to eq(false)
    end
  end

  describe ".flag_likely_crawlers!" do
    let(:session_id) { SecureRandom.hex(16) }
    let(:crawler_score) { CrawlerScorer::BOT_SCORE_THRESHOLD + 1 }

    def flag!
      described_class.flag_likely_crawlers!(window_start: 1.hour.ago, window_end: Time.now)
    end

    it "flags a search made from a session scored as a likely crawler" do
      Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)
      log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)

      flag!

      expect(log.reload.likely_crawler).to eq(true)
    end

    it "leaves a search whose session was scored below the crawler threshold" do
      Fabricate(
        :browser_pageview_event,
        session_id: session_id,
        score: CrawlerScorer::BOT_SCORE_THRESHOLD,
      )
      log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)

      flag!

      expect(log.reload.likely_crawler).to eq(false)
    end

    it "leaves a search that carries no session" do
      Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)
      log = Fabricate(:search_log, user: nil, session_id: nil, created_at: 30.minutes.ago)

      flag!

      expect(log.reload.likely_crawler).to eq(false)
    end

    it "leaves a search whose session has no scored pageviews at all" do
      Fabricate(:browser_pageview_event, session_id: SecureRandom.hex(16), score: crawler_score)
      log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)

      flag!

      expect(log.reload.likely_crawler).to eq(false)
    end

    it "flags a logged-in search whose session is scored as a likely crawler" do
      Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)
      log =
        Fabricate(
          :search_log,
          user: Fabricate(:user),
          ip_address: nil,
          session_id: session_id,
          created_at: 30.minutes.ago,
        )

      flag!

      expect(log.reload.likely_crawler).to eq(true)
    end

    it "leaves searches outside the window alone" do
      Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)
      log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 5.hours.ago)

      flag!

      expect(log.reload.likely_crawler).to eq(false)
    end
  end

  describe ".backfill_crawler!" do
    it "flags existing searches from known crawler user agents" do
      crawler =
        Fabricate(
          :search_log,
          user: nil,
          user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",
        )
      human = Fabricate(:search_log, user: nil, user_agent: "Mozilla/5.0 (Macintosh) Chrome/120.0")
      no_agent = Fabricate(:search_log, user: nil, user_agent: nil)
      blank_agent = Fabricate(:search_log, user: nil, user_agent: "")
      member = Fabricate(:search_log, user: Fabricate(:user), user_agent: "Googlebot/2.1")

      expect(described_class.backfill_crawler!).to eq(2)

      expect(crawler.reload.crawler).to eq(true)
      expect(human.reload.crawler).to eq(false)
      expect(no_agent.reload.crawler).to eq(false)
      expect(blank_agent.reload.crawler).to eq(false)
      expect(member.reload.crawler).to eq(true)
    end

    it "is idempotent" do
      Fabricate(:search_log, user: nil, user_agent: "Googlebot/2.1")

      described_class.backfill_crawler!

      expect(described_class.backfill_crawler!).to eq(0)
    end
  end

  describe ".term_details" do
    it "uses only the date for the period" do
      time = Time.utc(2019, 5, 23, 18, 15, 30)
      freeze_time(time)

      search_log = Fabricate(:search_log, created_at: time - 1.hour)
      search_log2 = Fabricate(:search_log, created_at: time + 1.hour)

      details = SearchLog.term_details(search_log.term, :daily)

      expect(details[:data].first[:y]).to eq(2)
    end

    it "correctly returns term details" do
      Fabricate(:search_log, term: "ruby")
      Fabricate(:search_log, term: "ruBy", user: Fabricate(:user))
      Fabricate(:search_log, term: "ruby core", ip_address: "127.0.0.3")

      Fabricate(
        :search_log,
        term: "ruBy",
        search_type: SearchLog.search_types[:full_page],
        ip_address: "127.0.0.2",
      )

      term_details = SearchLog.term_details("ruby")
      expect(term_details[:data][0][:y]).to eq(3)

      term_header_details = SearchLog.term_details("ruby", :all, :header)
      expect(term_header_details[:data][0][:y]).to eq(2)

      SearchLog
        .where("lower(term) = ?", "ruby")
        .where(ip_address: "127.0.0.2")
        .update_all(search_result_id: 24)

      term_click_through_details = SearchLog.term_details("ruby", :all, :click_through_only)
      expect(term_click_through_details[:period]).to eq("all")
      expect(term_click_through_details[:data][0][:y]).to eq(1)
    end

    it "returns only non-staff users' searches with the non_staff_only search type" do
      member = Fabricate(:user)
      admin = Fabricate(:admin)
      moderator = Fabricate(:moderator)
      Fabricate(:search_log, term: "ruby", user: member)
      Fabricate(:search_log, term: "ruby", user: admin)
      Fabricate(:search_log, term: "ruby", user: moderator)
      Fabricate(:search_log, term: "ruby", user: nil)

      expect(
        SearchLog.term_details("ruby", :weekly, :non_staff_only)[:data].sum { |point| point[:y] },
      ).to eq(1)
    end

    it "returns non-staff and anonymous browser searches minus crawlers with the human_only search type" do
      SiteSetting.improved_crawler_detection = true
      member = Fabricate(:user)
      Fabricate(:search_log, term: "ruby", user: member)
      Fabricate(:search_log, term: "ruby", user: Fabricate(:admin))
      Fabricate(:search_log, term: "ruby", user: Fabricate(:moderator))
      Fabricate(:search_log, term: "ruby", user: nil, session_id: "browser-session")
      Fabricate(:search_log, term: "ruby", user: nil)
      Fabricate(:search_log, term: "ruby", user: nil, session_id: "browser-session", crawler: true)

      expect(
        SearchLog.term_details("ruby", :weekly, :human_only)[:data].sum { |point| point[:y] },
      ).to eq(2)
    end
  end

  describe "trending" do
    fab!(:user)
    def log_search(term, **opts)
      SearchLog.log(
        term: term,
        search_type: :header,
        ip_address: "127.0.0.1",
        user_agent:
          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
        session_id: "browser-session",
        **opts,
      )
    end

    before do
      log_search("ruby")
      log_search("php")
      log_search("java")
      log_search("ruby", user_id: user.id)
      log_search("swift")
      log_search("ruby", ip_address: "127.0.0.2")
    end

    it "considers time period" do
      expect(SearchLog.trending.to_a.count).to eq(4)

      SearchLog.where(term: "swift").update_all(created_at: 1.year.ago)
      expect(SearchLog.trending(:monthly).to_a.count).to eq(3)
    end

    it "correctly returns trending data" do
      top_trending = SearchLog.trending.first
      expect(top_trending.term).to eq("ruby")
      expect(top_trending.searches).to eq(3)
      expect(top_trending.click_through).to eq(0)

      SearchLog.where(term: "ruby", ip_address: "127.0.0.1").update_all(search_result_id: 12)
      SearchLog.where(term: "ruby", user_id: user.id).update_all(search_result_id: 12)
      SearchLog.where(term: "ruby", ip_address: "127.0.0.2").update_all(search_result_id: 24)
      top_trending = SearchLog.trending.first
      expect(top_trending.click_through).to eq(3)
    end

    it "returns only non-staff users' searches with the non_staff_only search type" do
      admin = Fabricate(:admin)
      moderator = Fabricate(:moderator)
      Fabricate(:search_log, term: "admin-search", user: admin)
      Fabricate(:search_log, term: "moderator-search", user: moderator)
      Fabricate(:search_log, term: "anonymous-search", user: nil)

      results = SearchLog.trending(:all, :non_staff_only).to_a

      expect(results.map { |trend| [trend.term, trend.searches] }).to eq([["ruby", 1]])
    end

    it "returns non-staff and anonymous browser searches minus crawlers with the human_only search type" do
      SiteSetting.improved_crawler_detection = true
      Fabricate(:search_log, term: "admin-search", user: Fabricate(:admin))
      Fabricate(:search_log, term: "sessionless-search", user: nil)
      Fabricate(
        :search_log,
        term: "crawler-search",
        user: nil,
        session_id: "browser-session",
        crawler: true,
      )

      results = SearchLog.trending(:all, :human_only).to_a

      expect(results.map { |trend| [trend.term, trend.searches] }).to eq(
        [["ruby", 3], ["java", 1], ["php", 1], ["swift", 1]],
      )
    end

    it "falls back to members-only for human_only while crawler detection is disabled" do
      SiteSetting.improved_crawler_detection = false
      Fabricate(:search_log, term: "admin-search", user: Fabricate(:admin))
      Fabricate(:search_log, term: "anonymous-search", user: nil)

      results = SearchLog.trending(:all, :human_only).to_a

      expect(results.map { |trend| [trend.term, trend.searches] }).to eq([["ruby", 1]])
    end
  end

  describe "clean_up" do
    it "removes old logs" do
      SearchLog.log(term: "jawa", search_type: :header, ip_address: "127.0.0.1")
      SearchLog.log(term: "jedi", search_type: :header, ip_address: "127.0.0.1")
      SearchLog.log(term: "rey", search_type: :header, ip_address: "127.0.0.1")
      SearchLog.log(term: "finn", search_type: :header, ip_address: "127.0.0.1")

      SiteSetting.search_query_log_max_size = 5
      SearchLog.clean_up
      expect(SearchLog.count).to eq(4)

      SiteSetting.search_query_log_max_size = 2
      SearchLog.clean_up
      expect(SearchLog.count).to eq(2)
      expect(SearchLog.where(term: "rey").first).to be_present
      expect(SearchLog.where(term: "finn").first).to be_present
    end
  end
end