Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Allow fallback home routes for crawlers for custom homepage sites (#41234)

Penar Musaraj99c63660d4b4
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 4336b25798b..5dd53c0f0fe 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -78,13 +78,7 @@ class ApplicationController < ActionController::Base
helper_method :show_browser_update?
def use_crawler_layout?
- @use_crawler_layout ||=
- request.user_agent && (request.media_type.blank? || request.media_type.include?("html")) &&
- !%w[json rss].include?(params[:format]) &&
- (
- has_escaped_fragment? || params.key?("print") || show_browser_update? ||
- CrawlerDetection.crawler?(request.user_agent, request.headers["HTTP_VIA"])
- )
+ @use_crawler_layout ||= CrawlerDetection.crawler_layout_request?(request)
end
helper_method :use_crawler_layout?
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index 946ad1e5ebb..7259db46558 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -2066,6 +2066,7 @@ en:
include_secure_categories_in_tag_counts: "When enabled, count of topics for a tag will include topics that are in read restricted categories for all users. When disabled, normal users are only shown a count of topics for a tag where all the topics are in public categories."
display_personal_messages_tag_counts: "When enabled, count of personal messages tagged with a given tag will be displayed."
top_menu: "Determine which items appear in the homepage navigation, and in what order."
+ custom_homepage_crawler_route: "Route shown to web crawlers when a custom homepage is enabled."
post_menu: "Configure the visibility and order of default post menu items. Additional items added by plugins or themes are managed separately and won’t appear in this list."
post_menu_hidden_items: "The menu items to hide by default in the post menu unless an expansion ellipsis is clicked on. Additional items added by plugins or themes are managed separately and won’t appear in this list."
share_links: "Determine which items appear on the share dialog, and in what order."
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 04e092fa51b..3383437c36c 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -306,6 +306,11 @@ basic:
upcoming_change_default_override:
upcoming_change: "enable_unified_new"
new_default: "latest|new|hot|categories"
+ custom_homepage_crawler_route:
+ type: enum
+ default: "latest"
+ choices: "TopMenu.crawler_homepage_choices"
+ area: "interface"
post_menu:
client: true
type: list
diff --git a/lib/crawler_detection.rb b/lib/crawler_detection.rb
index 632eec33148..b326da98669 100644
--- a/lib/crawler_detection.rb
+++ b/lib/crawler_detection.rb
@@ -69,6 +69,17 @@ module CrawlerDetection
user_agent.match?(matcher)
end
+ def self.crawler_layout_request?(request)
+ return false if request.blank?
+ return false if request.user_agent.blank?
+ return false if request.media_type.present? && !request.media_type.include?("html")
+ return false if %w[json rss].include?(request.params[:format].to_s)
+
+ (SiteSetting.enable_escaped_fragments? && request.params.key?("_escaped_fragment_")) ||
+ request.params.key?("print") || show_browser_update?(request.user_agent) ||
+ crawler?(request.user_agent, request.headers["HTTP_VIA"])
+ end
+
# Given a user_agent that returns true from crawler?, should its request be allowed?
def self.allow_crawler?(user_agent)
if SiteSetting.allowed_crawler_user_agents.blank? &&
diff --git a/lib/homepage_helper.rb b/lib/homepage_helper.rb
index 8c301970f55..66d105f7155 100644
--- a/lib/homepage_helper.rb
+++ b/lib/homepage_helper.rb
@@ -4,18 +4,28 @@ class HomepageHelper
def self.resolve(request = nil, current_user = nil)
return "blank" if !current_user && SiteSetting.login_required?
- return "custom" if ThemeModifierHelper.new(request: request).custom_homepage
+ if ThemeModifierHelper.new(request: request).custom_homepage
+ return custom_homepage_route(request)
+ end
- enabled = false
enabled =
DiscoursePluginRegistry.apply_modifier(
:custom_homepage_enabled,
- enabled,
+ false,
request: request,
current_user: current_user,
)
- return "custom" if enabled
+
+ return custom_homepage_route(request) if enabled
current_user ? SiteSetting.homepage : SiteSetting.anonymous_homepage
end
+
+ def self.custom_homepage_route(request)
+ if CrawlerDetection.crawler_layout_request?(request)
+ return SiteSetting.custom_homepage_crawler_route
+ end
+
+ "custom"
+ end
end
diff --git a/lib/top_menu.rb b/lib/top_menu.rb
index 96fa6aa30f9..3deac7ac903 100644
--- a/lib/top_menu.rb
+++ b/lib/top_menu.rb
@@ -12,4 +12,8 @@ module TopMenu
end
base
end
+
+ def self.crawler_homepage_choices
+ choices & (Discourse.anonymous_filters.map(&:to_s) + %w[new categories])
+ end
end
diff --git a/spec/lib/crawler_detection_spec.rb b/spec/lib/crawler_detection_spec.rb
index cf733e93174..1cc5248c5a9 100644
--- a/spec/lib/crawler_detection_spec.rb
+++ b/spec/lib/crawler_detection_spec.rb
@@ -94,6 +94,42 @@ RSpec.describe CrawlerDetection do
end
end
+ describe ".crawler_layout_request?" do
+ it "returns true for crawler HTML requests" do
+ request = ActionDispatch::TestRequest.create("HTTP_USER_AGENT" => "Googlebot")
+
+ expect(CrawlerDetection.crawler_layout_request?(request)).to eq(true)
+ end
+
+ it "returns false for JSON requests" do
+ request =
+ ActionDispatch::TestRequest.create(
+ "HTTP_USER_AGENT" => "Googlebot",
+ "action_dispatch.request.path_parameters" => {
+ format: "json",
+ },
+ )
+
+ expect(CrawlerDetection.crawler_layout_request?(request)).to eq(false)
+ end
+
+ it "honors the escaped fragments site setting" do
+ SiteSetting.enable_escaped_fragments = false
+ request =
+ ActionDispatch::TestRequest.create(
+ "HTTP_USER_AGENT" =>
+ "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36",
+ )
+ request.params["_escaped_fragment_"] = ""
+
+ expect(CrawlerDetection.crawler_layout_request?(request)).to eq(false)
+
+ SiteSetting.enable_escaped_fragments = true
+
+ expect(CrawlerDetection.crawler_layout_request?(request)).to eq(true)
+ end
+ end
+
describe ".allow_crawler?" do
it "returns true if allowlist and blocklist are blank" do
expect(
diff --git a/spec/lib/homepage_helper_spec.rb b/spec/lib/homepage_helper_spec.rb
index b35bde83c2a..029e29b2d91 100644
--- a/spec/lib/homepage_helper_spec.rb
+++ b/spec/lib/homepage_helper_spec.rb
@@ -8,12 +8,19 @@ RSpec.describe HomepageHelper do
expect(HomepageHelper.resolve).to eq("latest")
end
- context "when theme has a custom homepage" do
- before { ThemeModifierHelper.any_instance.expects(:custom_homepage).returns(true) }
+ context "when a theme has a custom homepage" do
+ before { ThemeModifierHelper.any_instance.stubs(:custom_homepage).returns(true) }
it "returns custom" do
expect(HomepageHelper.resolve).to eq("custom")
end
+
+ it "returns the configured crawler route for crawler requests" do
+ SiteSetting.custom_homepage_crawler_route = "categories"
+ request = ActionDispatch::TestRequest.create("HTTP_USER_AGENT" => "Googlebot")
+
+ expect(HomepageHelper.resolve(request)).to eq("categories")
+ end
end
context "when a plugin modifies the custom_homepage_enabled to true" do
diff --git a/spec/models/site_setting_spec.rb b/spec/models/site_setting_spec.rb
index 9ac6527e338..b7bfe9ed273 100644
--- a/spec/models/site_setting_spec.rb
+++ b/spec/models/site_setting_spec.rb
@@ -80,6 +80,20 @@ RSpec.describe SiteSetting do
end
end
+ describe "custom_homepage_crawler_route" do
+ it "allows public top menu routes" do
+ SiteSetting.custom_homepage_crawler_route = "categories"
+
+ expect(SiteSetting.custom_homepage_crawler_route).to eq("categories")
+ end
+
+ it "does not allow authenticated-only routes" do
+ expect { SiteSetting.custom_homepage_crawler_route = "bookmarks" }.to raise_error(
+ Discourse::InvalidParameters,
+ )
+ end
+ end
+
describe "min_redirected_to_top_period" do
context "when has_enough_top_topics" do
before do
diff --git a/spec/requests/home_page_controller_spec.rb b/spec/requests/home_page_controller_spec.rb
index 66107395373..148bf2a41d1 100644
--- a/spec/requests/home_page_controller_spec.rb
+++ b/spec/requests/home_page_controller_spec.rb
@@ -62,6 +62,18 @@ RSpec.describe HomePageController do
)
end
+ it "uses the configured crawler route when a custom homepage is enabled" do
+ ThemeModifierHelper.any_instance.stubs(:custom_homepage).returns(true)
+ SiteSetting.custom_homepage_crawler_route = "categories"
+ category = Fabricate(:category, name: "Crawler Category")
+
+ get "/", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(category.name)
+ expect(response.body).not_to include("crawler-view-anon-menu")
+ end
+
it "should not display the site description on another route" do
get "/top", headers: { "HTTP_USER_AGENT" => "Googlebot" }