Skip to content

A platform for community discussion. Free, open, simple.

DEV: Add topic_noindex and category_noindex modifiers (#45465)

Arpit Jalanf82a27a8c223
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index f85765af1e0..35c14003717 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -1002,14 +1002,17 @@ class ApplicationController < ActionController::Base
def add_noindex_header
if request.get? && !response.headers["X-Robots-Tag"]
- if SiteSetting.allow_index_in_robots_txt
- response.headers["X-Robots-Tag"] = "noindex"
- else
- response.headers["X-Robots-Tag"] = "noindex, nofollow"
- end
+ response.headers["X-Robots-Tag"] = noindex_header_value
end
end
+ # "noindex", or "noindex, nofollow" when the whole site is kept out of
+ # search engines, so a page-level noindex never drops the site-wide
+ # directive.
+ def noindex_header_value
+ SiteSetting.allow_index_in_robots_txt ? "noindex" : "noindex, nofollow"
+ end
+
def add_noindex_header_to_non_canonical
canonical = @canonical_url || @default_canonical
if canonical.present? && canonical != request.url &&
@@ -1018,6 +1021,23 @@ class ApplicationController < ActionController::Base
end
end
+ # Whether search engines are told not to index a topic. Core only keeps
+ # unlisted topics out; the `topic_noindex` modifier receives that decision
+ # and the topic, so a plugin can send noindex for other topics too. It runs
+ # on every request that reaches the controller, so a topic that is flagged
+ # or moved follows once the anonymous cache (one minute) has expired.
+ def topic_noindex?(topic)
+ DiscoursePluginRegistry.apply_modifier(:topic_noindex, !topic.visible, topic)
+ end
+
+ # Whether search engines are told not to index a topic list that loads a
+ # category: its own pages, `?category=` lists and tag pages within it.
+ # Core indexes them all; the `category_noindex` modifier receives `false`
+ # and the category, so a plugin can send noindex for a category.
+ def category_noindex?(category)
+ DiscoursePluginRegistry.apply_modifier(:category_noindex, false, category)
+ end
+
def set_cross_origin_opener_policy_header
if current_user.present? && SiteSetting.cross_origin_opener_unsafe_none_groups_map.any? &&
current_user.in_any_groups?(SiteSetting.cross_origin_opener_unsafe_none_groups_map)
diff --git a/app/controllers/topics_controller.rb b/app/controllers/topics_controller.rb
index 600bd9b5b41..91f8482812a 100644
--- a/app/controllers/topics_controller.rb
+++ b/app/controllers/topics_controller.rb
@@ -224,7 +224,7 @@ class TopicsController < ApplicationController
@topic_view.draft = Draft.get(current_user, @topic_view.draft_key, @topic_view.draft_sequence)
end
- response.headers["X-Robots-Tag"] = "noindex" unless @topic_view.topic.visible
+ response.headers["X-Robots-Tag"] = noindex_header_value if topic_noindex?(@topic_view.topic)
canonical_url UrlHelper.absolute_without_cdn(@topic_view.canonical_path)
# The canonical can be an embedding page, which has no translated versions.
diff --git a/lib/topic_list_responder.rb b/lib/topic_list_responder.rb
index f994c03b387..52d1e94b6f3 100644
--- a/lib/topic_list_responder.rb
+++ b/lib/topic_list_responder.rb
@@ -4,6 +4,10 @@ module TopicListResponder
def respond_with_list(list)
discourse_expires_in 1.minute
+ if list.category && category_noindex?(list.category)
+ response.headers["X-Robots-Tag"] = noindex_header_value
+ end
+
respond_to do |format|
format.html do
@list = list
diff --git a/spec/requests/list_controller_spec.rb b/spec/requests/list_controller_spec.rb
index 9792aa70272..a149bfd7fe0 100644
--- a/spec/requests/list_controller_spec.rb
+++ b/spec/requests/list_controller_spec.rb
@@ -1960,6 +1960,80 @@ RSpec.describe ListController do
end
end
+ describe "category_noindex modifier" do
+ fab!(:category, :category_with_definition)
+ fab!(:subcategory) { Fabricate(:category_with_definition, parent_category_id: category.id) }
+ fab!(:topic_in_category) { Fabricate(:topic, category: category) }
+
+ let(:plugin) { Plugin::Instance.new }
+ let(:modifier_args) { [] }
+ let(:noindex_category_ids) { [] }
+ let(:modifier) do
+ Proc.new do |noindex, modifier_category|
+ modifier_args << [noindex, modifier_category]
+ noindex || noindex_category_ids.include?(modifier_category.id)
+ end
+ end
+
+ before { DiscoursePluginRegistry.register_modifier(plugin, :category_noindex, &modifier) }
+ after { DiscoursePluginRegistry.unregister_modifier(plugin, :category_noindex, &modifier) }
+
+ it "receives false and the category, and sends no noindex by default" do
+ get "/c/#{category.slug}/#{category.id}", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+ expect(response.status).to eq(200)
+ expect(modifier_args).to eq([[false, category]])
+ expect(response.headers["X-Robots-Tag"]).to eq(nil)
+ end
+
+ it "receives the subcategory for its own lists" do
+ get "/c/#{category.slug}/#{subcategory.slug}/#{subcategory.id}"
+
+ expect(response.status).to eq(200)
+ expect(modifier_args).to eq([[false, subcategory]])
+ end
+
+ it "lets a plugin send noindex on every topic list of the category" do
+ SiteSetting.enable_markdown_endpoints = true
+ noindex_category_ids << category.id
+
+ [
+ "/c/#{category.slug}/#{category.id}",
+ "/c/#{category.slug}/#{category.id}?page=1",
+ "/c/#{category.slug}/#{category.id}/none",
+ "/c/#{category.slug}/#{category.id}/l/top",
+ "/c/#{category.slug}/#{category.id}/l/latest.json",
+ "/c/#{category.slug}/#{category.id}.md",
+ "/latest?category=#{category.id}",
+ "/top?category=#{category.slug}",
+ ].each do |path|
+ get path, headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+ expect(response.status).to eq(200), path
+ expect(response.headers["X-Robots-Tag"]).to eq("noindex"), path
+ end
+ end
+
+ it "keeps the site-wide nofollow when the site is not indexable" do
+ SiteSetting.allow_index_in_robots_txt = false
+ noindex_category_ids << category.id
+
+ get "/c/#{category.slug}/#{category.id}.json"
+
+ expect(response.headers["X-Robots-Tag"]).to eq("noindex, nofollow")
+ end
+
+ it "is not consulted for topic lists outside a category" do
+ noindex_category_ids << category.id
+
+ get "/latest", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+ expect(response.status).to eq(200)
+ expect(modifier_args).to be_empty
+ expect(response.headers["X-Robots-Tag"]).to eq(nil)
+ end
+ end
+
describe "shared drafts" do
fab!(:category1, :category)
fab!(:category2, :category)
diff --git a/spec/requests/tags_controller_spec.rb b/spec/requests/tags_controller_spec.rb
index 5261cba3895..c998a0313ac 100644
--- a/spec/requests/tags_controller_spec.rb
+++ b/spec/requests/tags_controller_spec.rb
@@ -831,6 +831,64 @@ RSpec.describe TagsController do
end
end
+ describe "category_noindex modifier" do
+ fab!(:tag) { Fabricate(:tag, name: "test") }
+ fab!(:topic_in_category) { Fabricate(:topic, category: category, tags: [tag]) }
+
+ let(:plugin) { Plugin::Instance.new }
+ let(:modifier_args) { [] }
+ let(:noindex_category_ids) { [] }
+ let(:modifier) do
+ Proc.new do |noindex, modifier_category|
+ modifier_args << [noindex, modifier_category]
+ noindex || noindex_category_ids.include?(modifier_category.id)
+ end
+ end
+
+ before { DiscoursePluginRegistry.register_modifier(plugin, :category_noindex, &modifier) }
+ after { DiscoursePluginRegistry.unregister_modifier(plugin, :category_noindex, &modifier) }
+
+ it "receives false and the category, and sends no noindex by default" do
+ get "/tags/c/#{category.slug}/#{category.id}/#{tag.slug_for_url}/#{tag.id}",
+ headers: {
+ "HTTP_USER_AGENT" => "Googlebot",
+ }
+
+ expect(response.status).to eq(200)
+ expect(modifier_args).to eq([[false, category]])
+ expect(response.headers["X-Robots-Tag"]).to eq(nil)
+ end
+
+ it "lets a plugin send noindex on the tag pages within the category" do
+ noindex_category_ids << category.id
+ category_path = "/tags/c/#{category.slug}/#{category.id}"
+
+ [
+ "#{category_path}/#{tag.slug_for_url}/#{tag.id}",
+ "#{category_path}/none/#{tag.slug_for_url}/#{tag.id}",
+ "#{category_path}/#{tag.slug_for_url}/#{tag.id}/l/top",
+ "#{category_path}/#{tag.slug_for_url}/#{tag.id}.json",
+ "#{category_path}/#{tag.name}.json",
+ "/tag/#{tag.slug_for_url}/#{tag.id}.json?category=#{category.id}",
+ ].each do |path|
+ get path, headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+ expect(response.status).to eq(200), path
+ expect(response.headers["X-Robots-Tag"]).to eq("noindex"), path
+ end
+ end
+
+ it "is not consulted for tag pages outside a category" do
+ noindex_category_ids << category.id
+
+ get "/tag/#{tag.slug_for_url}/#{tag.id}", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+ expect(response.status).to eq(200)
+ expect(modifier_args).to be_empty
+ expect(response.headers["X-Robots-Tag"]).to eq(nil)
+ end
+ end
+
describe "#info" do
fab!(:tag) { Fabricate(:tag, name: "test") }
let(:synonym) { Fabricate(:tag, name: "synonym", target_tag: tag) }
diff --git a/spec/requests/topics_controller_spec.rb b/spec/requests/topics_controller_spec.rb
index 276d961c31d..3d8c131b3e7 100644
--- a/spec/requests/topics_controller_spec.rb
+++ b/spec/requests/topics_controller_spec.rb
@@ -4691,6 +4691,71 @@ RSpec.describe TopicsController do
expect(response.headers["X-Robots-Tag"]).to eq("noindex, nofollow")
end
+ describe "topic_noindex modifier" do
+ fab!(:first_post) { Fabricate(:post, topic: topic) }
+ fab!(:second_post) { Fabricate(:post, topic: topic) }
+
+ let(:plugin) { Plugin::Instance.new }
+ let(:modifier_args) { [] }
+ let(:noindex_topic_ids) { [] }
+ let(:modifier) do
+ Proc.new do |noindex, modifier_topic|
+ modifier_args << [noindex, modifier_topic]
+ noindex || noindex_topic_ids.include?(modifier_topic.id)
+ end
+ end
+
+ before { DiscoursePluginRegistry.register_modifier(plugin, :topic_noindex, &modifier) }
+ after { DiscoursePluginRegistry.unregister_modifier(plugin, :topic_noindex, &modifier) }
+
+ it "receives core's decision and the topic" do
+ get "/t/#{topic.slug}/#{topic.id}.json"
+
+ expect(response.status).to eq(200)
+ expect(modifier_args).to eq([[false, topic]])
+ expect(response.headers["X-Robots-Tag"]).to eq(nil)
+
+ get "/t/#{invisible_topic.slug}/#{invisible_topic.id}.json"
+
+ expect(response.status).to eq(200)
+ expect(modifier_args.last).to eq([true, invisible_topic])
+ expect(response.headers["X-Robots-Tag"]).to eq("noindex")
+ end
+
+ it "lets a plugin send noindex on every version of the topic page" do
+ SiteSetting.enable_markdown_endpoints = true
+ TopicView.stubs(:chunk_size).returns(1)
+ noindex_topic_ids << topic.id
+
+ [
+ "/t/#{topic.slug}/#{topic.id}",
+ "/t/#{topic.slug}/#{topic.id}?page=2",
+ "/t/#{topic.slug}/#{topic.id}/2",
+ "/t/#{topic.slug}/#{topic.id}/print",
+ "/t/#{topic.slug}/#{topic.id}.json",
+ "/t/#{topic.slug}/#{topic.id}.md",
+ ].each do |path|
+ get path, headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+ expect(response.status).to eq(200), path
+ expect(response.headers["X-Robots-Tag"]).to eq("noindex"), path
+ end
+ end
+
+ it "keeps the site-wide nofollow when the site is not indexable" do
+ SiteSetting.allow_index_in_robots_txt = false
+ noindex_topic_ids << topic.id
+
+ get "/t/#{topic.slug}/#{topic.id}.json"
+
+ expect(response.headers["X-Robots-Tag"]).to eq("noindex, nofollow")
+
+ get "/t/#{invisible_topic.slug}/#{invisible_topic.id}.json"
+
+ expect(response.headers["X-Robots-Tag"]).to eq("noindex, nofollow")
+ end
+ end
+
it "doesn't store an incoming link when there's no referer" do
expect { get "/t/#{topic.id}.json" }.not_to change(IncomingLink, :count)
expect(response.status).to eq(200)