A platform for community discussion. Free, open, simple.
DEV: Add topic_noindex and category_noindex modifiers (#45465)
Arpit Jalanf82a27a8c223
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rbindex f85765af1e0..35c14003717 100644--- a/app/controllers/application_controller.rb+++ b/app/controllers/application_controller.rb@@ -1002,14 +1002,17 @@ class ApplicationController < ActionController::Basedef add_noindex_headerif request.get? && !response.headers["X-Robots-Tag"]- if SiteSetting.allow_index_in_robots_txt- response.headers["X-Robots-Tag"] = "noindex"- else- response.headers["X-Robots-Tag"] = "noindex, nofollow"- end+ response.headers["X-Robots-Tag"] = noindex_header_valueendend+ # "noindex", or "noindex, nofollow" when the whole site is kept out of+ # search engines, so a page-level noindex never drops the site-wide+ # directive.+ def noindex_header_value+ SiteSetting.allow_index_in_robots_txt ? "noindex" : "noindex, nofollow"+ end+def add_noindex_header_to_non_canonicalcanonical = @canonical_url || @default_canonicalif canonical.present? && canonical != request.url &&@@ -1018,6 +1021,23 @@ class ApplicationController < ActionController::Baseendend+ # Whether search engines are told not to index a topic. Core only keeps+ # unlisted topics out; the `topic_noindex` modifier receives that decision+ # and the topic, so a plugin can send noindex for other topics too. It runs+ # on every request that reaches the controller, so a topic that is flagged+ # or moved follows once the anonymous cache (one minute) has expired.+ def topic_noindex?(topic)+ DiscoursePluginRegistry.apply_modifier(:topic_noindex, !topic.visible, topic)+ end++ # Whether search engines are told not to index a topic list that loads a+ # category: its own pages, `?category=` lists and tag pages within it.+ # Core indexes them all; the `category_noindex` modifier receives `false`+ # and the category, so a plugin can send noindex for a category.+ def category_noindex?(category)+ DiscoursePluginRegistry.apply_modifier(:category_noindex, false, category)+ end+def set_cross_origin_opener_policy_headerif current_user.present? && SiteSetting.cross_origin_opener_unsafe_none_groups_map.any? &¤t_user.in_any_groups?(SiteSetting.cross_origin_opener_unsafe_none_groups_map)diff --git a/app/controllers/topics_controller.rb b/app/controllers/topics_controller.rbindex 600bd9b5b41..91f8482812a 100644--- a/app/controllers/topics_controller.rb+++ b/app/controllers/topics_controller.rb@@ -224,7 +224,7 @@ class TopicsController < ApplicationController@topic_view.draft = Draft.get(current_user, @topic_view.draft_key, @topic_view.draft_sequence)end- response.headers["X-Robots-Tag"] = "noindex" unless @topic_view.topic.visible+ response.headers["X-Robots-Tag"] = noindex_header_value if topic_noindex?(@topic_view.topic)canonical_url UrlHelper.absolute_without_cdn(@topic_view.canonical_path)# The canonical can be an embedding page, which has no translated versions.diff --git a/lib/topic_list_responder.rb b/lib/topic_list_responder.rbindex f994c03b387..52d1e94b6f3 100644--- a/lib/topic_list_responder.rb+++ b/lib/topic_list_responder.rb@@ -4,6 +4,10 @@ module TopicListResponderdef respond_with_list(list)discourse_expires_in 1.minute+ if list.category && category_noindex?(list.category)+ response.headers["X-Robots-Tag"] = noindex_header_value+ end+respond_to do |format|format.html do@list = listdiff --git a/spec/requests/list_controller_spec.rb b/spec/requests/list_controller_spec.rbindex 9792aa70272..a149bfd7fe0 100644--- a/spec/requests/list_controller_spec.rb+++ b/spec/requests/list_controller_spec.rb@@ -1960,6 +1960,80 @@ RSpec.describe ListController doendend+ describe "category_noindex modifier" do+ fab!(:category, :category_with_definition)+ fab!(:subcategory) { Fabricate(:category_with_definition, parent_category_id: category.id) }+ fab!(:topic_in_category) { Fabricate(:topic, category: category) }++ let(:plugin) { Plugin::Instance.new }+ let(:modifier_args) { [] }+ let(:noindex_category_ids) { [] }+ let(:modifier) do+ Proc.new do |noindex, modifier_category|+ modifier_args << [noindex, modifier_category]+ noindex || noindex_category_ids.include?(modifier_category.id)+ end+ end++ before { DiscoursePluginRegistry.register_modifier(plugin, :category_noindex, &modifier) }+ after { DiscoursePluginRegistry.unregister_modifier(plugin, :category_noindex, &modifier) }++ it "receives false and the category, and sends no noindex by default" do+ get "/c/#{category.slug}/#{category.id}", headers: { "HTTP_USER_AGENT" => "Googlebot" }++ expect(response.status).to eq(200)+ expect(modifier_args).to eq([[false, category]])+ expect(response.headers["X-Robots-Tag"]).to eq(nil)+ end++ it "receives the subcategory for its own lists" do+ get "/c/#{category.slug}/#{subcategory.slug}/#{subcategory.id}"++ expect(response.status).to eq(200)+ expect(modifier_args).to eq([[false, subcategory]])+ end++ it "lets a plugin send noindex on every topic list of the category" do+ SiteSetting.enable_markdown_endpoints = true+ noindex_category_ids << category.id++ [+ "/c/#{category.slug}/#{category.id}",+ "/c/#{category.slug}/#{category.id}?page=1",+ "/c/#{category.slug}/#{category.id}/none",+ "/c/#{category.slug}/#{category.id}/l/top",+ "/c/#{category.slug}/#{category.id}/l/latest.json",+ "/c/#{category.slug}/#{category.id}.md",+ "/latest?category=#{category.id}",+ "/top?category=#{category.slug}",+ ].each do |path|+ get path, headers: { "HTTP_USER_AGENT" => "Googlebot" }++ expect(response.status).to eq(200), path+ expect(response.headers["X-Robots-Tag"]).to eq("noindex"), path+ end+ end++ it "keeps the site-wide nofollow when the site is not indexable" do+ SiteSetting.allow_index_in_robots_txt = false+ noindex_category_ids << category.id++ get "/c/#{category.slug}/#{category.id}.json"++ expect(response.headers["X-Robots-Tag"]).to eq("noindex, nofollow")+ end++ it "is not consulted for topic lists outside a category" do+ noindex_category_ids << category.id++ get "/latest", headers: { "HTTP_USER_AGENT" => "Googlebot" }++ expect(response.status).to eq(200)+ expect(modifier_args).to be_empty+ expect(response.headers["X-Robots-Tag"]).to eq(nil)+ end+ end+describe "shared drafts" dofab!(:category1, :category)fab!(:category2, :category)diff --git a/spec/requests/tags_controller_spec.rb b/spec/requests/tags_controller_spec.rbindex 5261cba3895..c998a0313ac 100644--- a/spec/requests/tags_controller_spec.rb+++ b/spec/requests/tags_controller_spec.rb@@ -831,6 +831,64 @@ RSpec.describe TagsController doendend+ describe "category_noindex modifier" do+ fab!(:tag) { Fabricate(:tag, name: "test") }+ fab!(:topic_in_category) { Fabricate(:topic, category: category, tags: [tag]) }++ let(:plugin) { Plugin::Instance.new }+ let(:modifier_args) { [] }+ let(:noindex_category_ids) { [] }+ let(:modifier) do+ Proc.new do |noindex, modifier_category|+ modifier_args << [noindex, modifier_category]+ noindex || noindex_category_ids.include?(modifier_category.id)+ end+ end++ before { DiscoursePluginRegistry.register_modifier(plugin, :category_noindex, &modifier) }+ after { DiscoursePluginRegistry.unregister_modifier(plugin, :category_noindex, &modifier) }++ it "receives false and the category, and sends no noindex by default" do+ get "/tags/c/#{category.slug}/#{category.id}/#{tag.slug_for_url}/#{tag.id}",+ headers: {+ "HTTP_USER_AGENT" => "Googlebot",+ }++ expect(response.status).to eq(200)+ expect(modifier_args).to eq([[false, category]])+ expect(response.headers["X-Robots-Tag"]).to eq(nil)+ end++ it "lets a plugin send noindex on the tag pages within the category" do+ noindex_category_ids << category.id+ category_path = "/tags/c/#{category.slug}/#{category.id}"++ [+ "#{category_path}/#{tag.slug_for_url}/#{tag.id}",+ "#{category_path}/none/#{tag.slug_for_url}/#{tag.id}",+ "#{category_path}/#{tag.slug_for_url}/#{tag.id}/l/top",+ "#{category_path}/#{tag.slug_for_url}/#{tag.id}.json",+ "#{category_path}/#{tag.name}.json",+ "/tag/#{tag.slug_for_url}/#{tag.id}.json?category=#{category.id}",+ ].each do |path|+ get path, headers: { "HTTP_USER_AGENT" => "Googlebot" }++ expect(response.status).to eq(200), path+ expect(response.headers["X-Robots-Tag"]).to eq("noindex"), path+ end+ end++ it "is not consulted for tag pages outside a category" do+ noindex_category_ids << category.id++ get "/tag/#{tag.slug_for_url}/#{tag.id}", headers: { "HTTP_USER_AGENT" => "Googlebot" }++ expect(response.status).to eq(200)+ expect(modifier_args).to be_empty+ expect(response.headers["X-Robots-Tag"]).to eq(nil)+ end+ end+describe "#info" dofab!(:tag) { Fabricate(:tag, name: "test") }let(:synonym) { Fabricate(:tag, name: "synonym", target_tag: tag) }diff --git a/spec/requests/topics_controller_spec.rb b/spec/requests/topics_controller_spec.rbindex 276d961c31d..3d8c131b3e7 100644--- a/spec/requests/topics_controller_spec.rb+++ b/spec/requests/topics_controller_spec.rb@@ -4691,6 +4691,71 @@ RSpec.describe TopicsController doexpect(response.headers["X-Robots-Tag"]).to eq("noindex, nofollow")end+ describe "topic_noindex modifier" do+ fab!(:first_post) { Fabricate(:post, topic: topic) }+ fab!(:second_post) { Fabricate(:post, topic: topic) }++ let(:plugin) { Plugin::Instance.new }+ let(:modifier_args) { [] }+ let(:noindex_topic_ids) { [] }+ let(:modifier) do+ Proc.new do |noindex, modifier_topic|+ modifier_args << [noindex, modifier_topic]+ noindex || noindex_topic_ids.include?(modifier_topic.id)+ end+ end++ before { DiscoursePluginRegistry.register_modifier(plugin, :topic_noindex, &modifier) }+ after { DiscoursePluginRegistry.unregister_modifier(plugin, :topic_noindex, &modifier) }++ it "receives core's decision and the topic" do+ get "/t/#{topic.slug}/#{topic.id}.json"++ expect(response.status).to eq(200)+ expect(modifier_args).to eq([[false, topic]])+ expect(response.headers["X-Robots-Tag"]).to eq(nil)++ get "/t/#{invisible_topic.slug}/#{invisible_topic.id}.json"++ expect(response.status).to eq(200)+ expect(modifier_args.last).to eq([true, invisible_topic])+ expect(response.headers["X-Robots-Tag"]).to eq("noindex")+ end++ it "lets a plugin send noindex on every version of the topic page" do+ SiteSetting.enable_markdown_endpoints = true+ TopicView.stubs(:chunk_size).returns(1)+ noindex_topic_ids << topic.id++ [+ "/t/#{topic.slug}/#{topic.id}",+ "/t/#{topic.slug}/#{topic.id}?page=2",+ "/t/#{topic.slug}/#{topic.id}/2",+ "/t/#{topic.slug}/#{topic.id}/print",+ "/t/#{topic.slug}/#{topic.id}.json",+ "/t/#{topic.slug}/#{topic.id}.md",+ ].each do |path|+ get path, headers: { "HTTP_USER_AGENT" => "Googlebot" }++ expect(response.status).to eq(200), path+ expect(response.headers["X-Robots-Tag"]).to eq("noindex"), path+ end+ end++ it "keeps the site-wide nofollow when the site is not indexable" do+ SiteSetting.allow_index_in_robots_txt = false+ noindex_topic_ids << topic.id++ get "/t/#{topic.slug}/#{topic.id}.json"++ expect(response.headers["X-Robots-Tag"]).to eq("noindex, nofollow")++ get "/t/#{invisible_topic.slug}/#{invisible_topic.id}.json"++ expect(response.headers["X-Robots-Tag"]).to eq("noindex, nofollow")+ end+ end+it "doesn't store an incoming link when there's no referer" doexpect { get "/t/#{topic.id}.json" }.not_to change(IncomingLink, :count)expect(response.status).to eq(200)