A platform for community discussion. Free, open, simple.
FIX: Keep sitemap pages complete and their lastmod dates accurate (#45152)
Arpit Jalance5f8cfc3207
diff --git a/app/controllers/sitemap_controller.rb b/app/controllers/sitemap_controller.rbindex 30fed7190f2..343f59bc9b6 100644--- a/app/controllers/sitemap_controller.rb+++ b/app/controllers/sitemap_controller.rb@@ -19,7 +19,7 @@ class SitemapController < ApplicationController@output =Rails.cache- .fetch("sitemap/#{sitemap.name}/#{sitemap.max_page_size}", expires_in: 24.hours) do+ .fetch(sitemap.page_cache_key, expires_in: 24.hours) do@topics = sitemap.topicsrender :page, content_type: "text/xml; charset=UTF-8"enddiff --git a/app/models/sitemap.rb b/app/models/sitemap.rbindex 88c3d7fb079..5b58166d153 100644--- a/app/models/sitemap.rb+++ b/app/models/sitemap.rb@@ -10,14 +10,10 @@ class Sitemap < ActiveRecord::Basenames_used.each { |name| touch(name) }- count = Category.where(read_restricted: false).sum(:topic_count)- max_page_size = SiteSetting.sitemap_page_size- size, mod = count.divmod(max_page_size)- size += 1 if mod > 0-- size.times do |index|- page_name = (index + 1).to_s- touch(page_name)+ numbered_pages_last_posted_at.each do |page_name, last_posted_at|+ sitemap = find_or_initialize_by(name: page_name)+ sitemap.update!(last_posted_at: last_posted_at, enabled: true)+ Rails.cache.delete(sitemap.page_cache_key) if sitemap.saved_change_to_last_posted_at?names_used << page_nameend@@ -29,6 +25,27 @@ class Sitemap < ActiveRecord::Basesitemap.update!(last_posted_at: sitemap.last_posted_topic || 3.days.ago, enabled: true)endend++ def indexable_topics+ Topic.where(visible: true).joins(:category).where(categories: { read_restricted: false })+ end++ private++ # One pass over the listed topics gives every page's newest topic, so the cost+ # grows with the number of topics rather than with topics times pages.+ def numbered_pages_last_posted_at+ page_number =+ "(ROW_NUMBER() OVER (ORDER BY topics.id) - 1) / #{SiteSetting.sitemap_page_size.to_i} + 1"+ pages = indexable_topics.select(:updated_at, "#{page_number} AS page")++ Topic+ .unscoped+ .from(pages, :pages)+ .group("pages.page")+ .pluck(Arel.sql("pages.page"), Arel.sql("MAX(pages.updated_at)"))+ .map { |page, last_posted_at| [page.to_s, last_posted_at] }+ endenddef topics@@ -49,11 +66,14 @@ class Sitemap < ActiveRecord::BaseSiteSetting.sitemap_page_sizeend+ def page_cache_key+ "sitemap/#{name}/#{max_page_size}"+ end+privatedef sitemap_topics- indexable_topics =- Topic.where(visible: true).joins(:category).where(categories: { read_restricted: false })+ indexable_topics = self.class.indexable_topicsif name == RECENT_SITEMAP_NAMEindexable_topics.where("bumped_at > ?", 3.days.ago).order(bumped_at: :desc)diff --git a/spec/models/sitemap_spec.rb b/spec/models/sitemap_spec.rbindex 9c831ab7015..54b837f201b 100644--- a/spec/models/sitemap_spec.rb+++ b/spec/models/sitemap_spec.rb@@ -47,6 +47,32 @@ RSpec.describe Sitemap doexpect(first_page).to be_nilend+ it "creates enough pages for every listed topic, including category definitions" do+ SiteSetting.sitemap_page_size = 1+ Fabricate(:category_with_definition)++ described_class.regenerate_sitemaps++ page_names =+ Sitemap+ .where(enabled: true)+ .where.not(name: [Sitemap::NEWS_SITEMAP_NAME, Sitemap::RECENT_SITEMAP_NAME])+ .pluck(:name)+ expect(page_names).to contain_exactly("1", "2")+ end++ it "sets each page's last_posted_at to the newest topic on that page" do+ SiteSetting.sitemap_page_size = 1+ newer_topic = Fabricate(:topic)+ topic.update_columns(updated_at: 2.days.ago)+ newer_topic.update_columns(updated_at: 1.day.ago)++ described_class.regenerate_sitemaps++ expect(Sitemap.find_by(name: "1").last_posted_at).to eq_time(topic.updated_at)+ expect(Sitemap.find_by(name: "2").last_posted_at).to eq_time(newer_topic.updated_at)+ end+it "disable empty pages" dounused_page = Sitemap.touch("10")diff --git a/spec/requests/sitemap_controller_spec.rb b/spec/requests/sitemap_controller_spec.rbindex b622be98581..669eee84924 100644--- a/spec/requests/sitemap_controller_spec.rb+++ b/spec/requests/sitemap_controller_spec.rb@@ -48,8 +48,6 @@ RSpec.describe SitemapController doenddescribe "#page" do- before { Discourse.cache.delete("sitemap/1/#{SiteSetting.sitemap_page_size}") }-it "returns a 404 if the sitemap doesn't exist" doget "/sitemap_999.xml"@@ -69,6 +67,23 @@ RSpec.describe SitemapController doexpect(loc).to eq("#{Discourse.base_url}/t/#{topic.slug}/#{topic.id}")expect(last_mod).to eq(topic.bumped_at.xmlschema)end++ it "refreshes a cached page once its newest topic changes" do+ topic = Fabricate(:topic)+ topic.update_columns(updated_at: 1.hour.ago)+ Sitemap.regenerate_sitemaps+ get "/sitemap_1.xml"++ new_topic = Fabricate(:topic)+ Sitemap.regenerate_sitemaps+ get "/sitemap_1.xml"++ locs = Nokogiri::XML::Document.parse(response.body).css("loc").map(&:text)+ expect(locs).to contain_exactly(+ "#{Discourse.base_url}/t/#{topic.slug}/#{topic.id}",+ "#{Discourse.base_url}/t/#{new_topic.slug}/#{new_topic.id}",+ )+ endenddescribe "#recent" do