Skip to content

A platform for community discussion. Free, open, simple.

FIX: Keep sitemap pages complete and their lastmod dates accurate (#45152)

Arpit Jalance5f8cfc3207
diff --git a/app/controllers/sitemap_controller.rb b/app/controllers/sitemap_controller.rb
index 30fed7190f2..343f59bc9b6 100644
--- a/app/controllers/sitemap_controller.rb
+++ b/app/controllers/sitemap_controller.rb
@@ -19,7 +19,7 @@ class SitemapController < ApplicationController
@output =
Rails
.cache
- .fetch("sitemap/#{sitemap.name}/#{sitemap.max_page_size}", expires_in: 24.hours) do
+ .fetch(sitemap.page_cache_key, expires_in: 24.hours) do
@topics = sitemap.topics
render :page, content_type: "text/xml; charset=UTF-8"
end
diff --git a/app/models/sitemap.rb b/app/models/sitemap.rb
index 88c3d7fb079..5b58166d153 100644
--- a/app/models/sitemap.rb
+++ b/app/models/sitemap.rb
@@ -10,14 +10,10 @@ class Sitemap < ActiveRecord::Base
names_used.each { |name| touch(name) }
- count = Category.where(read_restricted: false).sum(:topic_count)
- max_page_size = SiteSetting.sitemap_page_size
- size, mod = count.divmod(max_page_size)
- size += 1 if mod > 0
-
- size.times do |index|
- page_name = (index + 1).to_s
- touch(page_name)
+ numbered_pages_last_posted_at.each do |page_name, last_posted_at|
+ sitemap = find_or_initialize_by(name: page_name)
+ sitemap.update!(last_posted_at: last_posted_at, enabled: true)
+ Rails.cache.delete(sitemap.page_cache_key) if sitemap.saved_change_to_last_posted_at?
names_used << page_name
end
@@ -29,6 +25,27 @@ class Sitemap < ActiveRecord::Base
sitemap.update!(last_posted_at: sitemap.last_posted_topic || 3.days.ago, enabled: true)
end
end
+
+ def indexable_topics
+ Topic.where(visible: true).joins(:category).where(categories: { read_restricted: false })
+ end
+
+ private
+
+ # One pass over the listed topics gives every page's newest topic, so the cost
+ # grows with the number of topics rather than with topics times pages.
+ def numbered_pages_last_posted_at
+ page_number =
+ "(ROW_NUMBER() OVER (ORDER BY topics.id) - 1) / #{SiteSetting.sitemap_page_size.to_i} + 1"
+ pages = indexable_topics.select(:updated_at, "#{page_number} AS page")
+
+ Topic
+ .unscoped
+ .from(pages, :pages)
+ .group("pages.page")
+ .pluck(Arel.sql("pages.page"), Arel.sql("MAX(pages.updated_at)"))
+ .map { |page, last_posted_at| [page.to_s, last_posted_at] }
+ end
end
def topics
@@ -49,11 +66,14 @@ class Sitemap < ActiveRecord::Base
SiteSetting.sitemap_page_size
end
+ def page_cache_key
+ "sitemap/#{name}/#{max_page_size}"
+ end
+
private
def sitemap_topics
- indexable_topics =
- Topic.where(visible: true).joins(:category).where(categories: { read_restricted: false })
+ indexable_topics = self.class.indexable_topics
if name == RECENT_SITEMAP_NAME
indexable_topics.where("bumped_at > ?", 3.days.ago).order(bumped_at: :desc)
diff --git a/spec/models/sitemap_spec.rb b/spec/models/sitemap_spec.rb
index 9c831ab7015..54b837f201b 100644
--- a/spec/models/sitemap_spec.rb
+++ b/spec/models/sitemap_spec.rb
@@ -47,6 +47,32 @@ RSpec.describe Sitemap do
expect(first_page).to be_nil
end
+ it "creates enough pages for every listed topic, including category definitions" do
+ SiteSetting.sitemap_page_size = 1
+ Fabricate(:category_with_definition)
+
+ described_class.regenerate_sitemaps
+
+ page_names =
+ Sitemap
+ .where(enabled: true)
+ .where.not(name: [Sitemap::NEWS_SITEMAP_NAME, Sitemap::RECENT_SITEMAP_NAME])
+ .pluck(:name)
+ expect(page_names).to contain_exactly("1", "2")
+ end
+
+ it "sets each page's last_posted_at to the newest topic on that page" do
+ SiteSetting.sitemap_page_size = 1
+ newer_topic = Fabricate(:topic)
+ topic.update_columns(updated_at: 2.days.ago)
+ newer_topic.update_columns(updated_at: 1.day.ago)
+
+ described_class.regenerate_sitemaps
+
+ expect(Sitemap.find_by(name: "1").last_posted_at).to eq_time(topic.updated_at)
+ expect(Sitemap.find_by(name: "2").last_posted_at).to eq_time(newer_topic.updated_at)
+ end
+
it "disable empty pages" do
unused_page = Sitemap.touch("10")
diff --git a/spec/requests/sitemap_controller_spec.rb b/spec/requests/sitemap_controller_spec.rb
index b622be98581..669eee84924 100644
--- a/spec/requests/sitemap_controller_spec.rb
+++ b/spec/requests/sitemap_controller_spec.rb
@@ -48,8 +48,6 @@ RSpec.describe SitemapController do
end
describe "#page" do
- before { Discourse.cache.delete("sitemap/1/#{SiteSetting.sitemap_page_size}") }
-
it "returns a 404 if the sitemap doesn't exist" do
get "/sitemap_999.xml"
@@ -69,6 +67,23 @@ RSpec.describe SitemapController do
expect(loc).to eq("#{Discourse.base_url}/t/#{topic.slug}/#{topic.id}")
expect(last_mod).to eq(topic.bumped_at.xmlschema)
end
+
+ it "refreshes a cached page once its newest topic changes" do
+ topic = Fabricate(:topic)
+ topic.update_columns(updated_at: 1.hour.ago)
+ Sitemap.regenerate_sitemaps
+ get "/sitemap_1.xml"
+
+ new_topic = Fabricate(:topic)
+ Sitemap.regenerate_sitemaps
+ get "/sitemap_1.xml"
+
+ locs = Nokogiri::XML::Document.parse(response.body).css("loc").map(&:text)
+ expect(locs).to contain_exactly(
+ "#{Discourse.base_url}/t/#{topic.slug}/#{topic.id}",
+ "#{Discourse.base_url}/t/#{new_topic.slug}/#{new_topic.id}",
+ )
+ end
end
describe "#recent" do