A platform for community discussion. Free, open, simple.
FIX: Preserve RSS feed content when importing and expanding topics (#42808)
Régis Hanol1c1fa6abe256
diff --git a/app/models/topic.rb b/app/models/topic.rbindex 7fb7d7f75ec..54ed5496142 100644--- a/app/models/topic.rb+++ b/app/models/topic.rb@@ -1845,7 +1845,13 @@ class Topic < ActiveRecord::Baseenddef expandable_first_post?- SiteSetting.embed_truncate? && has_topic_embed?+ embed = topic_embed+ return false if embed.nil?++ # A nil state preserves site-wide behavior for embeds without recorded state.+ return SiteSetting.embed_truncate? if embed.content_truncated.nil?++ embed.content_truncated?enddef message_archived?(user)diff --git a/app/models/topic_embed.rb b/app/models/topic_embed.rbindex a294a88e604..74a16023b6d 100644--- a/app/models/topic_embed.rb+++ b/app/models/topic_embed.rb@@ -43,12 +43,30 @@ class TopicEmbed < ActiveRecord::Baseend# Import an article from a source (RSS/Atom/Other)- def self.import(user, url, title, contents, category_id: nil, cook_method: nil, tags: nil)+ def self.import(+ user,+ url,+ title,+ contents,+ category_id: nil,+ cook_method: nil,+ tags: nil,+ truncate: cook_method.nil?+ )return unless url =~ %r{\Ahttps?\://}- original_contents = contents.dup.truncate(EMBED_CONTENT_CACHE_MAX_LENGTH)- contents = first_paragraph_from(contents) if SiteSetting.embed_truncate && cook_method.nil?- contents ||= ""+ contents = contents.to_s+ original_contents = contents.truncate(EMBED_CONTENT_CACHE_MAX_LENGTH)+ content_truncated = false++ if SiteSetting.embed_truncate && truncate+ excerpt = first_paragraph_from(contents)+ if text_truncated?(contents, excerpt)+ contents = excerpt+ content_truncated = true+ end+ end+contents = contents.dup << imported_from_html(url)url = normalize_url(url)@@ -76,7 +94,7 @@ class TopicEmbed < ActiveRecord::Basetitle: title.presence || url,raw: absolutize_urls(url, contents),skip_validations: true,- cook_method: cook_method,+ cook_method:,category: category_id || eh.try(:category_id),tags: SiteSetting.tagging_enabled ? tags : nil,embed_url: url,@@ -84,22 +102,15 @@ class TopicEmbed < ActiveRecord::Base}create_args[:visible] = false if SiteSetting.import_embed_unlisted?- # always return `args` when using this modifier, e.g:- #- # plugin.register_modifier(:topic_embed_import_create_args) do |args|- # args[:title] = "MODIFIED: #{args[:title]}"- #- # args # returning args is important to prevent errors- # end+ # Modifiers that return nil preserve the default arguments.create_args =DiscoursePluginRegistry.apply_modifier(:topic_embed_import_create_args, create_args) ||create_argspost = PostCreator.create(user, create_args)- post.topic.topic_embed.update!(embed_content_cache: original_contents)+ post.topic.topic_embed.update!(embed_content_cache: original_contents, content_truncated:)endelse- absolutize_urls(url, contents)post = embed.postif eh = EmbeddableHost.record_for_url(url)@@ -107,35 +118,51 @@ class TopicEmbed < ActiveRecord::Baseuser = eh.user.presence || userend- # Update the topic if it changed- if post&.topic- if post.user != user- PostOwnerChanger.new(- post_ids: [post.id],- topic_id: post.topic_id,- new_owner: user,- acting_user: Discourse.system_user,- ).change_owner!-- # make sure the post returned has the right author- post.reload- end+ return post unless post&.topic++ if post.user != user+ PostOwnerChanger.new(+ post_ids: [post.id],+ topic_id: post.topic_id,+ new_owner: user,+ acting_user: Discourse.system_user,+ ).change_owner!- existing_tag_names = post.topic.tags.pluck(:name).sort- incoming_tag_names = Array(tags).map { |tag| tag.respond_to?(:name) ? tag.name : tag }.sort+ post.reload+ end- tags_changed = !tags.nil? && existing_tag_names != incoming_tag_names+ existing_tag_names = post.topic.tags.pluck(:name).sort+ incoming_tag_names = Array(tags).map { |tag| tag.respond_to?(:name) ? tag.name : tag }.sort- if (content_sha1 != embed.content_sha1) || (title && title != post&.topic&.title) ||- tags_changed- changes = { raw: absolutize_urls(url, contents) }+ tags_changed = !tags.nil? && existing_tag_names != incoming_tag_names+ content_changed = content_sha1 != embed.content_sha1+ title_changed = title.present? && title != post.topic.title+ cook_method_changed = !cook_method.nil? && cook_method != post.cook_method- changes[:tags] = incoming_tag_names if SiteSetting.tagging_enabled && tags_changed- changes[:title] = title if title.present?+ post.cook_method = cook_method if cook_method_changed- post.revise(user, changes, skip_validations: true, bypass_rate_limiter: true)- embed.update!(content_sha1: content_sha1, embed_content_cache: original_contents)- end+ if content_changed || title_changed || tags_changed+ changes = { raw: absolutize_urls(url, contents) }++ changes[:tags] = incoming_tag_names if SiteSetting.tagging_enabled && tags_changed+ changes[:title] = title if title_changed++ post.revise(user, changes, skip_validations: true, bypass_rate_limiter: true)+ end++ post.save! if cook_method_changed && post.has_changes_to_save?+ post.rebake! if cook_method_changed++ embed.assign_attributes(+ content_sha1:,+ embed_content_cache: original_contents,+ content_truncated:,+ )+ embed_changed = embed.changed?+ embed.save! if embed_changed++ if embed_changed || cook_method_changed+ Discourse.cache.delete(expanded_cache_key(post.topic_id))endend@@ -333,10 +360,10 @@ class TopicEmbed < ActiveRecord::Baseenddef self.first_paragraph_from(html)- doc = Nokogiri.HTML5(html)+ fragment = Nokogiri::HTML5.fragment(html)result = +""- doc+ fragment.css("p").each do |p|if p.text.present?@@ -346,27 +373,54 @@ class TopicEmbed < ActiveRecord::Baseendreturn result if result.present?- # If there is no first paragraph, return the first div (onebox)- doc.css("div").first.to_s+ # Oneboxes may contain a div without any paragraphs.+ first_div = fragment.at_css("div")+ return first_div.to_s if first_div++ return html.to_s.split(/\R[[:blank:]]*\R/, 2).first if fragment.element_children.empty?++ fragment.to_html+ end++ def self.text_truncated?(contents, excerpt)+ return false if excerpt.blank?++ Nokogiri::HTML5.fragment(contents).text.squish != Nokogiri::HTML5.fragment(excerpt).text.squishend+ private_class_method :text_truncated?def self.expanded_for(post)Discourse.cache- .fetch("embed-topic:#{post.topic_id}", expires_in: 10.minutes) do- url = TopicEmbed.where(topic_id: post.topic_id).pick(:embed_url)- response = TopicEmbed.find_remote(url)-- body = response.body- if post&.topic&.topic_embed && body.present?- post.topic.topic_embed.update!(- embed_content_cache: body.truncate(EMBED_CONTENT_CACHE_MAX_LENGTH),- )- end- body << TopicEmbed.imported_from_html(url)- body+ .fetch(expanded_cache_key(post.topic_id), expires_in: 10.minutes) do+ embed = post.topic.topic_embed+ raise Discourse::NotFound if embed.nil?++ body = content_for_expansion(embed).dup << imported_from_html(embed.embed_url)+ body = absolutize_urls(embed.embed_url, post.cook(body))++ post.cook_method == Post.cook_methods[:raw_html] ? PrettyText.sanitize(body) : bodyendend++ def self.content_for_expansion(embed)+ return embed.embed_content_cache if embed.embed_content_cache.present?++ raise Discourse::NotFound unless embed.content_truncated.nil?++ body = find_remote(embed.embed_url)&.body+ raise Discourse::NotFound if body.blank?++ body = body.truncate(EMBED_CONTENT_CACHE_MAX_LENGTH)+ embed.update!(embed_content_cache: body)+ body+ end+ private_class_method :content_for_expansion++ def self.expanded_cache_key(topic_id)+ "embed-topic:#{topic_id}"+ end+ private_class_method :expanded_cache_keyend# == Schema Information@@ -375,6 +429,7 @@ end## id :integer not null, primary key# content_sha1 :string(40)+# content_truncated :boolean# deleted_at :datetime# embed_content_cache :text# embed_url :string(1000) not nulldiff --git a/config/locales/client.en.yml b/config/locales/client.en.ymlindex 02adf30e594..07787233cea 100644--- a/config/locales/client.en.yml+++ b/config/locales/client.en.yml@@ -4918,7 +4918,7 @@ en:reply_as_new_private_message: "Reply as new message to the same recipients"continue_discussion: "Continuing the discussion from %{postLink}:"follow_quote: "go to the quoted post"- show_full: "Show Full Post"+ show_more: "Show more…"show_hidden: "View ignored content."deleted_by_author_simple: "(post deleted by author)"collapse: "collapse"diff --git a/db/migrate/20260821164114_add_content_truncated_to_topic_embeds.rb b/db/migrate/20260821164114_add_content_truncated_to_topic_embeds.rbnew file mode 100644index 00000000000..6f788092364--- /dev/null+++ b/db/migrate/20260821164114_add_content_truncated_to_topic_embeds.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++class AddContentTruncatedToTopicEmbeds < ActiveRecord::Migration[8.0]+ def change+ add_column :topic_embeds, :content_truncated, :boolean+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 02acbf00d25..9b77808f3fd 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -10212,7 +10212,8 @@ CREATE TABLE public.topic_embeds (updated_at timestamp without time zone NOT NULL,deleted_at timestamp without time zone,deleted_by_id integer,- embed_content_cache text+ embed_content_cache text,+ content_truncated boolean);@@ -23360,6 +23361,7 @@ INSERT INTO "schema_migrations" (version) VALUES('20260824091843'),('20260824072257'),('20260824051214'),+('20260821164114'),('20260820171539'),('20260820143851'),('20260818143417'),diff --git a/frontend/discourse/app/components/post.gjs b/frontend/discourse/app/components/post.gjsindex 40b4670cd17..bd4f1fa4215 100644--- a/frontend/discourse/app/components/post.gjs+++ b/frontend/discourse/app/components/post.gjs@@ -257,7 +257,7 @@ export default class Post extends Component {}@action- async expandFirstPost() {+ expandFirstPost() {this.expandedFirstPost = new TrackedAsyncData(this.args.post.expand());}@@ -598,7 +598,7 @@ export default class Post extends Component {@translatedLabel={{ifthis.expandedFirstPost.isPending(i18n "loading")- (concat (i18n "post.show_full") "...")+ (i18n "post.show_more")}}/>{{/if}}diff --git a/frontend/discourse/app/models/post.js b/frontend/discourse/app/models/post.jsindex 1323de21443..7ad1483202f 100644--- a/frontend/discourse/app/models/post.js+++ b/frontend/discourse/app/models/post.js@@ -566,8 +566,13 @@ export default class Post extends RestModel {// Expands the first post's content, if embedded and shortened.async expand() {- const post = await ajax(`/posts/${this.id}/expand-embed`);- this.cooked = `<section class="expanded-embed">${post.cooked}</section>`;+ try {+ const post = await ajax(`/posts/${this.id}/expand-embed`);+ this.cooked = `<section class="expanded-embed">${post.cooked}</section>`;+ } catch (error) {+ popupAjaxError.call(this, error);+ throw error;+ }}// Recover a deleted postdiff --git a/frontend/discourse/tests/integration/components/post/post-test.gjs b/frontend/discourse/tests/integration/components/post/post-test.gjsindex abc45fb4515..ae0cef93b9c 100644--- a/frontend/discourse/tests/integration/components/post/post-test.gjs+++ b/frontend/discourse/tests/integration/components/post/post-test.gjs@@ -8,6 +8,7 @@ import {} from "@ember/test-helpers";import hbs from "htmlbars-inline-precompile";import { module, test } from "qunit";+import sinon from "sinon";import Post from "discourse/components/post";import DMenus from "discourse/float-kit/components/d-menus";import { forceMobile, resetMobile } from "discourse/lib/mobile";@@ -678,10 +679,38 @@ module("Integration | Component | Post", function (hooks) {await renderComponent(this.post);+ assert+ .dom(".expand-post")+ .hasText(+ i18n("post.show_more"),+ "labels the feed-content expansion accurately"+ );await click(".topic-body .expand-post");assert.dom(".expand-post").doesNotExist("button is gone");});+ test("reports an error and keeps the expansion retryable", async function (assert) {+ const message = "Could not show more.";+ const dialog = getOwner(this).lookup("service:dialog");+ sinon.stub(dialog, "alert");+ pretender.get("/posts/:post_id/expand-embed", () =>+ response(422, { errors: [message] })+ );+ this.post.post_number = 1;+ this.post.topic.expandable_first_post = true;++ await renderComponent(this.post);+ await click(".topic-body .expand-post");++ assert.dom(".expand-post").exists("keeps the button available for retry");+ assert.true(+ dialog.alert.calledWith(+ i18n("generic_error_with_reason", { error: message })+ ),+ "shows the expansion error"+ );+ });+test("can't show admin menu when you can't manage", async function (assert) {this.currentUser.admin = false;diff --git a/lib/feed_item_accessor.rb b/lib/feed_item_accessor.rbindex 93c11c6b377..06fcc50acdc 100644--- a/lib/feed_item_accessor.rb+++ b/lib/feed_item_accessor.rb@@ -11,6 +11,11 @@ class FeedItemAccessortry_attribute_or_self(element(element_name), :content)end+ def element_type(element_name)+ value = element(element_name)+ value.type if value.respond_to?(:type)+ end+def linkif rss_item.respond_to?(:links)link = rss_item.links&.find { |l| l.rel == "alternate" && l.type == "text/html" }diff --git a/plugins/discourse-rss-polling/app/jobs/jobs/discourse_rss_polling/poll_feed.rb b/plugins/discourse-rss-polling/app/jobs/jobs/discourse_rss_polling/poll_feed.rbindex ce75dc85399..c8ff9c61382 100644--- a/plugins/discourse-rss-polling/app/jobs/jobs/discourse_rss_polling/poll_feed.rb+++ b/plugins/discourse-rss-polling/app/jobs/jobs/discourse_rss_polling/poll_feed.rb@@ -102,8 +102,6 @@ module Jobsnextend- cook_method = feed_item.is_youtube? ? Post.cook_methods[:regular] : nil-updated_tags = discourse_tagsupdated_tags = nil if already_imported&.key?(feed_item)@@ -113,10 +111,11 @@ module Jobsauthor,feed_item.url,feed_item.title,- CGI.unescapeHTML(feed_item.content),+ feed_item.decoded_content,category_id: discourse_category_id,tags: updated_tags,- cook_method: cook_method,+ cook_method: feed_item.cook_method,+ truncate: feed_item.truncate_content?,)rescue => eoutcomes << feed_item.outcome(status: :failed, reason: e.message.to_s.truncate(200))diff --git a/plugins/discourse-rss-polling/app/models/discourse_rss_polling/feed_item.rb b/plugins/discourse-rss-polling/app/models/discourse_rss_polling/feed_item.rbindex b03109e5684..07f9bb87747 100644--- a/plugins/discourse-rss-polling/app/models/discourse_rss_polling/feed_item.rb+++ b/plugins/discourse-rss-polling/app/models/discourse_rss_polling/feed_item.rb@@ -17,16 +17,30 @@ module DiscourseRssPollingdef contentreturn @content if defined?(@content)- @content =- if is_youtube?- url- else- value = nil- CONTENT_ELEMENT_TAG_NAMES.each do |tag_name|- break if value = @accessor.element_content(tag_name)- end- value&.force_encoding("UTF-8")&.scrub- end+ value = @accessor.element_content(content_element_name) if content_element_name+ value ||= url+ @content = value.to_s.dup.force_encoding("UTF-8").scrub if value+ end++ def decoded_content+ @decoded_content ||= CGI.unescapeHTML(content.to_s)+ end++ def cook_method+ return Post.cook_methods[:regular] if content_element_name.nil?++ case @accessor.element_type(content_element_name).to_s.downcase+ when "text", "text/plain"+ Post.cook_methods[:regular]+ when "html", "text/html", "xhtml", "application/xhtml+xml"+ nil+ else+ Post.cook_methods[:regular] if markdown_compatible?+ end+ end++ def truncate_content?+ Nokogiri::HTML5.fragment(decoded_content).text.squish.length > MINIMUM_TEXT_LENGTH_TO_TRUNCATEenddef title@@ -58,10 +72,6 @@ module DiscourseRssPolling@accessor.element_content(:itunes_image)&.hrefend- def is_youtube?- url&.starts_with?("https://www.youtube.com/watch")- end-def pubdatereturn @pubdate if defined?(@pubdate)@@ -97,5 +107,41 @@ module DiscourseRssPollingprivateCONTENT_ELEMENT_TAG_NAMES = %i[content_encoded content description summary]+ MINIMUM_TEXT_LENGTH_TO_TRUNCATE = 500+ MARKDOWN_COMPATIBLE_TAG_NAMES = %w[+ a+ abbr+ b+ br+ code+ del+ em+ i+ img+ ins+ kbd+ mark+ q+ s+ small+ span+ strong+ sub+ sup+ time+ u+ ]++ def content_element_name+ return @content_element_name if defined?(@content_element_name)++ @content_element_name =+ CONTENT_ELEMENT_TAG_NAMES.find { |tag_name| @accessor.element_content(tag_name).present? }+ end++ def markdown_compatible?+ tag_names = decoded_content.scan(%r{</?([a-z][a-z0-9-]*)(?:\s[^>]*)?/?>}i).flatten+ tag_names.all? { |tag_name| MARKDOWN_COMPATIBLE_TAG_NAMES.include?(tag_name.downcase) }+ endendenddiff --git a/plugins/discourse-rss-polling/spec/jobs/poll_feed_spec.rb b/plugins/discourse-rss-polling/spec/jobs/poll_feed_spec.rbindex c5d58bfe1c0..14b4da5d5d6 100644--- a/plugins/discourse-rss-polling/spec/jobs/poll_feed_spec.rb+++ b/plugins/discourse-rss-polling/spec/jobs/poll_feed_spec.rb@@ -29,6 +29,55 @@ RSpec.describe Jobs::DiscourseRssPolling::PollFeed do)end+ context "when inferring how to import feed content" do+ let(:raw_feed) { <<~XML }+ <?xml version="1.0" encoding="UTF-8"?>+ <rss version="2.0">+ <channel>+ <title>Example feed</title>+ <link>https://example.com</link>+ <description>Example feed</description>+ <item>+ <title>Feed item</title>+ <link>https://example.com/feed-item</link>+ <description><![CDATA[#{description}]]></description>+ </item>+ </channel>+ </rss>+ XML+ let(:post) do+ job.execute(feed_url: feed_url, author_username: author.username)+ author.topics.last.first_post+ end++ before { SiteSetting.embed_truncate = true }++ context "with an untyped Markdown description" do+ let(:description) { "**Meetup overview**\n\n#{"Talk details. " * 50}" }++ it "imports an excerpt and expands to the complete feed content" do+ expect(post.cook_method).to eq(Post.cook_methods[:regular])+ expect(post.raw).not_to include("Talk details")+ expect(post.cooked).to have_tag("strong", text: "Meetup overview")+ expect(post.topic.expandable_first_post?).to eq(true)+ expect(TopicEmbed.expanded_for(post)).to include("Talk details")+ end+ end++ context "with a short teaser" do+ let(:description) { <<~HTML }+ <h2>Legibility of effort</h2>+ <p>A short description of the article.</p>+ <a href="https://example.com/feed-item">Read the full post!</a>+ HTML++ it "imports the complete feed content without an expansion button" do+ expect(post.raw).to include("Read the full post!")+ expect(post.topic.expandable_first_post?).to eq(false)+ end+ end+ end+context "with use_pubdate set to false" dobefore doSiteSetting.rss_polling_use_pubdate = falsediff --git a/plugins/discourse-rss-polling/spec/models/feed_item_spec.rb b/plugins/discourse-rss-polling/spec/models/feed_item_spec.rbindex 30d8e8869de..4f03f23f3ec 100644--- a/plugins/discourse-rss-polling/spec/models/feed_item_spec.rb+++ b/plugins/discourse-rss-polling/spec/models/feed_item_spec.rb@@ -9,7 +9,9 @@ RSpec.describe DiscourseRssPolling::FeedItem doit { expect(feed_item.content).to eq(expected[:content]) }it { expect(feed_item.url).to eq(expected[:url]) }it { expect(feed_item.title).to eq(expected[:title]) }-+ if expected.key?(:cook_method)+ it { expect(feed_item.cook_method).to eq(expected[:cook_method]) }+ endit { expect(feed_item.categories).to eq(expected[:categories]) } if expected.key?(:categories)end@@ -27,6 +29,7 @@ RSpec.describe DiscourseRssPolling::FeedItem docontent: "<p>This is the body & content. </p>",url: "https://blog.discourse.org/2017/09/poll-feed-spec-fixture/",title: "Poll Feed Spec Fixture",+ cook_method: nil,)end@@ -40,6 +43,7 @@ RSpec.describe DiscourseRssPolling::FeedItem docontent: "Here are some random descriptions...",url: "https://blog.discourse.org/2017/09/poll-feed-spec-fixture/",title: "Wellington: “Progress is hard!” Other cities: “Hold my beer”",+ cook_method: nil,)end@@ -52,7 +56,15 @@ RSpec.describe DiscourseRssPolling::FeedItem docontent: "<p>This is the body & content. </p>",url: "https://blog.discourse.org/2017/09/poll-feed-spec-fixture/",title: "Poll Feed Spec Fixture",+ cook_method: nil,)++ it "uses regular cooking when the content is declared as text" do+ raw_feed_item.content.type = "text"+ feed_item = DiscourseRssPolling::FeedItem.new(raw_feed_item)++ expect(feed_item.cook_method).to eq(Post.cook_methods[:regular])+ endendcontext "with ATOM item with summary element" do@@ -65,6 +77,7 @@ RSpec.describe DiscourseRssPolling::FeedItem docontent: "Here are some random descriptions...",url: "https://blog.discourse.org/2017/09/poll-feed-spec-fixture/",title: "Poll Feed Spec Fixture",+ cook_method: nil,)end@@ -92,6 +105,7 @@ RSpec.describe DiscourseRssPolling::FeedItem docontent: "https://www.youtube.com/watch?v=K56soYl0U1w",url: "https://www.youtube.com/watch?v=K56soYl0U1w",title: "The Ramones - Blitzkrieg Bop (With Lyrics)",+ cook_method: Post.cook_methods[:regular],)end@@ -105,6 +119,48 @@ RSpec.describe DiscourseRssPolling::FeedItem docontent: "https://www.youtube.com/watch?v=peYYl2vrIt4",url: "https://www.youtube.com/watch?v=peYYl2vrIt4",title: "An Uncontroversial Opinion – AMD RX 6600 XT Announcement",+ cook_method: Post.cook_methods[:regular],)end++ describe "content inference" do+ let(:url) { "https://example.com/feed-item" }+ let(:raw_feed_item) { Struct.new(:description, :link).new(description, url) }+ let(:feed_item) { described_class.new(raw_feed_item) }++ context "with an untyped Markdown description" do+ let(:description) { "Let's gather again - **in person** & online." }++ it "uses regular cooking and decodes entities for import" do+ expect(feed_item.cook_method).to eq(Post.cook_methods[:regular])+ expect(feed_item.decoded_content).to eq("Let's gather again - **in person** & online.")+ expect(feed_item.truncate_content?).to eq(false)+ end+ end++ context "with a long description" do+ let(:description) { "Introduction\n\n#{"More details. " * 50}" }++ it "truncates content with more than 500 visible characters" do+ expect(feed_item.truncate_content?).to eq(true)+ end+ end++ context "with an image-only description" do+ let(:description) { "<img src='https://example.com/comic.png'>" }++ it "uses regular cooking" do+ expect(feed_item.cook_method).to eq(Post.cook_methods[:regular])+ end+ end++ context "without a description" do+ let(:description) { nil }++ it "uses the item URL as regular content" do+ expect(feed_item.content).to eq(url)+ expect(feed_item.cook_method).to eq(Post.cook_methods[:regular])+ end+ end+ endenddiff --git a/spec/lib/feed_item_accessor_spec.rb b/spec/lib/feed_item_accessor_spec.rbindex bbaaeb7d26d..07e461dd781 100644--- a/spec/lib/feed_item_accessor_spec.rb+++ b/spec/lib/feed_item_accessor_spec.rb@@ -12,6 +12,10 @@ RSpec.describe FeedItemAccessor doit { expect(item_accessor.element_content("title")).to eq(atom_feed_item.title.content) }end+ describe "#element_type" do+ it { expect(item_accessor.element_type("content")).to eq("html") }+ end+describe "#link" doit { expect(item_accessor.link).to eq(atom_feed_item.link.href) }end@@ -26,6 +30,10 @@ RSpec.describe FeedItemAccessor doit { expect(item_accessor.element_content("title")).to eq(rss_feed_item.title) }end+ describe "#element_type" do+ it { expect(item_accessor.element_type("content_encoded")).to be_nil }+ end+describe "#link" doit { expect(item_accessor.link).to eq(rss_feed_item.link) }enddiff --git a/spec/models/topic_embed_spec.rb b/spec/models/topic_embed_spec.rbindex dcbe34232df..171449e3d04 100644--- a/spec/models/topic_embed_spec.rb+++ b/spec/models/topic_embed_spec.rb@@ -111,9 +111,9 @@ RSpec.describe TopicEmbed do# It caches the embed contentexpect(post.topic.topic_embed.embed_content_cache).to eq(contents)+ expect(post.topic.topic_embed.content_truncated).to eq(false)# It converts relative URLs to absolute when expanded- stub_request(:get, url).to_return(status: 200, body: contents)expect(TopicEmbed.expanded_for(post)).to have_tag("img",with: {@@ -153,6 +153,12 @@ RSpec.describe TopicEmbed doexpect(topic_embed.embed_content_cache).to eq("new contents")end+ it "updates an explicitly selected cook method on reimport" do+ TopicEmbed.import(user, url, title, contents, cook_method: Post.cook_methods[:regular])++ expect(post.reload.cook_method).to eq(Post.cook_methods[:regular])+ end+it "Should leave uppercase Feed Entry URL untouched in content" docased_url = "http://eviltrout.com/ABCD"post = TopicEmbed.import(user, cased_url, title, "some random content")@@ -477,6 +483,7 @@ RSpec.describe TopicEmbed dopost = TopicEmbed.import(user, url, title, long_content)expect(post.raw).not_to include(long_content)+ expect(post.topic.topic_embed.content_truncated).to eq(true)endit "keeps everything in the imported post when truncation is disabled" do@@ -484,6 +491,7 @@ RSpec.describe TopicEmbed dopost = TopicEmbed.import(user, url, title, long_content)expect(post.raw).to include(long_content)+ expect(post.topic.topic_embed.content_truncated).to eq(false)endit "looks at first div when there is no paragraph" do@@ -494,6 +502,63 @@ RSpec.describe TopicEmbed doexpect(post.raw).to include("testing it")end++ it "keeps the complete markup when truncation would only remove a wrapper" do+ wrapped_content = "<div><p>#{"Complete paragraph. " * 30}</p></div>"+ SiteSetting.embed_truncate = true++ post = TopicEmbed.import(user, url, title, wrapped_content)++ expect(post.raw).to include(wrapped_content)+ expect(post.topic.topic_embed.content_truncated).to eq(false)+ end++ it "keeps image-only content" do+ SiteSetting.embed_truncate = true+ post = TopicEmbed.import(user, url, title, "<img src='/comic.png' alt='comic'>")++ expect(post.raw).to have_tag(+ "img",+ with: {+ src: "http://eviltrout.com/comic.png",+ alt: "comic",+ },+ )+ expect(post.topic.topic_embed.content_truncated).to eq(false)+ end++ it "truncates independently of the cook method" do+ SiteSetting.embed_truncate = true+ post =+ TopicEmbed.import(+ user,+ url,+ title,+ "**Meetup overview**\n\nTalk details",+ cook_method: Post.cook_methods[:regular],+ truncate: true,+ )++ expect(post.raw).not_to include("Talk details")+ expect(post.cooked).to have_tag("strong", text: "Meetup overview")+ expect(post.topic.topic_embed.content_truncated).to eq(true)+ end++ it "refreshes expanded content when the excerpt is unchanged" do+ SiteSetting.embed_truncate = true+ first_paragraph = "<p>#{"Same introduction. " * 10}</p>"+ original_contents = "#{first_paragraph}<p>Original ending.</p>"+ updated_contents = "#{first_paragraph}<p>Updated ending.</p>"++ post = TopicEmbed.import(user, url, title, original_contents)+ expect(TopicEmbed.expanded_for(post)).to include("Original ending.")++ TopicEmbed.import(user, url, title, updated_contents)++ expect(post.reload.raw).not_to include("Updated ending.")+ expect(post.topic.topic_embed.reload.embed_content_cache).to eq(updated_contents)+ expect(TopicEmbed.expanded_for(post)).to include("Updated ending.")+ endendend@@ -853,19 +918,67 @@ RSpec.describe TopicEmbed dofab!(:tag)it "returns embed content" do- stub_request(:get, url).to_return(status: 200, body: contents)+ stub_request(:get, url).to_return(status: 200, body: "<p>remote content</p>")+ post = TopicEmbed.import(user, url, title, contents)+ expanded = TopicEmbed.expanded_for(post)++ expect(expanded).to include(contents)+ expect(expanded).not_to include("remote content")+ expect(WebMock).not_to have_requested(:get, url)+ end++ it "falls back to the remote page for a legacy embed without cached content" do+ remote_contents =+ "<html><body><article><p>Content from the remote page.</p></article></body></html>"+ stub_request(:get, url).to_return(status: 200, body: remote_contents)post = TopicEmbed.import(user, url, title, contents)- expect(TopicEmbed.expanded_for(post)).to include(contents)+ post.topic.topic_embed.update!(embed_content_cache: nil, content_truncated: nil)++ expect(TopicEmbed.expanded_for(post)).to include("Content from the remote page.")+ expect(post.topic.topic_embed.reload.embed_content_cache).to include(+ "Content from the remote page.",+ )end- it "updates the embed content cache" do- stub_request(:get, url)- .to_return(status: 200, body: contents)- .then- .to_return(status: 200, body: "contents changed")+ it "does not fetch the remote page for an imported item with missing cached content" do+ stub_request(:get, url).to_return(status: 200, body: "<p>remote content</p>")post = TopicEmbed.import(user, url, title, contents)- TopicEmbed.expanded_for(post)- expect(post.topic.topic_embed.reload.embed_content_cache).to include("contents changed")+ post.topic.topic_embed.update!(embed_content_cache: nil, content_truncated: true)++ expect { TopicEmbed.expanded_for(post) }.to raise_error(Discourse::NotFound)+ expect(WebMock).not_to have_requested(:get, url)+ end++ it "cooks cached Markdown using the post cook method" do+ post =+ TopicEmbed.import(+ user,+ url,+ title,+ "**Meetup overview**\n\nTalk details",+ cook_method: Post.cook_methods[:regular],+ truncate: true,+ )++ expanded = TopicEmbed.expanded_for(post)++ expect(expanded).to have_tag("strong", text: "Meetup overview")+ expect(expanded).to include("Talk details")+ end++ it "sanitizes cached raw HTML" do+ post =+ TopicEmbed.import(+ user,+ url,+ title,+ "<p>Safe content</p><script>window.evil = true</script>",+ )++ expanded = TopicEmbed.expanded_for(post)++ expect(expanded).to include("Safe content")+ expect(expanded).not_to have_tag("script")endendenddiff --git a/spec/models/topic_spec.rb b/spec/models/topic_spec.rbindex f6824d62f8c..5838aaa5e00 100644--- a/spec/models/topic_spec.rb+++ b/spec/models/topic_spec.rb@@ -3197,31 +3197,32 @@ RSpec.describe Topic doenddescribe "expandable_first_post?" do- let(:topic) { Fabricate.build(:topic) }+ it "is false without a topic embed" do+ expect(Fabricate.build(:topic).expandable_first_post?).to eq(false)+ end++ it "uses the recorded truncation state when available" do+ post = Fabricate(:post)+ embed = Fabricate(:topic_embed, post:, content_truncated: true)+ SiteSetting.embed_truncate = false++ expect(post.topic.expandable_first_post?).to eq(true)++ embed.update!(content_truncated: false)+ SiteSetting.embed_truncate = true- it "is false if embeddable_host is blank" do- expect(topic.expandable_first_post?).to eq(false)+ expect(post.topic.expandable_first_post?).to eq(false)end- describe "with an embeddable host" do- before do- Fabricate(:embeddable_host)- SiteSetting.embed_truncate = true- topic.stubs(:has_topic_embed?).returns(true)- end+ it "uses the site setting for legacy embeds with unknown truncation state" do+ post = Fabricate(:post)+ Fabricate(:topic_embed, post:, content_truncated: nil)- it "is true with the correct settings and topic_embed" do- expect(topic.expandable_first_post?).to eq(true)- end- it "is false if embed_truncate? is false" do- SiteSetting.embed_truncate = false- expect(topic.expandable_first_post?).to eq(false)- end+ SiteSetting.embed_truncate = true+ expect(post.topic.expandable_first_post?).to eq(true)- it "is false if has_topic_embed? is false" do- topic.stubs(:has_topic_embed?).returns(false)- expect(topic.expandable_first_post?).to eq(false)- end+ SiteSetting.embed_truncate = false+ expect(post.topic.expandable_first_post?).to eq(false)endend