Skip to content

A platform for community discussion. Free, open, simple.

FIX: Preserve RSS feed content when importing and expanding topics (#42808)

Régis Hanol1c1fa6abe256
diff --git a/app/models/topic.rb b/app/models/topic.rb
index 7fb7d7f75ec..54ed5496142 100644
--- a/app/models/topic.rb
+++ b/app/models/topic.rb
@@ -1845,7 +1845,13 @@ class Topic < ActiveRecord::Base
end
def expandable_first_post?
- SiteSetting.embed_truncate? && has_topic_embed?
+ embed = topic_embed
+ return false if embed.nil?
+
+ # A nil state preserves site-wide behavior for embeds without recorded state.
+ return SiteSetting.embed_truncate? if embed.content_truncated.nil?
+
+ embed.content_truncated?
end
def message_archived?(user)
diff --git a/app/models/topic_embed.rb b/app/models/topic_embed.rb
index a294a88e604..74a16023b6d 100644
--- a/app/models/topic_embed.rb
+++ b/app/models/topic_embed.rb
@@ -43,12 +43,30 @@ class TopicEmbed < ActiveRecord::Base
end
# Import an article from a source (RSS/Atom/Other)
- def self.import(user, url, title, contents, category_id: nil, cook_method: nil, tags: nil)
+ def self.import(
+ user,
+ url,
+ title,
+ contents,
+ category_id: nil,
+ cook_method: nil,
+ tags: nil,
+ truncate: cook_method.nil?
+ )
return unless url =~ %r{\Ahttps?\://}
- original_contents = contents.dup.truncate(EMBED_CONTENT_CACHE_MAX_LENGTH)
- contents = first_paragraph_from(contents) if SiteSetting.embed_truncate && cook_method.nil?
- contents ||= ""
+ contents = contents.to_s
+ original_contents = contents.truncate(EMBED_CONTENT_CACHE_MAX_LENGTH)
+ content_truncated = false
+
+ if SiteSetting.embed_truncate && truncate
+ excerpt = first_paragraph_from(contents)
+ if text_truncated?(contents, excerpt)
+ contents = excerpt
+ content_truncated = true
+ end
+ end
+
contents = contents.dup << imported_from_html(url)
url = normalize_url(url)
@@ -76,7 +94,7 @@ class TopicEmbed < ActiveRecord::Base
title: title.presence || url,
raw: absolutize_urls(url, contents),
skip_validations: true,
- cook_method: cook_method,
+ cook_method:,
category: category_id || eh.try(:category_id),
tags: SiteSetting.tagging_enabled ? tags : nil,
embed_url: url,
@@ -84,22 +102,15 @@ class TopicEmbed < ActiveRecord::Base
}
create_args[:visible] = false if SiteSetting.import_embed_unlisted?
- # always return `args` when using this modifier, e.g:
- #
- # plugin.register_modifier(:topic_embed_import_create_args) do |args|
- # args[:title] = "MODIFIED: #{args[:title]}"
- #
- # args # returning args is important to prevent errors
- # end
+ # Modifiers that return nil preserve the default arguments.
create_args =
DiscoursePluginRegistry.apply_modifier(:topic_embed_import_create_args, create_args) ||
create_args
post = PostCreator.create(user, create_args)
- post.topic.topic_embed.update!(embed_content_cache: original_contents)
+ post.topic.topic_embed.update!(embed_content_cache: original_contents, content_truncated:)
end
else
- absolutize_urls(url, contents)
post = embed.post
if eh = EmbeddableHost.record_for_url(url)
@@ -107,35 +118,51 @@ class TopicEmbed < ActiveRecord::Base
user = eh.user.presence || user
end
- # Update the topic if it changed
- if post&.topic
- if post.user != user
- PostOwnerChanger.new(
- post_ids: [post.id],
- topic_id: post.topic_id,
- new_owner: user,
- acting_user: Discourse.system_user,
- ).change_owner!
-
- # make sure the post returned has the right author
- post.reload
- end
+ return post unless post&.topic
+
+ if post.user != user
+ PostOwnerChanger.new(
+ post_ids: [post.id],
+ topic_id: post.topic_id,
+ new_owner: user,
+ acting_user: Discourse.system_user,
+ ).change_owner!
- existing_tag_names = post.topic.tags.pluck(:name).sort
- incoming_tag_names = Array(tags).map { |tag| tag.respond_to?(:name) ? tag.name : tag }.sort
+ post.reload
+ end
- tags_changed = !tags.nil? && existing_tag_names != incoming_tag_names
+ existing_tag_names = post.topic.tags.pluck(:name).sort
+ incoming_tag_names = Array(tags).map { |tag| tag.respond_to?(:name) ? tag.name : tag }.sort
- if (content_sha1 != embed.content_sha1) || (title && title != post&.topic&.title) ||
- tags_changed
- changes = { raw: absolutize_urls(url, contents) }
+ tags_changed = !tags.nil? && existing_tag_names != incoming_tag_names
+ content_changed = content_sha1 != embed.content_sha1
+ title_changed = title.present? && title != post.topic.title
+ cook_method_changed = !cook_method.nil? && cook_method != post.cook_method
- changes[:tags] = incoming_tag_names if SiteSetting.tagging_enabled && tags_changed
- changes[:title] = title if title.present?
+ post.cook_method = cook_method if cook_method_changed
- post.revise(user, changes, skip_validations: true, bypass_rate_limiter: true)
- embed.update!(content_sha1: content_sha1, embed_content_cache: original_contents)
- end
+ if content_changed || title_changed || tags_changed
+ changes = { raw: absolutize_urls(url, contents) }
+
+ changes[:tags] = incoming_tag_names if SiteSetting.tagging_enabled && tags_changed
+ changes[:title] = title if title_changed
+
+ post.revise(user, changes, skip_validations: true, bypass_rate_limiter: true)
+ end
+
+ post.save! if cook_method_changed && post.has_changes_to_save?
+ post.rebake! if cook_method_changed
+
+ embed.assign_attributes(
+ content_sha1:,
+ embed_content_cache: original_contents,
+ content_truncated:,
+ )
+ embed_changed = embed.changed?
+ embed.save! if embed_changed
+
+ if embed_changed || cook_method_changed
+ Discourse.cache.delete(expanded_cache_key(post.topic_id))
end
end
@@ -333,10 +360,10 @@ class TopicEmbed < ActiveRecord::Base
end
def self.first_paragraph_from(html)
- doc = Nokogiri.HTML5(html)
+ fragment = Nokogiri::HTML5.fragment(html)
result = +""
- doc
+ fragment
.css("p")
.each do |p|
if p.text.present?
@@ -346,27 +373,54 @@ class TopicEmbed < ActiveRecord::Base
end
return result if result.present?
- # If there is no first paragraph, return the first div (onebox)
- doc.css("div").first.to_s
+ # Oneboxes may contain a div without any paragraphs.
+ first_div = fragment.at_css("div")
+ return first_div.to_s if first_div
+
+ return html.to_s.split(/\R[[:blank:]]*\R/, 2).first if fragment.element_children.empty?
+
+ fragment.to_html
+ end
+
+ def self.text_truncated?(contents, excerpt)
+ return false if excerpt.blank?
+
+ Nokogiri::HTML5.fragment(contents).text.squish != Nokogiri::HTML5.fragment(excerpt).text.squish
end
+ private_class_method :text_truncated?
def self.expanded_for(post)
Discourse
.cache
- .fetch("embed-topic:#{post.topic_id}", expires_in: 10.minutes) do
- url = TopicEmbed.where(topic_id: post.topic_id).pick(:embed_url)
- response = TopicEmbed.find_remote(url)
-
- body = response.body
- if post&.topic&.topic_embed && body.present?
- post.topic.topic_embed.update!(
- embed_content_cache: body.truncate(EMBED_CONTENT_CACHE_MAX_LENGTH),
- )
- end
- body << TopicEmbed.imported_from_html(url)
- body
+ .fetch(expanded_cache_key(post.topic_id), expires_in: 10.minutes) do
+ embed = post.topic.topic_embed
+ raise Discourse::NotFound if embed.nil?
+
+ body = content_for_expansion(embed).dup << imported_from_html(embed.embed_url)
+ body = absolutize_urls(embed.embed_url, post.cook(body))
+
+ post.cook_method == Post.cook_methods[:raw_html] ? PrettyText.sanitize(body) : body
end
end
+
+ def self.content_for_expansion(embed)
+ return embed.embed_content_cache if embed.embed_content_cache.present?
+
+ raise Discourse::NotFound unless embed.content_truncated.nil?
+
+ body = find_remote(embed.embed_url)&.body
+ raise Discourse::NotFound if body.blank?
+
+ body = body.truncate(EMBED_CONTENT_CACHE_MAX_LENGTH)
+ embed.update!(embed_content_cache: body)
+ body
+ end
+ private_class_method :content_for_expansion
+
+ def self.expanded_cache_key(topic_id)
+ "embed-topic:#{topic_id}"
+ end
+ private_class_method :expanded_cache_key
end
# == Schema Information
@@ -375,6 +429,7 @@ end
#
# id :integer not null, primary key
# content_sha1 :string(40)
+# content_truncated :boolean
# deleted_at :datetime
# embed_content_cache :text
# embed_url :string(1000) not null
diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml
index 02adf30e594..07787233cea 100644
--- a/config/locales/client.en.yml
+++ b/config/locales/client.en.yml
@@ -4918,7 +4918,7 @@ en:
reply_as_new_private_message: "Reply as new message to the same recipients"
continue_discussion: "Continuing the discussion from %{postLink}:"
follow_quote: "go to the quoted post"
- show_full: "Show Full Post"
+ show_more: "Show more…"
show_hidden: "View ignored content."
deleted_by_author_simple: "(post deleted by author)"
collapse: "collapse"
diff --git a/db/migrate/20260821164114_add_content_truncated_to_topic_embeds.rb b/db/migrate/20260821164114_add_content_truncated_to_topic_embeds.rb
new file mode 100644
index 00000000000..6f788092364
--- /dev/null
+++ b/db/migrate/20260821164114_add_content_truncated_to_topic_embeds.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddContentTruncatedToTopicEmbeds < ActiveRecord::Migration[8.0]
+ def change
+ add_column :topic_embeds, :content_truncated, :boolean
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 02acbf00d25..9b77808f3fd 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -10212,7 +10212,8 @@ CREATE TABLE public.topic_embeds (
updated_at timestamp without time zone NOT NULL,
deleted_at timestamp without time zone,
deleted_by_id integer,
- embed_content_cache text
+ embed_content_cache text,
+ content_truncated boolean
);
@@ -23360,6 +23361,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20260824091843'),
('20260824072257'),
('20260824051214'),
+('20260821164114'),
('20260820171539'),
('20260820143851'),
('20260818143417'),
diff --git a/frontend/discourse/app/components/post.gjs b/frontend/discourse/app/components/post.gjs
index 40b4670cd17..bd4f1fa4215 100644
--- a/frontend/discourse/app/components/post.gjs
+++ b/frontend/discourse/app/components/post.gjs
@@ -257,7 +257,7 @@ export default class Post extends Component {
}
@action
- async expandFirstPost() {
+ expandFirstPost() {
this.expandedFirstPost = new TrackedAsyncData(this.args.post.expand());
}
@@ -598,7 +598,7 @@ export default class Post extends Component {
@translatedLabel={{if
this.expandedFirstPost.isPending
(i18n "loading")
- (concat (i18n "post.show_full") "...")
+ (i18n "post.show_more")
}}
/>
{{/if}}
diff --git a/frontend/discourse/app/models/post.js b/frontend/discourse/app/models/post.js
index 1323de21443..7ad1483202f 100644
--- a/frontend/discourse/app/models/post.js
+++ b/frontend/discourse/app/models/post.js
@@ -566,8 +566,13 @@ export default class Post extends RestModel {
// Expands the first post's content, if embedded and shortened.
async expand() {
- const post = await ajax(`/posts/${this.id}/expand-embed`);
- this.cooked = `<section class="expanded-embed">${post.cooked}</section>`;
+ try {
+ const post = await ajax(`/posts/${this.id}/expand-embed`);
+ this.cooked = `<section class="expanded-embed">${post.cooked}</section>`;
+ } catch (error) {
+ popupAjaxError.call(this, error);
+ throw error;
+ }
}
// Recover a deleted post
diff --git a/frontend/discourse/tests/integration/components/post/post-test.gjs b/frontend/discourse/tests/integration/components/post/post-test.gjs
index abc45fb4515..ae0cef93b9c 100644
--- a/frontend/discourse/tests/integration/components/post/post-test.gjs
+++ b/frontend/discourse/tests/integration/components/post/post-test.gjs
@@ -8,6 +8,7 @@ import {
} from "@ember/test-helpers";
import hbs from "htmlbars-inline-precompile";
import { module, test } from "qunit";
+import sinon from "sinon";
import Post from "discourse/components/post";
import DMenus from "discourse/float-kit/components/d-menus";
import { forceMobile, resetMobile } from "discourse/lib/mobile";
@@ -678,10 +679,38 @@ module("Integration | Component | Post", function (hooks) {
await renderComponent(this.post);
+ assert
+ .dom(".expand-post")
+ .hasText(
+ i18n("post.show_more"),
+ "labels the feed-content expansion accurately"
+ );
await click(".topic-body .expand-post");
assert.dom(".expand-post").doesNotExist("button is gone");
});
+ test("reports an error and keeps the expansion retryable", async function (assert) {
+ const message = "Could not show more.";
+ const dialog = getOwner(this).lookup("service:dialog");
+ sinon.stub(dialog, "alert");
+ pretender.get("/posts/:post_id/expand-embed", () =>
+ response(422, { errors: [message] })
+ );
+ this.post.post_number = 1;
+ this.post.topic.expandable_first_post = true;
+
+ await renderComponent(this.post);
+ await click(".topic-body .expand-post");
+
+ assert.dom(".expand-post").exists("keeps the button available for retry");
+ assert.true(
+ dialog.alert.calledWith(
+ i18n("generic_error_with_reason", { error: message })
+ ),
+ "shows the expansion error"
+ );
+ });
+
test("can't show admin menu when you can't manage", async function (assert) {
this.currentUser.admin = false;
diff --git a/lib/feed_item_accessor.rb b/lib/feed_item_accessor.rb
index 93c11c6b377..06fcc50acdc 100644
--- a/lib/feed_item_accessor.rb
+++ b/lib/feed_item_accessor.rb
@@ -11,6 +11,11 @@ class FeedItemAccessor
try_attribute_or_self(element(element_name), :content)
end
+ def element_type(element_name)
+ value = element(element_name)
+ value.type if value.respond_to?(:type)
+ end
+
def link
if rss_item.respond_to?(:links)
link = rss_item.links&.find { |l| l.rel == "alternate" && l.type == "text/html" }
diff --git a/plugins/discourse-rss-polling/app/jobs/jobs/discourse_rss_polling/poll_feed.rb b/plugins/discourse-rss-polling/app/jobs/jobs/discourse_rss_polling/poll_feed.rb
index ce75dc85399..c8ff9c61382 100644
--- a/plugins/discourse-rss-polling/app/jobs/jobs/discourse_rss_polling/poll_feed.rb
+++ b/plugins/discourse-rss-polling/app/jobs/jobs/discourse_rss_polling/poll_feed.rb
@@ -102,8 +102,6 @@ module Jobs
next
end
- cook_method = feed_item.is_youtube? ? Post.cook_methods[:regular] : nil
-
updated_tags = discourse_tags
updated_tags = nil if already_imported&.key?(feed_item)
@@ -113,10 +111,11 @@ module Jobs
author,
feed_item.url,
feed_item.title,
- CGI.unescapeHTML(feed_item.content),
+ feed_item.decoded_content,
category_id: discourse_category_id,
tags: updated_tags,
- cook_method: cook_method,
+ cook_method: feed_item.cook_method,
+ truncate: feed_item.truncate_content?,
)
rescue => e
outcomes << feed_item.outcome(status: :failed, reason: e.message.to_s.truncate(200))
diff --git a/plugins/discourse-rss-polling/app/models/discourse_rss_polling/feed_item.rb b/plugins/discourse-rss-polling/app/models/discourse_rss_polling/feed_item.rb
index b03109e5684..07f9bb87747 100644
--- a/plugins/discourse-rss-polling/app/models/discourse_rss_polling/feed_item.rb
+++ b/plugins/discourse-rss-polling/app/models/discourse_rss_polling/feed_item.rb
@@ -17,16 +17,30 @@ module DiscourseRssPolling
def content
return @content if defined?(@content)
- @content =
- if is_youtube?
- url
- else
- value = nil
- CONTENT_ELEMENT_TAG_NAMES.each do |tag_name|
- break if value = @accessor.element_content(tag_name)
- end
- value&.force_encoding("UTF-8")&.scrub
- end
+ value = @accessor.element_content(content_element_name) if content_element_name
+ value ||= url
+ @content = value.to_s.dup.force_encoding("UTF-8").scrub if value
+ end
+
+ def decoded_content
+ @decoded_content ||= CGI.unescapeHTML(content.to_s)
+ end
+
+ def cook_method
+ return Post.cook_methods[:regular] if content_element_name.nil?
+
+ case @accessor.element_type(content_element_name).to_s.downcase
+ when "text", "text/plain"
+ Post.cook_methods[:regular]
+ when "html", "text/html", "xhtml", "application/xhtml+xml"
+ nil
+ else
+ Post.cook_methods[:regular] if markdown_compatible?
+ end
+ end
+
+ def truncate_content?
+ Nokogiri::HTML5.fragment(decoded_content).text.squish.length > MINIMUM_TEXT_LENGTH_TO_TRUNCATE
end
def title
@@ -58,10 +72,6 @@ module DiscourseRssPolling
@accessor.element_content(:itunes_image)&.href
end
- def is_youtube?
- url&.starts_with?("https://www.youtube.com/watch")
- end
-
def pubdate
return @pubdate if defined?(@pubdate)
@@ -97,5 +107,41 @@ module DiscourseRssPolling
private
CONTENT_ELEMENT_TAG_NAMES = %i[content_encoded content description summary]
+ MINIMUM_TEXT_LENGTH_TO_TRUNCATE = 500
+ MARKDOWN_COMPATIBLE_TAG_NAMES = %w[
+ a
+ abbr
+ b
+ br
+ code
+ del
+ em
+ i
+ img
+ ins
+ kbd
+ mark
+ q
+ s
+ small
+ span
+ strong
+ sub
+ sup
+ time
+ u
+ ]
+
+ def content_element_name
+ return @content_element_name if defined?(@content_element_name)
+
+ @content_element_name =
+ CONTENT_ELEMENT_TAG_NAMES.find { |tag_name| @accessor.element_content(tag_name).present? }
+ end
+
+ def markdown_compatible?
+ tag_names = decoded_content.scan(%r{</?([a-z][a-z0-9-]*)(?:\s[^>]*)?/?>}i).flatten
+ tag_names.all? { |tag_name| MARKDOWN_COMPATIBLE_TAG_NAMES.include?(tag_name.downcase) }
+ end
end
end
diff --git a/plugins/discourse-rss-polling/spec/jobs/poll_feed_spec.rb b/plugins/discourse-rss-polling/spec/jobs/poll_feed_spec.rb
index c5d58bfe1c0..14b4da5d5d6 100644
--- a/plugins/discourse-rss-polling/spec/jobs/poll_feed_spec.rb
+++ b/plugins/discourse-rss-polling/spec/jobs/poll_feed_spec.rb
@@ -29,6 +29,55 @@ RSpec.describe Jobs::DiscourseRssPolling::PollFeed do
)
end
+ context "when inferring how to import feed content" do
+ let(:raw_feed) { <<~XML }
+ <?xml version="1.0" encoding="UTF-8"?>
+ <rss version="2.0">
+ <channel>
+ <title>Example feed</title>
+ <link>https://example.com</link>
+ <description>Example feed</description>
+ <item>
+ <title>Feed item</title>
+ <link>https://example.com/feed-item</link>
+ <description><![CDATA[#{description}]]></description>
+ </item>
+ </channel>
+ </rss>
+ XML
+ let(:post) do
+ job.execute(feed_url: feed_url, author_username: author.username)
+ author.topics.last.first_post
+ end
+
+ before { SiteSetting.embed_truncate = true }
+
+ context "with an untyped Markdown description" do
+ let(:description) { "**Meetup overview**\n\n#{"Talk details. " * 50}" }
+
+ it "imports an excerpt and expands to the complete feed content" do
+ expect(post.cook_method).to eq(Post.cook_methods[:regular])
+ expect(post.raw).not_to include("Talk details")
+ expect(post.cooked).to have_tag("strong", text: "Meetup overview")
+ expect(post.topic.expandable_first_post?).to eq(true)
+ expect(TopicEmbed.expanded_for(post)).to include("Talk details")
+ end
+ end
+
+ context "with a short teaser" do
+ let(:description) { <<~HTML }
+ <h2>Legibility of effort</h2>
+ <p>A short description of the article.</p>
+ <a href="https://example.com/feed-item">Read the full post!</a>
+ HTML
+
+ it "imports the complete feed content without an expansion button" do
+ expect(post.raw).to include("Read the full post!")
+ expect(post.topic.expandable_first_post?).to eq(false)
+ end
+ end
+ end
+
context "with use_pubdate set to false" do
before do
SiteSetting.rss_polling_use_pubdate = false
diff --git a/plugins/discourse-rss-polling/spec/models/feed_item_spec.rb b/plugins/discourse-rss-polling/spec/models/feed_item_spec.rb
index 30d8e8869de..4f03f23f3ec 100644
--- a/plugins/discourse-rss-polling/spec/models/feed_item_spec.rb
+++ b/plugins/discourse-rss-polling/spec/models/feed_item_spec.rb
@@ -9,7 +9,9 @@ RSpec.describe DiscourseRssPolling::FeedItem do
it { expect(feed_item.content).to eq(expected[:content]) }
it { expect(feed_item.url).to eq(expected[:url]) }
it { expect(feed_item.title).to eq(expected[:title]) }
-
+ if expected.key?(:cook_method)
+ it { expect(feed_item.cook_method).to eq(expected[:cook_method]) }
+ end
it { expect(feed_item.categories).to eq(expected[:categories]) } if expected.key?(:categories)
end
@@ -27,6 +29,7 @@ RSpec.describe DiscourseRssPolling::FeedItem do
content: "<p>This is the body &amp; content. </p>",
url: "https://blog.discourse.org/2017/09/poll-feed-spec-fixture/",
title: "Poll Feed Spec Fixture",
+ cook_method: nil,
)
end
@@ -40,6 +43,7 @@ RSpec.describe DiscourseRssPolling::FeedItem do
content: "Here are some random descriptions...",
url: "https://blog.discourse.org/2017/09/poll-feed-spec-fixture/",
title: "Wellington: “Progress is hard!” Other cities: “Hold my beer”",
+ cook_method: nil,
)
end
@@ -52,7 +56,15 @@ RSpec.describe DiscourseRssPolling::FeedItem do
content: "<p>This is the body &amp; content. </p>",
url: "https://blog.discourse.org/2017/09/poll-feed-spec-fixture/",
title: "Poll Feed Spec Fixture",
+ cook_method: nil,
)
+
+ it "uses regular cooking when the content is declared as text" do
+ raw_feed_item.content.type = "text"
+ feed_item = DiscourseRssPolling::FeedItem.new(raw_feed_item)
+
+ expect(feed_item.cook_method).to eq(Post.cook_methods[:regular])
+ end
end
context "with ATOM item with summary element" do
@@ -65,6 +77,7 @@ RSpec.describe DiscourseRssPolling::FeedItem do
content: "Here are some random descriptions...",
url: "https://blog.discourse.org/2017/09/poll-feed-spec-fixture/",
title: "Poll Feed Spec Fixture",
+ cook_method: nil,
)
end
@@ -92,6 +105,7 @@ RSpec.describe DiscourseRssPolling::FeedItem do
content: "https://www.youtube.com/watch?v=K56soYl0U1w",
url: "https://www.youtube.com/watch?v=K56soYl0U1w",
title: "The Ramones - Blitzkrieg Bop (With Lyrics)",
+ cook_method: Post.cook_methods[:regular],
)
end
@@ -105,6 +119,48 @@ RSpec.describe DiscourseRssPolling::FeedItem do
content: "https://www.youtube.com/watch?v=peYYl2vrIt4",
url: "https://www.youtube.com/watch?v=peYYl2vrIt4",
title: "An Uncontroversial Opinion – AMD RX 6600 XT Announcement",
+ cook_method: Post.cook_methods[:regular],
)
end
+
+ describe "content inference" do
+ let(:url) { "https://example.com/feed-item" }
+ let(:raw_feed_item) { Struct.new(:description, :link).new(description, url) }
+ let(:feed_item) { described_class.new(raw_feed_item) }
+
+ context "with an untyped Markdown description" do
+ let(:description) { "Let's gather again - **in person** &amp; online." }
+
+ it "uses regular cooking and decodes entities for import" do
+ expect(feed_item.cook_method).to eq(Post.cook_methods[:regular])
+ expect(feed_item.decoded_content).to eq("Let's gather again - **in person** & online.")
+ expect(feed_item.truncate_content?).to eq(false)
+ end
+ end
+
+ context "with a long description" do
+ let(:description) { "Introduction\n\n#{"More details. " * 50}" }
+
+ it "truncates content with more than 500 visible characters" do
+ expect(feed_item.truncate_content?).to eq(true)
+ end
+ end
+
+ context "with an image-only description" do
+ let(:description) { "<img src='https://example.com/comic.png'>" }
+
+ it "uses regular cooking" do
+ expect(feed_item.cook_method).to eq(Post.cook_methods[:regular])
+ end
+ end
+
+ context "without a description" do
+ let(:description) { nil }
+
+ it "uses the item URL as regular content" do
+ expect(feed_item.content).to eq(url)
+ expect(feed_item.cook_method).to eq(Post.cook_methods[:regular])
+ end
+ end
+ end
end
diff --git a/spec/lib/feed_item_accessor_spec.rb b/spec/lib/feed_item_accessor_spec.rb
index bbaaeb7d26d..07e461dd781 100644
--- a/spec/lib/feed_item_accessor_spec.rb
+++ b/spec/lib/feed_item_accessor_spec.rb
@@ -12,6 +12,10 @@ RSpec.describe FeedItemAccessor do
it { expect(item_accessor.element_content("title")).to eq(atom_feed_item.title.content) }
end
+ describe "#element_type" do
+ it { expect(item_accessor.element_type("content")).to eq("html") }
+ end
+
describe "#link" do
it { expect(item_accessor.link).to eq(atom_feed_item.link.href) }
end
@@ -26,6 +30,10 @@ RSpec.describe FeedItemAccessor do
it { expect(item_accessor.element_content("title")).to eq(rss_feed_item.title) }
end
+ describe "#element_type" do
+ it { expect(item_accessor.element_type("content_encoded")).to be_nil }
+ end
+
describe "#link" do
it { expect(item_accessor.link).to eq(rss_feed_item.link) }
end
diff --git a/spec/models/topic_embed_spec.rb b/spec/models/topic_embed_spec.rb
index dcbe34232df..171449e3d04 100644
--- a/spec/models/topic_embed_spec.rb
+++ b/spec/models/topic_embed_spec.rb
@@ -111,9 +111,9 @@ RSpec.describe TopicEmbed do
# It caches the embed content
expect(post.topic.topic_embed.embed_content_cache).to eq(contents)
+ expect(post.topic.topic_embed.content_truncated).to eq(false)
# It converts relative URLs to absolute when expanded
- stub_request(:get, url).to_return(status: 200, body: contents)
expect(TopicEmbed.expanded_for(post)).to have_tag(
"img",
with: {
@@ -153,6 +153,12 @@ RSpec.describe TopicEmbed do
expect(topic_embed.embed_content_cache).to eq("new contents")
end
+ it "updates an explicitly selected cook method on reimport" do
+ TopicEmbed.import(user, url, title, contents, cook_method: Post.cook_methods[:regular])
+
+ expect(post.reload.cook_method).to eq(Post.cook_methods[:regular])
+ end
+
it "Should leave uppercase Feed Entry URL untouched in content" do
cased_url = "http://eviltrout.com/ABCD"
post = TopicEmbed.import(user, cased_url, title, "some random content")
@@ -477,6 +483,7 @@ RSpec.describe TopicEmbed do
post = TopicEmbed.import(user, url, title, long_content)
expect(post.raw).not_to include(long_content)
+ expect(post.topic.topic_embed.content_truncated).to eq(true)
end
it "keeps everything in the imported post when truncation is disabled" do
@@ -484,6 +491,7 @@ RSpec.describe TopicEmbed do
post = TopicEmbed.import(user, url, title, long_content)
expect(post.raw).to include(long_content)
+ expect(post.topic.topic_embed.content_truncated).to eq(false)
end
it "looks at first div when there is no paragraph" do
@@ -494,6 +502,63 @@ RSpec.describe TopicEmbed do
expect(post.raw).to include("testing it")
end
+
+ it "keeps the complete markup when truncation would only remove a wrapper" do
+ wrapped_content = "<div><p>#{"Complete paragraph. " * 30}</p></div>"
+ SiteSetting.embed_truncate = true
+
+ post = TopicEmbed.import(user, url, title, wrapped_content)
+
+ expect(post.raw).to include(wrapped_content)
+ expect(post.topic.topic_embed.content_truncated).to eq(false)
+ end
+
+ it "keeps image-only content" do
+ SiteSetting.embed_truncate = true
+ post = TopicEmbed.import(user, url, title, "<img src='/comic.png' alt='comic'>")
+
+ expect(post.raw).to have_tag(
+ "img",
+ with: {
+ src: "http://eviltrout.com/comic.png",
+ alt: "comic",
+ },
+ )
+ expect(post.topic.topic_embed.content_truncated).to eq(false)
+ end
+
+ it "truncates independently of the cook method" do
+ SiteSetting.embed_truncate = true
+ post =
+ TopicEmbed.import(
+ user,
+ url,
+ title,
+ "**Meetup overview**\n\nTalk details",
+ cook_method: Post.cook_methods[:regular],
+ truncate: true,
+ )
+
+ expect(post.raw).not_to include("Talk details")
+ expect(post.cooked).to have_tag("strong", text: "Meetup overview")
+ expect(post.topic.topic_embed.content_truncated).to eq(true)
+ end
+
+ it "refreshes expanded content when the excerpt is unchanged" do
+ SiteSetting.embed_truncate = true
+ first_paragraph = "<p>#{"Same introduction. " * 10}</p>"
+ original_contents = "#{first_paragraph}<p>Original ending.</p>"
+ updated_contents = "#{first_paragraph}<p>Updated ending.</p>"
+
+ post = TopicEmbed.import(user, url, title, original_contents)
+ expect(TopicEmbed.expanded_for(post)).to include("Original ending.")
+
+ TopicEmbed.import(user, url, title, updated_contents)
+
+ expect(post.reload.raw).not_to include("Updated ending.")
+ expect(post.topic.topic_embed.reload.embed_content_cache).to eq(updated_contents)
+ expect(TopicEmbed.expanded_for(post)).to include("Updated ending.")
+ end
end
end
@@ -853,19 +918,67 @@ RSpec.describe TopicEmbed do
fab!(:tag)
it "returns embed content" do
- stub_request(:get, url).to_return(status: 200, body: contents)
+ stub_request(:get, url).to_return(status: 200, body: "<p>remote content</p>")
+ post = TopicEmbed.import(user, url, title, contents)
+ expanded = TopicEmbed.expanded_for(post)
+
+ expect(expanded).to include(contents)
+ expect(expanded).not_to include("remote content")
+ expect(WebMock).not_to have_requested(:get, url)
+ end
+
+ it "falls back to the remote page for a legacy embed without cached content" do
+ remote_contents =
+ "<html><body><article><p>Content from the remote page.</p></article></body></html>"
+ stub_request(:get, url).to_return(status: 200, body: remote_contents)
post = TopicEmbed.import(user, url, title, contents)
- expect(TopicEmbed.expanded_for(post)).to include(contents)
+ post.topic.topic_embed.update!(embed_content_cache: nil, content_truncated: nil)
+
+ expect(TopicEmbed.expanded_for(post)).to include("Content from the remote page.")
+ expect(post.topic.topic_embed.reload.embed_content_cache).to include(
+ "Content from the remote page.",
+ )
end
- it "updates the embed content cache" do
- stub_request(:get, url)
- .to_return(status: 200, body: contents)
- .then
- .to_return(status: 200, body: "contents changed")
+ it "does not fetch the remote page for an imported item with missing cached content" do
+ stub_request(:get, url).to_return(status: 200, body: "<p>remote content</p>")
post = TopicEmbed.import(user, url, title, contents)
- TopicEmbed.expanded_for(post)
- expect(post.topic.topic_embed.reload.embed_content_cache).to include("contents changed")
+ post.topic.topic_embed.update!(embed_content_cache: nil, content_truncated: true)
+
+ expect { TopicEmbed.expanded_for(post) }.to raise_error(Discourse::NotFound)
+ expect(WebMock).not_to have_requested(:get, url)
+ end
+
+ it "cooks cached Markdown using the post cook method" do
+ post =
+ TopicEmbed.import(
+ user,
+ url,
+ title,
+ "**Meetup overview**\n\nTalk details",
+ cook_method: Post.cook_methods[:regular],
+ truncate: true,
+ )
+
+ expanded = TopicEmbed.expanded_for(post)
+
+ expect(expanded).to have_tag("strong", text: "Meetup overview")
+ expect(expanded).to include("Talk details")
+ end
+
+ it "sanitizes cached raw HTML" do
+ post =
+ TopicEmbed.import(
+ user,
+ url,
+ title,
+ "<p>Safe content</p><script>window.evil = true</script>",
+ )
+
+ expanded = TopicEmbed.expanded_for(post)
+
+ expect(expanded).to include("Safe content")
+ expect(expanded).not_to have_tag("script")
end
end
end
diff --git a/spec/models/topic_spec.rb b/spec/models/topic_spec.rb
index f6824d62f8c..5838aaa5e00 100644
--- a/spec/models/topic_spec.rb
+++ b/spec/models/topic_spec.rb
@@ -3197,31 +3197,32 @@ RSpec.describe Topic do
end
describe "expandable_first_post?" do
- let(:topic) { Fabricate.build(:topic) }
+ it "is false without a topic embed" do
+ expect(Fabricate.build(:topic).expandable_first_post?).to eq(false)
+ end
+
+ it "uses the recorded truncation state when available" do
+ post = Fabricate(:post)
+ embed = Fabricate(:topic_embed, post:, content_truncated: true)
+ SiteSetting.embed_truncate = false
+
+ expect(post.topic.expandable_first_post?).to eq(true)
+
+ embed.update!(content_truncated: false)
+ SiteSetting.embed_truncate = true
- it "is false if embeddable_host is blank" do
- expect(topic.expandable_first_post?).to eq(false)
+ expect(post.topic.expandable_first_post?).to eq(false)
end
- describe "with an embeddable host" do
- before do
- Fabricate(:embeddable_host)
- SiteSetting.embed_truncate = true
- topic.stubs(:has_topic_embed?).returns(true)
- end
+ it "uses the site setting for legacy embeds with unknown truncation state" do
+ post = Fabricate(:post)
+ Fabricate(:topic_embed, post:, content_truncated: nil)
- it "is true with the correct settings and topic_embed" do
- expect(topic.expandable_first_post?).to eq(true)
- end
- it "is false if embed_truncate? is false" do
- SiteSetting.embed_truncate = false
- expect(topic.expandable_first_post?).to eq(false)
- end
+ SiteSetting.embed_truncate = true
+ expect(post.topic.expandable_first_post?).to eq(true)
- it "is false if has_topic_embed? is false" do
- topic.stubs(:has_topic_embed?).returns(false)
- expect(topic.expandable_first_post?).to eq(false)
- end
+ SiteSetting.embed_truncate = false
+ expect(post.topic.expandable_first_post?).to eq(false)
end
end