Skip to content

A platform for community discussion. Free, open, simple.

FIX: Resolve embedded comments through URL aliases (#44046)

Keegan Georgeb3b4111b252c
diff --git a/app/models/topic_embed.rb b/app/models/topic_embed.rb
index a47662cd0de..a3395e5dc27 100644
--- a/app/models/topic_embed.rb
+++ b/app/models/topic_embed.rb
@@ -7,6 +7,7 @@ class TopicEmbed < ActiveRecord::Base
belongs_to :topic
belongs_to :post
+ has_many :topic_embed_aliases, dependent: :delete_all
validates :embed_url, presence: true
validates :embed_url, uniqueness: true
validates :embed_content_cache, length: { maximum: EMBED_CONTENT_CACHE_MAX_LENGTH }
@@ -320,7 +321,20 @@ class TopicEmbed < ActiveRecord::Base
embed_url = response.url if original_uri.host == canonical_uri.host
end
- TopicEmbed.import(import_user, embed_url, response.title, response.body)
+ transaction do
+ post = TopicEmbed.import(import_user, embed_url, response.title, response.body)
+ topic_embed = post&.topic&.topic_embed
+ alias_attributes = TopicEmbedAlias.key_attributes(url)
+
+ if topic_embed && alias_attributes[:url_key] != embed_url_key(topic_embed.embed_url)
+ TopicEmbedAlias.upsert(
+ alias_attributes.merge(topic_embed_id: topic_embed.id),
+ unique_by: :url_hash,
+ )
+ end
+
+ post
+ end
end
# Convert any relative URLs to absolute. RSS is annoying for this.
@@ -363,7 +377,10 @@ class TopicEmbed < ActiveRecord::Base
end
def self.topic_embed_by_url(embed_url)
- with_embed_urls([embed_url]).min_by(&:id)
+ with_embed_urls([embed_url]).min_by(&:id) ||
+ joins(:topic_embed_aliases).find_by(
+ topic_embed_aliases: TopicEmbedAlias.key_attributes(embed_url),
+ )
end
def self.with_embed_urls(urls)
diff --git a/app/models/topic_embed_alias.rb b/app/models/topic_embed_alias.rb
new file mode 100644
index 00000000000..ee92681a994
--- /dev/null
+++ b/app/models/topic_embed_alias.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class TopicEmbedAlias < ActiveRecord::Base
+ belongs_to :topic_embed
+
+ def self.key_attributes(url)
+ url_key = TopicEmbed.embed_url_key(url)
+ { url_key:, url_hash: Digest::SHA256.hexdigest(url_key) }
+ end
+end
+
+# == Schema Information
+#
+# Table name: topic_embed_aliases
+#
+# id :bigint not null, primary key
+# url_hash :string(64) not null
+# url_key :text not null
+# topic_embed_id :bigint not null
+#
+# Indexes
+#
+# index_topic_embed_aliases_on_topic_embed_id (topic_embed_id)
+# index_topic_embed_aliases_on_url_hash (url_hash) UNIQUE
+#
diff --git a/db/migrate/20260923141924_create_topic_embed_aliases.rb b/db/migrate/20260923141924_create_topic_embed_aliases.rb
new file mode 100644
index 00000000000..b84b365fd4a
--- /dev/null
+++ b/db/migrate/20260923141924_create_topic_embed_aliases.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CreateTopicEmbedAliases < ActiveRecord::Migration[8.1]
+ def change
+ create_table :topic_embed_aliases do |t|
+ t.bigint :topic_embed_id, null: false
+ t.text :url_key, null: false
+ t.string :url_hash, limit: 64, null: false
+ end
+
+ add_index :topic_embed_aliases, :topic_embed_id
+ add_index :topic_embed_aliases, :url_hash, unique: true
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 5cce8f7ac14..ced344e628b 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -11087,6 +11087,37 @@ CREATE SEQUENCE public.topic_custom_fields_id_seq
ALTER SEQUENCE public.topic_custom_fields_id_seq OWNED BY public.topic_custom_fields.id;
+--
+-- Name: topic_embed_aliases; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.topic_embed_aliases (
+ id bigint NOT NULL,
+ topic_embed_id bigint NOT NULL,
+ url_key text NOT NULL,
+ url_hash character varying(64) NOT NULL
+);
+
+
+--
+-- Name: topic_embed_aliases_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.topic_embed_aliases_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: topic_embed_aliases_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.topic_embed_aliases_id_seq OWNED BY public.topic_embed_aliases.id;
+
+
--
-- Name: topic_embeds; Type: TABLE; Schema: public; Owner: -
--
@@ -15736,6 +15767,13 @@ ALTER TABLE ONLY public.topic_allowed_users ALTER COLUMN id SET DEFAULT nextval(
ALTER TABLE ONLY public.topic_custom_fields ALTER COLUMN id SET DEFAULT nextval('public.topic_custom_fields_id_seq'::regclass);
+--
+-- Name: topic_embed_aliases id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.topic_embed_aliases ALTER COLUMN id SET DEFAULT nextval('public.topic_embed_aliases_id_seq'::regclass);
+
+
--
-- Name: topic_embeds id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -18549,6 +18587,14 @@ ALTER TABLE ONLY public.topic_custom_fields
ADD CONSTRAINT topic_custom_fields_pkey PRIMARY KEY (id);
+--
+-- Name: topic_embed_aliases topic_embed_aliases_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.topic_embed_aliases
+ ADD CONSTRAINT topic_embed_aliases_pkey PRIMARY KEY (id);
+
+
--
-- Name: topic_embeds topic_embeds_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -23835,6 +23881,20 @@ CREATE INDEX index_topic_custom_fields_on_topic_id_and_name ON public.topic_cust
CREATE UNIQUE INDEX index_topic_custom_fields_on_topic_id_and_slack_thread_id ON public.topic_custom_fields USING btree (topic_id, name) WHERE ((name)::text ~~ 'slack_thread_id_%'::text);
+--
+-- Name: index_topic_embed_aliases_on_topic_embed_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_topic_embed_aliases_on_topic_embed_id ON public.topic_embed_aliases USING btree (topic_embed_id);
+
+
+--
+-- Name: index_topic_embed_aliases_on_url_hash; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX index_topic_embed_aliases_on_url_hash ON public.topic_embed_aliases USING btree (url_hash);
+
+
--
-- Name: index_topic_embeds_on_embed_url; Type: INDEX; Schema: public; Owner: -
--
@@ -25504,6 +25564,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260923141924'),
('20260923080644'),
('20260923080642'),
('20260922233816'),
diff --git a/lib/topic_retriever.rb b/lib/topic_retriever.rb
index 5e7945e3a58..6badbb3e0ba 100644
--- a/lib/topic_retriever.rb
+++ b/lib/topic_retriever.rb
@@ -32,7 +32,7 @@ class TopicRetriever
def perform_retrieve
# It's possible another process or job found the embed already. So if that happened bail out.
- return if TopicEmbed.where(embed_url: @embed_url).exists?
+ return if TopicEmbed.topic_id_for_embed(@embed_url)
username =
SiteSetting.embed_by_username.presence || SiteSetting.site_contact_username.presence ||
diff --git a/spec/lib/topic_retriever_spec.rb b/spec/lib/topic_retriever_spec.rb
index c1021255a9d..740bf0c3a26 100644
--- a/spec/lib/topic_retriever_spec.rb
+++ b/spec/lib/topic_retriever_spec.rb
@@ -59,6 +59,16 @@ RSpec.describe TopicRetriever do
end
end
+ it "does not fetch an already imported URL alias" do
+ url = "https://example.com/articles/entry?view=full"
+ Fabricate(:embeddable_host, host: "example.com")
+ topic_embed = Fabricate(:topic_embed, embed_url: "https://example.com/articles/entry")
+ topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(url))
+
+ expect { TopicRetriever.new(url, no_throttle: true).retrieve }.not_to change { Topic.count }
+ expect(WebMock).not_to have_requested(:get, url)
+ end
+
it "works with URLs with whitespaces" do
expect { TopicRetriever.new(" https://example.com ").retrieve }.not_to raise_error
end
diff --git a/spec/models/topic_embed_spec.rb b/spec/models/topic_embed_spec.rb
index 181417ff8d1..c106b3d087f 100644
--- a/spec/models/topic_embed_spec.rb
+++ b/spec/models/topic_embed_spec.rb
@@ -878,6 +878,200 @@ RSpec.describe TopicEmbed do
end
end
+ describe ".import_remote" do
+ fab!(:user)
+ fab!(:embeddable_host) { Fabricate(:embeddable_host, host: "example.com") }
+
+ let(:original_url) { "https://example.com/articles/entry?view=full" }
+ let(:canonical_url) { "https://example.com/articles/entry" }
+ let(:content) do
+ "<html><title>Embedded article</title><body><p>Article content</p></body></html>"
+ end
+
+ before do
+ stub_request(:get, original_url).to_return(
+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),
+ )
+ stub_request(:head, canonical_url).to_return(status: 200)
+ stub_request(:get, canonical_url).to_return(body: content)
+ end
+
+ it "finds an imported topic using either the requested or canonical URL" do
+ post = TopicEmbed.import_remote(original_url, user: user)
+
+ expect(post.topic.topic_embed.embed_url).to eq(canonical_url)
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
+ expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(post.topic_id)
+ end
+
+ it "supports requested URLs longer than the alias index limit" do
+ long_original_url = "https://example.com/articles/entry?source=#{"a" * 1_100}"
+ stub_request(:get, long_original_url).to_return(
+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),
+ )
+
+ post = TopicEmbed.import_remote(long_original_url, user: user)
+
+ expect(TopicEmbedAlias.find_by(topic_embed: post.topic.topic_embed).url_key.length).to be >
+ 1_000
+ expect(TopicEmbed.topic_id_for_embed(long_original_url)).to eq(post.topic_id)
+ expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(post.topic_id)
+ end
+
+ it "reassigns a requested URL to the latest successfully imported canonical topic" do
+ other_url = "https://example.com/articles/entry?source=other"
+ original_post = TopicEmbed.import_remote(original_url, user: user)
+ original_embed = original_post.topic.topic_embed
+ original_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(other_url))
+ replacement_url = "https://example.com/articles/replacement"
+ stub_request(:get, original_url).to_return(
+ body: %(<html><head><link rel="canonical" href="#{replacement_url}"></head></html>),
+ )
+ stub_request(:head, replacement_url).to_return(status: 200)
+ stub_request(:get, replacement_url).to_return(body: content)
+
+ replacement_post = TopicEmbed.import_remote(original_url, user: user)
+
+ expect(replacement_post.topic_id).not_to eq(original_post.topic_id)
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(replacement_post.topic_id)
+ expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(original_post.topic_id)
+ expect(TopicEmbed.topic_id_for_embed(other_url)).to eq(original_post.topic_id)
+ expect(TopicEmbedAlias.where(TopicEmbedAlias.key_attributes(original_url)).count).to eq(1)
+ end
+
+ it "keeps an existing alias when a later remote import fails" do
+ original_post = TopicEmbed.import_remote(original_url, user: user)
+ stub_request(:get, original_url).to_return(status: 404)
+
+ expect(TopicEmbed.import_remote(original_url, user: user)).to be_nil
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(original_post.topic_id)
+ expect(TopicEmbedAlias.where(TopicEmbedAlias.key_attributes(original_url)).count).to eq(1)
+ end
+
+ it "remembers the requested URL when the canonical topic already exists" do
+ post = TopicEmbed.import_remote(canonical_url, user: user)
+
+ expect { TopicEmbed.import_remote(original_url, user: user) }.not_to change { Topic.count }
+
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
+ end
+
+ it "remembers multiple URLs without creating duplicate topics or aliases" do
+ other_url = "https://example.com/articles/entry?format=print"
+ stub_request(:get, other_url).to_return(
+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),
+ )
+ post = TopicEmbed.import_remote(original_url, user: user)
+
+ expect do
+ TopicEmbed.import_remote(other_url, user: user)
+ TopicEmbed.import_remote(original_url, user: user)
+ end.not_to change { Topic.count }
+
+ expect(post.topic.topic_embed.topic_embed_aliases.count).to eq(2)
+ expect(TopicEmbed.topic_id_for_embed(other_url)).to eq(post.topic_id)
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
+ end
+
+ it "remembers the requested URL after an HTTP redirect" do
+ destination = "https://example.com/archive/entry"
+ stub_request(:get, original_url).to_return(
+ status: 302,
+ headers: {
+ "Location" => destination,
+ },
+ )
+ stub_request(:get, destination).to_return(body: content)
+
+ post = TopicEmbed.import_remote(original_url, user: user)
+
+ expect(post.topic.topic_embed.embed_url).to eq(destination)
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
+ end
+
+ it "does not create an alias when the requested URL is already canonical" do
+ expect { TopicEmbed.import_remote(canonical_url, user: user) }.not_to change {
+ TopicEmbedAlias.count
+ }
+ end
+
+ it "does not create an alias when the remote page cannot be fetched" do
+ stub_request(:get, original_url).to_return(status: 404)
+
+ expect do
+ expect(TopicEmbed.import_remote(original_url, user: user)).to be_nil
+ end.not_to change { TopicEmbedAlias.count }
+ end
+
+ it "does not alias a canonical URL on another host" do
+ destination = "https://other.example.com/articles/entry"
+ stub_request(:get, original_url).to_return(
+ body: %(<html><head><link rel="canonical" href="#{destination}"></head></html>),
+ )
+ stub_request(:head, destination).to_return(status: 200)
+ stub_request(:get, destination).to_return(body: content)
+
+ post = TopicEmbed.import_remote(original_url, user: user)
+
+ expect(post.topic.topic_embed.embed_url).to eq(original_url)
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
+ expect(TopicEmbed.topic_id_for_embed(destination)).to be_nil
+ expect(TopicEmbedAlias.count).to eq(0)
+ end
+ end
+
+ describe ".topic_id_for_embed with aliases" do
+ fab!(:topic_embed) { Fabricate(:topic_embed, embed_url: "https://example.com/articles/entry") }
+ let(:original_url) { "https://example.com/entry?view=full" }
+
+ before { topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(original_url)) }
+
+ it "normalizes aliases in the same way as primary embed URLs" do
+ expect(TopicEmbed.topic_id_for_embed("http://EXAMPLE.com/entry?view=full")).to eq(
+ topic_embed.topic_id,
+ )
+ end
+
+ it "does not discard meaningful query parameters" do
+ expect(TopicEmbed.topic_id_for_embed("https://example.com/entry?view=other")).to be_nil
+ expect(TopicEmbed.topic_id_for_embed("https://example.com/entry")).to be_nil
+ end
+
+ it "prefers an existing primary URL over an alias" do
+ direct_embed = Fabricate(:topic_embed, embed_url: original_url)
+
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(direct_embed.topic_id)
+ end
+
+ it "requires the full key to match the stored hash" do
+ lookup_url = "https://example.com/entry?view=collision"
+ stored_url = "https://example.com/entry?view=stored"
+ inconsistent_attributes =
+ TopicEmbedAlias.key_attributes(stored_url).merge(
+ url_hash: TopicEmbedAlias.key_attributes(lookup_url)[:url_hash],
+ )
+ topic_embed.topic_embed_aliases.create!(inconsistent_attributes)
+
+ expect(TopicEmbed.topic_id_for_embed(lookup_url)).to be_nil
+ end
+
+ it "does not resolve deleted embeds and restores aliases when recovered" do
+ topic_embed.trash!
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to be_nil
+
+ topic_embed.recover!
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(topic_embed.topic_id)
+ end
+
+ it "removes aliases when the imported topic is destroyed" do
+ topic_embed.topic.destroy!
+
+ expect(TopicEmbed.topic_id_for_embed(original_url)).to be_nil
+ expect(TopicEmbedAlias.where(topic_embed_id: topic_embed.id)).to be_empty
+ expect(TopicEmbed.where(id: topic_embed.id)).to be_empty
+ end
+ end
+
describe ".absolutize_urls" do
it "handles badly formed URIs" do
invalid_url = "http://source.com/#double#anchor"
diff --git a/spec/requests/embed_controller_spec.rb b/spec/requests/embed_controller_spec.rb
index b51e1aed7b9..46c0b981725 100644
--- a/spec/requests/embed_controller_spec.rb
+++ b/spec/requests/embed_controller_spec.rb
@@ -321,6 +321,10 @@ RSpec.describe EmbedController do
describe "full_app redirect" do
fab!(:embeddable_host)
+ let(:original_url) { "https://example.com/articles/entry?view=full" }
+ let(:canonical_url) { "https://example.com/articles/entry" }
+ let(:generic_embeddable_host) { Fabricate(:embeddable_host, host: "example.com") }
+
before { SiteSetting.embed_full_app = true }
it "redirects to topic URL with embed_mode when full_app is present" do
@@ -338,6 +342,125 @@ RSpec.describe EmbedController do
expect(response).to redirect_to("#{topic_embed.topic.url}?embed_mode=true")
end
+ it "renders loading before redirecting a canonical URL alias" do
+ generic_embeddable_host
+ Jobs.run_immediately!
+ stub_request(:get, original_url).to_return(
+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),
+ )
+ stub_request(:head, canonical_url).to_return(status: 200)
+ stub_request(:get, canonical_url).to_return(
+ body: "<html><title>Embedded article</title><body><p>Article content</p></body></html>",
+ )
+
+ get "/embed/comments",
+ params: {
+ embed_url: original_url,
+ full_app: "true",
+ },
+ headers: {
+ "REFERER" => original_url,
+ }
+
+ expect(response).to have_http_status(:ok)
+ expect(response.body).to include("data-embed-state='loading'")
+
+ topic_embed = TopicEmbed.find_by(embed_url: canonical_url)
+
+ get "/embed/comments",
+ params: {
+ embed_url: original_url,
+ full_app: "true",
+ },
+ headers: {
+ "REFERER" => original_url,
+ }
+
+ expect(response).to redirect_to("#{topic_embed.topic.url}?embed_mode=true")
+ end
+
+ it "renders an imported canonical URL alias in classic mode" do
+ generic_embeddable_host
+ Jobs.run_immediately!
+ stub_request(:get, original_url).to_return(
+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),
+ )
+ stub_request(:head, canonical_url).to_return(status: 200)
+ stub_request(:get, canonical_url).to_return(
+ body: "<html><title>Embedded article</title><body><p>Article content</p></body></html>",
+ )
+
+ get "/embed/comments",
+ params: {
+ embed_url: original_url,
+ },
+ headers: {
+ "REFERER" => original_url,
+ }
+
+ expect(response.body).to include("data-embed-state='loading'")
+
+ get "/embed/comments",
+ params: {
+ embed_url: original_url,
+ },
+ headers: {
+ "REFERER" => original_url,
+ }
+
+ expect(response).to have_http_status(:ok)
+ expect(response.body).not_to include("data-embed-state='loading'")
+ end
+
+ it "returns not found for a canonical URL alias in a private category" do
+ generic_embeddable_host
+ restricted_category = Fabricate(:private_category, group: Fabricate(:group))
+ restricted_topic = Fabricate(:topic, category: restricted_category)
+ restricted_post = Fabricate(:post, topic: restricted_topic)
+ topic_embed =
+ Fabricate(
+ :topic_embed,
+ topic: restricted_topic,
+ post: restricted_post,
+ embed_url: canonical_url,
+ )
+ topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(original_url))
+
+ get "/embed/comments",
+ params: {
+ embed_url: original_url,
+ full_app: "true",
+ },
+ headers: {
+ "REFERER" => original_url,
+ }
+
+ expect(response).to have_http_status(:not_found)
+ end
+
+ it "rejects a canonical URL alias after its original path is no longer allowed" do
+ embeddable_host =
+ Fabricate(
+ :embeddable_host,
+ host: "example.com",
+ allowed_paths: %r{\A/articles/entry(?:\?.*)?\z}.source,
+ )
+ topic_embed = Fabricate(:topic_embed, embed_url: canonical_url)
+ topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(original_url))
+ embeddable_host.update!(allowed_paths: %r{\A/other/}.source)
+
+ get "/embed/comments",
+ params: {
+ embed_url: original_url,
+ full_app: "true",
+ },
+ headers: {
+ "REFERER" => original_url,
+ }
+
+ expect(response).to have_http_status(:bad_request)
+ end
+
it "redirects to topic URL with embed_mode when using topic_id" do
get "/embed/comments",
params: {