A platform for community discussion. Free, open, simple.
FIX: Resolve embedded comments through URL aliases (#44046)
Keegan Georgeb3b4111b252c
diff --git a/app/models/topic_embed.rb b/app/models/topic_embed.rbindex a47662cd0de..a3395e5dc27 100644--- a/app/models/topic_embed.rb+++ b/app/models/topic_embed.rb@@ -7,6 +7,7 @@ class TopicEmbed < ActiveRecord::Basebelongs_to :topicbelongs_to :post+ has_many :topic_embed_aliases, dependent: :delete_allvalidates :embed_url, presence: truevalidates :embed_url, uniqueness: truevalidates :embed_content_cache, length: { maximum: EMBED_CONTENT_CACHE_MAX_LENGTH }@@ -320,7 +321,20 @@ class TopicEmbed < ActiveRecord::Baseembed_url = response.url if original_uri.host == canonical_uri.hostend- TopicEmbed.import(import_user, embed_url, response.title, response.body)+ transaction do+ post = TopicEmbed.import(import_user, embed_url, response.title, response.body)+ topic_embed = post&.topic&.topic_embed+ alias_attributes = TopicEmbedAlias.key_attributes(url)++ if topic_embed && alias_attributes[:url_key] != embed_url_key(topic_embed.embed_url)+ TopicEmbedAlias.upsert(+ alias_attributes.merge(topic_embed_id: topic_embed.id),+ unique_by: :url_hash,+ )+ end++ post+ endend# Convert any relative URLs to absolute. RSS is annoying for this.@@ -363,7 +377,10 @@ class TopicEmbed < ActiveRecord::Baseenddef self.topic_embed_by_url(embed_url)- with_embed_urls([embed_url]).min_by(&:id)+ with_embed_urls([embed_url]).min_by(&:id) ||+ joins(:topic_embed_aliases).find_by(+ topic_embed_aliases: TopicEmbedAlias.key_attributes(embed_url),+ )enddef self.with_embed_urls(urls)diff --git a/app/models/topic_embed_alias.rb b/app/models/topic_embed_alias.rbnew file mode 100644index 00000000000..ee92681a994--- /dev/null+++ b/app/models/topic_embed_alias.rb@@ -0,0 +1,25 @@+# frozen_string_literal: true++class TopicEmbedAlias < ActiveRecord::Base+ belongs_to :topic_embed++ def self.key_attributes(url)+ url_key = TopicEmbed.embed_url_key(url)+ { url_key:, url_hash: Digest::SHA256.hexdigest(url_key) }+ end+end++# == Schema Information+#+# Table name: topic_embed_aliases+#+# id :bigint not null, primary key+# url_hash :string(64) not null+# url_key :text not null+# topic_embed_id :bigint not null+#+# Indexes+#+# index_topic_embed_aliases_on_topic_embed_id (topic_embed_id)+# index_topic_embed_aliases_on_url_hash (url_hash) UNIQUE+#diff --git a/db/migrate/20260923141924_create_topic_embed_aliases.rb b/db/migrate/20260923141924_create_topic_embed_aliases.rbnew file mode 100644index 00000000000..b84b365fd4a--- /dev/null+++ b/db/migrate/20260923141924_create_topic_embed_aliases.rb@@ -0,0 +1,14 @@+# frozen_string_literal: true++class CreateTopicEmbedAliases < ActiveRecord::Migration[8.1]+ def change+ create_table :topic_embed_aliases do |t|+ t.bigint :topic_embed_id, null: false+ t.text :url_key, null: false+ t.string :url_hash, limit: 64, null: false+ end++ add_index :topic_embed_aliases, :topic_embed_id+ add_index :topic_embed_aliases, :url_hash, unique: true+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 5cce8f7ac14..ced344e628b 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -11087,6 +11087,37 @@ CREATE SEQUENCE public.topic_custom_fields_id_seqALTER SEQUENCE public.topic_custom_fields_id_seq OWNED BY public.topic_custom_fields.id;+--+-- Name: topic_embed_aliases; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.topic_embed_aliases (+ id bigint NOT NULL,+ topic_embed_id bigint NOT NULL,+ url_key text NOT NULL,+ url_hash character varying(64) NOT NULL+);+++--+-- Name: topic_embed_aliases_id_seq; Type: SEQUENCE; Schema: public; Owner: -+--++CREATE SEQUENCE public.topic_embed_aliases_id_seq+ START WITH 1+ INCREMENT BY 1+ NO MINVALUE+ NO MAXVALUE+ CACHE 1;+++--+-- Name: topic_embed_aliases_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -+--++ALTER SEQUENCE public.topic_embed_aliases_id_seq OWNED BY public.topic_embed_aliases.id;++---- Name: topic_embeds; Type: TABLE; Schema: public; Owner: ---@@ -15736,6 +15767,13 @@ ALTER TABLE ONLY public.topic_allowed_users ALTER COLUMN id SET DEFAULT nextval(ALTER TABLE ONLY public.topic_custom_fields ALTER COLUMN id SET DEFAULT nextval('public.topic_custom_fields_id_seq'::regclass);+--+-- Name: topic_embed_aliases id; Type: DEFAULT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.topic_embed_aliases ALTER COLUMN id SET DEFAULT nextval('public.topic_embed_aliases_id_seq'::regclass);++---- Name: topic_embeds id; Type: DEFAULT; Schema: public; Owner: ---@@ -18549,6 +18587,14 @@ ALTER TABLE ONLY public.topic_custom_fieldsADD CONSTRAINT topic_custom_fields_pkey PRIMARY KEY (id);+--+-- Name: topic_embed_aliases topic_embed_aliases_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.topic_embed_aliases+ ADD CONSTRAINT topic_embed_aliases_pkey PRIMARY KEY (id);++---- Name: topic_embeds topic_embeds_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -23835,6 +23881,20 @@ CREATE INDEX index_topic_custom_fields_on_topic_id_and_name ON public.topic_custCREATE UNIQUE INDEX index_topic_custom_fields_on_topic_id_and_slack_thread_id ON public.topic_custom_fields USING btree (topic_id, name) WHERE ((name)::text ~~ 'slack_thread_id_%'::text);+--+-- Name: index_topic_embed_aliases_on_topic_embed_id; Type: INDEX; Schema: public; Owner: -+--++CREATE INDEX index_topic_embed_aliases_on_topic_embed_id ON public.topic_embed_aliases USING btree (topic_embed_id);+++--+-- Name: index_topic_embed_aliases_on_url_hash; Type: INDEX; Schema: public; Owner: -+--++CREATE UNIQUE INDEX index_topic_embed_aliases_on_url_hash ON public.topic_embed_aliases USING btree (url_hash);++---- Name: index_topic_embeds_on_embed_url; Type: INDEX; Schema: public; Owner: ---@@ -25504,6 +25564,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260923141924'),('20260923080644'),('20260923080642'),('20260922233816'),diff --git a/lib/topic_retriever.rb b/lib/topic_retriever.rbindex 5e7945e3a58..6badbb3e0ba 100644--- a/lib/topic_retriever.rb+++ b/lib/topic_retriever.rb@@ -32,7 +32,7 @@ class TopicRetrieverdef perform_retrieve# It's possible another process or job found the embed already. So if that happened bail out.- return if TopicEmbed.where(embed_url: @embed_url).exists?+ return if TopicEmbed.topic_id_for_embed(@embed_url)username =SiteSetting.embed_by_username.presence || SiteSetting.site_contact_username.presence ||diff --git a/spec/lib/topic_retriever_spec.rb b/spec/lib/topic_retriever_spec.rbindex c1021255a9d..740bf0c3a26 100644--- a/spec/lib/topic_retriever_spec.rb+++ b/spec/lib/topic_retriever_spec.rb@@ -59,6 +59,16 @@ RSpec.describe TopicRetriever doendend+ it "does not fetch an already imported URL alias" do+ url = "https://example.com/articles/entry?view=full"+ Fabricate(:embeddable_host, host: "example.com")+ topic_embed = Fabricate(:topic_embed, embed_url: "https://example.com/articles/entry")+ topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(url))++ expect { TopicRetriever.new(url, no_throttle: true).retrieve }.not_to change { Topic.count }+ expect(WebMock).not_to have_requested(:get, url)+ end+it "works with URLs with whitespaces" doexpect { TopicRetriever.new(" https://example.com ").retrieve }.not_to raise_errorenddiff --git a/spec/models/topic_embed_spec.rb b/spec/models/topic_embed_spec.rbindex 181417ff8d1..c106b3d087f 100644--- a/spec/models/topic_embed_spec.rb+++ b/spec/models/topic_embed_spec.rb@@ -878,6 +878,200 @@ RSpec.describe TopicEmbed doendend+ describe ".import_remote" do+ fab!(:user)+ fab!(:embeddable_host) { Fabricate(:embeddable_host, host: "example.com") }++ let(:original_url) { "https://example.com/articles/entry?view=full" }+ let(:canonical_url) { "https://example.com/articles/entry" }+ let(:content) do+ "<html><title>Embedded article</title><body><p>Article content</p></body></html>"+ end++ before do+ stub_request(:get, original_url).to_return(+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),+ )+ stub_request(:head, canonical_url).to_return(status: 200)+ stub_request(:get, canonical_url).to_return(body: content)+ end++ it "finds an imported topic using either the requested or canonical URL" do+ post = TopicEmbed.import_remote(original_url, user: user)++ expect(post.topic.topic_embed.embed_url).to eq(canonical_url)+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)+ expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(post.topic_id)+ end++ it "supports requested URLs longer than the alias index limit" do+ long_original_url = "https://example.com/articles/entry?source=#{"a" * 1_100}"+ stub_request(:get, long_original_url).to_return(+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),+ )++ post = TopicEmbed.import_remote(long_original_url, user: user)++ expect(TopicEmbedAlias.find_by(topic_embed: post.topic.topic_embed).url_key.length).to be >+ 1_000+ expect(TopicEmbed.topic_id_for_embed(long_original_url)).to eq(post.topic_id)+ expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(post.topic_id)+ end++ it "reassigns a requested URL to the latest successfully imported canonical topic" do+ other_url = "https://example.com/articles/entry?source=other"+ original_post = TopicEmbed.import_remote(original_url, user: user)+ original_embed = original_post.topic.topic_embed+ original_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(other_url))+ replacement_url = "https://example.com/articles/replacement"+ stub_request(:get, original_url).to_return(+ body: %(<html><head><link rel="canonical" href="#{replacement_url}"></head></html>),+ )+ stub_request(:head, replacement_url).to_return(status: 200)+ stub_request(:get, replacement_url).to_return(body: content)++ replacement_post = TopicEmbed.import_remote(original_url, user: user)++ expect(replacement_post.topic_id).not_to eq(original_post.topic_id)+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(replacement_post.topic_id)+ expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(original_post.topic_id)+ expect(TopicEmbed.topic_id_for_embed(other_url)).to eq(original_post.topic_id)+ expect(TopicEmbedAlias.where(TopicEmbedAlias.key_attributes(original_url)).count).to eq(1)+ end++ it "keeps an existing alias when a later remote import fails" do+ original_post = TopicEmbed.import_remote(original_url, user: user)+ stub_request(:get, original_url).to_return(status: 404)++ expect(TopicEmbed.import_remote(original_url, user: user)).to be_nil+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(original_post.topic_id)+ expect(TopicEmbedAlias.where(TopicEmbedAlias.key_attributes(original_url)).count).to eq(1)+ end++ it "remembers the requested URL when the canonical topic already exists" do+ post = TopicEmbed.import_remote(canonical_url, user: user)++ expect { TopicEmbed.import_remote(original_url, user: user) }.not_to change { Topic.count }++ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)+ end++ it "remembers multiple URLs without creating duplicate topics or aliases" do+ other_url = "https://example.com/articles/entry?format=print"+ stub_request(:get, other_url).to_return(+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),+ )+ post = TopicEmbed.import_remote(original_url, user: user)++ expect do+ TopicEmbed.import_remote(other_url, user: user)+ TopicEmbed.import_remote(original_url, user: user)+ end.not_to change { Topic.count }++ expect(post.topic.topic_embed.topic_embed_aliases.count).to eq(2)+ expect(TopicEmbed.topic_id_for_embed(other_url)).to eq(post.topic_id)+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)+ end++ it "remembers the requested URL after an HTTP redirect" do+ destination = "https://example.com/archive/entry"+ stub_request(:get, original_url).to_return(+ status: 302,+ headers: {+ "Location" => destination,+ },+ )+ stub_request(:get, destination).to_return(body: content)++ post = TopicEmbed.import_remote(original_url, user: user)++ expect(post.topic.topic_embed.embed_url).to eq(destination)+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)+ end++ it "does not create an alias when the requested URL is already canonical" do+ expect { TopicEmbed.import_remote(canonical_url, user: user) }.not_to change {+ TopicEmbedAlias.count+ }+ end++ it "does not create an alias when the remote page cannot be fetched" do+ stub_request(:get, original_url).to_return(status: 404)++ expect do+ expect(TopicEmbed.import_remote(original_url, user: user)).to be_nil+ end.not_to change { TopicEmbedAlias.count }+ end++ it "does not alias a canonical URL on another host" do+ destination = "https://other.example.com/articles/entry"+ stub_request(:get, original_url).to_return(+ body: %(<html><head><link rel="canonical" href="#{destination}"></head></html>),+ )+ stub_request(:head, destination).to_return(status: 200)+ stub_request(:get, destination).to_return(body: content)++ post = TopicEmbed.import_remote(original_url, user: user)++ expect(post.topic.topic_embed.embed_url).to eq(original_url)+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)+ expect(TopicEmbed.topic_id_for_embed(destination)).to be_nil+ expect(TopicEmbedAlias.count).to eq(0)+ end+ end++ describe ".topic_id_for_embed with aliases" do+ fab!(:topic_embed) { Fabricate(:topic_embed, embed_url: "https://example.com/articles/entry") }+ let(:original_url) { "https://example.com/entry?view=full" }++ before { topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(original_url)) }++ it "normalizes aliases in the same way as primary embed URLs" do+ expect(TopicEmbed.topic_id_for_embed("http://EXAMPLE.com/entry?view=full")).to eq(+ topic_embed.topic_id,+ )+ end++ it "does not discard meaningful query parameters" do+ expect(TopicEmbed.topic_id_for_embed("https://example.com/entry?view=other")).to be_nil+ expect(TopicEmbed.topic_id_for_embed("https://example.com/entry")).to be_nil+ end++ it "prefers an existing primary URL over an alias" do+ direct_embed = Fabricate(:topic_embed, embed_url: original_url)++ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(direct_embed.topic_id)+ end++ it "requires the full key to match the stored hash" do+ lookup_url = "https://example.com/entry?view=collision"+ stored_url = "https://example.com/entry?view=stored"+ inconsistent_attributes =+ TopicEmbedAlias.key_attributes(stored_url).merge(+ url_hash: TopicEmbedAlias.key_attributes(lookup_url)[:url_hash],+ )+ topic_embed.topic_embed_aliases.create!(inconsistent_attributes)++ expect(TopicEmbed.topic_id_for_embed(lookup_url)).to be_nil+ end++ it "does not resolve deleted embeds and restores aliases when recovered" do+ topic_embed.trash!+ expect(TopicEmbed.topic_id_for_embed(original_url)).to be_nil++ topic_embed.recover!+ expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(topic_embed.topic_id)+ end++ it "removes aliases when the imported topic is destroyed" do+ topic_embed.topic.destroy!++ expect(TopicEmbed.topic_id_for_embed(original_url)).to be_nil+ expect(TopicEmbedAlias.where(topic_embed_id: topic_embed.id)).to be_empty+ expect(TopicEmbed.where(id: topic_embed.id)).to be_empty+ end+ end+describe ".absolutize_urls" doit "handles badly formed URIs" doinvalid_url = "http://source.com/#double#anchor"diff --git a/spec/requests/embed_controller_spec.rb b/spec/requests/embed_controller_spec.rbindex b51e1aed7b9..46c0b981725 100644--- a/spec/requests/embed_controller_spec.rb+++ b/spec/requests/embed_controller_spec.rb@@ -321,6 +321,10 @@ RSpec.describe EmbedController dodescribe "full_app redirect" dofab!(:embeddable_host)+ let(:original_url) { "https://example.com/articles/entry?view=full" }+ let(:canonical_url) { "https://example.com/articles/entry" }+ let(:generic_embeddable_host) { Fabricate(:embeddable_host, host: "example.com") }+before { SiteSetting.embed_full_app = true }it "redirects to topic URL with embed_mode when full_app is present" do@@ -338,6 +342,125 @@ RSpec.describe EmbedController doexpect(response).to redirect_to("#{topic_embed.topic.url}?embed_mode=true")end+ it "renders loading before redirecting a canonical URL alias" do+ generic_embeddable_host+ Jobs.run_immediately!+ stub_request(:get, original_url).to_return(+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),+ )+ stub_request(:head, canonical_url).to_return(status: 200)+ stub_request(:get, canonical_url).to_return(+ body: "<html><title>Embedded article</title><body><p>Article content</p></body></html>",+ )++ get "/embed/comments",+ params: {+ embed_url: original_url,+ full_app: "true",+ },+ headers: {+ "REFERER" => original_url,+ }++ expect(response).to have_http_status(:ok)+ expect(response.body).to include("data-embed-state='loading'")++ topic_embed = TopicEmbed.find_by(embed_url: canonical_url)++ get "/embed/comments",+ params: {+ embed_url: original_url,+ full_app: "true",+ },+ headers: {+ "REFERER" => original_url,+ }++ expect(response).to redirect_to("#{topic_embed.topic.url}?embed_mode=true")+ end++ it "renders an imported canonical URL alias in classic mode" do+ generic_embeddable_host+ Jobs.run_immediately!+ stub_request(:get, original_url).to_return(+ body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),+ )+ stub_request(:head, canonical_url).to_return(status: 200)+ stub_request(:get, canonical_url).to_return(+ body: "<html><title>Embedded article</title><body><p>Article content</p></body></html>",+ )++ get "/embed/comments",+ params: {+ embed_url: original_url,+ },+ headers: {+ "REFERER" => original_url,+ }++ expect(response.body).to include("data-embed-state='loading'")++ get "/embed/comments",+ params: {+ embed_url: original_url,+ },+ headers: {+ "REFERER" => original_url,+ }++ expect(response).to have_http_status(:ok)+ expect(response.body).not_to include("data-embed-state='loading'")+ end++ it "returns not found for a canonical URL alias in a private category" do+ generic_embeddable_host+ restricted_category = Fabricate(:private_category, group: Fabricate(:group))+ restricted_topic = Fabricate(:topic, category: restricted_category)+ restricted_post = Fabricate(:post, topic: restricted_topic)+ topic_embed =+ Fabricate(+ :topic_embed,+ topic: restricted_topic,+ post: restricted_post,+ embed_url: canonical_url,+ )+ topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(original_url))++ get "/embed/comments",+ params: {+ embed_url: original_url,+ full_app: "true",+ },+ headers: {+ "REFERER" => original_url,+ }++ expect(response).to have_http_status(:not_found)+ end++ it "rejects a canonical URL alias after its original path is no longer allowed" do+ embeddable_host =+ Fabricate(+ :embeddable_host,+ host: "example.com",+ allowed_paths: %r{\A/articles/entry(?:\?.*)?\z}.source,+ )+ topic_embed = Fabricate(:topic_embed, embed_url: canonical_url)+ topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(original_url))+ embeddable_host.update!(allowed_paths: %r{\A/other/}.source)++ get "/embed/comments",+ params: {+ embed_url: original_url,+ full_app: "true",+ },+ headers: {+ "REFERER" => original_url,+ }++ expect(response).to have_http_status(:bad_request)+ end+it "redirects to topic URL with embed_mode when using topic_id" doget "/embed/comments",params: {