Skip to content

A platform for community discussion. Free, open, simple.

49.0 KB
Raw Blame
# frozen_string_literal: true

require "stringio"

RSpec.describe TopicEmbed do
  it { is_expected.to belong_to :topic }
  it { is_expected.to belong_to :post }
  it { is_expected.to validate_presence_of :embed_url }

  describe ".import" do
    fab!(:user) { Fabricate(:user, refresh_auto_groups: true) }
    let(:title) { "How to turn a fish from good to evil in 30 seconds" }
    let(:url) { "http://eviltrout.com/123" }
    let(:contents) do
      "<p>hello world new post <a href='/hello'>hello</a> <img src='images/wat.jpg'></p>"
    end

    fab!(:embeddable_host)
    fab!(:category)
    fab!(:tag)

    it "returns nil when the URL is malformed" do
      expect(TopicEmbed.import(user, "invalid url", title, contents)).to eq(nil)
      expect(TopicEmbed.count).to eq(0)
    end

    it "Allows figure, figcaption, details HTML tags" do
      html = <<~HTML
        <html>
        <head>
           <title>Some title</title>
        </head>
        <body>
          <div class='content'>
            <p>some content</p>
            <figure>
              <img src="/a.png">
              <figcaption>Some caption</figcaption>
            </figure>
            <details>
              some details
            </details>
          </div>
        </body>
        </html>
      HTML

      parsed = TopicEmbed.parse_html(html, "https://blog.discourse.com/somepost.html")

      # div inception is inserted by the readability gem
      expected = <<~HTML
        <div><div>
          <div>
            <p>some content</p>
            <figure>
              <img src="https://blog.discourse.com/a.png">
              <figcaption>Some caption</figcaption>
            </figure>
            <details>
              some details
            </details>
          </div>
        </div></div>
      HTML
      expect(parsed.body.strip).to eq(expected.strip)
    end

    # ideally, articles get a heavier weightage than td elements
    # so to force that, we do not allow td elements to be scored
    it "does not score td tags" do
      html = <<~HTML
        <html>
        <head>
           <title>Some title</title>
        </head>
        <body>
          <article>
            article content
            <table>
              <tr>
                <td>
                  <p>cats</p>
                  <p>cats</p>
                </td>
              </tr>
            </table>
          </article>
        </body>
        </html>
      HTML

      parsed = TopicEmbed.parse_html(html, "https://blog.discourse.com/somepost.html")
      expected = "<div><div> article content cats cats </div></div>"

      expect(parsed.body.squish).to eq(expected.squish)
    end

    context "with a charset in the Content-Type header" do
      let(:html) { <<~HTML.b }
          <html>
          <head>
            <title>Iran’s seven-day offer has ‘significant poison pills’</title>
          </head>
          <body>
            <p>The deal has what diplomats call “significant poison pills” built into the text of the offer, and negotiators on both sides say the seven-day window may be shorter than the headline suggests.</p>
          </body>
          </html>
        HTML

      it "decodes the title using the declared charset" do
        parsed =
          TopicEmbed.parse_html(
            html,
            "https://blog.discourse.com/somepost.html",
            content_type: "text/html; charset=UTF-8",
          )
        expect(parsed.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")
      end

      it "decodes the body using the declared charset" do
        parsed =
          TopicEmbed.parse_html(
            html,
            "https://blog.discourse.com/somepost.html",
            content_type: "text/html; charset=UTF-8",
          )
        expect(parsed.body).to include("“significant poison pills”")
      end
    end

    context "with a misleading single-byte charset in the header" do
      it "prefers the page's UTF-8 declaration when the bytes are valid UTF-8" do
        html = <<~HTML.b
          <html>
          <head>
            <meta charset="utf-8">
            <title>Iran’s seven-day offer has ‘significant poison pills’</title>
          </head>
          <body>
            <p>The deal has what diplomats call “significant poison pills” built into the text of the offer, and negotiators on both sides say the seven-day window may be shorter than the headline suggests.</p>
          </body>
          </html>
        HTML

        parsed =
          TopicEmbed.parse_html(
            html,
            "https://blog.discourse.com/somepost.html",
            content_type: "text/html; charset=ISO-8859-1",
          )

        expect(parsed.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")
        expect(parsed.body).to include("“significant poison pills”")
      end

      it "keeps decoding a genuine single-byte page as declared" do
        html =
          +"<html><head><title>caf\xE9</title></head><body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body></html>".b

        parsed =
          TopicEmbed.parse_html(
            html,
            "https://blog.discourse.com/somepost.html",
            content_type: "text/html; charset=ISO-8859-1",
          )

        expect(parsed.title).to eq("café")
      end

      it "keeps the header charset when the page declares none" do
        html = <<~HTML.b
          <html>
          <head>
            <title>Iran’s seven-day offer has ‘significant poison pills’</title>
          </head>
          <body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body>
          </html>
        HTML

        parsed =
          TopicEmbed.parse_html(
            html,
            "https://blog.discourse.com/somepost.html",
            content_type: "text/html; charset=ISO-8859-1",
          )

        expect(parsed.title).to eq(
          "Iranâ\u0080\u0099s seven-day offer has â\u0080\u0098significant poison pillsâ\u0080\u0099",
        )
      end
    end

    it "falls back to a meta charset when no Content-Type header is present" do
      html = <<~HTML.b
        <html>
        <head>
          <meta charset="utf-8">
          <title>Iran’s seven-day offer</title>
        </head>
        <body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body>
        </html>
      HTML

      parsed = TopicEmbed.parse_html(html, "https://blog.discourse.com/somepost.html")

      expect(parsed.title).to eq("Iran’s seven-day offer")
    end

    it "degrades to sniffing when the header charset has no UTF-8 converter" do
      html =
        +"<html><head><title>News headline</title></head><body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body></html>".b

      parsed =
        TopicEmbed.parse_html(
          html,
          "https://blog.discourse.com/somepost.html",
          content_type: "text/html; charset=UTF-7",
        )

      expect(parsed.title).to eq("News headline")
    end

    it "sniffs the whole document when the declared charset mis-parses the bytes" do
      html =
        +"<html><head><title>News headline</title></head><body><p>The deal has poison pills built into the text of the offer and negotiators say the seven-day window may be shorter.</p></body></html>".b

      parsed =
        TopicEmbed.parse_html(
          html,
          "https://blog.discourse.com/somepost.html",
          content_type: "text/html; charset=UTF-32BE",
        )

      expect(parsed.title).to eq("News headline")
      expect(parsed.body).to include("shorter")
    end

    context "when creating a post" do
      let!(:post) { TopicEmbed.import(user, url, title, contents) }
      let(:topic_embed) { TopicEmbed.find_by(post: post) }

      it "works as expected with a new URL" do
        expect(post).to be_present

        # It uses raw_html rendering
        expect(post.cook_method).to eq(Post.cook_methods[:raw_html])
        expect(post.cooked).to eq(post.raw)

        # It converts relative URLs to absolute
        expect(post.cooked).to have_tag("a", with: { href: "http://eviltrout.com/hello" })
        expect(post.cooked).to have_tag("img", with: { src: "http://eviltrout.com/images/wat.jpg" })

        # It caches the embed content
        expect(post.topic.topic_embed.embed_content_cache).to eq(contents)
        expect(post.topic.topic_embed.content_truncated).to eq(false)

        # It converts relative URLs to absolute when expanded
        expect(TopicEmbed.expanded_for(post)).to have_tag(
          "img",
          with: {
            src: "http://eviltrout.com/images/wat.jpg",
          },
        )

        expect(post.topic.has_topic_embed?).to eq(true)
        expect(TopicEmbed.where(topic_id: post.topic_id)).to be_present

        expect(post.topic.category).to eq(embeddable_host.category)
        expect(post.topic).not_to be_visible
      end

      it "Supports updating the post content" do
        expect do
          TopicEmbed.import(user, url, "New title received", "<p>muhahaha new contents!</p>")
        end.to change { topic_embed.reload.content_sha1 }
        expect(topic_embed.topic.title).to eq("New title received")

        expect(topic_embed.post.cooked).to match(/new contents/)
      end

      it "Supports updating the post author" do
        new_user = Fabricate(:user)
        TopicEmbed.import(new_user, url, title, contents)

        topic_embed.reload
        expect(topic_embed.post.user).to eq(new_user)
        expect(topic_embed.post.topic.user).to eq(new_user)
      end

      it "Supports updating the embed content cache" do
        expect do TopicEmbed.import(user, url, title, "new contents") end.to change {
          topic_embed.reload.embed_content_cache
        }
        expect(topic_embed.embed_content_cache).to eq("new contents")
      end

      it "updates an explicitly selected cook method on reimport" do
        TopicEmbed.import(user, url, title, contents, cook_method: Post.cook_methods[:regular])

        expect(post.reload.cook_method).to eq(Post.cook_methods[:regular])
      end

      it "leaves an uppercase feed entry URL untouched in content" do
        cased_url = "http://eviltrout.com/ABCD"
        post = TopicEmbed.import(user, cased_url, title, "some random content")
        expect(post.cooked).to match(/#{cased_url}/)
      end

      it "leaves a lowercase feed entry URL untouched in content" do
        cased_url = "http://eviltrout.com/abcd"
        post = TopicEmbed.import(user, cased_url, title, "some random content")
        expect(post.cooked).to match(/#{cased_url}/)
      end

      it "falls back to the url when the title is blank" do
        blank_title_url = "http://eviltrout.com/blank-title"
        imported_post = TopicEmbed.import(user, blank_title_url, "", contents)

        expect(imported_post.topic.title).to eq(blank_title_url)
      end

      it "preserves an existing title when a later import has a blank title" do
        imported_post = TopicEmbed.import(user, url, title, contents)

        TopicEmbed.import(user, url, "", "<p>updated content</p>")

        expect(imported_post.topic.reload.title).to eq(title)
      end

      shared_examples "topic is unlisted" do
        it "unlists the topic until someone replies" do
          Jobs.run_immediately!
          imported_post =
            TopicEmbed.import(user, "http://eviltrout.com/abcd", title, "some random content")
          expect(imported_post.topic).not_to be_visible
          pc =
            PostCreator.new(
              Fabricate(:user),
              raw: "this is a reply that will make the topic visible",
              topic_id: imported_post.topic_id,
              reply_to_post_number: 1,
            )
          pc.create
          expect(imported_post.topic.reload).to be_visible
        end
      end

      context "when import embed unlisted is true" do
        before { SiteSetting.import_embed_unlisted = true }

        include_examples "topic is unlisted"

        context "when embed unlisted is false" do
          before { SiteSetting.embed_unlisted = false }

          include_examples "topic is unlisted"
        end
      end

      context "when import embed unlisted is false" do
        before { SiteSetting.import_embed_unlisted = false }

        context "when embed unlisted is false" do
          before { SiteSetting.embed_unlisted = false }

          it "lists the topic" do
            Jobs.run_immediately!
            imported_post =
              TopicEmbed.import(user, "http://eviltrout.com/abcd", title, "some random content")
            expect(imported_post.topic).to be_visible
          end
        end

        context "when embed unlisted is true" do
          before { SiteSetting.embed_unlisted = true }

          include_examples "topic is unlisted"
        end
      end

      it "creates the topic in the category passed as a parameter" do
        Jobs.run_immediately!
        imported_post =
          TopicEmbed.import(
            user,
            "http://eviltrout.com/abcd",
            title,
            "some random content",
            category_id: category.id,
          )
        expect(imported_post.topic.category).not_to eq(embeddable_host.category)
        expect(imported_post.topic.category).to eq(category)
      end

      it "does not create duplicate topics with different protocols in the embed_url" do
        Jobs.run_immediately!
        expect {
          TopicEmbed.import(user, "http://eviltrout.com/abcd", title, "some random content")
        }.to change { Topic.all.count }.by(1)

        expect {
          TopicEmbed.import(user, "https://eviltrout.com/abcd", title, "some random content")
        }.to_not change { Topic.all.count }
      end

      it "creates the topic with the tag passed as a parameter" do
        Jobs.run_immediately!
        SiteSetting.tagging_enabled = true
        imported_post =
          TopicEmbed.import(
            user,
            "http://eviltrout.com/abcd",
            title,
            "some random content",
            tags: [tag.name],
          )
        expect(imported_post.topic.tags).to include(tag)
      end

      it "respects overriding the cook_method when asked" do
        Jobs.run_immediately!
        SiteSetting.embed_support_markdown = false
        stub_request(:get, "https://www.youtube.com/watch?v=K56soYl0U1w").to_return(
          status: 200,
          body: "",
          headers: {
          },
        )
        stub_request(:get, "https://www.youtube.com/embed/K56soYl0U1w").to_return(
          status: 200,
          body: "",
          headers: {
          },
        )
        stub_request(
          :get,
          "https://www.youtube.com/oembed?url=https://www.youtube.com/watch?v=K56soYl0U1w",
        ).to_return(status: 200, body: "", headers: {})

        imported_post =
          TopicEmbed.import(
            user,
            "http://eviltrout.com/abcd",
            title,
            "https://www.youtube.com/watch?v=K56soYl0U1w",
            cook_method: Post.cook_methods[:regular],
          )
        expect(imported_post.cooked).to match(/onebox|iframe/)
      end

      describe "topic_embed_import_create_args modifier" do
        after { DiscoursePluginRegistry.clear_modifiers! }

        it "can alter the args used to create the topic" do
          plugin = Plugin::Instance.new
          plugin.register_modifier(:topic_embed_import_create_args) do |args|
            args[:title] = "MODIFIED: #{args[:title]}"

            args
          end

          Jobs.run_immediately!
          imported_post =
            TopicEmbed.import(
              user,
              "http://eviltrout.com/abcd",
              title,
              "some random content",
              category_id: category.id,
            )
          expect(imported_post.topic.title).to eq("MODIFIED: #{title}")
        end

        it "reverts to defaults when the modifier returns nil" do
          plugin = Plugin::Instance.new
          plugin.register_modifier(:topic_embed_import_create_args) { |args| nil }

          Jobs.run_immediately!
          imported_post =
            TopicEmbed.import(
              user,
              "http://eviltrout.com/abcd",
              title,
              "some random content",
              category_id: category.id,
            )
          expect(imported_post.topic.title).to eq(title)
        end
      end
    end

    context "when post creation supports markdown rendering" do
      before { SiteSetting.embed_support_markdown = true }

      it "works as expected" do
        post = TopicEmbed.import(user, url, title, "some random content")
        expect(post).to be_present

        # It uses regular rendering
        expect(post.cook_method).to eq(Post.cook_methods[:regular])
      end
    end

    context "when importing a topic embed with string tags" do
      fab!(:tag1) { Fabricate(:tag, name: "interesting") }
      fab!(:tag2) { Fabricate(:tag, name: "article") }
      let(:tags) { [tag1.name, tag2.name] }

      it "associates the specified tags with the existing topic" do
        imported_page = TopicEmbed.import(user, url, title, contents, tags: tags)
        expect(imported_page.topic.tags).to match_array([tag1, tag2])
      end
    end

    context "when updating an existing topic embed with string tags" do
      fab!(:tag1) { Fabricate(:tag, name: "interesting") }
      fab!(:tag2) { Fabricate(:tag, name: "article") }
      let(:tags) { [tag1, tag2] }

      before { TopicEmbed.import(user, url, title, contents, tags: [tag1.name]) }

      it "associates the specified tags with the existing topic" do
        imported_page = TopicEmbed.import(user, url, title, contents, tags: tags)
        expect(imported_page.topic.tags).to match_array([tag1, tag2])
      end

      it "does not update tags if tags are nil or unspecified" do
        imported_page = TopicEmbed.import(user, url, title, contents)
        expect(imported_page.topic.tags).to match_array([tag1])
        imported_page = TopicEmbed.import(user, url, title, contents, tags: nil)
        expect(imported_page.topic.tags).to match_array([tag1])
      end

      it "does update tags if tags are empty" do
        imported_page = TopicEmbed.import(user, url, title, contents, tags: [])
        expect(imported_page.topic.tags).to match_array([])
      end

      it "does not revise the post when the tags cannot all be saved" do
        SiteSetting.max_tags_per_topic = 1
        TopicEmbed.import(user, url, title, contents, tags: tags)

        Post.any_instance.expects(:revise).never
        TopicEmbed.import(user, url, title, contents, tags: tags)
      end

      it "does not revise the post when the existing tags match the incoming tags" do
        TopicEmbed.import(user, url, title, contents, tags: tags)
        SiteSetting.max_tags_per_topic = 1

        Post.any_instance.expects(:revise).never
        TopicEmbed.import(user, url, title, contents, tags: tags)
      end
    end

    context "with specified user and tags" do
      fab!(:tag1) { Fabricate(:tag, name: "interesting") }
      fab!(:tag2) { Fabricate(:tag, name: "article") }

      let!(:new_user) { Fabricate(:user) }
      let(:tags) { [tag1.name, tag2.name] }
      let(:imported_post) { TopicEmbed.import(new_user, url, title, contents, tags: tags) }

      it "assigns the specified user as the author" do
        expect(imported_post.user).to eq(new_user)
      end

      it "associates the specified tags with the topic" do
        expect(imported_post.topic.tags).to contain_exactly(tag1, tag2)
      end
    end

    context "when the embeddable host specifies the user and tags" do
      fab!(:tag1) { Fabricate(:tag, name: "interesting") }
      fab!(:tag2) { Fabricate(:tag, name: "article") }
      fab!(:embeddable_host) { Fabricate(:embeddable_host, host: "tag-eviltrout.com") }

      let!(:new_user) { Fabricate(:user) }
      let(:tags) { [tag1.name, tag2.name] }
      let(:embed_url_with_tags) { "http://tag-eviltrout.com/abcd" }

      let(:imported_post) do
        # passing user = system and tags = nil to ensure we're getting the user and tags from the embeddable host
        # and not from the TopicEmbed.import method in the tests
        TopicEmbed.import(Discourse.system_user, embed_url_with_tags, title, contents, tags: nil)
      end

      before do
        embeddable_host.user = new_user
        embeddable_host.tags = [tag1, tag2]
        embeddable_host.save!
      end

      it "assigns the specified user as the author" do
        expect(imported_post.user).to eq(new_user)
      end

      it "associates the specified tags with the topic" do
        expect(imported_post.topic.tags).to contain_exactly(tag1, tag2)
      end
    end

    context "when updating an existing post with new tags and a different user" do
      fab!(:tag1) { Fabricate(:tag, name: "interesting") }
      fab!(:tag2) { Fabricate(:tag, name: "article") }

      let!(:admin) { Fabricate(:admin) }
      let!(:new_admin) { Fabricate(:admin) }
      let(:tags) { [tag1.name, tag2.name] }

      before { SiteSetting.tagging_enabled = true }

      it "updates the user and adds new tags" do
        original_post = TopicEmbed.import(admin, url, title, contents)

        expect(original_post.user).to eq(admin)
        expect(original_post.topic.tags).to be_empty

        embeddable_host.update!(
          tags: [tag1, tag2],
          user: new_admin,
          category: category,
          host: "eviltrout.com",
        )

        edited_post = TopicEmbed.import(admin, url, title, contents)

        expect(edited_post.user).to eq(new_admin)
        expect(edited_post.topic.tags).to match_array([tag1, tag2])
      end
    end

    describe "embedded content truncation" do
      MAX_LENGTH_BEFORE_TRUNCATION = 100

      let(:long_content) { "<p>#{"a" * MAX_LENGTH_BEFORE_TRUNCATION}</p>\n<p>more</p>" }

      it "truncates the imported post when truncation is enabled" do
        SiteSetting.embed_truncate = true
        post = TopicEmbed.import(user, url, title, long_content)

        expect(post.raw).not_to include(long_content)
        expect(post.topic.topic_embed.content_truncated).to eq(true)
      end

      it "keeps everything in the imported post when truncation is disabled" do
        SiteSetting.embed_truncate = false
        post = TopicEmbed.import(user, url, title, long_content)

        expect(post.raw).to include(long_content)
        expect(post.topic.topic_embed.content_truncated).to eq(false)
      end

      it "looks at first div when there is no paragraph" do
        no_para = "<div><h>testing it</h></div>"

        SiteSetting.embed_truncate = true
        post = TopicEmbed.import(user, url, title, no_para)

        expect(post.raw).to include("testing it")
      end

      it "keeps the complete markup when truncation would only remove a wrapper" do
        wrapped_content = "<div><p>#{"Complete paragraph. " * 30}</p></div>"
        SiteSetting.embed_truncate = true

        post = TopicEmbed.import(user, url, title, wrapped_content)

        expect(post.raw).to include(wrapped_content)
        expect(post.topic.topic_embed.content_truncated).to eq(false)
      end

      it "keeps image-only content" do
        SiteSetting.embed_truncate = true
        post = TopicEmbed.import(user, url, title, "<img src='/comic.png' alt='comic'>")

        expect(post.raw).to have_tag(
          "img",
          with: {
            src: "http://eviltrout.com/comic.png",
            alt: "comic",
          },
        )
        expect(post.topic.topic_embed.content_truncated).to eq(false)
      end

      it "truncates independently of the cook method" do
        SiteSetting.embed_truncate = true
        post =
          TopicEmbed.import(
            user,
            url,
            title,
            "**Meetup overview**\n\nTalk details",
            cook_method: Post.cook_methods[:regular],
            truncate: true,
          )

        expect(post.raw).not_to include("Talk details")
        expect(post.cooked).to have_tag("strong", text: "Meetup overview")
        expect(post.topic.topic_embed.content_truncated).to eq(true)
      end

      it "refreshes expanded content when the excerpt is unchanged" do
        SiteSetting.embed_truncate = true
        first_paragraph = "<p>#{"Same introduction. " * 10}</p>"
        original_contents = "#{first_paragraph}<p>Original ending.</p>"
        updated_contents = "#{first_paragraph}<p>Updated ending.</p>"

        post = TopicEmbed.import(user, url, title, original_contents)
        expect(TopicEmbed.expanded_for(post)).to include("Original ending.")

        TopicEmbed.import(user, url, title, updated_contents)

        expect(post.reload.raw).not_to include("Updated ending.")
        expect(post.topic.topic_embed.reload.embed_content_cache).to eq(updated_contents)
        expect(TopicEmbed.expanded_for(post)).to include("Updated ending.")
      end
    end
  end

  describe ".topic_id_for_embed" do
    it "returns correct topic id irrespective of url protocol" do
      topic_embed = Fabricate(:topic_embed, embed_url: "http://example.com/post/248")

      expect(TopicEmbed.topic_id_for_embed("http://exAMPle.com/post/248")).to eq(
        topic_embed.topic_id,
      )
      expect(TopicEmbed.topic_id_for_embed("https://example.com/post/248/")).to eq(
        topic_embed.topic_id,
      )

      expect(TopicEmbed.topic_id_for_embed("http://example.com/post/248/2")).to eq(nil)
      expect(TopicEmbed.topic_id_for_embed("http://examples.com/post/248")).to eq(nil)
      expect(TopicEmbed.topic_id_for_embed("http://example.com/post/24")).to eq(nil)
      expect(TopicEmbed.topic_id_for_embed("http://example.com/post")).to eq(nil)
    end

    it "finds the topic id when the stored embed_url has mixed case" do
      topic_embed = Fabricate(:topic_embed, embed_url: "https://Example.com/Post/248")

      expect(TopicEmbed.topic_id_for_embed("http://example.com/post/248")).to eq(
        topic_embed.topic_id,
      )
    end

    it "returns the oldest topic id when both http and https embeds exist" do
      http_embed = Fabricate(:topic_embed, embed_url: "http://example.com/post/248")
      Fabricate(:topic_embed, embed_url: "https://example.com/post/248")

      expect(TopicEmbed.topic_id_for_embed("https://example.com/post/248")).to eq(
        http_embed.topic_id,
      )
    end

    it "finds the topic id when the embed_url contains a query string" do
      topic_embed = Fabricate(:topic_embed, embed_url: "http://example.com/post/248?key=foo")
      expect(TopicEmbed.topic_id_for_embed("http://example.com/post/248?key=foo")).to eq(
        topic_embed.topic_id,
      )
    end
  end

  describe ".find_remote" do
    fab!(:embeddable_host)

    describe ".title_scrub" do
      let(:url) { "http://eviltrout.com/123" }
      let(:contents) do
        "<title>Through the Looking Glass - Classic Books</title><body>some content here</body>"
      end

      before { stub_request(:get, url).to_return(status: 200, body: contents) }

      it "doesn't scrub the title by default" do
        response = TopicEmbed.find_remote(url)
        expect(response.title).to eq("Through the Looking Glass - Classic Books")
      end

      it "scrubs the title when the option is enabled" do
        SiteSetting.embed_title_scrubber = " - Classic Books$"
        response = TopicEmbed.find_remote(url)
        expect(response.title).to eq("Through the Looking Glass")
      end

      it "doesn't follow redirect when making request" do
        FinalDestination.any_instance.stubs(:resolve).returns(URI("https://redirect.com"))
        stub_request(:get, "https://redirect.com/").to_return(
          status: 301,
          body: "<title>Moved permanently</title>",
          headers: {
            "Location" => "https://www.example.org/",
          },
        )
        response = TopicEmbed.find_remote(url)
        expect(response.title).to eq("Moved permanently")
      end
    end

    context "with a charset declared in the HTTP Content-Type header" do
      let(:url) { "http://eviltrout.com/123" }
      let(:contents) do
        "<title>Iran’s seven-day offer has ‘significant poison pills’</title><body>some content here</body>"
      end

      before do
        stub_request(:get, url).to_return(
          status: 200,
          headers: {
            "Content-Type" => "text/html; charset=UTF-8",
          },
          body: contents.b,
        )
      end

      it "decodes the title as UTF-8" do
        response = TopicEmbed.find_remote(url)
        expect(response.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")
      end
    end

    context 'with post with allowed classes "foo" and "emoji"' do
      fab!(:user)
      let(:url) { "http://eviltrout.com/123" }
      let(:contents) do
        "my normal size emoji <p class='foo'>Hi</p> <img class='emoji other foo' src='/images/smiley.jpg'>"
      end
      let(:response) { TopicEmbed.find_remote(url) }

      before do
        SiteSetting.allowed_embed_classnames = "emoji, foo"
        stub_request(:get, url).to_return(status: 200, body: contents)
      end

      it "has no author tag" do
        expect(response.author).to be_blank
      end

      it "img node has emoji class" do
        expect(response.body).to have_tag("img", with: { class: "emoji" })
      end

      it "img node has foo class" do
        expect(response.body).to have_tag("img", with: { class: "foo" })
      end

      it "p node has foo class" do
        expect(response.body).to have_tag("p", with: { class: "foo" })
      end

      it "nodes removes classes other than emoji" do
        expect(response.body).to have_tag("img", without: { class: "other" })
      end
    end

    context "with post with author metadata" do
      fab!(:user) { Fabricate(:user, username: "eviltrout") }
      let(:url) { "http://eviltrout.com/321" }
      let(:contents) do
        '<html><head><meta name="author" content="eviltrout"></head><body>rich and morty</body></html>'
      end

      before(:each) { stub_request(:get, url).to_return(status: 200, body: contents) }

      it "has no author tag" do
        response = TopicEmbed.find_remote(url)

        expect(response.author).to eq(user)
      end
    end

    context "with post with no allowed classes" do
      fab!(:user)
      let(:url) { "http://eviltrout.com/123" }
      let(:contents) do
        "my normal size emoji <p class='foo'>Hi</p> <img class='emoji other foo' src='/images/smiley.jpg'>"
      end
      let(:response) { TopicEmbed.find_remote(url) }

      before(:each) do
        SiteSetting.allowed_embed_classnames = ""
        stub_request(:get, url).to_return(status: 200, body: contents)
      end

      it 'img node doesn\'t have emoji class' do
        expect(response.body).to have_tag("img", without: { class: "emoji" })
      end

      it 'img node doesn\'t have foo class' do
        expect(response.body).to have_tag("img", without: { class: "foo" })
      end

      it 'p node doesn\'t foo class' do
        expect(response.body).to have_tag("p", without: { class: "foo" })
      end

      it 'img node doesn\'t have other class' do
        expect(response.body).to have_tag("img", without: { class: "other" })
      end
    end

    context "with non-ascii URL" do
      let(:url) { "http://eviltrout.com/test/ماهی" }
      let(:contents) { "<title>سلام</title><body>این یک پاراگراف آزمون است.</body>" }

      before { stub_request(:get, url).to_return(status: 200, body: contents) }

      it "doesn't throw an error" do
        response = TopicEmbed.find_remote(url)
        expect(response.title).to eq("سلام")
      end
    end

    context "with encoded URL" do
      let(:url) { "http://example.com/hello%20world" }
      let(:contents) { "<title>Hello World!</title><body></body>" }

      before { stub_request(:get, url).to_return(status: 200, body: contents) }

      it "doesn't throw an error" do
        response = TopicEmbed.find_remote(url)
        expect(response.title).to eq("Hello World!")
      end
    end

    context "with non-http URL" do
      it "throws an error" do
        url = "/test.txt"

        expect(TopicEmbed.find_remote(url)).to be_nil
      end
    end

    context "with emails" do
      let(:url) { "http://example.com/foo" }
      let(:contents) do
        '<p><a href="mailto:foo%40example.com">URL encoded @ symbol</a></p><p><a href="mailto:[email protected]">normal mailto link</a></p>'
      end

      before { stub_request(:get, url).to_return(status: 200, body: contents) }

      it "handles mailto links" do
        response = TopicEmbed.find_remote(url)

        expect(response.body).to have_tag("a", with: { href: "mailto:[email protected]" })
        expect(response.body).to have_tag("a", with: { href: "mailto:[email protected]" })
      end
    end

    context "with malformed href" do
      let(:url) { "http://example.com/foo" }
      let(:contents) { '<p><a href="(http://foo.bar)">Baz</a></p>' }

      before { stub_request(:get, url).to_return(status: 200, body: contents) }

      it "doesn’t raise an exception" do
        expect { TopicEmbed.find_remote(url) }.not_to raise_error
      end
    end

    context "with canonical links" do
      fab!(:user)
      let(:title) { "How to turn a fish from good to evil in 30 seconds" }
      let(:url) { "http://eviltrout.com/123?asd" }
      let(:canonical_url) { "http://eviltrout.com/123" }
      let(:url2) { "http://eviltrout.com/blog?post=1&canonical=false" }
      let(:canonical_url2) { "http://eviltrout.com/blog?post=1" }
      let(:content) { "<head><link rel=\"canonical\" href=\"#{canonical_url}\"></head>" }
      let(:content2) { "<head><link rel=\"canonical\" href=\"#{canonical_url2}\"></head>" }
      let(:canonical_content) { "<title>Canonical</title><body></body>" }

      before do
        stub_request(:get, url).to_return(status: 200, body: content)
        stub_request(:head, canonical_url)
        stub_request(:get, canonical_url).to_return(status: 200, body: canonical_content)

        stub_request(:get, url2).to_return(status: 200, body: content2)
        stub_request(:head, canonical_url2)
        stub_request(:get, canonical_url2).to_return(status: 200, body: canonical_content)
      end

      it "fetches canonical content" do
        response = TopicEmbed.find_remote(url)

        expect(response.title).to eq("Canonical")
        expect(response.url).to eq(canonical_url)
      end

      it "does not create duplicate topics when url differs from canonical_url" do
        Jobs.run_immediately!
        expect { TopicEmbed.import_remote(canonical_url, { title: title, user: user }) }.to change {
          Topic.all.count
        }.by(1)

        expect { TopicEmbed.import_remote(url, { title: title, user: user }) }.to_not change {
          Topic.all.count
        }
      end

      it "does not create duplicate topics when url contains extra params" do
        Jobs.run_immediately!
        expect {
          TopicEmbed.import_remote(canonical_url2, { title: title, user: user })
        }.to change { Topic.all.count }.by(1)

        expect { TopicEmbed.import_remote(url2, { title: title, user: user }) }.to_not change {
          Topic.all.count
        }
      end
    end

    context "when canonical URL points to a different domain" do
      fab!(:user)
      let(:title) { "Some article title" }
      let(:original_url) { "http://staging.example.com/article/123" }
      let(:canonical_url) { "http://production.example.com/article/123" }
      let(:original_content) do
        "<head><link rel=\"canonical\" href=\"#{canonical_url}\"></head><body>Article content</body>"
      end
      let(:canonical_content) do
        "<title>#{title}</title><body>Article content from canonical</body>"
      end

      before do
        stub_request(:get, original_url).to_return(status: 200, body: original_content)
        stub_request(:head, canonical_url)
        stub_request(:get, canonical_url).to_return(status: 200, body: canonical_content)
      end

      it "stores the original URL as embed_url so embeds can be found" do
        Jobs.run_immediately!
        post = TopicEmbed.import_remote(original_url, { title: title, user: user })

        topic_embed = TopicEmbed.find_by(topic_id: post.topic_id)
        expect(topic_embed.embed_url).to include("staging.example.com")

        found_topic_id = TopicEmbed.topic_id_for_embed(original_url)
        expect(found_topic_id).to eq(post.topic_id)
      end
    end
  end

  describe ".import_remote" do
    fab!(:user)
    fab!(:embeddable_host) { Fabricate(:embeddable_host, host: "example.com") }

    let(:original_url) { "https://example.com/articles/entry?view=full" }
    let(:canonical_url) { "https://example.com/articles/entry" }
    let(:content) do
      "<html><title>Embedded article</title><body><p>Article content</p></body></html>"
    end

    before do
      stub_request(:get, original_url).to_return(
        body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),
      )
      stub_request(:head, canonical_url).to_return(status: 200)
      stub_request(:get, canonical_url).to_return(body: content)
    end

    it "finds an imported topic using either the requested or canonical URL" do
      post = TopicEmbed.import_remote(original_url, user: user)

      expect(post.topic.topic_embed.embed_url).to eq(canonical_url)
      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
      expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(post.topic_id)
    end

    it "supports requested URLs longer than the alias index limit" do
      long_original_url = "https://example.com/articles/entry?source=#{"a" * 1_100}"
      stub_request(:get, long_original_url).to_return(
        body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),
      )

      post = TopicEmbed.import_remote(long_original_url, user: user)

      expect(TopicEmbedAlias.find_by(topic_embed: post.topic.topic_embed).url_key.length).to be >
        1_000
      expect(TopicEmbed.topic_id_for_embed(long_original_url)).to eq(post.topic_id)
      expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(post.topic_id)
    end

    it "reassigns a requested URL to the latest successfully imported canonical topic" do
      other_url = "https://example.com/articles/entry?source=other"
      original_post = TopicEmbed.import_remote(original_url, user: user)
      original_embed = original_post.topic.topic_embed
      original_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(other_url))
      replacement_url = "https://example.com/articles/replacement"
      stub_request(:get, original_url).to_return(
        body: %(<html><head><link rel="canonical" href="#{replacement_url}"></head></html>),
      )
      stub_request(:head, replacement_url).to_return(status: 200)
      stub_request(:get, replacement_url).to_return(body: content)

      replacement_post = TopicEmbed.import_remote(original_url, user: user)

      expect(replacement_post.topic_id).not_to eq(original_post.topic_id)
      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(replacement_post.topic_id)
      expect(TopicEmbed.topic_id_for_embed(canonical_url)).to eq(original_post.topic_id)
      expect(TopicEmbed.topic_id_for_embed(other_url)).to eq(original_post.topic_id)
      expect(TopicEmbedAlias.where(TopicEmbedAlias.key_attributes(original_url)).count).to eq(1)
    end

    it "keeps an existing alias when a later remote import fails" do
      original_post = TopicEmbed.import_remote(original_url, user: user)
      stub_request(:get, original_url).to_return(status: 404)

      expect(TopicEmbed.import_remote(original_url, user: user)).to be_nil
      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(original_post.topic_id)
      expect(TopicEmbedAlias.where(TopicEmbedAlias.key_attributes(original_url)).count).to eq(1)
    end

    it "remembers the requested URL when the canonical topic already exists" do
      post = TopicEmbed.import_remote(canonical_url, user: user)

      expect { TopicEmbed.import_remote(original_url, user: user) }.not_to change { Topic.count }

      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
    end

    it "remembers multiple URLs without creating duplicate topics or aliases" do
      other_url = "https://example.com/articles/entry?format=print"
      stub_request(:get, other_url).to_return(
        body: %(<html><head><link rel="canonical" href="#{canonical_url}"></head></html>),
      )
      post = TopicEmbed.import_remote(original_url, user: user)

      expect do
        TopicEmbed.import_remote(other_url, user: user)
        TopicEmbed.import_remote(original_url, user: user)
      end.not_to change { Topic.count }

      expect(post.topic.topic_embed.topic_embed_aliases.count).to eq(2)
      expect(TopicEmbed.topic_id_for_embed(other_url)).to eq(post.topic_id)
      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
    end

    it "remembers the requested URL after an HTTP redirect" do
      destination = "https://example.com/archive/entry"
      stub_request(:get, original_url).to_return(
        status: 302,
        headers: {
          "Location" => destination,
        },
      )
      stub_request(:get, destination).to_return(body: content)

      post = TopicEmbed.import_remote(original_url, user: user)

      expect(post.topic.topic_embed.embed_url).to eq(destination)
      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
    end

    it "does not create an alias when the requested URL is already canonical" do
      expect { TopicEmbed.import_remote(canonical_url, user: user) }.not_to change {
        TopicEmbedAlias.count
      }
    end

    it "does not create an alias when the remote page cannot be fetched" do
      stub_request(:get, original_url).to_return(status: 404)

      expect do
        expect(TopicEmbed.import_remote(original_url, user: user)).to be_nil
      end.not_to change { TopicEmbedAlias.count }
    end

    it "does not alias a canonical URL on another host" do
      destination = "https://other.example.com/articles/entry"
      stub_request(:get, original_url).to_return(
        body: %(<html><head><link rel="canonical" href="#{destination}"></head></html>),
      )
      stub_request(:head, destination).to_return(status: 200)
      stub_request(:get, destination).to_return(body: content)

      post = TopicEmbed.import_remote(original_url, user: user)

      expect(post.topic.topic_embed.embed_url).to eq(original_url)
      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(post.topic_id)
      expect(TopicEmbed.topic_id_for_embed(destination)).to be_nil
      expect(TopicEmbedAlias.count).to eq(0)
    end
  end

  describe ".topic_id_for_embed with aliases" do
    fab!(:topic_embed) { Fabricate(:topic_embed, embed_url: "https://example.com/articles/entry") }
    let(:original_url) { "https://example.com/entry?view=full" }

    before { topic_embed.topic_embed_aliases.create!(TopicEmbedAlias.key_attributes(original_url)) }

    it "normalizes aliases in the same way as primary embed URLs" do
      expect(TopicEmbed.topic_id_for_embed("http://EXAMPLE.com/entry?view=full")).to eq(
        topic_embed.topic_id,
      )
    end

    it "does not discard meaningful query parameters" do
      expect(TopicEmbed.topic_id_for_embed("https://example.com/entry?view=other")).to be_nil
      expect(TopicEmbed.topic_id_for_embed("https://example.com/entry")).to be_nil
    end

    it "prefers an existing primary URL over an alias" do
      direct_embed = Fabricate(:topic_embed, embed_url: original_url)

      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(direct_embed.topic_id)
    end

    it "requires the full key to match the stored hash" do
      lookup_url = "https://example.com/entry?view=collision"
      stored_url = "https://example.com/entry?view=stored"
      inconsistent_attributes =
        TopicEmbedAlias.key_attributes(stored_url).merge(
          url_hash: TopicEmbedAlias.key_attributes(lookup_url)[:url_hash],
        )
      topic_embed.topic_embed_aliases.create!(inconsistent_attributes)

      expect(TopicEmbed.topic_id_for_embed(lookup_url)).to be_nil
    end

    it "does not resolve deleted embeds and restores aliases when recovered" do
      topic_embed.trash!
      expect(TopicEmbed.topic_id_for_embed(original_url)).to be_nil

      topic_embed.recover!
      expect(TopicEmbed.topic_id_for_embed(original_url)).to eq(topic_embed.topic_id)
    end

    it "removes aliases when the imported topic is destroyed" do
      topic_embed.topic.destroy!

      expect(TopicEmbed.topic_id_for_embed(original_url)).to be_nil
      expect(TopicEmbedAlias.where(topic_embed_id: topic_embed.id)).to be_empty
      expect(TopicEmbed.where(id: topic_embed.id)).to be_empty
    end
  end

  describe ".absolutize_urls" do
    it "handles badly formed URIs" do
      invalid_url = "http://source.com/#double#anchor"
      contents = "hello world new post <a href='/hello'>hello</a>"

      raw = TopicEmbed.absolutize_urls(invalid_url, contents)
      expect(raw).to eq("hello world new post <a href=\"http://source.com/hello\">hello</a>")
    end

    it "handles malformed links" do
      url = "https://somesource.com"

      contents = <<~HTML
        hello world new post <a href="mailto:[email protected]>">hello</a>
        some image <img src="https:/><invalidimagesrc/">
      HTML

      raw = TopicEmbed.absolutize_urls(url, contents)
      expect(raw).to eq(contents)
    end
  end

  describe ".imported_from_html" do
    after { I18n.reload! }

    it "uses the default site locale for the 'imported_from' footer" do
      TranslationOverride.upsert!(
        "en",
        "embed.imported_from",
        "English translation of embed.imported_from with %{link}",
      )
      TranslationOverride.upsert!(
        "de",
        "embed.imported_from",
        "German translation of embed.imported_from with %{link}",
      )

      I18n.locale = :en
      expected_html = TopicEmbed.imported_from_html("some_url")

      I18n.locale = :de
      expect(TopicEmbed.imported_from_html("some_url")).to eq(expected_html)
    end

    it "normalize_encodes the url" do
      html =
        TopicEmbed.imported_from_html(
          'http://www.discourse.org/%23<%2Fa><img%20src%3Dx%20onerror%3Dalert("document.domain")%3B>',
        )
      expected_html =
        "\n<hr>\n<small>This is a companion discussion topic for the original entry at <a href='http://www.discourse.org/%23%3C/a%3E%3Cimg%20src=x%20onerror=alert(%22document.domain%22);%3E'>http://www.discourse.org/%23%3C/a%3E%3Cimg%20src=x%20onerror=alert(%22document.domain%22);%3E</a></small>\n"
      expect(html).to eq(expected_html)
    end

    it "escapes the URL in the raw HTML footer" do
      url = "http://eviltrout.com/'onmouseover='window.topicEmbedXssExecuted=true"
      link = Nokogiri::HTML5.fragment(TopicEmbed.imported_from_html(url)).at_css("a")

      expect(link["href"]).to eq(url)
      expect(link["onmouseover"]).to be_nil
      expect(link.text).to eq(url)
    end
  end

  describe ".expanded_for" do
    fab!(:user)
    let(:title) { "How to turn a fish from good to evil in 30 seconds" }
    let(:url) { "http://eviltrout.com/123" }
    let(:contents) { "<p>hello world new post :D</p>" }

    fab!(:embeddable_host)
    fab!(:category)
    fab!(:tag)

    it "returns embed content" do
      stub_request(:get, url).to_return(status: 200, body: "<p>remote content</p>")
      post = TopicEmbed.import(user, url, title, contents)
      expanded = TopicEmbed.expanded_for(post)

      expect(expanded).to include(contents)
      expect(expanded).not_to include("remote content")
      expect(WebMock).not_to have_requested(:get, url)
    end

    it "falls back to the remote page for a legacy embed without cached content" do
      remote_contents =
        "<html><body><article><p>Content from the remote page.</p></article></body></html>"
      stub_request(:get, url).to_return(status: 200, body: remote_contents)
      post = TopicEmbed.import(user, url, title, contents)
      post.topic.topic_embed.update!(embed_content_cache: nil, content_truncated: nil)

      expect(TopicEmbed.expanded_for(post)).to include("Content from the remote page.")
      expect(post.topic.topic_embed.reload.embed_content_cache).to include(
        "Content from the remote page.",
      )
    end

    it "does not fetch the remote page for an imported item with missing cached content" do
      stub_request(:get, url).to_return(status: 200, body: "<p>remote content</p>")
      post = TopicEmbed.import(user, url, title, contents)
      post.topic.topic_embed.update!(embed_content_cache: nil, content_truncated: true)

      expect { TopicEmbed.expanded_for(post) }.to raise_error(Discourse::NotFound)
      expect(WebMock).not_to have_requested(:get, url)
    end

    it "cooks cached Markdown using the post cook method" do
      post =
        TopicEmbed.import(
          user,
          url,
          title,
          "**Meetup overview**\n\nTalk details",
          cook_method: Post.cook_methods[:regular],
          truncate: true,
        )

      expanded = TopicEmbed.expanded_for(post)

      expect(expanded).to have_tag("strong", text: "Meetup overview")
      expect(expanded).to include("Talk details")
    end

    it "sanitizes cached raw HTML" do
      post =
        TopicEmbed.import(
          user,
          url,
          title,
          "<p>Safe content</p><script>window.evil = true</script>",
        )

      expanded = TopicEmbed.expanded_for(post)

      expect(expanded).to include("Safe content")
      expect(expanded).not_to have_tag("script")
    end
  end
end