Skip to content

A platform for community discussion. Free, open, simple.

12.7 KB
Raw Blame
# frozen_string_literal: true

RSpec.describe SharedAiConversation, type: :model do
  fab!(:claude_2) { Fabricate(:llm_model, name: "claude-2", display_name: "Claude-2") }

  before { prepare_ai_bot_fixtures(bots: [claude_2]) }

  fab!(:user)
  fab!(:agent) { Fabricate(:ai_agent, default_llm: claude_2).tap(&:ensure_user!) }

  let(:bad_user_input) { <<~HTML }
    Just trying something `<marquee style="font-size: 200px; color: red;" scrollamount=20>h4cked</marquee>`
  HTML
  let(:raw_with_details) { <<~HTML }
    <details>
    <summary>GitHub pull request diff</summary>
    <p><a href="https://github.com/discourse/discourse-ai/pull/521">discourse/discourse-ai 521</a></p>
    </details>
    <p>This is some other text</p>
  HTML

  let!(:topic) { Fabricate(:private_message_topic, user: user, recipient: agent.user) }
  let!(:post1) { Fabricate(:post, topic: topic, user: user, post_number: 1, raw: bad_user_input) }
  let!(:post2) do
    Fabricate(
      :post,
      topic: topic,
      user: agent.user,
      post_number: 2,
      raw: raw_with_details,
      custom_fields: {
        DiscourseAi::AiBot::POST_AI_AGENT_ID_FIELD => agent.id,
        DiscourseAi::AiBot::POST_AI_LLM_MODEL_ID_FIELD => claude_2.id,
        DiscourseAi::AiBot::POST_AI_LLM_NAME_FIELD => claude_2.display_name,
      },
    )
  end

  describe ".share_conversation" do
    it "creates a new conversation if one does not exist" do
      expect { described_class.share_conversation(user, topic) }.to change {
        described_class.count
      }.by(1)
    end

    it "generates a good onebox" do
      conversation = described_class.share_conversation(user, topic)
      onebox = conversation.onebox
      expect(onebox).not_to include("GitHub pull request diff")
      expect(onebox).not_to include("<details>")

      expect(onebox).to include("AI Conversation with Claude-2")
    end

    it "updates an existing conversation if one exists" do
      conversation = described_class.share_conversation(user, topic)
      expect(conversation.share_key).to be_present

      topic.update!(title: "New title")

      expect { described_class.share_conversation(user, topic) }.to_not change {
        described_class.count
      }
      expect(conversation.reload.title).to eq("New title")
      expect(conversation.share_key).to be_present
    end

    it "includes the correct conversation data" do
      conversation = described_class.share_conversation(user, topic)
      expect(conversation.llm_name).to eq("Claude-2")
      expect(conversation.title).to eq(topic.title)
      expect(conversation.context.size).to eq(2)
      expect(conversation.context[0]["id"]).to eq(post1.id)
      expect(conversation.context[1]["id"]).to eq(post2.id)

      populated_context = conversation.populated_context

      expect(populated_context[0].id).to eq(post1.id)
      expect(populated_context[0].user.id).to eq(post1.user.id)
      expect(populated_context[1].id).to eq(post2.id)
      expect(populated_context[1].user.id).to eq(post2.user.id)
    end

    it "summarizes mixed-model conversations from per-response snapshots" do
      second_model = Fabricate(:llm_model, display_name: "Second model")
      third_post =
        Fabricate(
          :post,
          topic: topic,
          user: agent.user,
          post_number: 3,
          raw: "A response generated by the second model.",
          custom_fields: {
            DiscourseAi::AiBot::POST_AI_AGENT_ID_FIELD => agent.id,
            DiscourseAi::AiBot::POST_AI_LLM_MODEL_ID_FIELD => second_model.id,
            DiscourseAi::AiBot::POST_AI_LLM_NAME_FIELD => second_model.display_name,
          },
        )

      conversation = described_class.share_conversation(user, topic)
      claude_2.update!(display_name: "Renamed model")

      expect(conversation.llm_name).to eq(I18n.t("discourse_ai.multiple_models"))
      expect(conversation.context.find { |post| post["id"] == post2.id }["llm_name"]).to eq(
        "Claude-2",
      )
      expect(conversation.context.find { |post| post["id"] == third_post.id }["llm_name"]).to eq(
        "Second model",
      )
    end

    it "does not infer response attribution from topic defaults or human custom fields" do
      topic.custom_fields[DiscourseAi::AiBot::TOPIC_AI_AGENT_ID_FIELD] = agent.id
      topic.save_custom_fields
      historical_response =
        Fabricate(
          :post,
          topic:,
          user: agent.user,
          post_number: 3,
          custom_fields: {
            DiscourseAi::AiBot::POST_AI_LLM_NAME_FIELD => claude_2.display_name,
          },
        )
      human_post =
        Fabricate(
          :post,
          topic:,
          user: user,
          post_number: 4,
          custom_fields: {
            DiscourseAi::AiBot::POST_AI_AGENT_ID_FIELD => agent.id,
            DiscourseAi::AiBot::POST_AI_LLM_NAME_FIELD => "Forged model",
          },
        )

      failed_response =
        Fabricate(
          :post,
          topic:,
          user: agent.user,
          post_number: 5,
          custom_fields: {
            DiscourseAi::AiBot::POST_AI_AGENT_ID_FIELD => agent.id,
          },
        )

      conversation = described_class.share_conversation(user, topic)
      historical_data = conversation.context.find { |post| post["id"] == historical_response.id }
      human_data = conversation.context.find { |post| post["id"] == human_post.id }
      failed_data = conversation.context.find { |post| post["id"] == failed_response.id }

      expect(historical_data).to include("llm_name" => claude_2.display_name)
      expect(historical_data).not_to have_key("agent")
      expect(human_data).not_to have_key("agent")
      expect(human_data).not_to have_key("llm_name")
      expect(failed_data).to include("agent" => agent.name)
      expect(failed_data).not_to have_key("llm_name")
    end

    it "shares artifacts publicly when conversation is shared" do
      # Create a post with an AI artifact
      artifact =
        Fabricate(
          :ai_artifact,
          post: post1,
          user: user,
          metadata: {
            public: false,
            something: "good",
          },
        )

      _post_with_artifact =
        Fabricate(
          :post,
          topic: topic,
          post_number: 3,
          raw: "Here's an artifact",
          cooked:
            "<div class='ai-artifact' data-ai-artifact-id='#{artifact.id}' data-ai-artifact-version='1'></div>",
        )

      expect(artifact.public?).to be_falsey

      conversation = described_class.share_conversation(user, topic)
      artifact.reload

      expect(artifact.metadata["something"]).to eq("good")
      expect(artifact.public?).to be_truthy

      described_class.destroy_conversation(conversation)
      artifact.reload

      expect(artifact.metadata["something"]).to eq("good")
      expect(artifact.public?).to be_falsey
    end

    it "does not share artifacts publicly when refreshing the share fails" do
      SiteSetting.ai_artifact_security = "lax"

      conversation = described_class.share_conversation(user, topic)
      conversation.update_column(:share_key, "")

      artifact =
        Fabricate(
          :ai_artifact,
          post: post1,
          user: user,
          metadata: {
            public: false,
            something: "good",
          },
        )

      Fabricate(
        :post,
        topic: topic,
        post_number: 3,
        raw: "Here's an artifact",
        cooked: "<div class='ai-artifact' data-ai-artifact-id='#{artifact.id}'></div>",
      )

      described_class.share_conversation(user, topic)

      expect(artifact.reload.metadata["something"]).to eq("good")
      expect(artifact.public?).to be_falsey
    end

    it "escapes HTML" do
      conversation = described_class.share_conversation(user, topic)
      onebox = conversation.onebox
      expect(onebox).not_to include("</marquee>")
      expect(onebox).to include("AI Conversation with Claude-2")
    end

    it "escapes HTML in the title to prevent XSS" do
      conversation = described_class.share_conversation(user, topic)
      xss_title = "<img src=x onerror=alert(1)>"
      conversation.update!(title: xss_title)
      onebox = conversation.onebox

      expect(onebox).not_to include(xss_title)
      expect(onebox).to include(ERB::Util.html_escape(xss_title))
    end
  end

  describe "#publicly_visible?" do
    let(:visibility_topic) { Fabricate(:private_message_topic, user: user, recipient: agent.user) }
    let!(:visibility_post) { Fabricate(:post, topic: visibility_topic, user: user) }
    let(:conversation) { described_class.share_conversation(user, visibility_topic) }

    before do
      group = Fabricate(:group)
      group.add(user)
      user.reload
      SiteSetting.ai_bot_public_sharing_allowed_groups = group.id.to_s
    end

    it "accepts matching preloaded topics and posts while rejecting unrelated preloads" do
      expect(conversation.publicly_visible?).to eq(true)
      expect(
        conversation.publicly_visible?(
          topic: visibility_topic,
          posts_by_id: {
            visibility_post.id => visibility_post,
          },
        ),
      ).to eq(true)

      other_topic = Fabricate(:private_message_topic, user: user, recipient: agent.user)
      other_post = Fabricate(:post, topic: other_topic, user: user)
      expect(
        conversation.publicly_visible?(
          topic: other_topic,
          posts_by_id: {
            visibility_post.id => visibility_post,
          },
        ),
      ).to eq(false)
      expect(
        conversation.publicly_visible?(
          topic: visibility_topic,
          posts_by_id: {
            visibility_post.id => other_post,
          },
        ),
      ).to eq(false)
      same_topic_post = Fabricate(:post, topic: visibility_topic, user: user)
      expect(
        conversation.publicly_visible?(
          topic: visibility_topic,
          posts_by_id: {
            visibility_post.id => same_topic_post,
          },
        ),
      ).to eq(false)
    end

    it "rejects hidden context posts the source user cannot see" do
      conversation
      visibility_post.update_columns(user_id: agent.user.id, hidden: true)
      expect(user.guardian.can_share_ai_bot_conversation?(visibility_topic)).to eq(true)
      expect(user.guardian.can_see?(visibility_post.reload)).to eq(false)

      expect(conversation.publicly_visible?).to eq(false)
      expect(
        conversation.publicly_visible?(
          topic: visibility_topic,
          posts_by_id: {
            visibility_post.id => visibility_post,
          },
        ),
      ).to eq(false)
    end

    it "rejects trashed context posts even when supplied as preloaded records" do
      conversation
      visibility_post.trash!

      expect(conversation.publicly_visible?).to eq(false)
      expect(
        conversation.publicly_visible?(
          topic: visibility_topic,
          posts_by_id: {
            visibility_post.id => visibility_post,
          },
        ),
      ).to eq(false)
    end

    it "rejects context posts belonging to another topic" do
      other_topic = Fabricate(:private_message_topic, user: user, recipient: agent.user)
      other_post = Fabricate(:post, topic: other_topic, user: user)
      conversation.update!(context: [{ id: other_post.id, cooked: other_post.cooked }])

      expect(conversation.publicly_visible?).to eq(false)
      expect(
        conversation.publicly_visible?(
          topic: visibility_topic,
          posts_by_id: {
            other_post.id => other_post,
          },
        ),
      ).to eq(false)
    end

    it "rejects a trashed topic even when the topic was preloaded before deletion" do
      conversation
      visibility_topic.trash!

      expect(conversation.publicly_visible?).to eq(false)
      expect(
        conversation.publicly_visible?(
          topic: visibility_topic,
          posts_by_id: {
            visibility_post.id => visibility_post,
          },
        ),
      ).to eq(false)
    end
  end

  describe "#onebox" do
    it "escapes title and username" do
      malicious_username = %(user"><img src=x onerror=alert(1)>)
      malicious_title = %(title</a><script>alert("x")</script>)
      user.update_columns(username: malicious_username, username_lower: malicious_username.downcase)
      topic.update_column(:title, malicious_title)

      Fabricate(:post, topic: topic, user: user, post_number: 3, raw: "this is a safe post")

      conversation = described_class.share_conversation(user, topic)

      onebox = conversation.onebox

      expect(onebox).to include("user&quot;&gt;&lt;img src=x onerror=alert(1)&gt;")
      expect(onebox).to include("title&lt;/a&gt;&lt;script&gt;alert(&quot;x&quot;)&lt;/script&gt;")

      expect(onebox).not_to include("<img src=x onerror=alert(1)>")
      expect(onebox).not_to include(%(<script>alert("x")</script>))
    end
  end
end