Skip to content

A platform for community discussion. Free, open, simple.

13.0 KB
Raw Blame
# frozen_string_literal: true

RSpec.describe DiscourseAi::Summarization::FoldContent do
  subject(:summarizer) { DiscourseAi::Summarization.topic_summary(topic) }

  let!(:llm_model) { assign_fake_provider_to(:ai_default_llm_model) }

  fab!(:topic) { Fabricate(:topic, highest_post_number: 2) }
  fab!(:post_1) { Fabricate(:post, topic: topic, post_number: 1, raw: "This is a text") }

  before do
    enable_current_plugin
    SiteSetting.ai_summarization_enabled = true
  end

  describe "#summarize" do
    before do
      context =
        DiscourseAi::Agents::BotContext.new(
          user: user,
          messages: summarizer.strategy.as_llm_messages(summarizer.strategy.targets_data),
        )
      prompt = summarizer.bot.agent.craft_prompt(context, llm: summarizer.bot.llm)
      response_format =
        if summarizer.bot.agent.response_format.present?
          DiscourseAi::Agents::Bot.build_json_schema(summarizer.bot.agent.response_format)
        end
      input_tokens, _, output_tokens =
        summarizer.bot.llm.prompt_capacity(
          prompt,
          max_tokens: 64,
          max_tokens_is_total: true,
          response_format:,
        )
      llm_model.update!(
        max_prompt_tokens: input_tokens + output_tokens + 128,
        max_output_tokens: 64,
      )
    end

    let(:summary) { "this is a summary" }

    fab!(:user)

    it "summarizes the content" do
      result =
        DiscourseAi::Completions::Llm.with_prepared_responses([summary]) do |spy, _, prompts|
          summarizer
            .summarize(user)
            .tap do
              expect(spy.completions).to eq(1)
              expect(prompts.first.messages.to_s).to include(post_1.raw)
            end
        end

      expect(result.summarized_text).to eq(summary)
    end

    it "fits the serialized prompt when post text expands during JSON encoding" do
      SiteSetting.ai_summarization_agent = Fabricate(:ai_agent).id
      bounded_summarizer = DiscourseAi::Summarization.topic_summary(topic)
      oversized_post =
        Fabricate(
          :post,
          topic: topic,
          post_number: 2,
          raw: "A post with whitespace #{"\n" * 2000}end",
        )
      context =
        DiscourseAi::Agents::BotContext.new(
          user: user,
          messages: bounded_summarizer.strategy.as_llm_messages([]),
        )
      prompt = bounded_summarizer.bot.agent.craft_prompt(context, llm: bounded_summarizer.bot.llm)
      input_tokens, _, output_tokens =
        bounded_summarizer.bot.llm.prompt_capacity(
          prompt,
          max_tokens: 256,
          max_tokens_is_total: true,
        )
      llm_model.update!(
        max_prompt_tokens: input_tokens + output_tokens + 384,
        max_output_tokens: 256,
      )

      bounded_summarizer.bot.llm.llm_model.reload
      result =
        DiscourseAi::Completions::Llm.with_prepared_responses([summary]) do |spy, _, prompts|
          bounded_summarizer
            .summarize(user)
            .tap do
              expect(spy.completions).to eq(1)
              content = prompts.first.messages.last[:content]
              expect(content).to include(post_1.raw)
              expect(content).not_to include(oversized_post.raw)
            end
        end

      expect(result.summarized_text).to eq(summary)
    end

    it "rejects a model context that cannot fit the summary instructions" do
      summarizer.bot.llm.llm_model.update!(max_prompt_tokens: 1)

      expect { summarizer.summarize(user) }.to raise_error(
        DiscourseAi::Completions::ContextPreparation::Error,
      ) { |error| expect(error.reason).to eq("hard_overflow") }
      expect(AiSummary.where(target: topic)).to be_empty
    end

    it "lets a forced-tool gist inspect delegated images before setting the summary" do
      upload =
        UploadCreator.new(
          file_from_fixtures(
            "1x1.jpg",
            "images",
            Rails.root.join("plugins/discourse-ai/spec/fixtures"),
          ),
          "gist.jpg",
        ).create_for(user.id)
      post_1.update_columns(
        user_id: user.id,
        raw: "#{"context " * 20}![image](#{upload.short_url})",
      )
      native_model = Fabricate(:llm_model, vision_enabled: true)
      delegated_model = Fabricate(:llm_model, vision_llm_model: native_model)
      gist_agent =
        AiAgent.find(DiscourseAi::Agents::Agent.system_agents[DiscourseAi::Agents::ShortSummarizer])
      gist_agent.update_column(:vision_enabled, true)
      AiAgent.agent_cache.flush!
      SiteSetting.ai_summary_gists_agent = gist_agent.id
      image_call =
        DiscourseAi::Completions::ToolCall.new(
          id: "view_1",
          name: "view_image",
          parameters: {
            images: [upload.id.to_s],
            question: "What information in this image is relevant to the discussion?",
          },
        )
      summary_call =
        DiscourseAi::Completions::ToolCall.new(
          id: "summary_1",
          name: "set_topic_summary",
          parameters: {
            summary: "Image-aware gist",
          },
        )

      result =
        DiscourseAi::Completions::Llm.with_prepared_responses(
          [image_call, "The image contains relevant context.", summary_call],
        ) do |spy|
          DiscourseAi::Summarization
            .topic_gist(topic, locale: "en", llm_model: delegated_model)
            .summarize(user)
            .tap { expect(spy.completions).to eq(3) }
        end

      expect(result.summarized_text).to eq("Image-aware gist")
    end

    it "forces the gist result after an image-aware attempt omits the output tool" do
      upload =
        UploadCreator.new(
          file_from_fixtures(
            "1x1.jpg",
            "images",
            Rails.root.join("plugins/discourse-ai/spec/fixtures"),
          ),
          "fallback-gist.jpg",
        ).create_for(user.id)
      post_1.update_columns(
        user_id: user.id,
        raw: "#{"context " * 20}![image](#{upload.short_url})",
      )
      native_model = Fabricate(:llm_model, vision_enabled: true)
      delegated_model = Fabricate(:llm_model, vision_llm_model: native_model)
      custom_agent = Fabricate(:ai_agent, vision_enabled: true)
      SiteSetting.ai_summary_gists_agent = custom_agent.id
      summary_call =
        DiscourseAi::Completions::ToolCall.new(
          id: "summary_1",
          name: "set_topic_summary",
          parameters: {
            summary: "Forced fallback gist",
          },
        )

      allow_any_instance_of(DiscourseAi::Completions::Llm).to receive(
        :generate,
      ).and_wrap_original do |original, *args, **kwargs, &block|
        result = original.call(*args, **kwargs, &block)
        kwargs[:execution_context]&.token_usage_tracker&.add_effective(
          request: 40_000,
          response: 30_000,
        )
        result
      end

      result =
        DiscourseAi::Completions::Llm.with_prepared_responses(
          ["Output without the required tool", summary_call],
        ) do |spy|
          DiscourseAi::Summarization
            .topic_gist(topic, locale: "en", llm_model: delegated_model)
            .summarize(user)
            .tap { expect(spy.completions).to eq(2) }
        end

      expect(result.summarized_text).to eq("Forced fallback gist")
    end

    it "forces the gist result immediately when the agent does not allow images" do
      upload =
        UploadCreator.new(
          file_from_fixtures(
            "1x1.jpg",
            "images",
            Rails.root.join("plugins/discourse-ai/spec/fixtures"),
          ),
          "policy-off-gist.jpg",
        ).create_for(user.id)
      post_1.update_columns(
        user_id: user.id,
        raw: "#{"context " * 20}![image](#{upload.short_url})",
      )
      native_model = Fabricate(:llm_model, vision_enabled: true)
      delegated_model = Fabricate(:llm_model, vision_llm_model: native_model)
      custom_agent = Fabricate(:ai_agent, vision_enabled: false)
      SiteSetting.ai_summary_gists_agent = custom_agent.id
      summary_call =
        DiscourseAi::Completions::ToolCall.new(
          id: "summary_1",
          name: "set_topic_summary",
          parameters: {
            summary: "Policy-off gist",
          },
        )

      result =
        DiscourseAi::Completions::Llm.with_prepared_responses([summary_call]) do |spy|
          DiscourseAi::Summarization
            .topic_gist(topic, locale: "en", llm_model: delegated_model)
            .summarize(user)
            .tap { expect(spy.completions).to eq(1) }
        end

      expect(result.summarized_text).to eq("Policy-off gist")
    end

    it "captures a tool-backed topic gist without structured output" do
      custom_agent =
        Fabricate(:ai_agent, response_format: [{ "key" => "fragile", "type" => "string" }])
      SiteSetting.ai_summary_gists_agent = custom_agent.id
      gist_summarizer = DiscourseAi::Summarization.topic_gist(topic, locale: "ja")
      expect(gist_summarizer.bot.agent.response_format).to be_nil
      expect(gist_summarizer.bot.returns_json?).to eq(false)
      tool_call =
        DiscourseAi::Completions::ToolCall.new(
          id: "call_1",
          name: "set_topic_summary",
          parameters: {
            summary: "日本語の要約",
          },
        )

      result =
        DiscourseAi::Completions::Llm.with_prepared_responses([tool_call]) do |spy|
          gist_summarizer
            .summarize(user)
            .tap do
              expect(spy.completions).to eq(1)
              expect(spy.model_params[:response_format]).to be_nil
            end
        end

      expect(result).to have_attributes(summarized_text: "日本語の要約", locale: "ja")
    end

    it "does not persist a gist when the model omits the summary tool" do
      gist_summarizer = DiscourseAi::Summarization.topic_gist(topic, locale: "ja")

      expect do
        DiscourseAi::Completions::Llm.with_prepared_responses(["plain text"]) do
          gist_summarizer.summarize(user)
        end
      end.to raise_error(DiscourseAi::Summarization::FoldContent::MissingToolOutput)

      expect(AiSummary.gist.where(target: topic, locale: "ja")).to be_empty
    end
  end

  describe "#existing_summary" do
    it "finds a gist stored under an equivalent regional locale" do
      existing_gist =
        Fabricate(
          :topic_ai_gist,
          target: topic,
          locale: "pt",
          highest_target_number: topic.highest_post_number,
          original_content_sha: AiSummary.build_sha("1"),
        )
      regional_summarizer = DiscourseAi::Summarization.topic_gist(topic, locale: "pt_BR")

      expect(regional_summarizer.existing_summary).to eq(existing_gist)
    end

    it "finds a complete summary stored under an equivalent regional locale" do
      existing_summary =
        Fabricate(
          :ai_summary,
          target: topic,
          locale: "pt",
          highest_target_number: topic.highest_post_number,
          original_content_sha: AiSummary.build_sha("1"),
        )
      regional_summarizer = DiscourseAi::Summarization.topic_summary(topic, locale: "pt_BR")

      expect(regional_summarizer.existing_summary).to eq(existing_summary)
    end

    context "when a summary already exists" do
      fab!(:ai_summary) do
        Fabricate(
          :ai_summary,
          target: topic,
          locale: SiteSetting.default_locale,
          highest_target_number: topic.highest_post_number,
          original_content_sha: AiSummary.build_sha("1"),
        )
      end

      it "doesn't mark it as outdated" do
        expect(summarizer.existing_summary.outdated).to eq(false)
      end

      context "when it's outdated because there are new targets" do
        before { Fabricate(:post, topic: topic, post_number: 2, raw: "This is a text") }

        it "marks it as outdated" do
          expect(summarizer.existing_summary.outdated).to eq(true)
        end
      end

      context "when it's outdated because existing content changes" do
        it "marks it as outdated" do
          ai_summary.update!(updated_at: 20.minutes.ago)
          post_1.update!(last_version_at: 5.minutes.ago)

          expect(summarizer.existing_summary.outdated).to eq(true)
        end
      end
    end
  end

  describe "#truncate" do
    it "leaves content within the token limit untouched" do
      text = "用 [keyd](https://man.archlinux.org/man/extra/keyd/keyd.1.en) 改键"

      expect(summarizer.truncate({ text: text.dup })[:text]).to eq(text)
    end

    it "doesn't let the tokenizer rewrite content within the token limit" do
      llm_model.update!(tokenizer: "DiscourseAi::Tokenizer::AnthropicTokenizer")
      text = "改键,很好用。" * 100

      expect(summarizer.truncate({ text: text.dup })[:text]).to eq(text)
    end

    it "keeps the start and the end of longer content" do
      item = summarizer.truncate({ text: "start #{"a " * 1500}#{"b " * 1500}end" })

      expect(item[:text]).to start_with("start a")
      expect(item[:text]).to end_with("b end")
    end

    it "preserves grapheme clusters at the split point" do
      item = summarizer.truncate({ text: "#{"🧩" * 600}⚖️#{"a" * 600}" })

      expect(item[:text]).to include(" ⚖️a")
    end
  end
end