A platform for community discussion. Free, open, simple.
FEATURE: Add opt-in IAM role assumption for S3 operations (#39174)
Alan Guo Xiang Tanfae7cd138dc8
diff --git a/Gemfile b/Gemfileindex b7f47a31c01..1a24cd992a6 100644--- a/Gemfile+++ b/Gemfile@@ -63,6 +63,7 @@ gem "fastimage"gem "aws-sdk-s3", require: falsegem "aws-sdk-sns", require: false+gem "aws-sdk-sts", require: falsegem "aws-sdk-mediaconvert", require: falsegem "aws-sdk-bedrockruntime", require: falsegem "excon"diff --git a/Gemfile.lock b/Gemfile.lockindex 2d2432f7d3b..d5611e421b5 100644--- a/Gemfile.lock+++ b/Gemfile.lock@@ -84,6 +84,9 @@ GEMaws-sdk-sns (1.96.0)aws-sdk-core (~> 3, >= 3.216.0)aws-sigv4 (~> 1.5)+ aws-sdk-sts (1.12.0)+ aws-sdk-core (~> 3, >= 3.110.0)+ aws-sigv4 (~> 1.1)aws-sigv4 (1.12.1)aws-eventstream (~> 1, >= 1.0.2)base64 (0.3.0)@@ -782,6 +785,7 @@ DEPENDENCIESaws-sdk-mediaconvertaws-sdk-s3aws-sdk-sns+ aws-sdk-stsbetter_errorsbootsnapbullet@@ -954,6 +958,7 @@ CHECKSUMSaws-sdk-mediaconvert (1.165.0) sha256=d955a571cd8b0407c0778e1d0f2333a70bf9ab48e36c81da8c5b317db24001e4aws-sdk-s3 (1.182.0) sha256=d0fc3579395cb6cb69bf6e975240ce031fc673190e74c8dddbdd6c18572b450daws-sdk-sns (1.96.0) sha256=f92b3c7203c53181b1cafb3dbbea85f330002ad696175bda829cfef359fa6dd4+ aws-sdk-sts (1.12.0) sha256=9bbd64223dd2423540291d9faaac4a169ace8cfe123b1075c9a127fcd65e061faws-sigv4 (1.12.1) sha256=6973ff95cb0fd0dc58ba26e90e9510a2219525d07620c8babeb70ef831826c00base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97bbenchmark (0.5.0) sha256=465df122341aedcb81a2a24b4d3bd19b6c67c1530713fd533f3ff034e419236cdiff --git a/app/services/video_conversion/aws_media_convert_adapter.rb b/app/services/video_conversion/aws_media_convert_adapter.rbindex 123e82c0b62..c08eac211a7 100644--- a/app/services/video_conversion/aws_media_convert_adapter.rb+++ b/app/services/video_conversion/aws_media_convert_adapter.rb@@ -349,13 +349,8 @@ module VideoConversiondef create_basic_client(endpoint: nil)client_options = { region: SiteSetting.s3_region }client_options[:endpoint] = endpoint if endpoint.present?-- if !SiteSetting.s3_use_iam_profile- client_options[:credentials] = Aws::Credentials.new(- SiteSetting.s3_access_key_id,- SiteSetting.s3_secret_access_key,- )- end+ creds = S3Helper.s3_credentials(SiteSetting)+ client_options[:credentials] = creds if credsAws::MediaConvert::Client.new(client_options)enddiff --git a/config/discourse_defaults.conf b/config/discourse_defaults.confindex 186acce8d7d..5ea8c8273ab 100644--- a/config/discourse_defaults.conf+++ b/config/discourse_defaults.conf@@ -243,6 +243,8 @@ s3_endpoint =s3_http_continue_timeout =s3_install_cors_rule =enable_s3_transfer_acceleration =+s3_role_arn =+s3_role_session_name =# Optionally, specify a separate CDN to be used for static JS assets stored on S3s3_asset_cdn_url =diff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex 2bbd643a0a4..ed27a4e7dea 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -2144,6 +2144,8 @@ en:s3_upload_bucket: "The Amazon S3 bucket name that files will be uploaded into. WARNING: must be lowercase, no periods, no underscores."s3_access_key_id: "The Amazon S3 access key id that will be used to upload images, attachments, and backups."s3_secret_access_key: "The Amazon S3 secret access key that will be used to upload images, attachments, and backups."+ s3_role_arn: "The ARN of the IAM role to assume for S3 operations. When set alongside access keys, Discourse will use STS AssumeRole to obtain temporary credentials."+ s3_role_session_name: "Optional session name used when assuming an IAM role for S3 operations. Appears in CloudTrail logs for auditing. Falls back to the server hostname if blank."s3_region: "The Amazon S3 region name that will be used to upload images and backups."s3_cdn_url: "The CDN URL to use for all s3 assets (for example: https://cdn.somewhere.com). WARNING: after changing this setting you must rebake all old posts."s3_use_cdn_url_for_all_uploads: "Use CDN URL for all the files uploaded to s3 instead of only for images."diff --git a/config/site_settings.yml b/config/site_settings.ymlindex a3323506fb0..c87e8eadff8 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -2243,6 +2243,15 @@ files:s3_secret_access_key:default: ""secret: true+ s3_role_arn:+ default: ""+ min: 20+ max: 2048 # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html+ s3_role_session_name:+ default: ""+ min: 2+ max: 64+ regex: '^[\w+=,.@\-]+$' # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.htmls3_region:default: "us-east-1"enum: "S3RegionSiteSetting"diff --git a/lib/s3_helper.rb b/lib/s3_helper.rbindex 27d92d187dd..36c496b7259 100644--- a/lib/s3_helper.rb+++ b/lib/s3_helper.rb@@ -1,6 +1,7 @@# frozen_string_literal: truerequire "aws-sdk-s3"+require "aws-sdk-sts"class S3HelperFIFTEEN_MEGABYTES = 15 * 1024 * 1024@@ -297,14 +298,35 @@ class S3Helperopts[:http_continue_timeout] = SiteSetting.s3_http_continue_timeoutopts[:use_dualstack_endpoint] = SiteSetting.Upload.use_dualstack_endpoint- unless obj.s3_use_iam_profile- opts[:access_key_id] = obj.s3_access_key_id- opts[:secret_access_key] = obj.s3_secret_access_key- end+ creds = s3_credentials(obj)+ opts[:credentials] = creds if credsoptsend+ def self.s3_credentials(obj, stub_responses: false)+ return nil if obj.s3_use_iam_profile++ if obj.s3_role_arn.present?+ # RoleSessionName max 64 chars: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html+ session_name = obj.s3_role_session_name.presence || Discourse.os_hostname[0...64]+ sts_client =+ Aws::STS::Client.new(+ region: obj.s3_region,+ access_key_id: obj.s3_access_key_id,+ secret_access_key: obj.s3_secret_access_key,+ stub_responses: stub_responses,+ )+ Aws::AssumeRoleCredentials.new(+ role_arn: obj.s3_role_arn,+ role_session_name: session_name,+ client: sts_client,+ )+ else+ Aws::Credentials.new(obj.s3_access_key_id, obj.s3_secret_access_key)+ end+ end+def download_file(filename, destination_path, failure_message = nil)object(filename).download_file(destination_path)rescue => errdiff --git a/spec/lib/s3_helper_spec.rb b/spec/lib/s3_helper_spec.rbindex c065853c258..b2bb1c4ad09 100644--- a/spec/lib/s3_helper_spec.rb+++ b/spec/lib/s3_helper_spec.rb@@ -448,4 +448,55 @@ RSpec.describe "S3Helper" do)endend++ describe ".s3_credentials" do+ it "returns AssumeRoleCredentials when s3_role_arn is configured" do+ SiteSetting.s3_region = "us-east-1"+ SiteSetting.s3_access_key_id = "some-key"+ SiteSetting.s3_secret_access_key = "some-secret"+ SiteSetting.s3_role_arn = "arn:aws:iam::123456789012:role/some-role"+ SiteSetting.s3_role_session_name = "some-session"++ creds = S3Helper.s3_credentials(SiteSetting, stub_responses: true)++ expect(creds).to be_a(Aws::AssumeRoleCredentials)+ expect(creds.assume_role_params[:role_arn]).to eq("arn:aws:iam::123456789012:role/some-role")+ expect(creds.assume_role_params[:role_session_name]).to eq("some-session")+ expect(creds.client.config.access_key_id).to eq("some-key")+ expect(creds.client.config.secret_access_key).to eq("some-secret")+ end++ it "returns static Aws::Credentials when s3_role_arn is blank" do+ SiteSetting.s3_region = "us-east-1"+ SiteSetting.s3_access_key_id = "some-key"+ SiteSetting.s3_secret_access_key = "some-secret"+ SiteSetting.s3_role_arn = ""++ creds = S3Helper.s3_credentials(SiteSetting)++ expect(creds).to be_a(Aws::Credentials)+ expect(creds.access_key_id).to eq("some-key")+ expect(creds.secret_access_key).to eq("some-secret")+ end++ it "returns nil when s3_use_iam_profile is true" do+ SiteSetting.s3_use_iam_profile = true+ SiteSetting.s3_role_arn = "arn:aws:iam::123456789012:role/some-role"++ expect(S3Helper.s3_credentials(SiteSetting)).to be_nil+ end++ it "falls back to Discourse.os_hostname when s3_role_session_name is blank" do+ SiteSetting.s3_region = "us-east-1"+ SiteSetting.s3_access_key_id = "some-key"+ SiteSetting.s3_secret_access_key = "some-secret"+ SiteSetting.s3_role_arn = "arn:aws:iam::123456789012:role/some-role"+ SiteSetting.s3_role_session_name = ""+ Discourse.stubs(:os_hostname).returns("some-host")++ creds = S3Helper.s3_credentials(SiteSetting, stub_responses: true)++ expect(creds.assume_role_params[:role_session_name]).to eq("some-host")+ end+ endenddiff --git a/spec/services/video_conversion/aws_media_convert_adapter_spec.rb b/spec/services/video_conversion/aws_media_convert_adapter_spec.rbindex 7124b5748d7..442e0f434fb 100644--- a/spec/services/video_conversion/aws_media_convert_adapter_spec.rb+++ b/spec/services/video_conversion/aws_media_convert_adapter_spec.rb@@ -226,27 +226,19 @@ RSpec.describe VideoConversion::AwsMediaConvertAdapter doendend- context "when using IAM profile" do- before do- allow(SiteSetting).to receive(:s3_use_iam_profile).and_return(true)- allow(SiteSetting).to receive(:s3_access_key_id).and_return("")- allow(SiteSetting).to receive(:s3_secret_access_key).and_return("")- upload.update!(- url: "//#{s3_bucket}.s3.#{s3_region}.amazonaws.com/uploads/default/original/test.mp4",- original_filename: "test.mp4",- )- allow(mediaconvert_job).to receive(:id).and_return(job_id)- allow(mediaconvert_client).to receive(:create_job).and_return(mediaconvert_job_response)- end+ it "uses S3Helper.s3_credentials for client credentials" do+ upload.update!(+ url: "//#{s3_bucket}.s3.#{s3_region}.amazonaws.com/uploads/default/original/test.mp4",+ original_filename: "test.mp4",+ )+ allow(mediaconvert_job).to receive(:id).and_return(job_id)+ allow(mediaconvert_client).to receive(:create_job).and_return(mediaconvert_job_response)- it "creates MediaConvert client without explicit credentials" do- expected_client_options = { region: s3_region, endpoint: "https://mediaconvert.endpoint" }+ allow(S3Helper).to receive(:s3_credentials).with(SiteSetting).and_call_original- adapter.convert+ adapter.convert- expect(Aws::MediaConvert::Client).to have_received(:new).with(expected_client_options)- expect(adapter.convert).to be true- end+ expect(S3Helper).to have_received(:s3_credentials).with(SiteSetting)endend@@ -575,58 +567,4 @@ RSpec.describe VideoConversion::AwsMediaConvertAdapter doendendend-- describe "#create_basic_client" do- context "when using IAM profile" do- before do- allow(SiteSetting).to receive(:s3_use_iam_profile).and_return(true)- allow(SiteSetting).to receive(:s3_access_key_id).and_return("")- allow(SiteSetting).to receive(:s3_secret_access_key).and_return("")- end-- it "creates client without endpoint when endpoint is nil" do- allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)-- adapter.send(:create_basic_client, endpoint: nil)-- expect(Aws::MediaConvert::Client).to have_received(:new).with({ region: s3_region })- end-- it "creates client without endpoint when endpoint is empty string" do- allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)-- adapter.send(:create_basic_client, endpoint: "")-- expect(Aws::MediaConvert::Client).to have_received(:new).with({ region: s3_region })- end-- it "creates client with endpoint when endpoint is present" do- endpoint = "https://mediaconvert.us-west-2.amazonaws.com"- allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)-- adapter.send(:create_basic_client, endpoint: endpoint)-- expect(Aws::MediaConvert::Client).to have_received(:new).with(- { region: s3_region, endpoint: endpoint },- )- end- end-- context "when not using IAM profile" do- before do- allow(SiteSetting).to receive(:s3_use_iam_profile).and_return(false)- allow(SiteSetting).to receive(:s3_access_key_id).and_return("test-key")- allow(SiteSetting).to receive(:s3_secret_access_key).and_return("test-secret")- end-- it "creates client with credentials" do- allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)-- adapter.send(:create_basic_client, endpoint: nil)-- # Verify that credentials are included in the client creation- expect(Aws::MediaConvert::Client).to have_received(:new)- end- end- endend