Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Add opt-in IAM role assumption for S3 operations (#39174)

Alan Guo Xiang Tanfae7cd138dc8
diff --git a/Gemfile b/Gemfile
index b7f47a31c01..1a24cd992a6 100644
--- a/Gemfile
+++ b/Gemfile
@@ -63,6 +63,7 @@ gem "fastimage"
gem "aws-sdk-s3", require: false
gem "aws-sdk-sns", require: false
+gem "aws-sdk-sts", require: false
gem "aws-sdk-mediaconvert", require: false
gem "aws-sdk-bedrockruntime", require: false
gem "excon"
diff --git a/Gemfile.lock b/Gemfile.lock
index 2d2432f7d3b..d5611e421b5 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -84,6 +84,9 @@ GEM
aws-sdk-sns (1.96.0)
aws-sdk-core (~> 3, >= 3.216.0)
aws-sigv4 (~> 1.5)
+ aws-sdk-sts (1.12.0)
+ aws-sdk-core (~> 3, >= 3.110.0)
+ aws-sigv4 (~> 1.1)
aws-sigv4 (1.12.1)
aws-eventstream (~> 1, >= 1.0.2)
base64 (0.3.0)
@@ -782,6 +785,7 @@ DEPENDENCIES
aws-sdk-mediaconvert
aws-sdk-s3
aws-sdk-sns
+ aws-sdk-sts
better_errors
bootsnap
bullet
@@ -954,6 +958,7 @@ CHECKSUMS
aws-sdk-mediaconvert (1.165.0) sha256=d955a571cd8b0407c0778e1d0f2333a70bf9ab48e36c81da8c5b317db24001e4
aws-sdk-s3 (1.182.0) sha256=d0fc3579395cb6cb69bf6e975240ce031fc673190e74c8dddbdd6c18572b450d
aws-sdk-sns (1.96.0) sha256=f92b3c7203c53181b1cafb3dbbea85f330002ad696175bda829cfef359fa6dd4
+ aws-sdk-sts (1.12.0) sha256=9bbd64223dd2423540291d9faaac4a169ace8cfe123b1075c9a127fcd65e061f
aws-sigv4 (1.12.1) sha256=6973ff95cb0fd0dc58ba26e90e9510a2219525d07620c8babeb70ef831826c00
base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b
benchmark (0.5.0) sha256=465df122341aedcb81a2a24b4d3bd19b6c67c1530713fd533f3ff034e419236c
diff --git a/app/services/video_conversion/aws_media_convert_adapter.rb b/app/services/video_conversion/aws_media_convert_adapter.rb
index 123e82c0b62..c08eac211a7 100644
--- a/app/services/video_conversion/aws_media_convert_adapter.rb
+++ b/app/services/video_conversion/aws_media_convert_adapter.rb
@@ -349,13 +349,8 @@ module VideoConversion
def create_basic_client(endpoint: nil)
client_options = { region: SiteSetting.s3_region }
client_options[:endpoint] = endpoint if endpoint.present?
-
- if !SiteSetting.s3_use_iam_profile
- client_options[:credentials] = Aws::Credentials.new(
- SiteSetting.s3_access_key_id,
- SiteSetting.s3_secret_access_key,
- )
- end
+ creds = S3Helper.s3_credentials(SiteSetting)
+ client_options[:credentials] = creds if creds
Aws::MediaConvert::Client.new(client_options)
end
diff --git a/config/discourse_defaults.conf b/config/discourse_defaults.conf
index 186acce8d7d..5ea8c8273ab 100644
--- a/config/discourse_defaults.conf
+++ b/config/discourse_defaults.conf
@@ -243,6 +243,8 @@ s3_endpoint =
s3_http_continue_timeout =
s3_install_cors_rule =
enable_s3_transfer_acceleration =
+s3_role_arn =
+s3_role_session_name =
# Optionally, specify a separate CDN to be used for static JS assets stored on S3
s3_asset_cdn_url =
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index 2bbd643a0a4..ed27a4e7dea 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -2144,6 +2144,8 @@ en:
s3_upload_bucket: "The Amazon S3 bucket name that files will be uploaded into. WARNING: must be lowercase, no periods, no underscores."
s3_access_key_id: "The Amazon S3 access key id that will be used to upload images, attachments, and backups."
s3_secret_access_key: "The Amazon S3 secret access key that will be used to upload images, attachments, and backups."
+ s3_role_arn: "The ARN of the IAM role to assume for S3 operations. When set alongside access keys, Discourse will use STS AssumeRole to obtain temporary credentials."
+ s3_role_session_name: "Optional session name used when assuming an IAM role for S3 operations. Appears in CloudTrail logs for auditing. Falls back to the server hostname if blank."
s3_region: "The Amazon S3 region name that will be used to upload images and backups."
s3_cdn_url: "The CDN URL to use for all s3 assets (for example: https://cdn.somewhere.com). WARNING: after changing this setting you must rebake all old posts."
s3_use_cdn_url_for_all_uploads: "Use CDN URL for all the files uploaded to s3 instead of only for images."
diff --git a/config/site_settings.yml b/config/site_settings.yml
index a3323506fb0..c87e8eadff8 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -2243,6 +2243,15 @@ files:
s3_secret_access_key:
default: ""
secret: true
+ s3_role_arn:
+ default: ""
+ min: 20
+ max: 2048 # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
+ s3_role_session_name:
+ default: ""
+ min: 2
+ max: 64
+ regex: '^[\w+=,.@\-]+$' # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
s3_region:
default: "us-east-1"
enum: "S3RegionSiteSetting"
diff --git a/lib/s3_helper.rb b/lib/s3_helper.rb
index 27d92d187dd..36c496b7259 100644
--- a/lib/s3_helper.rb
+++ b/lib/s3_helper.rb
@@ -1,6 +1,7 @@
# frozen_string_literal: true
require "aws-sdk-s3"
+require "aws-sdk-sts"
class S3Helper
FIFTEEN_MEGABYTES = 15 * 1024 * 1024
@@ -297,14 +298,35 @@ class S3Helper
opts[:http_continue_timeout] = SiteSetting.s3_http_continue_timeout
opts[:use_dualstack_endpoint] = SiteSetting.Upload.use_dualstack_endpoint
- unless obj.s3_use_iam_profile
- opts[:access_key_id] = obj.s3_access_key_id
- opts[:secret_access_key] = obj.s3_secret_access_key
- end
+ creds = s3_credentials(obj)
+ opts[:credentials] = creds if creds
opts
end
+ def self.s3_credentials(obj, stub_responses: false)
+ return nil if obj.s3_use_iam_profile
+
+ if obj.s3_role_arn.present?
+ # RoleSessionName max 64 chars: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
+ session_name = obj.s3_role_session_name.presence || Discourse.os_hostname[0...64]
+ sts_client =
+ Aws::STS::Client.new(
+ region: obj.s3_region,
+ access_key_id: obj.s3_access_key_id,
+ secret_access_key: obj.s3_secret_access_key,
+ stub_responses: stub_responses,
+ )
+ Aws::AssumeRoleCredentials.new(
+ role_arn: obj.s3_role_arn,
+ role_session_name: session_name,
+ client: sts_client,
+ )
+ else
+ Aws::Credentials.new(obj.s3_access_key_id, obj.s3_secret_access_key)
+ end
+ end
+
def download_file(filename, destination_path, failure_message = nil)
object(filename).download_file(destination_path)
rescue => err
diff --git a/spec/lib/s3_helper_spec.rb b/spec/lib/s3_helper_spec.rb
index c065853c258..b2bb1c4ad09 100644
--- a/spec/lib/s3_helper_spec.rb
+++ b/spec/lib/s3_helper_spec.rb
@@ -448,4 +448,55 @@ RSpec.describe "S3Helper" do
)
end
end
+
+ describe ".s3_credentials" do
+ it "returns AssumeRoleCredentials when s3_role_arn is configured" do
+ SiteSetting.s3_region = "us-east-1"
+ SiteSetting.s3_access_key_id = "some-key"
+ SiteSetting.s3_secret_access_key = "some-secret"
+ SiteSetting.s3_role_arn = "arn:aws:iam::123456789012:role/some-role"
+ SiteSetting.s3_role_session_name = "some-session"
+
+ creds = S3Helper.s3_credentials(SiteSetting, stub_responses: true)
+
+ expect(creds).to be_a(Aws::AssumeRoleCredentials)
+ expect(creds.assume_role_params[:role_arn]).to eq("arn:aws:iam::123456789012:role/some-role")
+ expect(creds.assume_role_params[:role_session_name]).to eq("some-session")
+ expect(creds.client.config.access_key_id).to eq("some-key")
+ expect(creds.client.config.secret_access_key).to eq("some-secret")
+ end
+
+ it "returns static Aws::Credentials when s3_role_arn is blank" do
+ SiteSetting.s3_region = "us-east-1"
+ SiteSetting.s3_access_key_id = "some-key"
+ SiteSetting.s3_secret_access_key = "some-secret"
+ SiteSetting.s3_role_arn = ""
+
+ creds = S3Helper.s3_credentials(SiteSetting)
+
+ expect(creds).to be_a(Aws::Credentials)
+ expect(creds.access_key_id).to eq("some-key")
+ expect(creds.secret_access_key).to eq("some-secret")
+ end
+
+ it "returns nil when s3_use_iam_profile is true" do
+ SiteSetting.s3_use_iam_profile = true
+ SiteSetting.s3_role_arn = "arn:aws:iam::123456789012:role/some-role"
+
+ expect(S3Helper.s3_credentials(SiteSetting)).to be_nil
+ end
+
+ it "falls back to Discourse.os_hostname when s3_role_session_name is blank" do
+ SiteSetting.s3_region = "us-east-1"
+ SiteSetting.s3_access_key_id = "some-key"
+ SiteSetting.s3_secret_access_key = "some-secret"
+ SiteSetting.s3_role_arn = "arn:aws:iam::123456789012:role/some-role"
+ SiteSetting.s3_role_session_name = ""
+ Discourse.stubs(:os_hostname).returns("some-host")
+
+ creds = S3Helper.s3_credentials(SiteSetting, stub_responses: true)
+
+ expect(creds.assume_role_params[:role_session_name]).to eq("some-host")
+ end
+ end
end
diff --git a/spec/services/video_conversion/aws_media_convert_adapter_spec.rb b/spec/services/video_conversion/aws_media_convert_adapter_spec.rb
index 7124b5748d7..442e0f434fb 100644
--- a/spec/services/video_conversion/aws_media_convert_adapter_spec.rb
+++ b/spec/services/video_conversion/aws_media_convert_adapter_spec.rb
@@ -226,27 +226,19 @@ RSpec.describe VideoConversion::AwsMediaConvertAdapter do
end
end
- context "when using IAM profile" do
- before do
- allow(SiteSetting).to receive(:s3_use_iam_profile).and_return(true)
- allow(SiteSetting).to receive(:s3_access_key_id).and_return("")
- allow(SiteSetting).to receive(:s3_secret_access_key).and_return("")
- upload.update!(
- url: "//#{s3_bucket}.s3.#{s3_region}.amazonaws.com/uploads/default/original/test.mp4",
- original_filename: "test.mp4",
- )
- allow(mediaconvert_job).to receive(:id).and_return(job_id)
- allow(mediaconvert_client).to receive(:create_job).and_return(mediaconvert_job_response)
- end
+ it "uses S3Helper.s3_credentials for client credentials" do
+ upload.update!(
+ url: "//#{s3_bucket}.s3.#{s3_region}.amazonaws.com/uploads/default/original/test.mp4",
+ original_filename: "test.mp4",
+ )
+ allow(mediaconvert_job).to receive(:id).and_return(job_id)
+ allow(mediaconvert_client).to receive(:create_job).and_return(mediaconvert_job_response)
- it "creates MediaConvert client without explicit credentials" do
- expected_client_options = { region: s3_region, endpoint: "https://mediaconvert.endpoint" }
+ allow(S3Helper).to receive(:s3_credentials).with(SiteSetting).and_call_original
- adapter.convert
+ adapter.convert
- expect(Aws::MediaConvert::Client).to have_received(:new).with(expected_client_options)
- expect(adapter.convert).to be true
- end
+ expect(S3Helper).to have_received(:s3_credentials).with(SiteSetting)
end
end
@@ -575,58 +567,4 @@ RSpec.describe VideoConversion::AwsMediaConvertAdapter do
end
end
end
-
- describe "#create_basic_client" do
- context "when using IAM profile" do
- before do
- allow(SiteSetting).to receive(:s3_use_iam_profile).and_return(true)
- allow(SiteSetting).to receive(:s3_access_key_id).and_return("")
- allow(SiteSetting).to receive(:s3_secret_access_key).and_return("")
- end
-
- it "creates client without endpoint when endpoint is nil" do
- allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)
-
- adapter.send(:create_basic_client, endpoint: nil)
-
- expect(Aws::MediaConvert::Client).to have_received(:new).with({ region: s3_region })
- end
-
- it "creates client without endpoint when endpoint is empty string" do
- allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)
-
- adapter.send(:create_basic_client, endpoint: "")
-
- expect(Aws::MediaConvert::Client).to have_received(:new).with({ region: s3_region })
- end
-
- it "creates client with endpoint when endpoint is present" do
- endpoint = "https://mediaconvert.us-west-2.amazonaws.com"
- allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)
-
- adapter.send(:create_basic_client, endpoint: endpoint)
-
- expect(Aws::MediaConvert::Client).to have_received(:new).with(
- { region: s3_region, endpoint: endpoint },
- )
- end
- end
-
- context "when not using IAM profile" do
- before do
- allow(SiteSetting).to receive(:s3_use_iam_profile).and_return(false)
- allow(SiteSetting).to receive(:s3_access_key_id).and_return("test-key")
- allow(SiteSetting).to receive(:s3_secret_access_key).and_return("test-secret")
- end
-
- it "creates client with credentials" do
- allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)
-
- adapter.send(:create_basic_client, endpoint: nil)
-
- # Verify that credentials are included in the client creation
- expect(Aws::MediaConvert::Client).to have_received(:new)
- end
- end
- end
end