Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3control): take TagResource and UntagResource tag condition keys from generated data (#6677)

galsuchetzky-actf926968167dc
diff --git a/.github/workflows/iam-data-drift.yml b/.github/workflows/iam-data-drift.yml
index aef293dd87..74a834fede 100644
--- a/.github/workflows/iam-data-drift.yml
+++ b/.github/workflows/iam-data-drift.yml
@@ -1,12 +1,13 @@
name: IAM Data Drift
-# Checks the nine vendored files that come from AWS's live service reference rather than from a
+# Checks the ten vendored files that come from AWS's live service reference rather than from a
# pinned dependency: the IAM service namespace list, the map of which resource each IAM action is
# authorized against, the map of operations AWS authorizes as an action of another name, the
# table of the operations that pass a role and the services they pass it to, the table of the
# operations that pass a Lambda network connector, the table of the
-# tagging actions operations that tag what they create need, and the table of the resources each
-# operation's request names, the DynamoDB condition key table, and the IAM condition key table. That
+# tagging actions operations that tag what they create need, the table of the resources each
+# operation's request names, the DynamoDB condition key table, the IAM condition key table, and
+# the S3 Control tag condition key table. That
# needs the live service reference index, so unlike every other check here it can
# fail without anything in the repository having changed: an AWS outage, or a namespace AWS has
# retired. It therefore runs on its own, never as part of a job that ordinary changes trigger.
@@ -62,6 +63,9 @@ on:
- 'src/main/resources/aws/iam-condition-keys.json'
- 'tools/aws/regen_iam_condition_keys.py'
- 'tools/aws/test_regen_iam_condition_keys.py'
+ - 'src/main/resources/aws/s3control-tag-condition-keys.json'
+ - 'tools/aws/regen_s3control_tag_condition_keys.py'
+ - 'tools/aws/test_regen_s3control_tag_condition_keys.py'
- 'tools/aws/regen_partitions.py'
- 'tools/aws/regen_query_services.py'
- '.github/workflows/iam-data-drift.yml'
diff --git a/.github/workflows/partition-literals.yml b/.github/workflows/partition-literals.yml
index c12cd3d0a8..aa3f511943 100644
--- a/.github/workflows/partition-literals.yml
+++ b/.github/workflows/partition-literals.yml
@@ -105,6 +105,8 @@ jobs:
- name: Check IAM condition key request members
run: make iam-condition-keys-check
+ - name: Check S3 Control tag condition request members
+ run: make s3control-tag-condition-keys-check
- name: Check partitions.json against a fresh generation
run: make aws-data-check
diff --git a/Makefile b/Makefile
index b634e15c01..eb9852837e 100644
--- a/Makefile
+++ b/Makefile
@@ -425,6 +425,9 @@ DYNAMODB_CONDITION_KEYS_PATHS := src/main/resources/aws/dynamodb-condition-keys.
IAM_CONDITION_KEYS_PATHS := src/main/resources/aws/iam-condition-keys.json \
tools/aws/regen_iam_condition_keys.py tools/aws/test_regen_iam_condition_keys.py \
tools/aws/regen_partitions.py tools/aws/regen_query_services.py
+S3CONTROL_TAG_CONDITION_KEYS_PATHS := src/main/resources/aws/s3control-tag-condition-keys.json \
+ tools/aws/regen_s3control_tag_condition_keys.py tools/aws/test_regen_s3control_tag_condition_keys.py \
+ tools/aws/regen_iam_tag_on_create.py $(IAM_PASSED_ROLES_GENERATOR)
IAM_REQUEST_RESOURCES_PATHS := src/main/resources/aws/iam-request-resources.json \
tools/aws/regen_iam_request_resources.py $(IAM_PASSED_ROLES_GENERATOR) \
tools/aws/test_regen_iam_request_resources.py
@@ -433,7 +436,7 @@ iam-data-verify: ## CI gate (online): every vendored IAM file must still match A
@# Every check runs even when an earlier one fails, so one file's drift cannot hide another's
@# state. Sequential steps in CI stop at the first failure, which once left the action
@# resource gate unexercised while the namespace list was stale.
- @namespaces=0; actions=0; authorized=0; passed=0; connectors=0; tagged=0; requested=0; dynamodb=0; conditionkeys=0; \
+ @namespaces=0; actions=0; authorized=0; passed=0; connectors=0; tagged=0; requested=0; dynamodb=0; conditionkeys=0; s3control=0; \
covers() { \
[ -z "$(strip $(CHANGED))" ] && return 0; \
for path in $(CHANGED); do case " $$1 " in *" $$path "*) return 0;; esac; done; \
@@ -467,9 +470,12 @@ iam-data-verify: ## CI gate (online): every vendored IAM file must still match A
if covers "$(IAM_CONDITION_KEYS_PATHS)" "the IAM condition key table"; then \
$(MAKE) --no-print-directory iam-condition-keys-verify || conditionkeys=$$?; \
fi; \
+ if covers "$(S3CONTROL_TAG_CONDITION_KEYS_PATHS)" "the S3 Control tag condition key table"; then \
+ $(MAKE) --no-print-directory s3control-tag-condition-keys-verify || s3control=$$?; \
+ fi; \
if [ $$namespaces -ne 0 ] || [ $$actions -ne 0 ] || [ $$authorized -ne 0 ] || [ $$passed -ne 0 ] \
|| [ $$connectors -ne 0 ] || [ $$tagged -ne 0 ] || [ $$requested -ne 0 ] || [ $$dynamodb -ne 0 ] \
- || [ $$conditionkeys -ne 0 ]; then \
+ || [ $$conditionkeys -ne 0 ] || [ $$s3control -ne 0 ]; then \
echo ""; \
echo "error: vendored IAM data does not match upstream:"; \
if [ $$namespaces -ne 0 ]; then \
@@ -495,6 +501,7 @@ iam-data-verify: ## CI gate (online): every vendored IAM file must still match A
fi; \
if [ $$dynamodb -ne 0 ]; then echo " the DynamoDB condition key table"; fi; \
if [ $$conditionkeys -ne 0 ]; then echo " the IAM condition key table"; fi; \
+ if [ $$s3control -ne 0 ]; then echo " the S3 Control tag condition key table"; fi; \
exit 1; \
fi
@@ -680,3 +687,11 @@ iam-condition-keys-check: ## Check the IAM condition key request members against
$(PYTHON) tools/aws/regen_iam_condition_keys.py --check
iam-condition-keys-verify: ## Check every IAM condition key and action pair against AWS
$(PYTHON) tools/aws/regen_iam_condition_keys.py --verify
+
+.PHONY: s3control-tag-condition-keys-sync s3control-tag-condition-keys-check s3control-tag-condition-keys-verify
+s3control-tag-condition-keys-sync: ## Regenerate the S3 Control tag condition keys from AWS and botocore
+ $(PYTHON) tools/aws/regen_s3control_tag_condition_keys.py
+s3control-tag-condition-keys-check: ## Check the S3 Control tag condition request members against pinned botocore
+ $(PYTHON) tools/aws/regen_s3control_tag_condition_keys.py --check
+s3control-tag-condition-keys-verify: ## Check the S3 Control tag condition action keys against AWS
+ $(PYTHON) tools/aws/regen_s3control_tag_condition_keys.py --verify
diff --git a/docs/services/iam.md b/docs/services/iam.md
index c72d785ba2..92e3bd2e85 100644
--- a/docs/services/iam.md
+++ b/docs/services/iam.md
@@ -2238,8 +2238,9 @@ floci populates:
- `s3:prefix`, `s3:delimiter`, `s3:max-keys`: from the S3 request parameters.
- `aws:RequestTag/<key>`: the tags named in the request itself, before they are applied, for
`ec2:RunInstances` (`TagSpecification.N`), `ec2:CreateTags` (`Tag.N`) and
- `s3:PutBucketTagging` (the `<Tagging>` body), and for every create in the tag-on-create table
- whose action takes it, as described under the tag-on-create check above.
+ `s3:PutBucketTagging` (the `<Tagging>` body), for S3 Control's `s3:TagResource` (its `Tags` body),
+ and for every create in the tag-on-create table whose action takes it, as described under the
+ tag-on-create check above.
- `aws:ResourceTag/<key>`: the target resource's current tags, for `ec2:CreateTags`,
`ec2:DeleteTags`, `ec2:TerminateInstances` and `ec2:DescribeInstances` (the first
`ResourceId.N` or `InstanceId.N`), and for `s3:GetBucketTagging`, `s3:DeleteBucket` and the
@@ -2314,8 +2315,11 @@ floci populates:
cannot match: an absent key has no equal or like value. That absence is the enforcement rather
than a gap in it.
- `aws:TagKeys`: the case-sensitive requested tag keys for EC2 `RunInstances` and
- `CreateTags`, S3 `PutBucketTagging`, and every create in the tag-on-create table whose action
- takes it. `ForAllValues:StringEquals` can restrict the
+ `CreateTags`, S3 `PutBucketTagging`, S3 Control `TagResource` (its `Tags` body) and
+ `UntagResource` (its `tagKeys` query parameters), and every create in the tag-on-create table
+ whose action takes it. S3 Control's come from `s3control-tag-condition-keys.json`, which
+ `tools/aws/regen_s3control_tag_condition_keys.py` builds from the Service Reference's condition
+ keys and botocore's `s3control` request members; S3's own were measured against AWS. `ForAllValues:StringEquals` can restrict the
complete requested set without a `Null` guard; no tags still means absent context.
- `dynamodb:LeadingKeys`, `dynamodb:Attributes`, `dynamodb:Select`: from the DynamoDB request
body, for `GetItem`, `PutItem`, `UpdateItem`, `DeleteItem`, `Query`, `BatchGetItem` and
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/IamConditionContextResolver.java b/src/main/java/io/github/hectorvent/floci/core/common/IamConditionContextResolver.java
index f4b0af29b2..14a1dd706d 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/IamConditionContextResolver.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/IamConditionContextResolver.java
@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.ec2.model.Tag;
import io.github.hectorvent.floci.services.iam.IamActionRegistry;
import io.github.hectorvent.floci.services.iam.ResourceArnBuilder;
+import io.github.hectorvent.floci.services.s3.S3ControlTagConditionKeys;
import io.github.hectorvent.floci.services.s3.S3ObjectActions;
import io.github.hectorvent.floci.services.s3.S3ObjectKeys;
import io.github.hectorvent.floci.services.s3.S3Service;
@@ -296,6 +297,12 @@ public class IamConditionContextResolver {
// ── S3 ──────────────────────────────────────────────────────────────────────
private Map<String, List<String>> s3ConditionContext(String action, ContainerRequestContext ctx) {
+ // S3 Control's request tag keys come from the generated table; S3's own below were measured
+ // against AWS, and do not all match the Service Reference.
+ if (IamActionRegistry.isS3ControlRequest("s3", ctx)) {
+ return S3ControlTagConditionKeys.conditions(action, () -> bodyText(ctx),
+ ctx.getUriInfo().getQueryParameters());
+ }
Map<String, List<String>> conditions = s3ActionConditionContext(action, ctx);
// A version action is checked only for a request naming a version, and carries its id.
if (S3ObjectActions.isVersionAction(action) && ctx.getUriInfo() != null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeys.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeys.java
new file mode 100644
index 0000000000..f608a210b4
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeys.java
@@ -0,0 +1,89 @@
+package io.github.hectorvent.floci.services.s3;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.VendoredResources;
+import io.github.hectorvent.floci.core.common.XmlParser;
+import jakarta.ws.rs.core.MultivaluedMap;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+/**
+ * The request tag condition keys of an S3 Control request, {@code aws:RequestTag/<key>} and
+ * {@code aws:TagKeys}, read from the table {@code tools/aws/regen_s3control_tag_condition_keys.py}
+ * generates: the actions the Service Reference gives either key, and the request members botocore's
+ * {@code s3control} model carries them in. S3 Control's routes are checked as the action of their
+ * operation's name, so an action names its operation's members.
+ */
+public final class S3ControlTagConditionKeys {
+
+ private static final String REQUEST_TAG = "aws:RequestTag/${TagKey}";
+ private static final String TAG_KEYS = "aws:TagKeys";
+ private static final String REQUEST_TAG_PREFIX = "aws:RequestTag/";
+
+ private static final JsonNode TABLE = VendoredResources.read("aws/s3control-tag-condition-keys.json",
+ "S3 Control tag condition keys", input -> {
+ JsonNode document = new ObjectMapper().readTree(input);
+ if (!document.path("actions").isObject() || document.path("actions").isEmpty()
+ || !document.path("members").isObject() || document.path("members").isEmpty()) {
+ throw new IllegalStateException("Malformed S3 Control tag condition key table");
+ }
+ return document;
+ });
+
+ private S3ControlTagConditionKeys() {}
+
+ /**
+ * The request tag keys {@code action} takes, from the members of the request that carry them:
+ * the tags in {@code body}, read as {@code S3ControlController} reads them, and the tag keys in
+ * {@code query}. Null when the action takes neither key or the request carries no tags.
+ */
+ public static Map<String, List<String>> conditions(String action, Supplier<String> body,
+ MultivaluedMap<String, String> query) {
+ JsonNode members = TABLE.path("members").path(action == null ? "" : action);
+ Map<String, List<String>> conditions = new LinkedHashMap<>();
+ if (takes(action, REQUEST_TAG)) {
+ for (JsonNode member : members.path(REQUEST_TAG)) {
+ bodyTags(member, body).forEach((key, value) -> conditions.put(REQUEST_TAG_PREFIX + key, List.of(value)));
+ }
+ }
+ if (takes(action, TAG_KEYS)) {
+ List<String> keys = new ArrayList<>();
+ for (JsonNode member : members.path(TAG_KEYS)) {
+ List<String> named = "querystring".equals(member.path("location").asText())
+ ? query.getOrDefault(member.path("member").asText(), List.of())
+ : List.copyOf(bodyTags(member, body).keySet());
+ named.stream().filter(key -> !keys.contains(key)).forEach(keys::add);
+ }
+ if (!keys.isEmpty()) {
+ conditions.put(TAG_KEYS, keys);
+ }
+ }
+ return conditions.isEmpty() ? null : conditions;
+ }
+
+ private static boolean takes(String action, String key) {
+ for (JsonNode taken : TABLE.path("actions").path(action == null ? "" : action)) {
+ if (key.equals(taken.asText())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** The tags a body member holds, by the element its list wraps each tag in. */
+ private static Map<String, String> bodyTags(JsonNode member, Supplier<String> body) {
+ String path = member.path("member").asText();
+ int open = path.indexOf('[');
+ if (open < 0 || !member.has("key") || !member.has("value")) {
+ return Map.of();
+ }
+ String element = path.substring(open + 1, path.length() - 1);
+ return XmlParser.extractPairs(body.get(), element.isEmpty() ? "member" : element,
+ member.path("key").asText(), member.path("value").asText());
+ }
+}
diff --git a/src/main/resources/aws/s3control-tag-condition-keys.json b/src/main/resources/aws/s3control-tag-condition-keys.json
new file mode 100644
index 0000000000..2e7db68ad0
--- /dev/null
+++ b/src/main/resources/aws/s3control-tag-condition-keys.json
@@ -0,0 +1,352 @@
+{
+ "_source": {
+ "generator": "tools/aws/regen_s3control_tag_condition_keys.py",
+ "reference": "https://servicereference.us-east-1.amazonaws.com/v1/s3/s3.json",
+ "botocore": "botocore 1.43.113 (installed package)",
+ "note": "make s3control-tag-condition-keys-check re-derives members from botocore; make s3control-tag-condition-keys-verify checks actions against AWS."
+ },
+ "actions": {
+ "s3:AssociateAccessGrantsIdentityCenter": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:CreateAccessGrant": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:CreateAccessGrantsInstance": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:CreateAccessGrantsLocation": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:CreateAccessPoint": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:CreateBucket": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:CreateJob": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:CreateStorageLensGroup": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DeleteAccessGrant": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DeleteAccessGrantsInstance": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DeleteAccessGrantsInstanceResourcePolicy": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DeleteAccessGrantsLocation": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DeleteJobTagging": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DeleteStorageLensConfiguration": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DeleteStorageLensConfigurationTagging": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DeleteStorageLensGroup": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DescribeJob": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:DissociateAccessGrantsIdentityCenter": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetAccessGrant": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetAccessGrantsInstance": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetAccessGrantsInstanceForPrefix": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetAccessGrantsInstanceResourcePolicy": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetAccessGrantsLocation": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetDataAccess": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetJobTagging": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetStorageLensConfiguration": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetStorageLensConfigurationTagging": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetStorageLensDashboard": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:GetStorageLensGroup": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:ListAccessGrants": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:ListAccessGrantsLocations": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:ListCallerAccessGrants": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:ListTagsForResource": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:PutAccessGrantsInstanceResourcePolicy": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:PutJobTagging": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:PutStorageLensConfiguration": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:PutStorageLensConfigurationTagging": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:TagResource": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:UntagResource": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:UpdateAccessGrantsLocation": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:UpdateJobPriority": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:UpdateJobStatus": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ],
+ "s3:UpdateStorageLensGroup": [
+ "aws:RequestTag/${TagKey}",
+ "aws:TagKeys"
+ ]
+ },
+ "members": {
+ "s3:CreateAccessGrant": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:CreateAccessGrantsInstance": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:CreateAccessGrantsLocation": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:CreateAccessPoint": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:CreateJob": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:CreateStorageLensGroup": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:PutJobTagging": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:PutStorageLensConfiguration": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:PutStorageLensConfigurationTagging": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:TagResource": {
+ "aws:RequestTag/${TagKey}": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ],
+ "aws:TagKeys": [
+ {
+ "member": "Tags[Tag]",
+ "key": "Key",
+ "value": "Value"
+ }
+ ]
+ },
+ "s3:UntagResource": {
+ "aws:TagKeys": [
+ {
+ "member": "tagKeys",
+ "location": "querystring"
+ }
+ ]
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlIamEnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlIamEnforcementIntegrationTest.java
index 5fdf568f6f..4e1746b35a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlIamEnforcementIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlIamEnforcementIntegrationTest.java
@@ -503,6 +503,74 @@ class S3ControlIamEnforcementIntegrationTest {
.body(containsString("<Code>NoSuchBucket</Code>"));
}
+ /**
+ * TagResource is checked with the tags it sends as {@code aws:RequestTag/<key>} and
+ * {@code aws:TagKeys}, the condition keys the Service Authorization Reference lists for it.
+ */
+ @Test
+ void tagResourceIsCheckedAgainstTheTagsItsRequestSends() {
+ String bucket = bucketWithTags("control-request-tag");
+ String user = userWith("""
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Action":"s3:TagResource","Resource":"arn:aws:s3:::%1$s",
+ "Condition":{"StringEquals":{"aws:RequestTag/Owner":"team"}}}]}"""
+ .formatted(bucket));
+
+ given()
+ .header("Authorization", auth(user, "s3"))
+ .header("x-amz-account-id", ACCOUNT)
+ .contentType("application/xml")
+ .body("<Tagging><Tags><Tag><Key>Owner</Key><Value>team</Value></Tag></Tags></Tagging>")
+ .when()
+ .post("/v20180820/tags/arn:aws:s3:::" + bucket)
+ .then()
+ .statusCode(204);
+
+ given()
+ .header("Authorization", auth(user, "s3"))
+ .header("x-amz-account-id", ACCOUNT)
+ .contentType("application/xml")
+ .body("<Tagging><Tags><Tag><Key>Owner</Key><Value>other</Value></Tag></Tags></Tagging>")
+ .when()
+ .post("/v20180820/tags/arn:aws:s3:::" + bucket)
+ .then()
+ .statusCode(403)
+ .body(containsString("is not authorized to perform: s3:TagResource"));
+ }
+
+ /**
+ * UntagResource is checked with the keys it removes as {@code aws:TagKeys}, the condition key
+ * the Service Authorization Reference lists for it.
+ */
+ @Test
+ void untagResourceIsCheckedAgainstTheKeysItsRequestNames() {
+ String bucket = bucketWithTags("control-tag-keys");
+ String user = userWith("""
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Action":"s3:UntagResource","Resource":"arn:aws:s3:::%1$s",
+ "Condition":{"ForAllValues:StringEquals":{"aws:TagKeys":["Env"]}}}]}"""
+ .formatted(bucket));
+
+ given()
+ .header("Authorization", auth(user, "s3"))
+ .header("x-amz-account-id", ACCOUNT)
+ .queryParam("tagKeys", "Env", "Owner")
+ .when()
+ .delete("/v20180820/tags/arn:aws:s3:::" + bucket)
+ .then()
+ .statusCode(403)
+ .body(containsString("is not authorized to perform: s3:UntagResource"));
+
+ given()
+ .header("Authorization", auth(user, "s3"))
+ .header("x-amz-account-id", ACCOUNT)
+ .queryParam("tagKeys", "Env")
+ .when()
+ .delete("/v20180820/tags/arn:aws:s3:::" + bucket)
+ .then()
+ .statusCode(204);
+ }
+
/** A bucket created by the root stand-in, tagged through the data plane, as Terraform reads it. */
private String bucketWithTags(String prefix) {
String bucket = prefix + "-" + UUID.randomUUID().toString().substring(0, 8);
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeysTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeysTest.java
new file mode 100644
index 0000000000..d70260d399
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeysTest.java
@@ -0,0 +1,42 @@
+package io.github.hectorvent.floci.services.s3;
+
+import jakarta.ws.rs.core.MultivaluedHashMap;
+import jakarta.ws.rs.core.MultivaluedMap;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+class S3ControlTagConditionKeysTest {
+
+ private static final String TAGS = "<TagResourceRequest><Tags>"
+ + "<Tag><Key>Owner</Key><Value>team</Value></Tag>"
+ + "<Tag><Key>Env</Key><Value>dev</Value></Tag>"
+ + "</Tags></TagResourceRequest>";
+
+ @Test
+ void tagResourceTakesEachTagOfItsBodyAndTheirKeys() {
+ assertEquals(Map.of("aws:RequestTag/Owner", List.of("team"), "aws:RequestTag/Env", List.of("dev"),
+ "aws:TagKeys", List.of("Owner", "Env")),
+ S3ControlTagConditionKeys.conditions("s3:TagResource", () -> TAGS, new MultivaluedHashMap<>()));
+ }
+
+ @Test
+ void untagResourceTakesTheKeysItsQueryNamesAndNoRequestTag() {
+ MultivaluedMap<String, String> query = new MultivaluedHashMap<>();
+ query.addAll("tagKeys", "Owner", "Env");
+ assertEquals(Map.of("aws:TagKeys", List.of("Owner", "Env")),
+ S3ControlTagConditionKeys.conditions("s3:UntagResource", () -> TAGS, query));
+ }
+
+ @Test
+ void aRequestCarryingNoTagsAndAnActionTakingNoKeyGetNothing() {
+ assertNull(S3ControlTagConditionKeys.conditions("s3:TagResource", () -> "", new MultivaluedHashMap<>()));
+ assertNull(S3ControlTagConditionKeys.conditions("s3:UntagResource", () -> "", new MultivaluedHashMap<>()));
+ assertNull(S3ControlTagConditionKeys.conditions("s3:ListAccessPoints", () -> TAGS,
+ new MultivaluedHashMap<>()));
+ }
+}
diff --git a/tools/aws/regen_s3control_tag_condition_keys.py b/tools/aws/regen_s3control_tag_condition_keys.py
new file mode 100644
index 0000000000..d6ddf0b95b
--- /dev/null
+++ b/tools/aws/regen_s3control_tag_condition_keys.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""Regenerate src/main/resources/aws/s3control-tag-condition-keys.json from AWS's Service Reference and botocore.
+
+An S3 Control request is checked with the tags it carries as `aws:RequestTag/<key>` and `aws:TagKeys`
+when its action takes them. The table comes from two sources, each answering what the other cannot:
+
+- AWS's Service Reference for `s3`, which gives every action's condition keys, for the action and for
+ each resource type it acts on, but not where a request carries the tags:
+
+ https://servicereference.us-east-1.amazonaws.com/v1/s3/s3.json
+
+- botocore's `s3control` model (Apache-2.0, https://github.com/boto/botocore, the pinned release),
+ which gives each request's members but not which of them AWS checks.
+
+`actions` holds every `s3` action that takes either key. `members` holds, for every S3 Control
+operation, as `s3:<Operation>`, the request members that feed each key: a top-level body member
+holding tags, found as `regen_iam_tag_on_create.py` finds them (a list of key and value structures,
+named by its list element, or a map of strings), feeds both keys, and a top-level query string list
+of tag keys feeds `aws:TagKeys`. A member nested deeper describes something else: a job's manifest,
+a Storage Lens configuration's filters.
+
+Only S3 Control's model is read. S3's own requests carry the tag keys `IamConditionContextResolver`
+gives them, which were measured against AWS and do not all match the reference: `s3:PutBucketTagging`
+takes `aws:RequestTag/<key>` and `aws:TagKeys` on AWS though the reference lists neither for it.
+
+The Service Reference is live rather than a pinned dependency. The two gates split along the sources:
+
+- `--check` is offline and needs the pinned botocore: `members` must be exactly what botocore gives.
+- `--verify` is online and stdlib-only: `actions` must be exactly what the reference gives.
+
+Run from anywhere in the repo:
+ python3 tools/aws/regen_s3control_tag_condition_keys.py # rewrite the vendored file in place
+ python3 tools/aws/regen_s3control_tag_condition_keys.py --check # offline: members match botocore
+ python3 tools/aws/regen_s3control_tag_condition_keys.py --verify # online: actions match AWS
+ python3 tools/aws/regen_s3control_tag_condition_keys.py --source <s3.json> # read a saved reference
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import urllib.request
+from pathlib import Path
+
+from regen_iam_tag_on_create import (REQUEST_TAG_KEYS, TAG_MEMBER_RE, _collection, _condition_keys,
+ _resource_keys, _serialized)
+from regen_partitions import resolve_botocore_data
+from regen_query_services import latest_model_dir, load_model
+
+OUTPUT = Path(__file__).resolve().parents[2] / "src/main/resources/aws/s3control-tag-condition-keys.json"
+URL = "https://servicereference.us-east-1.amazonaws.com/v1/s3/s3.json"
+NAMESPACE = "s3"
+MODEL = "s3control"
+REQUEST_TAG = "aws:RequestTag/${TagKey}"
+TAG_KEYS = "aws:TagKeys"
+# A request member that lists the keys of the tags the operation removes.
+TAG_KEY_LIST_RE = re.compile(r"^\w*tagkeys$", re.IGNORECASE)
+BODY_LOCATIONS = (None, "")
+
+
+def action_keys(reference: dict) -> dict[str, list[str]]:
+ """Every action of the namespace that takes a request tag key, with the ones it takes."""
+ resource_keys = _resource_keys([reference])
+ found = {}
+ for action in sorted(reference["Actions"], key=lambda entry: entry["Name"]):
+ keys = REQUEST_TAG_KEYS & _condition_keys(NAMESPACE, action, resource_keys)
+ if keys:
+ found[f"{NAMESPACE}:{action['Name']}"] = sorted(keys)
+ return found
+
+
+def request_members(model: dict) -> dict[str, dict[str, list[dict]]]:
+ """The request members of every operation that feed each request tag key."""
+ found = {}
+ for operation, spec in sorted(model["operations"].items()):
+ if "input" not in spec:
+ continue
+ tags, tag_keys = [], []
+ for name, member in sorted(model["shapes"][spec["input"]["shape"]].get("members", {}).items()):
+ location = member.get("location")
+ if location in BODY_LOCATIONS and TAG_MEMBER_RE.match(name):
+ collection = _collection(model, member["shape"])
+ if collection is not None:
+ tags.append({"member": _serialized(name, member, model) + collection.pop("step"), **collection})
+ elif location == "querystring" and TAG_KEY_LIST_RE.match(name) and _is_string_list(model, member):
+ tag_keys.append({"member": member.get("locationName", name), "location": "querystring"})
+ keys = {}
+ if tags:
+ keys[REQUEST_TAG] = tags
+ if tags or tag_keys:
+ keys[TAG_KEYS] = tags + tag_keys
+ if keys:
+ found[f"{NAMESPACE}:{operation}"] = keys
+ return found
+
+
+def _is_string_list(model: dict, member: dict) -> bool:
+ shape = model["shapes"][member["shape"]]
+ return shape["type"] == "list" and model["shapes"][shape["member"]["shape"]]["type"] == "string"
+
+
+def render(document: dict) -> str:
+ return json.dumps(document, indent=2) + "\n"
+
+
+def validate(document: dict) -> list[str]:
+ """Problems that make the vendored file unusable, rather than merely out of date."""
+ problems = []
+ for section in ("actions", "members"):
+ if not isinstance(document.get(section), dict) or not document[section]:
+ problems.append(f"{section} is missing or empty")
+ if problems:
+ return problems
+ for operation in ("s3:TagResource", "s3:UntagResource"):
+ if operation not in document["actions"]:
+ problems.append(f"{operation} takes no request tag key")
+ if operation not in document["members"]:
+ problems.append(f"{operation} names no member that carries tags")
+ return problems
+
+
+def _reference(source: Path | None) -> dict:
+ if source is not None:
+ return json.loads(source.read_text(encoding="utf-8"))
+ request = urllib.request.Request(URL, headers={"Accept": "application/json"})
+ with urllib.request.urlopen(request, timeout=30) as response:
+ return json.load(response)
+
+
+def _model(botocore_data: Path | None) -> tuple[dict, str]:
+ data, provenance = resolve_botocore_data(botocore_data)
+ return load_model(latest_model_dir(data, MODEL)), provenance
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--check", action="store_true", help="offline: members must match the pinned botocore")
+ parser.add_argument("--verify", action="store_true", help="online: actions must match the Service Reference")
+ parser.add_argument("--source", type=Path, help="a saved s3.json Service Reference document")
+ parser.add_argument("--botocore-data", type=Path, help="botocore data directory")
+ args = parser.parse_args(argv)
+
+ if args.check or args.verify:
+ document = json.loads(OUTPUT.read_text(encoding="utf-8"))
+ problems = validate(document)
+ if args.check:
+ model, provenance = _model(args.botocore_data)
+ if document["members"] != request_members(model):
+ problems.append(f"members differ from {provenance}")
+ if args.verify and document["actions"] != action_keys(_reference(args.source)):
+ problems.append("actions differ from the AWS Service Reference")
+ for problem in problems:
+ print(f"{OUTPUT.name}: {problem}", file=sys.stderr)
+ return 1 if problems else 0
+
+ model, provenance = _model(args.botocore_data)
+ document = {"_source": {"generator": "tools/aws/regen_s3control_tag_condition_keys.py", "reference": URL,
+ "botocore": provenance,
+ "note": "make s3control-tag-condition-keys-check re-derives members from botocore; "
+ "make s3control-tag-condition-keys-verify checks actions against AWS."},
+ "actions": action_keys(_reference(args.source)), "members": request_members(model)}
+ problems = validate(document)
+ if problems:
+ for problem in problems:
+ print(f"error: {problem}", file=sys.stderr)
+ return 1
+ OUTPUT.write_text(render(document), encoding="utf-8")
+ print(f"wrote {OUTPUT.name} from {provenance}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/aws/test_regen_s3control_tag_condition_keys.py b/tools/aws/test_regen_s3control_tag_condition_keys.py
new file mode 100644
index 0000000000..5f95992513
--- /dev/null
+++ b/tools/aws/test_regen_s3control_tag_condition_keys.py
@@ -0,0 +1,87 @@
+"""Action keys and request members for the S3 Control tag condition key table."""
+import json
+import subprocess
+
+import pytest
+
+from regen_s3control_tag_condition_keys import OUTPUT, action_keys, request_members, validate
+
+
+def test_action_and_resource_keys_are_combined_and_filtered():
+ reference = {"Name": "s3", "Resources": [{"Name": "job", "ConditionKeys": ["aws:RequestTag/${TagKey}", "s3:JobSuspendedCause"]}],
+ "Actions": [{"Name": "TagResource", "ActionConditionKeys": ["aws:TagKeys", "s3:authType"]},
+ {"Name": "PutJobTagging", "Resources": [{"Name": "job"}]},
+ {"Name": "ListAccessPoints", "ActionConditionKeys": ["s3:authType"]}]}
+ assert action_keys(reference) == {"s3:PutJobTagging": ["aws:RequestTag/${TagKey}"],
+ "s3:TagResource": ["aws:TagKeys"]}
+
+
+def test_body_tags_feed_both_keys_and_query_tag_keys_feed_tag_keys():
+ model = {"metadata": {"protocol": "rest-xml"}, "operations": {
+ "TagResource": {"input": {"shape": "TagIn"}},
+ "UntagResource": {"input": {"shape": "UntagIn"}},
+ "CreateJob": {"input": {"shape": "JobIn"}},
+ "ListAccessPoints": {"input": {"shape": "ListIn"}}}, "shapes": {
+ "TagIn": {"type": "structure", "members": {"ResourceArn": {"shape": "String", "location": "uri"},
+ "Tags": {"shape": "TagList"}}},
+ "UntagIn": {"type": "structure", "members": {
+ "TagKeys": {"shape": "KeyList", "location": "querystring", "locationName": "tagKeys"}}},
+ "JobIn": {"type": "structure", "members": {"Manifest": {"shape": "Manifest"}}},
+ "Manifest": {"type": "structure", "members": {"Tags": {"shape": "TagList"}}},
+ "ListIn": {"type": "structure", "members": {"Bucket": {"shape": "String", "location": "querystring"}}},
+ "TagList": {"type": "list", "member": {"shape": "Tag", "locationName": "Tag"}},
+ "Tag": {"type": "structure", "members": {"Key": {"shape": "String"}, "Value": {"shape": "String"}}},
+ "KeyList": {"type": "list", "member": {"shape": "String"}},
+ "String": {"type": "string"}}}
+ tags = {"member": "Tags[Tag]", "key": "Key", "value": "Value"}
+ assert request_members(model) == {
+ "s3:TagResource": {"aws:RequestTag/${TagKey}": [tags], "aws:TagKeys": [tags]},
+ "s3:UntagResource": {"aws:TagKeys": [{"member": "tagKeys", "location": "querystring"}]}}
+
+
+def test_the_vendored_table_reads_tag_resource_and_untag_resource():
+ document = json.loads(OUTPUT.read_text())
+ assert validate(document) == []
+ assert document["actions"]["s3:TagResource"] == ["aws:RequestTag/${TagKey}", "aws:TagKeys"]
+ assert "aws:TagKeys" in document["actions"]["s3:UntagResource"]
+ assert document["members"]["s3:TagResource"]["aws:RequestTag/${TagKey}"] == [
+ {"member": "Tags[Tag]", "key": "Key", "value": "Value"}]
+ assert document["members"]["s3:UntagResource"] == {
+ "aws:TagKeys": [{"member": "tagKeys", "location": "querystring"}]}
+
+
[email protected]("changed", [
+ "src/main/resources/aws/s3control-tag-condition-keys.json",
+ "tools/aws/regen_s3control_tag_condition_keys.py",
+ "tools/aws/test_regen_s3control_tag_condition_keys.py",
+ "tools/aws/regen_iam_tag_on_create.py",
+ "",
+])
+def test_aggregate_gate_checks_the_table_and_its_imports(tmp_path, changed):
+ result = run_aggregate_gate(tmp_path, changed, fail=True)
+ assert "checked:s3control-tag-condition-keys-verify" in result.stdout
+ assert result.returncode != 0
+ assert "the S3 Control tag condition key table" in result.stdout
+
+
[email protected]("changed", [
+ "docs/services/s3.md",
+ "src/main/resources/aws/dynamodb-condition-keys.json",
+ "src/main/resources/aws/s3control-tag-condition-keys.json.bak",
+])
+def test_aggregate_gate_does_not_check_the_table_for_unrelated_files(tmp_path, changed):
+ result = run_aggregate_gate(tmp_path, changed, fail=True)
+ assert "checked:s3control-tag-condition-keys-verify" not in result.stdout
+ assert result.returncode == 0
+
+
+def run_aggregate_gate(tmp_path, changed, fail):
+ recursive_make = tmp_path / "check-gate"
+ recursive_make.write_text(
+ '#!/bin/sh\nprintf "checked:%s\\n" "$2"\n'
+ + ('[ "$2" != s3control-tag-condition-keys-verify ]\n' if fail else 'exit 0\n'))
+ recursive_make.chmod(0o700)
+ return subprocess.run(
+ ["make", "--no-print-directory", "iam-data-verify", "CHANGED=" + changed,
+ "MAKE=" + str(recursive_make)],
+ cwd=OUTPUT.parents[4], capture_output=True, text=True, check=False)