Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3control): take TagResource and UntagResource tag condition keys from generated data (#6677)
galsuchetzky-actf926968167dc
diff --git a/.github/workflows/iam-data-drift.yml b/.github/workflows/iam-data-drift.ymlindex aef293dd87..74a834fede 100644--- a/.github/workflows/iam-data-drift.yml+++ b/.github/workflows/iam-data-drift.yml@@ -1,12 +1,13 @@name: IAM Data Drift-# Checks the nine vendored files that come from AWS's live service reference rather than from a+# Checks the ten vendored files that come from AWS's live service reference rather than from a# pinned dependency: the IAM service namespace list, the map of which resource each IAM action is# authorized against, the map of operations AWS authorizes as an action of another name, the# table of the operations that pass a role and the services they pass it to, the table of the# operations that pass a Lambda network connector, the table of the-# tagging actions operations that tag what they create need, and the table of the resources each-# operation's request names, the DynamoDB condition key table, and the IAM condition key table. That+# tagging actions operations that tag what they create need, the table of the resources each+# operation's request names, the DynamoDB condition key table, the IAM condition key table, and+# the S3 Control tag condition key table. That# needs the live service reference index, so unlike every other check here it can# fail without anything in the repository having changed: an AWS outage, or a namespace AWS has# retired. It therefore runs on its own, never as part of a job that ordinary changes trigger.@@ -62,6 +63,9 @@ on:- 'src/main/resources/aws/iam-condition-keys.json'- 'tools/aws/regen_iam_condition_keys.py'- 'tools/aws/test_regen_iam_condition_keys.py'+ - 'src/main/resources/aws/s3control-tag-condition-keys.json'+ - 'tools/aws/regen_s3control_tag_condition_keys.py'+ - 'tools/aws/test_regen_s3control_tag_condition_keys.py'- 'tools/aws/regen_partitions.py'- 'tools/aws/regen_query_services.py'- '.github/workflows/iam-data-drift.yml'diff --git a/.github/workflows/partition-literals.yml b/.github/workflows/partition-literals.ymlindex c12cd3d0a8..aa3f511943 100644--- a/.github/workflows/partition-literals.yml+++ b/.github/workflows/partition-literals.yml@@ -105,6 +105,8 @@ jobs:- name: Check IAM condition key request membersrun: make iam-condition-keys-check+ - name: Check S3 Control tag condition request members+ run: make s3control-tag-condition-keys-check- name: Check partitions.json against a fresh generationrun: make aws-data-checkdiff --git a/Makefile b/Makefileindex b634e15c01..eb9852837e 100644--- a/Makefile+++ b/Makefile@@ -425,6 +425,9 @@ DYNAMODB_CONDITION_KEYS_PATHS := src/main/resources/aws/dynamodb-condition-keys.IAM_CONDITION_KEYS_PATHS := src/main/resources/aws/iam-condition-keys.json \tools/aws/regen_iam_condition_keys.py tools/aws/test_regen_iam_condition_keys.py \tools/aws/regen_partitions.py tools/aws/regen_query_services.py+S3CONTROL_TAG_CONDITION_KEYS_PATHS := src/main/resources/aws/s3control-tag-condition-keys.json \+ tools/aws/regen_s3control_tag_condition_keys.py tools/aws/test_regen_s3control_tag_condition_keys.py \+ tools/aws/regen_iam_tag_on_create.py $(IAM_PASSED_ROLES_GENERATOR)IAM_REQUEST_RESOURCES_PATHS := src/main/resources/aws/iam-request-resources.json \tools/aws/regen_iam_request_resources.py $(IAM_PASSED_ROLES_GENERATOR) \tools/aws/test_regen_iam_request_resources.py@@ -433,7 +436,7 @@ iam-data-verify: ## CI gate (online): every vendored IAM file must still match A@# Every check runs even when an earlier one fails, so one file's drift cannot hide another's@# state. Sequential steps in CI stop at the first failure, which once left the action@# resource gate unexercised while the namespace list was stale.- @namespaces=0; actions=0; authorized=0; passed=0; connectors=0; tagged=0; requested=0; dynamodb=0; conditionkeys=0; \+ @namespaces=0; actions=0; authorized=0; passed=0; connectors=0; tagged=0; requested=0; dynamodb=0; conditionkeys=0; s3control=0; \covers() { \[ -z "$(strip $(CHANGED))" ] && return 0; \for path in $(CHANGED); do case " $$1 " in *" $$path "*) return 0;; esac; done; \@@ -467,9 +470,12 @@ iam-data-verify: ## CI gate (online): every vendored IAM file must still match Aif covers "$(IAM_CONDITION_KEYS_PATHS)" "the IAM condition key table"; then \$(MAKE) --no-print-directory iam-condition-keys-verify || conditionkeys=$$?; \fi; \+ if covers "$(S3CONTROL_TAG_CONDITION_KEYS_PATHS)" "the S3 Control tag condition key table"; then \+ $(MAKE) --no-print-directory s3control-tag-condition-keys-verify || s3control=$$?; \+ fi; \if [ $$namespaces -ne 0 ] || [ $$actions -ne 0 ] || [ $$authorized -ne 0 ] || [ $$passed -ne 0 ] \|| [ $$connectors -ne 0 ] || [ $$tagged -ne 0 ] || [ $$requested -ne 0 ] || [ $$dynamodb -ne 0 ] \- || [ $$conditionkeys -ne 0 ]; then \+ || [ $$conditionkeys -ne 0 ] || [ $$s3control -ne 0 ]; then \echo ""; \echo "error: vendored IAM data does not match upstream:"; \if [ $$namespaces -ne 0 ]; then \@@ -495,6 +501,7 @@ iam-data-verify: ## CI gate (online): every vendored IAM file must still match Afi; \if [ $$dynamodb -ne 0 ]; then echo " the DynamoDB condition key table"; fi; \if [ $$conditionkeys -ne 0 ]; then echo " the IAM condition key table"; fi; \+ if [ $$s3control -ne 0 ]; then echo " the S3 Control tag condition key table"; fi; \exit 1; \fi@@ -680,3 +687,11 @@ iam-condition-keys-check: ## Check the IAM condition key request members against$(PYTHON) tools/aws/regen_iam_condition_keys.py --checkiam-condition-keys-verify: ## Check every IAM condition key and action pair against AWS$(PYTHON) tools/aws/regen_iam_condition_keys.py --verify++.PHONY: s3control-tag-condition-keys-sync s3control-tag-condition-keys-check s3control-tag-condition-keys-verify+s3control-tag-condition-keys-sync: ## Regenerate the S3 Control tag condition keys from AWS and botocore+ $(PYTHON) tools/aws/regen_s3control_tag_condition_keys.py+s3control-tag-condition-keys-check: ## Check the S3 Control tag condition request members against pinned botocore+ $(PYTHON) tools/aws/regen_s3control_tag_condition_keys.py --check+s3control-tag-condition-keys-verify: ## Check the S3 Control tag condition action keys against AWS+ $(PYTHON) tools/aws/regen_s3control_tag_condition_keys.py --verifydiff --git a/docs/services/iam.md b/docs/services/iam.mdindex c72d785ba2..92e3bd2e85 100644--- a/docs/services/iam.md+++ b/docs/services/iam.md@@ -2238,8 +2238,9 @@ floci populates:- `s3:prefix`, `s3:delimiter`, `s3:max-keys`: from the S3 request parameters.- `aws:RequestTag/<key>`: the tags named in the request itself, before they are applied, for`ec2:RunInstances` (`TagSpecification.N`), `ec2:CreateTags` (`Tag.N`) and- `s3:PutBucketTagging` (the `<Tagging>` body), and for every create in the tag-on-create table- whose action takes it, as described under the tag-on-create check above.+ `s3:PutBucketTagging` (the `<Tagging>` body), for S3 Control's `s3:TagResource` (its `Tags` body),+ and for every create in the tag-on-create table whose action takes it, as described under the+ tag-on-create check above.- `aws:ResourceTag/<key>`: the target resource's current tags, for `ec2:CreateTags`,`ec2:DeleteTags`, `ec2:TerminateInstances` and `ec2:DescribeInstances` (the first`ResourceId.N` or `InstanceId.N`), and for `s3:GetBucketTagging`, `s3:DeleteBucket` and the@@ -2314,8 +2315,11 @@ floci populates:cannot match: an absent key has no equal or like value. That absence is the enforcement ratherthan a gap in it.- `aws:TagKeys`: the case-sensitive requested tag keys for EC2 `RunInstances` and- `CreateTags`, S3 `PutBucketTagging`, and every create in the tag-on-create table whose action- takes it. `ForAllValues:StringEquals` can restrict the+ `CreateTags`, S3 `PutBucketTagging`, S3 Control `TagResource` (its `Tags` body) and+ `UntagResource` (its `tagKeys` query parameters), and every create in the tag-on-create table+ whose action takes it. S3 Control's come from `s3control-tag-condition-keys.json`, which+ `tools/aws/regen_s3control_tag_condition_keys.py` builds from the Service Reference's condition+ keys and botocore's `s3control` request members; S3's own were measured against AWS. `ForAllValues:StringEquals` can restrict thecomplete requested set without a `Null` guard; no tags still means absent context.- `dynamodb:LeadingKeys`, `dynamodb:Attributes`, `dynamodb:Select`: from the DynamoDB requestbody, for `GetItem`, `PutItem`, `UpdateItem`, `DeleteItem`, `Query`, `BatchGetItem` anddiff --git a/src/main/java/io/github/hectorvent/floci/core/common/IamConditionContextResolver.java b/src/main/java/io/github/hectorvent/floci/core/common/IamConditionContextResolver.javaindex f4b0af29b2..14a1dd706d 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/IamConditionContextResolver.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/IamConditionContextResolver.java@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.services.ec2.model.Tag;import io.github.hectorvent.floci.services.iam.IamActionRegistry;import io.github.hectorvent.floci.services.iam.ResourceArnBuilder;+import io.github.hectorvent.floci.services.s3.S3ControlTagConditionKeys;import io.github.hectorvent.floci.services.s3.S3ObjectActions;import io.github.hectorvent.floci.services.s3.S3ObjectKeys;import io.github.hectorvent.floci.services.s3.S3Service;@@ -296,6 +297,12 @@ public class IamConditionContextResolver {// ── S3 ──────────────────────────────────────────────────────────────────────private Map<String, List<String>> s3ConditionContext(String action, ContainerRequestContext ctx) {+ // S3 Control's request tag keys come from the generated table; S3's own below were measured+ // against AWS, and do not all match the Service Reference.+ if (IamActionRegistry.isS3ControlRequest("s3", ctx)) {+ return S3ControlTagConditionKeys.conditions(action, () -> bodyText(ctx),+ ctx.getUriInfo().getQueryParameters());+ }Map<String, List<String>> conditions = s3ActionConditionContext(action, ctx);// A version action is checked only for a request naming a version, and carries its id.if (S3ObjectActions.isVersionAction(action) && ctx.getUriInfo() != null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeys.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeys.javanew file mode 100644index 0000000000..f608a210b4--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeys.java@@ -0,0 +1,89 @@+package io.github.hectorvent.floci.services.s3;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.VendoredResources;+import io.github.hectorvent.floci.core.common.XmlParser;+import jakarta.ws.rs.core.MultivaluedMap;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.function.Supplier;++/**+ * The request tag condition keys of an S3 Control request, {@code aws:RequestTag/<key>} and+ * {@code aws:TagKeys}, read from the table {@code tools/aws/regen_s3control_tag_condition_keys.py}+ * generates: the actions the Service Reference gives either key, and the request members botocore's+ * {@code s3control} model carries them in. S3 Control's routes are checked as the action of their+ * operation's name, so an action names its operation's members.+ */+public final class S3ControlTagConditionKeys {++ private static final String REQUEST_TAG = "aws:RequestTag/${TagKey}";+ private static final String TAG_KEYS = "aws:TagKeys";+ private static final String REQUEST_TAG_PREFIX = "aws:RequestTag/";++ private static final JsonNode TABLE = VendoredResources.read("aws/s3control-tag-condition-keys.json",+ "S3 Control tag condition keys", input -> {+ JsonNode document = new ObjectMapper().readTree(input);+ if (!document.path("actions").isObject() || document.path("actions").isEmpty()+ || !document.path("members").isObject() || document.path("members").isEmpty()) {+ throw new IllegalStateException("Malformed S3 Control tag condition key table");+ }+ return document;+ });++ private S3ControlTagConditionKeys() {}++ /**+ * The request tag keys {@code action} takes, from the members of the request that carry them:+ * the tags in {@code body}, read as {@code S3ControlController} reads them, and the tag keys in+ * {@code query}. Null when the action takes neither key or the request carries no tags.+ */+ public static Map<String, List<String>> conditions(String action, Supplier<String> body,+ MultivaluedMap<String, String> query) {+ JsonNode members = TABLE.path("members").path(action == null ? "" : action);+ Map<String, List<String>> conditions = new LinkedHashMap<>();+ if (takes(action, REQUEST_TAG)) {+ for (JsonNode member : members.path(REQUEST_TAG)) {+ bodyTags(member, body).forEach((key, value) -> conditions.put(REQUEST_TAG_PREFIX + key, List.of(value)));+ }+ }+ if (takes(action, TAG_KEYS)) {+ List<String> keys = new ArrayList<>();+ for (JsonNode member : members.path(TAG_KEYS)) {+ List<String> named = "querystring".equals(member.path("location").asText())+ ? query.getOrDefault(member.path("member").asText(), List.of())+ : List.copyOf(bodyTags(member, body).keySet());+ named.stream().filter(key -> !keys.contains(key)).forEach(keys::add);+ }+ if (!keys.isEmpty()) {+ conditions.put(TAG_KEYS, keys);+ }+ }+ return conditions.isEmpty() ? null : conditions;+ }++ private static boolean takes(String action, String key) {+ for (JsonNode taken : TABLE.path("actions").path(action == null ? "" : action)) {+ if (key.equals(taken.asText())) {+ return true;+ }+ }+ return false;+ }++ /** The tags a body member holds, by the element its list wraps each tag in. */+ private static Map<String, String> bodyTags(JsonNode member, Supplier<String> body) {+ String path = member.path("member").asText();+ int open = path.indexOf('[');+ if (open < 0 || !member.has("key") || !member.has("value")) {+ return Map.of();+ }+ String element = path.substring(open + 1, path.length() - 1);+ return XmlParser.extractPairs(body.get(), element.isEmpty() ? "member" : element,+ member.path("key").asText(), member.path("value").asText());+ }+}diff --git a/src/main/resources/aws/s3control-tag-condition-keys.json b/src/main/resources/aws/s3control-tag-condition-keys.jsonnew file mode 100644index 0000000000..2e7db68ad0--- /dev/null+++ b/src/main/resources/aws/s3control-tag-condition-keys.json@@ -0,0 +1,352 @@+{+ "_source": {+ "generator": "tools/aws/regen_s3control_tag_condition_keys.py",+ "reference": "https://servicereference.us-east-1.amazonaws.com/v1/s3/s3.json",+ "botocore": "botocore 1.43.113 (installed package)",+ "note": "make s3control-tag-condition-keys-check re-derives members from botocore; make s3control-tag-condition-keys-verify checks actions against AWS."+ },+ "actions": {+ "s3:AssociateAccessGrantsIdentityCenter": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:CreateAccessGrant": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:CreateAccessGrantsInstance": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:CreateAccessGrantsLocation": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:CreateAccessPoint": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:CreateBucket": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:CreateJob": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:CreateStorageLensGroup": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DeleteAccessGrant": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DeleteAccessGrantsInstance": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DeleteAccessGrantsInstanceResourcePolicy": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DeleteAccessGrantsLocation": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DeleteJobTagging": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DeleteStorageLensConfiguration": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DeleteStorageLensConfigurationTagging": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DeleteStorageLensGroup": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DescribeJob": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:DissociateAccessGrantsIdentityCenter": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetAccessGrant": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetAccessGrantsInstance": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetAccessGrantsInstanceForPrefix": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetAccessGrantsInstanceResourcePolicy": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetAccessGrantsLocation": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetDataAccess": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetJobTagging": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetStorageLensConfiguration": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetStorageLensConfigurationTagging": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetStorageLensDashboard": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:GetStorageLensGroup": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:ListAccessGrants": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:ListAccessGrantsLocations": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:ListCallerAccessGrants": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:ListTagsForResource": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:PutAccessGrantsInstanceResourcePolicy": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:PutJobTagging": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:PutStorageLensConfiguration": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:PutStorageLensConfigurationTagging": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:TagResource": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:UntagResource": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:UpdateAccessGrantsLocation": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:UpdateJobPriority": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:UpdateJobStatus": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ],+ "s3:UpdateStorageLensGroup": [+ "aws:RequestTag/${TagKey}",+ "aws:TagKeys"+ ]+ },+ "members": {+ "s3:CreateAccessGrant": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:CreateAccessGrantsInstance": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:CreateAccessGrantsLocation": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:CreateAccessPoint": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:CreateJob": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:CreateStorageLensGroup": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:PutJobTagging": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:PutStorageLensConfiguration": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:PutStorageLensConfigurationTagging": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:TagResource": {+ "aws:RequestTag/${TagKey}": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ],+ "aws:TagKeys": [+ {+ "member": "Tags[Tag]",+ "key": "Key",+ "value": "Value"+ }+ ]+ },+ "s3:UntagResource": {+ "aws:TagKeys": [+ {+ "member": "tagKeys",+ "location": "querystring"+ }+ ]+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlIamEnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlIamEnforcementIntegrationTest.javaindex 5fdf568f6f..4e1746b35a 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlIamEnforcementIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlIamEnforcementIntegrationTest.java@@ -503,6 +503,74 @@ class S3ControlIamEnforcementIntegrationTest {.body(containsString("<Code>NoSuchBucket</Code>"));}+ /**+ * TagResource is checked with the tags it sends as {@code aws:RequestTag/<key>} and+ * {@code aws:TagKeys}, the condition keys the Service Authorization Reference lists for it.+ */+ @Test+ void tagResourceIsCheckedAgainstTheTagsItsRequestSends() {+ String bucket = bucketWithTags("control-request-tag");+ String user = userWith("""+ {"Version":"2012-10-17","Statement":[+ {"Effect":"Allow","Action":"s3:TagResource","Resource":"arn:aws:s3:::%1$s",+ "Condition":{"StringEquals":{"aws:RequestTag/Owner":"team"}}}]}"""+ .formatted(bucket));++ given()+ .header("Authorization", auth(user, "s3"))+ .header("x-amz-account-id", ACCOUNT)+ .contentType("application/xml")+ .body("<Tagging><Tags><Tag><Key>Owner</Key><Value>team</Value></Tag></Tags></Tagging>")+ .when()+ .post("/v20180820/tags/arn:aws:s3:::" + bucket)+ .then()+ .statusCode(204);++ given()+ .header("Authorization", auth(user, "s3"))+ .header("x-amz-account-id", ACCOUNT)+ .contentType("application/xml")+ .body("<Tagging><Tags><Tag><Key>Owner</Key><Value>other</Value></Tag></Tags></Tagging>")+ .when()+ .post("/v20180820/tags/arn:aws:s3:::" + bucket)+ .then()+ .statusCode(403)+ .body(containsString("is not authorized to perform: s3:TagResource"));+ }++ /**+ * UntagResource is checked with the keys it removes as {@code aws:TagKeys}, the condition key+ * the Service Authorization Reference lists for it.+ */+ @Test+ void untagResourceIsCheckedAgainstTheKeysItsRequestNames() {+ String bucket = bucketWithTags("control-tag-keys");+ String user = userWith("""+ {"Version":"2012-10-17","Statement":[+ {"Effect":"Allow","Action":"s3:UntagResource","Resource":"arn:aws:s3:::%1$s",+ "Condition":{"ForAllValues:StringEquals":{"aws:TagKeys":["Env"]}}}]}"""+ .formatted(bucket));++ given()+ .header("Authorization", auth(user, "s3"))+ .header("x-amz-account-id", ACCOUNT)+ .queryParam("tagKeys", "Env", "Owner")+ .when()+ .delete("/v20180820/tags/arn:aws:s3:::" + bucket)+ .then()+ .statusCode(403)+ .body(containsString("is not authorized to perform: s3:UntagResource"));++ given()+ .header("Authorization", auth(user, "s3"))+ .header("x-amz-account-id", ACCOUNT)+ .queryParam("tagKeys", "Env")+ .when()+ .delete("/v20180820/tags/arn:aws:s3:::" + bucket)+ .then()+ .statusCode(204);+ }+/** A bucket created by the root stand-in, tagged through the data plane, as Terraform reads it. */private String bucketWithTags(String prefix) {String bucket = prefix + "-" + UUID.randomUUID().toString().substring(0, 8);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeysTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeysTest.javanew file mode 100644index 0000000000..d70260d399--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControlTagConditionKeysTest.java@@ -0,0 +1,42 @@+package io.github.hectorvent.floci.services.s3;++import jakarta.ws.rs.core.MultivaluedHashMap;+import jakarta.ws.rs.core.MultivaluedMap;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;++class S3ControlTagConditionKeysTest {++ private static final String TAGS = "<TagResourceRequest><Tags>"+ + "<Tag><Key>Owner</Key><Value>team</Value></Tag>"+ + "<Tag><Key>Env</Key><Value>dev</Value></Tag>"+ + "</Tags></TagResourceRequest>";++ @Test+ void tagResourceTakesEachTagOfItsBodyAndTheirKeys() {+ assertEquals(Map.of("aws:RequestTag/Owner", List.of("team"), "aws:RequestTag/Env", List.of("dev"),+ "aws:TagKeys", List.of("Owner", "Env")),+ S3ControlTagConditionKeys.conditions("s3:TagResource", () -> TAGS, new MultivaluedHashMap<>()));+ }++ @Test+ void untagResourceTakesTheKeysItsQueryNamesAndNoRequestTag() {+ MultivaluedMap<String, String> query = new MultivaluedHashMap<>();+ query.addAll("tagKeys", "Owner", "Env");+ assertEquals(Map.of("aws:TagKeys", List.of("Owner", "Env")),+ S3ControlTagConditionKeys.conditions("s3:UntagResource", () -> TAGS, query));+ }++ @Test+ void aRequestCarryingNoTagsAndAnActionTakingNoKeyGetNothing() {+ assertNull(S3ControlTagConditionKeys.conditions("s3:TagResource", () -> "", new MultivaluedHashMap<>()));+ assertNull(S3ControlTagConditionKeys.conditions("s3:UntagResource", () -> "", new MultivaluedHashMap<>()));+ assertNull(S3ControlTagConditionKeys.conditions("s3:ListAccessPoints", () -> TAGS,+ new MultivaluedHashMap<>()));+ }+}diff --git a/tools/aws/regen_s3control_tag_condition_keys.py b/tools/aws/regen_s3control_tag_condition_keys.pynew file mode 100644index 0000000000..d6ddf0b95b--- /dev/null+++ b/tools/aws/regen_s3control_tag_condition_keys.py@@ -0,0 +1,174 @@+#!/usr/bin/env python3+"""Regenerate src/main/resources/aws/s3control-tag-condition-keys.json from AWS's Service Reference and botocore.++An S3 Control request is checked with the tags it carries as `aws:RequestTag/<key>` and `aws:TagKeys`+when its action takes them. The table comes from two sources, each answering what the other cannot:++- AWS's Service Reference for `s3`, which gives every action's condition keys, for the action and for+ each resource type it acts on, but not where a request carries the tags:++ https://servicereference.us-east-1.amazonaws.com/v1/s3/s3.json++- botocore's `s3control` model (Apache-2.0, https://github.com/boto/botocore, the pinned release),+ which gives each request's members but not which of them AWS checks.++`actions` holds every `s3` action that takes either key. `members` holds, for every S3 Control+operation, as `s3:<Operation>`, the request members that feed each key: a top-level body member+holding tags, found as `regen_iam_tag_on_create.py` finds them (a list of key and value structures,+named by its list element, or a map of strings), feeds both keys, and a top-level query string list+of tag keys feeds `aws:TagKeys`. A member nested deeper describes something else: a job's manifest,+a Storage Lens configuration's filters.++Only S3 Control's model is read. S3's own requests carry the tag keys `IamConditionContextResolver`+gives them, which were measured against AWS and do not all match the reference: `s3:PutBucketTagging`+takes `aws:RequestTag/<key>` and `aws:TagKeys` on AWS though the reference lists neither for it.++The Service Reference is live rather than a pinned dependency. The two gates split along the sources:++- `--check` is offline and needs the pinned botocore: `members` must be exactly what botocore gives.+- `--verify` is online and stdlib-only: `actions` must be exactly what the reference gives.++Run from anywhere in the repo:+ python3 tools/aws/regen_s3control_tag_condition_keys.py # rewrite the vendored file in place+ python3 tools/aws/regen_s3control_tag_condition_keys.py --check # offline: members match botocore+ python3 tools/aws/regen_s3control_tag_condition_keys.py --verify # online: actions match AWS+ python3 tools/aws/regen_s3control_tag_condition_keys.py --source <s3.json> # read a saved reference+"""+from __future__ import annotations++import argparse+import json+import re+import sys+import urllib.request+from pathlib import Path++from regen_iam_tag_on_create import (REQUEST_TAG_KEYS, TAG_MEMBER_RE, _collection, _condition_keys,+ _resource_keys, _serialized)+from regen_partitions import resolve_botocore_data+from regen_query_services import latest_model_dir, load_model++OUTPUT = Path(__file__).resolve().parents[2] / "src/main/resources/aws/s3control-tag-condition-keys.json"+URL = "https://servicereference.us-east-1.amazonaws.com/v1/s3/s3.json"+NAMESPACE = "s3"+MODEL = "s3control"+REQUEST_TAG = "aws:RequestTag/${TagKey}"+TAG_KEYS = "aws:TagKeys"+# A request member that lists the keys of the tags the operation removes.+TAG_KEY_LIST_RE = re.compile(r"^\w*tagkeys$", re.IGNORECASE)+BODY_LOCATIONS = (None, "")+++def action_keys(reference: dict) -> dict[str, list[str]]:+ """Every action of the namespace that takes a request tag key, with the ones it takes."""+ resource_keys = _resource_keys([reference])+ found = {}+ for action in sorted(reference["Actions"], key=lambda entry: entry["Name"]):+ keys = REQUEST_TAG_KEYS & _condition_keys(NAMESPACE, action, resource_keys)+ if keys:+ found[f"{NAMESPACE}:{action['Name']}"] = sorted(keys)+ return found+++def request_members(model: dict) -> dict[str, dict[str, list[dict]]]:+ """The request members of every operation that feed each request tag key."""+ found = {}+ for operation, spec in sorted(model["operations"].items()):+ if "input" not in spec:+ continue+ tags, tag_keys = [], []+ for name, member in sorted(model["shapes"][spec["input"]["shape"]].get("members", {}).items()):+ location = member.get("location")+ if location in BODY_LOCATIONS and TAG_MEMBER_RE.match(name):+ collection = _collection(model, member["shape"])+ if collection is not None:+ tags.append({"member": _serialized(name, member, model) + collection.pop("step"), **collection})+ elif location == "querystring" and TAG_KEY_LIST_RE.match(name) and _is_string_list(model, member):+ tag_keys.append({"member": member.get("locationName", name), "location": "querystring"})+ keys = {}+ if tags:+ keys[REQUEST_TAG] = tags+ if tags or tag_keys:+ keys[TAG_KEYS] = tags + tag_keys+ if keys:+ found[f"{NAMESPACE}:{operation}"] = keys+ return found+++def _is_string_list(model: dict, member: dict) -> bool:+ shape = model["shapes"][member["shape"]]+ return shape["type"] == "list" and model["shapes"][shape["member"]["shape"]]["type"] == "string"+++def render(document: dict) -> str:+ return json.dumps(document, indent=2) + "\n"+++def validate(document: dict) -> list[str]:+ """Problems that make the vendored file unusable, rather than merely out of date."""+ problems = []+ for section in ("actions", "members"):+ if not isinstance(document.get(section), dict) or not document[section]:+ problems.append(f"{section} is missing or empty")+ if problems:+ return problems+ for operation in ("s3:TagResource", "s3:UntagResource"):+ if operation not in document["actions"]:+ problems.append(f"{operation} takes no request tag key")+ if operation not in document["members"]:+ problems.append(f"{operation} names no member that carries tags")+ return problems+++def _reference(source: Path | None) -> dict:+ if source is not None:+ return json.loads(source.read_text(encoding="utf-8"))+ request = urllib.request.Request(URL, headers={"Accept": "application/json"})+ with urllib.request.urlopen(request, timeout=30) as response:+ return json.load(response)+++def _model(botocore_data: Path | None) -> tuple[dict, str]:+ data, provenance = resolve_botocore_data(botocore_data)+ return load_model(latest_model_dir(data, MODEL)), provenance+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ parser.add_argument("--check", action="store_true", help="offline: members must match the pinned botocore")+ parser.add_argument("--verify", action="store_true", help="online: actions must match the Service Reference")+ parser.add_argument("--source", type=Path, help="a saved s3.json Service Reference document")+ parser.add_argument("--botocore-data", type=Path, help="botocore data directory")+ args = parser.parse_args(argv)++ if args.check or args.verify:+ document = json.loads(OUTPUT.read_text(encoding="utf-8"))+ problems = validate(document)+ if args.check:+ model, provenance = _model(args.botocore_data)+ if document["members"] != request_members(model):+ problems.append(f"members differ from {provenance}")+ if args.verify and document["actions"] != action_keys(_reference(args.source)):+ problems.append("actions differ from the AWS Service Reference")+ for problem in problems:+ print(f"{OUTPUT.name}: {problem}", file=sys.stderr)+ return 1 if problems else 0++ model, provenance = _model(args.botocore_data)+ document = {"_source": {"generator": "tools/aws/regen_s3control_tag_condition_keys.py", "reference": URL,+ "botocore": provenance,+ "note": "make s3control-tag-condition-keys-check re-derives members from botocore; "+ "make s3control-tag-condition-keys-verify checks actions against AWS."},+ "actions": action_keys(_reference(args.source)), "members": request_members(model)}+ problems = validate(document)+ if problems:+ for problem in problems:+ print(f"error: {problem}", file=sys.stderr)+ return 1+ OUTPUT.write_text(render(document), encoding="utf-8")+ print(f"wrote {OUTPUT.name} from {provenance}")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/aws/test_regen_s3control_tag_condition_keys.py b/tools/aws/test_regen_s3control_tag_condition_keys.pynew file mode 100644index 0000000000..5f95992513--- /dev/null+++ b/tools/aws/test_regen_s3control_tag_condition_keys.py@@ -0,0 +1,87 @@+"""Action keys and request members for the S3 Control tag condition key table."""+import json+import subprocess++import pytest++from regen_s3control_tag_condition_keys import OUTPUT, action_keys, request_members, validate+++def test_action_and_resource_keys_are_combined_and_filtered():+ reference = {"Name": "s3", "Resources": [{"Name": "job", "ConditionKeys": ["aws:RequestTag/${TagKey}", "s3:JobSuspendedCause"]}],+ "Actions": [{"Name": "TagResource", "ActionConditionKeys": ["aws:TagKeys", "s3:authType"]},+ {"Name": "PutJobTagging", "Resources": [{"Name": "job"}]},+ {"Name": "ListAccessPoints", "ActionConditionKeys": ["s3:authType"]}]}+ assert action_keys(reference) == {"s3:PutJobTagging": ["aws:RequestTag/${TagKey}"],+ "s3:TagResource": ["aws:TagKeys"]}+++def test_body_tags_feed_both_keys_and_query_tag_keys_feed_tag_keys():+ model = {"metadata": {"protocol": "rest-xml"}, "operations": {+ "TagResource": {"input": {"shape": "TagIn"}},+ "UntagResource": {"input": {"shape": "UntagIn"}},+ "CreateJob": {"input": {"shape": "JobIn"}},+ "ListAccessPoints": {"input": {"shape": "ListIn"}}}, "shapes": {+ "TagIn": {"type": "structure", "members": {"ResourceArn": {"shape": "String", "location": "uri"},+ "Tags": {"shape": "TagList"}}},+ "UntagIn": {"type": "structure", "members": {+ "TagKeys": {"shape": "KeyList", "location": "querystring", "locationName": "tagKeys"}}},+ "JobIn": {"type": "structure", "members": {"Manifest": {"shape": "Manifest"}}},+ "Manifest": {"type": "structure", "members": {"Tags": {"shape": "TagList"}}},+ "ListIn": {"type": "structure", "members": {"Bucket": {"shape": "String", "location": "querystring"}}},+ "TagList": {"type": "list", "member": {"shape": "Tag", "locationName": "Tag"}},+ "Tag": {"type": "structure", "members": {"Key": {"shape": "String"}, "Value": {"shape": "String"}}},+ "KeyList": {"type": "list", "member": {"shape": "String"}},+ "String": {"type": "string"}}}+ tags = {"member": "Tags[Tag]", "key": "Key", "value": "Value"}+ assert request_members(model) == {+ "s3:TagResource": {"aws:RequestTag/${TagKey}": [tags], "aws:TagKeys": [tags]},+ "s3:UntagResource": {"aws:TagKeys": [{"member": "tagKeys", "location": "querystring"}]}}+++def test_the_vendored_table_reads_tag_resource_and_untag_resource():+ document = json.loads(OUTPUT.read_text())+ assert validate(document) == []+ assert document["actions"]["s3:TagResource"] == ["aws:RequestTag/${TagKey}", "aws:TagKeys"]+ assert "aws:TagKeys" in document["actions"]["s3:UntagResource"]+ assert document["members"]["s3:TagResource"]["aws:RequestTag/${TagKey}"] == [+ {"member": "Tags[Tag]", "key": "Key", "value": "Value"}]+ assert document["members"]["s3:UntagResource"] == {+ "aws:TagKeys": [{"member": "tagKeys", "location": "querystring"}]}++[email protected]("changed", [+ "src/main/resources/aws/s3control-tag-condition-keys.json",+ "tools/aws/regen_s3control_tag_condition_keys.py",+ "tools/aws/test_regen_s3control_tag_condition_keys.py",+ "tools/aws/regen_iam_tag_on_create.py",+ "",+])+def test_aggregate_gate_checks_the_table_and_its_imports(tmp_path, changed):+ result = run_aggregate_gate(tmp_path, changed, fail=True)+ assert "checked:s3control-tag-condition-keys-verify" in result.stdout+ assert result.returncode != 0+ assert "the S3 Control tag condition key table" in result.stdout++[email protected]("changed", [+ "docs/services/s3.md",+ "src/main/resources/aws/dynamodb-condition-keys.json",+ "src/main/resources/aws/s3control-tag-condition-keys.json.bak",+])+def test_aggregate_gate_does_not_check_the_table_for_unrelated_files(tmp_path, changed):+ result = run_aggregate_gate(tmp_path, changed, fail=True)+ assert "checked:s3control-tag-condition-keys-verify" not in result.stdout+ assert result.returncode == 0+++def run_aggregate_gate(tmp_path, changed, fail):+ recursive_make = tmp_path / "check-gate"+ recursive_make.write_text(+ '#!/bin/sh\nprintf "checked:%s\\n" "$2"\n'+ + ('[ "$2" != s3control-tag-condition-keys-verify ]\n' if fail else 'exit 0\n'))+ recursive_make.chmod(0o700)+ return subprocess.run(+ ["make", "--no-print-directory", "iam-data-verify", "CHANGED=" + changed,+ "MAKE=" + str(recursive_make)],+ cwd=OUTPUT.parents[4], capture_output=True, text=True, check=False)