Light, fluffy, and always free - The AWS Local Emulator alternative
feat(datasync): add the DataSync management plane (#3377)
lex00128d3ef18134
diff --git a/README.md b/README.mdindex 3562cd1114..3800690d9e 100644--- a/README.md+++ b/README.md@@ -237,7 +237,7 @@ Floci supports local emulation for application services, data services, eventing| Containers and compute | ECS, EC2, Lightsail, EKS, MWAA, ECR, EFS, CodeBuild, CodeDeploy, CodePipeline, CodeGuru Reviewer, AWS Batch, Auto Scaling, Application Auto Scaling, Elastic Beanstalk, ELB v2, ELB Classic || Data, analytics, and AI | Athena, Glue, Lake Formation, EMR, EMR Serverless, Redshift, Redshift Data API, Firehose, Managed Service for Apache Flink, OpenSearch, S3 Tables, S3 Vectors, Textract, Transcribe, Comprehend, Rekognition, Translate, Bedrock, Bedrock Runtime, Bedrock AgentCore || Databases and caching | RDS, RDS Data API, Neptune, DocumentDB, MemoryDB, ElastiCache |-| Messaging and transfer | SES, Kinesis, MSK, Amazon MQ, Transfer Family, IoT Core, Amazon Connect, Amazon AppIntegrations |+| Messaging and transfer | SES, Kinesis, MSK, Amazon MQ, Transfer Family, DataSync, IoT Core, Amazon Connect, Amazon AppIntegrations || Security and governance | AWS Network Firewall, AWS RAM, Service Quotas, WAF v2, GuardDuty, Amazon Inspector, CloudTrail, CloudFront, Resource Groups Tagging API, Resource Explorer 2, CloudHSM v2, Organizations, AWS Account Management, IAM Access Analyzer, IAM Identity Center (SSO Admin, OIDC, Access Portal, SCIM), Identity Store, Amazon Macie, Amazon Detective, Security Hub, Amazon Verified Permissions, Control Catalog, Control Tower, Service Catalog, AWS Marketplace || Cost and billing | AWS Budgets, Pricing, Cost Explorer, Cost and Usage Reports, BCM Data Exports || Resilience, backup, and config | AWS FIS, AWS Backup, AWS Config, AppConfig, AppConfigData, CloudFormation, Cloud Control API |diff --git a/compatibility-tests/sdk-test-java/pom.xml b/compatibility-tests/sdk-test-java/pom.xmlindex cf8c9720f9..334324f35c 100644--- a/compatibility-tests/sdk-test-java/pom.xml+++ b/compatibility-tests/sdk-test-java/pom.xml@@ -560,6 +560,11 @@<groupId>software.amazon.awssdk</groupId><artifactId>appintegrations</artifactId></dependency>+ <!-- DataSync client -->+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>datasync</artifactId>+ </dependency><!-- JDBC drivers for RDS data-plane tests --><dependency><groupId>org.postgresql</groupId>diff --git a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.javaindex 3b21999c0d..5896826697 100644--- a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java+++ b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java@@ -106,6 +106,7 @@ import software.amazon.awssdk.services.scheduler.SchedulerClient;import software.amazon.awssdk.services.appconfig.AppConfigClient;import software.amazon.awssdk.services.appconfigdata.AppConfigDataClient;import software.amazon.awssdk.services.appintegrations.AppIntegrationsClient;+import software.amazon.awssdk.services.datasync.DataSyncClient;import software.amazon.awssdk.services.autoscaling.AutoScalingClient;import software.amazon.awssdk.services.backup.BackupClient;import software.amazon.awssdk.services.elasticloadbalancingv2.ElasticLoadBalancingV2Client;@@ -1373,4 +1374,12 @@ public final class TestFixtures {.credentialsProvider(CREDENTIALS).build();}++ public static DataSyncClient dataSyncClient() {+ return DataSyncClient.builder()+ .endpointOverride(ENDPOINT)+ .region(REGION)+ .credentialsProvider(CREDENTIALS)+ .build();+ }}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataSyncTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataSyncTest.javanew file mode 100644index 0000000000..d19622dd92--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataSyncTest.java@@ -0,0 +1,232 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;+import software.amazon.awssdk.services.datasync.DataSyncClient;+import software.amazon.awssdk.services.datasync.model.AgentStatus;+import software.amazon.awssdk.services.datasync.model.CreateTaskResponse;+import software.amazon.awssdk.services.datasync.model.DescribeAgentResponse;+import software.amazon.awssdk.services.datasync.model.DescribeLocationNfsResponse;+import software.amazon.awssdk.services.datasync.model.DescribeLocationS3Response;+import software.amazon.awssdk.services.datasync.model.DescribeTaskResponse;+import software.amazon.awssdk.services.datasync.model.InvalidRequestException;+import software.amazon.awssdk.services.datasync.model.S3StorageClass;+import software.amazon.awssdk.services.datasync.model.TagListEntry;+import software.amazon.awssdk.services.datasync.model.TaskMode;+import software.amazon.awssdk.services.datasync.model.TaskStatus;+import software.amazon.awssdk.services.datasync.model.VerifyMode;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++@DisplayName("DataSync")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class DataSyncTest {++ private static DataSyncClient datasync;+ private static String suffix;+ private static String agentArn;+ private static String s3LocationArn;+ private static String nfsLocationArn;+ private static String taskArn;+ private static String enhancedTaskArn;++ @BeforeAll+ static void setup() {+ datasync = TestFixtures.dataSyncClient();+ suffix = String.valueOf(System.currentTimeMillis());+ }++ @AfterAll+ static void cleanup() {+ if (datasync == null) {+ return;+ }+ deleteQuietly("task", enhancedTaskArn, arn -> datasync.deleteTask(r -> r.taskArn(arn)));+ deleteQuietly("task", taskArn, arn -> datasync.deleteTask(r -> r.taskArn(arn)));+ deleteQuietly("location", nfsLocationArn, arn -> datasync.deleteLocation(r -> r.locationArn(arn)));+ deleteQuietly("location", s3LocationArn, arn -> datasync.deleteLocation(r -> r.locationArn(arn)));+ deleteQuietly("agent", agentArn, arn -> datasync.deleteAgent(r -> r.agentArn(arn)));+ datasync.close();+ }++ private static void deleteQuietly(String kind, String arn, java.util.function.Consumer<String> delete) {+ if (arn == null) {+ return;+ }+ try {+ delete.accept(arn);+ } catch (RuntimeException alreadyGone) {+ System.out.println("DataSync cleanup left a " + kind + " behind: " + arn+ + " (" + alreadyGone.getMessage() + ")");+ }+ }++ @Test+ @Order(1)+ void createAgentIsOnlineOnTheFirstDescribe() {+ agentArn = datasync.createAgent(r -> r+ .activationKey("AAAAA-1AAAA-BB1CC-DDDDD-EEEEE")+ .agentName("sdk-test-agent-" + suffix)+ .tags(TagListEntry.builder().key("team").value("platform").build()))+ .agentArn();++ assertThat(agentArn).contains(":datasync:").contains(":agent/agent-");++ DescribeAgentResponse described = datasync.describeAgent(r -> r.agentArn(agentArn));++ assertThat(described.agentArn()).isEqualTo(agentArn);+ assertThat(described.name()).isEqualTo("sdk-test-agent-" + suffix);+ assertThat(described.status()).isEqualTo(AgentStatus.ONLINE);+ assertThat(described.endpointType()).isNotNull();+ assertThat(described.creationTime()).isNotNull();+ }++ @Test+ @Order(2)+ void createS3LocationAppliesTheDocumentedStorageClassDefault() {+ s3LocationArn = datasync.createLocationS3(r -> r+ .s3BucketArn("arn:aws:s3:::sdk-test-datasync-" + suffix)+ .subdirectory("/backups")+ .s3Config(c -> c.bucketAccessRoleArn("arn:aws:iam::000000000000:role/datasync")))+ .locationArn();++ assertThat(s3LocationArn).contains(":location/loc-");++ DescribeLocationS3Response described = datasync.describeLocationS3(r -> r.locationArn(s3LocationArn));++ assertThat(described.locationArn()).isEqualTo(s3LocationArn);+ assertThat(described.locationUri()).startsWith("s3://sdk-test-datasync-" + suffix);+ assertThat(described.s3StorageClass()).isEqualTo(S3StorageClass.STANDARD);+ assertThat(described.s3Config().bucketAccessRoleArn())+ .isEqualTo("arn:aws:iam::000000000000:role/datasync");+ assertThat(described.creationTime()).isNotNull();+ }++ @Test+ @Order(3)+ void createNfsLocationReadsBackItsAgentAndMountOptions() {+ nfsLocationArn = datasync.createLocationNfs(r -> r+ .serverHostname("nfs-" + suffix + ".example.com")+ .subdirectory("/export/home")+ .onPremConfig(c -> c.agentArns(agentArn)))+ .locationArn();++ DescribeLocationNfsResponse described = datasync.describeLocationNfs(r -> r.locationArn(nfsLocationArn));++ assertThat(described.locationUri()).startsWith("nfs://nfs-" + suffix + ".example.com");+ assertThat(described.onPremConfig().agentArns()).containsExactly(agentArn);+ assertThat(described.mountOptions().versionAsString()).isEqualTo("AUTOMATIC");+ }++ @Test+ @Order(4)+ void basicTaskIsAvailableAndVerifiesPointInTime() {+ CreateTaskResponse created = datasync.createTask(r -> r+ .sourceLocationArn(nfsLocationArn)+ .destinationLocationArn(s3LocationArn)+ .name("sdk-test-task-" + suffix)+ .tags(TagListEntry.builder().key("env").value("test").build()));++ taskArn = created.taskArn();+ assertThat(taskArn).contains(":task/task-");++ DescribeTaskResponse described = datasync.describeTask(r -> r.taskArn(taskArn));++ assertThat(described.status()).isEqualTo(TaskStatus.AVAILABLE);+ assertThat(described.taskMode()).isEqualTo(TaskMode.BASIC);+ assertThat(described.sourceLocationArn()).isEqualTo(nfsLocationArn);+ assertThat(described.destinationLocationArn()).isEqualTo(s3LocationArn);+ assertThat(described.options().verifyMode()).isEqualTo(VerifyMode.POINT_IN_TIME_CONSISTENT);+ }++ @Test+ @Order(5)+ void enhancedTaskVerifiesOnlyFilesTransferred() {+ enhancedTaskArn = datasync.createTask(r -> r+ .sourceLocationArn(nfsLocationArn)+ .destinationLocationArn(s3LocationArn)+ .name("sdk-test-task-enhanced-" + suffix)+ .taskMode(TaskMode.ENHANCED))+ .taskArn();++ DescribeTaskResponse described = datasync.describeTask(r -> r.taskArn(enhancedTaskArn));++ assertThat(described.taskMode()).isEqualTo(TaskMode.ENHANCED);+ assertThat(described.options().verifyMode()).isEqualTo(VerifyMode.ONLY_FILES_TRANSFERRED);+ }++ @Test+ @Order(6)+ void enhancedTaskRejectsPointInTimeConsistentVerification() {+ assertThatThrownBy(() -> datasync.createTask(r -> r+ .sourceLocationArn(nfsLocationArn)+ .destinationLocationArn(s3LocationArn)+ .name("sdk-test-task-rejected-" + suffix)+ .taskMode(TaskMode.ENHANCED)+ .options(o -> o.verifyMode(VerifyMode.POINT_IN_TIME_CONSISTENT))))+ .isInstanceOf(InvalidRequestException.class);++ assertThatThrownBy(() -> datasync.updateTask(r -> r+ .taskArn(enhancedTaskArn)+ .options(o -> o.verifyMode(VerifyMode.POINT_IN_TIME_CONSISTENT))))+ .isInstanceOf(InvalidRequestException.class);+ }++ @Test+ @Order(7)+ void updateTaskKeepsTheModeItWasCreatedWith() {+ datasync.updateTask(r -> r+ .taskArn(enhancedTaskArn)+ .name("sdk-test-task-enhanced-renamed-" + suffix));++ DescribeTaskResponse described = datasync.describeTask(r -> r.taskArn(enhancedTaskArn));++ assertThat(described.name()).isEqualTo("sdk-test-task-enhanced-renamed-" + suffix);+ assertThat(described.taskMode()).isEqualTo(TaskMode.ENHANCED);+ assertThat(described.options().verifyMode()).isEqualTo(VerifyMode.ONLY_FILES_TRANSFERRED);+ }++ @Test+ @Order(8)+ void listTasksIncludesBothTasks() {+ assertThat(datasync.listTasks(r -> r.maxResults(100)).tasks())+ .extracting(t -> t.taskArn())+ .contains(taskArn, enhancedTaskArn);+ }++ @Test+ @Order(9)+ void tagRoundTripOnTheTask() {+ datasync.tagResource(r -> r+ .resourceArn(taskArn)+ .tags(TagListEntry.builder().key("owner").value("platform").build()));++ assertThat(datasync.listTagsForResource(r -> r.resourceArn(taskArn)).tags())+ .contains(TagListEntry.builder().key("env").value("test").build())+ .contains(TagListEntry.builder().key("owner").value("platform").build());++ datasync.untagResource(r -> r.resourceArn(taskArn).keys("env"));++ assertThat(datasync.listTagsForResource(r -> r.resourceArn(taskArn)).tags())+ .extracting(TagListEntry::key)+ .contains("owner")+ .doesNotContain("env");+ }++ @Test+ @Order(10)+ void deleteTaskThenDescribeIsAnInvalidRequest() {+ datasync.deleteTask(r -> r.taskArn(taskArn));+ String deleted = taskArn;+ taskArn = null;++ assertThatThrownBy(() -> datasync.describeTask(r -> r.taskArn(deleted)))+ .isInstanceOf(InvalidRequestException.class);+ }+}diff --git a/docs/services/datasync.md b/docs/services/datasync.mdnew file mode 100644index 0000000000..98501620f5--- /dev/null+++ b/docs/services/datasync.md@@ -0,0 +1,157 @@+# AWS DataSync++**Protocol:** JSON 1.1 (`X-Amz-Target: FmrsService.<Action>`)+**Endpoint:** `http://localhost:4566/` (SigV4 service `datasync`)++DataSync's Smithy service shape is `FmrsService`, so that is the target prefix the SDK and+the CLI send, not the service name. Floci emulates the management plane: agents, locations,+tasks and tagging. The task-execution data plane is not emulated.++## Supported Actions++<!-- floci:actions:start -->+| Action | Description |+| --- | --- |+| `CreateAgent` | Activate an agent; it reports `ONLINE` immediately |+| `DescribeAgent` | Read an agent's status, endpoint type and platform version |+| `UpdateAgent` | Rename an agent |+| `DeleteAgent` | Delete an agent |+| `ListAgents` | Page through agents, ordered by ARN |+| `CreateLocationAzureBlob` | Create an Azure Blob Storage location |+| `CreateLocationEfs` | Create an Amazon EFS location |+| `CreateLocationFsxLustre` | Create an FSx for Lustre location |+| `CreateLocationFsxOntap` | Create an FSx for NetApp ONTAP location |+| `CreateLocationFsxOpenZfs` | Create an FSx for OpenZFS location |+| `CreateLocationFsxWindows` | Create an FSx for Windows File Server location |+| `CreateLocationHdfs` | Create a Hadoop HDFS location |+| `CreateLocationNfs` | Create an NFS location |+| `CreateLocationObjectStorage` | Create a self-managed object storage location |+| `CreateLocationS3` | Create an Amazon S3 location |+| `CreateLocationSmb` | Create an SMB file server location |+| `DescribeLocationAzureBlob` | Read back an Azure Blob location's configuration |+| `DescribeLocationEfs` | Read back an EFS location's configuration |+| `DescribeLocationFsxLustre` | Read back an FSx for Lustre location's configuration |+| `DescribeLocationFsxOntap` | Read back an FSx for ONTAP location, including the derived `FsxFilesystemArn` |+| `DescribeLocationFsxOpenZfs` | Read back an FSx for OpenZFS location's configuration |+| `DescribeLocationFsxWindows` | Read back an FSx for Windows location's configuration |+| `DescribeLocationHdfs` | Read back an HDFS location's configuration |+| `DescribeLocationNfs` | Read back an NFS location's configuration |+| `DescribeLocationObjectStorage` | Read back an object storage location's configuration |+| `DescribeLocationS3` | Read back an S3 location's configuration |+| `DescribeLocationSmb` | Read back an SMB location's configuration |+| `UpdateLocationAzureBlob` | Change an Azure Blob location and rebuild its `LocationUri` |+| `UpdateLocationEfs` | Change an EFS location and rebuild its `LocationUri` |+| `UpdateLocationFsxLustre` | Change an FSx for Lustre location and rebuild its `LocationUri` |+| `UpdateLocationFsxOntap` | Change an FSx for ONTAP location and rebuild its `LocationUri` |+| `UpdateLocationFsxOpenZfs` | Change an FSx for OpenZFS location and rebuild its `LocationUri` |+| `UpdateLocationFsxWindows` | Change an FSx for Windows location and rebuild its `LocationUri` |+| `UpdateLocationHdfs` | Change an HDFS location and rebuild its `LocationUri` |+| `UpdateLocationNfs` | Change an NFS location and rebuild its `LocationUri` |+| `UpdateLocationObjectStorage` | Change an object storage location and rebuild its `LocationUri` |+| `UpdateLocationS3` | Change an S3 location and rebuild its `LocationUri` |+| `UpdateLocationSmb` | Change an SMB location and rebuild its `LocationUri` |+| `ListLocations` | Page through locations, optionally filtered by `LocationUri`, `LocationType` or `CreationTime` |+| `DeleteLocation` | Delete a location |+| `CreateTask` | Create a transfer task; it reports `AVAILABLE` immediately |+| `DescribeTask` | Read a task's status, locations, options and filters |+| `UpdateTask` | Change a task's name, options, filters, schedule or reporting |+| `DeleteTask` | Delete a task |+| `ListTasks` | Page through tasks, optionally filtered by `LocationId` or `CreationTime` |+| `TagResource` | Tag an agent, location or task by ARN |+| `UntagResource` | Remove tags from an agent, location or task by ARN |+| `ListTagsForResource` | List an agent's, location's or task's tags |+<!-- floci:actions:end -->++Agents report `ONLINE` and tasks report `AVAILABLE` on the first read after they are+created, so SDK and Terraform waiters, including the one behind `aws_datasync_task`,+complete on their first poll. Tags passed on any create are honoured and come back from+`ListTagsForResource`.++Each `DescribeLocation*` projects the create request that produced the location, so the+configuration you sent is the configuration you read back. `LocationUri` is derived from+the same request the way AWS documents it, per location type:++| Location type | `LocationUri` |+|---|---|+| S3 | `s3://<bucket>/<subdirectory>` |+| EFS | `efs://<region>.<file-system-id>/<subdirectory>` |+| FSx for Lustre | `fsxl://<region>.<file-system-id>/<subdirectory>` |+| FSx for OpenZFS | `fsxz://<region>.<file-system-id>/<subdirectory>` |+| FSx for Windows | `fsxw://<region>.<file-system-id>/<subdirectory>` |+| FSx for ONTAP | `fsxn://<region>.<file-system-id>.<svm-id>/<subdirectory>` |+| HDFS | `hdfs://<namenode-hostname>:<port>/<subdirectory>` |+| NFS | `nfs://<server-hostname>/<subdirectory>` |+| SMB | `smb://<server-hostname>/<subdirectory>` |+| Object storage | `object-storage://<server-hostname>/<bucket>/<subdirectory>` |+| Azure Blob | `azure-blob://<account-host>/<container>/<subdirectory>` |++DataSync's documented server-side defaults are applied on create and returned on describe:+`S3StorageClass` `STANDARD`, Azure Blob `BLOCK` and `HOT`, object storage `HTTPS` on port+443 (80 for `HTTP`), SMB `NTLM` authentication, NFS and SMB mount version `AUTOMATIC`, HDFS+block size 128 MiB with replication factor 3 and `PRIVACY` QOP, EFS `InTransitEncryption`+`NONE`, and the full `Options` block on a task. `DescribeLocationFsxOntap` derives+`FsxFilesystemArn` from the storage virtual machine ARN, as AWS does.++A task's `VerifyMode` default follows its `TaskMode`. A `BASIC` task, which is what+`CreateTask` assumes when the request omits `TaskMode`, defaults to+`POINT_IN_TIME_CONSISTENT`. An `ENHANCED` task defaults to `ONLY_FILES_TRANSFERRED` and+cannot use `POINT_IN_TIME_CONSISTENT` at all, so asking for that combination fails with+`InvalidRequestException`. `UpdateTaskRequest` carries no `TaskMode`, so a task keeps the+mode it was created with and `UpdateTask` applies the same rule against that stored mode.++## Not emulated++The task-execution data plane (`StartTaskExecution`, `DescribeTaskExecution`,+`ListTaskExecutions`, `CancelTaskExecution` and `UpdateTaskExecution`) transfers real bytes+between real storage systems, which floci has no way to model. These operations return+`UnknownOperationException` rather than a stub success, so a caller fails fast instead of+waiting on a transfer that will never progress.++Credential members are accepted on create but never stored or returned: `Password`,+`SecretKey`, Azure `SasConfiguration`, `KerberosKeytab` and `KerberosKrb5Conf`, plus the+nested `Protocol.SMB.Password` on the FSx protocol block. AWS also omits them from every+describe response. Because no secret is created, `ManagedSecretConfig` is not returned;+`CmkSecretConfig` and `CustomSecretConfig` are echoed as sent.++Agent ARNs referenced by a location (`AgentArns`, `OnPremConfig.AgentArns`) are not checked+against the agents floci knows about, so a location can point at an agent that was+provisioned elsewhere. Location ARNs on `CreateTask` are checked, and an unknown one fails+with `InvalidRequestException`.++DataSync models only `InvalidRequestException` and `InternalException`, so a missing+resource, a required member that was not sent, and a describe aimed at the wrong location+type all return `InvalidRequestException` with HTTP 400. There is no+`ResourceNotFoundException` in this API.++## Configuration++| Variable | Default | Description |+|---|---|---|+| `FLOCI_SERVICES_DATASYNC_ENABLED` | `true` | Enable or disable the service |++## Examples++```bash+export AWS_ENDPOINT_URL=http://localhost:4566++aws datasync create-agent --activation-key AAAAA-1AAAA-BB1CC-DDDDD-EEEEE --agent-name my-agent++aws datasync create-location-s3 \+ --s3-bucket-arn arn:aws:s3:::my-bucket \+ --subdirectory /backups \+ --s3-config BucketAccessRoleArn=arn:aws:iam::000000000000:role/datasync++aws datasync create-location-nfs \+ --server-hostname nfs.example.com \+ --subdirectory /export/home \+ --on-prem-config AgentArns=arn:aws:datasync:us-east-1:000000000000:agent/agent-...++aws datasync create-task \+ --source-location-arn arn:aws:datasync:us-east-1:000000000000:location/loc-... \+ --destination-location-arn arn:aws:datasync:us-east-1:000000000000:location/loc-... \+ --name my-task++aws datasync describe-task --task-arn arn:aws:datasync:us-east-1:000000000000:task/task-...++aws datasync list-locations+```diff --git a/docs/services/index.md b/docs/services/index.mdindex 1869eab0e6..ac898c1320 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -132,6 +132,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [Cost and Usage Reports](cur.md) | `POST /` + `X-Amz-Target: AWSOrigamiServiceGatewayService.*` | JSON 1.1 | 6 || [BCM Data Exports](bcm-data-exports.md) | `POST /` + `X-Amz-Target: AWSBillingAndCostManagementDataExports.*` | JSON 1.1 | 7 || [Transfer Family](transfer.md) | `POST /` + `X-Amz-Target: TransferService.*` | JSON 1.1 | 17 |+| [DataSync](datasync.md) | `POST /` + `X-Amz-Target: FmrsService.*` | JSON 1.1 | 48 || [IoT Core](iot.md) | `/things/...`, `/endpoint`, rules/policies REST paths | REST JSON | 62 || [IoT Data](iot.md) | `/things/{thingName}/shadow`, MQTT topics | REST JSON | 11 |diff --git a/mkdocs.yml b/mkdocs.ymlindex 12e875b556..f42f797d54 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -195,6 +195,7 @@ nav:- AWS IoT Core: services/iot.md- AppSync: services/appsync.md- Transfer Family: services/transfer.md+ - DataSync: services/datasync.md- AWS Config: services/config.md- EMR: services/emr.md- EMR Serverless: services/emr-serverless.mddiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex d169987b9d..1c1646d949 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -743,6 +743,7 @@ public interface EmulatorConfig {AppIntegrationsServiceConfig appintegrations();CognitoIdentityServiceConfig cognitoidentity();GlobalAcceleratorServiceConfig globalaccelerator();+ DataSyncServiceConfig datasync();ApsServiceConfig aps();@@ -772,6 +773,11 @@ public interface EmulatorConfig {boolean enabled();}+ interface DataSyncServiceConfig {+ @WithDefault("true")+ boolean enabled();+ }+interface SsoAdminServiceConfig {@WithDefault("true")boolean enabled();diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.javaindex a828d8d789..a8fb8afa5c 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java@@ -52,6 +52,7 @@ import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerJsonHandimport io.github.hectorvent.floci.services.organizations.OrganizationsJsonHandler;import io.github.hectorvent.floci.services.cognitoidentity.CognitoIdentityJsonHandler;import io.github.hectorvent.floci.services.globalaccelerator.GlobalAcceleratorJsonHandler;+import io.github.hectorvent.floci.services.datasync.DataSyncJsonHandler;import io.github.hectorvent.floci.services.route53resolver.Route53ResolverJsonHandler;import io.github.hectorvent.floci.services.networkfirewall.NetworkFirewallJsonHandler;import io.github.hectorvent.floci.services.servicecatalog.ServiceCatalogJsonHandler;@@ -125,6 +126,7 @@ public class AwsJson11Controller {private final Route53ResolverJsonHandler route53ResolverJsonHandler;private final CognitoIdentityJsonHandler cognitoIdentityJsonHandler;private final GlobalAcceleratorJsonHandler globalAcceleratorJsonHandler;+ private final DataSyncJsonHandler dataSyncJsonHandler;private final NetworkFirewallJsonHandler networkFirewallJsonHandler;private final ServiceCatalogJsonHandler serviceCatalogJsonHandler;private final CloudControlJsonHandler cloudControlJsonHandler;@@ -178,6 +180,7 @@ public class AwsJson11Controller {Route53ResolverJsonHandler route53ResolverJsonHandler,CognitoIdentityJsonHandler cognitoIdentityJsonHandler,GlobalAcceleratorJsonHandler globalAcceleratorJsonHandler,+ DataSyncJsonHandler dataSyncJsonHandler,NetworkFirewallJsonHandler networkFirewallJsonHandler,ServiceCatalogJsonHandler serviceCatalogJsonHandler,CloudControlJsonHandler cloudControlJsonHandler,@@ -235,6 +238,7 @@ public class AwsJson11Controller {this.route53ResolverJsonHandler = route53ResolverJsonHandler;this.cognitoIdentityJsonHandler = cognitoIdentityJsonHandler;this.globalAcceleratorJsonHandler = globalAcceleratorJsonHandler;+ this.dataSyncJsonHandler = dataSyncJsonHandler;this.networkFirewallJsonHandler = networkFirewallJsonHandler;this.serviceCatalogJsonHandler = serviceCatalogJsonHandler;this.cloudControlJsonHandler = cloudControlJsonHandler;@@ -327,6 +331,7 @@ public class AwsJson11Controller {// segment and are never partitioned by the region the caller signed with, so the// handler takes no region.case "globalaccelerator" -> globalAcceleratorJsonHandler.handle(action, request);+ case "datasync" -> dataSyncJsonHandler.handle(action, request, region);case "network-firewall" -> networkFirewallJsonHandler.handle(action, request, region, regionResolver.getAccountId());case "servicecatalog" -> serviceCatalogJsonHandler.handle(diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.javaindex 16d48caf80..9ef752f4b9 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java@@ -636,6 +636,12 @@ public class ResolvedServiceCatalog {"globalaccelerator", config.storage().mode(), 5000L, null, ServiceProtocol.JSON,protocols(ServiceProtocol.JSON),Set.of("GlobalAccelerator_V20180706."), Set.of("globalaccelerator"), Set.of(), Set.of()),+ // DataSync's Smithy service shape is FmrsService, so that is the target prefix+ // the SDK sends, not the service name.+ descriptor("datasync", "datasync", config.services().datasync().enabled(), true,+ "datasync", config.storage().mode(), 5000L, null, ServiceProtocol.JSON,+ protocols(ServiceProtocol.JSON),+ Set.of("FmrsService."), Set.of("datasync"), Set.of(), Set.of()),descriptor("network-firewall", "networkfirewall", config.services().networkfirewall().enabled(), true,"networkfirewall", config.storage().mode(), 5000L, null, ServiceProtocol.JSON,protocols(ServiceProtocol.JSON),diff --git a/src/main/java/io/github/hectorvent/floci/services/datasync/DataSyncJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/datasync/DataSyncJsonHandler.javanew file mode 100644index 0000000000..5f7f7f623e--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/datasync/DataSyncJsonHandler.java@@ -0,0 +1,265 @@+package io.github.hectorvent.floci.services.datasync;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.datasync.model.DataSyncAgent;+import io.github.hectorvent.floci.services.datasync.model.DataSyncLocation;+import io.github.hectorvent.floci.services.datasync.model.DataSyncLocationType;+import io.github.hectorvent.floci.services.datasync.model.DataSyncTask;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import jakarta.ws.rs.core.Response;++import java.time.Instant;+import java.util.List;+import java.util.Map;++/**+ * Translates the DataSync JSON wire protocol ({@code X-Amz-Target: FmrsService.<Action>})+ * to and from {@link DataSyncService}.+ *+ * <p>The task-execution data plane ({@code StartTaskExecution}, {@code DescribeTaskExecution},+ * {@code ListTaskExecutions}, {@code CancelTaskExecution}, {@code UpdateTaskExecution}) moves+ * real bytes between real storage systems and is not emulated. Those operations fall through+ * to a clean {@code UnknownOperationException} rather than a stub success, so callers fail+ * fast instead of stranding a waiter on a transfer that will never progress.+ *+ * <p>Every location action is spelled out as its own switch label rather than matched by+ * prefix: the action tables in {@code docs/services/datasync.md} are generated from these+ * labels, and a prefix match would document none of them.+ */+@ApplicationScoped+public class DataSyncJsonHandler {++ private static final String CREATE_LOCATION = "CreateLocation";+ private static final String DESCRIBE_LOCATION = "DescribeLocation";+ private static final String UPDATE_LOCATION = "UpdateLocation";++ private final DataSyncService dataSyncService;+ private final ObjectMapper mapper;++ @Inject+ public DataSyncJsonHandler(DataSyncService dataSyncService, ObjectMapper mapper) {+ this.dataSyncService = dataSyncService;+ this.mapper = mapper;+ }++ public Response handle(String action, JsonNode request, String region) {+ return switch (action) {+ case "CreateAgent" -> {+ DataSyncAgent agent = dataSyncService.createAgent(request, region);+ yield ok(mapper.createObjectNode().put("AgentArn", agent.getAgentArn()));+ }+ case "DescribeAgent" -> ok(describeAgent(dataSyncService.getAgent(text(request, "AgentArn"))));+ case "UpdateAgent" -> {+ dataSyncService.updateAgent(text(request, "AgentArn"), text(request, "Name"));+ yield ok(mapper.createObjectNode());+ }+ case "DeleteAgent" -> {+ dataSyncService.deleteAgent(text(request, "AgentArn"));+ yield ok(mapper.createObjectNode());+ }+ case "ListAgents" -> {+ var page = dataSyncService.listAgents(text(request, "NextToken"),+ request.path("MaxResults").asInt(0));+ ObjectNode response = mapper.createObjectNode();+ ArrayNode entries = response.putArray("Agents");+ for (DataSyncAgent agent : page.items()) {+ ObjectNode entry = entries.addObject();+ entry.put("AgentArn", agent.getAgentArn());+ entry.put("Name", agent.getName() != null ? agent.getName() : "");+ entry.put("Status", agent.getStatus());+ entry.putObject("Platform").put("Version", agent.getPlatformVersion());+ }+ putNextToken(response, page.nextToken());+ yield ok(response);+ }++ case "CreateLocationAzureBlob", "CreateLocationEfs", "CreateLocationFsxLustre",+ "CreateLocationFsxOntap", "CreateLocationFsxOpenZfs", "CreateLocationFsxWindows",+ "CreateLocationHdfs", "CreateLocationNfs", "CreateLocationObjectStorage",+ "CreateLocationS3", "CreateLocationSmb" -> {+ DataSyncLocation location = dataSyncService.createLocation(+ locationType(action, CREATE_LOCATION), request, region);+ yield ok(mapper.createObjectNode().put("LocationArn", location.getLocationArn()));+ }+ case "DescribeLocationAzureBlob", "DescribeLocationEfs", "DescribeLocationFsxLustre",+ "DescribeLocationFsxOntap", "DescribeLocationFsxOpenZfs", "DescribeLocationFsxWindows",+ "DescribeLocationHdfs", "DescribeLocationNfs", "DescribeLocationObjectStorage",+ "DescribeLocationS3", "DescribeLocationSmb" -> ok(describeLocation(dataSyncService.getLocation(+ text(request, "LocationArn"), locationType(action, DESCRIBE_LOCATION))));+ case "UpdateLocationAzureBlob", "UpdateLocationEfs", "UpdateLocationFsxLustre",+ "UpdateLocationFsxOntap", "UpdateLocationFsxOpenZfs", "UpdateLocationFsxWindows",+ "UpdateLocationHdfs", "UpdateLocationNfs", "UpdateLocationObjectStorage",+ "UpdateLocationS3", "UpdateLocationSmb" -> {+ dataSyncService.updateLocation(locationType(action, UPDATE_LOCATION), request);+ yield ok(mapper.createObjectNode());+ }+ case "ListLocations" -> {+ var page = dataSyncService.listLocations(request.get("Filters"),+ text(request, "NextToken"), request.path("MaxResults").asInt(0));+ ObjectNode response = mapper.createObjectNode();+ ArrayNode entries = response.putArray("Locations");+ for (DataSyncLocation location : page.items()) {+ ObjectNode entry = entries.addObject();+ entry.put("LocationArn", location.getLocationArn());+ entry.put("LocationUri", location.getLocationUri());+ }+ putNextToken(response, page.nextToken());+ yield ok(response);+ }+ case "DeleteLocation" -> {+ dataSyncService.deleteLocation(text(request, "LocationArn"));+ yield ok(mapper.createObjectNode());+ }++ case "CreateTask" -> {+ DataSyncTask task = dataSyncService.createTask(request, region);+ yield ok(mapper.createObjectNode().put("TaskArn", task.getTaskArn()));+ }+ case "DescribeTask" -> ok(describeTask(dataSyncService.getTask(text(request, "TaskArn"))));+ case "UpdateTask" -> {+ dataSyncService.updateTask(request);+ yield ok(mapper.createObjectNode());+ }+ case "DeleteTask" -> {+ dataSyncService.deleteTask(text(request, "TaskArn"));+ yield ok(mapper.createObjectNode());+ }+ case "ListTasks" -> {+ var page = dataSyncService.listTasks(request.get("Filters"),+ text(request, "NextToken"), request.path("MaxResults").asInt(0));+ ObjectNode response = mapper.createObjectNode();+ ArrayNode entries = response.putArray("Tasks");+ for (DataSyncTask task : page.items()) {+ ObjectNode entry = entries.addObject();+ entry.put("TaskArn", task.getTaskArn());+ entry.put("Status", task.getStatus());+ entry.put("Name", task.getName() != null ? task.getName() : "");+ entry.put("TaskMode", task.getTaskMode());+ }+ putNextToken(response, page.nextToken());+ yield ok(response);+ }++ case "TagResource" -> {+ dataSyncService.tagResource(text(request, "ResourceArn"),+ DataSyncService.tagsOf(request.path("Tags")));+ yield ok(mapper.createObjectNode());+ }+ case "UntagResource" -> {+ dataSyncService.untagResource(text(request, "ResourceArn"),+ DataSyncService.stringList(request.path("Keys")));+ yield ok(mapper.createObjectNode());+ }+ case "ListTagsForResource" -> {+ Map<String, String> tags = dataSyncService.listTagsForResource(text(request, "ResourceArn"));+ ObjectNode response = mapper.createObjectNode();+ ArrayNode entries = response.putArray("Tags");+ tags.forEach((key, value) -> entries.addObject().put("Key", key).put("Value", value));+ yield ok(response);+ }++ default -> throw new AwsException("UnknownOperationException",+ "Operation " + action + " is not supported by floci", 400);+ };+ }++ private static DataSyncLocationType locationType(String action, String operationPrefix) {+ String suffix = action.substring(operationPrefix.length());+ return DataSyncLocationType.fromOperationSuffix(suffix)+ .orElseThrow(() -> new AwsException("UnknownOperationException",+ "Operation " + action + " is not supported by floci", 400));+ }++ private ObjectNode describeAgent(DataSyncAgent agent) {+ ObjectNode response = mapper.createObjectNode();+ response.put("AgentArn", agent.getAgentArn());+ response.put("Name", agent.getName() != null ? agent.getName() : "");+ response.put("Status", agent.getStatus());+ response.put("CreationTime", epochSeconds(agent.getCreationTime()));+ response.put("LastConnectionTime", epochSeconds(agent.getLastConnectionTime()));+ response.put("EndpointType", agent.getEndpointType());+ response.putObject("Platform").put("Version", agent.getPlatformVersion());+ if (agent.getVpcEndpointId() != null) {+ ObjectNode privateLink = response.putObject("PrivateLinkConfig");+ privateLink.put("VpcEndpointId", agent.getVpcEndpointId());+ putStrings(privateLink, "SubnetArns", agent.getSubnetArns());+ putStrings(privateLink, "SecurityGroupArns", agent.getSecurityGroupArns());+ }+ return response;+ }++ private ObjectNode describeLocation(DataSyncLocation location) {+ ObjectNode response = mapper.createObjectNode();+ response.put("LocationArn", location.getLocationArn());+ response.put("LocationUri", location.getLocationUri());+ JsonNode configuration = location.getConfiguration();+ for (String member : location.getLocationType().describeMembers()) {+ JsonNode value = configuration.get(member);+ if (value != null && !value.isNull()) {+ response.set(member, value.deepCopy());+ }+ }+ response.put("CreationTime", epochSeconds(location.getCreationTime()));+ return response;+ }++ private ObjectNode describeTask(DataSyncTask task) {+ ObjectNode response = mapper.createObjectNode();+ response.put("TaskArn", task.getTaskArn());+ response.put("Status", task.getStatus());+ response.put("Name", task.getName() != null ? task.getName() : "");+ response.put("TaskMode", task.getTaskMode());+ response.put("SourceLocationArn", task.getSourceLocationArn());+ response.put("DestinationLocationArn", task.getDestinationLocationArn());+ if (task.getCloudWatchLogGroupArn() != null) {+ response.put("CloudWatchLogGroupArn", task.getCloudWatchLogGroupArn());+ }+ response.putArray("SourceNetworkInterfaceArns");+ response.putArray("DestinationNetworkInterfaceArns");+ response.set("Options", task.getOptions() != null+ ? task.getOptions().deepCopy() : mapper.createObjectNode());+ response.set("Excludes", task.getExcludes() != null+ ? task.getExcludes().deepCopy() : mapper.createArrayNode());+ response.set("Includes", task.getIncludes() != null+ ? task.getIncludes().deepCopy() : mapper.createArrayNode());+ if (task.getSchedule() != null) {+ response.set("Schedule", task.getSchedule().deepCopy());+ }+ if (task.getManifestConfig() != null) {+ response.set("ManifestConfig", task.getManifestConfig().deepCopy());+ }+ if (task.getTaskReportConfig() != null) {+ response.set("TaskReportConfig", task.getTaskReportConfig().deepCopy());+ }+ response.put("CreationTime", epochSeconds(task.getCreationTime()));+ return response;+ }++ private static void putStrings(ObjectNode parent, String member, List<String> values) {+ ArrayNode array = parent.putArray(member);+ values.forEach(array::add);+ }++ private static void putNextToken(ObjectNode response, String nextToken) {+ if (nextToken != null) {+ response.put("NextToken", nextToken);+ }+ }++ private static double epochSeconds(Instant instant) {+ return (instant != null ? instant : Instant.EPOCH).toEpochMilli() / 1000.0;+ }++ private static String text(JsonNode request, String member) {+ return request.hasNonNull(member) ? request.get(member).asText() : null;+ }++ private static Response ok(ObjectNode body) {+ return Response.ok(body).build();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/datasync/DataSyncService.java b/src/main/java/io/github/hectorvent/floci/services/datasync/DataSyncService.javanew file mode 100644index 0000000000..1b082b1ae1--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/datasync/DataSyncService.java@@ -0,0 +1,739 @@+package io.github.hectorvent.floci.services.datasync;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.datasync.model.DataSyncAgent;+import io.github.hectorvent.floci.services.datasync.model.DataSyncLocation;+import io.github.hectorvent.floci.services.datasync.model.DataSyncLocationType;+import io.github.hectorvent.floci.services.datasync.model.DataSyncTaggable;+import io.github.hectorvent.floci.services.datasync.model.DataSyncTask;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.security.SecureRandom;+import java.time.Instant;+import java.time.format.DateTimeParseException;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Set;+import java.util.function.Function;++/**+ * AWS DataSync management plane.+ *+ * <p>Agents report {@code ONLINE} and tasks {@code AVAILABLE} from the moment they are+ * created, so SDK and Terraform waiters finish on their first poll. Every+ * {@code DescribeLocation*} answer is projected out of the create request that produced+ * the location, with the location's {@code LocationUri} derived from the same request.+ * Credential members ({@code Password}, {@code SecretKey}, SAS tokens, Kerberos files)+ * are dropped before the configuration is stored, matching AWS, which never reads them+ * back.+ */+@ApplicationScoped+public class DataSyncService {++ private static final Logger LOG = Logger.getLogger(DataSyncService.class);++ private static final String HEX_ALPHABET = "0123456789abcdef";+ private static final int RESOURCE_ID_LENGTH = 17;+ private static final SecureRandom RANDOM = new SecureRandom();++ public static final String AGENT_STATUS_ONLINE = "ONLINE";+ public static final String TASK_STATUS_AVAILABLE = "AVAILABLE";+ public static final String DEFAULT_TASK_MODE = "BASIC";+ public static final String TASK_MODE_ENHANCED = "ENHANCED";+ public static final String VERIFY_MODE_POINT_IN_TIME_CONSISTENT = "POINT_IN_TIME_CONSISTENT";+ public static final String VERIFY_MODE_ONLY_FILES_TRANSFERRED = "ONLY_FILES_TRANSFERRED";++ private static final Set<String> TASK_MODES = Set.of(DEFAULT_TASK_MODE, TASK_MODE_ENHANCED);++ /** Reported by DescribeAgent and ListAgents; floci runs no agent software. */+ public static final String AGENT_PLATFORM_VERSION = "1.0.0";++ private static final int DEFAULT_MAX_RESULTS = 100;++ private static final Set<String> CREDENTIAL_MEMBERS = Set.of(+ "Password", "SecretKey", "SasConfiguration", "KerberosKeytab", "KerberosKrb5Conf");++ private final StorageBackend<String, DataSyncAgent> agents;+ private final StorageBackend<String, DataSyncLocation> locations;+ private final StorageBackend<String, DataSyncTask> tasks;+ private final RegionResolver regionResolver;+ private final ObjectMapper mapper;++ @Inject+ public DataSyncService(StorageFactory storageFactory, RegionResolver regionResolver, ObjectMapper mapper) {+ this.agents = storageFactory.create("datasync", "datasync-agents.json",+ new TypeReference<Map<String, DataSyncAgent>>() {});+ this.locations = storageFactory.create("datasync", "datasync-locations.json",+ new TypeReference<Map<String, DataSyncLocation>>() {});+ this.tasks = storageFactory.create("datasync", "datasync-tasks.json",+ new TypeReference<Map<String, DataSyncTask>>() {});+ this.regionResolver = regionResolver;+ this.mapper = mapper;+ }++ /** A page of list results plus the token that resumes after it, or {@code null} at the end. */+ public record Page<T>(List<T> items, String nextToken) {+ }++ // ---- Agents ----++ public DataSyncAgent createAgent(JsonNode request, String region) {+ String activationKey = requiredText(request, "ActivationKey", "CreateAgent");+ DataSyncAgent agent = new DataSyncAgent();+ agent.setAgentArn(regionResolver.buildArn("datasync", region, "agent/agent-" + randomResourceId()));+ agent.setName(request.path("AgentName").asText(""));+ agent.setStatus(AGENT_STATUS_ONLINE);+ agent.setActivationKey(activationKey);+ agent.setVpcEndpointId(optionalText(request, "VpcEndpointId"));+ agent.setSubnetArns(stringList(request.path("SubnetArns")));+ agent.setSecurityGroupArns(stringList(request.path("SecurityGroupArns")));+ agent.setEndpointType(agent.getVpcEndpointId() != null ? "PRIVATE_LINK" : "PUBLIC");+ agent.setPlatformVersion(AGENT_PLATFORM_VERSION);+ agent.setCreationTime(Instant.now());+ agent.setLastConnectionTime(agent.getCreationTime());+ agent.setTags(tagsOf(request.path("Tags")));++ agents.put(agent.getAgentArn(), agent);+ LOG.infov("Created DataSync agent: {0}", agent.getAgentArn());+ return agent;+ }++ public DataSyncAgent getAgent(String agentArn) {+ requireArn(agentArn, "AgentArn");+ return agents.get(agentArn).orElseThrow(() -> new AwsException("InvalidRequestException",+ "Agent " + agentArn + " is not found.", 400));+ }++ public DataSyncAgent updateAgent(String agentArn, String name) {+ DataSyncAgent agent = getAgent(agentArn);+ if (name != null) {+ agent.setName(name);+ }+ agents.put(agentArn, agent);+ return agent;+ }++ public void deleteAgent(String agentArn) {+ getAgent(agentArn);+ agents.delete(agentArn);+ LOG.infov("Deleted DataSync agent: {0}", agentArn);+ }++ public Page<DataSyncAgent> listAgents(String nextToken, int maxResults) {+ List<DataSyncAgent> all = agents.scan(key -> true);+ all.sort(Comparator.comparing(DataSyncAgent::getAgentArn));+ return paginate(all, DataSyncAgent::getAgentArn, nextToken, maxResults);+ }++ // ---- Locations ----++ public DataSyncLocation createLocation(DataSyncLocationType type, JsonNode request, String region) {+ for (String member : type.requiredMembers()) {+ if (!hasValue(request, member)) {+ throw new AwsException("InvalidRequestException",+ member + " is required for CreateLocation" + type.operationSuffix() + ".", 400);+ }+ }++ ObjectNode configuration = mapper.createObjectNode();+ request.properties().forEach(member -> {+ if (!"Tags".equals(member.getKey())) {+ configuration.set(member.getKey(), member.getValue().deepCopy());+ }+ });+ stripCredentials(configuration);+ applyLocationDefaults(type, configuration, region);++ DataSyncLocation location = new DataSyncLocation();+ location.setLocationArn(regionResolver.buildArn("datasync", region, "location/loc-" + randomResourceId()));+ location.setLocationType(type);+ location.setRegion(region);+ location.setConfiguration(configuration);+ location.setLocationUri(buildLocationUri(type, configuration, region));+ location.setCreationTime(Instant.now());+ location.setTags(tagsOf(request.path("Tags")));++ locations.put(location.getLocationArn(), location);+ LOG.infov("Created DataSync {0} location: {1}", type.operationSuffix(), location.getLocationArn());+ return location;+ }++ public DataSyncLocation getLocation(String locationArn) {+ requireArn(locationArn, "LocationArn");+ return locations.get(locationArn).orElseThrow(() -> new AwsException("InvalidRequestException",+ "Location " + locationArn + " is not found.", 400));+ }++ public DataSyncLocation getLocation(String locationArn, DataSyncLocationType expected) {+ DataSyncLocation location = getLocation(locationArn);+ if (location.getLocationType() != expected) {+ throw new AwsException("InvalidRequestException",+ "Location " + locationArn + " is a " + location.getLocationType().operationSuffix()+ + " location, not a " + expected.operationSuffix() + " location.", 400);+ }+ return location;+ }++ public DataSyncLocation updateLocation(DataSyncLocationType type, JsonNode request) {+ String locationArn = requiredText(request, "LocationArn", "UpdateLocation" + type.operationSuffix());+ DataSyncLocation location = getLocation(locationArn, type);++ ObjectNode configuration = location.getConfiguration().deepCopy();+ request.properties().forEach(member -> {+ if (!"LocationArn".equals(member.getKey())) {+ configuration.set(member.getKey(), member.getValue().deepCopy());+ }+ });+ stripCredentials(configuration);+ applyLocationDefaults(type, configuration, location.getRegion());++ location.setConfiguration(configuration);+ location.setLocationUri(buildLocationUri(type, configuration, location.getRegion()));+ locations.put(locationArn, location);+ LOG.infov("Updated DataSync {0} location: {1}", type.operationSuffix(), locationArn);+ return location;+ }++ public void deleteLocation(String locationArn) {+ getLocation(locationArn);+ locations.delete(locationArn);+ LOG.infov("Deleted DataSync location: {0}", locationArn);+ }++ public Page<DataSyncLocation> listLocations(JsonNode filters, String nextToken, int maxResults) {+ List<DataSyncLocation> all = new ArrayList<>();+ for (DataSyncLocation location : locations.scan(key -> true)) {+ if (matchesLocationFilters(location, filters)) {+ all.add(location);+ }+ }+ all.sort(Comparator.comparing(DataSyncLocation::getLocationArn));+ return paginate(all, DataSyncLocation::getLocationArn, nextToken, maxResults);+ }++ // ---- Tasks ----++ public DataSyncTask createTask(JsonNode request, String region) {+ String sourceArn = requiredText(request, "SourceLocationArn", "CreateTask");+ String destinationArn = requiredText(request, "DestinationLocationArn", "CreateTask");+ getLocation(sourceArn);+ getLocation(destinationArn);++ DataSyncTask task = new DataSyncTask();+ task.setTaskArn(regionResolver.buildArn("datasync", region, "task/task-" + randomResourceId()));+ task.setName(request.path("Name").asText(""));+ task.setStatus(TASK_STATUS_AVAILABLE);+ String taskMode = requestedTaskMode(request);+ task.setTaskMode(taskMode);+ task.setSourceLocationArn(sourceArn);+ task.setDestinationLocationArn(destinationArn);+ task.setCloudWatchLogGroupArn(optionalText(request, "CloudWatchLogGroupArn"));+ task.setOptions(mergedOptions(request.get("Options"), taskMode));+ task.setExcludes(filterList(request.get("Excludes")));+ task.setIncludes(filterList(request.get("Includes")));+ task.setSchedule(copyOrNull(request.get("Schedule")));+ task.setManifestConfig(copyOrNull(request.get("ManifestConfig")));+ task.setTaskReportConfig(copyOrNull(request.get("TaskReportConfig")));+ task.setCreationTime(Instant.now());+ task.setTags(tagsOf(request.path("Tags")));++ tasks.put(task.getTaskArn(), task);+ LOG.infov("Created DataSync task: {0}", task.getTaskArn());+ return task;+ }++ public DataSyncTask getTask(String taskArn) {+ requireArn(taskArn, "TaskArn");+ return tasks.get(taskArn).orElseThrow(() -> new AwsException("InvalidRequestException",+ "Task " + taskArn + " is not found.", 400));+ }++ public DataSyncTask updateTask(JsonNode request) {+ String taskArn = requiredText(request, "TaskArn", "UpdateTask");+ DataSyncTask task = getTask(taskArn);++ if (request.hasNonNull("Name")) {+ task.setName(request.get("Name").asText());+ }+ if (request.hasNonNull("CloudWatchLogGroupArn")) {+ task.setCloudWatchLogGroupArn(request.get("CloudWatchLogGroupArn").asText());+ }+ if (request.hasNonNull("Options")) {+ task.setOptions(mergedOptions(request.get("Options"), taskModeOf(task)));+ }+ if (request.hasNonNull("Excludes")) {+ task.setExcludes(filterList(request.get("Excludes")));+ }+ if (request.hasNonNull("Includes")) {+ task.setIncludes(filterList(request.get("Includes")));+ }+ if (request.hasNonNull("Schedule")) {+ task.setSchedule(copyOrNull(request.get("Schedule")));+ }+ if (request.hasNonNull("ManifestConfig")) {+ task.setManifestConfig(copyOrNull(request.get("ManifestConfig")));+ }+ if (request.hasNonNull("TaskReportConfig")) {+ task.setTaskReportConfig(copyOrNull(request.get("TaskReportConfig")));+ }++ tasks.put(taskArn, task);+ LOG.infov("Updated DataSync task: {0}", taskArn);+ return task;+ }++ public void deleteTask(String taskArn) {+ getTask(taskArn);+ tasks.delete(taskArn);+ LOG.infov("Deleted DataSync task: {0}", taskArn);+ }++ public Page<DataSyncTask> listTasks(JsonNode filters, String nextToken, int maxResults) {+ List<DataSyncTask> all = new ArrayList<>();+ for (DataSyncTask task : tasks.scan(key -> true)) {+ if (matchesTaskFilters(task, filters)) {+ all.add(task);+ }+ }+ all.sort(Comparator.comparing(DataSyncTask::getTaskArn));+ return paginate(all, DataSyncTask::getTaskArn, nextToken, maxResults);+ }++ // ---- Tags ----++ public Map<String, String> listTagsForResource(String resourceArn) {+ return new LinkedHashMap<>(findTaggable(resourceArn).getTags());+ }++ public void tagResource(String resourceArn, Map<String, String> tags) {+ DataSyncTaggable resource = findTaggable(resourceArn);+ resource.getTags().putAll(tags);+ persist(resourceArn, resource);+ }++ public void untagResource(String resourceArn, List<String> tagKeys) {+ DataSyncTaggable resource = findTaggable(resourceArn);+ tagKeys.forEach(resource.getTags()::remove);+ persist(resourceArn, resource);+ }++ private DataSyncTaggable findTaggable(String resourceArn) {+ requireArn(resourceArn, "ResourceArn");+ return agents.get(resourceArn).<DataSyncTaggable>map(agent -> agent)+ .or(() -> locations.get(resourceArn).map(location -> location))+ .or(() -> tasks.get(resourceArn).map(task -> task))+ .orElseThrow(() -> new AwsException("InvalidRequestException",+ "Resource " + resourceArn + " is not found.", 400));+ }++ private void persist(String resourceArn, DataSyncTaggable resource) {+ if (resource instanceof DataSyncAgent agent) {+ agents.put(resourceArn, agent);+ } else if (resource instanceof DataSyncLocation location) {+ locations.put(resourceArn, location);+ } else {+ tasks.put(resourceArn, (DataSyncTask) resource);+ }+ }++ // ---- Location derivation ----++ private void applyLocationDefaults(DataSyncLocationType type, ObjectNode configuration, String region) {+ switch (type) {+ case AZURE_BLOB -> {+ defaultText(configuration, "BlobType", "BLOCK");+ defaultText(configuration, "AccessTier", "HOT");+ }+ case EFS -> defaultText(configuration, "InTransitEncryption", "NONE");+ case FSX_ONTAP -> {+ String svmArn = configuration.path("StorageVirtualMachineArn").asText("");+ String fileSystemId = fsxFileSystemId(svmArn);+ if (!fileSystemId.isEmpty()) {+ configuration.put("FsxFilesystemArn", "arn:aws:fsx:" + arnRegion(svmArn, region) + ":"+ + arnAccountId(svmArn) + ":file-system/" + fileSystemId);+ }+ }+ case HDFS -> {+ if (!configuration.hasNonNull("BlockSize")) {+ configuration.put("BlockSize", 134217728);+ }+ if (!configuration.hasNonNull("ReplicationFactor")) {+ configuration.put("ReplicationFactor", 3);+ }+ ObjectNode qop = objectMember(configuration, "QopConfiguration");+ defaultText(qop, "RpcProtection", "PRIVACY");+ defaultText(qop, "DataTransferProtection", "PRIVACY");+ }+ case NFS -> defaultMountOptions(configuration);+ case OBJECT_STORAGE -> {+ defaultText(configuration, "ServerProtocol", "HTTPS");+ if (!configuration.hasNonNull("ServerPort")) {+ configuration.put("ServerPort",+ "HTTP".equals(configuration.path("ServerProtocol").asText()) ? 80 : 443);+ }+ }+ case S3 -> defaultText(configuration, "S3StorageClass", "STANDARD");+ case SMB -> {+ defaultText(configuration, "AuthenticationType", "NTLM");+ defaultMountOptions(configuration);+ }+ default -> {+ // FSx Lustre, FSx Windows and FSx OpenZFS carry no server-side defaults.+ }+ }+ }++ private void defaultMountOptions(ObjectNode configuration) {+ defaultText(objectMember(configuration, "MountOptions"), "Version", "AUTOMATIC");+ }++ private static ObjectNode objectMember(ObjectNode parent, String member) {+ return parent.get(member) instanceof ObjectNode existing ? existing : parent.putObject(member);+ }++ private static void defaultText(ObjectNode node, String member, String value) {+ if (!node.hasNonNull(member) || node.get(member).asText().isEmpty()) {+ node.put(member, value);+ }+ }++ private void stripCredentials(ObjectNode configuration) {+ CREDENTIAL_MEMBERS.forEach(configuration::remove);+ JsonNode protocol = configuration.get("Protocol");+ if (protocol instanceof ObjectNode protocolNode && protocolNode.get("SMB") instanceof ObjectNode smb) {+ smb.remove("Password");+ }+ }++ String buildLocationUri(DataSyncLocationType type, JsonNode configuration, String region) {+ return switch (type) {+ case AZURE_BLOB -> {+ String authority = configuration.path("ContainerUrl").asText("")+ .replaceFirst("^https?://", "");+ yield "azure-blob://" + trimTrailingSlashes(authority) + subdirectory(configuration);+ }+ case EFS -> fileSystemUri(type, configuration.path("EfsFilesystemArn").asText(""), region, configuration);+ case FSX_LUSTRE, FSX_OPEN_ZFS, FSX_WINDOWS ->+ fileSystemUri(type, configuration.path("FsxFilesystemArn").asText(""), region, configuration);+ case FSX_ONTAP -> {+ String svmArn = configuration.path("StorageVirtualMachineArn").asText("");+ yield "fsxn://" + arnRegion(svmArn, region) + "." + fsxFileSystemId(svmArn) + "."+ + fsxStorageVirtualMachineId(svmArn) + subdirectory(configuration);+ }+ case HDFS -> {+ JsonNode nameNode = configuration.path("NameNodes").path(0);+ int port = nameNode.path("Port").asInt(0);+ yield "hdfs://" + nameNode.path("Hostname").asText("")+ + (port > 0 ? ":" + port : "") + subdirectory(configuration);+ }+ case NFS -> "nfs://" + configuration.path("ServerHostname").asText("") + subdirectory(configuration);+ case SMB -> "smb://" + configuration.path("ServerHostname").asText("") + subdirectory(configuration);+ case OBJECT_STORAGE -> "object-storage://" + configuration.path("ServerHostname").asText("")+ + "/" + configuration.path("BucketName").asText("") + subdirectory(configuration);+ case S3 -> {+ String subdirectory = subdirectory(configuration);+ yield "s3://" + s3BucketName(configuration.path("S3BucketArn").asText(""))+ + ("/".equals(subdirectory) ? "" : subdirectory);+ }+ };+ }++ private String fileSystemUri(DataSyncLocationType type, String fileSystemArn, String region,+ JsonNode configuration) {+ String fileSystemId = arnResource(fileSystemArn);+ int slash = fileSystemId.lastIndexOf('/');+ if (slash >= 0) {+ fileSystemId = fileSystemId.substring(slash + 1);+ }+ return type.uriScheme() + "://" + arnRegion(fileSystemArn, region) + "." + fileSystemId+ + subdirectory(configuration);+ }++ private static String subdirectory(JsonNode configuration) {+ String subdirectory = configuration.path("Subdirectory").asText("").trim();+ while (subdirectory.endsWith("/")) {+ subdirectory = subdirectory.substring(0, subdirectory.length() - 1);+ }+ if (subdirectory.isEmpty()) {+ return "/";+ }+ return subdirectory.startsWith("/") ? subdirectory : "/" + subdirectory;+ }++ private static String trimTrailingSlashes(String value) {+ String trimmed = value;+ while (trimmed.endsWith("/")) {+ trimmed = trimmed.substring(0, trimmed.length() - 1);+ }+ return trimmed;+ }++ private static String s3BucketName(String bucketArn) {+ String resource = arnResource(bucketArn);+ return resource.isEmpty() ? bucketArn : resource;+ }++ private static String fsxFileSystemId(String storageVirtualMachineArn) {+ String[] segments = arnResource(storageVirtualMachineArn).split("/");+ return segments.length > 1 ? segments[1] : "";+ }++ private static String fsxStorageVirtualMachineId(String storageVirtualMachineArn) {+ String[] segments = arnResource(storageVirtualMachineArn).split("/");+ return segments.length > 2 ? segments[2] : "";+ }++ private static String arnRegion(String arn, String fallback) {+ String[] segments = arn.split(":", 6);+ return segments.length > 3 && !segments[3].isEmpty() ? segments[3] : fallback;+ }++ private String arnAccountId(String arn) {+ String[] segments = arn.split(":", 6);+ return segments.length > 4 && !segments[4].isEmpty() ? segments[4] : regionResolver.getAccountId();+ }++ private static String arnResource(String arn) {+ String[] segments = arn.split(":", 6);+ return segments.length > 5 ? segments[5] : "";+ }++ // ---- Filters ----++ private boolean matchesLocationFilters(DataSyncLocation location, JsonNode filters) {+ if (filters == null || !filters.isArray()) {+ return true;+ }+ for (JsonNode filter : filters) {+ String name = filter.path("Name").asText("");+ List<String> values = stringList(filter.path("Values"));+ String operator = filter.path("Operator").asText("Equals");+ boolean matched = switch (name) {+ case "LocationUri" -> matchesString(location.getLocationUri(), values, operator);+ case "LocationType" -> matchesString(location.getLocationType().operationSuffix(), values, operator)+ || matchesString(location.getLocationType().uriScheme(), values, operator);+ case "CreationTime" -> matchesInstant(location.getCreationTime(), values, operator);+ default -> throw new AwsException("InvalidRequestException",+ "Unsupported ListLocations filter name: " + name, 400);+ };+ if (!matched) {+ return false;+ }+ }+ return true;+ }++ private boolean matchesTaskFilters(DataSyncTask task, JsonNode filters) {+ if (filters == null || !filters.isArray()) {+ return true;+ }+ for (JsonNode filter : filters) {+ String name = filter.path("Name").asText("");+ List<String> values = stringList(filter.path("Values"));+ String operator = filter.path("Operator").asText("Equals");+ boolean matched = switch (name) {+ case "LocationId" -> matchesString(task.getSourceLocationArn(), values, operator)+ || matchesString(task.getDestinationLocationArn(), values, operator);+ case "CreationTime" -> matchesInstant(task.getCreationTime(), values, operator);+ default -> throw new AwsException("InvalidRequestException",+ "Unsupported ListTasks filter name: " + name, 400);+ };+ if (!matched) {+ return false;+ }+ }+ return true;+ }++ private static boolean matchesString(String attribute, List<String> values, String operator) {+ String actual = attribute != null ? attribute : "";+ return switch (operator) {+ case "Equals", "In" -> values.stream().anyMatch(actual::equalsIgnoreCase);+ case "NotEquals" -> values.stream().noneMatch(actual::equalsIgnoreCase);+ case "Contains" -> values.stream()+ .anyMatch(value -> actual.toLowerCase(Locale.ROOT).contains(value.toLowerCase(Locale.ROOT)));+ case "NotContains" -> values.stream()+ .noneMatch(value -> actual.toLowerCase(Locale.ROOT).contains(value.toLowerCase(Locale.ROOT)));+ case "BeginsWith" -> values.stream()+ .anyMatch(value -> actual.toLowerCase(Locale.ROOT).startsWith(value.toLowerCase(Locale.ROOT)));+ default -> throw new AwsException("InvalidRequestException",+ "Unsupported filter operator for a string attribute: " + operator, 400);+ };+ }++ private static boolean matchesInstant(Instant attribute, List<String> values, String operator) {+ Instant actual = attribute != null ? attribute : Instant.EPOCH;+ return switch (operator) {+ case "Equals", "In" -> values.stream().anyMatch(value -> actual.equals(parseInstant(value)));+ case "NotEquals" -> values.stream().noneMatch(value -> actual.equals(parseInstant(value)));+ case "LessThan" -> values.stream().allMatch(value -> actual.isBefore(parseInstant(value)));+ case "LessThanOrEqual" -> values.stream().allMatch(value -> !actual.isAfter(parseInstant(value)));+ case "GreaterThan" -> values.stream().allMatch(value -> actual.isAfter(parseInstant(value)));+ case "GreaterThanOrEqual" -> values.stream().allMatch(value -> !actual.isBefore(parseInstant(value)));+ default -> throw new AwsException("InvalidRequestException",+ "Unsupported filter operator for CreationTime: " + operator, 400);+ };+ }++ private static Instant parseInstant(String value) {+ try {+ return Instant.parse(value);+ } catch (DateTimeParseException notIso) {+ try {+ return Instant.ofEpochMilli((long) (Double.parseDouble(value) * 1000));+ } catch (NumberFormatException notEpoch) {+ throw new AwsException("InvalidRequestException",+ "CreationTime filter value is neither an ISO-8601 instant nor an epoch timestamp: "+ + value, 400);+ }+ }+ }++ // ---- Helpers ----++ private JsonNode mergedOptions(JsonNode requested, String taskMode) {+ ObjectNode options = mapper.createObjectNode();+ options.put("VerifyMode", defaultVerifyMode(taskMode));+ options.put("OverwriteMode", "ALWAYS");+ options.put("Atime", "BEST_EFFORT");+ options.put("Mtime", "PRESERVE");+ options.put("Uid", "INT_VALUE");+ options.put("Gid", "INT_VALUE");+ options.put("PreserveDeletedFiles", "PRESERVE");+ options.put("PreserveDevices", "NONE");+ options.put("PosixPermissions", "PRESERVE");+ options.put("BytesPerSecond", -1L);+ options.put("TaskQueueing", "ENABLED");+ options.put("LogLevel", "OFF");+ options.put("TransferMode", "CHANGED");+ options.put("SecurityDescriptorCopyFlags", "OWNER_DACL");+ options.put("ObjectTags", "PRESERVE");+ if (requested != null && requested.isObject()) {+ requested.properties().forEach(member -> options.set(member.getKey(), member.getValue().deepCopy()));+ }+ requireSupportedVerifyMode(options.path("VerifyMode").asText(), taskMode);+ return options;+ }++ private static String requestedTaskMode(JsonNode request) {+ String taskMode = request.path("TaskMode").asText(DEFAULT_TASK_MODE);+ if (!TASK_MODES.contains(taskMode)) {+ throw new AwsException("InvalidRequestException",+ "TaskMode " + taskMode + " is not a supported task mode. Supported modes are "+ + DEFAULT_TASK_MODE + " and " + TASK_MODE_ENHANCED + ".", 400);+ }+ return taskMode;+ }++ /**+ * The mode a stored task was created with. {@code UpdateTaskRequest} carries no+ * {@code TaskMode}, so a task keeps the mode it was created with for life.+ */+ private static String taskModeOf(DataSyncTask task) {+ return task.getTaskMode() != null ? task.getTaskMode() : DEFAULT_TASK_MODE;+ }++ private static String defaultVerifyMode(String taskMode) {+ return TASK_MODE_ENHANCED.equals(taskMode)+ ? VERIFY_MODE_ONLY_FILES_TRANSFERRED+ : VERIFY_MODE_POINT_IN_TIME_CONSISTENT;+ }++ private static void requireSupportedVerifyMode(String verifyMode, String taskMode) {+ if (TASK_MODE_ENHANCED.equals(taskMode) && VERIFY_MODE_POINT_IN_TIME_CONSISTENT.equals(verifyMode)) {+ throw new AwsException("InvalidRequestException",+ "VerifyMode " + VERIFY_MODE_POINT_IN_TIME_CONSISTENT + " is not supported for "+ + TASK_MODE_ENHANCED + " mode tasks.", 400);+ }+ }++ private JsonNode filterList(JsonNode requested) {+ return requested != null && requested.isArray() ? requested.deepCopy() : mapper.createArrayNode();+ }++ private static JsonNode copyOrNull(JsonNode requested) {+ return requested != null && !requested.isNull() ? requested.deepCopy() : null;+ }++ private static <T> Page<T> paginate(List<T> all, Function<T, String> identity,+ String nextToken, int maxResults) {+ int start = 0;+ if (nextToken != null && !nextToken.isEmpty()) {+ for (int i = 0; i < all.size(); i++) {+ if (identity.apply(all.get(i)).equals(nextToken)) {+ start = i + 1;+ break;+ }+ }+ }+ int limit = maxResults > 0 ? maxResults : DEFAULT_MAX_RESULTS;+ int end = Math.min(start + limit, all.size());+ List<T> page = new ArrayList<>(all.subList(Math.min(start, all.size()), end));+ String token = end < all.size() && !page.isEmpty() ? identity.apply(page.get(page.size() - 1)) : null;+ return new Page<>(page, token);+ }++ public static Map<String, String> tagsOf(JsonNode tagList) {+ Map<String, String> tags = new LinkedHashMap<>();+ if (tagList != null && tagList.isArray()) {+ for (JsonNode tag : tagList) {+ String key = tag.path("Key").asText("");+ if (!key.isEmpty()) {+ tags.put(key, tag.path("Value").asText(""));+ }+ }+ }+ return tags;+ }++ public static List<String> stringList(JsonNode arrayNode) {+ List<String> values = new ArrayList<>();+ if (arrayNode != null && arrayNode.isArray()) {+ arrayNode.forEach(element -> values.add(element.asText("")));+ }+ return values;+ }++ private static boolean hasValue(JsonNode request, String member) {+ JsonNode value = request.get(member);+ return value != null && !value.isNull() && !(value.isTextual() && value.asText().isEmpty());+ }++ private static String requiredText(JsonNode request, String member, String operation) {+ if (!hasValue(request, member)) {+ throw new AwsException("InvalidRequestException", member + " is required for " + operation + ".", 400);+ }+ return request.get(member).asText();+ }++ private static String optionalText(JsonNode request, String member) {+ return request.hasNonNull(member) ? request.get(member).asText() : null;+ }++ private static void requireArn(String arn, String member) {+ if (arn == null || arn.isBlank()) {+ throw new AwsException("InvalidRequestException", member + " is required.", 400);+ }+ }++ private static String randomResourceId() {+ StringBuilder id = new StringBuilder(RESOURCE_ID_LENGTH);+ for (int i = 0; i < RESOURCE_ID_LENGTH; i++) {+ id.append(HEX_ALPHABET.charAt(RANDOM.nextInt(HEX_ALPHABET.length())));+ }+ return id.toString();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncAgent.java b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncAgent.javanew file mode 100644index 0000000000..1075ee0316--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncAgent.java@@ -0,0 +1,126 @@+package io.github.hectorvent.floci.services.datasync.model;++import io.quarkus.runtime.annotations.RegisterForReflection;+import java.time.Instant;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++@RegisterForReflection+public class DataSyncAgent implements DataSyncTaggable {++ private String agentArn;+ private String name;+ private String status;+ private String activationKey;+ private String endpointType;+ private String vpcEndpointId;+ private List<String> subnetArns = new ArrayList<>();+ private List<String> securityGroupArns = new ArrayList<>();+ private String platformVersion;+ private Instant creationTime;+ private Instant lastConnectionTime;+ private Map<String, String> tags = new LinkedHashMap<>();++ public String getAgentArn() {+ return agentArn;+ }++ public void setAgentArn(String agentArn) {+ this.agentArn = agentArn;+ }++ public String getName() {+ return name;+ }++ public void setName(String name) {+ this.name = name;+ }++ public String getStatus() {+ return status;+ }++ public void setStatus(String status) {+ this.status = status;+ }++ public String getActivationKey() {+ return activationKey;+ }++ public void setActivationKey(String activationKey) {+ this.activationKey = activationKey;+ }++ public String getEndpointType() {+ return endpointType;+ }++ public void setEndpointType(String endpointType) {+ this.endpointType = endpointType;+ }++ public String getVpcEndpointId() {+ return vpcEndpointId;+ }++ public void setVpcEndpointId(String vpcEndpointId) {+ this.vpcEndpointId = vpcEndpointId;+ }++ public List<String> getSubnetArns() {+ return subnetArns;+ }++ public void setSubnetArns(List<String> subnetArns) {+ this.subnetArns = subnetArns != null ? subnetArns : new ArrayList<>();+ }++ public List<String> getSecurityGroupArns() {+ return securityGroupArns;+ }++ public void setSecurityGroupArns(List<String> securityGroupArns) {+ this.securityGroupArns = securityGroupArns != null ? securityGroupArns : new ArrayList<>();+ }++ public String getPlatformVersion() {+ return platformVersion;+ }++ public void setPlatformVersion(String platformVersion) {+ this.platformVersion = platformVersion;+ }++ public Instant getCreationTime() {+ return creationTime;+ }++ public void setCreationTime(Instant creationTime) {+ this.creationTime = creationTime;+ }++ public Instant getLastConnectionTime() {+ return lastConnectionTime;+ }++ public void setLastConnectionTime(Instant lastConnectionTime) {+ this.lastConnectionTime = lastConnectionTime;+ }++ @Override+ public Map<String, String> getTags() {+ if (tags == null) {+ tags = new LinkedHashMap<>();+ }+ return tags;+ }++ @Override+ public void setTags(Map<String, String> tags) {+ this.tags = tags != null ? tags : new LinkedHashMap<>();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncLocation.java b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncLocation.javanew file mode 100644index 0000000000..603b53755b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncLocation.java@@ -0,0 +1,86 @@+package io.github.hectorvent.floci.services.datasync.model;++import com.fasterxml.jackson.databind.JsonNode;++import io.quarkus.runtime.annotations.RegisterForReflection;+import java.time.Instant;+import java.util.LinkedHashMap;+import java.util.Map;++/**+ * A DataSync transfer location. {@code configuration} holds the create request with its+ * credential members stripped and the service's documented defaults applied; every+ * {@code DescribeLocation*} answer is projected out of it.+ */+@RegisterForReflection+public class DataSyncLocation implements DataSyncTaggable {++ private String locationArn;+ private DataSyncLocationType locationType;+ private String locationUri;+ private String region;+ private JsonNode configuration;+ private Instant creationTime;+ private Map<String, String> tags = new LinkedHashMap<>();++ public String getLocationArn() {+ return locationArn;+ }++ public void setLocationArn(String locationArn) {+ this.locationArn = locationArn;+ }++ public DataSyncLocationType getLocationType() {+ return locationType;+ }++ public void setLocationType(DataSyncLocationType locationType) {+ this.locationType = locationType;+ }++ public String getLocationUri() {+ return locationUri;+ }++ public void setLocationUri(String locationUri) {+ this.locationUri = locationUri;+ }++ public String getRegion() {+ return region;+ }++ public void setRegion(String region) {+ this.region = region;+ }++ public JsonNode getConfiguration() {+ return configuration;+ }++ public void setConfiguration(JsonNode configuration) {+ this.configuration = configuration;+ }++ public Instant getCreationTime() {+ return creationTime;+ }++ public void setCreationTime(Instant creationTime) {+ this.creationTime = creationTime;+ }++ @Override+ public Map<String, String> getTags() {+ if (tags == null) {+ tags = new LinkedHashMap<>();+ }+ return tags;+ }++ @Override+ public void setTags(Map<String, String> tags) {+ this.tags = tags != null ? tags : new LinkedHashMap<>();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncLocationType.java b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncLocationType.javanew file mode 100644index 0000000000..1c49caa896--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncLocationType.java@@ -0,0 +1,104 @@+package io.github.hectorvent.floci.services.datasync.model;++import io.quarkus.runtime.annotations.RegisterForReflection;+import java.util.List;+import java.util.Optional;++/**+ * The DataSync location families, one per {@code CreateLocation*} operation.+ *+ * <p>Each constant carries the operation suffix that names its Create/Describe/Update+ * trio, the URI scheme AWS builds the location's {@code LocationUri} from, the members+ * the model marks required on create, and the members its {@code DescribeLocation*}+ * response echoes back from the create request.+ */+@RegisterForReflection+public enum DataSyncLocationType {++ AZURE_BLOB("AzureBlob", "azure-blob",+ List.of("ContainerUrl", "AuthenticationType"),+ List.of("AuthenticationType", "BlobType", "AccessTier", "AgentArns",+ "CmkSecretConfig", "CustomSecretConfig")),++ EFS("Efs", "efs",+ List.of("EfsFilesystemArn", "Ec2Config"),+ List.of("Ec2Config", "AccessPointArn", "FileSystemAccessRoleArn", "InTransitEncryption")),++ FSX_LUSTRE("FsxLustre", "fsxl",+ List.of("FsxFilesystemArn", "SecurityGroupArns"),+ List.of("SecurityGroupArns")),++ FSX_ONTAP("FsxOntap", "fsxn",+ List.of("Protocol", "SecurityGroupArns", "StorageVirtualMachineArn"),+ List.of("Protocol", "SecurityGroupArns", "StorageVirtualMachineArn", "FsxFilesystemArn")),++ FSX_OPEN_ZFS("FsxOpenZfs", "fsxz",+ List.of("FsxFilesystemArn", "Protocol", "SecurityGroupArns"),+ List.of("SecurityGroupArns", "Protocol")),++ FSX_WINDOWS("FsxWindows", "fsxw",+ List.of("FsxFilesystemArn", "SecurityGroupArns", "User"),+ List.of("SecurityGroupArns", "User", "Domain")),++ HDFS("Hdfs", "hdfs",+ List.of("NameNodes", "AuthenticationType", "AgentArns"),+ List.of("NameNodes", "BlockSize", "ReplicationFactor", "KmsKeyProviderUri",+ "QopConfiguration", "AuthenticationType", "SimpleUser", "KerberosPrincipal",+ "AgentArns")),++ NFS("Nfs", "nfs",+ List.of("Subdirectory", "ServerHostname", "OnPremConfig"),+ List.of("OnPremConfig", "MountOptions")),++ OBJECT_STORAGE("ObjectStorage", "object-storage",+ List.of("ServerHostname", "BucketName"),+ List.of("AccessKey", "ServerPort", "ServerProtocol", "AgentArns", "ServerCertificate",+ "CmkSecretConfig", "CustomSecretConfig")),++ S3("S3", "s3",+ List.of("S3BucketArn", "S3Config"),+ List.of("S3StorageClass", "S3Config", "AgentArns")),++ SMB("Smb", "smb",+ List.of("Subdirectory", "ServerHostname", "AgentArns"),+ List.of("AgentArns", "User", "Domain", "MountOptions", "DnsIpAddresses",+ "KerberosPrincipal", "AuthenticationType", "CmkSecretConfig", "CustomSecretConfig"));++ private final String operationSuffix;+ private final String uriScheme;+ private final List<String> requiredMembers;+ private final List<String> describeMembers;++ DataSyncLocationType(String operationSuffix, String uriScheme,+ List<String> requiredMembers, List<String> describeMembers) {+ this.operationSuffix = operationSuffix;+ this.uriScheme = uriScheme;+ this.requiredMembers = requiredMembers;+ this.describeMembers = describeMembers;+ }++ public static Optional<DataSyncLocationType> fromOperationSuffix(String suffix) {+ for (DataSyncLocationType type : values()) {+ if (type.operationSuffix.equals(suffix)) {+ return Optional.of(type);+ }+ }+ return Optional.empty();+ }++ public String operationSuffix() {+ return operationSuffix;+ }++ public String uriScheme() {+ return uriScheme;+ }++ public List<String> requiredMembers() {+ return requiredMembers;+ }++ public List<String> describeMembers() {+ return describeMembers;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncTaggable.java b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncTaggable.javanew file mode 100644index 0000000000..1a2c3a14b1--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncTaggable.java@@ -0,0 +1,14 @@+package io.github.hectorvent.floci.services.datasync.model;++import java.util.Map;++/**+ * A DataSync resource addressable by the service's own {@code TagResource},+ * {@code UntagResource} and {@code ListTagsForResource} operations.+ */+public interface DataSyncTaggable {++ Map<String, String> getTags();++ void setTags(Map<String, String> tags);+}diff --git a/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncTask.java b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncTask.javanew file mode 100644index 0000000000..10a66230e1--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/datasync/model/DataSyncTask.java@@ -0,0 +1,153 @@+package io.github.hectorvent.floci.services.datasync.model;++import com.fasterxml.jackson.databind.JsonNode;++import io.quarkus.runtime.annotations.RegisterForReflection;+import java.time.Instant;+import java.util.LinkedHashMap;+import java.util.Map;++@RegisterForReflection+public class DataSyncTask implements DataSyncTaggable {++ private String taskArn;+ private String name;+ private String status;+ private String taskMode;+ private String sourceLocationArn;+ private String destinationLocationArn;+ private String cloudWatchLogGroupArn;+ private JsonNode options;+ private JsonNode excludes;+ private JsonNode includes;+ private JsonNode schedule;+ private JsonNode manifestConfig;+ private JsonNode taskReportConfig;+ private Instant creationTime;+ private Map<String, String> tags = new LinkedHashMap<>();++ public String getTaskArn() {+ return taskArn;+ }++ public void setTaskArn(String taskArn) {+ this.taskArn = taskArn;+ }++ public String getName() {+ return name;+ }++ public void setName(String name) {+ this.name = name;+ }++ public String getStatus() {+ return status;+ }++ public void setStatus(String status) {+ this.status = status;+ }++ public String getTaskMode() {+ return taskMode;+ }++ public void setTaskMode(String taskMode) {+ this.taskMode = taskMode;+ }++ public String getSourceLocationArn() {+ return sourceLocationArn;+ }++ public void setSourceLocationArn(String sourceLocationArn) {+ this.sourceLocationArn = sourceLocationArn;+ }++ public String getDestinationLocationArn() {+ return destinationLocationArn;+ }++ public void setDestinationLocationArn(String destinationLocationArn) {+ this.destinationLocationArn = destinationLocationArn;+ }++ public String getCloudWatchLogGroupArn() {+ return cloudWatchLogGroupArn;+ }++ public void setCloudWatchLogGroupArn(String cloudWatchLogGroupArn) {+ this.cloudWatchLogGroupArn = cloudWatchLogGroupArn;+ }++ public JsonNode getOptions() {+ return options;+ }++ public void setOptions(JsonNode options) {+ this.options = options;+ }++ public JsonNode getExcludes() {+ return excludes;+ }++ public void setExcludes(JsonNode excludes) {+ this.excludes = excludes;+ }++ public JsonNode getIncludes() {+ return includes;+ }++ public void setIncludes(JsonNode includes) {+ this.includes = includes;+ }++ public JsonNode getSchedule() {+ return schedule;+ }++ public void setSchedule(JsonNode schedule) {+ this.schedule = schedule;+ }++ public JsonNode getManifestConfig() {+ return manifestConfig;+ }++ public void setManifestConfig(JsonNode manifestConfig) {+ this.manifestConfig = manifestConfig;+ }++ public JsonNode getTaskReportConfig() {+ return taskReportConfig;+ }++ public void setTaskReportConfig(JsonNode taskReportConfig) {+ this.taskReportConfig = taskReportConfig;+ }++ public Instant getCreationTime() {+ return creationTime;+ }++ public void setCreationTime(Instant creationTime) {+ this.creationTime = creationTime;+ }++ @Override+ public Map<String, String> getTags() {+ if (tags == null) {+ tags = new LinkedHashMap<>();+ }+ return tags;+ }++ @Override+ public void setTags(Map<String, String> tags) {+ this.tags = tags != null ? tags : new LinkedHashMap<>();+ }+}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 71c110e309..a8227e5af3 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -91,6 +91,7 @@ quarkus:- --initialize-at-run-time=io.github.hectorvent.floci.services.emr.EmrService- --initialize-at-run-time=io.github.hectorvent.floci.services.elasticbeanstalk.ElasticBeanstalkService- --initialize-at-run-time=io.github.hectorvent.floci.services.elb.ElbClassicService+ - --initialize-at-run-time=io.github.hectorvent.floci.services.datasync.DataSyncService- --initialize-at-run-time=graphql.util.IdGenerator- --initialize-at-run-time=io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials- --initialize-at-run-time=io.github.hectorvent.floci.services.ssoportal.SsoPortalService@@ -689,6 +690,8 @@ floci:enabled: trueglobalaccelerator:enabled: true+ datasync:+ enabled: trueaps:enabled: truelakeformation:diff --git a/src/test/java/io/github/hectorvent/floci/services/datasync/DataSyncIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/datasync/DataSyncIntegrationTest.javanew file mode 100644index 0000000000..57e23c6087--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/datasync/DataSyncIntegrationTest.java@@ -0,0 +1,682 @@+package io.github.hectorvent.floci.services.datasync;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.github.hectorvent.floci.testing.RestAssuredJsonUtils.awsAction;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.emptyIterable;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.greaterThan;+import static org.hamcrest.Matchers.hasItem;+import static org.hamcrest.Matchers.matchesPattern;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.nullValue;++@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class DataSyncIntegrationTest {++ private static final String TARGET = "FmrsService";++ private static final String IAM_ROLE_ARN = "arn:aws:iam::000000000000:role/datasync-bucket-access";+ private static final String EFS_FILESYSTEM_ARN =+ "arn:aws:elasticfilesystem:us-east-1:000000000000:file-system/fs-0123456789abcdef0";+ private static final String FSX_FILESYSTEM_ARN =+ "arn:aws:fsx:us-east-1:000000000000:file-system/fs-0123456789abcdef0";+ private static final String STORAGE_VIRTUAL_MACHINE_ARN =+ "arn:aws:fsx:us-east-1:000000000000:storage-virtual-machine/fs-0123456789abcdef0/svm-0123456789abcdef0";+ private static final String SUBNET_ARN = "arn:aws:ec2:us-east-1:000000000000:subnet/subnet-0123456789abcdef0";+ private static final String SECURITY_GROUP_ARN =+ "arn:aws:ec2:us-east-1:000000000000:security-group/sg-0123456789abcdef0";++ private static String agentArn;+ private static String s3LocationArn;+ private static String efsLocationArn;+ private static String nfsLocationArn;+ private static String smbLocationArn;+ private static String objectStorageLocationArn;+ private static String hdfsLocationArn;+ private static String azureBlobLocationArn;+ private static String fsxLustreLocationArn;+ private static String fsxOpenZfsLocationArn;+ private static String fsxWindowsLocationArn;+ private static String fsxOntapLocationArn;+ private static String taskArn;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ private static String createLocation(String action, String body) {+ return awsAction(TARGET, action, body)+ .then()+ .statusCode(200)+ .body("LocationArn", matchesPattern("^arn:aws:datasync:[^:]+:\\d{12}:location/loc-[0-9a-f]{17}$"))+ .extract().path("LocationArn");+ }++ private static io.restassured.response.Response describeLocation(String action, String locationArn) {+ return awsAction(TARGET, action, "{\"LocationArn\": \"" + locationArn + "\"}");+ }++ // ---- Agents ----++ @Test+ @Order(1)+ void createAgent() {+ agentArn = awsAction(TARGET, "CreateAgent", """+ {+ "ActivationKey": "AAAAA-1AAAA-BB1CC-DDDDD-EEEEE",+ "AgentName": "floci-agent",+ "Tags": [{"Key": "team", "Value": "data-movement"}]+ }+ """)+ .then()+ .statusCode(200)+ .body("AgentArn", matchesPattern("^arn:aws:datasync:[^:]+:\\d{12}:agent/agent-[0-9a-f]{17}$"))+ .extract().path("AgentArn");+ }++ @Test+ @Order(2)+ void describeAgentReportsOnlineOnTheFirstRead() {+ awsAction(TARGET, "DescribeAgent", "{\"AgentArn\": \"" + agentArn + "\"}")+ .then()+ .statusCode(200)+ .body("AgentArn", equalTo(agentArn))+ .body("Name", equalTo("floci-agent"))+ .body("Status", equalTo("ONLINE"))+ .body("EndpointType", equalTo("PUBLIC"))+ .body("Platform.Version", equalTo(DataSyncService.AGENT_PLATFORM_VERSION))+ .body("CreationTime", greaterThan(0.0f))+ .body("LastConnectionTime", greaterThan(0.0f));+ }++ @Test+ @Order(3)+ void listAgentsIncludesTheCreatedAgent() {+ awsAction(TARGET, "ListAgents", "{}")+ .then()+ .statusCode(200)+ .body("Agents.AgentArn", hasItem(agentArn))+ .body("Agents.find { it.AgentArn == '" + agentArn + "' }.Status", equalTo("ONLINE"));+ }++ @Test+ @Order(4)+ void updateAgentRenamesTheAgent() {+ awsAction(TARGET, "UpdateAgent",+ "{\"AgentArn\": \"" + agentArn + "\", \"Name\": \"floci-agent-renamed\"}")+ .then()+ .statusCode(200);++ awsAction(TARGET, "DescribeAgent", "{\"AgentArn\": \"" + agentArn + "\"}")+ .then()+ .statusCode(200)+ .body("Name", equalTo("floci-agent-renamed"));+ }++ @Test+ @Order(5)+ void listTagsForAgentReturnsTheCreateTags() {+ awsAction(TARGET, "ListTagsForResource", "{\"ResourceArn\": \"" + agentArn + "\"}")+ .then()+ .statusCode(200)+ .body("Tags[0].Key", equalTo("team"))+ .body("Tags[0].Value", equalTo("data-movement"));+ }++ // ---- Locations ----++ @Test+ @Order(10)+ void s3LocationRoundTrip() {+ s3LocationArn = createLocation("CreateLocationS3", """+ {+ "S3BucketArn": "arn:aws:s3:::floci-datasync-bucket",+ "Subdirectory": "/backups",+ "S3Config": {"BucketAccessRoleArn": "%s"},+ "Tags": [{"Key": "env", "Value": "test"}]+ }+ """.formatted(IAM_ROLE_ARN));++ describeLocation("DescribeLocationS3", s3LocationArn)+ .then()+ .statusCode(200)+ .body("LocationArn", equalTo(s3LocationArn))+ .body("LocationUri", equalTo("s3://floci-datasync-bucket/backups"))+ .body("S3StorageClass", equalTo("STANDARD"))+ .body("S3Config.BucketAccessRoleArn", equalTo(IAM_ROLE_ARN))+ .body("CreationTime", greaterThan(0.0f));+ }++ @Test+ @Order(11)+ void efsLocationRoundTrip() {+ efsLocationArn = createLocation("CreateLocationEfs", """+ {+ "EfsFilesystemArn": "%s",+ "Subdirectory": "/exports",+ "Ec2Config": {"SubnetArn": "%s", "SecurityGroupArns": ["%s"]}+ }+ """.formatted(EFS_FILESYSTEM_ARN, SUBNET_ARN, SECURITY_GROUP_ARN));++ describeLocation("DescribeLocationEfs", efsLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("efs://us-east-1.fs-0123456789abcdef0/exports"))+ .body("Ec2Config.SubnetArn", equalTo(SUBNET_ARN))+ .body("Ec2Config.SecurityGroupArns[0]", equalTo(SECURITY_GROUP_ARN))+ .body("InTransitEncryption", equalTo("NONE"));+ }++ @Test+ @Order(12)+ void nfsLocationRoundTrip() {+ nfsLocationArn = createLocation("CreateLocationNfs", """+ {+ "Subdirectory": "/export/home",+ "ServerHostname": "nfs.example.com",+ "OnPremConfig": {"AgentArns": ["%s"]}+ }+ """.formatted(agentArn));++ describeLocation("DescribeLocationNfs", nfsLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("nfs://nfs.example.com/export/home"))+ .body("OnPremConfig.AgentArns[0]", equalTo(agentArn))+ .body("MountOptions.Version", equalTo("AUTOMATIC"));+ }++ @Test+ @Order(13)+ void smbLocationRoundTripDoesNotEchoThePassword() {+ smbLocationArn = createLocation("CreateLocationSmb", """+ {+ "Subdirectory": "/share",+ "ServerHostname": "smb.example.com",+ "User": "floci",+ "Domain": "EXAMPLE",+ "Password": "sup3rs3cret",+ "AgentArns": ["%s"]+ }+ """.formatted(agentArn));++ describeLocation("DescribeLocationSmb", smbLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("smb://smb.example.com/share"))+ .body("User", equalTo("floci"))+ .body("Domain", equalTo("EXAMPLE"))+ .body("AuthenticationType", equalTo("NTLM"))+ .body("MountOptions.Version", equalTo("AUTOMATIC"))+ .body("AgentArns[0]", equalTo(agentArn))+ .body("Password", nullValue());+ }++ @Test+ @Order(14)+ void objectStorageLocationRoundTripDoesNotEchoTheSecretKey() {+ objectStorageLocationArn = createLocation("CreateLocationObjectStorage", """+ {+ "ServerHostname": "objects.example.com",+ "BucketName": "floci-bucket",+ "Subdirectory": "/incoming",+ "AccessKey": "AKIAFLOCI",+ "SecretKey": "sup3rs3cret",+ "AgentArns": ["%s"]+ }+ """.formatted(agentArn));++ describeLocation("DescribeLocationObjectStorage", objectStorageLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("object-storage://objects.example.com/floci-bucket/incoming"))+ .body("ServerProtocol", equalTo("HTTPS"))+ .body("ServerPort", equalTo(443))+ .body("AccessKey", equalTo("AKIAFLOCI"))+ .body("SecretKey", nullValue());+ }++ @Test+ @Order(15)+ void hdfsLocationRoundTripAppliesDocumentedDefaults() {+ hdfsLocationArn = createLocation("CreateLocationHdfs", """+ {+ "Subdirectory": "/user/hadoop",+ "NameNodes": [{"Hostname": "namenode.example.com", "Port": 8020}],+ "AuthenticationType": "SIMPLE",+ "SimpleUser": "hadoop",+ "AgentArns": ["%s"]+ }+ """.formatted(agentArn));++ describeLocation("DescribeLocationHdfs", hdfsLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("hdfs://namenode.example.com:8020/user/hadoop"))+ .body("NameNodes[0].Hostname", equalTo("namenode.example.com"))+ .body("NameNodes[0].Port", equalTo(8020))+ .body("BlockSize", equalTo(134217728))+ .body("ReplicationFactor", equalTo(3))+ .body("QopConfiguration.RpcProtection", equalTo("PRIVACY"))+ .body("QopConfiguration.DataTransferProtection", equalTo("PRIVACY"))+ .body("AuthenticationType", equalTo("SIMPLE"))+ .body("SimpleUser", equalTo("hadoop"));+ }++ @Test+ @Order(16)+ void azureBlobLocationRoundTrip() {+ azureBlobLocationArn = createLocation("CreateLocationAzureBlob", """+ {+ "ContainerUrl": "https://flociaccount.blob.core.windows.net/flocicontainer",+ "AuthenticationType": "SAS",+ "SasConfiguration": {"Token": "sv=2021&sig=secret"},+ "Subdirectory": "/data",+ "AgentArns": ["%s"]+ }+ """.formatted(agentArn));++ describeLocation("DescribeLocationAzureBlob", azureBlobLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri",+ equalTo("azure-blob://flociaccount.blob.core.windows.net/flocicontainer/data"))+ .body("AuthenticationType", equalTo("SAS"))+ .body("BlobType", equalTo("BLOCK"))+ .body("AccessTier", equalTo("HOT"))+ .body("AgentArns[0]", equalTo(agentArn))+ .body("SasConfiguration", nullValue());+ }++ @Test+ @Order(17)+ void fsxLustreLocationRoundTrip() {+ fsxLustreLocationArn = createLocation("CreateLocationFsxLustre", """+ {+ "FsxFilesystemArn": "%s",+ "SecurityGroupArns": ["%s"],+ "Subdirectory": "/scratch"+ }+ """.formatted(FSX_FILESYSTEM_ARN, SECURITY_GROUP_ARN));++ describeLocation("DescribeLocationFsxLustre", fsxLustreLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("fsxl://us-east-1.fs-0123456789abcdef0/scratch"))+ .body("SecurityGroupArns[0]", equalTo(SECURITY_GROUP_ARN));+ }++ @Test+ @Order(18)+ void fsxOpenZfsLocationRoundTrip() {+ fsxOpenZfsLocationArn = createLocation("CreateLocationFsxOpenZfs", """+ {+ "FsxFilesystemArn": "%s",+ "SecurityGroupArns": ["%s"],+ "Protocol": {"NFS": {"MountOptions": {"Version": "AUTOMATIC"}}},+ "Subdirectory": "/fsx/folderA"+ }+ """.formatted(FSX_FILESYSTEM_ARN, SECURITY_GROUP_ARN));++ describeLocation("DescribeLocationFsxOpenZfs", fsxOpenZfsLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("fsxz://us-east-1.fs-0123456789abcdef0/fsx/folderA"))+ .body("Protocol.NFS.MountOptions.Version", equalTo("AUTOMATIC"));+ }++ @Test+ @Order(19)+ void fsxWindowsLocationRoundTrip() {+ fsxWindowsLocationArn = createLocation("CreateLocationFsxWindows", """+ {+ "FsxFilesystemArn": "%s",+ "SecurityGroupArns": ["%s"],+ "User": "floci",+ "Domain": "EXAMPLE",+ "Password": "sup3rs3cret",+ "Subdirectory": "/share"+ }+ """.formatted(FSX_FILESYSTEM_ARN, SECURITY_GROUP_ARN));++ describeLocation("DescribeLocationFsxWindows", fsxWindowsLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("fsxw://us-east-1.fs-0123456789abcdef0/share"))+ .body("User", equalTo("floci"))+ .body("Domain", equalTo("EXAMPLE"))+ .body("Password", nullValue());+ }++ @Test+ @Order(20)+ void fsxOntapLocationDerivesTheFileSystemArn() {+ fsxOntapLocationArn = createLocation("CreateLocationFsxOntap", """+ {+ "StorageVirtualMachineArn": "%s",+ "SecurityGroupArns": ["%s"],+ "Protocol": {"SMB": {"User": "floci", "Domain": "EXAMPLE", "Password": "sup3rs3cret"}},+ "Subdirectory": "/vol1"+ }+ """.formatted(STORAGE_VIRTUAL_MACHINE_ARN, SECURITY_GROUP_ARN));++ describeLocation("DescribeLocationFsxOntap", fsxOntapLocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri",+ equalTo("fsxn://us-east-1.fs-0123456789abcdef0.svm-0123456789abcdef0/vol1"))+ .body("StorageVirtualMachineArn", equalTo(STORAGE_VIRTUAL_MACHINE_ARN))+ .body("FsxFilesystemArn", equalTo(FSX_FILESYSTEM_ARN))+ .body("Protocol.SMB.User", equalTo("floci"))+ .body("Protocol.SMB.Password", nullValue());+ }++ @Test+ @Order(21)+ void updateLocationS3RebuildsTheLocationUri() {+ awsAction(TARGET, "UpdateLocationS3", """+ {+ "LocationArn": "%s",+ "Subdirectory": "/archive",+ "S3StorageClass": "GLACIER"+ }+ """.formatted(s3LocationArn))+ .then()+ .statusCode(200);++ describeLocation("DescribeLocationS3", s3LocationArn)+ .then()+ .statusCode(200)+ .body("LocationUri", equalTo("s3://floci-datasync-bucket/archive"))+ .body("S3StorageClass", equalTo("GLACIER"))+ .body("S3Config.BucketAccessRoleArn", equalTo(IAM_ROLE_ARN));+ }++ @Test+ @Order(22)+ void listLocationsSeesEveryCreatedLocation() {+ awsAction(TARGET, "ListLocations", "{}")+ .then()+ .statusCode(200)+ .body("Locations.LocationArn", hasItem(s3LocationArn))+ .body("Locations.LocationArn", hasItem(efsLocationArn))+ .body("Locations.LocationArn", hasItem(nfsLocationArn))+ .body("Locations.LocationArn", hasItem(smbLocationArn))+ .body("Locations.LocationArn", hasItem(objectStorageLocationArn))+ .body("Locations.LocationArn", hasItem(hdfsLocationArn))+ .body("Locations.LocationArn", hasItem(azureBlobLocationArn))+ .body("Locations.LocationArn", hasItem(fsxLustreLocationArn))+ .body("Locations.LocationArn", hasItem(fsxOpenZfsLocationArn))+ .body("Locations.LocationArn", hasItem(fsxWindowsLocationArn))+ .body("Locations.LocationArn", hasItem(fsxOntapLocationArn));+ }++ @Test+ @Order(23)+ void listLocationsHonoursALocationTypeFilter() {+ awsAction(TARGET, "ListLocations", """+ {"Filters": [{"Name": "LocationType", "Values": ["Efs"], "Operator": "Equals"}]}+ """)+ .then()+ .statusCode(200)+ .body("Locations.LocationArn", hasItem(efsLocationArn))+ .body("Locations.LocationArn", not(hasItem(s3LocationArn)));+ }++ @Test+ @Order(24)+ void describeLocationRejectsAMismatchedLocationType() {+ describeLocation("DescribeLocationS3", efsLocationArn)+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }++ @Test+ @Order(25)+ void describeMissingLocationIsAnInvalidRequest() {+ describeLocation("DescribeLocationS3",+ "arn:aws:datasync:us-east-1:000000000000:location/loc-00000000000000000")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", containsString("is not found"));+ }++ @Test+ @Order(26)+ void createLocationRejectsAMissingRequiredMember() {+ awsAction(TARGET, "CreateLocationS3",+ "{\"S3Config\": {\"BucketAccessRoleArn\": \"" + IAM_ROLE_ARN + "\"}}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", containsString("S3BucketArn"));+ }++ // ---- Tasks ----++ @Test+ @Order(30)+ void createTaskReportsAvailableOnTheFirstRead() {+ taskArn = awsAction(TARGET, "CreateTask", """+ {+ "SourceLocationArn": "%s",+ "DestinationLocationArn": "%s",+ "Name": "floci-task",+ "Options": {"LogLevel": "TRANSFER", "VerifyMode": "NONE"},+ "Tags": [{"Key": "env", "Value": "test"}]+ }+ """.formatted(nfsLocationArn, s3LocationArn))+ .then()+ .statusCode(200)+ .body("TaskArn", matchesPattern("^arn:aws:datasync:[^:]+:\\d{12}:task/task-[0-9a-f]{17}$"))+ .extract().path("TaskArn");++ awsAction(TARGET, "DescribeTask", "{\"TaskArn\": \"" + taskArn + "\"}")+ .then()+ .statusCode(200)+ .body("TaskArn", equalTo(taskArn))+ .body("Status", equalTo("AVAILABLE"))+ .body("Name", equalTo("floci-task"))+ .body("TaskMode", equalTo("BASIC"))+ .body("SourceLocationArn", equalTo(nfsLocationArn))+ .body("DestinationLocationArn", equalTo(s3LocationArn))+ .body("SourceNetworkInterfaceArns", emptyIterable())+ .body("DestinationNetworkInterfaceArns", emptyIterable())+ .body("Excludes", emptyIterable())+ .body("Includes", emptyIterable())+ .body("Options.LogLevel", equalTo("TRANSFER"))+ .body("Options.VerifyMode", equalTo("NONE"))+ .body("Options.OverwriteMode", equalTo("ALWAYS"))+ .body("Options.PreserveDeletedFiles", equalTo("PRESERVE"))+ .body("Options.TransferMode", equalTo("CHANGED"))+ .body("Options.BytesPerSecond", equalTo(-1))+ .body("CreationTime", greaterThan(0.0f));+ }++ @Test+ @Order(31)+ void createTaskRejectsAnUnknownLocation() {+ awsAction(TARGET, "CreateTask", """+ {+ "SourceLocationArn": "arn:aws:datasync:us-east-1:000000000000:location/loc-00000000000000000",+ "DestinationLocationArn": "%s"+ }+ """.formatted(s3LocationArn))+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }++ @Test+ @Order(32)+ void listTasksIncludesTheCreatedTask() {+ awsAction(TARGET, "ListTasks", "{}")+ .then()+ .statusCode(200)+ .body("Tasks.TaskArn", hasItem(taskArn))+ .body("Tasks.find { it.TaskArn == '" + taskArn + "' }.Status", equalTo("AVAILABLE"));+ }++ @Test+ @Order(33)+ void updateTaskRenamesAndReschedulesTheTask() {+ awsAction(TARGET, "UpdateTask", """+ {+ "TaskArn": "%s",+ "Name": "floci-task-renamed",+ "Schedule": {"ScheduleExpression": "cron(0 12 ? * SUN *)", "Status": "ENABLED"},+ "Excludes": [{"FilterType": "SIMPLE_PATTERN", "Value": "/tmp"}]+ }+ """.formatted(taskArn))+ .then()+ .statusCode(200);++ awsAction(TARGET, "DescribeTask", "{\"TaskArn\": \"" + taskArn + "\"}")+ .then()+ .statusCode(200)+ .body("Name", equalTo("floci-task-renamed"))+ .body("Schedule.ScheduleExpression", equalTo("cron(0 12 ? * SUN *)"))+ .body("Excludes[0].FilterType", equalTo("SIMPLE_PATTERN"))+ .body("Excludes[0].Value", equalTo("/tmp"));+ }++ @Test+ @Order(34)+ void taskTagRoundTrip() {+ awsAction(TARGET, "TagResource", """+ {"ResourceArn": "%s", "Tags": [{"Key": "owner", "Value": "platform"}]}+ """.formatted(taskArn))+ .then()+ .statusCode(200);++ awsAction(TARGET, "ListTagsForResource", "{\"ResourceArn\": \"" + taskArn + "\"}")+ .then()+ .statusCode(200)+ .body("Tags.Key", hasItem("env"))+ .body("Tags.Key", hasItem("owner"))+ .body("Tags.find { it.Key == 'owner' }.Value", equalTo("platform"));++ awsAction(TARGET, "UntagResource", "{\"ResourceArn\": \"" + taskArn + "\", \"Keys\": [\"env\"]}")+ .then()+ .statusCode(200);++ awsAction(TARGET, "ListTagsForResource", "{\"ResourceArn\": \"" + taskArn + "\"}")+ .then()+ .statusCode(200)+ .body("Tags.Key", not(hasItem("env")))+ .body("Tags.Key", hasItem("owner"));+ }++ @Test+ @Order(35)+ void enhancedTaskDefaultsToOnlyFilesTransferredAndRejectsPointInTimeConsistent() {+ String enhancedTaskArn = awsAction(TARGET, "CreateTask", """+ {+ "SourceLocationArn": "%s",+ "DestinationLocationArn": "%s",+ "Name": "floci-task-enhanced",+ "TaskMode": "ENHANCED"+ }+ """.formatted(nfsLocationArn, s3LocationArn))+ .then()+ .statusCode(200)+ .extract().path("TaskArn");++ awsAction(TARGET, "DescribeTask", "{\"TaskArn\": \"" + enhancedTaskArn + "\"}")+ .then()+ .statusCode(200)+ .body("TaskMode", equalTo("ENHANCED"))+ .body("Options.VerifyMode", equalTo("ONLY_FILES_TRANSFERRED"));++ awsAction(TARGET, "UpdateTask", """+ {"TaskArn": "%s", "Options": {"VerifyMode": "POINT_IN_TIME_CONSISTENT"}}+ """.formatted(enhancedTaskArn))+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", containsString("POINT_IN_TIME_CONSISTENT"));++ awsAction(TARGET, "CreateTask", """+ {+ "SourceLocationArn": "%s",+ "DestinationLocationArn": "%s",+ "TaskMode": "ENHANCED",+ "Options": {"VerifyMode": "POINT_IN_TIME_CONSISTENT"}+ }+ """.formatted(nfsLocationArn, s3LocationArn))+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));++ awsAction(TARGET, "DeleteTask", "{\"TaskArn\": \"" + enhancedTaskArn + "\"}")+ .then()+ .statusCode(200);+ }++ // ---- Data plane and teardown ----++ @Test+ @Order(40)+ void taskExecutionOperationsAreNotEmulated() {+ awsAction(TARGET, "StartTaskExecution", "{\"TaskArn\": \"" + taskArn + "\"}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("UnknownOperationException"));+ }++ @Test+ @Order(41)+ void deleteTaskRemovesIt() {+ awsAction(TARGET, "DeleteTask", "{\"TaskArn\": \"" + taskArn + "\"}")+ .then()+ .statusCode(200);++ awsAction(TARGET, "DescribeTask", "{\"TaskArn\": \"" + taskArn + "\"}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }++ @Test+ @Order(42)+ void deleteLocationRemovesIt() {+ awsAction(TARGET, "DeleteLocation", "{\"LocationArn\": \"" + s3LocationArn + "\"}")+ .then()+ .statusCode(200);++ describeLocation("DescribeLocationS3", s3LocationArn)+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));++ awsAction(TARGET, "ListLocations", "{}")+ .then()+ .statusCode(200)+ .body("Locations.LocationArn", not(hasItem(s3LocationArn)));+ }++ @Test+ @Order(43)+ void deleteAgentRemovesIt() {+ awsAction(TARGET, "DeleteAgent", "{\"AgentArn\": \"" + agentArn + "\"}")+ .then()+ .statusCode(200);++ awsAction(TARGET, "DescribeAgent", "{\"AgentArn\": \"" + agentArn + "\"}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/datasync/DataSyncServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/datasync/DataSyncServiceTest.javanew file mode 100644index 0000000000..e0ceafcc2d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/datasync/DataSyncServiceTest.java@@ -0,0 +1,448 @@+package io.github.hectorvent.floci.services.datasync;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.datasync.model.DataSyncAgent;+import io.github.hectorvent.floci.services.datasync.model.DataSyncLocation;+import io.github.hectorvent.floci.services.datasync.model.DataSyncLocationType;+import io.github.hectorvent.floci.services.datasync.model.DataSyncTask;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.when;++class DataSyncServiceTest {++ private static final String REGION = "us-east-1";+ private static final String FSX_FILESYSTEM_ARN =+ "arn:aws:fsx:us-east-1:000000000000:file-system/fs-0123456789abcdef0";+ private static final String STORAGE_VIRTUAL_MACHINE_ARN =+ "arn:aws:fsx:us-east-1:000000000000:storage-virtual-machine/fs-0123456789abcdef0/svm-0123456789abcdef0";++ private final ObjectMapper mapper = new ObjectMapper();+ private DataSyncService service;++ @BeforeEach+ void setUp() {+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);+ // A fresh backend per call: agents, locations and tasks are three separate stores,+ // and sharing one would let a location ARN resolve out of the agent store.+ when(storageFactory.create(Mockito.anyString(), Mockito.anyString(), Mockito.any()))+ .thenAnswer(invocation -> AccountAwareStorageBackend.inMemory("000000000000"));+ service = new DataSyncService(storageFactory, new RegionResolver(REGION, "000000000000"), mapper);+ }++ private JsonNode json(String body) {+ try {+ return mapper.readTree(body);+ } catch (Exception e) {+ throw new IllegalArgumentException("test fixture is not valid JSON: " + body, e);+ }+ }++ private DataSyncLocation createLocation(DataSyncLocationType type, String body) {+ return service.createLocation(type, json(body), REGION);+ }++ @Test+ void createAgentIsOnlineAndPublicWithoutAVpcEndpoint() {+ DataSyncAgent agent = service.createAgent(+ json("{\"ActivationKey\": \"AAAAA-1AAAA-BB1CC-DDDDD-EEEEE\", \"AgentName\": \"floci\"}"), REGION);++ assertTrue(agent.getAgentArn().startsWith("arn:aws:datasync:us-east-1:000000000000:agent/agent-"));+ assertEquals(DataSyncService.AGENT_STATUS_ONLINE, agent.getStatus());+ assertEquals("PUBLIC", agent.getEndpointType());+ assertEquals("floci", agent.getName());+ }++ @Test+ void createAgentWithAVpcEndpointReportsPrivateLink() {+ DataSyncAgent agent = service.createAgent(json("""+ {"ActivationKey": "AAAAA-1AAAA-BB1CC-DDDDD-EEEEE", "VpcEndpointId": "vpce-0123456789abcdef0"}+ """), REGION);++ assertEquals("PRIVATE_LINK", agent.getEndpointType());+ assertEquals("vpce-0123456789abcdef0", agent.getVpcEndpointId());+ }++ @Test+ void createAgentWithoutAnActivationKeyIsAnInvalidRequest() {+ AwsException error = assertThrows(AwsException.class,+ () -> service.createAgent(json("{\"AgentName\": \"floci\"}"), REGION));++ assertEquals("InvalidRequestException", error.getErrorCode());+ assertTrue(error.getMessage().contains("ActivationKey"));+ }++ @Test+ void s3LocationDerivesItsUriFromTheBucketArnAndSubdirectory() {+ DataSyncLocation location = createLocation(DataSyncLocationType.S3, """+ {"S3BucketArn": "arn:aws:s3:::floci-bucket", "Subdirectory": "/backups/",+ "S3Config": {"BucketAccessRoleArn": "arn:aws:iam::000000000000:role/datasync"}}+ """);++ assertEquals("s3://floci-bucket/backups", location.getLocationUri());+ assertEquals("STANDARD", location.getConfiguration().path("S3StorageClass").asText());+ }++ @Test+ void s3LocationWithoutASubdirectoryOmitsTheTrailingSlash() {+ DataSyncLocation location = createLocation(DataSyncLocationType.S3, """+ {"S3BucketArn": "arn:aws:s3:::floci-bucket",+ "S3Config": {"BucketAccessRoleArn": "arn:aws:iam::000000000000:role/datasync"}}+ """);++ assertEquals("s3://floci-bucket", location.getLocationUri());+ }++ @Test+ void fsxOntapLocationDerivesTheFileSystemArnFromTheSvmArn() {+ DataSyncLocation location = createLocation(DataSyncLocationType.FSX_ONTAP, """+ {"StorageVirtualMachineArn": "%s", "SecurityGroupArns": ["sg"],+ "Protocol": {"SMB": {"User": "floci", "Password": "sup3rs3cret"}}, "Subdirectory": "/vol1"}+ """.formatted(STORAGE_VIRTUAL_MACHINE_ARN));++ assertEquals("fsxn://us-east-1.fs-0123456789abcdef0.svm-0123456789abcdef0/vol1",+ location.getLocationUri());+ assertEquals(FSX_FILESYSTEM_ARN, location.getConfiguration().path("FsxFilesystemArn").asText());+ }++ @Test+ void nestedProtocolPasswordIsStrippedBeforeTheConfigurationIsStored() {+ DataSyncLocation location = createLocation(DataSyncLocationType.FSX_ONTAP, """+ {"StorageVirtualMachineArn": "%s", "SecurityGroupArns": ["sg"],+ "Protocol": {"SMB": {"User": "floci", "Password": "sup3rs3cret"}}}+ """.formatted(STORAGE_VIRTUAL_MACHINE_ARN));++ JsonNode smb = location.getConfiguration().path("Protocol").path("SMB");+ assertEquals("floci", smb.path("User").asText());+ assertFalse(smb.has("Password"));+ }++ @Test+ void topLevelCredentialMembersAreStrippedBeforeTheConfigurationIsStored() {+ DataSyncLocation location = createLocation(DataSyncLocationType.OBJECT_STORAGE, """+ {"ServerHostname": "objects.example.com", "BucketName": "floci-bucket",+ "AccessKey": "AKIAFLOCI", "SecretKey": "sup3rs3cret"}+ """);++ assertEquals("AKIAFLOCI", location.getConfiguration().path("AccessKey").asText());+ assertFalse(location.getConfiguration().has("SecretKey"));+ }++ @Test+ void objectStorageDefaultsToHttpsOnPort443() {+ DataSyncLocation location = createLocation(DataSyncLocationType.OBJECT_STORAGE, """+ {"ServerHostname": "objects.example.com", "BucketName": "floci-bucket", "Subdirectory": "/incoming"}+ """);++ assertEquals("object-storage://objects.example.com/floci-bucket/incoming", location.getLocationUri());+ assertEquals("HTTPS", location.getConfiguration().path("ServerProtocol").asText());+ assertEquals(443, location.getConfiguration().path("ServerPort").asInt());+ }++ @Test+ void objectStorageOverHttpDefaultsToPort80() {+ DataSyncLocation location = createLocation(DataSyncLocationType.OBJECT_STORAGE, """+ {"ServerHostname": "objects.example.com", "BucketName": "floci-bucket", "ServerProtocol": "HTTP"}+ """);++ assertEquals(80, location.getConfiguration().path("ServerPort").asInt());+ }++ @Test+ void hdfsAppliesTheDocumentedBlockSizeReplicationAndQopDefaults() {+ DataSyncLocation location = createLocation(DataSyncLocationType.HDFS, """+ {"NameNodes": [{"Hostname": "namenode.example.com", "Port": 8020}],+ "AuthenticationType": "SIMPLE", "AgentArns": ["agent"], "Subdirectory": "/user/hadoop"}+ """);++ assertEquals("hdfs://namenode.example.com:8020/user/hadoop", location.getLocationUri());+ JsonNode configuration = location.getConfiguration();+ assertEquals(134217728, configuration.path("BlockSize").asInt());+ assertEquals(3, configuration.path("ReplicationFactor").asInt());+ assertEquals("PRIVACY", configuration.path("QopConfiguration").path("RpcProtection").asText());+ assertEquals("PRIVACY", configuration.path("QopConfiguration").path("DataTransferProtection").asText());+ }++ @Test+ void createLocationWithoutARequiredMemberIsAnInvalidRequest() {+ AwsException error = assertThrows(AwsException.class,+ () -> createLocation(DataSyncLocationType.S3,+ "{\"S3Config\": {\"BucketAccessRoleArn\": \"arn:aws:iam::000000000000:role/datasync\"}}"));++ assertEquals("InvalidRequestException", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertTrue(error.getMessage().contains("S3BucketArn"));+ }++ @Test+ void createLocationFsxWindowsDoesNotRequireAPassword() {+ DataSyncLocation location = createLocation(DataSyncLocationType.FSX_WINDOWS, """+ {"FsxFilesystemArn": "%s", "SecurityGroupArns": ["sg"], "User": "floci", "Subdirectory": "/share"}+ """.formatted(FSX_FILESYSTEM_ARN));++ assertEquals("fsxw://us-east-1.fs-0123456789abcdef0/share", location.getLocationUri());+ }++ @Test+ void updateLocationMergesTheRequestAndRebuildsTheUri() {+ DataSyncLocation location = createLocation(DataSyncLocationType.S3, """+ {"S3BucketArn": "arn:aws:s3:::floci-bucket", "Subdirectory": "/backups",+ "S3Config": {"BucketAccessRoleArn": "arn:aws:iam::000000000000:role/datasync"}}+ """);++ DataSyncLocation updated = service.updateLocation(DataSyncLocationType.S3, json("""+ {"LocationArn": "%s", "Subdirectory": "/archive", "S3StorageClass": "GLACIER"}+ """.formatted(location.getLocationArn())));++ assertEquals("s3://floci-bucket/archive", updated.getLocationUri());+ assertEquals("GLACIER", updated.getConfiguration().path("S3StorageClass").asText());+ assertEquals("arn:aws:iam::000000000000:role/datasync",+ updated.getConfiguration().path("S3Config").path("BucketAccessRoleArn").asText());+ }++ @Test+ void describingALocationAsTheWrongTypeIsAnInvalidRequest() {+ DataSyncLocation location = createLocation(DataSyncLocationType.NFS, """+ {"Subdirectory": "/export", "ServerHostname": "nfs.example.com",+ "OnPremConfig": {"AgentArns": ["agent"]}}+ """);++ AwsException error = assertThrows(AwsException.class,+ () -> service.getLocation(location.getLocationArn(), DataSyncLocationType.S3));++ assertEquals("InvalidRequestException", error.getErrorCode());+ }++ @Test+ void createTaskIsAvailableAndMergesTheRequestedOptionsOverTheDefaults() {+ DataSyncLocation source = createLocation(DataSyncLocationType.NFS, """+ {"Subdirectory": "/export", "ServerHostname": "nfs.example.com",+ "OnPremConfig": {"AgentArns": ["agent"]}}+ """);+ DataSyncLocation destination = createLocation(DataSyncLocationType.S3, """+ {"S3BucketArn": "arn:aws:s3:::floci-bucket",+ "S3Config": {"BucketAccessRoleArn": "arn:aws:iam::000000000000:role/datasync"}}+ """);++ DataSyncTask task = service.createTask(json("""+ {"SourceLocationArn": "%s", "DestinationLocationArn": "%s", "Name": "nightly",+ "Options": {"LogLevel": "TRANSFER", "VerifyMode": "NONE"}}+ """.formatted(source.getLocationArn(), destination.getLocationArn())), REGION);++ assertEquals(DataSyncService.TASK_STATUS_AVAILABLE, task.getStatus());+ assertEquals(DataSyncService.DEFAULT_TASK_MODE, task.getTaskMode());+ assertEquals("TRANSFER", task.getOptions().path("LogLevel").asText());+ assertEquals("NONE", task.getOptions().path("VerifyMode").asText());+ assertEquals("ALWAYS", task.getOptions().path("OverwriteMode").asText());+ assertEquals(-1L, task.getOptions().path("BytesPerSecond").asLong());+ }++ @Test+ void createTaskAgainstAnUnknownLocationIsAnInvalidRequest() {+ AwsException error = assertThrows(AwsException.class, () -> service.createTask(json("""+ {"SourceLocationArn": "arn:aws:datasync:us-east-1:000000000000:location/loc-00000000000000000",+ "DestinationLocationArn": "arn:aws:datasync:us-east-1:000000000000:location/loc-00000000000000001"}+ """), REGION));++ assertEquals("InvalidRequestException", error.getErrorCode());+ }++ private JsonNode taskRequest(String extraMembers) {+ DataSyncLocation source = createLocation(DataSyncLocationType.NFS, """+ {"Subdirectory": "/export", "ServerHostname": "nfs.example.com",+ "OnPremConfig": {"AgentArns": ["agent"]}}+ """);+ DataSyncLocation destination = createLocation(DataSyncLocationType.S3, """+ {"S3BucketArn": "arn:aws:s3:::floci-bucket",+ "S3Config": {"BucketAccessRoleArn": "arn:aws:iam::000000000000:role/datasync"}}+ """);+ return json("""+ {"SourceLocationArn": "%s", "DestinationLocationArn": "%s", "Name": "nightly"%s}+ """.formatted(source.getLocationArn(), destination.getLocationArn(), extraMembers));+ }++ @Test+ void createTaskWithoutATaskModeIsBasicAndVerifiesPointInTime() {+ DataSyncTask task = service.createTask(taskRequest(""), REGION);++ assertEquals(DataSyncService.DEFAULT_TASK_MODE, task.getTaskMode());+ assertEquals(DataSyncService.VERIFY_MODE_POINT_IN_TIME_CONSISTENT,+ task.getOptions().path("VerifyMode").asText());+ }++ @Test+ void createBasicTaskDefaultsToPointInTimeConsistentVerification() {+ DataSyncTask task = service.createTask(taskRequest(", \"TaskMode\": \"BASIC\""), REGION);++ assertEquals("BASIC", task.getTaskMode());+ assertEquals(DataSyncService.VERIFY_MODE_POINT_IN_TIME_CONSISTENT,+ task.getOptions().path("VerifyMode").asText());+ }++ @Test+ void createEnhancedTaskDefaultsToOnlyFilesTransferredVerification() {+ DataSyncTask task = service.createTask(taskRequest(", \"TaskMode\": \"ENHANCED\""), REGION);++ assertEquals(DataSyncService.TASK_MODE_ENHANCED, task.getTaskMode());+ assertEquals(DataSyncService.VERIFY_MODE_ONLY_FILES_TRANSFERRED,+ task.getOptions().path("VerifyMode").asText());+ }++ @Test+ void createBasicTaskKeepsAnExplicitVerifyMode() {+ DataSyncTask task = service.createTask(+ taskRequest(", \"Options\": {\"VerifyMode\": \"ONLY_FILES_TRANSFERRED\"}"), REGION);++ assertEquals(DataSyncService.VERIFY_MODE_ONLY_FILES_TRANSFERRED,+ task.getOptions().path("VerifyMode").asText());+ }++ @Test+ void createEnhancedTaskKeepsAnExplicitVerifyModeItSupports() {+ DataSyncTask task = service.createTask(taskRequest(+ ", \"TaskMode\": \"ENHANCED\", \"Options\": {\"VerifyMode\": \"NONE\"}"), REGION);++ assertEquals("NONE", task.getOptions().path("VerifyMode").asText());+ }++ @Test+ void createEnhancedTaskRejectsPointInTimeConsistentVerification() {+ JsonNode request = taskRequest(+ ", \"TaskMode\": \"ENHANCED\", \"Options\": {\"VerifyMode\": \"POINT_IN_TIME_CONSISTENT\"}");++ AwsException error = assertThrows(AwsException.class, () -> service.createTask(request, REGION));++ assertEquals("InvalidRequestException", error.getErrorCode());+ assertTrue(error.getMessage().contains("POINT_IN_TIME_CONSISTENT"), error.getMessage());+ }++ @Test+ void createTaskRejectsAnUnknownTaskMode() {+ JsonNode request = taskRequest(", \"TaskMode\": \"TURBO\"");++ AwsException error = assertThrows(AwsException.class, () -> service.createTask(request, REGION));++ assertEquals("InvalidRequestException", error.getErrorCode());+ }++ @Test+ void updateTaskKeepsTheVerifyModeDefaultOfTheModeTheTaskWasCreatedWith() {+ DataSyncTask task = service.createTask(taskRequest(", \"TaskMode\": \"ENHANCED\""), REGION);++ DataSyncTask updated = service.updateTask(json("""+ {"TaskArn": "%s", "Options": {"LogLevel": "TRANSFER"}}+ """.formatted(task.getTaskArn())));++ assertEquals(DataSyncService.TASK_MODE_ENHANCED, updated.getTaskMode());+ assertEquals(DataSyncService.VERIFY_MODE_ONLY_FILES_TRANSFERRED,+ updated.getOptions().path("VerifyMode").asText());+ }++ @Test+ void updateTaskRejectsAVerifyModeTheStoredTaskModeDoesNotSupport() {+ DataSyncTask task = service.createTask(taskRequest(", \"TaskMode\": \"ENHANCED\""), REGION);+ JsonNode request = json("""+ {"TaskArn": "%s", "Options": {"VerifyMode": "POINT_IN_TIME_CONSISTENT"}}+ """.formatted(task.getTaskArn()));++ AwsException error = assertThrows(AwsException.class, () -> service.updateTask(request));++ assertEquals("InvalidRequestException", error.getErrorCode());+ }++ @Test+ void listLocationsFiltersByLocationType() {+ DataSyncLocation nfs = createLocation(DataSyncLocationType.NFS, """+ {"Subdirectory": "/export", "ServerHostname": "nfs.example.com",+ "OnPremConfig": {"AgentArns": ["agent"]}}+ """);+ createLocation(DataSyncLocationType.S3, """+ {"S3BucketArn": "arn:aws:s3:::floci-bucket",+ "S3Config": {"BucketAccessRoleArn": "arn:aws:iam::000000000000:role/datasync"}}+ """);++ var page = service.listLocations(+ json("[{\"Name\": \"LocationType\", \"Values\": [\"Nfs\"], \"Operator\": \"Equals\"}]"), null, 0);++ assertEquals(List.of(nfs.getLocationArn()),+ page.items().stream().map(DataSyncLocation::getLocationArn).toList());+ assertNull(page.nextToken());+ }++ @Test+ void listLocationsRejectsAnUnknownFilterName() {+ createLocation(DataSyncLocationType.S3, """+ {"S3BucketArn": "arn:aws:s3:::floci-bucket",+ "S3Config": {"BucketAccessRoleArn": "arn:aws:iam::000000000000:role/datasync"}}+ """);++ AwsException error = assertThrows(AwsException.class, () -> service.listLocations(+ json("[{\"Name\": \"Nope\", \"Values\": [\"x\"]}]"), null, 0));++ assertEquals("InvalidRequestException", error.getErrorCode());+ }++ @Test+ void listAgentsPagesOnMaxResultsAndResumesFromTheNextToken() {+ for (int i = 0; i < 3; i++) {+ service.createAgent(json("{\"ActivationKey\": \"AAAAA-1AAAA-BB1CC-DDDDD-EEEEE\"}"), REGION);+ }++ var first = service.listAgents(null, 2);+ assertEquals(2, first.items().size());+ assertEquals(first.items().get(1).getAgentArn(), first.nextToken());++ var second = service.listAgents(first.nextToken(), 2);+ assertEquals(1, second.items().size());+ assertNull(second.nextToken());+ }++ @Test+ void tagsSurviveTheirCreateAndRoundTripThroughTagAndUntag() {+ DataSyncAgent agent = service.createAgent(json("""+ {"ActivationKey": "AAAAA-1AAAA-BB1CC-DDDDD-EEEEE",+ "Tags": [{"Key": "env", "Value": "test"}]}+ """), REGION);++ assertEquals(Map.of("env", "test"), service.listTagsForResource(agent.getAgentArn()));++ service.tagResource(agent.getAgentArn(), Map.of("owner", "platform"));+ assertEquals("platform", service.listTagsForResource(agent.getAgentArn()).get("owner"));++ service.untagResource(agent.getAgentArn(), List.of("env"));+ assertEquals(Map.of("owner", "platform"), service.listTagsForResource(agent.getAgentArn()));+ }++ @Test+ void taggingAnUnknownResourceIsAnInvalidRequest() {+ AwsException error = assertThrows(AwsException.class, () -> service.listTagsForResource(+ "arn:aws:datasync:us-east-1:000000000000:task/task-00000000000000000"));++ assertEquals("InvalidRequestException", error.getErrorCode());+ }++ @Test+ void deleteAgentRemovesItFromTheStore() {+ DataSyncAgent agent = service.createAgent(+ json("{\"ActivationKey\": \"AAAAA-1AAAA-BB1CC-DDDDD-EEEEE\"}"), REGION);++ service.deleteAgent(agent.getAgentArn());++ assertThrows(AwsException.class, () -> service.getAgent(agent.getAgentArn()));+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 8cb49d903c..418d3f6e0f 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -426,6 +426,8 @@ floci:enabled: trueglobalaccelerator:enabled: true+ datasync:+ enabled: trueaps:enabled: truelakeformation:diff --git a/tools/docs/services.yaml b/tools/docs/services.yamlindex 051f516475..94a49263cc 100644--- a/tools/docs/services.yaml+++ b/tools/docs/services.yaml@@ -289,6 +289,10 @@ services:doc: docs/services/globalaccelerator.mdsources:- { path: src/main/java/io/github/hectorvent/floci/services/globalaccelerator/GlobalAcceleratorJsonHandler.java, mode: switch }+ - service: datasync+ doc: docs/services/datasync.md+ sources:+ - { path: src/main/java/io/github/hectorvent/floci/services/datasync/DataSyncJsonHandler.java, mode: switch }- service: route53doc: docs/services/route53.mdsources: