Light, fluffy, and always free - The AWS Local Emulator alternative
feat(s3tables): emulate metadata service (#2322)
Dixit R Jainb45de1bb7dcd
diff --git a/README.md b/README.mdindex 78a2f3b7f7..b3579ab3eb 100644--- a/README.md+++ b/README.md@@ -223,7 +223,7 @@ Floci supports local emulation for application services, data services, eventing| Events and workflows | EventBridge, EventBridge Pipes, EventBridge Scheduler, Step Functions, SWF, CloudWatch Logs, CloudWatch Metrics || API and identity | API Gateway REST, API Gateway v2, AppSync, Cognito, ACM, Route53, Cloud Map || Containers and compute | ECS, EC2, Lightsail, EKS, MWAA, ECR, CodeBuild, CodeDeploy, CodePipeline, AWS Batch, Auto Scaling, Application Auto Scaling, Elastic Beanstalk, ELB v2 |-| Data, analytics, and AI | Athena, Glue, EMR, Firehose, Managed Service for Apache Flink, OpenSearch, S3 Vectors, Textract, Transcribe, Bedrock Runtime |+| Data, analytics, and AI | Athena, Glue, EMR, Firehose, Managed Service for Apache Flink, OpenSearch, S3 Tables, S3 Vectors, Textract, Transcribe, Bedrock Runtime || Databases and caching | RDS, RDS Data API, Neptune, DocumentDB, MemoryDB, ElastiCache || Messaging and transfer | SES, Kinesis, MSK, Amazon MQ, Transfer Family, IoT Core || Security and governance | WAF v2, CloudTrail, CloudFront, Resource Groups Tagging API |diff --git a/compatibility-tests/sdk-test-java/pom.xml b/compatibility-tests/sdk-test-java/pom.xmlindex 82772be5f7..328b515ab0 100644--- a/compatibility-tests/sdk-test-java/pom.xml+++ b/compatibility-tests/sdk-test-java/pom.xml@@ -369,6 +369,11 @@<groupId>software.amazon.awssdk</groupId><artifactId>s3vectors</artifactId></dependency>+ <!-- S3 Tables client -->+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>s3tables</artifactId>+ </dependency><!-- AppConfig and AppConfigData clients --><dependency><groupId>software.amazon.awssdk</groupId>diff --git a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.javaindex 81c816eb1b..1c984ffe80 100644--- a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java+++ b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java@@ -84,6 +84,7 @@ import software.amazon.awssdk.services.route53.model.RRType;import software.amazon.awssdk.services.route53.model.ResourceRecord;import software.amazon.awssdk.services.route53.model.ResourceRecordSet;import software.amazon.awssdk.services.s3vectors.S3VectorsClient;+import software.amazon.awssdk.services.s3tables.S3TablesClient;import software.amazon.awssdk.core.SdkBytes;import software.amazon.awssdk.services.lambda.model.CreateFunctionRequest;@@ -922,4 +923,12 @@ public final class TestFixtures {.credentialsProvider(CREDENTIALS).build();}++ public static S3TablesClient s3tablesClient() {+ return S3TablesClient.builder()+ .endpointOverride(ENDPOINT)+ .region(REGION)+ .credentialsProvider(CREDENTIALS)+ .build();+ }}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3TablesTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3TablesTest.javanew file mode 100644index 0000000000..1bc2814d27--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3TablesTest.java@@ -0,0 +1,233 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;+import software.amazon.awssdk.services.s3tables.S3TablesClient;+import software.amazon.awssdk.services.s3tables.model.*;++import java.util.Map;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class S3TablesTest {+ private static S3TablesClient client;+ private static String bucketName;+ private static String bucketArn;+ private static String tableName;+ private static String versionToken;+ private static final String NAMESPACE = "analytics";++ @BeforeAll+ static void setup() {+ client = TestFixtures.s3tablesClient();+ bucketName = TestFixtures.uniqueName("table-bucket");+ tableName = TestFixtures.uniqueName("events");+ }++ @AfterAll+ static void cleanup() {+ if (client == null) {+ return;+ }+ if (bucketArn != null) {+ cleanup("table " + tableName, () -> client.deleteTable(DeleteTableRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build()));+ cleanup("namespace " + NAMESPACE, () -> client.deleteNamespace(DeleteNamespaceRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).build()));+ cleanup("table bucket " + bucketArn, () -> client.deleteTableBucket(DeleteTableBucketRequest.builder()+ .tableBucketARN(bucketArn).build()));+ }+ client.close();+ }++ private static void cleanup(String resource, Runnable deletion) {+ try {+ deletion.run();+ } catch (Exception exception) {+ System.err.printf("S3 Tables SDK test cleanup failed for %s: %s%n", resource, exception.getMessage());+ }+ }++ @Test+ @Order(1)+ void createsAndGetsTableBucket() {+ CreateTableBucketResponse created = client.createTableBucket(+ CreateTableBucketRequest.builder()+ .name(bucketName)+ .tags(Map.of("suite", "sdk-java"))+ .encryptionConfiguration(EncryptionConfiguration.builder().sseAlgorithm(SSEAlgorithm.AES256).build())+ .storageClassConfiguration(StorageClassConfiguration.builder().storageClass(StorageClass.STANDARD).build())+ .build());+ bucketArn = created.arn();+ assertThat(bucketArn).contains("arn:aws:s3tables:").endsWith("bucket/" + bucketName);++ GetTableBucketResponse fetched = client.getTableBucket(+ GetTableBucketRequest.builder().tableBucketARN(bucketArn).build());+ assertThat(fetched.name()).isEqualTo(bucketName);+ assertThat(fetched.arn()).isEqualTo(bucketArn);+ }++ @Test+ @Order(2)+ void createsNamespaceAndTable() {+ CreateNamespaceResponse namespace = client.createNamespace(+ CreateNamespaceRequest.builder().tableBucketARN(bucketArn).namespace(NAMESPACE).build());+ assertThat(namespace.namespace()).containsExactly(NAMESPACE);++ CreateTableResponse table = client.createTable(CreateTableRequest.builder()+ .tableBucketARN(bucketArn)+ .namespace(NAMESPACE)+ .name(tableName)+ .format(OpenTableFormat.ICEBERG)+ .metadata(TableMetadata.builder().iceberg(IcebergMetadata.builder()+ .properties(Map.of("format-version", "2")).build()).build())+ .encryptionConfiguration(EncryptionConfiguration.builder().sseAlgorithm(SSEAlgorithm.AES256).build())+ .storageClassConfiguration(StorageClassConfiguration.builder().storageClass(StorageClass.STANDARD).build())+ .tags(Map.of("suite", "sdk-java"))+ .build());+ versionToken = table.versionToken();+ assertThat(table.tableARN()).isEqualTo(bucketArn + "/table/" + tableName);+ assertThat(versionToken).isNotBlank();++ GetTableResponse fetched = client.getTable(GetTableRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build());+ assertThat(fetched.tableARN()).isEqualTo(table.tableARN());+ assertThat(fetched.namespace()).containsExactly(NAMESPACE);+ assertThat(fetched.format()).isEqualTo(OpenTableFormat.ICEBERG);+ }++ @Test+ @Order(3)+ void getsAndListsResourcesWithPrefixes() {+ assertThat(client.listTableBuckets(ListTableBucketsRequest.builder().prefix("table-bucket").build()).tableBuckets())+ .anyMatch(bucket -> bucketName.equals(bucket.name()));+ assertThat(client.getNamespace(GetNamespaceRequest.builder().tableBucketARN(bucketArn).namespace(NAMESPACE).build())+ .namespace()).containsExactly(NAMESPACE);+ assertThat(client.listNamespaces(ListNamespacesRequest.builder().tableBucketARN(bucketArn).prefix("anal").build())+ .namespaces()).anyMatch(namespace -> namespace.namespace().contains(NAMESPACE));+ assertThat(client.listTables(ListTablesRequest.builder().tableBucketARN(bucketArn).namespace(NAMESPACE)+ .prefix("events").build()).tables())+ .anyMatch(table -> tableName.equals(table.name()));+ }++ @Test+ @Order(4)+ void updatesAndGetsMetadataLocationAndRejectsStaleVersionToken() {+ UpdateTableMetadataLocationResponse updated = client.updateTableMetadataLocation(+ UpdateTableMetadataLocationRequest.builder()+ .tableBucketARN(bucketArn)+ .namespace(NAMESPACE)+ .name(tableName)+ .metadataLocation("s3://warehouse/" + tableName + "/v2.metadata.json")+ .versionToken(versionToken)+ .build());+ assertThat(updated.metadataLocation()).endsWith("v2.metadata.json");+ assertThat(updated.versionToken()).isNotEqualTo(versionToken);+ assertThat(client.getTableMetadataLocation(GetTableMetadataLocationRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build())+ .metadataLocation()).isEqualTo(updated.metadataLocation());++ assertThatThrownBy(() -> client.updateTableMetadataLocation(+ UpdateTableMetadataLocationRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName)+ .metadataLocation("s3://warehouse/stale.metadata.json")+ .versionToken(versionToken).build()))+ .isInstanceOf(ConflictException.class);+ versionToken = updated.versionToken();+ }++ @Test+ @Order(5)+ void roundTripsAndDeletesBucketAndTablePolicies() {+ String policy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}";+ client.putTableBucketPolicy(PutTableBucketPolicyRequest.builder()+ .tableBucketARN(bucketArn).resourcePolicy(policy).build());+ assertThat(client.getTableBucketPolicy(GetTableBucketPolicyRequest.builder().tableBucketARN(bucketArn).build())+ .resourcePolicy()).isEqualTo(policy);++ client.putTablePolicy(PutTablePolicyRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).resourcePolicy(policy).build());+ assertThat(client.getTablePolicy(GetTablePolicyRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build()).resourcePolicy()).isEqualTo(policy);+ }++ @Test+ @Order(6)+ void roundTripsTypedMaintenanceConfigurations() {+ TableBucketMaintenanceConfigurationValue bucketValue = TableBucketMaintenanceConfigurationValue.builder()+ .status(MaintenanceStatus.ENABLED).build();+ client.putTableBucketMaintenanceConfiguration(PutTableBucketMaintenanceConfigurationRequest.builder()+ .tableBucketARN(bucketArn)+ .type(TableBucketMaintenanceType.ICEBERG_UNREFERENCED_FILE_REMOVAL)+ .value(bucketValue)+ .build());+ assertThat(client.getTableBucketMaintenanceConfiguration(GetTableBucketMaintenanceConfigurationRequest.builder()+ .tableBucketARN(bucketArn).build())+ .configuration().get(TableBucketMaintenanceType.ICEBERG_UNREFERENCED_FILE_REMOVAL).status())+ .isEqualTo(MaintenanceStatus.ENABLED);++ TableMaintenanceConfigurationValue tableValue = TableMaintenanceConfigurationValue.builder()+ .status(MaintenanceStatus.ENABLED).build();+ client.putTableMaintenanceConfiguration(PutTableMaintenanceConfigurationRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName)+ .type(TableMaintenanceType.ICEBERG_COMPACTION)+ .value(tableValue)+ .build());+ assertThat(client.getTableMaintenanceConfiguration(GetTableMaintenanceConfigurationRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build())+ .configuration().get(TableMaintenanceType.ICEBERG_COMPACTION).status())+ .isEqualTo(MaintenanceStatus.ENABLED);+ }++ @Test+ @Order(7)+ void renamesTableAndRotatesItsVersionToken() {+ String originalToken = versionToken;+ String renamedTableName = tableName + "-renamed";+ client.renameTable(RenameTableRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName)+ .newName(renamedTableName).versionToken(originalToken).build());++ GetTableResponse renamed = client.getTable(GetTableRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(renamedTableName).build());+ assertThat(renamed.name()).isEqualTo(renamedTableName);+ assertThat(renamed.versionToken()).isNotEqualTo(originalToken);+ assertThatThrownBy(() -> client.getTable(GetTableRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build()))+ .isInstanceOf(NotFoundException.class);+ tableName = renamedTableName;+ versionToken = renamed.versionToken();+ }++ @Test+ @Order(8)+ void deletesPoliciesAndResourceHierarchy() {+ client.deleteTablePolicy(DeleteTablePolicyRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build());+ assertThatThrownBy(() -> client.getTablePolicy(GetTablePolicyRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build()))+ .isInstanceOf(NotFoundException.class);++ client.deleteTableBucketPolicy(DeleteTableBucketPolicyRequest.builder().tableBucketARN(bucketArn).build());+ assertThatThrownBy(() -> client.getTableBucketPolicy(GetTableBucketPolicyRequest.builder().tableBucketARN(bucketArn).build()))+ .isInstanceOf(NotFoundException.class);++ client.deleteTable(DeleteTableRequest.builder().tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build());+ assertThatThrownBy(() -> client.getTable(GetTableRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).name(tableName).build()))+ .isInstanceOf(NotFoundException.class);+ client.deleteNamespace(DeleteNamespaceRequest.builder().tableBucketARN(bucketArn).namespace(NAMESPACE).build());+ assertThatThrownBy(() -> client.getNamespace(GetNamespaceRequest.builder()+ .tableBucketARN(bucketArn).namespace(NAMESPACE).build()))+ .isInstanceOf(NotFoundException.class);+ client.deleteTableBucket(DeleteTableBucketRequest.builder().tableBucketARN(bucketArn).build());+ assertThatThrownBy(() -> client.getTableBucket(GetTableBucketRequest.builder().tableBucketARN(bucketArn).build()))+ .isInstanceOf(NotFoundException.class);+ }+}diff --git a/compatibility-tests/sdk-test-node/package.json b/compatibility-tests/sdk-test-node/package.jsonindex fd9bb1faf5..a463e820f5 100644--- a/compatibility-tests/sdk-test-node/package.json+++ b/compatibility-tests/sdk-test-node/package.json@@ -21,13 +21,14 @@"@aws-sdk/client-kinesis": "^3.500.0","@aws-sdk/client-kms": "^3.500.0","@aws-sdk/client-lambda": "^3.500.0",+ "@aws-sdk/client-neptune": "^3.500.0","@aws-sdk/client-pipes": "^3.500.0","@aws-sdk/client-s3": "^3.500.0",+ "@aws-sdk/client-s3tables": "3.1111.0","@aws-sdk/client-secrets-manager": "^3.500.0","@aws-sdk/client-sns": "^3.500.0","@aws-sdk/client-sqs": "^3.500.0","@aws-sdk/client-ssm": "^3.500.0",- "@aws-sdk/client-neptune": "^3.500.0","@aws-sdk/client-sts": "^3.500.0","selfsigned": "^5.5.0","ws": "^8.0.0"diff --git a/compatibility-tests/sdk-test-node/tests/s3tables.test.ts b/compatibility-tests/sdk-test-node/tests/s3tables.test.tsnew file mode 100644index 0000000000..beebd4ce7a--- /dev/null+++ b/compatibility-tests/sdk-test-node/tests/s3tables.test.ts@@ -0,0 +1,198 @@+import { afterAll, beforeAll, describe, expect, it } from 'vitest';+import {+ ConflictException,+ CreateNamespaceCommand,+ CreateTableBucketCommand,+ CreateTableCommand,+ DeleteNamespaceCommand,+ DeleteTableBucketCommand,+ DeleteTableBucketPolicyCommand,+ DeleteTableCommand,+ DeleteTablePolicyCommand,+ GetNamespaceCommand,+ GetTableBucketCommand,+ GetTableBucketMaintenanceConfigurationCommand,+ GetTableBucketPolicyCommand,+ GetTableCommand,+ GetTableMaintenanceConfigurationCommand,+ GetTableMetadataLocationCommand,+ GetTablePolicyCommand,+ ListNamespacesCommand,+ ListTableBucketsCommand,+ ListTablesCommand,+ NotFoundException,+ PutTableBucketMaintenanceConfigurationCommand,+ PutTableBucketPolicyCommand,+ PutTableMaintenanceConfigurationCommand,+ PutTablePolicyCommand,+ RenameTableCommand,+ S3TablesClient,+ UpdateTableMetadataLocationCommand,+} from '@aws-sdk/client-s3tables';+import { makeClient, uniqueName } from './setup';++describe('S3 Tables lifecycle', () => {+ let client: S3TablesClient;+ let bucketArn: string;+ const bucketName = uniqueName('table-bucket');+ const namespace = 'analytics';+ let tableName = uniqueName('events');+ let versionToken: string;++ beforeAll(() => {+ client = makeClient(S3TablesClient);+ });++ afterAll(async () => {+ if (!bucketArn) return;+ await client.send(new DeleteTableCommand({ tableBucketARN: bucketArn, namespace, name: tableName })).catch(() => undefined);+ await client.send(new DeleteNamespaceCommand({ tableBucketARN: bucketArn, namespace })).catch(() => undefined);+ await client.send(new DeleteTableBucketCommand({ tableBucketARN: bucketArn })).catch(() => undefined);+ });++ it('creates a table bucket, namespace, and Iceberg table', async () => {+ const bucket = await client.send(new CreateTableBucketCommand({+ name: bucketName,+ tags: { suite: 'sdk-node' },+ encryptionConfiguration: { sseAlgorithm: 'AES256' },+ storageClassConfiguration: { storageClass: 'STANDARD' },+ }));+ bucketArn = bucket.arn!;+ expect(bucketArn).toContain(`bucket/${bucketName}`);++ const createdNamespace = await client.send(new CreateNamespaceCommand({+ tableBucketARN: bucketArn,+ namespace: [namespace],+ }));+ expect(createdNamespace.namespace).toEqual([namespace]);++ const table = await client.send(new CreateTableCommand({+ tableBucketARN: bucketArn,+ namespace,+ name: tableName,+ format: 'ICEBERG',+ metadata: { iceberg: { properties: { 'format-version': '2' } } },+ encryptionConfiguration: { sseAlgorithm: 'AES256' },+ storageClassConfiguration: { storageClass: 'STANDARD' },+ tags: { suite: 'sdk-node' },+ }));+ versionToken = table.versionToken!;+ expect(table.tableARN).toBe(`${bucketArn}/table/${tableName}`);+ });++ it('gets resources and applies list prefixes', async () => {+ const bucket = await client.send(new GetTableBucketCommand({ tableBucketARN: bucketArn }));+ expect(bucket.name).toBe(bucketName);++ const buckets = await client.send(new ListTableBucketsCommand({ prefix: 'table-bucket' }));+ expect(buckets.tableBuckets?.some((item) => item.name === bucketName)).toBe(true);++ const fetchedNamespace = await client.send(new GetNamespaceCommand({ tableBucketARN: bucketArn, namespace }));+ expect(fetchedNamespace.namespace).toEqual([namespace]);+ const namespaces = await client.send(new ListNamespacesCommand({ tableBucketARN: bucketArn, prefix: 'anal' }));+ expect(namespaces.namespaces?.some((item) => item.namespace?.includes(namespace))).toBe(true);++ const tables = await client.send(new ListTablesCommand({ tableBucketARN: bucketArn, namespace, prefix: 'events' }));+ expect(tables.tables?.some((table) => table.name === tableName)).toBe(true);+ const table = await client.send(new GetTableCommand({ tableBucketARN: bucketArn, namespace, name: tableName }));+ expect(table.tableARN).toBe(`${bucketArn}/table/${tableName}`);+ });++ it('round-trips policies and metadata version tokens', async () => {+ const policy = '{"Version":"2012-10-17","Statement":[]}';+ await client.send(new PutTableBucketPolicyCommand({ tableBucketARN: bucketArn, resourcePolicy: policy }));+ expect((await client.send(new GetTableBucketPolicyCommand({ tableBucketARN: bucketArn }))).resourcePolicy).toBe(policy);+ await client.send(new PutTablePolicyCommand({ tableBucketARN: bucketArn, namespace, name: tableName, resourcePolicy: policy }));+ expect((await client.send(new GetTablePolicyCommand({ tableBucketARN: bucketArn, namespace, name: tableName }))).resourcePolicy).toBe(policy);++ const updated = await client.send(new UpdateTableMetadataLocationCommand({+ tableBucketARN: bucketArn,+ namespace,+ name: tableName,+ metadataLocation: `s3://warehouse/${tableName}/v2.metadata.json`,+ versionToken,+ }));+ expect(updated.versionToken).not.toBe(versionToken);+ expect((await client.send(new GetTableMetadataLocationCommand({+ tableBucketARN: bucketArn, namespace, name: tableName,+ }))).metadataLocation).toBe(updated.metadataLocation);++ await expect(client.send(new UpdateTableMetadataLocationCommand({+ tableBucketARN: bucketArn,+ namespace,+ name: tableName,+ metadataLocation: `s3://warehouse/${tableName}/stale.metadata.json`,+ versionToken,+ }))).rejects.toBeInstanceOf(ConflictException);+ versionToken = updated.versionToken!;+ });++ it('round-trips typed maintenance configuration maps', async () => {+ await client.send(new PutTableBucketMaintenanceConfigurationCommand({+ tableBucketARN: bucketArn,+ type: 'icebergUnreferencedFileRemoval',+ value: { status: 'enabled' },+ }));+ const bucketMaintenance = await client.send(new GetTableBucketMaintenanceConfigurationCommand({+ tableBucketARN: bucketArn,+ }));+ expect(bucketMaintenance.configuration?.icebergUnreferencedFileRemoval?.status).toBe('enabled');++ await client.send(new PutTableMaintenanceConfigurationCommand({+ tableBucketARN: bucketArn,+ namespace,+ name: tableName,+ type: 'icebergCompaction',+ value: { status: 'enabled' },+ }));+ const tableMaintenance = await client.send(new GetTableMaintenanceConfigurationCommand({+ tableBucketARN: bucketArn,+ namespace,+ name: tableName,+ }));+ expect(tableMaintenance.configuration?.icebergCompaction?.status).toBe('enabled');+ });++ it('renames a table and rotates its version token', async () => {+ const originalToken = versionToken;+ const renamedTableName = `${tableName}-renamed`;+ await client.send(new RenameTableCommand({+ tableBucketARN: bucketArn,+ namespace,+ name: tableName,+ newName: renamedTableName,+ versionToken: originalToken,+ }));++ const renamed = await client.send(new GetTableCommand({+ tableBucketARN: bucketArn,+ namespace,+ name: renamedTableName,+ }));+ expect(renamed.name).toBe(renamedTableName);+ expect(renamed.versionToken).not.toBe(originalToken);+ await expect(client.send(new GetTableCommand({ tableBucketARN: bucketArn, namespace, name: tableName })))+ .rejects.toBeInstanceOf(NotFoundException);+ tableName = renamedTableName;+ versionToken = renamed.versionToken!;+ });++ it('deletes policies and the resource hierarchy', async () => {+ await client.send(new DeleteTablePolicyCommand({ tableBucketARN: bucketArn, namespace, name: tableName }));+ await expect(client.send(new GetTablePolicyCommand({ tableBucketARN: bucketArn, namespace, name: tableName })))+ .rejects.toBeInstanceOf(NotFoundException);+ await client.send(new DeleteTableBucketPolicyCommand({ tableBucketARN: bucketArn }));+ await expect(client.send(new GetTableBucketPolicyCommand({ tableBucketARN: bucketArn })))+ .rejects.toBeInstanceOf(NotFoundException);++ await client.send(new DeleteTableCommand({ tableBucketARN: bucketArn, namespace, name: tableName }));+ await expect(client.send(new GetTableCommand({ tableBucketARN: bucketArn, namespace, name: tableName })))+ .rejects.toBeInstanceOf(NotFoundException);+ await client.send(new DeleteNamespaceCommand({ tableBucketARN: bucketArn, namespace }));+ await expect(client.send(new GetNamespaceCommand({ tableBucketARN: bucketArn, namespace })))+ .rejects.toBeInstanceOf(NotFoundException);+ await client.send(new DeleteTableBucketCommand({ tableBucketARN: bucketArn }));+ await expect(client.send(new GetTableBucketCommand({ tableBucketARN: bucketArn })))+ .rejects.toBeInstanceOf(NotFoundException);+ });+});diff --git a/docs/services/index.md b/docs/services/index.mdindex f55d22aad9..508ffc9812 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -15,6 +15,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [SNS](sns.md) | `POST /` with `Action=` param | Query / JSON | 17 || [S3](s3.md) | `/{bucket}/{key}` | REST XML | 58 || [S3 Vectors](s3vectors.md) | `POST /{OperationName}` | REST JSON | 12 |+| [S3 Tables](s3tables.md) | `/buckets`, `/namespaces/*`, `/tables/*` | REST JSON | 25 || [DynamoDB](dynamodb.md) | `POST /` + `X-Amz-Target: DynamoDB_20120810.*` | JSON 1.1 | 28 || [DynamoDB Streams](dynamodb.md#streams) | `POST /` + `X-Amz-Target: DynamoDBStreams_20120810.*` | JSON 1.1 | 4 || [Lambda](lambda.md) | `/2015-03-31/functions/...` | REST JSON | 30 |diff --git a/docs/services/s3tables.md b/docs/services/s3tables.mdnew file mode 100644index 0000000000..5060235717--- /dev/null+++ b/docs/services/s3tables.md@@ -0,0 +1,60 @@+# S3 Tables++**Protocol:** REST JSON+**Signing service:** `s3tables`+**Endpoint:** `http://localhost:4566`++Floci emulates the S3 Tables control plane for local development, Terraform+convergence, and SDK integration tests. It is a separate AWS service from S3:+S3 Tables requests are signed as `s3tables` and use REST JSON routes such as+`/buckets` and `/tables/{tableBucketARN}/{namespace}`.++## Metadata-only scope++This service stores table-bucket, namespace, table, policy, maintenance+configuration, and metadata-location state. It does **not** run an Apache+Iceberg engine or create data-plane artifacts: manifests, warehouse objects,+queries, compaction, snapshot expiry, replication, encryption-management, and+metrics are intentionally outside this implementation.++## Supported operations++| Area | Operations |+|---|---|+| Table buckets | CreateTableBucket, GetTableBucket, ListTableBuckets, DeleteTableBucket |+| Bucket policy | PutTableBucketPolicy, GetTableBucketPolicy, DeleteTableBucketPolicy |+| Bucket maintenance | PutTableBucketMaintenanceConfiguration, GetTableBucketMaintenanceConfiguration |+| Namespaces | CreateNamespace, GetNamespace, ListNamespaces, DeleteNamespace |+| Tables | CreateTable, GetTable, ListTables, DeleteTable, RenameTable |+| Metadata location | GetTableMetadataLocation, UpdateTableMetadataLocation |+| Table policy | PutTablePolicy, GetTablePolicy, DeleteTablePolicy |+| Table maintenance | PutTableMaintenanceConfiguration, GetTableMaintenanceConfiguration |++Tables must use `ICEBERG` format. `UpdateTableMetadataLocation` implements+optimistic concurrency: supply the table's current `versionToken`; a stale+token returns `ConflictException`.++## Example++```bash+export AWS_ENDPOINT_URL=http://localhost:4566+export AWS_DEFAULT_REGION=us-east-1+export AWS_ACCESS_KEY_ID=test+export AWS_SECRET_ACCESS_KEY=test++bucket_arn=$(aws s3tables create-table-bucket --name local-warehouse --query arn --output text)+aws s3tables create-namespace --table-bucket-arn "$bucket_arn" --namespace analytics+aws s3tables create-table \+ --table-bucket-arn "$bucket_arn" \+ --namespace analytics \+ --name events \+ --format ICEBERG+```++## Configuration++| Variable | Default | Description |+|---|---|---|+| `FLOCI_SERVICES_S3TABLES_ENABLED` | `true` | Enables the S3 Tables REST JSON service. |+| `FLOCI_STORAGE_SERVICES_S3TABLES_MODE` | Global storage mode | Optional per-service storage-mode override. |+| `FLOCI_STORAGE_SERVICES_S3TABLES_FLUSH_INTERVAL_MS` | `5000` | Persistence flush interval for hybrid storage. |diff --git a/mkdocs.yml b/mkdocs.ymlindex 015029f406..ba581cbe8e 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -79,6 +79,7 @@ nav:- SNS: services/sns.md- SES: services/ses.md- S3: services/s3.md+ - S3 Tables: services/s3tables.md- DynamoDB: services/dynamodb.md- Lambda: services/lambda.md- Lambda MicroVMs: services/lambda-microvms.mddiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 3704be00fd..4b4b75016e 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -272,6 +272,7 @@ public interface EmulatorConfig {LightsailStorageConfig lightsail();CodePipelineStorageConfig codepipeline();S3VectorsStorageConfig s3vectors();+ S3TablesStorageConfig s3tables();EcsStorageConfig ecs();CodeBuildStorageConfig codebuild();ConfigStorageConfig config();@@ -440,6 +441,13 @@ public interface EmulatorConfig {long flushIntervalMs();}+ interface S3TablesStorageConfig {+ Optional<String> mode();++ @WithDefault("5000")+ long flushIntervalMs();+ }+interface EcsStorageConfig {Optional<String> mode();@@ -595,6 +603,7 @@ public interface EmulatorConfig {LightsailServiceConfig lightsail();UiServiceConfig ui();S3VectorsServiceConfig s3vectors();+ S3TablesServiceConfig s3tables();IotServiceConfig iot();IotDataServiceConfig iotdata();RumServiceConfig rum();@@ -645,6 +654,11 @@ public interface EmulatorConfig {boolean enabled();}+ interface S3TablesServiceConfig {+ @WithDefault("true")+ boolean enabled();+ }+interface TransferServiceConfig {@WithDefault("true")boolean enabled();diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.javaindex b5d9384ac0..fd8e6f5428 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java@@ -25,6 +25,7 @@ import io.github.hectorvent.floci.services.appsync.AppSyncController;import io.github.hectorvent.floci.services.rdsdata.RdsDataController;import io.github.hectorvent.floci.services.rum.RumController;import io.github.hectorvent.floci.services.s3vectors.S3VectorsController;+import io.github.hectorvent.floci.services.s3tables.S3TablesController;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -417,6 +418,11 @@ public class ResolvedServiceCatalog {config.storage().services().s3vectors().flushIntervalMs(), null, ServiceProtocol.REST_JSON,protocols(ServiceProtocol.REST_JSON),Set.of(), Set.of("s3vectors"), Set.of(), Set.of(S3VectorsController.class)),+ descriptor("s3tables", "s3tables", config.services().s3tables().enabled(), true,+ "s3tables", storageMode(config.storage().services().s3tables().mode(), config.storage().mode()),+ config.storage().services().s3tables().flushIntervalMs(), null, ServiceProtocol.REST_JSON,+ protocols(ServiceProtocol.REST_JSON),+ Set.of(), Set.of("s3tables"), Set.of(), Set.of(S3TablesController.class)),descriptor("iot", "iot", config.services().iot().enabled(), true,"iot", config.storage().mode(), 5000L, null, ServiceProtocol.REST_JSON,protocols(ServiceProtocol.REST_JSON),diff --git a/src/main/java/io/github/hectorvent/floci/services/s3tables/S3TablesController.java b/src/main/java/io/github/hectorvent/floci/services/s3tables/S3TablesController.javanew file mode 100644index 0000000000..5699ce0f8a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3tables/S3TablesController.java@@ -0,0 +1,325 @@+package io.github.hectorvent.floci.services.s3tables;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.s3tables.model.Namespace;+import io.github.hectorvent.floci.services.s3tables.model.S3Table;+import io.github.hectorvent.floci.services.s3tables.model.TableBucket;+import jakarta.inject.Inject;+import jakarta.ws.rs.Consumes;+import jakarta.ws.rs.DELETE;+import jakarta.ws.rs.Encoded;+import jakarta.ws.rs.GET;+import jakarta.ws.rs.PUT;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.PathParam;+import jakarta.ws.rs.Produces;+import jakarta.ws.rs.QueryParam;+import jakarta.ws.rs.core.Context;+import jakarta.ws.rs.core.HttpHeaders;+import jakarta.ws.rs.core.MediaType;+import jakarta.ws.rs.core.Response;++import java.net.URLDecoder;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++@Path("/")+@Produces(MediaType.APPLICATION_JSON)+@Consumes(MediaType.APPLICATION_JSON)+public class S3TablesController {+ private final S3TablesService service;+ private final RegionResolver regionResolver;++ @Inject+ public S3TablesController(S3TablesService service, RegionResolver regionResolver) {+ this.service = service;+ this.regionResolver = regionResolver;+ }++ @PUT+ @Path("/buckets")+ public Response createTableBucket(@Context HttpHeaders headers, Map<String, Object> request) {+ TableBucket bucket = service.createTableBucket(string(request, "name"), request.get("encryptionConfiguration"),+ request.get("storageClassConfiguration"), stringMap(request.get("tags")), region(headers));+ return Response.ok(Map.of("arn", bucket.getArn())).build();+ }++ @GET+ @Path("/buckets")+ public Response listTableBuckets(@Context HttpHeaders headers, @QueryParam("prefix") String prefix) {+ List<Map<String, Object>> buckets = service.listTableBuckets(prefix, region(headers)).stream()+ .map(this::bucketRepresentation).toList();+ return Response.ok(Map.of("tableBuckets", buckets)).build();+ }++ @GET+ @Path("/buckets/{tableBucketARN}")+ public Response getTableBucket(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn) {+ return Response.ok(bucketRepresentation(service.getTableBucket(decode(tableBucketArn), region(headers)))).build();+ }++ @DELETE+ @Path("/buckets/{tableBucketARN}")+ public Response deleteTableBucket(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn) {+ service.deleteTableBucket(decode(tableBucketArn), region(headers));+ return Response.noContent().build();+ }++ @PUT+ @Path("/buckets/{tableBucketARN}/policy")+ public Response putTableBucketPolicy(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ Map<String, Object> request) {+ service.putTableBucketPolicy(decode(tableBucketArn), string(request, "resourcePolicy"), region(headers));+ return Response.ok().build();+ }++ @GET+ @Path("/buckets/{tableBucketARN}/policy")+ public Response getTableBucketPolicy(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn) {+ return Response.ok(Map.of("resourcePolicy", service.getTableBucketPolicy(decode(tableBucketArn), region(headers)))).build();+ }++ @DELETE+ @Path("/buckets/{tableBucketARN}/policy")+ public Response deleteTableBucketPolicy(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn) {+ service.deleteTableBucketPolicy(decode(tableBucketArn), region(headers));+ return Response.noContent().build();+ }++ @PUT+ @Path("/buckets/{tableBucketARN}/maintenance/{type}")+ public Response putTableBucketMaintenanceConfiguration(@Context HttpHeaders headers,+ @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("type") String type, Map<String, Object> request) {+ service.putTableBucketMaintenance(decode(tableBucketArn), type, request.get("value"), region(headers));+ return Response.ok().build();+ }++ @GET+ @Path("/buckets/{tableBucketARN}/maintenance")+ public Response getTableBucketMaintenanceConfiguration(@Context HttpHeaders headers,+ @Encoded @PathParam("tableBucketARN") String tableBucketArn) {+ String arn = decode(tableBucketArn);+ return Response.ok(Map.of("tableBucketARN", arn,+ "configuration", service.getTableBucketMaintenanceConfigurations(arn, region(headers)))).build();+ }++ @PUT+ @Path("/namespaces/{tableBucketARN}")+ public Response createNamespace(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ Map<String, Object> request) {+ String arn = decode(tableBucketArn);+ Namespace namespace = service.createNamespace(arn, stringList(request.get("namespace")), region(headers));+ return Response.ok(namespaceRepresentation(arn, namespace)).build();+ }++ @GET+ @Path("/namespaces/{tableBucketARN}")+ public Response listNamespaces(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @QueryParam("prefix") String prefix) {+ String arn = decode(tableBucketArn);+ List<Map<String, Object>> namespaces = service.listNamespaces(arn, prefix, region(headers)).stream()+ .map(namespace -> namespaceRepresentation(arn, namespace)).toList();+ return Response.ok(Map.of("namespaces", namespaces)).build();+ }++ @GET+ @Path("/namespaces/{tableBucketARN}/{namespace}")+ public Response getNamespace(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace) {+ String arn = decode(tableBucketArn);+ return Response.ok(namespaceRepresentation(arn, service.getNamespace(arn, namespace, region(headers)))).build();+ }++ @DELETE+ @Path("/namespaces/{tableBucketARN}/{namespace}")+ public Response deleteNamespace(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace) {+ service.deleteNamespace(decode(tableBucketArn), namespace, region(headers));+ return Response.noContent().build();+ }++ @PUT+ @Path("/tables/{tableBucketARN}/{namespace}")+ public Response createTable(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, Map<String, Object> request) {+ S3Table table = service.createTable(decode(tableBucketArn), namespace, string(request, "name"), string(request, "format"),+ request.get("metadata"), request.get("encryptionConfiguration"), request.get("storageClassConfiguration"),+ stringMap(request.get("tags")), region(headers));+ return Response.ok(Map.of("tableARN", table.getArn(), "versionToken", table.getVersionToken())).build();+ }++ @GET+ @Path("/get-table")+ public Response getTable(@Context HttpHeaders headers, @QueryParam("tableBucketARN") String tableBucketArn,+ @QueryParam("namespace") String namespace, @QueryParam("name") String name) {+ return Response.ok(tableRepresentation(service.getTable(decode(tableBucketArn), namespace, name, region(headers)))).build();+ }++ @GET+ @Path("/tables/{tableBucketARN}")+ public Response listTables(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @QueryParam("namespace") String namespace, @QueryParam("prefix") String prefix) {+ List<Map<String, Object>> tables = service.listTables(decode(tableBucketArn), namespace, prefix, region(headers)).stream()+ .map(this::tableRepresentation).toList();+ return Response.ok(Map.of("tables", tables)).build();+ }++ @DELETE+ @Path("/tables/{tableBucketARN}/{namespace}/{name}")+ public Response deleteTable(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name) {+ service.deleteTable(decode(tableBucketArn), namespace, name, region(headers));+ return Response.noContent().build();+ }++ @PUT+ @Path("/tables/{tableBucketARN}/{namespace}/{name}/rename")+ public Response renameTable(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name,+ Map<String, Object> request) {+ service.renameTable(decode(tableBucketArn), namespace, name, string(request, "newNamespaceName"),+ string(request, "newName"), string(request, "versionToken"), region(headers));+ return Response.noContent().build();+ }++ @GET+ @Path("/tables/{tableBucketARN}/{namespace}/{name}/metadata-location")+ public Response getTableMetadataLocation(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name) {+ S3Table table = service.getTable(decode(tableBucketArn), namespace, name, region(headers));+ return Response.ok(metadataLocationRepresentation(table)).build();+ }++ @PUT+ @Path("/tables/{tableBucketARN}/{namespace}/{name}/metadata-location")+ public Response updateTableMetadataLocation(@Context HttpHeaders headers,+ @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name,+ Map<String, Object> request) {+ S3Table table = service.updateTableMetadataLocation(decode(tableBucketArn), namespace, name,+ string(request, "metadataLocation"), string(request, "versionToken"), region(headers));+ Map<String, Object> response = metadataLocationRepresentation(table);+ response.put("name", table.getName());+ response.put("namespace", List.of(table.getNamespace()));+ response.put("tableARN", table.getArn());+ return Response.ok(response).build();+ }++ @PUT+ @Path("/tables/{tableBucketARN}/{namespace}/{name}/policy")+ public Response putTablePolicy(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name,+ Map<String, Object> request) {+ service.putTablePolicy(decode(tableBucketArn), namespace, name, string(request, "resourcePolicy"), region(headers));+ return Response.ok().build();+ }++ @GET+ @Path("/tables/{tableBucketARN}/{namespace}/{name}/policy")+ public Response getTablePolicy(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name) {+ return Response.ok(Map.of("resourcePolicy", service.getTablePolicy(decode(tableBucketArn), namespace, name, region(headers)))).build();+ }++ @DELETE+ @Path("/tables/{tableBucketARN}/{namespace}/{name}/policy")+ public Response deleteTablePolicy(@Context HttpHeaders headers, @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name) {+ service.deleteTablePolicy(decode(tableBucketArn), namespace, name, region(headers));+ return Response.noContent().build();+ }++ @PUT+ @Path("/tables/{tableBucketARN}/{namespace}/{name}/maintenance/{type}")+ public Response putTableMaintenanceConfiguration(@Context HttpHeaders headers,+ @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name,+ @PathParam("type") String type, Map<String, Object> request) {+ service.putTableMaintenance(decode(tableBucketArn), namespace, name, type, request.get("value"), region(headers));+ return Response.ok().build();+ }++ @GET+ @Path("/tables/{tableBucketARN}/{namespace}/{name}/maintenance")+ public Response getTableMaintenanceConfiguration(@Context HttpHeaders headers,+ @Encoded @PathParam("tableBucketARN") String tableBucketArn,+ @PathParam("namespace") String namespace, @PathParam("name") String name) {+ String arn = decode(tableBucketArn);+ return Response.ok(Map.of("tableARN", service.getTable(arn, namespace, name, region(headers)).getArn(),+ "configuration", service.getTableMaintenanceConfigurations(arn, namespace, name, region(headers)))).build();+ }++ private Map<String, Object> bucketRepresentation(TableBucket bucket) {+ Map<String, Object> response = new LinkedHashMap<>();+ response.put("arn", bucket.getArn());+ response.put("name", bucket.getName());+ response.put("createdAt", bucket.getCreatedAt());+ response.put("ownerAccountId", bucket.getOwnerAccountId());+ putIfNotNull(response, "encryptionConfiguration", bucket.getEncryptionConfiguration());+ putIfNotNull(response, "storageClassConfiguration", bucket.getStorageClassConfiguration());+ return response;+ }++ private Map<String, Object> namespaceRepresentation(String tableBucketArn, Namespace namespace) {+ Map<String, Object> response = new LinkedHashMap<>();+ response.put("namespace", List.of(namespace.getName()));+ response.put("tableBucketARN", tableBucketArn);+ response.put("createdAt", namespace.getCreatedAt());+ return response;+ }++ private Map<String, Object> tableRepresentation(S3Table table) {+ Map<String, Object> response = new LinkedHashMap<>();+ response.put("name", table.getName());+ response.put("type", "customer");+ response.put("tableARN", table.getArn());+ response.put("namespace", List.of(table.getNamespace()));+ response.put("versionToken", table.getVersionToken());+ putIfNotNull(response, "metadataLocation", table.getMetadataLocation());+ response.put("createdAt", table.getCreatedAt());+ response.put("modifiedAt", table.getModifiedAt());+ response.put("ownerAccountId", table.getOwnerAccountId());+ response.put("format", table.getFormat());+ return response;+ }++ private Map<String, Object> metadataLocationRepresentation(S3Table table) {+ Map<String, Object> response = new LinkedHashMap<>();+ putIfNotNull(response, "metadataLocation", table.getMetadataLocation());+ response.put("versionToken", table.getVersionToken());+ return response;+ }++ private void putIfNotNull(Map<String, Object> target, String key, Object value) {+ if (value != null) {+ target.put(key, value);+ }+ }++ private String region(HttpHeaders headers) { return regionResolver.resolveRegion(headers); }+ private String decode(String value) { return value == null ? null : URLDecoder.decode(value, StandardCharsets.UTF_8); }+ private String string(Map<String, Object> value, String key) { return value == null ? null : (String) value.get(key); }++ @SuppressWarnings("unchecked")+ private Map<String, String> stringMap(Object value) {+ if (!(value instanceof Map<?, ?> map)) {+ return Map.of();+ }+ Map<String, String> result = new LinkedHashMap<>();+ map.forEach((key, entry) -> result.put(String.valueOf(key), String.valueOf(entry)));+ return result;+ }++ private List<String> stringList(Object value) {+ if (!(value instanceof List<?> list)) {+ return List.of();+ }+ List<String> result = new ArrayList<>();+ list.forEach(entry -> result.add(String.valueOf(entry)));+ return result;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3tables/S3TablesService.java b/src/main/java/io/github/hectorvent/floci/services/s3tables/S3TablesService.javanew file mode 100644index 0000000000..307b1c73f4--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3tables/S3TablesService.java@@ -0,0 +1,375 @@+package io.github.hectorvent.floci.services.s3tables;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.s3tables.model.Namespace;+import io.github.hectorvent.floci.services.s3tables.model.S3Table;+import io.github.hectorvent.floci.services.s3tables.model.TableBucket;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Instant;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.UUID;++@ApplicationScoped+public class S3TablesService {+ private static final Logger LOG = Logger.getLogger(S3TablesService.class);+ private static final String BUCKET_PREFIX = "bucket/";++ private final StorageBackend<String, TableBucket> store;+ private final RegionResolver regionResolver;++ @Inject+ public S3TablesService(StorageFactory factory, RegionResolver regionResolver) {+ this.store = factory.create("s3tables", "s3tables.json", new TypeReference<Map<String, TableBucket>>() {});+ this.regionResolver = regionResolver;+ }++ public synchronized TableBucket createTableBucket(String name, Object encryptionConfiguration,+ Object storageClassConfiguration, Map<String, String> tags, String region) {+ validateBucketName(name);+ String key = storageKey(region, name);+ if (store.get(key).isPresent()) {+ throw conflict("The table bucket " + name + " already exists.");+ }+ TableBucket bucket = new TableBucket(name, bucketArn(region, name), now(), regionResolver.getAccountId(),+ encryptionConfiguration, storageClassConfiguration, tags);+ store.put(key, bucket);+ LOG.infov("Created S3 Tables bucket {0} in region {1}", name, region);+ return bucket;+ }++ public synchronized TableBucket getTableBucket(String tableBucketArn, String region) {+ String name = resourceName(tableBucketArn, BUCKET_PREFIX);+ TableBucket bucket = store.get(storageKey(region, name))+ .orElseThrow(() -> notFound("The table bucket does not exist."));+ if (!bucket.getArn().equals(tableBucketArn)) {+ throw notFound("The table bucket does not exist.");+ }+ return bucket;+ }++ public synchronized List<TableBucket> listTableBuckets(String prefix, String region) {+ return store.scan(key -> key.startsWith(region + "::")).stream()+ .filter(bucket -> prefix == null || bucket.getName().startsWith(prefix))+ .sorted(Comparator.comparing(TableBucket::getName))+ .toList();+ }++ public synchronized void deleteTableBucket(String tableBucketArn, String region) {+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ if (!bucket.getNamespaces().isEmpty()) {+ throw conflict("The table bucket is not empty.");+ }+ store.delete(storageKey(region, bucket.getName()));+ }++ public synchronized Namespace createNamespace(String tableBucketArn, List<String> namespace, String region) {+ String namespaceName = namespaceName(namespace);+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ if (bucket.getNamespaces().containsKey(namespaceName)) {+ throw conflict("The namespace already exists.");+ }+ Namespace value = new Namespace(namespaceName, now());+ bucket.getNamespaces().put(namespaceName, value);+ persist(bucket, region);+ return value;+ }++ public synchronized Namespace getNamespace(String tableBucketArn, String namespaceName, String region) {+ return getNamespace(getTableBucket(tableBucketArn, region), namespaceName);+ }++ public synchronized List<Namespace> listNamespaces(String tableBucketArn, String prefix, String region) {+ return getTableBucket(tableBucketArn, region).getNamespaces().values().stream()+ .filter(namespace -> prefix == null || namespace.getName().startsWith(prefix))+ .sorted(Comparator.comparing(Namespace::getName))+ .toList();+ }++ public synchronized void deleteNamespace(String tableBucketArn, String namespaceName, String region) {+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ Namespace namespace = getNamespace(bucket, namespaceName);+ if (!namespace.getTables().isEmpty()) {+ throw conflict("The namespace is not empty.");+ }+ bucket.getNamespaces().remove(namespaceName);+ persist(bucket, region);+ }++ public synchronized S3Table createTable(String tableBucketArn, String namespaceName, String name, String format,+ Object metadata, Object encryptionConfiguration, Object storageClassConfiguration,+ Map<String, String> tags, String region) {+ requireName(name, "Table name is required.");+ if (!"ICEBERG".equals(format)) {+ throw badRequest("format must be ICEBERG.");+ }+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ Namespace namespace = getNamespace(bucket, namespaceName);+ if (namespace.getTables().containsKey(name)) {+ throw conflict("The table already exists.");+ }+ S3Table table = new S3Table(name, bucket.getArn() + "/table/" + name, namespaceName, format,+ versionToken(), metadataLocation(metadata), now(), regionResolver.getAccountId(), metadata,+ encryptionConfiguration, storageClassConfiguration, tags);+ namespace.getTables().put(name, table);+ persist(bucket, region);+ return table;+ }++ public synchronized S3Table getTable(String tableBucketArn, String namespaceName, String name, String region) {+ return getTable(getTableBucket(tableBucketArn, region), namespaceName, name);+ }++ public synchronized List<S3Table> listTables(String tableBucketArn, String namespaceName, String prefix, String region) {+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ List<S3Table> tables = new ArrayList<>();+ if (namespaceName == null) {+ bucket.getNamespaces().values().forEach(namespace -> tables.addAll(namespace.getTables().values()));+ } else {+ tables.addAll(getNamespace(bucket, namespaceName).getTables().values());+ }+ return tables.stream().filter(table -> prefix == null || table.getName().startsWith(prefix))+ .sorted(Comparator.comparing(S3Table::getName)).toList();+ }++ public synchronized void deleteTable(String tableBucketArn, String namespaceName, String name, String region) {+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ Namespace namespace = getNamespace(bucket, namespaceName);+ getTable(bucket, namespaceName, name);+ namespace.getTables().remove(name);+ persist(bucket, region);+ }++ public synchronized S3Table renameTable(String tableBucketArn, String namespaceName, String name, String newNamespaceName,+ String newName, String versionToken, String region) {+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ S3Table table = getTable(bucket, namespaceName, name);+ assertCurrentToken(table, versionToken);+ String targetNamespaceName = newNamespaceName == null ? namespaceName : newNamespaceName;+ String targetName = newName == null ? name : newName;+ requireName(targetName, "newName is required.");+ Namespace sourceNamespace = getNamespace(bucket, namespaceName);+ Namespace targetNamespace = getNamespace(bucket, targetNamespaceName);+ if ((!namespaceName.equals(targetNamespaceName) || !name.equals(targetName))+ && targetNamespace.getTables().containsKey(targetName)) {+ throw conflict("The target table already exists.");+ }+ sourceNamespace.getTables().remove(name);+ table.setNamespace(targetNamespaceName);+ table.setName(targetName);+ table.setArn(bucket.getArn() + "/table/" + targetName);+ touch(table);+ targetNamespace.getTables().put(targetName, table);+ persist(bucket, region);+ return table;+ }++ public synchronized S3Table updateTableMetadataLocation(String tableBucketArn, String namespaceName, String name,+ String metadataLocation, String versionToken, String region) {+ if (metadataLocation == null || metadataLocation.isBlank() || versionToken == null || versionToken.isBlank()) {+ throw badRequest("metadataLocation and versionToken are required.");+ }+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ S3Table table = getTable(bucket, namespaceName, name);+ assertCurrentToken(table, versionToken);+ table.setMetadataLocation(metadataLocation);+ touch(table);+ persist(bucket, region);+ return table;+ }++ public synchronized void putTableBucketPolicy(String tableBucketArn, String resourcePolicy, String region) {+ if (resourcePolicy == null) {+ throw badRequest("resourcePolicy is required.");+ }+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ bucket.setResourcePolicy(resourcePolicy);+ persist(bucket, region);+ }++ public synchronized String getTableBucketPolicy(String tableBucketArn, String region) {+ String policy = getTableBucket(tableBucketArn, region).getResourcePolicy();+ if (policy == null) {+ throw notFound("The table bucket policy does not exist.");+ }+ return policy;+ }++ public synchronized void deleteTableBucketPolicy(String tableBucketArn, String region) {+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ if (bucket.getResourcePolicy() == null) {+ throw notFound("The table bucket policy does not exist.");+ }+ bucket.setResourcePolicy(null);+ persist(bucket, region);+ }++ public synchronized void putTableBucketMaintenance(String tableBucketArn, String type, Object value, String region) {+ requireName(type, "Maintenance configuration type is required.");+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ bucket.getMaintenanceConfigurations().put(type, value);+ persist(bucket, region);+ }++ public synchronized Object getTableBucketMaintenance(String tableBucketArn, String type, String region) {+ Object value = getTableBucket(tableBucketArn, region).getMaintenanceConfigurations().get(type);+ if (value == null) {+ throw notFound("The table bucket maintenance configuration does not exist.");+ }+ return value;+ }++ public synchronized Map<String, Object> getTableBucketMaintenanceConfigurations(String tableBucketArn, String region) {+ return new LinkedHashMap<>(getTableBucket(tableBucketArn, region).getMaintenanceConfigurations());+ }++ public synchronized void putTablePolicy(String tableBucketArn, String namespaceName, String name, String resourcePolicy, String region) {+ if (resourcePolicy == null) {+ throw badRequest("resourcePolicy is required.");+ }+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ S3Table table = getTable(bucket, namespaceName, name);+ table.setResourcePolicy(resourcePolicy);+ persist(bucket, region);+ }++ public synchronized String getTablePolicy(String tableBucketArn, String namespaceName, String name, String region) {+ String policy = getTable(tableBucketArn, namespaceName, name, region).getResourcePolicy();+ if (policy == null) {+ throw notFound("The table policy does not exist.");+ }+ return policy;+ }++ public synchronized void deleteTablePolicy(String tableBucketArn, String namespaceName, String name, String region) {+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ S3Table table = getTable(bucket, namespaceName, name);+ if (table.getResourcePolicy() == null) {+ throw notFound("The table policy does not exist.");+ }+ table.setResourcePolicy(null);+ persist(bucket, region);+ }++ public synchronized void putTableMaintenance(String tableBucketArn, String namespaceName, String name, String type,+ Object value, String region) {+ requireName(type, "Maintenance configuration type is required.");+ TableBucket bucket = getTableBucket(tableBucketArn, region);+ S3Table table = getTable(bucket, namespaceName, name);+ table.getMaintenanceConfigurations().put(type, value);+ persist(bucket, region);+ }++ public synchronized Object getTableMaintenance(String tableBucketArn, String namespaceName, String name, String type, String region) {+ Object value = getTable(tableBucketArn, namespaceName, name, region).getMaintenanceConfigurations().get(type);+ if (value == null) {+ throw notFound("The table maintenance configuration does not exist.");+ }+ return value;+ }++ public synchronized Map<String, Object> getTableMaintenanceConfigurations(String tableBucketArn, String namespaceName,+ String name, String region) {+ return new LinkedHashMap<>(getTable(tableBucketArn, namespaceName, name, region).getMaintenanceConfigurations());+ }++ private Namespace getNamespace(TableBucket bucket, String name) {+ Namespace namespace = bucket.getNamespaces().get(name);+ if (namespace == null) {+ throw notFound("The namespace does not exist.");+ }+ return namespace;+ }++ private S3Table getTable(TableBucket bucket, String namespaceName, String name) {+ S3Table table = getNamespace(bucket, namespaceName).getTables().get(name);+ if (table == null) {+ throw notFound("The table does not exist.");+ }+ return table;+ }++ private String bucketArn(String region, String name) {+ return AwsArnUtils.Arn.of("s3tables", region, regionResolver.getAccountId(), BUCKET_PREFIX + name).toString();+ }++ private String storageKey(String region, String name) {+ return region + "::" + name;+ }++ private String resourceName(String arn, String prefix) {+ if (arn == null || !arn.startsWith("arn:") || !arn.contains(":" + prefix)) {+ throw notFound("The table bucket does not exist.");+ }+ return arn.substring(arn.lastIndexOf(prefix) + prefix.length());+ }++ private void persist(TableBucket bucket, String region) {+ store.put(storageKey(region, bucket.getName()), bucket);+ }++ private String namespaceName(List<String> namespace) {+ if (namespace == null || namespace.size() != 1) {+ throw badRequest("namespace must contain exactly one name.");+ }+ requireName(namespace.getFirst(), "Namespace name is required.");+ return namespace.getFirst();+ }++ private void validateBucketName(String name) {+ if (name == null || !name.matches("[a-z0-9][a-z0-9-]{1,61}[a-z0-9]")) {+ throw badRequest("Table bucket names must be 3-63 lowercase letters, numbers, or hyphens.");+ }+ }++ private void requireName(String value, String message) {+ if (value == null || value.isBlank()) {+ throw badRequest(message);+ }+ }++ @SuppressWarnings("unchecked")+ private String metadataLocation(Object metadata) {+ if (metadata instanceof Map<?, ?> map) {+ Object direct = map.get("metadataLocation");+ if (direct instanceof String value) {+ return value;+ }+ for (Object value : map.values()) {+ String nested = metadataLocation(value);+ if (nested != null) {+ return nested;+ }+ }+ }+ return null;+ }++ private void assertCurrentToken(S3Table table, String suppliedToken) {+ if (suppliedToken != null && !suppliedToken.equals(table.getVersionToken())) {+ throw conflict("The supplied version token is stale.");+ }+ }++ private void touch(S3Table table) {+ table.setVersionToken(versionToken());+ table.setModifiedAt(now());+ }++ private String now() { return Instant.now().toString(); }+ private String versionToken() { return UUID.randomUUID().toString(); }+ private AwsException badRequest(String message) { return new AwsException("BadRequestException", message, 400); }+ private AwsException conflict(String message) { return new AwsException("ConflictException", message, 409); }+ private AwsException notFound(String message) { return new AwsException("NotFoundException", message, 404); }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3tables/model/Namespace.java b/src/main/java/io/github/hectorvent/floci/services/s3tables/model/Namespace.javanew file mode 100644index 0000000000..9dbb749241--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3tables/model/Namespace.java@@ -0,0 +1,28 @@+package io.github.hectorvent.floci.services.s3tables.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.LinkedHashMap;+import java.util.Map;++@RegisterForReflection+public class Namespace {+ private String name;+ private String createdAt;+ private Map<String, S3Table> tables = new LinkedHashMap<>();++ public Namespace() {+ }++ public Namespace(String name, String createdAt) {+ this.name = name;+ this.createdAt = createdAt;+ }++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getCreatedAt() { return createdAt; }+ public void setCreatedAt(String createdAt) { this.createdAt = createdAt; }+ public Map<String, S3Table> getTables() { return tables; }+ public void setTables(Map<String, S3Table> tables) { this.tables = tables == null ? new LinkedHashMap<>() : new LinkedHashMap<>(tables); }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3tables/model/S3Table.java b/src/main/java/io/github/hectorvent/floci/services/s3tables/model/S3Table.javanew file mode 100644index 0000000000..8262c0bafe--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3tables/model/S3Table.java@@ -0,0 +1,79 @@+package io.github.hectorvent.floci.services.s3tables.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.LinkedHashMap;+import java.util.Map;++@RegisterForReflection+public class S3Table {+ private String name;+ private String arn;+ private String namespace;+ private String format;+ private String versionToken;+ private String metadataLocation;+ private String createdAt;+ private String modifiedAt;+ private String ownerAccountId;+ private Object metadata;+ private Object encryptionConfiguration;+ private Object storageClassConfiguration;+ private Map<String, String> tags = new LinkedHashMap<>();+ private String resourcePolicy;+ private Map<String, Object> maintenanceConfigurations = new LinkedHashMap<>();++ public S3Table() {+ }++ public S3Table(String name, String arn, String namespace, String format, String versionToken,+ String metadataLocation, String createdAt, String ownerAccountId, Object metadata,+ Object encryptionConfiguration, Object storageClassConfiguration, Map<String, String> tags) {+ this.name = name;+ this.arn = arn;+ this.namespace = namespace;+ this.format = format;+ this.versionToken = versionToken;+ this.metadataLocation = metadataLocation;+ this.createdAt = createdAt;+ this.modifiedAt = createdAt;+ this.ownerAccountId = ownerAccountId;+ this.metadata = metadata;+ this.encryptionConfiguration = encryptionConfiguration;+ this.storageClassConfiguration = storageClassConfiguration;+ if (tags != null) {+ this.tags = new LinkedHashMap<>(tags);+ }+ }++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getArn() { return arn; }+ public void setArn(String arn) { this.arn = arn; }+ public String getNamespace() { return namespace; }+ public void setNamespace(String namespace) { this.namespace = namespace; }+ public String getFormat() { return format; }+ public void setFormat(String format) { this.format = format; }+ public String getVersionToken() { return versionToken; }+ public void setVersionToken(String versionToken) { this.versionToken = versionToken; }+ public String getMetadataLocation() { return metadataLocation; }+ public void setMetadataLocation(String metadataLocation) { this.metadataLocation = metadataLocation; }+ public String getCreatedAt() { return createdAt; }+ public void setCreatedAt(String createdAt) { this.createdAt = createdAt; }+ public String getModifiedAt() { return modifiedAt; }+ public void setModifiedAt(String modifiedAt) { this.modifiedAt = modifiedAt; }+ public String getOwnerAccountId() { return ownerAccountId; }+ public void setOwnerAccountId(String ownerAccountId) { this.ownerAccountId = ownerAccountId; }+ public Object getMetadata() { return metadata; }+ public void setMetadata(Object metadata) { this.metadata = metadata; }+ public Object getEncryptionConfiguration() { return encryptionConfiguration; }+ public void setEncryptionConfiguration(Object encryptionConfiguration) { this.encryptionConfiguration = encryptionConfiguration; }+ public Object getStorageClassConfiguration() { return storageClassConfiguration; }+ public void setStorageClassConfiguration(Object storageClassConfiguration) { this.storageClassConfiguration = storageClassConfiguration; }+ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags == null ? new LinkedHashMap<>() : new LinkedHashMap<>(tags); }+ public String getResourcePolicy() { return resourcePolicy; }+ public void setResourcePolicy(String resourcePolicy) { this.resourcePolicy = resourcePolicy; }+ public Map<String, Object> getMaintenanceConfigurations() { return maintenanceConfigurations; }+ public void setMaintenanceConfigurations(Map<String, Object> maintenanceConfigurations) { this.maintenanceConfigurations = maintenanceConfigurations == null ? new LinkedHashMap<>() : new LinkedHashMap<>(maintenanceConfigurations); }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3tables/model/TableBucket.java b/src/main/java/io/github/hectorvent/floci/services/s3tables/model/TableBucket.javanew file mode 100644index 0000000000..52430f124a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3tables/model/TableBucket.java@@ -0,0 +1,57 @@+package io.github.hectorvent.floci.services.s3tables.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.LinkedHashMap;+import java.util.Map;++@RegisterForReflection+public class TableBucket {+ private String name;+ private String arn;+ private String createdAt;+ private String ownerAccountId;+ private Object encryptionConfiguration;+ private Object storageClassConfiguration;+ private Map<String, String> tags = new LinkedHashMap<>();+ private String resourcePolicy;+ private Map<String, Object> maintenanceConfigurations = new LinkedHashMap<>();+ private Map<String, Namespace> namespaces = new LinkedHashMap<>();++ public TableBucket() {+ }++ public TableBucket(String name, String arn, String createdAt, String ownerAccountId,+ Object encryptionConfiguration, Object storageClassConfiguration, Map<String, String> tags) {+ this.name = name;+ this.arn = arn;+ this.createdAt = createdAt;+ this.ownerAccountId = ownerAccountId;+ this.encryptionConfiguration = encryptionConfiguration;+ this.storageClassConfiguration = storageClassConfiguration;+ if (tags != null) {+ this.tags = new LinkedHashMap<>(tags);+ }+ }++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getArn() { return arn; }+ public void setArn(String arn) { this.arn = arn; }+ public String getCreatedAt() { return createdAt; }+ public void setCreatedAt(String createdAt) { this.createdAt = createdAt; }+ public String getOwnerAccountId() { return ownerAccountId; }+ public void setOwnerAccountId(String ownerAccountId) { this.ownerAccountId = ownerAccountId; }+ public Object getEncryptionConfiguration() { return encryptionConfiguration; }+ public void setEncryptionConfiguration(Object encryptionConfiguration) { this.encryptionConfiguration = encryptionConfiguration; }+ public Object getStorageClassConfiguration() { return storageClassConfiguration; }+ public void setStorageClassConfiguration(Object storageClassConfiguration) { this.storageClassConfiguration = storageClassConfiguration; }+ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags == null ? new LinkedHashMap<>() : new LinkedHashMap<>(tags); }+ public String getResourcePolicy() { return resourcePolicy; }+ public void setResourcePolicy(String resourcePolicy) { this.resourcePolicy = resourcePolicy; }+ public Map<String, Object> getMaintenanceConfigurations() { return maintenanceConfigurations; }+ public void setMaintenanceConfigurations(Map<String, Object> maintenanceConfigurations) { this.maintenanceConfigurations = maintenanceConfigurations == null ? new LinkedHashMap<>() : new LinkedHashMap<>(maintenanceConfigurations); }+ public Map<String, Namespace> getNamespaces() { return namespaces; }+ public void setNamespaces(Map<String, Namespace> namespaces) { this.namespaces = namespaces == null ? new LinkedHashMap<>() : new LinkedHashMap<>(namespaces); }+}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex ebadbf168a..9f5a8eae37 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -116,6 +116,8 @@ floci:flush-interval-ms: 5000s3vectors:flush-interval-ms: 5000+ s3tables:+ flush-interval-ms: 5000ecs:flush-interval-ms: 5000codebuild:@@ -475,6 +477,8 @@ floci:schema-worker-shutdown-timeout-seconds: 30s3vectors:enabled: true+ s3tables:+ enabled: trueiot:enabled: truemqtt:diff --git a/src/test/java/io/github/hectorvent/floci/services/s3tables/S3TablesIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3tables/S3TablesIntegrationTest.javanew file mode 100644index 0000000000..b8669aa662--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3tables/S3TablesIntegrationTest.java@@ -0,0 +1,200 @@+package io.github.hectorvent.floci.services.s3tables;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestInstance;+import org.junit.jupiter.api.TestMethodOrder;++import java.net.URLEncoder;+import java.nio.charset.StandardCharsets;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.contains;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasKey;+import static org.hamcrest.Matchers.not;++@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+@TestInstance(TestInstance.Lifecycle.PER_CLASS)+class S3TablesIntegrationTest {++ private static final String JSON_CONTENT_TYPE = "application/json";+ private static final String BUCKET_NAME = "s3tables-integration-bucket";+ private static final String NAMESPACE = "analytics";+ private static final String TABLE_NAME = "events";+ private static final String ACCOUNT_ID = "000000000000";+ private static final String REGION = "us-east-1";+ private static final String BUCKET_ARN = "arn:aws:s3tables:%s:%s:bucket/%s"+ .formatted(REGION, ACCOUNT_ID, BUCKET_NAME);+ private static final String ENCODED_BUCKET_ARN = URLEncoder.encode(BUCKET_ARN, StandardCharsets.UTF_8);++ private String versionToken;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void createsTableBucketThroughTheS3TablesRoute() {+ given()+ .urlEncodingEnabled(false)+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ { "name": "%s" }+ """.formatted(BUCKET_NAME))+ .when()+ .put("/buckets")+ .then()+ .statusCode(200)+ .contentType(containsString(JSON_CONTENT_TYPE))+ .body("arn", equalTo(BUCKET_ARN));+ }++ @Test+ @Order(2)+ void managesNamespaceAndTableThroughEncodedBucketArnRoutes() {+ given()+ .urlEncodingEnabled(false)+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ { "namespace": ["%s"] }+ """.formatted(NAMESPACE))+ .when()+ .put("/namespaces/" + ENCODED_BUCKET_ARN)+ .then()+ .statusCode(200)+ .contentType(containsString(JSON_CONTENT_TYPE))+ .body("namespace", contains(NAMESPACE))+ .body("tableBucketARN", equalTo(BUCKET_ARN));++ versionToken = given()+ .urlEncodingEnabled(false)+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "name": "%s",+ "format": "ICEBERG",+ "metadata": { "iceberg": { "metadataLocation": "s3://warehouse/events/metadata/v1.json" } }+ }+ """.formatted(TABLE_NAME))+ .when()+ .put("/tables/" + ENCODED_BUCKET_ARN + "/" + NAMESPACE)+ .then()+ .statusCode(200)+ .contentType(containsString(JSON_CONTENT_TYPE))+ .body("tableARN", equalTo(BUCKET_ARN + "/table/" + TABLE_NAME))+ .body("versionToken", not(""))+ .extract()+ .path("versionToken");+ }++ @Test+ @Order(3)+ void updatesMetadataLocationUsingOptimisticVersionTokens() {+ String replacementToken = given()+ .urlEncodingEnabled(false)+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "metadataLocation": "s3://warehouse/events/metadata/v2.json",+ "versionToken": "%s"+ }+ """.formatted(versionToken))+ .when()+ .put("/tables/" + ENCODED_BUCKET_ARN + "/" + NAMESPACE + "/" + TABLE_NAME + "/metadata-location")+ .then()+ .statusCode(200)+ .body("metadataLocation", equalTo("s3://warehouse/events/metadata/v2.json"))+ .body("versionToken", not(equalTo(versionToken)))+ .extract()+ .path("versionToken");++ given()+ .urlEncodingEnabled(false)+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "metadataLocation": "s3://warehouse/events/metadata/v3.json",+ "versionToken": "%s"+ }+ """.formatted(versionToken))+ .when()+ .put("/tables/" + ENCODED_BUCKET_ARN + "/" + NAMESPACE + "/" + TABLE_NAME + "/metadata-location")+ .then()+ .statusCode(409)+ .body("__type", equalTo("ConflictException"));++ versionToken = replacementToken;+ }++ @Test+ @Order(4)+ void roundTripsTableBucketPolicy() {+ given()+ .urlEncodingEnabled(false)+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ { "resourcePolicy": "{\\\"Version\\\":\\\"2012-10-17\\\"}" }+ """)+ .when()+ .put("/buckets/" + ENCODED_BUCKET_ARN + "/policy")+ .then()+ .statusCode(200);++ given()+ .urlEncodingEnabled(false)+ .when()+ .get("/buckets/" + ENCODED_BUCKET_ARN + "/policy")+ .then()+ .statusCode(200)+ .contentType(containsString(JSON_CONTENT_TYPE))+ .body("resourcePolicy", equalTo("{\"Version\":\"2012-10-17\"}"));+ }++ @Test+ @Order(5)+ void roundTripsTypedMaintenanceConfigurationMaps() {+ given()+ .urlEncodingEnabled(false)+ .contentType(JSON_CONTENT_TYPE)+ .body("{ \"value\": { \"status\": \"enabled\" } }")+ .when()+ .put("/buckets/" + ENCODED_BUCKET_ARN + "/maintenance/icebergUnreferencedFileRemoval")+ .then()+ .statusCode(200);++ given()+ .urlEncodingEnabled(false)+ .when()+ .get("/buckets/" + ENCODED_BUCKET_ARN + "/maintenance")+ .then()+ .statusCode(200)+ .body("configuration.icebergUnreferencedFileRemoval.status", equalTo("enabled"));++ given()+ .urlEncodingEnabled(false)+ .contentType(JSON_CONTENT_TYPE)+ .body("{ \"value\": { \"status\": \"enabled\" } }")+ .when()+ .put("/tables/" + ENCODED_BUCKET_ARN + "/" + NAMESPACE + "/" + TABLE_NAME+ + "/maintenance/icebergCompaction")+ .then()+ .statusCode(200);++ given()+ .urlEncodingEnabled(false)+ .when()+ .get("/tables/" + ENCODED_BUCKET_ARN + "/" + NAMESPACE + "/" + TABLE_NAME + "/maintenance")+ .then()+ .statusCode(200)+ .body("configuration.icebergCompaction.status", equalTo("enabled"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3tables/S3TablesServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3tables/S3TablesServiceTest.javanew file mode 100644index 0000000000..b77b3e3e53--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3tables/S3TablesServiceTest.java@@ -0,0 +1,194 @@+package io.github.hectorvent.floci.services.s3tables;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.s3tables.model.S3Table;+import io.github.hectorvent.floci.services.s3tables.model.TableBucket;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;++class S3TablesServiceTest {+ private static final String ACCOUNT_ID = "000000000000";+ private static final String REGION = "us-east-1";+ private static final String BUCKET = "unit-test-table-bucket";++ private S3TablesService service;++ @BeforeEach+ void setUp() {+ StorageFactory storageFactory = new StorageFactory(null, null) {+ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return AccountAwareStorageBackend.inMemory(ACCOUNT_ID);+ }+ };+ service = new S3TablesService(storageFactory, new RegionResolver(REGION, ACCOUNT_ID));+ }++ @Test+ void isolatesSameBucketNameByRegion() {+ TableBucket east = service.createTableBucket(BUCKET, null, null, Map.of(), REGION);+ TableBucket west = service.createTableBucket(BUCKET, null, null, Map.of(), "eu-west-1");++ assertEquals("arn:aws:s3tables:us-east-1:000000000000:bucket/" + BUCKET, east.getArn());+ assertEquals("arn:aws:s3tables:eu-west-1:000000000000:bucket/" + BUCKET, west.getArn());+ assertEquals(List.of(east), service.listTableBuckets(null, REGION));+ assertEquals(List.of(west), service.listTableBuckets(null, "eu-west-1"));+ }++ @Test+ void rejectsDuplicateBucketsAndNonEmptyParents() {+ TableBucket bucket = createBucket();+ String arn = bucket.getArn();++ assertError("ConflictException", () -> service.createTableBucket(BUCKET, null, null, Map.of(), REGION));++ service.createNamespace(arn, List.of("analytics"), REGION);+ assertError("ConflictException", () -> service.deleteTableBucket(arn, REGION));++ createTable(arn, "analytics", "events");+ assertError("ConflictException", () -> service.deleteNamespace(arn, "analytics", REGION));++ service.deleteTable(arn, "analytics", "events", REGION);+ service.deleteNamespace(arn, "analytics", REGION);+ service.deleteTableBucket(arn, REGION);+ assertError("NotFoundException", () -> service.getTableBucket(arn, REGION));+ }++ @Test+ void updatesMetadataOnlyWithTheCurrentVersionToken() {+ String arn = createBucketWithNamespace("analytics");+ S3Table created = createTable(arn, "analytics", "events");+ String originalToken = created.getVersionToken();++ S3Table updated = service.updateTableMetadataLocation(arn, "analytics", "events",+ "s3://warehouse/events/metadata/v2.json", originalToken, REGION);++ assertEquals("s3://warehouse/events/metadata/v2.json", updated.getMetadataLocation());+ assertNotEquals(originalToken, updated.getVersionToken());+ assertError("ConflictException", () -> service.updateTableMetadataLocation(arn, "analytics", "events",+ "s3://warehouse/events/metadata/v3.json", originalToken, REGION));+ assertEquals("s3://warehouse/events/metadata/v2.json",+ service.getTable(arn, "analytics", "events", REGION).getMetadataLocation());+ }++ @Test+ void renameMovesTableAndRotatesItsVersionToken() {+ String arn = createBucketWithNamespace("analytics");+ service.createNamespace(arn, List.of("reporting"), REGION);+ S3Table created = createTable(arn, "analytics", "events");+ String originalToken = created.getVersionToken();++ S3Table renamed = service.renameTable(arn, "analytics", "events", "reporting", "daily_events",+ originalToken, REGION);++ assertEquals("reporting", renamed.getNamespace());+ assertEquals("daily_events", renamed.getName());+ assertEquals(arn + "/table/daily_events", renamed.getArn());+ assertNotEquals(originalToken, renamed.getVersionToken());+ assertError("NotFoundException", () -> service.getTable(arn, "analytics", "events", REGION));+ assertEquals(renamed, service.getTable(arn, "reporting", "daily_events", REGION));+ }++ @Test+ void roundTripsPoliciesAndTypedMaintenanceConfigurations() {+ String arn = createBucketWithNamespace("analytics");+ createTable(arn, "analytics", "events");+ Map<String, Object> bucketMaintenance = Map.of("unreferencedFileRemoval", Map.of("status", "enabled"));+ Map<String, Object> tableMaintenance = Map.of("status", "enabled");++ service.putTableBucketPolicy(arn, "{\"Version\":\"2012-10-17\"}", REGION);+ service.putTableBucketMaintenance(arn, "UNREFERENCED_FILE_REMOVAL", bucketMaintenance, REGION);+ service.putTablePolicy(arn, "analytics", "events", "{\"Statement\":[]}", REGION);+ service.putTableMaintenance(arn, "analytics", "events", "ICEBERG_COMPACTION", tableMaintenance, REGION);++ assertEquals("{\"Version\":\"2012-10-17\"}", service.getTableBucketPolicy(arn, REGION));+ assertEquals(bucketMaintenance, service.getTableBucketMaintenance(arn, "UNREFERENCED_FILE_REMOVAL", REGION));+ assertEquals(Map.of("UNREFERENCED_FILE_REMOVAL", bucketMaintenance),+ service.getTableBucketMaintenanceConfigurations(arn, REGION));+ assertEquals("{\"Statement\":[]}", service.getTablePolicy(arn, "analytics", "events", REGION));+ assertEquals(tableMaintenance,+ service.getTableMaintenance(arn, "analytics", "events", "ICEBERG_COMPACTION", REGION));+ assertEquals(Map.of("ICEBERG_COMPACTION", tableMaintenance),+ service.getTableMaintenanceConfigurations(arn, "analytics", "events", REGION));++ service.deleteTableBucketPolicy(arn, REGION);+ service.deleteTablePolicy(arn, "analytics", "events", REGION);+ assertError("NotFoundException", () -> service.getTableBucketPolicy(arn, REGION));+ assertError("NotFoundException", () -> service.getTablePolicy(arn, "analytics", "events", REGION));+ }++ private TableBucket createBucket() {+ return service.createTableBucket(BUCKET, null, null, Map.of(), REGION);+ }++ private String createBucketWithNamespace(String namespace) {+ TableBucket bucket = createBucket();+ service.createNamespace(bucket.getArn(), List.of(namespace), REGION);+ return bucket.getArn();+ }++ private S3Table createTable(String arn, String namespace, String name) {+ return service.createTable(arn, namespace, name, "ICEBERG",+ Map.of("iceberg", Map.of("metadataLocation", "s3://warehouse/" + name + "/metadata/v1.json")),+ null, null, Map.of(), REGION);+ }++ private void assertError(String expectedCode, org.junit.jupiter.api.function.Executable action) {+ AwsException exception = assertThrows(AwsException.class, action);+ assertEquals(expectedCode, exception.getErrorCode());+ }++ @Test+ void allowsOnlyOneConcurrentMetadataUpdatePerVersionToken() throws Exception {+ String arn = createBucketWithNamespace("analytics");+ S3Table created = createTable(arn, "analytics", "events");+ var ready = new java.util.concurrent.CountDownLatch(2);+ var start = new java.util.concurrent.CountDownLatch(1);+ var executor = java.util.concurrent.Executors.newFixedThreadPool(2);++ try {+ var first = executor.submit(() -> updateMetadataWithToken(arn, created.getVersionToken(),+ "s3://warehouse/events/metadata/v2.json", ready, start));+ var second = executor.submit(() -> updateMetadataWithToken(arn, created.getVersionToken(),+ "s3://warehouse/events/metadata/v3.json", ready, start));++ assertEquals(true, ready.await(5, java.util.concurrent.TimeUnit.SECONDS));+ start.countDown();+ List<String> outcomes = List.of(first.get(5, java.util.concurrent.TimeUnit.SECONDS),+ second.get(5, java.util.concurrent.TimeUnit.SECONDS));++ assertEquals(1L, outcomes.stream().filter("updated"::equals).count());+ assertEquals(1L, outcomes.stream().filter("ConflictException"::equals).count());+ } finally {+ executor.shutdownNow();+ }+ }++ private String updateMetadataWithToken(String arn, String token, String metadataLocation,+ java.util.concurrent.CountDownLatch ready,+ java.util.concurrent.CountDownLatch start) throws InterruptedException {+ ready.countDown();+ if (!start.await(5, java.util.concurrent.TimeUnit.SECONDS)) {+ return "timed out";+ }+ try {+ service.updateTableMetadataLocation(arn, "analytics", "events", metadataLocation, token, REGION);+ return "updated";+ } catch (AwsException exception) {+ return exception.getErrorCode();+ }+ }++}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 4bed797ec7..b1090c20c9 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -44,6 +44,8 @@ floci:flush-interval-ms: 60000s3vectors:flush-interval-ms: 60000+ s3tables:+ flush-interval-ms: 60000ecs:flush-interval-ms: 60000codebuild:@@ -288,6 +290,8 @@ floci:schema-worker-shutdown-timeout-seconds: 30s3vectors:enabled: true+ s3tables:+ enabled: trueiot:enabled: truemqtt: