Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat: Glue Schema Registry support (#621)

Tristan Burchb84d205b89f4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0dc1095dd2..7edee8ed8b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@
* **athena:** real SQL execution via `floci-duck` DuckDB sidecar — queries run against actual S3 data; Glue tables are auto-registered as DuckDB views at query time
* **athena:** lazy sidecar lifecycle — `floci/floci-duck:latest` container is started on first query and reused for subsequent executions; `FLOCI_SERVICES_ATHENA_DUCK_URL` bypasses container management
* **athena:** `mock` mode (`FLOCI_SERVICES_ATHENA_MOCK=true`) keeps the previous behavior for unit tests that only verify the state machine
+* **glue:** add Schema Registry support for registries, schemas, versions, compatibility checks, metadata, tags, and Java SerDe SDK compatibility
* **glue,athena:** SerDe-aware format inference — `inferReadFunction` now checks `SerdeInfo.SerializationLibrary` in addition to `InputFormat`; standard AWS JSON tables (`TextInputFormat` + `JsonSerDe`) correctly map to `read_json_auto`
* **bedrock-runtime:** add stub for Converse and InvokeModel ([#87](https://github.com/floci-io/floci/issues/87))
* **s3:** preserve explicit object server-side-encryption headers on PutObject, GetObject, HeadObject, CopyObject, and multipart uploads
diff --git a/README.md b/README.md
index 7815adcf42..79541685b9 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@
| RDS (PostgreSQL + MySQL + IAM auth) | ✅ | ❌ |
| MSK (Kafka + Redpanda) | ✅ | ❌ |
| Athena (real SQL via DuckDB sidecar + Glue views) | ✅ | ❌ |
-| Glue Data Catalog | ✅ | ❌ |
+| Glue Data Catalog + Schema Registry | ✅ | ❌ |
| Data Firehose (NDJSON delivery) | ✅ | ❌ |
| S3 Object Lock (COMPLIANCE / GOVERNANCE) | ✅ | ⚠️ Partial |
| DynamoDB Streams | ✅ | ⚠️ Partial |
@@ -203,7 +203,7 @@ All default images are configurable via environment variables, useful for pinnin
| **RDS** | **Real Docker containers** | PostgreSQL & MySQL, IAM auth, JDBC-compatible |
| **MSK** | **Real Docker containers** | Kafka compatible via Redpanda orchestration |
| **Athena** | In-process + **DuckDB sidecar** | Real SQL execution; Glue-backed views over S3 data; `read_parquet` / `read_json_auto` / `read_csv_auto` inferred from SerDe |
-| **Glue** | In-process | Data Catalog; tables consumed by Athena as DuckDB views at query time |
+| **Glue** | In-process | Data Catalog; Schema Registry for Avro / JSON Schema / Protobuf; tables consumed by Athena as DuckDB views at query time |
| **Data Firehose** | In-process | Streaming data delivery; records flushed as NDJSON to S3 |
| **ECS** | **Real Docker containers** | Clusters, task definitions, tasks, services, capacity providers, task sets |
| **EC2** | **Real Docker containers** | `RunInstances` launches real Docker containers; SSH key injection; UserData execution; IMDS (IMDSv1+IMDSv2, port 9169) with IAM credential serving; VPCs, subnets, security groups, AMIs, key pairs, internet gateways, route tables, Elastic IPs, tags |
diff --git a/compatibility-tests/sdk-test-java/pom.xml b/compatibility-tests/sdk-test-java/pom.xml
index 3528fbdbf6..d31b2c27f4 100644
--- a/compatibility-tests/sdk-test-java/pom.xml
+++ b/compatibility-tests/sdk-test-java/pom.xml
@@ -118,6 +118,11 @@
<groupId>software.amazon.awssdk</groupId>
<artifactId>glue</artifactId>
</dependency>
+ <dependency>
+ <groupId>software.amazon.glue</groupId>
+ <artifactId>schema-registry-serde</artifactId>
+ <version>1.1.27</version>
+ </dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>firehose</artifactId>
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueSchemaRegistryTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueSchemaRegistryTest.java
new file mode 100644
index 0000000000..970c8f05c5
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueSchemaRegistryTest.java
@@ -0,0 +1,995 @@
+package com.floci.test;
+
+import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryKafkaDeserializer;
+import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistryKafkaSerializer;
+import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants;
+import com.amazonaws.services.schemaregistry.utils.AvroRecordType;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.glue.model.CheckSchemaVersionValidityRequest;
+import software.amazon.awssdk.services.glue.model.Compatibility;
+import software.amazon.awssdk.services.glue.model.CreateRegistryRequest;
+import software.amazon.awssdk.services.glue.model.CreateSchemaRequest;
+import software.amazon.awssdk.services.glue.model.DataFormat;
+import software.amazon.awssdk.services.glue.model.DeleteRegistryRequest;
+import software.amazon.awssdk.services.glue.model.DeleteSchemaRequest;
+import software.amazon.awssdk.services.glue.model.DeleteSchemaVersionsRequest;
+import software.amazon.awssdk.services.glue.model.EntityNotFoundException;
+import software.amazon.awssdk.services.glue.model.GetRegistryRequest;
+import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionRequest;
+import software.amazon.awssdk.services.glue.model.GetSchemaRequest;
+import software.amazon.awssdk.services.glue.model.GetSchemaVersionRequest;
+import software.amazon.awssdk.services.glue.model.GetSchemaVersionsDiffRequest;
+import software.amazon.awssdk.services.glue.model.GetTagsRequest;
+import software.amazon.awssdk.services.glue.model.InvalidInputException;
+import software.amazon.awssdk.services.glue.model.ListRegistriesRequest;
+import software.amazon.awssdk.services.glue.model.ListSchemasRequest;
+import software.amazon.awssdk.services.glue.model.ListSchemaVersionsRequest;
+import software.amazon.awssdk.services.glue.model.MetadataKeyValuePair;
+import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataRequest;
+import software.amazon.awssdk.services.glue.model.QuerySchemaVersionMetadataRequest;
+import software.amazon.awssdk.services.glue.model.RegisterSchemaVersionRequest;
+import software.amazon.awssdk.services.glue.model.RegistryId;
+import software.amazon.awssdk.services.glue.model.RemoveSchemaVersionMetadataRequest;
+import software.amazon.awssdk.services.glue.model.SchemaDiffType;
+import software.amazon.awssdk.services.glue.model.SchemaId;
+import software.amazon.awssdk.services.glue.model.SchemaVersionNumber;
+import software.amazon.awssdk.services.glue.model.TagResourceRequest;
+import software.amazon.awssdk.services.glue.model.UntagResourceRequest;
+import software.amazon.awssdk.services.glue.model.UpdateRegistryRequest;
+import software.amazon.awssdk.services.glue.model.UpdateSchemaRequest;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@DisplayName("Glue Schema Registry")
+class GlueSchemaRegistryTest {
+
+ private static final String AVRO_V1 =
+ "{\"type\":\"record\",\"name\":\"Order\",\"namespace\":\"com.floci\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"}]}";
+
+ private static final String AVRO_V2 =
+ "{\"type\":\"record\",\"name\":\"Order\",\"namespace\":\"com.floci\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"amount\",\"type\":[\"null\",\"double\"],\"default\":null}]}";
+
+ private static final String AVRO_COMPAT_BASE =
+ "{\"type\":\"record\",\"name\":\"Customer\",\"namespace\":\"com.floci\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":\"string\"},"
+ + "{\"name\":\"phone\",\"type\":[\"null\",\"string\"],\"default\":null}"
+ + "]}";
+
+ private static final String AVRO_COMPAT_ADD_REQUIRED =
+ "{\"type\":\"record\",\"name\":\"Customer\",\"namespace\":\"com.floci\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":\"string\"},"
+ + "{\"name\":\"phone\",\"type\":[\"null\",\"string\"],\"default\":null},"
+ + "{\"name\":\"zip\",\"type\":\"string\"}"
+ + "]}";
+
+ private static final String AVRO_COMPAT_ADD_OPTIONAL =
+ "{\"type\":\"record\",\"name\":\"Customer\",\"namespace\":\"com.floci\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":\"string\"},"
+ + "{\"name\":\"phone\",\"type\":[\"null\",\"string\"],\"default\":null},"
+ + "{\"name\":\"zip\",\"type\":[\"null\",\"string\"],\"default\":null}"
+ + "]}";
+
+ private static final String AVRO_COMPAT_DELETE_REQUIRED =
+ "{\"type\":\"record\",\"name\":\"Customer\",\"namespace\":\"com.floci\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"phone\",\"type\":[\"null\",\"string\"],\"default\":null}"
+ + "]}";
+
+ private static final String AVRO_COMPAT_DELETE_OPTIONAL =
+ "{\"type\":\"record\",\"name\":\"Customer\",\"namespace\":\"com.floci\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":\"string\"}"
+ + "]}";
+
+ private static final String AVRO_TRANSITIVE_ID_ONLY =
+ "{\"type\":\"record\",\"name\":\"Account\",\"namespace\":\"com.floci\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"}]}";
+
+ private static final String AVRO_TRANSITIVE_ID_EMAIL_REQUIRED =
+ "{\"type\":\"record\",\"name\":\"Account\",\"namespace\":\"com.floci\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":\"string\"}"
+ + "]}";
+
+ private static final String AVRO_TRANSITIVE_ID_EMAIL_PHONE_OPTIONAL =
+ "{\"type\":\"record\",\"name\":\"Account\",\"namespace\":\"com.floci\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":\"string\"},"
+ + "{\"name\":\"phone\",\"type\":[\"null\",\"string\"],\"default\":null}"
+ + "]}";
+
+ private static final String AVRO_TRANSITIVE_ID_PHONE_OPTIONAL =
+ "{\"type\":\"record\",\"name\":\"Account\",\"namespace\":\"com.floci\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"phone\",\"type\":[\"null\",\"string\"],\"default\":null}"
+ + "]}";
+
+ private static final String JSON_SCHEMA =
+ "{\"$schema\":\"http://json-schema.org/draft-07/schema#\","
+ + "\"type\":\"object\","
+ + "\"properties\":{\"id\":{\"type\":\"integer\"}},"
+ + "\"required\":[\"id\"],"
+ + "\"additionalProperties\":false}";
+
+ private static final String PROTOBUF_SCHEMA =
+ "syntax = \"proto3\";\n"
+ + "package com.floci;\n"
+ + "message Customer {\n"
+ + " int64 id = 1;\n"
+ + " string email = 2;\n"
+ + "}\n";
+
+ private static final String JSON_COMPAT_BASE_CLOSED =
+ "{\"$schema\":\"http://json-schema.org/draft-07/schema#\","
+ + "\"type\":\"object\","
+ + "\"properties\":{\"firstName\":{\"type\":\"string\"}},"
+ + "\"additionalProperties\":false}";
+
+ private static final String JSON_COMPAT_BASE_OPEN =
+ "{\"$schema\":\"http://json-schema.org/draft-07/schema#\","
+ + "\"type\":\"object\","
+ + "\"properties\":{\"firstName\":{\"type\":\"string\"}},"
+ + "\"additionalProperties\":true}";
+
+ private static final String JSON_COMPAT_ADD_OPTIONAL_PHONE =
+ "{\"$schema\":\"http://json-schema.org/draft-07/schema#\","
+ + "\"type\":\"object\","
+ + "\"properties\":{\"firstName\":{\"type\":\"string\"},\"phone\":{\"type\":\"number\"}},"
+ + "\"additionalProperties\":false}";
+
+ private static final String JSON_COMPAT_WITH_OPTIONAL_PHONE =
+ "{\"$schema\":\"http://json-schema.org/draft-07/schema#\","
+ + "\"type\":\"object\","
+ + "\"properties\":{\"firstName\":{\"type\":\"string\"},\"phone\":{\"type\":\"number\"}},"
+ + "\"additionalProperties\":false}";
+
+ private static final String JSON_COMPAT_DELETE_OPTIONAL_PHONE_CLOSED =
+ "{\"$schema\":\"http://json-schema.org/draft-07/schema#\","
+ + "\"type\":\"object\","
+ + "\"properties\":{\"firstName\":{\"type\":\"string\"}},"
+ + "\"additionalProperties\":false}";
+
+ private static final String JSON_COMPAT_DELETE_OPTIONAL_PHONE_OPEN =
+ "{\"$schema\":\"http://json-schema.org/draft-07/schema#\","
+ + "\"type\":\"object\","
+ + "\"properties\":{\"firstName\":{\"type\":\"string\"}},"
+ + "\"additionalProperties\":true}";
+
+ private static final String PROTOBUF_COMPAT_BACKWARD_BASE =
+ "syntax = \"proto2\";\n"
+ + "package com.floci;\n"
+ + "message Person {\n"
+ + " required string first_name = 1;\n"
+ + " required string last_name = 2;\n"
+ + " required string email = 3;\n"
+ + " optional string phone = 4;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_COMPAT_REMOVE_REQUIRED_EMAIL =
+ "syntax = \"proto2\";\n"
+ + "package com.floci;\n"
+ + "message Person {\n"
+ + " required string first_name = 1;\n"
+ + " required string last_name = 2;\n"
+ + " optional string phone = 4;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_COMPAT_ADD_REQUIRED_ZIP =
+ "syntax = \"proto2\";\n"
+ + "package com.floci;\n"
+ + "message Person {\n"
+ + " required string first_name = 1;\n"
+ + " required string last_name = 2;\n"
+ + " required string email = 3;\n"
+ + " optional string phone = 4;\n"
+ + " required string zip = 5;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_COMPAT_ADD_OPTIONAL_ZIP =
+ "syntax = \"proto2\";\n"
+ + "package com.floci;\n"
+ + "message Person {\n"
+ + " required string first_name = 1;\n"
+ + " required string last_name = 2;\n"
+ + " required string email = 3;\n"
+ + " optional string phone = 4;\n"
+ + " optional string zip = 5;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_COMPAT_FORWARD_BASE =
+ "syntax = \"proto2\";\n"
+ + "package com.floci;\n"
+ + "message Person {\n"
+ + " required string first_name = 1;\n"
+ + " required string last_name = 2;\n"
+ + " optional string email = 3;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_COMPAT_ADD_REQUIRED_PHONE =
+ "syntax = \"proto2\";\n"
+ + "package com.floci;\n"
+ + "message Person {\n"
+ + " required string first_name = 1;\n"
+ + " required string last_name = 2;\n"
+ + " optional string email = 3;\n"
+ + " required string phone = 4;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_COMPAT_DELETE_REQUIRED_FIRST_NAME =
+ "syntax = \"proto2\";\n"
+ + "package com.floci;\n"
+ + "message Person {\n"
+ + " required string last_name = 2;\n"
+ + " optional string email = 3;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_COMPAT_OPTIONAL_FIRST_NAME_BASE =
+ "syntax = \"proto2\";\n"
+ + "package com.floci;\n"
+ + "message Person {\n"
+ + " optional string first_name = 1;\n"
+ + " required string last_name = 2;\n"
+ + " optional string email = 3;\n"
+ + "}\n";
+
+ @BeforeAll
+ static void configureAwsDefaultsForSchemaRegistrySerde() {
+ System.setProperty("aws.accessKeyId", "test");
+ System.setProperty("aws.secretAccessKey", "test");
+ System.setProperty("aws.secretKey", "test");
+ System.setProperty("aws.region", "us-east-1");
+ }
+
+ @Test
+ void sdkClientCanUseDefaultRegistryAndListRegistryResources() {
+ String schemaName = TestFixtures.uniqueName("default-orders");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ var created = glue.createSchema(CreateSchemaRequest.builder()
+ .schemaName(schemaName)
+ .dataFormat(DataFormat.AVRO)
+ .schemaDefinition(AVRO_V1)
+ .build());
+ try {
+ assertThat(created.registryName()).isEqualTo(AWSSchemaRegistryConstants.DEFAULT_REGISTRY_NAME);
+ assertThat(created.compatibility()).isEqualTo(Compatibility.BACKWARD);
+ assertThat(created.schemaCheckpoint()).isEqualTo(1L);
+ assertThat(created.latestSchemaVersion()).isEqualTo(1L);
+ assertThat(created.nextSchemaVersion()).isEqualTo(2L);
+
+ assertThat(glue.getRegistry(GetRegistryRequest.builder().build()).registryName())
+ .isEqualTo(AWSSchemaRegistryConstants.DEFAULT_REGISTRY_NAME);
+ var updatedRegistry = glue.updateRegistry(UpdateRegistryRequest.builder()
+ .registryId(RegistryId.builder()
+ .registryName(AWSSchemaRegistryConstants.DEFAULT_REGISTRY_NAME)
+ .build())
+ .description("default registry updated through SDK")
+ .build());
+ assertThat(updatedRegistry.registryName()).isEqualTo(AWSSchemaRegistryConstants.DEFAULT_REGISTRY_NAME);
+ assertThat(glue.getRegistry(GetRegistryRequest.builder().build()).description())
+ .isEqualTo("default registry updated through SDK");
+
+ assertThat(glue.listRegistries(ListRegistriesRequest.builder().build()).registries())
+ .anyMatch(registry -> AWSSchemaRegistryConstants.DEFAULT_REGISTRY_NAME.equals(registry.registryName()));
+ assertThat(glue.listSchemas(ListSchemasRequest.builder()
+ .registryId(RegistryId.builder()
+ .registryName(AWSSchemaRegistryConstants.DEFAULT_REGISTRY_NAME)
+ .build())
+ .build()).schemas())
+ .anyMatch(schema -> schemaName.equals(schema.schemaName()));
+ } finally {
+ glue.deleteSchema(DeleteSchemaRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(AWSSchemaRegistryConstants.DEFAULT_REGISTRY_NAME)
+ .schemaName(schemaName)
+ .build())
+ .build());
+ }
+ }
+ }
+
+ @Test
+ void sdkClientCanUseJsonAndProtobufSchemaFormats() {
+ String registryName = TestFixtures.uniqueName("java-gsr-format");
+ String jsonSchemaName = TestFixtures.uniqueName("json");
+ String protobufSchemaName = TestFixtures.uniqueName("protobuf");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .build());
+ try {
+ var json = glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(jsonSchemaName)
+ .dataFormat(DataFormat.JSON)
+ .compatibility(Compatibility.NONE)
+ .schemaDefinition(JSON_SCHEMA)
+ .build());
+ assertThat(json.dataFormat()).isEqualTo(DataFormat.JSON);
+ assertThat(glue.getSchemaVersion(GetSchemaVersionRequest.builder()
+ .schemaVersionId(json.schemaVersionId())
+ .build()).schemaDefinition()).isEqualTo(JSON_SCHEMA);
+
+ var protobuf = glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(protobufSchemaName)
+ .dataFormat(DataFormat.PROTOBUF)
+ .compatibility(Compatibility.NONE)
+ .schemaDefinition(PROTOBUF_SCHEMA)
+ .build());
+ assertThat(protobuf.dataFormat()).isEqualTo(DataFormat.PROTOBUF);
+ assertThat(glue.getSchemaVersion(GetSchemaVersionRequest.builder()
+ .schemaVersionId(protobuf.schemaVersionId())
+ .build()).schemaDefinition()).isEqualTo(PROTOBUF_SCHEMA);
+
+ assertThat(glue.checkSchemaVersionValidity(CheckSchemaVersionValidityRequest.builder()
+ .dataFormat(DataFormat.JSON)
+ .schemaDefinition(JSON_SCHEMA)
+ .build()).valid()).isTrue();
+ var invalidJson = glue.checkSchemaVersionValidity(CheckSchemaVersionValidityRequest.builder()
+ .dataFormat(DataFormat.JSON)
+ .schemaDefinition("{not-json")
+ .build());
+ assertThat(invalidJson.valid()).isFalse();
+ assertThat(invalidJson.error()).isNotBlank();
+
+ assertThat(glue.checkSchemaVersionValidity(CheckSchemaVersionValidityRequest.builder()
+ .dataFormat(DataFormat.PROTOBUF)
+ .schemaDefinition(PROTOBUF_SCHEMA)
+ .build()).valid()).isTrue();
+ } finally {
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+ }
+ }
+ }
+
+ @Test
+ void sdkClientEnforcesDocumentedJsonAndProtobufCompatibilityExamples() {
+ String registryName = TestFixtures.uniqueName("java-gsr-format-compat");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .build());
+ try {
+ assertSchemaEvolution(glue, registryName, "json-backward-closed",
+ DataFormat.JSON, Compatibility.BACKWARD,
+ JSON_COMPAT_BASE_CLOSED, JSON_COMPAT_ADD_OPTIONAL_PHONE, true);
+ assertSchemaEvolution(glue, registryName, "json-backward-open",
+ DataFormat.JSON, Compatibility.BACKWARD,
+ JSON_COMPAT_BASE_OPEN, JSON_COMPAT_ADD_OPTIONAL_PHONE, false);
+ assertSchemaEvolution(glue, registryName, "json-forward-closed",
+ DataFormat.JSON, Compatibility.FORWARD,
+ JSON_COMPAT_WITH_OPTIONAL_PHONE, JSON_COMPAT_DELETE_OPTIONAL_PHONE_CLOSED, true);
+ assertSchemaEvolution(glue, registryName, "json-forward-open",
+ DataFormat.JSON, Compatibility.FORWARD,
+ JSON_COMPAT_WITH_OPTIONAL_PHONE, JSON_COMPAT_DELETE_OPTIONAL_PHONE_OPEN, false);
+
+ assertSchemaEvolution(glue, registryName, "protobuf-backward-remove-required",
+ DataFormat.PROTOBUF, Compatibility.BACKWARD,
+ PROTOBUF_COMPAT_BACKWARD_BASE, PROTOBUF_COMPAT_REMOVE_REQUIRED_EMAIL, false);
+ assertSchemaEvolution(glue, registryName, "protobuf-backward-add-required",
+ DataFormat.PROTOBUF, Compatibility.BACKWARD,
+ PROTOBUF_COMPAT_BACKWARD_BASE, PROTOBUF_COMPAT_ADD_REQUIRED_ZIP, false);
+ assertSchemaEvolution(glue, registryName, "protobuf-backward-add-optional",
+ DataFormat.PROTOBUF, Compatibility.BACKWARD,
+ PROTOBUF_COMPAT_BACKWARD_BASE, PROTOBUF_COMPAT_ADD_OPTIONAL_ZIP, true);
+ assertSchemaEvolution(glue, registryName, "protobuf-forward-add-required",
+ DataFormat.PROTOBUF, Compatibility.FORWARD,
+ PROTOBUF_COMPAT_FORWARD_BASE, PROTOBUF_COMPAT_ADD_REQUIRED_PHONE, false);
+ assertSchemaEvolution(glue, registryName, "protobuf-forward-delete-required",
+ DataFormat.PROTOBUF, Compatibility.FORWARD,
+ PROTOBUF_COMPAT_FORWARD_BASE, PROTOBUF_COMPAT_DELETE_REQUIRED_FIRST_NAME, false);
+ assertSchemaEvolution(glue, registryName, "protobuf-forward-delete-optional",
+ DataFormat.PROTOBUF, Compatibility.FORWARD,
+ PROTOBUF_COMPAT_OPTIONAL_FIRST_NAME_BASE, PROTOBUF_COMPAT_DELETE_REQUIRED_FIRST_NAME, true);
+ } finally {
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+ }
+ }
+ }
+
+ @Test
+ void sdkClientRejectsSchemaDefinitionsOverDocumentedPayloadLimit() {
+ String registryName = TestFixtures.uniqueName("java-gsr-quota");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .build());
+ try {
+ assertThatThrownBy(() -> glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(TestFixtures.uniqueName("too-large"))
+ .dataFormat(DataFormat.AVRO)
+ .schemaDefinition("x".repeat(170_001))
+ .build()))
+ .isInstanceOf(InvalidInputException.class);
+ } finally {
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+ }
+ }
+ }
+
+ @ParameterizedTest(name = "SDK client can use {0} compatibility")
+ @EnumSource(value = Compatibility.class, names = {
+ "NONE",
+ "DISABLED",
+ "BACKWARD",
+ "BACKWARD_ALL",
+ "FORWARD",
+ "FORWARD_ALL",
+ "FULL",
+ "FULL_ALL"
+ })
+ void sdkClientCanCreateAndUpdateSchemasWithEachCompatibilityMode(Compatibility compatibility) {
+ String registryName = TestFixtures.uniqueName("java-gsr-compat");
+ String createdSchemaName = TestFixtures.uniqueName("created");
+ String updatedSchemaName = TestFixtures.uniqueName("updated");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .build());
+ try {
+ glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(createdSchemaName)
+ .dataFormat(DataFormat.AVRO)
+ .compatibility(compatibility)
+ .schemaDefinition(AVRO_V1)
+ .build());
+ assertThat(glue.getSchema(GetSchemaRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(createdSchemaName)
+ .build())
+ .build()).compatibility()).isEqualTo(compatibility);
+
+ glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(updatedSchemaName)
+ .dataFormat(DataFormat.AVRO)
+ .compatibility(Compatibility.BACKWARD)
+ .schemaDefinition(AVRO_V1)
+ .build());
+ glue.updateSchema(UpdateSchemaRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(updatedSchemaName)
+ .build())
+ .compatibility(compatibility)
+ .build());
+ assertThat(glue.getSchema(GetSchemaRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(updatedSchemaName)
+ .build())
+ .build()).compatibility()).isEqualTo(compatibility);
+ } finally {
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+ }
+ }
+ }
+
+ @ParameterizedTest(name = "{0} {1} is {3}")
+ @MethodSource("avroCompatibilityCases")
+ void sdkClientEnforcesAvroCompatibilityModePermutations(
+ Compatibility compatibility,
+ String change,
+ String nextSchemaDefinition,
+ boolean allowed) {
+ String registryName = TestFixtures.uniqueName("java-gsr-compat-rule");
+ String schemaName = TestFixtures.uniqueName(change.toLowerCase().replace('_', '-'));
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .build());
+ try {
+ glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(schemaName)
+ .dataFormat(DataFormat.AVRO)
+ .compatibility(compatibility)
+ .schemaDefinition(AVRO_COMPAT_BASE)
+ .build());
+
+ var register = RegisterSchemaVersionRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(schemaName)
+ .build())
+ .schemaDefinition(nextSchemaDefinition)
+ .build();
+ if (allowed) {
+ assertThat(glue.registerSchemaVersion(register).versionNumber()).isEqualTo(2L);
+ } else {
+ assertThatThrownBy(() -> glue.registerSchemaVersion(register))
+ .isInstanceOf(InvalidInputException.class);
+ }
+ } finally {
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+ }
+ }
+ }
+
+ @ParameterizedTest(name = "{1} checks all previous versions")
+ @MethodSource("avroTransitiveCompatibilityCases")
+ void sdkClientEnforcesAvroTransitiveCompatibilityModes(
+ Compatibility nonTransitiveCompatibility,
+ Compatibility transitiveCompatibility,
+ String firstSchemaDefinition,
+ String secondSchemaDefinition,
+ String candidateSchemaDefinition) {
+ String registryName = TestFixtures.uniqueName("java-gsr-transitive");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .build());
+ try {
+ String nonTransitiveSchemaName = TestFixtures.uniqueName("non-transitive");
+ seedSchemaHistory(glue, registryName, nonTransitiveSchemaName,
+ firstSchemaDefinition, secondSchemaDefinition);
+ glue.updateSchema(UpdateSchemaRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(nonTransitiveSchemaName)
+ .build())
+ .compatibility(nonTransitiveCompatibility)
+ .build());
+ assertThat(glue.registerSchemaVersion(RegisterSchemaVersionRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(nonTransitiveSchemaName)
+ .build())
+ .schemaDefinition(candidateSchemaDefinition)
+ .build()).versionNumber()).isEqualTo(3L);
+
+ String transitiveSchemaName = TestFixtures.uniqueName("transitive");
+ seedSchemaHistory(glue, registryName, transitiveSchemaName,
+ firstSchemaDefinition, secondSchemaDefinition);
+ glue.updateSchema(UpdateSchemaRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(transitiveSchemaName)
+ .build())
+ .compatibility(transitiveCompatibility)
+ .build());
+ assertThatThrownBy(() -> glue.registerSchemaVersion(RegisterSchemaVersionRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(transitiveSchemaName)
+ .build())
+ .schemaDefinition(candidateSchemaDefinition)
+ .build()))
+ .isInstanceOf(InvalidInputException.class);
+ } finally {
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+ }
+ }
+ }
+
+ @Test
+ void sdkClientCanManageTagsAndSchemaVersionMetadata() {
+ String registryName = TestFixtures.uniqueName("java-gsr-meta");
+ String schemaName = TestFixtures.uniqueName("meta-schema");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ var registry = glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .tags(Map.of("env", "test"))
+ .build());
+ try {
+ assertThat(glue.getTags(GetTagsRequest.builder()
+ .resourceArn(registry.registryArn())
+ .build()).tags()).containsEntry("env", "test");
+
+ glue.tagResource(TagResourceRequest.builder()
+ .resourceArn(registry.registryArn())
+ .tagsToAdd(Map.of("team", "platform"))
+ .build());
+ assertThat(glue.getTags(GetTagsRequest.builder()
+ .resourceArn(registry.registryArn())
+ .build()).tags())
+ .containsEntry("env", "test")
+ .containsEntry("team", "platform");
+
+ glue.untagResource(UntagResourceRequest.builder()
+ .resourceArn(registry.registryArn())
+ .tagsToRemove("env")
+ .build());
+ assertThat(glue.getTags(GetTagsRequest.builder()
+ .resourceArn(registry.registryArn())
+ .build()).tags())
+ .doesNotContainKey("env")
+ .containsEntry("team", "platform");
+
+ var schema = glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(schemaName)
+ .dataFormat(DataFormat.AVRO)
+ .compatibility(Compatibility.BACKWARD)
+ .schemaDefinition(AVRO_V1)
+ .tags(Map.of("purpose", "metadata"))
+ .build());
+ assertThat(glue.getTags(GetTagsRequest.builder()
+ .resourceArn(schema.schemaArn())
+ .build()).tags()).containsEntry("purpose", "metadata");
+
+ var putMetadata = glue.putSchemaVersionMetadata(PutSchemaVersionMetadataRequest.builder()
+ .schemaVersionId(schema.schemaVersionId())
+ .metadataKeyValue(MetadataKeyValuePair.builder()
+ .metadataKey("stage")
+ .metadataValue("prod")
+ .build())
+ .build());
+ assertThat(putMetadata.registryName()).isEqualTo(registryName);
+ assertThat(putMetadata.schemaName()).isEqualTo(schemaName);
+ assertThat(putMetadata.latestVersion()).isTrue();
+ var metadata = glue.querySchemaVersionMetadata(QuerySchemaVersionMetadataRequest.builder()
+ .schemaVersionId(schema.schemaVersionId())
+ .build());
+ assertThat(metadata.metadataInfoMap()).containsKey("stage");
+ assertThat(metadata.metadataInfoMap().get("stage").metadataValue()).isEqualTo("prod");
+
+ glue.putSchemaVersionMetadata(PutSchemaVersionMetadataRequest.builder()
+ .schemaVersionId(schema.schemaVersionId())
+ .metadataKeyValue(MetadataKeyValuePair.builder()
+ .metadataKey("stage")
+ .metadataValue("qa")
+ .build())
+ .build());
+ var updatedMetadata = glue.querySchemaVersionMetadata(QuerySchemaVersionMetadataRequest.builder()
+ .schemaVersionId(schema.schemaVersionId())
+ .metadataList(MetadataKeyValuePair.builder()
+ .metadataKey("stage")
+ .metadataValue("prod")
+ .build())
+ .build());
+ assertThat(updatedMetadata.metadataInfoMap().get("stage").metadataValue()).isEqualTo("qa");
+ assertThat(updatedMetadata.metadataInfoMap().get("stage").otherMetadataValueList())
+ .anyMatch(item -> "prod".equals(item.metadataValue()));
+
+ var removeMetadata = glue.removeSchemaVersionMetadata(RemoveSchemaVersionMetadataRequest.builder()
+ .schemaVersionId(schema.schemaVersionId())
+ .metadataKeyValue(MetadataKeyValuePair.builder()
+ .metadataKey("stage")
+ .metadataValue("qa")
+ .build())
+ .build());
+ assertThat(removeMetadata.registryName()).isEqualTo(registryName);
+ assertThat(removeMetadata.schemaName()).isEqualTo(schemaName);
+ assertThat(removeMetadata.latestVersion()).isTrue();
+ var afterRemoval = glue.querySchemaVersionMetadata(QuerySchemaVersionMetadataRequest.builder()
+ .schemaVersionId(schema.schemaVersionId())
+ .build());
+ assertThat(afterRemoval.metadataInfoMap().get("stage").metadataValue()).isEqualTo("prod");
+ } finally {
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+ }
+ }
+ }
+
+ @Test
+ void sdkClientCanManageSchemaRegistryWithPaginationAndCheckpointDeletion() {
+ String registryName = TestFixtures.uniqueName("java-gsr");
+ String schemaName = TestFixtures.uniqueName("orders");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ var registry = glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .build());
+ assertThat(registry.registryName()).isEqualTo(registryName);
+ assertThat(registry.registryArn()).contains(":registry/" + registryName);
+
+ var created = glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(schemaName)
+ .dataFormat(DataFormat.AVRO)
+ .compatibility(Compatibility.BACKWARD)
+ .schemaDefinition(AVRO_V1)
+ .build());
+ assertThat(created.schemaName()).isEqualTo(schemaName);
+ assertThat(created.schemaVersionId()).isNotBlank();
+
+ var registered = glue.registerSchemaVersion(RegisterSchemaVersionRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .schemaDefinition(AVRO_V2)
+ .build());
+ assertThat(registered.versionNumber()).isEqualTo(2L);
+
+ var byDefinition = glue.getSchemaByDefinition(GetSchemaByDefinitionRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .schemaDefinition(AVRO_V2)
+ .build());
+ assertThat(byDefinition.schemaVersionId()).isEqualTo(registered.schemaVersionId());
+ assertThat(byDefinition.dataFormat()).isEqualTo(DataFormat.AVRO);
+
+ var diff = glue.getSchemaVersionsDiff(GetSchemaVersionsDiffRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .firstSchemaVersionNumber(SchemaVersionNumber.builder().versionNumber(1L).build())
+ .secondSchemaVersionNumber(SchemaVersionNumber.builder().versionNumber(2L).build())
+ .schemaDiffType(SchemaDiffType.SYNTAX_DIFF)
+ .build());
+ assertThat(diff.diff()).contains("--- v1", "+++ v2", "amount");
+
+ var compatibilityUpdate = glue.updateSchema(UpdateSchemaRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .compatibility(Compatibility.FORWARD)
+ .build());
+ assertThat(compatibilityUpdate.schemaName()).isEqualTo(schemaName);
+ assertThat(compatibilityUpdate.registryName()).isEqualTo(registryName);
+ assertThat(glue.getSchema(GetSchemaRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .build()).compatibility()).isEqualTo(Compatibility.FORWARD);
+
+ var firstPage = glue.listSchemaVersions(ListSchemaVersionsRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .maxResults(1)
+ .build());
+ assertThat(firstPage.schemas()).hasSize(1);
+ assertThat(firstPage.nextToken()).isNotBlank();
+
+ var secondPage = glue.listSchemaVersions(ListSchemaVersionsRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .maxResults(1)
+ .nextToken(firstPage.nextToken())
+ .build());
+ assertThat(secondPage.schemas()).hasSize(1);
+ assertThat(secondPage.nextToken()).isNull();
+
+ var checkpointUpdate = glue.updateSchema(UpdateSchemaRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .schemaVersionNumber(SchemaVersionNumber.builder().versionNumber(2L).build())
+ .build());
+ assertThat(checkpointUpdate.schemaName()).isEqualTo(schemaName);
+ assertThat(checkpointUpdate.registryName()).isEqualTo(registryName);
+
+ var deleted = glue.deleteSchemaVersions(DeleteSchemaVersionsRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .versions("1")
+ .build());
+ assertThat(deleted.schemaVersionErrors()).isEmpty();
+
+ var latest = glue.getSchemaVersion(GetSchemaVersionRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .schemaVersionNumber(SchemaVersionNumber.builder().latestVersion(true).build())
+ .build());
+ assertThat(latest.versionNumber()).isEqualTo(2L);
+
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+
+ assertThatThrownBy(() -> glue.getSchema(GetSchemaRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .build()))
+ .isInstanceOf(EntityNotFoundException.class);
+ }
+ }
+
+ @Test
+ void kafkaAvroSerializerAndDeserializerRoundTripThroughFlociEndpoint() {
+ String registryName = TestFixtures.uniqueName("java-gsr-serde");
+ String schemaName = TestFixtures.uniqueName("orders-serde");
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ glue.createRegistry(CreateRegistryRequest.builder()
+ .registryName(registryName)
+ .build());
+
+ Schema schema = new Schema.Parser().parse(AVRO_V1);
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("id", 42L);
+
+ Map<String, Object> configs = new HashMap<>();
+ configs.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, TestFixtures.endpoint().toString());
+ configs.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1");
+ configs.put(AWSSchemaRegistryConstants.DATA_FORMAT, DataFormat.AVRO.name());
+ configs.put(AWSSchemaRegistryConstants.REGISTRY_NAME, registryName);
+ configs.put(AWSSchemaRegistryConstants.SCHEMA_NAME, schemaName);
+ configs.put(AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, true);
+ configs.put(AWSSchemaRegistryConstants.COMPRESSION_TYPE, AWSSchemaRegistryConstants.COMPRESSION.ZLIB.name());
+ configs.put(AWSSchemaRegistryConstants.AVRO_RECORD_TYPE, AvroRecordType.GENERIC_RECORD.getName());
+
+ GlueSchemaRegistryKafkaSerializer serializer = new GlueSchemaRegistryKafkaSerializer();
+ GlueSchemaRegistryKafkaDeserializer deserializer = new GlueSchemaRegistryKafkaDeserializer();
+ try {
+ serializer.configure(configs, false);
+ deserializer.configure(configs, false);
+
+ byte[] bytes = serializer.serialize("orders-topic", record);
+ Object decoded = deserializer.deserialize("orders-topic", bytes);
+
+ assertThat(decoded).isInstanceOf(GenericRecord.class);
+ assertThat(((GenericRecord) decoded).get("id")).isEqualTo(42L);
+ assertThat(glue.getSchema(GetSchemaRequest.builder()
+ .schemaId(SchemaId.builder().registryName(registryName).schemaName(schemaName).build())
+ .build()).schemaName()).isEqualTo(schemaName);
+ } finally {
+ serializer.close();
+ deserializer.close();
+ glue.deleteRegistry(DeleteRegistryRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .build());
+ }
+ }
+ }
+
+ private static Stream<Arguments> avroCompatibilityCases() {
+ return Stream.of(
+ arguments(Compatibility.NONE, "ADD_REQUIRED", AVRO_COMPAT_ADD_REQUIRED, true),
+ arguments(Compatibility.NONE, "ADD_OPTIONAL", AVRO_COMPAT_ADD_OPTIONAL, true),
+ arguments(Compatibility.NONE, "DELETE_REQUIRED", AVRO_COMPAT_DELETE_REQUIRED, true),
+ arguments(Compatibility.NONE, "DELETE_OPTIONAL", AVRO_COMPAT_DELETE_OPTIONAL, true),
+ arguments(Compatibility.DISABLED, "ADD_REQUIRED", AVRO_COMPAT_ADD_REQUIRED, false),
+ arguments(Compatibility.DISABLED, "ADD_OPTIONAL", AVRO_COMPAT_ADD_OPTIONAL, false),
+ arguments(Compatibility.DISABLED, "DELETE_REQUIRED", AVRO_COMPAT_DELETE_REQUIRED, false),
+ arguments(Compatibility.DISABLED, "DELETE_OPTIONAL", AVRO_COMPAT_DELETE_OPTIONAL, false),
+ arguments(Compatibility.BACKWARD, "ADD_REQUIRED", AVRO_COMPAT_ADD_REQUIRED, false),
+ arguments(Compatibility.BACKWARD, "ADD_OPTIONAL", AVRO_COMPAT_ADD_OPTIONAL, true),
+ arguments(Compatibility.BACKWARD, "DELETE_REQUIRED", AVRO_COMPAT_DELETE_REQUIRED, true),
+ arguments(Compatibility.BACKWARD, "DELETE_OPTIONAL", AVRO_COMPAT_DELETE_OPTIONAL, true),
+ arguments(Compatibility.BACKWARD_ALL, "ADD_REQUIRED", AVRO_COMPAT_ADD_REQUIRED, false),
+ arguments(Compatibility.BACKWARD_ALL, "ADD_OPTIONAL", AVRO_COMPAT_ADD_OPTIONAL, true),
+ arguments(Compatibility.BACKWARD_ALL, "DELETE_REQUIRED", AVRO_COMPAT_DELETE_REQUIRED, true),
+ arguments(Compatibility.BACKWARD_ALL, "DELETE_OPTIONAL", AVRO_COMPAT_DELETE_OPTIONAL, true),
+ arguments(Compatibility.FORWARD, "ADD_REQUIRED", AVRO_COMPAT_ADD_REQUIRED, true),
+ arguments(Compatibility.FORWARD, "ADD_OPTIONAL", AVRO_COMPAT_ADD_OPTIONAL, true),
+ arguments(Compatibility.FORWARD, "DELETE_REQUIRED", AVRO_COMPAT_DELETE_REQUIRED, false),
+ arguments(Compatibility.FORWARD, "DELETE_OPTIONAL", AVRO_COMPAT_DELETE_OPTIONAL, true),
+ arguments(Compatibility.FORWARD_ALL, "ADD_REQUIRED", AVRO_COMPAT_ADD_REQUIRED, true),
+ arguments(Compatibility.FORWARD_ALL, "ADD_OPTIONAL", AVRO_COMPAT_ADD_OPTIONAL, true),
+ arguments(Compatibility.FORWARD_ALL, "DELETE_REQUIRED", AVRO_COMPAT_DELETE_REQUIRED, false),
+ arguments(Compatibility.FORWARD_ALL, "DELETE_OPTIONAL", AVRO_COMPAT_DELETE_OPTIONAL, true),
+ arguments(Compatibility.FULL, "ADD_REQUIRED", AVRO_COMPAT_ADD_REQUIRED, false),
+ arguments(Compatibility.FULL, "ADD_OPTIONAL", AVRO_COMPAT_ADD_OPTIONAL, true),
+ arguments(Compatibility.FULL, "DELETE_REQUIRED", AVRO_COMPAT_DELETE_REQUIRED, false),
+ arguments(Compatibility.FULL, "DELETE_OPTIONAL", AVRO_COMPAT_DELETE_OPTIONAL, true),
+ arguments(Compatibility.FULL_ALL, "ADD_REQUIRED", AVRO_COMPAT_ADD_REQUIRED, false),
+ arguments(Compatibility.FULL_ALL, "ADD_OPTIONAL", AVRO_COMPAT_ADD_OPTIONAL, true),
+ arguments(Compatibility.FULL_ALL, "DELETE_REQUIRED", AVRO_COMPAT_DELETE_REQUIRED, false),
+ arguments(Compatibility.FULL_ALL, "DELETE_OPTIONAL", AVRO_COMPAT_DELETE_OPTIONAL, true)
+ );
+ }
+
+ private static Stream<Arguments> avroTransitiveCompatibilityCases() {
+ return Stream.of(
+ arguments(Compatibility.BACKWARD, Compatibility.BACKWARD_ALL,
+ AVRO_TRANSITIVE_ID_ONLY,
+ AVRO_TRANSITIVE_ID_EMAIL_REQUIRED,
+ AVRO_TRANSITIVE_ID_EMAIL_PHONE_OPTIONAL),
+ arguments(Compatibility.FORWARD, Compatibility.FORWARD_ALL,
+ AVRO_TRANSITIVE_ID_EMAIL_REQUIRED,
+ AVRO_TRANSITIVE_ID_ONLY,
+ AVRO_TRANSITIVE_ID_PHONE_OPTIONAL),
+ arguments(Compatibility.FULL, Compatibility.FULL_ALL,
+ AVRO_TRANSITIVE_ID_EMAIL_REQUIRED,
+ AVRO_TRANSITIVE_ID_ONLY,
+ AVRO_TRANSITIVE_ID_PHONE_OPTIONAL)
+ );
+ }
+
+ private static void seedSchemaHistory(
+ GlueClient glue,
+ String registryName,
+ String schemaName,
+ String firstSchemaDefinition,
+ String secondSchemaDefinition) {
+ glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(schemaName)
+ .dataFormat(DataFormat.AVRO)
+ .compatibility(Compatibility.NONE)
+ .schemaDefinition(firstSchemaDefinition)
+ .build());
+ glue.registerSchemaVersion(RegisterSchemaVersionRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(schemaName)
+ .build())
+ .schemaDefinition(secondSchemaDefinition)
+ .build());
+ }
+
+ private static void assertSchemaEvolution(
+ GlueClient glue,
+ String registryName,
+ String scenario,
+ DataFormat dataFormat,
+ Compatibility compatibility,
+ String firstSchemaDefinition,
+ String nextSchemaDefinition,
+ boolean allowed) {
+ String schemaName = TestFixtures.uniqueName(scenario);
+ glue.createSchema(CreateSchemaRequest.builder()
+ .registryId(RegistryId.builder().registryName(registryName).build())
+ .schemaName(schemaName)
+ .dataFormat(dataFormat)
+ .compatibility(compatibility)
+ .schemaDefinition(firstSchemaDefinition)
+ .build());
+
+ var register = RegisterSchemaVersionRequest.builder()
+ .schemaId(SchemaId.builder()
+ .registryName(registryName)
+ .schemaName(schemaName)
+ .build())
+ .schemaDefinition(nextSchemaDefinition)
+ .build();
+ if (allowed) {
+ assertThat(glue.registerSchemaVersion(register).versionNumber()).isEqualTo(2L);
+ } else {
+ assertThatThrownBy(() -> glue.registerSchemaVersion(register))
+ .isInstanceOf(InvalidInputException.class);
+ }
+ }
+
+ private static Arguments arguments(
+ Compatibility compatibility,
+ String change,
+ String nextSchemaDefinition,
+ boolean allowed) {
+ return Arguments.of(compatibility, change, nextSchemaDefinition, allowed);
+ }
+
+ private static Arguments arguments(
+ Compatibility nonTransitiveCompatibility,
+ Compatibility transitiveCompatibility,
+ String firstSchemaDefinition,
+ String secondSchemaDefinition,
+ String candidateSchemaDefinition) {
+ return Arguments.of(
+ nonTransitiveCompatibility,
+ transitiveCompatibility,
+ firstSchemaDefinition,
+ secondSchemaDefinition,
+ candidateSchemaDefinition);
+ }
+}
diff --git a/docs/index.md b/docs/index.md
index 8b159139d9..ad7eff959f 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -35,6 +35,10 @@ Floci emulates 41 AWS services. See the [Services Overview](services/index.md) f
| STS | Query |
| ElastiCache (Redis / Valkey) | Query + RESP proxy |
| RDS (PostgreSQL / MySQL) | Query + wire proxy |
+| MSK (Kafka / Redpanda) | REST JSON + Kafka |
+| Athena | JSON 1.1 |
+| Glue Data Catalog + Schema Registry | JSON 1.1 |
+| Data Firehose | JSON 1.1 |
| ECS | JSON 1.1 |
| EC2 | EC2 Query |
| ACM | JSON 1.1 |
@@ -91,4 +95,4 @@ aws --endpoint-url http://localhost:4566 s3 mb s3://my-bucket
All 41 AWS services are immediately available at `http://localhost:4566`.
[Get started →](getting-started/quick-start.md){ .md-button .md-button--primary }
-[View services →](services/index.md){ .md-button }
\ No newline at end of file
+[View services →](services/index.md){ .md-button }
diff --git a/docs/services/glue.md b/docs/services/glue.md
index a0ebf0c2fe..b4c33a532d 100644
--- a/docs/services/glue.md
+++ b/docs/services/glue.md
@@ -3,23 +3,35 @@
**Protocol:** JSON 1.1
**Endpoint:** `http://localhost:4566/`
-Floci emulates the AWS Glue Data Catalog, allowing you to manage metadata for your data lake locally.
+Floci emulates the AWS Glue Data Catalog and Glue Schema Registry, allowing you to manage local data lake metadata and schema-version workflows.
## Supported Actions
-### Databases
-`CreateDatabase` · `GetDatabase` · `GetDatabases` · `DeleteDatabase` · `UpdateDatabase`
+### Data Catalog
-### Tables
-`CreateTable` · `GetTable` · `GetTables` · `DeleteTable` · `UpdateTable`
+| Area | Actions |
+|---|---|
+| Databases | `CreateDatabase` · `GetDatabase` · `GetDatabases` |
+| Tables | `CreateTable` · `GetTable` · `GetTables` · `DeleteTable` |
+| Partitions | `CreatePartition` · `GetPartitions` |
+
+### Schema Registry
-### Partitions
-`CreatePartition` · `BatchCreatePartition` · `GetPartition` · `GetPartitions` · `DeletePartition`
+| Area | Actions |
+|---|---|
+| Registries | `CreateRegistry` · `GetRegistry` · `ListRegistries` · `UpdateRegistry` · `DeleteRegistry` |
+| Schemas | `CreateSchema` · `GetSchema` · `ListSchemas` · `UpdateSchema` · `DeleteSchema` |
+| Versions | `RegisterSchemaVersion` · `GetSchemaByDefinition` · `GetSchemaVersion` · `ListSchemaVersions` · `DeleteSchemaVersions` · `GetSchemaVersionsDiff` · `CheckSchemaVersionValidity` |
+| Metadata and tags | `PutSchemaVersionMetadata` · `RemoveSchemaVersionMetadata` · `QuerySchemaVersionMetadata` · `TagResource` · `UntagResource` · `GetTags` |
+
+Supported schema formats are `AVRO`, `JSON`, and `PROTOBUF`. Compatibility modes are `NONE`, `DISABLED`, `BACKWARD`, `BACKWARD_ALL`, `FORWARD`, `FORWARD_ALL`, `FULL`, and `FULL_ALL`.
## Integration with Athena
The Glue Data Catalog is automatically used by **Athena** to resolve table names to S3 locations and formats. When you submit an Athena query, Floci reads all Glue tables for the target database and generates DuckDB views on top of the underlying S3 objects before executing the SQL.
+Tables can reference a Schema Registry schema version through `StorageDescriptor.SchemaReference`. On `GetTable` and `GetTables`, Floci resolves the schema definition into Glue columns when possible.
+
The DuckDB read function is selected based on the table's `StorageDescriptor.InputFormat` and `StorageDescriptor.SerdeInfo.SerializationLibrary`:
| Condition | DuckDB function |
@@ -29,7 +41,7 @@ The DuckDB read function is selected based on the table's `StorageDescriptor.Inp
| `InputFormat` contains `hive` | `read_json_auto` |
| Anything else | `read_csv_auto` |
-## Example
+## Data Catalog Example
```bash
export AWS_ENDPOINT_URL=http://localhost:4566
@@ -78,3 +90,38 @@ aws glue create-table \
}' \
--endpoint-url $AWS_ENDPOINT_URL
```
+
+## Schema Registry Example
+
+```bash
+export AWS_ENDPOINT_URL=http://localhost:4566
+
+cat > /tmp/order.avsc <<'JSON'
+{"type":"record","name":"Order","namespace":"example","fields":[{"name":"id","type":"long"}]}
+JSON
+
+cat > /tmp/order-v2.avsc <<'JSON'
+{"type":"record","name":"Order","namespace":"example","fields":[{"name":"id","type":"long"},{"name":"amount","type":["null","double"],"default":null}]}
+JSON
+
+aws glue create-registry \
+ --registry-name local-registry \
+ --endpoint-url $AWS_ENDPOINT_URL
+
+aws glue create-schema \
+ --registry-id RegistryName=local-registry \
+ --schema-name orders \
+ --data-format AVRO \
+ --compatibility BACKWARD \
+ --schema-definition file:///tmp/order.avsc \
+ --endpoint-url $AWS_ENDPOINT_URL
+
+aws glue register-schema-version \
+ --schema-id RegistryName=local-registry,SchemaName=orders \
+ --schema-definition file:///tmp/order-v2.avsc \
+ --endpoint-url $AWS_ENDPOINT_URL
+
+aws glue list-schema-versions \
+ --schema-id RegistryName=local-registry,SchemaName=orders \
+ --endpoint-url $AWS_ENDPOINT_URL
+```
diff --git a/docs/services/index.md b/docs/services/index.md
index 2502deba4c..8885eac7c4 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -2,7 +2,7 @@
Floci emulates 42 AWS services on a single port (`4566`). All services use the real AWS wire protocol, your existing AWS CLI commands and SDK clients work without modification.
-This page is the canonical reference for supported service counts and operation counts. Other docs (and the README) should link here rather than duplicating the table.
+This page is the canonical reference for supported service and operation counts. Some services expose separate control-plane and data-plane rows below. Other docs (and the README) should link here rather than duplicating the table.
## Service Matrix
@@ -33,6 +33,10 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [CloudWatch Metrics](cloudwatch.md#metrics) | `POST /` with `Action=` or JSON 1.1 | Query / JSON | 11 |
| [ElastiCache](elasticache.md) | `POST /` with `Action=` param + TCP proxy | Query + RESP | 8 |
| [RDS](rds.md) | `POST /` with `Action=` param + TCP proxy | Query + wire | 14 |
+| [MSK](msk.md) | `/v1/clusters/...`, `/api/v2/clusters/...` + Redpanda broker | REST JSON + Kafka | 8 |
+| [Athena](athena.md) | `POST /` + `X-Amz-Target: AmazonAthena.*` | JSON 1.1 | 4 |
+| [Glue](glue.md) | `POST /` + `X-Amz-Target: AWSGlue.*` | JSON 1.1 | 32 |
+| [Data Firehose](firehose.md) | `POST /` + `X-Amz-Target: Firehose_20150804.*` | JSON 1.1 | 6 |
| [ECS](ecs.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerServiceV20141113.*` | JSON 1.1 | 58 |
| [EC2](ec2.md) | `POST /` with `Action=` param | EC2 Query | 61 |
| [ACM](acm.md) | `POST /` + `X-Amz-Target: CertificateManager.*` | JSON 1.1 | 12 |
diff --git a/pom.xml b/pom.xml
index 723fd35714..5c2af51ae4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -16,6 +16,7 @@
<maven.compiler.release>25</maven.compiler.release>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<quarkus.platform.version>3.34.6</quarkus.platform.version>
+ <apicurio.version>2.6.9.Final</apicurio.version>
</properties>
<dependencyManagement>
@@ -167,6 +168,32 @@
<version>2.1.41</version>
</dependency>
+ <!-- Glue Schema Registry: Apicurio format utilities (parsing, canonicalization, compatibility) -->
+ <dependency>
+ <groupId>io.apicurio</groupId>
+ <artifactId>apicurio-registry-schema-util-avro</artifactId>
+ <version>${apicurio.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.apicurio</groupId>
+ <artifactId>apicurio-registry-schema-util-json</artifactId>
+ <version>${apicurio.version}</version>
+ <exclusions>
+ <!-- Apicurio's logging library brings a Servlet-based audit filter that
+ fails to wire under Quarkus REST (Vert.x). Schema-util code does not
+ reference it. -->
+ <exclusion>
+ <groupId>io.apicurio</groupId>
+ <artifactId>apicurio-common-app-components-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.apicurio</groupId>
+ <artifactId>apicurio-registry-schema-util-protobuf</artifactId>
+ <version>${apicurio.version}</version>
+ </dependency>
+
<!-- Testing -->
<dependency>
<groupId>io.quarkus</groupId>
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 4d97d71fec..bf674b3822 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -6,21 +6,36 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.glue.model.Database;
import io.github.hectorvent.floci.services.glue.model.Partition;
import io.github.hectorvent.floci.services.glue.model.Table;
+import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.Registry;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.RegistryId;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.Schema;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaVersion;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.ws.rs.core.Response;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Objects;
@ApplicationScoped
public class GlueJsonHandler {
private final GlueService glueService;
+ private final GlueSchemaRegistryService schemaRegistryService;
private final ObjectMapper mapper;
@Inject
- public GlueJsonHandler(GlueService glueService, ObjectMapper mapper) {
+ public GlueJsonHandler(GlueService glueService,
+ GlueSchemaRegistryService schemaRegistryService,
+ ObjectMapper mapper) {
this.glueService = glueService;
+ this.schemaRegistryService = schemaRegistryService;
this.mapper = mapper;
}
@@ -73,7 +88,420 @@ public class GlueJsonHandler {
String tableName = request.get("TableName").asText();
yield Response.ok(Map.of("Partitions", glueService.getPartitions(dbName, tableName))).build();
}
+ case "CreateRegistry" -> handleCreateRegistry(request, region);
+ case "GetRegistry" -> handleGetRegistry(request, region);
+ case "ListRegistries" -> handleListRegistries(request);
+ case "UpdateRegistry" -> handleUpdateRegistry(request, region);
+ case "DeleteRegistry" -> handleDeleteRegistry(request, region);
+ case "CreateSchema" -> handleCreateSchema(request, region);
+ case "RegisterSchemaVersion" -> handleRegisterSchemaVersion(request, region);
+ case "GetSchemaByDefinition" -> handleGetSchemaByDefinition(request, region);
+ case "GetSchemaVersion" -> handleGetSchemaVersion(request, region);
+ case "GetSchema" -> handleGetSchema(request, region);
+ case "UpdateSchema" -> handleUpdateSchema(request, region);
+ case "ListSchemas" -> handleListSchemas(request, region);
+ case "ListSchemaVersions" -> handleListSchemaVersions(request, region);
+ case "DeleteSchema" -> handleDeleteSchema(request, region);
+ case "DeleteSchemaVersions" -> handleDeleteSchemaVersions(request, region);
+ case "GetSchemaVersionsDiff" -> handleGetSchemaVersionsDiff(request, region);
+ case "CheckSchemaVersionValidity" -> handleCheckSchemaVersionValidity(request);
+ case "PutSchemaVersionMetadata" -> handlePutSchemaVersionMetadata(request);
+ case "RemoveSchemaVersionMetadata" -> handleRemoveSchemaVersionMetadata(request);
+ case "QuerySchemaVersionMetadata" -> handleQuerySchemaVersionMetadata(request);
+ case "TagResource" -> handleTagResource(request);
+ case "UntagResource" -> handleUntagResource(request);
+ case "GetTags" -> handleGetTags(request);
default -> throw new AwsException("InvalidAction", "Action " + action + " is not supported", 400);
};
}
+
+ private Response handleCreateRegistry(JsonNode request, String region) {
+ String name = request.path("RegistryName").asText(null);
+ String description = request.path("Description").asText(null);
+ @SuppressWarnings("unchecked")
+ Map<String, String> tags = request.has("Tags")
+ ? mapper.convertValue(request.get("Tags"), Map.class)
+ : null;
+ Registry registry = schemaRegistryService.createRegistry(name, description, tags, region);
+ return Response.ok(registry).build();
+ }
+
+ private Response handleGetRegistry(JsonNode request, String region) throws Exception {
+ RegistryId registryId = readRegistryId(request);
+ Registry registry = schemaRegistryService.getRegistry(registryId, region);
+ return Response.ok(registry).build();
+ }
+
+ private Response handleListRegistries(JsonNode request) {
+ var page = schemaRegistryService.listRegistries(readMaxResults(request), readNextToken(request));
+ return Response.ok(pageResponse("Registries", registryListItems(page.items()), page.nextToken())).build();
+ }
+
+ private Response handleUpdateRegistry(JsonNode request, String region) throws Exception {
+ RegistryId registryId = readRegistryId(request);
+ String description = request.path("Description").asText(null);
+ Registry registry = schemaRegistryService.updateRegistry(registryId, description, region);
+ return Response.ok(Map.of(
+ "RegistryName", registry.getRegistryName(),
+ "RegistryArn", registry.getRegistryArn()
+ )).build();
+ }
+
+ private Response handleDeleteRegistry(JsonNode request, String region) throws Exception {
+ RegistryId registryId = readRegistryId(request);
+ Registry registry = schemaRegistryService.deleteRegistry(registryId, region);
+ return Response.ok(Map.of(
+ "RegistryName", registry.getRegistryName(),
+ "RegistryArn", registry.getRegistryArn(),
+ "Status", registry.getStatus()
+ )).build();
+ }
+
+ private RegistryId readRegistryId(JsonNode request) throws Exception {
+ JsonNode node = request.get("RegistryId");
+ if (node == null || node.isNull()) {
+ return null;
+ }
+ return mapper.treeToValue(node, RegistryId.class);
+ }
+
+ private SchemaId readSchemaId(JsonNode request) throws Exception {
+ JsonNode node = request.get("SchemaId");
+ if (node == null || node.isNull()) {
+ return null;
+ }
+ return mapper.treeToValue(node, SchemaId.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ private Response handleCreateSchema(JsonNode request, String region) throws Exception {
+ RegistryId registryId = readRegistryId(request);
+ String schemaName = request.path("SchemaName").asText(null);
+ String dataFormat = request.path("DataFormat").asText(null);
+ String compatibility = request.path("Compatibility").asText(null);
+ String description = request.path("Description").asText(null);
+ String definition = request.path("SchemaDefinition").asText(null);
+ Map<String, String> tags = request.has("Tags")
+ ? mapper.convertValue(request.get("Tags"), Map.class)
+ : null;
+ var result = schemaRegistryService.createSchema(
+ registryId, schemaName, dataFormat, compatibility, description, definition, tags, region);
+ Schema schema = result.schema();
+ SchemaVersion version = result.firstVersion();
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("RegistryName", schema.getRegistryName());
+ response.put("RegistryArn", schema.getRegistryArn());
+ response.put("SchemaName", schema.getSchemaName());
+ response.put("SchemaArn", schema.getSchemaArn());
+ if (schema.getDescription() != null) response.put("Description", schema.getDescription());
+ response.put("DataFormat", schema.getDataFormat());
+ response.put("Compatibility", schema.getCompatibility());
+ response.put("SchemaCheckpoint", schema.getSchemaCheckpoint());
+ response.put("LatestSchemaVersion", schema.getLatestSchemaVersion());
+ response.put("NextSchemaVersion", schema.getNextSchemaVersion());
+ response.put("SchemaStatus", schema.getSchemaStatus());
+ if (schema.getTags() != null) response.put("Tags", schema.getTags());
+ response.put("SchemaVersionId", version.getSchemaVersionId());
+ response.put("SchemaVersionStatus", version.getStatus());
+ return Response.ok(response).build();
+ }
+
+ private Response handleRegisterSchemaVersion(JsonNode request, String region) throws Exception {
+ SchemaId schemaId = readSchemaId(request);
+ String definition = request.path("SchemaDefinition").asText(null);
+ SchemaVersion version = schemaRegistryService.registerSchemaVersion(schemaId, definition, region);
+ return Response.ok(Map.of(
+ "SchemaVersionId", version.getSchemaVersionId(),
+ "VersionNumber", version.getVersionNumber(),
+ "Status", version.getStatus()
+ )).build();
+ }
+
+ private Response handleGetSchemaByDefinition(JsonNode request, String region) throws Exception {
+ SchemaId schemaId = readSchemaId(request);
+ String definition = request.path("SchemaDefinition").asText(null);
+ SchemaVersion version = schemaRegistryService.getSchemaByDefinition(schemaId, definition, region);
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("SchemaVersionId", version.getSchemaVersionId());
+ response.put("SchemaArn", version.getSchemaArn());
+ response.put("DataFormat", version.getDataFormat());
+ response.put("Status", version.getStatus());
+ response.put("CreatedTime", iso(version.getCreatedTime()));
+ return Response.ok(response).build();
+ }
+
+ private Response handleGetSchemaVersion(JsonNode request, String region) throws Exception {
+ String schemaVersionId = request.path("SchemaVersionId").asText(null);
+ SchemaId schemaId = readSchemaId(request);
+ Long versionNumber = null;
+ boolean latestVersion = false;
+ JsonNode svn = request.get("SchemaVersionNumber");
+ if (svn != null && !svn.isNull()) {
+ JsonNode vn = svn.get("VersionNumber");
+ if (vn != null && !vn.isNull()) {
+ versionNumber = vn.asLong();
+ }
+ JsonNode lv = svn.get("LatestVersion");
+ if (lv != null && !lv.isNull()) {
+ latestVersion = lv.asBoolean();
+ }
+ }
+ SchemaVersion version = schemaRegistryService.getSchemaVersion(
+ schemaId, schemaVersionId, versionNumber, latestVersion, region);
+ return Response.ok(version).build();
+ }
+
+ private Response handleGetSchema(JsonNode request, String region) throws Exception {
+ SchemaId schemaId = readSchemaId(request);
+ Schema schema = schemaRegistryService.getSchema(schemaId, region);
+ return Response.ok(schema).build();
+ }
+
+ private Response handleUpdateSchema(JsonNode request, String region) throws Exception {
+ SchemaId schemaId = readSchemaId(request);
+ String compatibility = request.path("Compatibility").asText(null);
+ String description = request.path("Description").asText(null);
+ Long checkpointVersion = readVersionNumber(request, "SchemaVersionNumber");
+ Schema schema = schemaRegistryService.updateSchema(
+ schemaId, compatibility, description, checkpointVersion, region);
+ return Response.ok(Map.of(
+ "SchemaArn", schema.getSchemaArn(),
+ "SchemaName", schema.getSchemaName(),
+ "RegistryName", schema.getRegistryName()
+ )).build();
+ }
+
+ private Response handleListSchemas(JsonNode request, String region) throws Exception {
+ RegistryId registryId = readRegistryId(request);
+ var page = schemaRegistryService.listSchemas(
+ registryId, region, readMaxResults(request), readNextToken(request));
+ return Response.ok(pageResponse("Schemas", schemaListItems(page.items()), page.nextToken())).build();
+ }
+
+ private Response handleListSchemaVersions(JsonNode request, String region) throws Exception {
+ SchemaId schemaId = readSchemaId(request);
+ var page = schemaRegistryService.listSchemaVersions(
+ schemaId, region, readMaxResults(request), readNextToken(request));
+ return Response.ok(pageResponse("Schemas", schemaVersionListItems(page.items()), page.nextToken())).build();
+ }
+
+ private Response handleDeleteSchema(JsonNode request, String region) throws Exception {
+ SchemaId schemaId = readSchemaId(request);
+ Schema schema = schemaRegistryService.deleteSchema(schemaId, region);
+ return Response.ok(Map.of(
+ "SchemaArn", schema.getSchemaArn(),
+ "SchemaName", schema.getSchemaName(),
+ "Status", schema.getSchemaStatus()
+ )).build();
+ }
+
+ private Response handleDeleteSchemaVersions(JsonNode request, String region) throws Exception {
+ SchemaId schemaId = readSchemaId(request);
+ String versions = request.path("Versions").asText(null);
+ var results = schemaRegistryService.deleteSchemaVersions(schemaId, versions, region);
+
+ List<Map<String, Object>> errors = new ArrayList<>();
+ for (var r : results) {
+ if (r.errorCode() != null) {
+ Map<String, Object> err = new LinkedHashMap<>();
+ err.put("VersionNumber", r.versionNumber());
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("ErrorCode", r.errorCode());
+ details.put("ErrorMessage", r.errorMessage());
+ err.put("ErrorDetails", details);
+ errors.add(err);
+ }
+ }
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("SchemaVersionErrors", errors);
+ return Response.ok(response).build();
+ }
+
+ private Response handleGetSchemaVersionsDiff(JsonNode request, String region) throws Exception {
+ SchemaId schemaId = readSchemaId(request);
+ Long first = readVersionNumber(request, "FirstSchemaVersionNumber");
+ Long second = readVersionNumber(request, "SecondSchemaVersionNumber");
+ String diff = schemaRegistryService.getSchemaVersionsDiff(schemaId, first, second, region);
+ return Response.ok(Map.of("Diff", diff)).build();
+ }
+
+ private Response handleCheckSchemaVersionValidity(JsonNode request) {
+ String dataFormat = request.path("DataFormat").asText(null);
+ String definition = request.path("SchemaDefinition").asText(null);
+ var result = schemaRegistryService.checkSchemaVersionValidity(dataFormat, definition);
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("Valid", result.valid());
+ if (result.error() != null) {
+ response.put("Error", result.error());
+ }
+ return Response.ok(response).build();
+ }
+
+ private Long readVersionNumber(JsonNode request, String field) {
+ JsonNode node = request.get(field);
+ if (node == null || node.isNull()) {
+ return null;
+ }
+ JsonNode vn = node.get("VersionNumber");
+ if (vn == null || vn.isNull()) {
+ return null;
+ }
+ return vn.asLong();
+ }
+
+ private Integer readMaxResults(JsonNode request) {
+ JsonNode node = request.get("MaxResults");
+ if (node == null || node.isNull()) {
+ return null;
+ }
+ return node.asInt();
+ }
+
+ private String readNextToken(JsonNode request) {
+ JsonNode node = request.get("NextToken");
+ if (node == null || node.isNull()) {
+ return null;
+ }
+ return node.asText(null);
+ }
+
+ private Map<String, Object> pageResponse(String field, List<?> items, String nextToken) {
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put(field, items);
+ if (nextToken != null) {
+ response.put("NextToken", nextToken);
+ }
+ return response;
+ }
+
+ private List<Map<String, Object>> registryListItems(List<Registry> registries) {
+ return registries.stream().map(this::registryListItem).toList();
+ }
+
+ private Map<String, Object> registryListItem(Registry registry) {
+ Map<String, Object> item = new LinkedHashMap<>();
+ putIfNotNull(item, "RegistryName", registry.getRegistryName());
+ putIfNotNull(item, "RegistryArn", registry.getRegistryArn());
+ putIfNotNull(item, "Description", registry.getDescription());
+ putIfNotNull(item, "Status", registry.getStatus());
+ putIfNotNull(item, "CreatedTime", iso(registry.getCreatedTime()));
+ putIfNotNull(item, "UpdatedTime", iso(registry.getUpdatedTime()));
+ return item;
+ }
+
+ private List<Map<String, Object>> schemaListItems(List<Schema> schemas) {
+ return schemas.stream().map(this::schemaListItem).toList();
+ }
+
+ private Map<String, Object> schemaListItem(Schema schema) {
+ Map<String, Object> item = new LinkedHashMap<>();
+ putIfNotNull(item, "RegistryName", schema.getRegistryName());
+ putIfNotNull(item, "SchemaName", schema.getSchemaName());
+ putIfNotNull(item, "SchemaArn", schema.getSchemaArn());
+ putIfNotNull(item, "Description", schema.getDescription());
+ putIfNotNull(item, "SchemaStatus", schema.getSchemaStatus());
+ putIfNotNull(item, "CreatedTime", iso(schema.getCreatedTime()));
+ putIfNotNull(item, "UpdatedTime", iso(schema.getUpdatedTime()));
+ return item;
+ }
+
+ private List<Map<String, Object>> schemaVersionListItems(List<SchemaVersion> versions) {
+ return versions.stream().map(this::schemaVersionListItem).toList();
+ }
+
+ private Map<String, Object> schemaVersionListItem(SchemaVersion version) {
+ Map<String, Object> item = new LinkedHashMap<>();
+ putIfNotNull(item, "SchemaArn", version.getSchemaArn());
+ putIfNotNull(item, "SchemaVersionId", version.getSchemaVersionId());
+ putIfNotNull(item, "VersionNumber", version.getVersionNumber());
+ putIfNotNull(item, "Status", version.getStatus());
+ putIfNotNull(item, "CreatedTime", iso(version.getCreatedTime()));
+ return item;
+ }
+
+ private static void putIfNotNull(Map<String, Object> target, String key, Object value) {
+ if (value != null) {
+ target.put(key, value);
+ }
+ }
+
+ private static String iso(Instant instant) {
+ return instant != null ? instant.toString() : null;
+ }
+
+ private Response handlePutSchemaVersionMetadata(JsonNode request) {
+ String svId = request.path("SchemaVersionId").asText(null);
+ JsonNode kv = request.get("MetadataKeyValue");
+ String key = kv != null ? kv.path("MetadataKey").asText(null) : null;
+ String value = kv != null ? kv.path("MetadataValue").asText(null) : null;
+ var result = schemaRegistryService.putSchemaVersionMetadata(svId, key, value);
+ return Response.ok(buildMetadataPutResponse(result)).build();
+ }
+
+ private Response handleRemoveSchemaVersionMetadata(JsonNode request) {
+ String svId = request.path("SchemaVersionId").asText(null);
+ JsonNode kv = request.get("MetadataKeyValue");
+ String key = kv != null ? kv.path("MetadataKey").asText(null) : null;
+ String value = kv != null ? kv.path("MetadataValue").asText(null) : null;
+ var result = schemaRegistryService.removeSchemaVersionMetadata(svId, key, value);
+ return Response.ok(buildMetadataPutResponse(result)).build();
+ }
+
+ private Response handleQuerySchemaVersionMetadata(JsonNode request) {
+ String svId = request.path("SchemaVersionId").asText(null);
+ List<GlueSchemaRegistryService.MetadataKeyValueFilter> filters = new ArrayList<>();
+ JsonNode list = request.get("MetadataList");
+ if (list != null && list.isArray()) {
+ for (JsonNode item : list) {
+ String k = item.path("MetadataKey").asText(null);
+ String v = item.path("MetadataValue").asText(null);
+ filters.add(new GlueSchemaRegistryService.MetadataKeyValueFilter(k, v));
+ }
+ }
+ Map<String, ?> infoMap = schemaRegistryService.querySchemaVersionMetadata(svId, filters);
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("MetadataInfoMap", infoMap);
+ response.put("SchemaVersionId", svId);
+ return Response.ok(response).build();
+ }
+
+ private Map<String, Object> buildMetadataPutResponse(GlueSchemaRegistryService.MetadataPutResult result) {
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("SchemaArn", result.version().getSchemaArn());
+ response.put("SchemaName", result.schema().getSchemaName());
+ response.put("RegistryName", result.schema().getRegistryName());
+ response.put("LatestVersion",
+ Objects.equals(result.version().getVersionNumber(), result.schema().getLatestSchemaVersion()));
+ response.put("SchemaVersionId", result.version().getSchemaVersionId());
+ response.put("VersionNumber", result.version().getVersionNumber());
+ response.put("MetadataKey", result.metadataKey());
+ response.put("MetadataValue", result.metadataValue());
+ return response;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Response handleTagResource(JsonNode request) {
+ String arn = request.path("ResourceArn").asText(null);
+ Map<String, String> tagsToAdd = request.has("TagsToAdd")
+ ? mapper.convertValue(request.get("TagsToAdd"), Map.class)
+ : null;
+ schemaRegistryService.tagResource(arn, tagsToAdd);
+ return Response.ok(Map.of()).build();
+ }
+
+ @SuppressWarnings("unchecked")
+ private Response handleUntagResource(JsonNode request) {
+ String arn = request.path("ResourceArn").asText(null);
+ List<String> tagsToRemove = request.has("TagsToRemove")
+ ? mapper.convertValue(request.get("TagsToRemove"), List.class)
+ : null;
+ schemaRegistryService.untagResource(arn, tagsToRemove);
+ return Response.ok(Map.of()).build();
+ }
+
+ private Response handleGetTags(JsonNode request) {
+ String arn = request.path("ResourceArn").asText(null);
+ Map<String, String> tags = schemaRegistryService.getTags(arn);
+ return Response.ok(Map.of("Tags", tags)).build();
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index dd1e864bae..e3425db223 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -2,18 +2,27 @@ package io.github.hectorvent.floci.services.glue;
import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.glue.model.Column;
import io.github.hectorvent.floci.services.glue.model.Database;
import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.SchemaReference;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
import io.github.hectorvent.floci.services.glue.model.Table;
+import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.glue.schemaregistry.SchemaToColumnsConverter;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaVersion;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
@ApplicationScoped
public class GlueService {
@@ -23,12 +32,30 @@ public class GlueService {
private final StorageBackend<String, Database> databaseStore;
private final StorageBackend<String, Table> tableStore;
private final StorageBackend<String, Partition> partitionStore;
+ private final GlueSchemaRegistryService schemaRegistryService;
+ private final RegionResolver regionResolver;
@Inject
- public GlueService(StorageFactory storageFactory) {
+ public GlueService(StorageFactory storageFactory,
+ GlueSchemaRegistryService schemaRegistryService,
+ RegionResolver regionResolver) {
this.databaseStore = storageFactory.create("glue", "databases.json", new TypeReference<Map<String, Database>>() {});
this.tableStore = storageFactory.create("glue", "tables.json", new TypeReference<Map<String, Table>>() {});
this.partitionStore = storageFactory.create("glue", "partitions.json", new TypeReference<Map<String, Partition>>() {});
+ this.schemaRegistryService = schemaRegistryService;
+ this.regionResolver = regionResolver;
+ }
+
+ GlueService(StorageBackend<String, Database> databaseStore,
+ StorageBackend<String, Table> tableStore,
+ StorageBackend<String, Partition> partitionStore,
+ GlueSchemaRegistryService schemaRegistryService,
+ RegionResolver regionResolver) {
+ this.databaseStore = databaseStore;
+ this.tableStore = tableStore;
+ this.partitionStore = partitionStore;
+ this.schemaRegistryService = schemaRegistryService;
+ this.regionResolver = regionResolver;
}
public void createDatabase(Database database) {
@@ -49,11 +76,12 @@ public class GlueService {
}
public void createTable(String databaseName, Table table) {
- getDatabase(databaseName); // Check if db exists
+ getDatabase(databaseName);
String key = databaseName + ":" + table.getName();
if (tableStore.get(key).isPresent()) {
throw new AwsException("AlreadyExistsException", "Table already exists: " + table.getName(), 400);
}
+ validateSchemaReference(table);
table.setDatabaseName(databaseName);
tableStore.put(key, table);
LOG.infov("Created Glue Table: {0}.{1}", databaseName, table.getName());
@@ -61,18 +89,23 @@ public class GlueService {
public Table getTable(String databaseName, String tableName) {
String key = databaseName + ":" + tableName;
- return tableStore.get(key)
+ Table table = tableStore.get(key)
.orElseThrow(() -> new AwsException("EntityNotFoundException", "Table not found: " + databaseName + "." + tableName, 400));
+ return withResolvedSchemaReference(table);
}
public List<Table> getTables(String databaseName) {
- return tableStore.scan(k -> k.startsWith(databaseName + ":"));
+ List<Table> tables = tableStore.scan(k -> k.startsWith(databaseName + ":"));
+ List<Table> resolved = new ArrayList<>(tables.size());
+ for (Table table : tables) {
+ resolved.add(withResolvedSchemaReference(table));
+ }
+ return resolved;
}
public void deleteTable(String databaseName, String tableName) {
String key = databaseName + ":" + tableName;
tableStore.delete(key);
- // Also delete partitions
partitionStore.scan(k -> k.startsWith(key + ":")).forEach(p -> {
partitionStore.delete(databaseName + ":" + tableName + ":" + String.join(",", p.getValues()));
});
@@ -80,7 +113,7 @@ public class GlueService {
}
public void createPartition(String databaseName, String tableName, Partition partition) {
- getTable(databaseName, tableName); // Check if table exists
+ getTable(databaseName, tableName);
String key = databaseName + ":" + tableName + ":" + String.join(",", partition.getValues());
partition.setDatabaseName(databaseName);
partition.setTableName(tableName);
@@ -91,4 +124,123 @@ public class GlueService {
String prefix = databaseName + ":" + tableName + ":";
return partitionStore.scan(k -> k.startsWith(prefix));
}
+
+ private void validateSchemaReference(Table table) {
+ SchemaReference ref = schemaReferenceOf(table);
+ if (ref == null) {
+ return;
+ }
+ // Throws EntityNotFoundException / InvalidInputException if reference is broken.
+ resolveSchemaVersion(ref);
+ }
+
+ private Table withResolvedSchemaReference(Table table) {
+ SchemaReference ref = schemaReferenceOf(table);
+ if (ref == null) {
+ return table;
+ }
+ try {
+ SchemaVersion version = resolveSchemaVersion(ref);
+ List<Column> columns = SchemaToColumnsConverter.toColumns(
+ version.getDataFormat(), version.getSchemaDefinition());
+ if (!columns.isEmpty()) {
+ Table resolved = copyTable(table);
+ resolved.getStorageDescriptor().setColumns(columns);
+ return resolved;
+ }
+ } catch (AwsException e) {
+ LOG.warnv("SchemaReference resolution failed for {0}.{1}: {2}",
+ table.getDatabaseName(), table.getName(), e.getMessage());
+ }
+ return table;
+ }
+
+ private SchemaVersion resolveSchemaVersion(SchemaReference ref) {
+ boolean latest = ref.getSchemaVersionId() == null && ref.getSchemaVersionNumber() == null;
+ return schemaRegistryService.getSchemaVersion(
+ ref.getSchemaId(), ref.getSchemaVersionId(),
+ ref.getSchemaVersionNumber(), latest, regionResolver.getDefaultRegion());
+ }
+
+ private static SchemaReference schemaReferenceOf(Table table) {
+ StorageDescriptor sd = table != null ? table.getStorageDescriptor() : null;
+ return sd != null ? sd.getSchemaReference() : null;
+ }
+
+ private static Table copyTable(Table source) {
+ Table copy = new Table();
+ copy.setName(source.getName());
+ copy.setDatabaseName(source.getDatabaseName());
+ copy.setDescription(source.getDescription());
+ copy.setCreateTime(source.getCreateTime());
+ copy.setUpdateTime(source.getUpdateTime());
+ copy.setLastAccessTime(source.getLastAccessTime());
+ copy.setPartitionKeys(copyColumns(source.getPartitionKeys()));
+ copy.setStorageDescriptor(copyStorageDescriptor(source.getStorageDescriptor()));
+ copy.setTableType(source.getTableType());
+ copy.setParameters(copyMap(source.getParameters()));
+ return copy;
+ }
+
+ private static StorageDescriptor copyStorageDescriptor(StorageDescriptor source) {
+ if (source == null) {
+ return null;
+ }
+ StorageDescriptor copy = new StorageDescriptor();
+ copy.setColumns(copyColumns(source.getColumns()));
+ copy.setLocation(source.getLocation());
+ copy.setInputFormat(source.getInputFormat());
+ copy.setOutputFormat(source.getOutputFormat());
+ copy.setCompressed(source.getCompressed());
+ copy.setNumberOfBuckets(source.getNumberOfBuckets());
+ copy.setSerdeInfo(copySerDeInfo(source.getSerdeInfo()));
+ copy.setParameters(copyMap(source.getParameters()));
+ copy.setSchemaReference(copySchemaReference(source.getSchemaReference()));
+ return copy;
+ }
+
+ private static StorageDescriptor.SerDeInfo copySerDeInfo(StorageDescriptor.SerDeInfo source) {
+ if (source == null) {
+ return null;
+ }
+ StorageDescriptor.SerDeInfo copy = new StorageDescriptor.SerDeInfo();
+ copy.setName(source.getName());
+ copy.setSerializationLibrary(source.getSerializationLibrary());
+ copy.setParameters(copyMap(source.getParameters()));
+ return copy;
+ }
+
+ private static SchemaReference copySchemaReference(SchemaReference source) {
+ if (source == null) {
+ return null;
+ }
+ SchemaReference copy = new SchemaReference();
+ SchemaId schemaId = source.getSchemaId();
+ if (schemaId != null) {
+ copy.setSchemaId(new SchemaId(
+ schemaId.getRegistryName(), schemaId.getSchemaName(), schemaId.getSchemaArn()));
+ }
+ copy.setSchemaVersionId(source.getSchemaVersionId());
+ copy.setSchemaVersionNumber(source.getSchemaVersionNumber());
+ return copy;
+ }
+
+ private static List<Column> copyColumns(List<Column> source) {
+ if (source == null) {
+ return null;
+ }
+ List<Column> copy = new ArrayList<>(source.size());
+ for (Column column : source) {
+ Column columnCopy = new Column();
+ columnCopy.setName(column.getName());
+ columnCopy.setType(column.getType());
+ columnCopy.setComment(column.getComment());
+ copy.add(columnCopy);
+ }
+ return copy;
+ }
+
+ private static Map<String, String> copyMap(Map<String, String> source) {
+ return source != null ? new LinkedHashMap<>(source) : null;
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/SchemaReference.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/SchemaReference.java
new file mode 100644
index 0000000000..0df2a5cfc8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/SchemaReference.java
@@ -0,0 +1,31 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SchemaReference {
+
+ @JsonProperty("SchemaId")
+ private SchemaId schemaId;
+
+ @JsonProperty("SchemaVersionId")
+ private String schemaVersionId;
+
+ @JsonProperty("SchemaVersionNumber")
+ private Long schemaVersionNumber;
+
+ public SchemaReference() {}
+
+ public SchemaId getSchemaId() { return schemaId; }
+ public void setSchemaId(SchemaId schemaId) { this.schemaId = schemaId; }
+
+ public String getSchemaVersionId() { return schemaVersionId; }
+ public void setSchemaVersionId(String schemaVersionId) { this.schemaVersionId = schemaVersionId; }
+
+ public Long getSchemaVersionNumber() { return schemaVersionNumber; }
+ public void setSchemaVersionNumber(Long schemaVersionNumber) { this.schemaVersionNumber = schemaVersionNumber; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/StorageDescriptor.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/StorageDescriptor.java
index 15fd9d0edd..9f93d4ba33 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/model/StorageDescriptor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/StorageDescriptor.java
@@ -23,6 +23,8 @@ public class StorageDescriptor {
private SerDeInfo serdeInfo;
@JsonProperty("Parameters")
private Map<String, String> parameters;
+ @JsonProperty("SchemaReference")
+ private SchemaReference schemaReference;
public StorageDescriptor() {}
@@ -42,6 +44,8 @@ public class StorageDescriptor {
public void setSerdeInfo(SerDeInfo serdeInfo) { this.serdeInfo = serdeInfo; }
public Map<String, String> getParameters() { return parameters; }
public void setParameters(Map<String, String> parameters) { this.parameters = parameters; }
+ public SchemaReference getSchemaReference() { return schemaReference; }
+ public void setSchemaReference(SchemaReference schemaReference) { this.schemaReference = schemaReference; }
@RegisterForReflection
public static class SerDeInfo {
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryService.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryService.java
new file mode 100644
index 0000000000..7f98154ec0
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryService.java
@@ -0,0 +1,1047 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.MetadataInfo;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.Registry;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.RegistryId;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.Schema;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaVersion;
+import jakarta.annotation.PostConstruct;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.regex.Pattern;
+
+@ApplicationScoped
+public class GlueSchemaRegistryService {
+
+ private static final Logger LOG = Logger.getLogger(GlueSchemaRegistryService.class);
+
+ static final String DEFAULT_REGISTRY_NAME = "default-registry";
+ private static final Pattern NAME_PATTERN = Pattern.compile("^[A-Za-z0-9_$#.\\-]+$");
+ private static final int NAME_MAX_LENGTH = 255;
+ private static final int DEFAULT_MAX_RESULTS = 100;
+ private static final int MAX_MAX_RESULTS = 100;
+ private static final int MAX_DELETE_SCHEMA_VERSIONS = 25;
+ private static final int MAX_SCHEMA_DEFINITION_LENGTH = 170_000;
+
+ private static final Set<String> DATA_FORMATS = Set.of("AVRO", "JSON", "PROTOBUF");
+ private static final Set<String> COMPATIBILITY_MODES = Set.of(
+ "NONE", "DISABLED", "BACKWARD", "BACKWARD_ALL",
+ "FORWARD", "FORWARD_ALL", "FULL", "FULL_ALL");
+
+ private final StorageBackend<String, Registry> registryStore;
+ private final StorageBackend<String, Schema> schemaStore;
+ private final StorageBackend<String, SchemaVersion> versionStore;
+ private final StorageBackend<String, Map<String, MetadataInfo>> metadataStore;
+ private final RegionResolver regionResolver;
+
+ private final Map<String, NavigableMap<Long, String>> versionByNumber = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, String>> versionByDefinitionHash = new ConcurrentHashMap<>();
+
+ @Inject
+ public GlueSchemaRegistryService(StorageFactory storageFactory, RegionResolver regionResolver) {
+ this(
+ storageFactory.create("glue", "registries.json", new TypeReference<Map<String, Registry>>() {}),
+ storageFactory.create("glue", "schemas.json", new TypeReference<Map<String, Schema>>() {}),
+ storageFactory.create("glue", "schema_versions.json", new TypeReference<Map<String, SchemaVersion>>() {}),
+ storageFactory.create("glue", "schema_metadata.json", new TypeReference<Map<String, Map<String, MetadataInfo>>>() {}),
+ regionResolver
+ );
+ }
+
+ GlueSchemaRegistryService(StorageBackend<String, Registry> registryStore,
+ RegionResolver regionResolver) {
+ this(registryStore,
+ new io.github.hectorvent.floci.core.storage.InMemoryStorage<>(),
+ new io.github.hectorvent.floci.core.storage.InMemoryStorage<>(),
+ new io.github.hectorvent.floci.core.storage.InMemoryStorage<>(),
+ regionResolver);
+ }
+
+ GlueSchemaRegistryService(StorageBackend<String, Registry> registryStore,
+ StorageBackend<String, Schema> schemaStore,
+ StorageBackend<String, SchemaVersion> versionStore,
+ StorageBackend<String, Map<String, MetadataInfo>> metadataStore,
+ RegionResolver regionResolver) {
+ this.registryStore = registryStore;
+ this.schemaStore = schemaStore;
+ this.versionStore = versionStore;
+ this.metadataStore = metadataStore;
+ this.regionResolver = regionResolver;
+ }
+
+ @PostConstruct
+ void afterCdiInit() {
+ rebuildVersionIndexes();
+ }
+
+ // ---- Registry --------------------------------------------------------
+
+ public Registry createRegistry(String name, String description, Map<String, String> tags, String region) {
+ validateName(name, "RegistryName");
+ if (registryStore.get(name).isPresent()) {
+ throw new AwsException("AlreadyExistsException", "Registry already exists: " + name, 400);
+ }
+ Registry registry = new Registry(name);
+ registry.setDescription(description);
+ registry.setTags(tags);
+ registry.setRegistryArn(buildRegistryArn(region, name));
+ registryStore.put(name, registry);
+ LOG.infov("Created Glue Registry: {0}", name);
+ return registry;
+ }
+
+ public Registry getRegistry(RegistryId registryId, String region) {
+ String name = resolveRegistryName(registryId);
+ if (DEFAULT_REGISTRY_NAME.equals(name) && registryStore.get(name).isEmpty()) {
+ return autoCreateDefaultRegistry(region);
+ }
+ return registryStore.get(name)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Registry not found: " + name, 400));
+ }
+
+ public List<Registry> listRegistries() {
+ return sortedRegistries();
+ }
+
+ public Page<Registry> listRegistries(Integer maxResults, String nextToken) {
+ return paginate(sortedRegistries(), maxResults, nextToken);
+ }
+
+ public Registry updateRegistry(RegistryId registryId, String description, String region) {
+ Registry registry = getRegistry(registryId, region);
+ registry.setDescription(description);
+ registry.setUpdatedTime(Instant.now());
+ registryStore.put(registry.getRegistryName(), registry);
+ return registry;
+ }
+
+ public synchronized Registry deleteRegistry(RegistryId registryId, String region) {
+ Registry registry = getRegistry(registryId, region);
+ String prefix = registry.getRegistryName() + ":";
+ for (Schema schema : schemaStore.scan(k -> k.startsWith(prefix))) {
+ deleteSchemaByKey(schemaKey(schema.getRegistryName(), schema.getSchemaName()), schema);
+ }
+ registry.setStatus("DELETING");
+ registryStore.delete(registry.getRegistryName());
+ LOG.infov("Deleted Glue Registry: {0}", registry.getRegistryName());
+ return registry;
+ }
+
+ // ---- Schema ----------------------------------------------------------
+
+ public synchronized SchemaWithFirstVersion createSchema(RegistryId registryId,
+ String schemaName,
+ String dataFormat,
+ String compatibility,
+ String description,
+ String definition,
+ Map<String, String> tags,
+ String region) {
+ validateName(schemaName, "SchemaName");
+ validateDataFormat(dataFormat);
+ String compat = compatibility != null ? compatibility : "BACKWARD";
+ validateCompatibility(compat);
+ validateDefinitionRequired(definition);
+ Registry registry = getRegistry(registryId, region);
+ String schemaKey = schemaKey(registry.getRegistryName(), schemaName);
+ if (schemaStore.get(schemaKey).isPresent()) {
+ throw new AwsException("AlreadyExistsException",
+ "Schema already exists: " + registry.getRegistryName() + "/" + schemaName, 400);
+ }
+ String defError = SchemaCompatibilityChecker.validateDefinition(definition, dataFormat);
+ if (defError != null) {
+ throw new AwsException("InvalidInputException",
+ "Schema definition is not valid " + dataFormat + ": " + defError, 400);
+ }
+
+ Instant now = Instant.now();
+ Schema schema = new Schema();
+ schema.setRegistryName(registry.getRegistryName());
+ schema.setRegistryArn(registry.getRegistryArn());
+ schema.setSchemaName(schemaName);
+ schema.setSchemaArn(buildSchemaArn(region, registry.getRegistryName(), schemaName));
+ schema.setDescription(description);
+ schema.setDataFormat(dataFormat);
+ schema.setCompatibility(compat);
+ schema.setSchemaStatus("AVAILABLE");
+ schema.setSchemaCheckpoint(1L);
+ schema.setLatestSchemaVersion(1L);
+ schema.setNextSchemaVersion(2L);
+ schema.setCreatedTime(now);
+ schema.setUpdatedTime(now);
+ schema.setTags(tags);
+
+ SchemaVersion version = new SchemaVersion();
+ version.setSchemaVersionId(UUID.randomUUID().toString());
+ version.setSchemaArn(schema.getSchemaArn());
+ version.setVersionNumber(1L);
+ version.setStatus("AVAILABLE");
+ version.setSchemaDefinition(definition);
+ version.setDataFormat(dataFormat);
+ version.setCreatedTime(now);
+
+ schemaStore.put(schemaKey, schema);
+ versionStore.put(version.getSchemaVersionId(), version);
+ indexVersion(schemaKey, version);
+ LOG.infov("Created Glue Schema {0}/{1} (version 1: {2})",
+ registry.getRegistryName(), schemaName, version.getSchemaVersionId());
+ return new SchemaWithFirstVersion(schema, version);
+ }
+
+ public Schema getSchema(SchemaId schemaId, String region) {
+ return resolveSchema(schemaId, region);
+ }
+
+ public List<Schema> listSchemas(RegistryId registryId, String region) {
+ Registry registry = getRegistry(registryId, region);
+ String prefix = registry.getRegistryName() + ":";
+ return sortedSchemas(prefix);
+ }
+
+ public Page<Schema> listSchemas(RegistryId registryId, String region, Integer maxResults, String nextToken) {
+ Registry registry = getRegistry(registryId, region);
+ String prefix = registry.getRegistryName() + ":";
+ return paginate(sortedSchemas(prefix), maxResults, nextToken);
+ }
+
+ public synchronized Schema updateSchema(SchemaId schemaId,
+ String compatibility,
+ String description,
+ String region) {
+ return updateSchema(schemaId, compatibility, description, null, region);
+ }
+
+ public synchronized Schema updateSchema(SchemaId schemaId,
+ String compatibility,
+ String description,
+ Long checkpointVersionNumber,
+ String region) {
+ Schema schema = resolveSchema(schemaId, region);
+ if (compatibility != null && !compatibility.isBlank()) {
+ validateCompatibility(compatibility);
+ schema.setCompatibility(compatibility);
+ }
+ if (description != null) {
+ schema.setDescription(description);
+ }
+ if (checkpointVersionNumber != null) {
+ validateCheckpointVersionExists(schema, checkpointVersionNumber);
+ schema.setSchemaCheckpoint(checkpointVersionNumber);
+ }
+ schema.setUpdatedTime(Instant.now());
+ schemaStore.put(schemaKey(schema.getRegistryName(), schema.getSchemaName()), schema);
+ return schema;
+ }
+
+ public synchronized Schema deleteSchema(SchemaId schemaId, String region) {
+ Schema schema = resolveSchema(schemaId, region);
+ String key = schemaKey(schema.getRegistryName(), schema.getSchemaName());
+ int versionsDeleted = deleteSchemaByKey(key, schema);
+ schema.setSchemaStatus("DELETING");
+ LOG.infov("Deleted Glue Schema {0}/{1} ({2} versions)",
+ schema.getRegistryName(), schema.getSchemaName(), versionsDeleted);
+ return schema;
+ }
+
+ // ---- Schema versions -------------------------------------------------
+
+ public synchronized SchemaVersion registerSchemaVersion(SchemaId schemaId, String definition, String region) {
+ validateDefinitionRequired(definition);
+ Schema schema = resolveSchema(schemaId, region);
+ String dataFormat = schema.getDataFormat();
+ String defError = SchemaCompatibilityChecker.validateDefinition(definition, dataFormat);
+ if (defError != null) {
+ throw new AwsException("InvalidInputException",
+ "Schema definition is not valid " + dataFormat + ": " + defError, 400);
+ }
+
+ String schemaKey = schemaKey(schema.getRegistryName(), schema.getSchemaName());
+ String hash = canonicalHash(definition, dataFormat);
+ Map<String, String> hashIndex = versionByDefinitionHash.computeIfAbsent(schemaKey, k -> new ConcurrentHashMap<>());
+ String existingId = hashIndex.get(hash);
+ if (existingId != null) {
+ return versionStore.get(existingId)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Indexed schema version vanished: " + existingId, 400));
+ }
+
+ if ("DISABLED".equals(schema.getCompatibility())) {
+ throw new AwsException("InvalidInputException",
+ "Compatibility mode DISABLED — auto-registration of new versions is not allowed", 400);
+ }
+
+ List<String> existingDefs = orderedDefinitions(schemaKey);
+ SchemaCompatibilityChecker.Result compat =
+ SchemaCompatibilityChecker.check(schema.getCompatibility(), existingDefs, definition, dataFormat);
+ if (!compat.compatible()) {
+ throw new AwsException("InvalidInputException",
+ "Schema is incompatible: " + compat.reason(), 400);
+ }
+
+ Instant now = Instant.now();
+ long nextVersion = schema.getNextSchemaVersion() != null ? schema.getNextSchemaVersion() : 1L;
+ SchemaVersion version = new SchemaVersion();
+ version.setSchemaVersionId(UUID.randomUUID().toString());
+ version.setSchemaArn(schema.getSchemaArn());
+ version.setVersionNumber(nextVersion);
+ version.setStatus("AVAILABLE");
+ version.setSchemaDefinition(definition);
+ version.setDataFormat(dataFormat);
+ version.setCreatedTime(now);
+
+ schema.setLatestSchemaVersion(nextVersion);
+ schema.setNextSchemaVersion(nextVersion + 1);
+ schema.setUpdatedTime(now);
+
+ schemaStore.put(schemaKey, schema);
+ versionStore.put(version.getSchemaVersionId(), version);
+ indexVersion(schemaKey, version);
+ LOG.infov("Registered Glue Schema version {0}/{1} v{2} ({3})",
+ schema.getRegistryName(), schema.getSchemaName(), nextVersion, version.getSchemaVersionId());
+ return version;
+ }
+
+ public SchemaVersion getSchemaVersion(SchemaId schemaId,
+ String schemaVersionId,
+ Long versionNumber,
+ boolean latestVersion,
+ String region) {
+ if (schemaVersionId != null && !schemaVersionId.isBlank()) {
+ return versionStore.get(schemaVersionId)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Schema version not found: " + schemaVersionId, 400));
+ }
+ Schema schema = resolveSchema(schemaId, region);
+ String schemaKey = schemaKey(schema.getRegistryName(), schema.getSchemaName());
+ NavigableMap<Long, String> byNumber = versionByNumber.get(schemaKey);
+ if (byNumber == null || byNumber.isEmpty()) {
+ throw new AwsException("EntityNotFoundException", "No schema versions for " + schemaKey, 400);
+ }
+ Long target;
+ if (latestVersion) {
+ target = byNumber.lastKey();
+ } else if (versionNumber != null) {
+ target = versionNumber;
+ } else {
+ throw new AwsException("InvalidInputException",
+ "Either SchemaVersionId, VersionNumber, or LatestVersion=true is required", 400);
+ }
+ String id = byNumber.get(target);
+ if (id == null) {
+ throw new AwsException("EntityNotFoundException",
+ "Schema version " + target + " not found for " + schemaKey, 400);
+ }
+ return versionStore.get(id)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Schema version vanished: " + id, 400));
+ }
+
+ public List<SchemaVersion> listSchemaVersions(SchemaId schemaId, String region) {
+ Schema schema = resolveSchema(schemaId, region);
+ return sortedSchemaVersions(schema);
+ }
+
+ public Page<SchemaVersion> listSchemaVersions(SchemaId schemaId, String region,
+ Integer maxResults, String nextToken) {
+ Schema schema = resolveSchema(schemaId, region);
+ return paginate(sortedSchemaVersions(schema), maxResults, nextToken);
+ }
+
+ private List<SchemaVersion> sortedSchemaVersions(Schema schema) {
+ String key = schemaKey(schema.getRegistryName(), schema.getSchemaName());
+ NavigableMap<Long, String> byNumber = versionByNumber.get(key);
+ if (byNumber == null || byNumber.isEmpty()) {
+ return List.of();
+ }
+ List<SchemaVersion> out = new ArrayList<>(byNumber.size());
+ for (String id : byNumber.values()) {
+ versionStore.get(id).ifPresent(out::add);
+ }
+ return out;
+ }
+
+ public synchronized List<VersionDeletionResult> deleteSchemaVersions(SchemaId schemaId,
+ String versionsExpression,
+ String region) {
+ if (versionsExpression == null || versionsExpression.isBlank()) {
+ throw new AwsException("InvalidInputException", "Versions expression is required", 400);
+ }
+ Schema schema = resolveSchema(schemaId, region);
+ String key = schemaKey(schema.getRegistryName(), schema.getSchemaName());
+ NavigableMap<Long, String> byNumber = versionByNumber.get(key);
+ if (byNumber == null) {
+ byNumber = new ConcurrentSkipListMap<>();
+ }
+ List<Long> versions = parseVersionsExpression(versionsExpression);
+ List<VersionDeletionResult> results = new ArrayList<>(versions.size());
+ Long latestRemaining = byNumber.isEmpty() ? null : byNumber.lastKey();
+
+ for (Long v : versions) {
+ // Glue forbids deleting the latest version while older versions exist.
+ if (latestRemaining != null && v.equals(latestRemaining) && byNumber.size() > 1) {
+ results.add(new VersionDeletionResult(v, "InvalidInputException",
+ "Cannot delete the latest version while older versions remain"));
+ continue;
+ }
+ if (schema.getSchemaCheckpoint() != null && v.equals(schema.getSchemaCheckpoint())) {
+ results.add(new VersionDeletionResult(v, "InvalidInputException",
+ "Cannot delete the checkpoint version"));
+ continue;
+ }
+ String id = byNumber.get(v);
+ if (id == null) {
+ results.add(new VersionDeletionResult(v, "EntityNotFoundException",
+ "Version " + v + " not found"));
+ continue;
+ }
+ String hash = versionStore.get(id)
+ .map(sv -> canonicalHash(sv.getSchemaDefinition(), sv.getDataFormat()))
+ .orElse(null);
+ versionStore.delete(id);
+ metadataStore.delete(id);
+ byNumber.remove(v);
+ if (hash != null) {
+ Map<String, String> hashIndex = versionByDefinitionHash.get(key);
+ if (hashIndex != null) {
+ hashIndex.remove(hash);
+ }
+ }
+ if (v.equals(latestRemaining)) {
+ latestRemaining = byNumber.isEmpty() ? null : byNumber.lastKey();
+ }
+ results.add(new VersionDeletionResult(v, null, null));
+ }
+
+ if (byNumber.isEmpty()) {
+ versionByNumber.remove(key);
+ versionByDefinitionHash.remove(key);
+ schema.setLatestSchemaVersion(null);
+ } else {
+ schema.setLatestSchemaVersion(byNumber.lastKey());
+ }
+ schema.setUpdatedTime(Instant.now());
+ schemaStore.put(key, schema);
+ return results;
+ }
+
+ public String getSchemaVersionsDiff(SchemaId schemaId, Long firstVersion, Long secondVersion, String region) {
+ if (firstVersion == null || secondVersion == null) {
+ throw new AwsException("InvalidInputException",
+ "Both FirstSchemaVersionNumber and SecondSchemaVersionNumber are required", 400);
+ }
+ SchemaVersion first = getSchemaVersion(schemaId, null, firstVersion, false, region);
+ SchemaVersion second = getSchemaVersion(schemaId, null, secondVersion, false, region);
+ return simpleUnifiedDiff(
+ "v" + first.getVersionNumber(), first.getSchemaDefinition(),
+ "v" + second.getVersionNumber(), second.getSchemaDefinition());
+ }
+
+ // ---- Metadata --------------------------------------------------------
+
+ public synchronized MetadataPutResult putSchemaVersionMetadata(String schemaVersionId,
+ String key,
+ String value) {
+ validateMetadataKeyValue(key, value);
+ SchemaVersion version = versionStore.get(schemaVersionId)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Schema version not found: " + schemaVersionId, 400));
+
+ Map<String, MetadataInfo> metadata = metadataStore.get(schemaVersionId).orElseGet(java.util.LinkedHashMap::new);
+ MetadataInfo info = metadata.get(key);
+ Instant now = Instant.now();
+ if (info == null) {
+ info = new MetadataInfo(value, now);
+ } else if (value.equals(info.getMetadataValue())) {
+ throw new AwsException("AlreadyExistsException",
+ "Metadata key/value pair already exists: " + key + "=" + value, 400);
+ } else {
+ // Demote current value into OtherMetadataValueList; check duplicate among history.
+ List<MetadataInfo.OtherMetadataValueListItem> history =
+ info.getOtherMetadataValueList() != null
+ ? new ArrayList<>(info.getOtherMetadataValueList())
+ : new ArrayList<>();
+ for (var item : history) {
+ if (value.equals(item.getMetadataValue())) {
+ throw new AwsException("AlreadyExistsException",
+ "Metadata key/value pair already exists: " + key + "=" + value, 400);
+ }
+ }
+ history.add(0,
+ new MetadataInfo.OtherMetadataValueListItem(info.getMetadataValue(), info.getCreatedTime()));
+ info = new MetadataInfo(value, now);
+ info.setOtherMetadataValueList(history);
+ }
+ metadata.put(key, info);
+ metadataStore.put(schemaVersionId, metadata);
+ return new MetadataPutResult(schemaForVersion(version), version, key, value);
+ }
+
+ public synchronized MetadataPutResult removeSchemaVersionMetadata(String schemaVersionId,
+ String key,
+ String value) {
+ validateMetadataKeyValue(key, value);
+ SchemaVersion version = versionStore.get(schemaVersionId)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Schema version not found: " + schemaVersionId, 400));
+
+ Map<String, MetadataInfo> metadata = metadataStore.get(schemaVersionId)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Metadata not found for schema version: " + schemaVersionId, 400));
+ MetadataInfo info = metadata.get(key);
+ if (info == null) {
+ throw new AwsException("EntityNotFoundException",
+ "Metadata key not found: " + key, 400);
+ }
+
+ if (value.equals(info.getMetadataValue())) {
+ // Remove current; promote first OtherMetadataValueList entry.
+ List<MetadataInfo.OtherMetadataValueListItem> history = info.getOtherMetadataValueList();
+ if (history == null || history.isEmpty()) {
+ metadata.remove(key);
+ } else {
+ MetadataInfo.OtherMetadataValueListItem promoted = history.get(0);
+ MetadataInfo replacement = new MetadataInfo(promoted.getMetadataValue(), promoted.getCreatedTime());
+ if (history.size() > 1) {
+ replacement.setOtherMetadataValueList(new ArrayList<>(history.subList(1, history.size())));
+ }
+ metadata.put(key, replacement);
+ }
+ } else {
+ // Remove from OtherMetadataValueList.
+ List<MetadataInfo.OtherMetadataValueListItem> history = info.getOtherMetadataValueList();
+ if (history == null) {
+ throw new AwsException("EntityNotFoundException",
+ "Metadata key/value not found: " + key + "=" + value, 400);
+ }
+ boolean removed = history.removeIf(it -> value.equals(it.getMetadataValue()));
+ if (!removed) {
+ throw new AwsException("EntityNotFoundException",
+ "Metadata key/value not found: " + key + "=" + value, 400);
+ }
+ info.setOtherMetadataValueList(history.isEmpty() ? null : history);
+ metadata.put(key, info);
+ }
+
+ if (metadata.isEmpty()) {
+ metadataStore.delete(schemaVersionId);
+ } else {
+ metadataStore.put(schemaVersionId, metadata);
+ }
+ return new MetadataPutResult(schemaForVersion(version), version, key, value);
+ }
+
+ public Map<String, MetadataInfo> querySchemaVersionMetadata(String schemaVersionId,
+ List<MetadataKeyValueFilter> metadataList) {
+ if (versionStore.get(schemaVersionId).isEmpty()) {
+ throw new AwsException("EntityNotFoundException",
+ "Schema version not found: " + schemaVersionId, 400);
+ }
+ Map<String, MetadataInfo> stored = metadataStore.get(schemaVersionId).orElse(Map.of());
+ if (metadataList == null || metadataList.isEmpty()) {
+ return stored;
+ }
+ Map<String, MetadataInfo> filtered = new java.util.LinkedHashMap<>();
+ for (var f : metadataList) {
+ MetadataInfo info = stored.get(f.metadataKey());
+ if (info == null) continue;
+ if (f.metadataValue() == null || f.metadataValue().isBlank()) {
+ filtered.put(f.metadataKey(), info);
+ } else if (f.metadataValue().equals(info.getMetadataValue())) {
+ filtered.put(f.metadataKey(), info);
+ } else if (info.getOtherMetadataValueList() != null) {
+ for (var item : info.getOtherMetadataValueList()) {
+ if (f.metadataValue().equals(item.getMetadataValue())) {
+ filtered.put(f.metadataKey(), info);
+ break;
+ }
+ }
+ }
+ }
+ return filtered;
+ }
+
+ // ---- Tags ------------------------------------------------------------
+
+ public synchronized void tagResource(String resourceArn, Map<String, String> tagsToAdd) {
+ if (tagsToAdd == null || tagsToAdd.isEmpty()) {
+ return;
+ }
+ TaggedResource resource = resolveTaggedResource(resourceArn);
+ Map<String, String> tags = resource.getTags();
+ if (tags == null) {
+ tags = new java.util.LinkedHashMap<>();
+ } else {
+ tags = new java.util.LinkedHashMap<>(tags);
+ }
+ tags.putAll(tagsToAdd);
+ resource.setTags(tags);
+ resource.persist();
+ }
+
+ public synchronized void untagResource(String resourceArn, List<String> tagKeysToRemove) {
+ if (tagKeysToRemove == null || tagKeysToRemove.isEmpty()) {
+ return;
+ }
+ TaggedResource resource = resolveTaggedResource(resourceArn);
+ Map<String, String> tags = resource.getTags();
+ if (tags == null || tags.isEmpty()) {
+ return;
+ }
+ Map<String, String> updated = new java.util.LinkedHashMap<>(tags);
+ for (String key : tagKeysToRemove) {
+ updated.remove(key);
+ }
+ resource.setTags(updated.isEmpty() ? null : updated);
+ resource.persist();
+ }
+
+ public Map<String, String> getTags(String resourceArn) {
+ TaggedResource resource = resolveTaggedResource(resourceArn);
+ Map<String, String> tags = resource.getTags();
+ return tags != null ? tags : Map.of();
+ }
+
+ public CheckValidityResult checkSchemaVersionValidity(String dataFormat, String definition) {
+ validateDataFormat(dataFormat);
+ if (definition == null || definition.isBlank()) {
+ return new CheckValidityResult(false, "SchemaDefinition is required");
+ }
+ String error = SchemaCompatibilityChecker.validateDefinition(definition, dataFormat);
+ if (error == null) {
+ return new CheckValidityResult(true, null);
+ }
+ return new CheckValidityResult(false, error);
+ }
+
+ public SchemaVersion getSchemaByDefinition(SchemaId schemaId, String definition, String region) {
+ validateDefinitionRequired(definition);
+ Schema schema = resolveSchema(schemaId, region);
+ String schemaKey = schemaKey(schema.getRegistryName(), schema.getSchemaName());
+ String hash = canonicalHash(definition, schema.getDataFormat());
+ Map<String, String> hashIndex = versionByDefinitionHash.get(schemaKey);
+ String id = hashIndex != null ? hashIndex.get(hash) : null;
+ if (id == null) {
+ throw new AwsException("EntityNotFoundException",
+ "Schema version is not found. Definition not found in " + schemaKey, 400);
+ }
+ return versionStore.get(id)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Schema version vanished: " + id, 400));
+ }
+
+ // ---- Helpers ---------------------------------------------------------
+
+ public record SchemaWithFirstVersion(Schema schema, SchemaVersion firstVersion) {}
+
+ public record VersionDeletionResult(long versionNumber, String errorCode, String errorMessage) {}
+
+ public record CheckValidityResult(boolean valid, String error) {}
+
+ public record MetadataPutResult(Schema schema, SchemaVersion version, String metadataKey, String metadataValue) {}
+
+ public record MetadataKeyValueFilter(String metadataKey, String metadataValue) {}
+
+ public record Page<T>(List<T> items, String nextToken) {}
+
+ private interface TaggedResource {
+ Map<String, String> getTags();
+ void setTags(Map<String, String> tags);
+ void persist();
+ }
+
+ private TaggedResource resolveTaggedResource(String resourceArn) {
+ if (resourceArn == null || resourceArn.isBlank()) {
+ throw new AwsException("InvalidInputException", "ResourceArn is required", 400);
+ }
+ if (resourceArn.contains(":registry/")) {
+ String name = parseRegistryNameFromArn(resourceArn);
+ Registry r = registryStore.get(name)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Registry not found: " + name, 400));
+ return new TaggedResource() {
+ @Override public Map<String, String> getTags() { return r.getTags(); }
+ @Override public void setTags(Map<String, String> tags) { r.setTags(tags); }
+ @Override public void persist() { registryStore.put(r.getRegistryName(), r); }
+ };
+ }
+ if (resourceArn.contains(":schema/")) {
+ String[] parts = parseSchemaArn(resourceArn);
+ String key = schemaKey(parts[0], parts[1]);
+ Schema s = schemaStore.get(key)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Schema not found: " + parts[0] + "/" + parts[1], 400));
+ return new TaggedResource() {
+ @Override public Map<String, String> getTags() { return s.getTags(); }
+ @Override public void setTags(Map<String, String> tags) { s.setTags(tags); }
+ @Override public void persist() { schemaStore.put(key, s); }
+ };
+ }
+ throw new AwsException("InvalidInputException",
+ "Resource ARN does not point to a Registry or Schema: " + resourceArn, 400);
+ }
+
+ private void validateMetadataKeyValue(String key, String value) {
+ if (key == null || key.isBlank()) {
+ throw new AwsException("InvalidInputException", "MetadataKey is required", 400);
+ }
+ if (value == null || value.isBlank()) {
+ throw new AwsException("InvalidInputException", "MetadataValue is required", 400);
+ }
+ }
+
+ private static List<Long> parseVersionsExpression(String expr) {
+ java.util.SortedSet<Long> versions = new java.util.TreeSet<>();
+ for (String token : expr.split(",")) {
+ token = token.trim();
+ if (token.isEmpty()) continue;
+ try {
+ if (token.contains("-")) {
+ String[] range = token.split("-", 2);
+ long start = Long.parseLong(range[0].trim());
+ long end = Long.parseLong(range[1].trim());
+ if (end < start) {
+ throw new AwsException("InvalidInputException",
+ "Invalid version range: " + token, 400);
+ }
+ for (long v = start; v <= end; v++) {
+ versions.add(v);
+ }
+ } else {
+ versions.add(Long.parseLong(token));
+ }
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidInputException",
+ "Invalid version expression: " + token, 400);
+ }
+ }
+ if (versions.size() > MAX_DELETE_SCHEMA_VERSIONS) {
+ throw new AwsException("InvalidInputException",
+ "Versions expression cannot expand to more than " + MAX_DELETE_SCHEMA_VERSIONS + " versions", 400);
+ }
+ return new ArrayList<>(versions);
+ }
+
+ private static String simpleUnifiedDiff(String labelA, String a, String labelB, String b) {
+ if (java.util.Objects.equals(a, b)) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append("--- ").append(labelA).append('\n');
+ sb.append("+++ ").append(labelB).append('\n');
+ java.util.List<String> linesA = a == null ? List.of() : List.of(a.split("\n", -1));
+ java.util.List<String> linesB = b == null ? List.of() : List.of(b.split("\n", -1));
+ int n = linesA.size();
+ int m = linesB.size();
+ // Compute LCS lengths.
+ int[][] dp = new int[n + 1][m + 1];
+ for (int i = n - 1; i >= 0; i--) {
+ for (int j = m - 1; j >= 0; j--) {
+ if (linesA.get(i).equals(linesB.get(j))) {
+ dp[i][j] = dp[i + 1][j + 1] + 1;
+ } else {
+ dp[i][j] = Math.max(dp[i + 1][j], dp[i][j + 1]);
+ }
+ }
+ }
+ int i = 0, j = 0;
+ while (i < n && j < m) {
+ if (linesA.get(i).equals(linesB.get(j))) {
+ sb.append(' ').append(linesA.get(i)).append('\n');
+ i++; j++;
+ } else if (dp[i + 1][j] >= dp[i][j + 1]) {
+ sb.append('-').append(linesA.get(i)).append('\n');
+ i++;
+ } else {
+ sb.append('+').append(linesB.get(j)).append('\n');
+ j++;
+ }
+ }
+ while (i < n) {
+ sb.append('-').append(linesA.get(i++)).append('\n');
+ }
+ while (j < m) {
+ sb.append('+').append(linesB.get(j++)).append('\n');
+ }
+ return sb.toString();
+ }
+
+ private Schema resolveSchema(SchemaId schemaId, String region) {
+ if (schemaId == null) {
+ throw new AwsException("InvalidInputException", "SchemaId is required", 400);
+ }
+ String registryName;
+ String schemaName;
+ if (schemaId.getSchemaArn() != null && !schemaId.getSchemaArn().isBlank()) {
+ String[] parsed = parseSchemaArn(schemaId.getSchemaArn());
+ registryName = parsed[0];
+ schemaName = parsed[1];
+ } else {
+ schemaName = schemaId.getSchemaName();
+ if (schemaName == null || schemaName.isBlank()) {
+ throw new AwsException("InvalidInputException",
+ "SchemaId requires SchemaArn or SchemaName", 400);
+ }
+ String requested = schemaId.getRegistryName();
+ if (requested == null || requested.isBlank()) {
+ Registry reg = getRegistry(null, region);
+ registryName = reg.getRegistryName();
+ } else {
+ registryName = requested;
+ if (registryStore.get(registryName).isEmpty()) {
+ throw new AwsException("EntityNotFoundException",
+ "Registry not found: " + registryName, 400);
+ }
+ }
+ }
+ String key = schemaKey(registryName, schemaName);
+ return schemaStore.get(key)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Schema is not found. " + registryName + "/" + schemaName, 400));
+ }
+
+ private String[] parseSchemaArn(String arn) {
+ int idx = arn.indexOf("schema/");
+ if (idx < 0) {
+ throw new AwsException("InvalidInputException", "Invalid schema ARN: " + arn, 400);
+ }
+ String tail = arn.substring(idx + "schema/".length());
+ int slash = tail.indexOf('/');
+ if (slash < 0 || slash == tail.length() - 1) {
+ throw new AwsException("InvalidInputException", "Invalid schema ARN: " + arn, 400);
+ }
+ return new String[] { tail.substring(0, slash), tail.substring(slash + 1) };
+ }
+
+ private List<String> orderedDefinitions(String schemaKey) {
+ NavigableMap<Long, String> byNumber = versionByNumber.get(schemaKey);
+ if (byNumber == null) {
+ return List.of();
+ }
+ List<String> defs = new ArrayList<>(byNumber.size());
+ for (String id : byNumber.values()) {
+ versionStore.get(id).ifPresent(v -> defs.add(v.getSchemaDefinition()));
+ }
+ return defs;
+ }
+
+ private List<Registry> sortedRegistries() {
+ List<Registry> registries = new ArrayList<>(registryStore.scan(k -> true));
+ registries.sort(Comparator.comparing(Registry::getRegistryName,
+ Comparator.nullsLast(String::compareTo)));
+ return registries;
+ }
+
+ private List<Schema> sortedSchemas(String prefix) {
+ List<Schema> schemas = new ArrayList<>(schemaStore.scan(k -> k.startsWith(prefix)));
+ schemas.sort(Comparator.comparing(Schema::getSchemaName,
+ Comparator.nullsLast(String::compareTo)));
+ return schemas;
+ }
+
+ private static <T> Page<T> paginate(List<T> all, Integer maxResults, String nextToken) {
+ int limit = maxResults == null ? DEFAULT_MAX_RESULTS : maxResults;
+ if (limit < 1 || limit > MAX_MAX_RESULTS) {
+ throw new AwsException("InvalidInputException",
+ "MaxResults must be between 1 and " + MAX_MAX_RESULTS, 400);
+ }
+ int start = 0;
+ if (nextToken != null && !nextToken.isBlank()) {
+ try {
+ start = Integer.parseInt(nextToken);
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidInputException", "Invalid NextToken", 400);
+ }
+ if (start < 0 || start > all.size()) {
+ throw new AwsException("InvalidInputException", "Invalid NextToken", 400);
+ }
+ }
+ int end = Math.min(start + limit, all.size());
+ String newToken = end < all.size() ? String.valueOf(end) : null;
+ return new Page<>(List.copyOf(all.subList(start, end)), newToken);
+ }
+
+ private void validateCheckpointVersionExists(Schema schema, long checkpointVersionNumber) {
+ if (checkpointVersionNumber < 1) {
+ throw new AwsException("InvalidInputException", "Schema checkpoint version must be at least 1", 400);
+ }
+ String key = schemaKey(schema.getRegistryName(), schema.getSchemaName());
+ NavigableMap<Long, String> byNumber = versionByNumber.get(key);
+ if (byNumber == null || !byNumber.containsKey(checkpointVersionNumber)) {
+ throw new AwsException("EntityNotFoundException",
+ "Schema version " + checkpointVersionNumber + " not found for " + key, 400);
+ }
+ }
+
+ private int deleteSchemaByKey(String key, Schema schema) {
+ int versionsDeleted = 0;
+ NavigableMap<Long, String> byNumber = versionByNumber.get(key);
+ if (byNumber != null) {
+ for (String versionId : new ArrayList<>(byNumber.values())) {
+ versionStore.delete(versionId);
+ metadataStore.delete(versionId);
+ versionsDeleted++;
+ }
+ }
+ versionByNumber.remove(key);
+ versionByDefinitionHash.remove(key);
+ schemaStore.delete(key);
+ return versionsDeleted;
+ }
+
+ private Schema schemaForVersion(SchemaVersion version) {
+ String[] parts = parseSchemaArn(version.getSchemaArn());
+ String key = schemaKey(parts[0], parts[1]);
+ return schemaStore.get(key)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Schema not found for version: " + version.getSchemaVersionId(), 400));
+ }
+
+ private void indexVersion(String schemaKey, SchemaVersion version) {
+ versionByNumber
+ .computeIfAbsent(schemaKey, k -> new ConcurrentSkipListMap<>())
+ .put(version.getVersionNumber(), version.getSchemaVersionId());
+ String hash = canonicalHash(version.getSchemaDefinition(), version.getDataFormat());
+ versionByDefinitionHash
+ .computeIfAbsent(schemaKey, k -> new ConcurrentHashMap<>())
+ .put(hash, version.getSchemaVersionId());
+ }
+
+ private void rebuildVersionIndexes() {
+ versionByNumber.clear();
+ versionByDefinitionHash.clear();
+ Map<String, String> arnToSchemaKey = new java.util.HashMap<>();
+ for (Schema s : schemaStore.scan(k -> true)) {
+ arnToSchemaKey.put(s.getSchemaArn(), schemaKey(s.getRegistryName(), s.getSchemaName()));
+ }
+ for (SchemaVersion v : versionStore.scan(k -> true)) {
+ String schemaKey = arnToSchemaKey.get(v.getSchemaArn());
+ if (schemaKey == null) {
+ continue;
+ }
+ indexVersion(schemaKey, v);
+ }
+ }
+
+ private static String canonicalHash(String definition, String dataFormat) {
+ String canonical;
+ try {
+ canonical = SchemaCompatibilityChecker.canonicalize(definition, dataFormat);
+ } catch (RuntimeException e) {
+ canonical = definition;
+ }
+ return sha256Hex(canonical);
+ }
+
+ private static String sha256Hex(String input) {
+ try {
+ MessageDigest md = MessageDigest.getInstance("SHA-256");
+ return HexFormat.of().formatHex(md.digest(input.getBytes(StandardCharsets.UTF_8)));
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("SHA-256 not available", e);
+ }
+ }
+
+ private Registry autoCreateDefaultRegistry(String region) {
+ Registry registry = new Registry(DEFAULT_REGISTRY_NAME);
+ registry.setRegistryArn(buildRegistryArn(region, DEFAULT_REGISTRY_NAME));
+ registryStore.put(DEFAULT_REGISTRY_NAME, registry);
+ LOG.infov("Auto-created default Glue Registry");
+ return registry;
+ }
+
+ String resolveRegistryName(RegistryId registryId) {
+ if (registryId == null) {
+ return DEFAULT_REGISTRY_NAME;
+ }
+ if (registryId.getRegistryArn() != null && !registryId.getRegistryArn().isBlank()) {
+ return parseRegistryNameFromArn(registryId.getRegistryArn());
+ }
+ if (registryId.getRegistryName() != null && !registryId.getRegistryName().isBlank()) {
+ return registryId.getRegistryName();
+ }
+ return DEFAULT_REGISTRY_NAME;
+ }
+
+ private String parseRegistryNameFromArn(String arn) {
+ int slash = arn.indexOf("registry/");
+ if (slash < 0) {
+ throw new AwsException("InvalidInputException", "Invalid registry ARN: " + arn, 400);
+ }
+ String name = arn.substring(slash + "registry/".length());
+ if (name.isBlank()) {
+ throw new AwsException("InvalidInputException", "Invalid registry ARN: " + arn, 400);
+ }
+ return name;
+ }
+
+ private String buildRegistryArn(String region, String name) {
+ return regionResolver.buildArn("glue", region, "registry/" + name);
+ }
+
+ private String buildSchemaArn(String region, String registryName, String schemaName) {
+ return regionResolver.buildArn("glue", region, "schema/" + registryName + "/" + schemaName);
+ }
+
+ private static String schemaKey(String registryName, String schemaName) {
+ return registryName + ":" + schemaName;
+ }
+
+ private void validateName(String name, String field) {
+ if (name == null || name.isBlank()) {
+ throw new AwsException("InvalidInputException", field + " is required", 400);
+ }
+ if (name.length() > NAME_MAX_LENGTH) {
+ throw new AwsException("InvalidInputException", field + " exceeds " + NAME_MAX_LENGTH + " characters", 400);
+ }
+ if (!NAME_PATTERN.matcher(name).matches()) {
+ throw new AwsException("InvalidInputException",
+ field + " must match " + NAME_PATTERN.pattern(), 400);
+ }
+ }
+
+ private void validateDataFormat(String dataFormat) {
+ if (dataFormat == null || !DATA_FORMATS.contains(dataFormat)) {
+ throw new AwsException("InvalidInputException",
+ "DataFormat must be one of " + DATA_FORMATS, 400);
+ }
+ }
+
+ private void validateCompatibility(String compatibility) {
+ if (!COMPATIBILITY_MODES.contains(compatibility)) {
+ throw new AwsException("InvalidInputException",
+ "Compatibility must be one of " + COMPATIBILITY_MODES, 400);
+ }
+ }
+
+ private void validateDefinitionRequired(String definition) {
+ if (definition == null || definition.isBlank()) {
+ throw new AwsException("InvalidInputException", "SchemaDefinition is required", 400);
+ }
+ if (definition.length() > MAX_SCHEMA_DEFINITION_LENGTH) {
+ throw new AwsException("InvalidInputException",
+ "SchemaDefinition exceeds " + MAX_SCHEMA_DEFINITION_LENGTH + " characters", 400);
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaCompatibilityChecker.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaCompatibilityChecker.java
new file mode 100644
index 0000000000..0063973af7
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaCompatibilityChecker.java
@@ -0,0 +1,146 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import io.apicurio.registry.content.ContentHandle;
+import io.apicurio.registry.content.canon.AvroContentCanonicalizer;
+import io.apicurio.registry.content.canon.ContentCanonicalizer;
+import io.apicurio.registry.content.canon.JsonContentCanonicalizer;
+import io.apicurio.registry.content.canon.ProtobufContentCanonicalizer;
+import io.apicurio.registry.rules.compatibility.AvroCompatibilityChecker;
+import io.apicurio.registry.rules.compatibility.CompatibilityChecker;
+import io.apicurio.registry.rules.compatibility.CompatibilityDifference;
+import io.apicurio.registry.rules.compatibility.CompatibilityExecutionResult;
+import io.apicurio.registry.rules.compatibility.CompatibilityLevel;
+import io.apicurio.registry.rules.compatibility.JsonSchemaCompatibilityChecker;
+import io.apicurio.registry.rules.compatibility.ProtobufCompatibilityChecker;
+import io.apicurio.registry.rules.validity.AvroContentValidator;
+import io.apicurio.registry.rules.validity.ContentValidator;
+import io.apicurio.registry.rules.validity.JsonSchemaContentValidator;
+import io.apicurio.registry.rules.validity.ProtobufContentValidator;
+import io.apicurio.registry.rules.validity.ValidityLevel;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Glue → Apicurio adapter for schema compatibility, validation, and canonicalization.
+ * Pure utility — no CDI. Stateless.
+ */
+public final class SchemaCompatibilityChecker {
+
+ public record Result(boolean compatible, String reason) {
+ public static Result ok() {
+ return new Result(true, null);
+ }
+ }
+
+ private SchemaCompatibilityChecker() {}
+
+ /**
+ * Check whether {@code newDefinition} is compatible with {@code existingDefinitions}
+ * under the given Glue compatibility {@code mode}.
+ *
+ * <p>{@code existingDefinitions} must be ordered by version ascending (oldest first,
+ * latest last). For non-transitive modes (BACKWARD/FORWARD/FULL) only the latest
+ * existing version is compared; for transitive modes (BACKWARD_ALL/FORWARD_ALL/FULL_ALL)
+ * every prior version is compared.
+ */
+ public static Result check(String mode, List<String> existingDefinitions, String newDefinition, String dataFormat) {
+ if (mode == null || existingDefinitions == null || existingDefinitions.isEmpty()) {
+ return Result.ok();
+ }
+ if ("NONE".equals(mode) || "DISABLED".equals(mode)) {
+ return Result.ok();
+ }
+ CompatibilityLevel level = toApicurioLevel(mode);
+ CompatibilityChecker checker = checkerFor(dataFormat);
+
+ List<ContentHandle> existing = existingDefinitions.stream()
+ .map(ContentHandle::create)
+ .collect(Collectors.toList());
+ ContentHandle proposed = ContentHandle.create(newDefinition);
+
+ CompatibilityExecutionResult result = checker.testCompatibility(level, existing, proposed, Map.of());
+ if (result.isCompatible()) {
+ return Result.ok();
+ }
+ return new Result(false, formatDifferences(result));
+ }
+
+ public static String canonicalize(String definition, String dataFormat) {
+ ContentCanonicalizer canon = canonicalizerFor(dataFormat);
+ ContentHandle handle = ContentHandle.create(definition);
+ return canon.canonicalize(handle, Map.of()).content();
+ }
+
+ /**
+ * Validate that {@code definition} is parseable for the declared {@code dataFormat}.
+ * @return null when valid; an error message when invalid.
+ */
+ public static String validateDefinition(String definition, String dataFormat) {
+ ContentValidator validator = validatorFor(dataFormat);
+ try {
+ validator.validate(ValidityLevel.SYNTAX_ONLY, ContentHandle.create(definition), Map.of());
+ return null;
+ } catch (RuntimeException e) {
+ return e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName();
+ }
+ }
+
+ private static CompatibilityLevel toApicurioLevel(String glueMode) {
+ return switch (glueMode) {
+ case "NONE", "DISABLED" -> CompatibilityLevel.NONE;
+ case "BACKWARD" -> CompatibilityLevel.BACKWARD;
+ case "BACKWARD_ALL" -> CompatibilityLevel.BACKWARD_TRANSITIVE;
+ case "FORWARD" -> CompatibilityLevel.FORWARD;
+ case "FORWARD_ALL" -> CompatibilityLevel.FORWARD_TRANSITIVE;
+ case "FULL" -> CompatibilityLevel.FULL;
+ case "FULL_ALL" -> CompatibilityLevel.FULL_TRANSITIVE;
+ default -> throw new IllegalArgumentException("Unknown compatibility mode: " + glueMode);
+ };
+ }
+
+ private static CompatibilityChecker checkerFor(String dataFormat) {
+ return switch (dataFormat) {
+ case "AVRO" -> new AvroCompatibilityChecker();
+ case "JSON" -> new JsonSchemaCompatibilityChecker();
+ case "PROTOBUF" -> new ProtobufCompatibilityChecker();
+ default -> throw new IllegalArgumentException("Unsupported DataFormat: " + dataFormat);
+ };
+ }
+
+ private static ContentCanonicalizer canonicalizerFor(String dataFormat) {
+ return switch (dataFormat) {
+ case "AVRO" -> new AvroContentCanonicalizer();
+ case "JSON" -> new JsonContentCanonicalizer();
+ case "PROTOBUF" -> new ProtobufContentCanonicalizer();
+ default -> throw new IllegalArgumentException("Unsupported DataFormat: " + dataFormat);
+ };
+ }
+
+ private static ContentValidator validatorFor(String dataFormat) {
+ return switch (dataFormat) {
+ case "AVRO" -> new AvroContentValidator();
+ case "JSON" -> new JsonSchemaContentValidator();
+ case "PROTOBUF" -> new ProtobufContentValidator();
+ default -> throw new IllegalArgumentException("Unsupported DataFormat: " + dataFormat);
+ };
+ }
+
+ private static String formatDifferences(CompatibilityExecutionResult result) {
+ if (result.getIncompatibleDifferences() == null || result.getIncompatibleDifferences().isEmpty()) {
+ return "Schema is incompatible";
+ }
+ return result.getIncompatibleDifferences().stream()
+ .map(d -> {
+ var rv = d.asRuleViolation();
+ String desc = rv != null ? rv.getDescription() : null;
+ String ctx = rv != null ? rv.getContext() : null;
+ if (desc == null) {
+ return d.toString();
+ }
+ return ctx == null ? desc : desc + " at " + ctx;
+ })
+ .collect(Collectors.joining("; "));
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaToColumnsConverter.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaToColumnsConverter.java
new file mode 100644
index 0000000000..85de4a3f8b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaToColumnsConverter.java
@@ -0,0 +1,267 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.squareup.wire.schema.Field;
+import com.squareup.wire.schema.Location;
+import com.squareup.wire.schema.internal.parser.FieldElement;
+import com.squareup.wire.schema.internal.parser.MessageElement;
+import com.squareup.wire.schema.internal.parser.ProtoFileElement;
+import com.squareup.wire.schema.internal.parser.ProtoParser;
+import com.squareup.wire.schema.internal.parser.TypeElement;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import org.apache.avro.LogicalType;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
+import org.jboss.logging.Logger;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Converts a registered schema definition (Avro / JSON Schema / Protobuf source) into a list
+ * of Glue Catalog {@link Column} objects with Hive-style type strings.
+ *
+ * <p>Top-level fields only — no flattening of nested records. Unknown / unsupported types
+ * fall back to {@code string}. Parse failures log a warning and return an empty list (the
+ * Catalog read should still succeed).
+ */
+public final class SchemaToColumnsConverter {
+
+ private static final Logger LOG = Logger.getLogger(SchemaToColumnsConverter.class);
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private SchemaToColumnsConverter() {}
+
+ public static List<Column> toColumns(String dataFormat, String definition) {
+ if (dataFormat == null || definition == null || definition.isBlank()) {
+ return List.of();
+ }
+ try {
+ return switch (dataFormat) {
+ case "AVRO" -> avroColumns(definition);
+ case "JSON" -> jsonColumns(definition);
+ case "PROTOBUF" -> protobufColumns(definition);
+ default -> List.of();
+ };
+ } catch (RuntimeException e) {
+ LOG.warnv("Failed to convert {0} schema to columns: {1}", dataFormat, e.getMessage());
+ return List.of();
+ }
+ }
+
+ // ---- Avro -----------------------------------------------------------
+
+ private static List<Column> avroColumns(String definition) {
+ Schema schema = new Schema.Parser().parse(definition);
+ if (schema.getType() != Schema.Type.RECORD) {
+ return List.of();
+ }
+ List<Column> out = new ArrayList<>(schema.getFields().size());
+ for (Schema.Field field : schema.getFields()) {
+ out.add(new Column(field.name(), avroToHive(field.schema())));
+ }
+ return out;
+ }
+
+ private static String avroToHive(Schema s) {
+ String logical = avroLogicalTypeToHive(s);
+ if (logical != null) {
+ return logical;
+ }
+ return switch (s.getType()) {
+ case LONG -> "bigint";
+ case INT -> "int";
+ case STRING, ENUM -> "string";
+ case BOOLEAN -> "boolean";
+ case DOUBLE -> "double";
+ case FLOAT -> "float";
+ case BYTES, FIXED -> "binary";
+ case NULL -> "string";
+ case UNION -> avroUnionToHive(s);
+ case RECORD -> avroRecordToHive(s);
+ case ARRAY -> "array<" + avroToHive(s.getElementType()) + ">";
+ case MAP -> "map<string," + avroToHive(s.getValueType()) + ">";
+ };
+ }
+
+ private static String avroLogicalTypeToHive(Schema s) {
+ LogicalType logical = s.getLogicalType();
+ if (logical == null) {
+ return null;
+ }
+ if (logical instanceof LogicalTypes.Decimal d) {
+ return "decimal(" + d.getPrecision() + "," + d.getScale() + ")";
+ }
+ return switch (logical.getName()) {
+ case "date" -> "date";
+ case "timestamp-millis", "timestamp-micros",
+ "local-timestamp-millis", "local-timestamp-micros" -> "timestamp";
+ // Hive has no TIME type — surface as string so downstream readers see the raw value.
+ case "time-millis", "time-micros", "uuid" -> "string";
+ default -> null;
+ };
+ }
+
+ private static String avroUnionToHive(Schema s) {
+ List<Schema> types = s.getTypes();
+ if (types.size() == 2) {
+ if (types.get(0).getType() == Schema.Type.NULL) return avroToHive(types.get(1));
+ if (types.get(1).getType() == Schema.Type.NULL) return avroToHive(types.get(0));
+ }
+ return "string";
+ }
+
+ private static String avroRecordToHive(Schema s) {
+ StringBuilder sb = new StringBuilder("struct<");
+ boolean first = true;
+ for (Schema.Field f : s.getFields()) {
+ if (!first) sb.append(",");
+ sb.append(f.name()).append(":").append(avroToHive(f.schema()));
+ first = false;
+ }
+ return sb.append(">").toString();
+ }
+
+ // ---- JSON Schema ----------------------------------------------------
+
+ private static List<Column> jsonColumns(String definition) {
+ JsonNode root;
+ try {
+ root = MAPPER.readTree(definition);
+ } catch (Exception e) {
+ LOG.warnv("Invalid JSON Schema: {0}", e.getMessage());
+ return List.of();
+ }
+ if (!"object".equals(root.path("type").asText())) {
+ return List.of();
+ }
+ JsonNode properties = root.get("properties");
+ if (properties == null || !properties.isObject()) {
+ return List.of();
+ }
+ List<Column> out = new ArrayList<>();
+ for (Iterator<Map.Entry<String, JsonNode>> it = properties.fields(); it.hasNext(); ) {
+ Map.Entry<String, JsonNode> e = it.next();
+ out.add(new Column(e.getKey(), jsonToHive(e.getValue())));
+ }
+ return out;
+ }
+
+ private static String jsonToHive(JsonNode node) {
+ String t = node.path("type").asText("string");
+ return switch (t) {
+ case "string" -> "string";
+ case "integer" -> "bigint";
+ case "number" -> "double";
+ case "boolean" -> "boolean";
+ case "array" -> {
+ JsonNode items = node.get("items");
+ yield "array<" + (items != null && items.isObject() ? jsonToHive(items) : "string") + ">";
+ }
+ case "object" -> jsonObjectToHive(node);
+ default -> "string";
+ };
+ }
+
+ private static String jsonObjectToHive(JsonNode node) {
+ JsonNode properties = node.get("properties");
+ if (properties == null || !properties.isObject() || !properties.fields().hasNext()) {
+ return "struct<>";
+ }
+ StringBuilder sb = new StringBuilder("struct<");
+ boolean first = true;
+ for (Iterator<Map.Entry<String, JsonNode>> it = properties.fields(); it.hasNext(); ) {
+ Map.Entry<String, JsonNode> e = it.next();
+ if (!first) sb.append(",");
+ sb.append(e.getKey()).append(":").append(jsonToHive(e.getValue()));
+ first = false;
+ }
+ return sb.append(">").toString();
+ }
+
+ // ---- Protobuf (parsed via Wire — bypasses Apicurio's well-known-deps issue) -------
+
+ private static List<Column> protobufColumns(String definition) {
+ ProtoFileElement file;
+ try {
+ Location loc = Location.get("schema.proto");
+ file = new ProtoParser(loc, definition.toCharArray()).readProtoFile();
+ } catch (Exception e) {
+ LOG.warnv("Invalid Protobuf schema: {0}", e.getMessage());
+ return List.of();
+ }
+ MessageElement target = firstMessage(file.getTypes());
+ if (target == null) {
+ return List.of();
+ }
+ Map<String, MessageElement> nestedByName = collectMessages(file.getTypes(), new HashMap<>());
+ List<Column> out = new ArrayList<>(target.getFields().size());
+ for (FieldElement f : target.getFields()) {
+ out.add(new Column(f.getName(), protobufToHive(f, nestedByName)));
+ }
+ return out;
+ }
+
+ private static MessageElement firstMessage(List<TypeElement> types) {
+ for (TypeElement t : types) {
+ if (t instanceof MessageElement m) return m;
+ }
+ return null;
+ }
+
+ private static Map<String, MessageElement> collectMessages(List<TypeElement> types, Map<String, MessageElement> acc) {
+ for (TypeElement t : types) {
+ if (t instanceof MessageElement m) {
+ acc.put(m.getName(), m);
+ collectMessages(m.getNestedTypes(), acc);
+ }
+ }
+ return acc;
+ }
+
+ private static String protobufToHive(FieldElement field, Map<String, MessageElement> messages) {
+ String base = protobufBaseTypeToHive(field.getType(), messages);
+ return field.getLabel() == Field.Label.REPEATED ? "array<" + base + ">" : base;
+ }
+
+ private static String protobufBaseTypeToHive(String type, Map<String, MessageElement> messages) {
+ if (type.startsWith("map<") && type.endsWith(">")) {
+ // Wire preserves the literal "map<key, value>" syntax in field types.
+ String inner = type.substring(4, type.length() - 1);
+ int comma = inner.indexOf(',');
+ if (comma > 0) {
+ String valueType = inner.substring(comma + 1).trim();
+ return "map<string," + protobufBaseTypeToHive(valueType, messages) + ">";
+ }
+ return "map<string,string>";
+ }
+ return switch (type) {
+ case "int32", "uint32", "sint32", "fixed32", "sfixed32" -> "int";
+ case "int64", "uint64", "sint64", "fixed64", "sfixed64" -> "bigint";
+ case "float" -> "float";
+ case "double" -> "double";
+ case "bool" -> "boolean";
+ case "string" -> "string";
+ case "bytes" -> "binary";
+ default -> {
+ MessageElement nested = messages.get(type);
+ yield nested != null ? protobufMessageToHive(nested, messages) : "string";
+ }
+ };
+ }
+
+ private static String protobufMessageToHive(MessageElement msg, Map<String, MessageElement> messages) {
+ StringBuilder sb = new StringBuilder("struct<");
+ boolean first = true;
+ for (FieldElement f : msg.getFields()) {
+ if (!first) sb.append(",");
+ sb.append(f.getName()).append(":").append(protobufToHive(f, messages));
+ first = false;
+ }
+ return sb.append(">").toString();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/MetadataInfo.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/MetadataInfo.java
new file mode 100644
index 0000000000..e455e6ef01
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/MetadataInfo.java
@@ -0,0 +1,66 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.List;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class MetadataInfo {
+
+ @JsonProperty("MetadataValue")
+ private String metadataValue;
+
+ @JsonProperty("CreatedTime")
+ @JsonFormat(shape = JsonFormat.Shape.STRING)
+ private Instant createdTime;
+
+ @JsonProperty("OtherMetadataValueList")
+ private List<OtherMetadataValueListItem> otherMetadataValueList;
+
+ public MetadataInfo() {}
+
+ public MetadataInfo(String metadataValue, Instant createdTime) {
+ this.metadataValue = metadataValue;
+ this.createdTime = createdTime;
+ }
+
+ public String getMetadataValue() { return metadataValue; }
+ public void setMetadataValue(String metadataValue) { this.metadataValue = metadataValue; }
+
+ public Instant getCreatedTime() { return createdTime; }
+ public void setCreatedTime(Instant createdTime) { this.createdTime = createdTime; }
+
+ public List<OtherMetadataValueListItem> getOtherMetadataValueList() { return otherMetadataValueList; }
+ public void setOtherMetadataValueList(List<OtherMetadataValueListItem> otherMetadataValueList) {
+ this.otherMetadataValueList = otherMetadataValueList;
+ }
+
+ @RegisterForReflection
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class OtherMetadataValueListItem {
+ @JsonProperty("MetadataValue")
+ private String metadataValue;
+
+ @JsonProperty("CreatedTime")
+ @JsonFormat(shape = JsonFormat.Shape.STRING)
+ private Instant createdTime;
+
+ public OtherMetadataValueListItem() {}
+
+ public OtherMetadataValueListItem(String metadataValue, Instant createdTime) {
+ this.metadataValue = metadataValue;
+ this.createdTime = createdTime;
+ }
+
+ public String getMetadataValue() { return metadataValue; }
+ public void setMetadataValue(String metadataValue) { this.metadataValue = metadataValue; }
+
+ public Instant getCreatedTime() { return createdTime; }
+ public void setCreatedTime(Instant createdTime) { this.createdTime = createdTime; }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/Registry.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/Registry.java
new file mode 100644
index 0000000000..c4140cda4f
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/Registry.java
@@ -0,0 +1,67 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.Map;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Registry {
+
+ @JsonProperty("RegistryName")
+ private String registryName;
+
+ @JsonProperty("RegistryArn")
+ private String registryArn;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("Status")
+ private String status;
+
+ @JsonProperty("CreatedTime")
+ @JsonFormat(shape = JsonFormat.Shape.STRING)
+ private Instant createdTime;
+
+ @JsonProperty("UpdatedTime")
+ @JsonFormat(shape = JsonFormat.Shape.STRING)
+ private Instant updatedTime;
+
+ @JsonProperty("Tags")
+ private Map<String, String> tags;
+
+ public Registry() {}
+
+ public Registry(String registryName) {
+ this.registryName = registryName;
+ this.createdTime = Instant.now();
+ this.updatedTime = this.createdTime;
+ this.status = "AVAILABLE";
+ }
+
+ public String getRegistryName() { return registryName; }
+ public void setRegistryName(String registryName) { this.registryName = registryName; }
+
+ public String getRegistryArn() { return registryArn; }
+ public void setRegistryArn(String registryArn) { this.registryArn = registryArn; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public String getStatus() { return status; }
+ public void setStatus(String status) { this.status = status; }
+
+ public Instant getCreatedTime() { return createdTime; }
+ public void setCreatedTime(Instant createdTime) { this.createdTime = createdTime; }
+
+ public Instant getUpdatedTime() { return updatedTime; }
+ public void setUpdatedTime(Instant updatedTime) { this.updatedTime = updatedTime; }
+
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/RegistryId.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/RegistryId.java
new file mode 100644
index 0000000000..80ace4f96d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/RegistryId.java
@@ -0,0 +1,29 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class RegistryId {
+
+ @JsonProperty("RegistryName")
+ private String registryName;
+
+ @JsonProperty("RegistryArn")
+ private String registryArn;
+
+ public RegistryId() {}
+
+ public RegistryId(String registryName, String registryArn) {
+ this.registryName = registryName;
+ this.registryArn = registryArn;
+ }
+
+ public String getRegistryName() { return registryName; }
+ public void setRegistryName(String registryName) { this.registryName = registryName; }
+
+ public String getRegistryArn() { return registryArn; }
+ public void setRegistryArn(String registryArn) { this.registryArn = registryArn; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/Schema.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/Schema.java
new file mode 100644
index 0000000000..0e61d396b3
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/Schema.java
@@ -0,0 +1,102 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.Map;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Schema {
+
+ @JsonProperty("RegistryName")
+ private String registryName;
+
+ @JsonProperty("RegistryArn")
+ private String registryArn;
+
+ @JsonProperty("SchemaName")
+ private String schemaName;
+
+ @JsonProperty("SchemaArn")
+ private String schemaArn;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("DataFormat")
+ private String dataFormat;
+
+ @JsonProperty("Compatibility")
+ private String compatibility;
+
+ @JsonProperty("SchemaStatus")
+ private String schemaStatus;
+
+ @JsonProperty("SchemaCheckpoint")
+ private Long schemaCheckpoint;
+
+ @JsonProperty("LatestSchemaVersion")
+ private Long latestSchemaVersion;
+
+ @JsonProperty("NextSchemaVersion")
+ private Long nextSchemaVersion;
+
+ @JsonProperty("CreatedTime")
+ @JsonFormat(shape = JsonFormat.Shape.STRING)
+ private Instant createdTime;
+
+ @JsonProperty("UpdatedTime")
+ @JsonFormat(shape = JsonFormat.Shape.STRING)
+ private Instant updatedTime;
+
+ @JsonProperty("Tags")
+ private Map<String, String> tags;
+
+ public Schema() {}
+
+ public String getRegistryName() { return registryName; }
+ public void setRegistryName(String registryName) { this.registryName = registryName; }
+
+ public String getRegistryArn() { return registryArn; }
+ public void setRegistryArn(String registryArn) { this.registryArn = registryArn; }
+
+ public String getSchemaName() { return schemaName; }
+ public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
+
+ public String getSchemaArn() { return schemaArn; }
+ public void setSchemaArn(String schemaArn) { this.schemaArn = schemaArn; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public String getDataFormat() { return dataFormat; }
+ public void setDataFormat(String dataFormat) { this.dataFormat = dataFormat; }
+
+ public String getCompatibility() { return compatibility; }
+ public void setCompatibility(String compatibility) { this.compatibility = compatibility; }
+
+ public String getSchemaStatus() { return schemaStatus; }
+ public void setSchemaStatus(String schemaStatus) { this.schemaStatus = schemaStatus; }
+
+ public Long getSchemaCheckpoint() { return schemaCheckpoint; }
+ public void setSchemaCheckpoint(Long schemaCheckpoint) { this.schemaCheckpoint = schemaCheckpoint; }
+
+ public Long getLatestSchemaVersion() { return latestSchemaVersion; }
+ public void setLatestSchemaVersion(Long latestSchemaVersion) { this.latestSchemaVersion = latestSchemaVersion; }
+
+ public Long getNextSchemaVersion() { return nextSchemaVersion; }
+ public void setNextSchemaVersion(Long nextSchemaVersion) { this.nextSchemaVersion = nextSchemaVersion; }
+
+ public Instant getCreatedTime() { return createdTime; }
+ public void setCreatedTime(Instant createdTime) { this.createdTime = createdTime; }
+
+ public Instant getUpdatedTime() { return updatedTime; }
+ public void setUpdatedTime(Instant updatedTime) { this.updatedTime = updatedTime; }
+
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/SchemaId.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/SchemaId.java
new file mode 100644
index 0000000000..dd8105cc33
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/SchemaId.java
@@ -0,0 +1,36 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SchemaId {
+
+ @JsonProperty("SchemaArn")
+ private String schemaArn;
+
+ @JsonProperty("SchemaName")
+ private String schemaName;
+
+ @JsonProperty("RegistryName")
+ private String registryName;
+
+ public SchemaId() {}
+
+ public SchemaId(String registryName, String schemaName, String schemaArn) {
+ this.registryName = registryName;
+ this.schemaName = schemaName;
+ this.schemaArn = schemaArn;
+ }
+
+ public String getSchemaArn() { return schemaArn; }
+ public void setSchemaArn(String schemaArn) { this.schemaArn = schemaArn; }
+
+ public String getSchemaName() { return schemaName; }
+ public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
+
+ public String getRegistryName() { return registryName; }
+ public void setRegistryName(String registryName) { this.registryName = registryName; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/SchemaVersion.java b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/SchemaVersion.java
new file mode 100644
index 0000000000..88135d122b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/schemaregistry/model/SchemaVersion.java
@@ -0,0 +1,58 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SchemaVersion {
+
+ @JsonProperty("SchemaVersionId")
+ private String schemaVersionId;
+
+ @JsonProperty("SchemaArn")
+ private String schemaArn;
+
+ @JsonProperty("VersionNumber")
+ private Long versionNumber;
+
+ @JsonProperty("Status")
+ private String status;
+
+ @JsonProperty("SchemaDefinition")
+ private String schemaDefinition;
+
+ @JsonProperty("DataFormat")
+ private String dataFormat;
+
+ @JsonProperty("CreatedTime")
+ @JsonFormat(shape = JsonFormat.Shape.STRING)
+ private Instant createdTime;
+
+ public SchemaVersion() {}
+
+ public String getSchemaVersionId() { return schemaVersionId; }
+ public void setSchemaVersionId(String schemaVersionId) { this.schemaVersionId = schemaVersionId; }
+
+ public String getSchemaArn() { return schemaArn; }
+ public void setSchemaArn(String schemaArn) { this.schemaArn = schemaArn; }
+
+ public Long getVersionNumber() { return versionNumber; }
+ public void setVersionNumber(Long versionNumber) { this.versionNumber = versionNumber; }
+
+ public String getStatus() { return status; }
+ public void setStatus(String status) { this.status = status; }
+
+ public String getSchemaDefinition() { return schemaDefinition; }
+ public void setSchemaDefinition(String schemaDefinition) { this.schemaDefinition = schemaDefinition; }
+
+ public String getDataFormat() { return dataFormat; }
+ public void setDataFormat(String dataFormat) { this.dataFormat = dataFormat; }
+
+ public Instant getCreatedTime() { return createdTime; }
+ public void setCreatedTime(Instant createdTime) { this.createdTime = createdTime; }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogSchemaBindingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogSchemaBindingIntegrationTest.java
new file mode 100644
index 0000000000..641a8c3417
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogSchemaBindingIntegrationTest.java
@@ -0,0 +1,154 @@
+package io.github.hectorvent.floci.services.glue;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class GlueCatalogSchemaBindingIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String REGISTRY = "binding-registry";
+ private static final String SCHEMA = "users";
+ private static final String DATABASE = "binding-db";
+ private static final String TABLE = "users_table";
+
+ private static final String AVRO_V1 =
+ "{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"User\\\",\\\"namespace\\\":\\\"x\\\","
+ + "\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\"},"
+ + "{\\\"name\\\":\\\"name\\\",\\\"type\\\":\\\"string\\\"}]}";
+
+ private static final String AVRO_V2 =
+ "{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"User\\\",\\\"namespace\\\":\\\"x\\\","
+ + "\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\"},"
+ + "{\\\"name\\\":\\\"name\\\",\\\"type\\\":\\\"string\\\"},"
+ + "{\\\"name\\\":\\\"email\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null}]}";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void seed_registryAndSchemaAndDatabase() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateRegistry")
+ .body("{ \"RegistryName\": \"" + REGISTRY + "\" }")
+ .when().post("/").then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateSchema")
+ .body("{"
+ + " \"RegistryId\": { \"RegistryName\": \"" + REGISTRY + "\" },"
+ + " \"SchemaName\": \"" + SCHEMA + "\","
+ + " \"DataFormat\": \"AVRO\","
+ + " \"Compatibility\": \"BACKWARD\","
+ + " \"SchemaDefinition\": \"" + AVRO_V1 + "\""
+ + " }")
+ .when().post("/").then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateDatabase")
+ .body("{ \"DatabaseInput\": { \"Name\": \"" + DATABASE + "\" } }")
+ .when().post("/").then().statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void createTableWithSchemaReferenceLatest() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateTable")
+ .body("{"
+ + " \"DatabaseName\": \"" + DATABASE + "\","
+ + " \"TableInput\": {"
+ + " \"Name\": \"" + TABLE + "\","
+ + " \"StorageDescriptor\": {"
+ + " \"Location\": \"s3://bucket/users/\","
+ + " \"Columns\": [],"
+ + " \"SchemaReference\": {"
+ + " \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" }"
+ + " }"
+ + " }"
+ + " } }")
+ .when().post("/").then().statusCode(200);
+ }
+
+ @Test
+ @Order(3)
+ void getTableReturnsColumnsDerivedFromV1() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetTable")
+ .body("{ \"DatabaseName\": \"" + DATABASE + "\", \"Name\": \"" + TABLE + "\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Table.Name", equalTo(TABLE))
+ .body("Table.StorageDescriptor.Columns", hasSize(2))
+ .body("Table.StorageDescriptor.Columns[0].Name", equalTo("id"))
+ .body("Table.StorageDescriptor.Columns[0].Type", equalTo("bigint"))
+ .body("Table.StorageDescriptor.Columns[1].Name", equalTo("name"))
+ .body("Table.StorageDescriptor.Columns[1].Type", equalTo("string"))
+ .body("Table.StorageDescriptor.SchemaReference", notNullValue());
+ }
+
+ @Test
+ @Order(4)
+ void registerV2AndGetTableReflectsLatestSchema() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.RegisterSchemaVersion")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaDefinition\": \"" + AVRO_V2 + "\" }")
+ .when().post("/").then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetTable")
+ .body("{ \"DatabaseName\": \"" + DATABASE + "\", \"Name\": \"" + TABLE + "\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Table.StorageDescriptor.Columns", hasSize(3))
+ .body("Table.StorageDescriptor.Columns[2].Name", equalTo("email"))
+ .body("Table.StorageDescriptor.Columns[2].Type", equalTo("string"));
+ }
+
+ @Test
+ @Order(5)
+ void createTableWithBrokenReferenceReturns400() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateTable")
+ .body("{"
+ + " \"DatabaseName\": \"" + DATABASE + "\","
+ + " \"TableInput\": {"
+ + " \"Name\": \"broken-table\","
+ + " \"StorageDescriptor\": {"
+ + " \"Location\": \"s3://bucket/x/\","
+ + " \"SchemaReference\": {"
+ + " \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"does-not-exist\" }"
+ + " }"
+ + " }"
+ + " } }")
+ .when().post("/").then()
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+
+ @Test
+ @Order(6)
+ void getTablesAppliesResolutionToEachTable() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetTables")
+ .body("{ \"DatabaseName\": \"" + DATABASE + "\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("TableList[0].StorageDescriptor.Columns", hasSize(3));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
new file mode 100644
index 0000000000..698d312224
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -0,0 +1,207 @@
+package io.github.hectorvent.floci.services.glue;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Database;
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.SchemaReference;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.RegistryId;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class GlueServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT_ID = "000000000000";
+
+ private static final String AVRO_V1 =
+ "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"x\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"}]}";
+
+ private static final String AVRO_V2 =
+ "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"x\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":[\"null\",\"string\"],\"default\":null}]}";
+
+ private GlueService glueService;
+ private GlueSchemaRegistryService schemaRegistryService;
+
+ @BeforeEach
+ void setUp() {
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);
+ StorageFactory storageFactory = new InMemoryStorageFactory();
+ schemaRegistryService = new GlueSchemaRegistryService(storageFactory, regionResolver);
+ glueService = new GlueService(
+ new InMemoryStorage<String, Database>(),
+ new InMemoryStorage<String, Table>(),
+ new InMemoryStorage<String, Partition>(),
+ schemaRegistryService, regionResolver);
+ glueService.createDatabase(new Database("db1"));
+ }
+
+ @Test
+ void getTableWithoutSchemaReferenceReturnsColumnsUnchanged() {
+ Table table = new Table();
+ table.setName("plain");
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setColumns(java.util.List.of(new Column("a", "string")));
+ table.setStorageDescriptor(sd);
+ glueService.createTable("db1", table);
+
+ Table fetched = glueService.getTable("db1", "plain");
+
+ assertEquals(1, fetched.getStorageDescriptor().getColumns().size());
+ assertEquals("a", fetched.getStorageDescriptor().getColumns().get(0).getName());
+ assertNull(fetched.getStorageDescriptor().getSchemaReference());
+ }
+
+ @Test
+ void getTableWithValidSchemaReferenceReturnsDerivedColumns() {
+ schemaRegistryService.createRegistry("r1", null, null, REGION);
+ schemaRegistryService.createSchema(new RegistryId("r1", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ Table table = tableReferencing("r1", "users", null, null);
+ glueService.createTable("db1", table);
+
+ Table fetched = glueService.getTable("db1", "withref");
+
+ assertEquals(1, fetched.getStorageDescriptor().getColumns().size());
+ assertEquals("id", fetched.getStorageDescriptor().getColumns().get(0).getName());
+ assertEquals("bigint", fetched.getStorageDescriptor().getColumns().get(0).getType());
+ assertNotNull(fetched.getStorageDescriptor().getSchemaReference());
+ }
+
+ @Test
+ void getTablePicksUpNewVersionWhenPinnedToLatest() {
+ schemaRegistryService.createRegistry("r1", null, null, REGION);
+ schemaRegistryService.createSchema(new RegistryId("r1", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ Table storedTable = tableReferencing("r1", "users", null, null);
+ glueService.createTable("db1", storedTable);
+
+ Table firstFetch = glueService.getTable("db1", "withref");
+
+ assertEquals(1, firstFetch.getStorageDescriptor().getColumns().size());
+ assertTrue(storedTable.getStorageDescriptor().getColumns() == null
+ || storedTable.getStorageDescriptor().getColumns().isEmpty());
+
+ // Register v2 — adds optional email field.
+ schemaRegistryService.registerSchemaVersion(
+ new SchemaId("r1", "users", null), AVRO_V2, REGION);
+
+ Table fetched = glueService.getTable("db1", "withref");
+
+ assertEquals(2, fetched.getStorageDescriptor().getColumns().size());
+ assertEquals("email", fetched.getStorageDescriptor().getColumns().get(1).getName());
+ }
+
+ @Test
+ void getTablePinnedToVersionNumberStaysOnThatVersion() {
+ schemaRegistryService.createRegistry("r1", null, null, REGION);
+ schemaRegistryService.createSchema(new RegistryId("r1", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ glueService.createTable("db1", tableReferencing("r1", "users", 1L, null));
+ schemaRegistryService.registerSchemaVersion(
+ new SchemaId("r1", "users", null), AVRO_V2, REGION);
+
+ Table fetched = glueService.getTable("db1", "withref");
+
+ assertEquals(1, fetched.getStorageDescriptor().getColumns().size(), "should still see v1");
+ }
+
+ @Test
+ void createTableWithBrokenSchemaReferenceThrows() {
+ Table table = tableReferencing("does-not-exist", "users", null, null);
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> glueService.createTable("db1", table));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void getTableWithStaleSchemaReferenceReturnsTableTolerantly() {
+ schemaRegistryService.createRegistry("r1", null, null, REGION);
+ schemaRegistryService.createSchema(new RegistryId("r1", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ glueService.createTable("db1", tableReferencing("r1", "users", null, null));
+
+ // Delete the underlying schema after the table was created.
+ schemaRegistryService.deleteSchema(new SchemaId("r1", "users", null), REGION);
+
+ Table fetched = glueService.getTable("db1", "withref");
+
+ // Tolerant path: table is returned, columns are whatever was stored at create
+ // time (in our case nothing — we never wrote columns explicitly).
+ assertNotNull(fetched);
+ assertNotNull(fetched.getStorageDescriptor().getSchemaReference());
+ assertTrue(fetched.getStorageDescriptor().getColumns() == null
+ || fetched.getStorageDescriptor().getColumns().isEmpty());
+ }
+
+ @Test
+ void getTablesAppliesResolutionToEachTable() {
+ schemaRegistryService.createRegistry("r1", null, null, REGION);
+ schemaRegistryService.createSchema(new RegistryId("r1", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ glueService.createTable("db1", tableReferencing("r1", "users", null, null));
+ Table plain = new Table();
+ plain.setName("plain");
+ StorageDescriptor sd = new StorageDescriptor();
+ plain.setStorageDescriptor(sd);
+ glueService.createTable("db1", plain);
+
+ var tables = glueService.getTables("db1");
+
+ assertEquals(2, tables.size());
+ for (Table t : tables) {
+ if ("withref".equals(t.getName())) {
+ assertEquals(1, t.getStorageDescriptor().getColumns().size());
+ }
+ }
+ }
+
+ private Table tableReferencing(String registryName, String schemaName, Long versionNumber, String versionId) {
+ Table table = new Table();
+ table.setName("withref");
+ StorageDescriptor sd = new StorageDescriptor();
+ SchemaReference ref = new SchemaReference();
+ SchemaId schemaId = new SchemaId(registryName, schemaName, null);
+ ref.setSchemaId(schemaId);
+ ref.setSchemaVersionNumber(versionNumber);
+ ref.setSchemaVersionId(versionId);
+ sd.setSchemaReference(ref);
+ table.setStorageDescriptor(sd);
+ return table;
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ public <K, V> StorageBackend<K, V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<K, V>> typeReference) {
+ return new InMemoryStorage<>();
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryAdminIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryAdminIntegrationTest.java
new file mode 100644
index 0000000000..92d0d673ee
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryAdminIntegrationTest.java
@@ -0,0 +1,207 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class GlueSchemaRegistryAdminIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String REGISTRY = "admin-it-registry";
+ private static final String SCHEMA = "users-admin";
+
+ private static final String AVRO_V1 =
+ "{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"User\\\",\\\"namespace\\\":\\\"x\\\","
+ + "\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\"}]}";
+
+ private static final String AVRO_V2 =
+ "{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"User\\\",\\\"namespace\\\":\\\"x\\\","
+ + "\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\"},"
+ + "{\\\"name\\\":\\\"email\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null}]}";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void seed_createRegistryAndSchemaWithTwoVersions() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateRegistry")
+ .body("{ \"RegistryName\": \"" + REGISTRY + "\" }")
+ .when().post("/").then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateSchema")
+ .body("{"
+ + " \"RegistryId\": { \"RegistryName\": \"" + REGISTRY + "\" },"
+ + " \"SchemaName\": \"" + SCHEMA + "\","
+ + " \"DataFormat\": \"AVRO\","
+ + " \"Compatibility\": \"BACKWARD\","
+ + " \"SchemaDefinition\": \"" + AVRO_V1 + "\""
+ + " }")
+ .when().post("/").then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.RegisterSchemaVersion")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaDefinition\": \"" + AVRO_V2 + "\" }")
+ .when().post("/").then().statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void getSchema() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchema")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" } }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("SchemaName", equalTo(SCHEMA))
+ .body("LatestSchemaVersion", equalTo(2));
+ }
+
+ @Test
+ @Order(3)
+ void updateSchemaCompatibility() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.UpdateSchema")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"Compatibility\": \"FORWARD\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("SchemaArn", containsString(":schema/" + REGISTRY + "/" + SCHEMA));
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchema")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" } }")
+ .when().post("/").then()
+ .body("Compatibility", equalTo("FORWARD"));
+ }
+
+ @Test
+ @Order(4)
+ void listSchemasIncludesCreated() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.ListSchemas")
+ .body("{ \"RegistryId\": { \"RegistryName\": \"" + REGISTRY + "\" } }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Schemas.SchemaName", hasItem(SCHEMA))
+ .body("Schemas.find { it.SchemaName == '" + SCHEMA + "' }.DataFormat", nullValue())
+ .body("Schemas.find { it.SchemaName == '" + SCHEMA + "' }.Tags", nullValue());
+ }
+
+ @Test
+ @Order(5)
+ void listSchemaVersionsReturnsBoth() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.ListSchemaVersions")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" } }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Schemas", hasSize(greaterThanOrEqualTo(2)))
+ .body("Schemas.VersionNumber", hasItem(1))
+ .body("Schemas.VersionNumber", hasItem(2))
+ .body("Schemas.find { it.VersionNumber == 1 }.SchemaArn", notNullValue())
+ .body("Schemas.find { it.VersionNumber == 1 }.SchemaVersionId", notNullValue())
+ .body("Schemas.find { it.VersionNumber == 1 }.Status", equalTo("AVAILABLE"))
+ .body("Schemas.find { it.VersionNumber == 1 }.CreatedTime", notNullValue())
+ .body("Schemas.find { it.VersionNumber == 1 }.DataFormat", nullValue())
+ .body("Schemas.find { it.VersionNumber == 1 }.SchemaDefinition", nullValue());
+ }
+
+ @Test
+ @Order(6)
+ void getSchemaVersionsDiffReturnsTextDiff() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchemaVersionsDiff")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"FirstSchemaVersionNumber\": { \"VersionNumber\": 1 },"
+ + " \"SecondSchemaVersionNumber\": { \"VersionNumber\": 2 },"
+ + " \"SchemaDiffType\": \"SYNTAX_DIFF\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Diff", notNullValue())
+ .body("Diff", containsString("---"));
+ }
+
+ @Test
+ @Order(7)
+ void checkSchemaVersionValidityForValidAvro() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CheckSchemaVersionValidity")
+ .body("{ \"DataFormat\": \"AVRO\", \"SchemaDefinition\": \"" + AVRO_V1 + "\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Valid", equalTo(true))
+ .body("Error", nullValue());
+ }
+
+ @Test
+ @Order(8)
+ void checkSchemaVersionValidityForInvalidAvro() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CheckSchemaVersionValidity")
+ .body("{ \"DataFormat\": \"AVRO\", \"SchemaDefinition\": \"{not-valid-avro\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Valid", equalTo(false))
+ .body("Error", notNullValue());
+ }
+
+ @Test
+ @Order(9)
+ void deleteSchemaVersionsRemovesVersion1() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.UpdateSchema")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaVersionNumber\": { \"VersionNumber\": 2 } }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("SchemaArn", containsString(":schema/" + REGISTRY + "/" + SCHEMA));
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.DeleteSchemaVersions")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"Versions\": \"1\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("SchemaArn", nullValue())
+ .body("SchemaVersionErrors", hasSize(0));
+ }
+
+ @Test
+ @Order(10)
+ void deleteSchemaCascades() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.DeleteSchema")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" } }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Status", equalTo("DELETING"));
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchema")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" } }")
+ .when().post("/").then()
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryIntegrationTest.java
new file mode 100644
index 0000000000..7fe3dc1544
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryIntegrationTest.java
@@ -0,0 +1,189 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.matchesPattern;
+import static org.hamcrest.Matchers.nullValue;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class GlueSchemaRegistryIntegrationTest {
+
+ private static final String REGISTRY_NAME = "integration-registry";
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void createRegistry() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateRegistry")
+ .body("{ \"RegistryName\": \"" + REGISTRY_NAME + "\", \"Description\": \"test\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("RegistryName", equalTo(REGISTRY_NAME))
+ .body("RegistryArn", containsString(":registry/" + REGISTRY_NAME))
+ .body("Status", equalTo("AVAILABLE"));
+ }
+
+ @Test
+ @Order(2)
+ void createDuplicateRegistryReturnsAlreadyExists() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateRegistry")
+ .body("{ \"RegistryName\": \"" + REGISTRY_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("AlreadyExistsException"));
+ }
+
+ @Test
+ @Order(3)
+ void createRegistryWithInvalidNameReturnsInvalidInput() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateRegistry")
+ .body("{ \"RegistryName\": \"bad name with spaces\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidInputException"));
+ }
+
+ @Test
+ @Order(4)
+ void getRegistryByName() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetRegistry")
+ .body("{ \"RegistryId\": { \"RegistryName\": \"" + REGISTRY_NAME + "\" } }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("RegistryName", equalTo(REGISTRY_NAME))
+ .body("Description", equalTo("test"))
+ .body("Status", equalTo("AVAILABLE"))
+ .body("CreatedTime", matchesPattern("^\\d{4}-\\d{2}-\\d{2}T.*Z$"));
+ }
+
+ @Test
+ @Order(5)
+ void listRegistriesIncludesCreated() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.ListRegistries")
+ .body("{}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("Registries.RegistryName", hasItem(REGISTRY_NAME))
+ .body("Registries.find { it.RegistryName == '" + REGISTRY_NAME + "' }.Tags", nullValue())
+ .body("Registries.find { it.RegistryName == '" + REGISTRY_NAME + "' }.CreatedTime",
+ matchesPattern("^\\d{4}-\\d{2}-\\d{2}T.*Z$"));
+ }
+
+ @Test
+ @Order(6)
+ void updateRegistryDescription() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.UpdateRegistry")
+ .body("{ \"RegistryId\": { \"RegistryName\": \"" + REGISTRY_NAME + "\" }, \"Description\": \"updated\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("RegistryName", equalTo(REGISTRY_NAME))
+ .body("RegistryArn", containsString(":registry/" + REGISTRY_NAME))
+ .body("Description", nullValue());
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetRegistry")
+ .body("{ \"RegistryId\": { \"RegistryName\": \"" + REGISTRY_NAME + "\" } }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("Description", equalTo("updated"));
+ }
+
+ @Test
+ @Order(7)
+ void getRegistryWithMalformedArnReturnsInvalidInput() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetRegistry")
+ .body("{ \"RegistryId\": { \"RegistryArn\": \"not-an-arn\" } }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidInputException"));
+ }
+
+ @Test
+ @Order(8)
+ void getRegistryWithoutIdAutoCreatesDefault() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetRegistry")
+ .body("{}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("RegistryName", equalTo("default-registry"));
+ }
+
+ @Test
+ @Order(9)
+ void deleteRegistryReturnsDeletingStatus() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.DeleteRegistry")
+ .body("{ \"RegistryId\": { \"RegistryName\": \"" + REGISTRY_NAME + "\" } }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("Status", equalTo("DELETING"));
+ }
+
+ @Test
+ @Order(10)
+ void getDeletedRegistryReturnsNotFound() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetRegistry")
+ .body("{ \"RegistryId\": { \"RegistryName\": \"" + REGISTRY_NAME + "\" } }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryMetadataAndTagsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryMetadataAndTagsIntegrationTest.java
new file mode 100644
index 0000000000..664f64faad
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryMetadataAndTagsIntegrationTest.java
@@ -0,0 +1,180 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.matchesPattern;
+import static org.hamcrest.Matchers.nullValue;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class GlueSchemaRegistryMetadataAndTagsIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String REGISTRY = "tags-it-registry";
+ private static final String SCHEMA = "tags-schema";
+ private static final String AVRO_V1 =
+ "{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"User\\\",\\\"namespace\\\":\\\"x\\\","
+ + "\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\"}]}";
+
+ private static String registryArn;
+ private static String schemaVersionId;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void seed() {
+ registryArn = given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateRegistry")
+ .body("{ \"RegistryName\": \"" + REGISTRY + "\" }")
+ .when().post("/").then().statusCode(200)
+ .extract().path("RegistryArn");
+
+ schemaVersionId = given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateSchema")
+ .body("{"
+ + " \"RegistryId\": { \"RegistryName\": \"" + REGISTRY + "\" },"
+ + " \"SchemaName\": \"" + SCHEMA + "\","
+ + " \"DataFormat\": \"AVRO\","
+ + " \"Compatibility\": \"BACKWARD\","
+ + " \"SchemaDefinition\": \"" + AVRO_V1 + "\""
+ + " }")
+ .when().post("/").then().statusCode(200)
+ .extract().path("SchemaVersionId");
+ }
+
+ @Test
+ @Order(2)
+ void tagRegistry() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.TagResource")
+ .body("{ \"ResourceArn\": \"" + registryArn + "\", \"TagsToAdd\": { \"env\": \"prod\", \"team\": \"platform\" } }")
+ .when().post("/").then().statusCode(200);
+ }
+
+ @Test
+ @Order(3)
+ void getTagsReturnsAddedTags() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetTags")
+ .body("{ \"ResourceArn\": \"" + registryArn + "\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Tags.env", equalTo("prod"))
+ .body("Tags.team", equalTo("platform"));
+ }
+
+ @Test
+ @Order(4)
+ void untagResourceRemovesKey() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.UntagResource")
+ .body("{ \"ResourceArn\": \"" + registryArn + "\", \"TagsToRemove\": [\"env\"] }")
+ .when().post("/").then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetTags")
+ .body("{ \"ResourceArn\": \"" + registryArn + "\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("Tags.env", nullValue())
+ .body("Tags.team", equalTo("platform"));
+ }
+
+ @Test
+ @Order(5)
+ void putSchemaVersionMetadata() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.PutSchemaVersionMetadata")
+ .body("{ \"SchemaVersionId\": \"" + schemaVersionId + "\","
+ + " \"MetadataKeyValue\": { \"MetadataKey\": \"owner\", \"MetadataValue\": \"alice\" } }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("MetadataKey", equalTo("owner"))
+ .body("MetadataValue", equalTo("alice"))
+ .body("RegistryName", equalTo(REGISTRY))
+ .body("SchemaName", equalTo(SCHEMA))
+ .body("LatestVersion", equalTo(true))
+ .body("SchemaVersionId", equalTo(schemaVersionId));
+ }
+
+ @Test
+ @Order(6)
+ void putDuplicateMetadataReturnsAlreadyExists() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.PutSchemaVersionMetadata")
+ .body("{ \"SchemaVersionId\": \"" + schemaVersionId + "\","
+ + " \"MetadataKeyValue\": { \"MetadataKey\": \"owner\", \"MetadataValue\": \"alice\" } }")
+ .when().post("/").then()
+ .statusCode(400)
+ .body("__type", equalTo("AlreadyExistsException"));
+ }
+
+ @Test
+ @Order(7)
+ void querySchemaVersionMetadataReturnsKey() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.QuerySchemaVersionMetadata")
+ .body("{ \"SchemaVersionId\": \"" + schemaVersionId + "\" }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("SchemaVersionId", equalTo(schemaVersionId))
+ .body("MetadataInfoMap.owner.MetadataValue", equalTo("alice"))
+ .body("MetadataInfoMap.owner.CreatedTime", matchesPattern("^\\d{4}-\\d{2}-\\d{2}T.*Z$"));
+ }
+
+ @Test
+ @Order(8)
+ void removeSchemaVersionMetadata() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.RemoveSchemaVersionMetadata")
+ .body("{ \"SchemaVersionId\": \"" + schemaVersionId + "\","
+ + " \"MetadataKeyValue\": { \"MetadataKey\": \"owner\", \"MetadataValue\": \"alice\" } }")
+ .when().post("/").then()
+ .statusCode(200)
+ .body("MetadataKey", equalTo("owner"))
+ .body("RegistryName", equalTo(REGISTRY))
+ .body("SchemaName", equalTo(SCHEMA))
+ .body("LatestVersion", equalTo(true));
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.QuerySchemaVersionMetadata")
+ .body("{ \"SchemaVersionId\": \"" + schemaVersionId + "\" }")
+ .when().post("/").then()
+ .body("MetadataInfoMap.owner", nullValue());
+ }
+
+ @Test
+ @Order(9)
+ void removeUnknownMetadataReturnsNotFound() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.RemoveSchemaVersionMetadata")
+ .body("{ \"SchemaVersionId\": \"" + schemaVersionId + "\","
+ + " \"MetadataKeyValue\": { \"MetadataKey\": \"missing\", \"MetadataValue\": \"x\" } }")
+ .when().post("/").then()
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+
+ @Test
+ @Order(10)
+ void getTagsOnUnknownArnReturnsInvalidInput() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetTags")
+ .body("{ \"ResourceArn\": \"not-an-arn\" }")
+ .when().post("/").then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidInputException"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistrySchemaIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistrySchemaIntegrationTest.java
new file mode 100644
index 0000000000..02b645dac9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistrySchemaIntegrationTest.java
@@ -0,0 +1,236 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class GlueSchemaRegistrySchemaIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String REGISTRY = "schema-it-registry";
+ private static final String SCHEMA = "users";
+
+ private static final String AVRO_V1 =
+ "{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"User\\\",\\\"namespace\\\":\\\"x\\\","
+ + "\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\"}]}";
+
+ private static final String AVRO_V2_OK =
+ "{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"User\\\",\\\"namespace\\\":\\\"x\\\","
+ + "\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\"},"
+ + "{\\\"name\\\":\\\"email\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null}]}";
+
+ private static final String AVRO_V2_BAD =
+ "{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"User\\\",\\\"namespace\\\":\\\"x\\\","
+ + "\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\"},"
+ + "{\\\"name\\\":\\\"email\\\",\\\"type\\\":\\\"string\\\"}]}";
+
+ private static String createdSchemaVersionId;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void createRegistryForSchemaTests() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateRegistry")
+ .body("{ \"RegistryName\": \"" + REGISTRY + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void createSchemaCreatesV1() {
+ createdSchemaVersionId = given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateSchema")
+ .body("{"
+ + " \"RegistryId\": { \"RegistryName\": \"" + REGISTRY + "\" },"
+ + " \"SchemaName\": \"" + SCHEMA + "\","
+ + " \"DataFormat\": \"AVRO\","
+ + " \"Compatibility\": \"BACKWARD\","
+ + " \"SchemaDefinition\": \"" + AVRO_V1 + "\""
+ + " }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("SchemaName", equalTo(SCHEMA))
+ .body("RegistryName", equalTo(REGISTRY))
+ .body("SchemaArn", containsString(":schema/" + REGISTRY + "/" + SCHEMA))
+ .body("DataFormat", equalTo("AVRO"))
+ .body("Compatibility", equalTo("BACKWARD"))
+ .body("SchemaStatus", equalTo("AVAILABLE"))
+ .body("LatestSchemaVersion", equalTo(1))
+ .body("NextSchemaVersion", equalTo(2))
+ .body("SchemaVersionId", notNullValue())
+ .body("SchemaVersionStatus", equalTo("AVAILABLE"))
+ .extract().path("SchemaVersionId");
+ }
+
+ @Test
+ @Order(3)
+ void getSchemaByDefinitionFindsExistingVersion() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchemaByDefinition")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaDefinition\": \"" + AVRO_V1 + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("SchemaVersionId", equalTo(createdSchemaVersionId))
+ .body("DataFormat", equalTo("AVRO"))
+ .body("Status", equalTo("AVAILABLE"));
+ }
+
+ @Test
+ @Order(4)
+ void getSchemaByDefinitionMissingReturnsNotFound() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchemaByDefinition")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaDefinition\": \"" + AVRO_V2_OK + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+
+ @Test
+ @Order(5)
+ void registerSchemaVersionAcceptsBackwardCompatible() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.RegisterSchemaVersion")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaDefinition\": \"" + AVRO_V2_OK + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("VersionNumber", equalTo(2))
+ .body("Status", equalTo("AVAILABLE"))
+ .body("SchemaVersionId", notNullValue());
+ }
+
+ @Test
+ @Order(6)
+ void registerSchemaVersionRejectsIncompatibleEvolution() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.RegisterSchemaVersion")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaDefinition\": \"" + AVRO_V2_BAD + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidInputException"));
+ }
+
+ @Test
+ @Order(7)
+ void registerSchemaVersionDuplicateReturnsExistingId() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.RegisterSchemaVersion")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaDefinition\": \"" + AVRO_V1 + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("SchemaVersionId", equalTo(createdSchemaVersionId))
+ .body("VersionNumber", equalTo(1));
+ }
+
+ @Test
+ @Order(8)
+ void getSchemaVersionByLatestReturnsV2() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchemaVersion")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaVersionNumber\": { \"LatestVersion\": true } }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("VersionNumber", equalTo(2))
+ .body("DataFormat", equalTo("AVRO"))
+ .body("Status", equalTo("AVAILABLE"))
+ .body("SchemaDefinition", notNullValue());
+ }
+
+ @Test
+ @Order(9)
+ void getSchemaVersionByNumberReturnsV1() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchemaVersion")
+ .body("{ \"SchemaId\": { \"RegistryName\": \"" + REGISTRY + "\", \"SchemaName\": \"" + SCHEMA + "\" },"
+ + " \"SchemaVersionNumber\": { \"VersionNumber\": 1 } }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("VersionNumber", equalTo(1))
+ .body("SchemaVersionId", equalTo(createdSchemaVersionId));
+ }
+
+ @Test
+ @Order(10)
+ void getSchemaVersionByIdReturnsThatVersion() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetSchemaVersion")
+ .body("{ \"SchemaVersionId\": \"" + createdSchemaVersionId + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("SchemaVersionId", equalTo(createdSchemaVersionId))
+ .body("VersionNumber", equalTo(1));
+ }
+
+ @Test
+ @Order(11)
+ void createSchemaWithInvalidAvroDefinitionReturns400() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateSchema")
+ .body("{"
+ + " \"RegistryId\": { \"RegistryName\": \"" + REGISTRY + "\" },"
+ + " \"SchemaName\": \"bad\","
+ + " \"DataFormat\": \"AVRO\","
+ + " \"Compatibility\": \"BACKWARD\","
+ + " \"SchemaDefinition\": \"{not-valid-avro\""
+ + " }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidInputException"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryServiceTest.java
new file mode 100644
index 0000000000..e644d02a05
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/GlueSchemaRegistryServiceTest.java
@@ -0,0 +1,909 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.Registry;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.RegistryId;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.Schema;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
+import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaVersion;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class GlueSchemaRegistryServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT_ID = "000000000000";
+
+ private GlueSchemaRegistryService service;
+
+ @BeforeEach
+ void setUp() {
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);
+ service = new GlueSchemaRegistryService(new InMemoryStorage<>(), regionResolver);
+ }
+
+ @Test
+ void createRegistryReturnsRegistryWithArnAndStatusAvailable() {
+ Registry registry = service.createRegistry("my-registry", "desc", Map.of("env", "test"), REGION);
+
+ assertEquals("my-registry", registry.getRegistryName());
+ assertEquals("desc", registry.getDescription());
+ assertEquals("AVAILABLE", registry.getStatus());
+ assertEquals(Map.of("env", "test"), registry.getTags());
+ assertEquals("arn:aws:glue:us-east-1:" + ACCOUNT_ID + ":registry/my-registry", registry.getRegistryArn());
+ assertNotNull(registry.getCreatedTime());
+ assertNotNull(registry.getUpdatedTime());
+ }
+
+ @Test
+ void createRegistryRejectsDuplicate() {
+ service.createRegistry("dup", null, null, REGION);
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.createRegistry("dup", null, null, REGION));
+ assertEquals("AlreadyExistsException", ex.getErrorCode());
+ assertEquals(400, ex.getHttpStatus());
+ }
+
+ @Test
+ void createRegistryRejectsBlankName() {
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.createRegistry("", null, null, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void createRegistryRejectsNullName() {
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.createRegistry(null, null, null, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void createRegistryRejectsInvalidCharacters() {
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.createRegistry("bad name with spaces", null, null, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void createRegistryAllowsDotAndHashCharacters() {
+ for (String name : List.of("valid.name", "valid#name")) {
+ Registry registry = service.createRegistry(name, null, null, REGION);
+ assertEquals(name, registry.getRegistryName());
+ }
+ }
+
+ @Test
+ void createRegistryRejectsExcessiveLength() {
+ String tooLong = "a".repeat(256);
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.createRegistry(tooLong, null, null, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void getRegistryByNameReturnsExisting() {
+ service.createRegistry("r1", "d1", null, REGION);
+
+ Registry registry = service.getRegistry(new RegistryId("r1", null), REGION);
+
+ assertEquals("r1", registry.getRegistryName());
+ assertEquals("d1", registry.getDescription());
+ }
+
+ @Test
+ void getRegistryByArnReturnsExisting() {
+ service.createRegistry("r1", null, null, REGION);
+ String arn = "arn:aws:glue:us-east-1:" + ACCOUNT_ID + ":registry/r1";
+
+ Registry registry = service.getRegistry(new RegistryId(null, arn), REGION);
+
+ assertEquals("r1", registry.getRegistryName());
+ }
+
+ @Test
+ void getRegistryNotFoundThrows() {
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.getRegistry(new RegistryId("missing", null), REGION));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void getRegistryWithNullIdAutoCreatesDefaultRegistry() {
+ Registry registry = service.getRegistry(null, REGION);
+
+ assertEquals("default-registry", registry.getRegistryName());
+ assertEquals("AVAILABLE", registry.getStatus());
+ assertEquals(1, service.listRegistries().size());
+ }
+
+ @Test
+ void getRegistryWithEmptyIdAutoCreatesDefaultRegistry() {
+ Registry registry = service.getRegistry(new RegistryId(null, null), REGION);
+
+ assertEquals("default-registry", registry.getRegistryName());
+ }
+
+ @Test
+ void getRegistryRejectsMalformedArn() {
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.getRegistry(new RegistryId(null, "not-an-arn"), REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void listRegistriesReturnsAll() {
+ service.createRegistry("a", null, null, REGION);
+ service.createRegistry("b", null, null, REGION);
+
+ List<Registry> registries = service.listRegistries();
+
+ assertEquals(2, registries.size());
+ }
+
+ @Test
+ void listRegistriesPaginatesWithNextToken() {
+ service.createRegistry("a", null, null, REGION);
+ service.createRegistry("b", null, null, REGION);
+ service.createRegistry("c", null, null, REGION);
+
+ var first = service.listRegistries(2, null);
+ var second = service.listRegistries(2, first.nextToken());
+
+ assertEquals(2, first.items().size());
+ assertEquals("2", first.nextToken());
+ assertEquals(1, second.items().size());
+ assertNull(second.nextToken());
+ assertEquals("c", second.items().get(0).getRegistryName());
+ }
+
+ @Test
+ void listRegistriesEmptyByDefault() {
+ assertTrue(service.listRegistries().isEmpty());
+ }
+
+ @Test
+ void updateRegistryChangesDescriptionAndUpdatedTime() throws InterruptedException {
+ service.createRegistry("r1", "old", null, REGION);
+ java.time.Instant beforeTime = service.getRegistry(new RegistryId("r1", null), REGION).getUpdatedTime();
+ Thread.sleep(10);
+
+ Registry updated = service.updateRegistry(new RegistryId("r1", null), "new", REGION);
+
+ assertEquals("new", updated.getDescription());
+ assertTrue(updated.getUpdatedTime().isAfter(beforeTime),
+ "updatedTime should advance: before=" + beforeTime + " after=" + updated.getUpdatedTime());
+ }
+
+ @Test
+ void deleteRegistryRemovesFromStore() {
+ service.createRegistry("r1", null, null, REGION);
+ Registry deleted = service.deleteRegistry(new RegistryId("r1", null), REGION);
+
+ assertEquals("DELETING", deleted.getStatus());
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.getRegistry(new RegistryId("r1", null), REGION));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void deleteRegistryCascadesToSchemasVersionsAndMetadata() {
+ service.createRegistry("r1", null, null, REGION);
+ var first = service.createSchema(new RegistryId("r1", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion();
+ service.putSchemaVersionMetadata(first.getSchemaVersionId(), "team", "platform");
+
+ service.deleteRegistry(new RegistryId("r1", null), REGION);
+ service.createRegistry("r1", null, null, REGION);
+
+ assertTrue(service.listSchemas(new RegistryId("r1", null), REGION).isEmpty());
+ AwsException versionEx = assertThrows(AwsException.class, () ->
+ service.getSchemaVersion(null, first.getSchemaVersionId(), null, false, REGION));
+ assertEquals("EntityNotFoundException", versionEx.getErrorCode());
+ AwsException metadataEx = assertThrows(AwsException.class, () ->
+ service.querySchemaVersionMetadata(first.getSchemaVersionId(), null));
+ assertEquals("EntityNotFoundException", metadataEx.getErrorCode());
+ }
+
+ @Test
+ void deleteRegistryNotFoundThrows() {
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.deleteRegistry(new RegistryId("nope", null), REGION));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void createRegistryWithNullDescriptionAndTagsSucceeds() {
+ Registry registry = service.createRegistry("r1", null, null, REGION);
+
+ assertEquals("r1", registry.getRegistryName());
+ assertNull(registry.getDescription());
+ assertNull(registry.getTags());
+ }
+
+ // ---- Schema / SchemaVersion ----
+
+ private static final String AVRO_V1 =
+ "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"com.example\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"}]}";
+
+ private static final String AVRO_V2_BACKWARD_OK =
+ "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"com.example\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":[\"null\",\"string\"],\"default\":null}]}";
+
+ private static final String AVRO_V2_BACKWARD_BAD =
+ "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"com.example\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":\"string\"}]}";
+
+ private Registry preCreateRegistry() {
+ return service.createRegistry("reg", null, null, REGION);
+ }
+
+ @Test
+ void createSchemaCreatesV1AndReturnsAvailable() {
+ preCreateRegistry();
+
+ var result = service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", "desc", AVRO_V1, null, REGION);
+
+ Schema schema = result.schema();
+ SchemaVersion v1 = result.firstVersion();
+ assertEquals("users", schema.getSchemaName());
+ assertEquals("reg", schema.getRegistryName());
+ assertEquals("AVRO", schema.getDataFormat());
+ assertEquals("BACKWARD", schema.getCompatibility());
+ assertEquals("AVAILABLE", schema.getSchemaStatus());
+ assertEquals(1L, schema.getLatestSchemaVersion());
+ assertEquals(2L, schema.getNextSchemaVersion());
+ assertEquals("arn:aws:glue:us-east-1:" + ACCOUNT_ID + ":schema/reg/users", schema.getSchemaArn());
+ assertEquals(1L, v1.getVersionNumber());
+ assertEquals("AVAILABLE", v1.getStatus());
+ assertNotNull(v1.getSchemaVersionId());
+ }
+
+ @Test
+ void createSchemaWithoutRegistryAutoCreatesDefaultRegistry() {
+ var result = service.createSchema(null, "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ assertEquals("default-registry", result.schema().getRegistryName());
+ }
+
+ @Test
+ void createSchemaDefaultsToBackwardWhenCompatibilityOmitted() {
+ preCreateRegistry();
+ var result = service.createSchema(new RegistryId("reg", null),
+ "s1", "AVRO", null, null, AVRO_V1, null, REGION);
+
+ assertEquals("BACKWARD", result.schema().getCompatibility());
+ }
+
+ @Test
+ void createSchemaRejectsDuplicate() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION));
+ assertEquals("AlreadyExistsException", ex.getErrorCode());
+ }
+
+ @Test
+ void createSchemaRejectsInvalidAvroDefinition() {
+ preCreateRegistry();
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, "{not-valid-avro", null, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void createSchemaRejectsUnknownDataFormat() {
+ preCreateRegistry();
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.createSchema(new RegistryId("reg", null),
+ "users", "BOGUS", "BACKWARD", null, AVRO_V1, null, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void createSchemaRejectsUnknownCompatibility() {
+ preCreateRegistry();
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "WAT", null, AVRO_V1, null, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void createSchemaAllowsDotAndHashCharacters() {
+ preCreateRegistry();
+ for (String name : List.of("valid.name", "valid#name")) {
+ Schema schema = service.createSchema(new RegistryId("reg", null),
+ name, "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).schema();
+ assertEquals(name, schema.getSchemaName());
+ }
+ }
+
+ @Test
+ void registerSchemaVersionAppendsBackwardCompatibleVersion() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ SchemaVersion v2 = service.registerSchemaVersion(
+ new SchemaId("reg", "users", null), AVRO_V2_BACKWARD_OK, REGION);
+
+ assertEquals(2L, v2.getVersionNumber());
+ assertEquals("AVAILABLE", v2.getStatus());
+ }
+
+ @Test
+ void registerSchemaVersionRejectsBackwardIncompatibleEvolution() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.registerSchemaVersion(
+ new SchemaId("reg", "users", null), AVRO_V2_BACKWARD_BAD, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void registerSchemaVersionDuplicateDefinitionReturnsExistingId() {
+ preCreateRegistry();
+ var v1 = service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion();
+
+ SchemaVersion same = service.registerSchemaVersion(
+ new SchemaId("reg", "users", null), AVRO_V1, REGION);
+
+ assertEquals(v1.getSchemaVersionId(), same.getSchemaVersionId());
+ assertEquals(1L, same.getVersionNumber());
+ }
+
+ @Test
+ void registerSchemaVersionWithDisabledCompatRejectsNewVersions() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "DISABLED", null, AVRO_V1, null, REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.registerSchemaVersion(
+ new SchemaId("reg", "users", null), AVRO_V2_BACKWARD_OK, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void registerSchemaVersionWithNoneCompatAcceptsAnyEvolution() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "NONE", null, AVRO_V1, null, REGION);
+
+ SchemaVersion v2 = service.registerSchemaVersion(
+ new SchemaId("reg", "users", null), AVRO_V2_BACKWARD_BAD, REGION);
+
+ assertEquals(2L, v2.getVersionNumber());
+ }
+
+ @Test
+ void getSchemaVersionByLatest() {
+ preCreateRegistry();
+ var first = service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion();
+ SchemaVersion v2 = service.registerSchemaVersion(
+ new SchemaId("reg", "users", null), AVRO_V2_BACKWARD_OK, REGION);
+
+ SchemaVersion latest = service.getSchemaVersion(
+ new SchemaId("reg", "users", null), null, null, true, REGION);
+
+ assertEquals(v2.getSchemaVersionId(), latest.getSchemaVersionId());
+ assertEquals(2L, latest.getVersionNumber());
+ assertNotNull(first.getSchemaVersionId());
+ }
+
+ @Test
+ void getSchemaVersionByNumber() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "users", null), AVRO_V2_BACKWARD_OK, REGION);
+
+ SchemaVersion v1 = service.getSchemaVersion(
+ new SchemaId("reg", "users", null), null, 1L, false, REGION);
+
+ assertEquals(1L, v1.getVersionNumber());
+ assertEquals(AVRO_V1, v1.getSchemaDefinition());
+ }
+
+ @Test
+ void getSchemaVersionByVersionId() {
+ preCreateRegistry();
+ var first = service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion();
+
+ SchemaVersion fetched = service.getSchemaVersion(
+ null, first.getSchemaVersionId(), null, false, REGION);
+
+ assertEquals(first.getSchemaVersionId(), fetched.getSchemaVersionId());
+ }
+
+ @Test
+ void getSchemaVersionWithoutSelectorThrows() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.getSchemaVersion(new SchemaId("reg", "users", null), null, null, false, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void getSchemaByDefinitionFindsExisting() {
+ preCreateRegistry();
+ var first = service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion();
+
+ SchemaVersion found = service.getSchemaByDefinition(
+ new SchemaId("reg", "users", null), AVRO_V1, REGION);
+
+ assertEquals(first.getSchemaVersionId(), found.getSchemaVersionId());
+ }
+
+ @Test
+ void getSchemaByDefinitionMatchesDespiteWhitespace() {
+ preCreateRegistry();
+ var first = service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion();
+
+ // Same Avro schema but with extra whitespace
+ String formatted = AVRO_V1.replace(",", ", ").replace(":", " : ");
+ SchemaVersion found = service.getSchemaByDefinition(
+ new SchemaId("reg", "users", null), formatted, REGION);
+
+ assertEquals(first.getSchemaVersionId(), found.getSchemaVersionId());
+ }
+
+ @Test
+ void getSchemaByDefinitionNotFoundThrows() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.getSchemaByDefinition(
+ new SchemaId("reg", "users", null), AVRO_V2_BACKWARD_OK, REGION));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void getSchemaByArnRoundTrips() {
+ preCreateRegistry();
+ var result = service.createSchema(new RegistryId("reg", null),
+ "users", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ Schema fetched = service.getSchema(
+ new SchemaId(null, null, result.schema().getSchemaArn()), REGION);
+
+ assertEquals("users", fetched.getSchemaName());
+ }
+
+ @Test
+ void registerSchemaVersionInUnknownRegistryThrows() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.registerSchemaVersion(
+ new SchemaId("missing", "users", null), AVRO_V1, REGION));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ // ---- PR 3: admin actions ----
+
+ @Test
+ void listSchemasReturnsSchemasInRegistry() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.createSchema(new RegistryId("reg", null), "b", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ List<Schema> schemas = service.listSchemas(new RegistryId("reg", null), REGION);
+ assertEquals(2, schemas.size());
+ }
+
+ @Test
+ void listSchemasOnlyReturnsTargetRegistry() {
+ preCreateRegistry();
+ service.createRegistry("other", null, null, REGION);
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.createSchema(new RegistryId("other", null), "x", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ List<Schema> regSchemas = service.listSchemas(new RegistryId("reg", null), REGION);
+ assertEquals(1, regSchemas.size());
+ assertEquals("a", regSchemas.get(0).getSchemaName());
+ }
+
+ @Test
+ void listSchemasPaginatesWithinRegistry() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.createSchema(new RegistryId("reg", null), "b", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.createSchema(new RegistryId("reg", null), "c", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ var first = service.listSchemas(new RegistryId("reg", null), REGION, 2, null);
+ var second = service.listSchemas(new RegistryId("reg", null), REGION, 2, first.nextToken());
+
+ assertEquals(2, first.items().size());
+ assertEquals("2", first.nextToken());
+ assertEquals(1, second.items().size());
+ assertEquals("c", second.items().get(0).getSchemaName());
+ }
+
+ @Test
+ void updateSchemaChangesCompatibility() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ Schema updated = service.updateSchema(new SchemaId("reg", "a", null), "FORWARD", null, REGION);
+ assertEquals("FORWARD", updated.getCompatibility());
+ }
+
+ @Test
+ void updateSchemaChangesDescription() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", "old", AVRO_V1, null, REGION);
+
+ Schema updated = service.updateSchema(new SchemaId("reg", "a", null), null, "new", REGION);
+ assertEquals("new", updated.getDescription());
+ assertEquals("BACKWARD", updated.getCompatibility());
+ }
+
+ @Test
+ void updateSchemaChangesCheckpointVersion() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null), AVRO_V2_BACKWARD_OK, REGION);
+
+ Schema updated = service.updateSchema(new SchemaId("reg", "a", null), null, null, 2L, REGION);
+
+ assertEquals(2L, updated.getSchemaCheckpoint());
+ }
+
+ @Test
+ void updateSchemaRejectsUnknownCompatibility() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.updateSchema(new SchemaId("reg", "a", null), "BOGUS", null, REGION));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void listSchemaVersionsReturnsInVersionOrder() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null), AVRO_V2_BACKWARD_OK, REGION);
+
+ List<SchemaVersion> versions = service.listSchemaVersions(new SchemaId("reg", "a", null), REGION);
+ assertEquals(2, versions.size());
+ assertEquals(1L, versions.get(0).getVersionNumber());
+ assertEquals(2L, versions.get(1).getVersionNumber());
+ }
+
+ @Test
+ void listSchemaVersionsPaginatesInVersionOrder() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "NONE", null, AVRO_V1, null, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null), AVRO_V2_BACKWARD_OK, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null),
+ AVRO_V2_BACKWARD_OK.replace("email", "phone"), REGION);
+
+ var first = service.listSchemaVersions(new SchemaId("reg", "a", null), REGION, 2, null);
+ var second = service.listSchemaVersions(new SchemaId("reg", "a", null), REGION, 2, first.nextToken());
+
+ assertEquals(2, first.items().size());
+ assertEquals("2", first.nextToken());
+ assertEquals(1, second.items().size());
+ assertEquals(3L, second.items().get(0).getVersionNumber());
+ }
+
+ @Test
+ void deleteSchemaCascadesToVersions() {
+ preCreateRegistry();
+ var first = service.createSchema(new RegistryId("reg", null),
+ "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion();
+
+ service.deleteSchema(new SchemaId("reg", "a", null), REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.getSchema(new SchemaId("reg", "a", null), REGION));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ AwsException ex2 = assertThrows(AwsException.class, () ->
+ service.getSchemaVersion(null, first.getSchemaVersionId(), null, false, REGION));
+ assertEquals("EntityNotFoundException", ex2.getErrorCode());
+ }
+
+ @Test
+ void deleteSchemaVersionsRemovesGivenVersions() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null), AVRO_V2_BACKWARD_OK, REGION);
+ service.updateSchema(new SchemaId("reg", "a", null), null, null, 2L, REGION);
+
+ var results = service.deleteSchemaVersions(new SchemaId("reg", "a", null), "1", REGION);
+ assertEquals(1, results.size());
+ assertNull(results.get(0).errorCode());
+
+ List<SchemaVersion> remaining = service.listSchemaVersions(new SchemaId("reg", "a", null), REGION);
+ assertEquals(1, remaining.size());
+ assertEquals(2L, remaining.get(0).getVersionNumber());
+ }
+
+ @Test
+ void deleteSchemaVersionsParsesRanges() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "NONE", null, AVRO_V1, null, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null), AVRO_V2_BACKWARD_OK, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null),
+ AVRO_V2_BACKWARD_OK.replace("email", "phone"), REGION);
+ service.updateSchema(new SchemaId("reg", "a", null), null, null, 3L, REGION);
+
+ var results = service.deleteSchemaVersions(new SchemaId("reg", "a", null), "1-2", REGION);
+ assertEquals(2, results.size());
+ assertEquals(1, service.listSchemaVersions(new SchemaId("reg", "a", null), REGION).size());
+ }
+
+ @Test
+ void deleteSchemaVersionsRejectsCheckpointVersion() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null), AVRO_V2_BACKWARD_OK, REGION);
+
+ var results = service.deleteSchemaVersions(new SchemaId("reg", "a", null), "1", REGION);
+
+ assertEquals(1, results.size());
+ assertEquals(1L, results.get(0).versionNumber());
+ assertEquals("InvalidInputException", results.get(0).errorCode());
+ assertEquals(2, service.listSchemaVersions(new SchemaId("reg", "a", null), REGION).size());
+ }
+
+ @Test
+ void deleteSchemaVersionsRejectsExpandedRangesOverTwentyFiveVersions() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.deleteSchemaVersions(new SchemaId("reg", "a", null), "1-26", REGION));
+
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void deleteSchemaVersionsReportsErrorsForMissingVersions() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ var results = service.deleteSchemaVersions(new SchemaId("reg", "a", null), "1,99", REGION);
+ assertEquals(2, results.size());
+ // Version 1 is the latest and only — cannot delete (latest constraint).
+ assertEquals(99L, results.get(1).versionNumber());
+ assertNotNull(results.get(1).errorCode());
+ }
+
+ @Test
+ void getSchemaVersionsDiffReturnsTextDiff() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+ service.registerSchemaVersion(new SchemaId("reg", "a", null), AVRO_V2_BACKWARD_OK, REGION);
+
+ String diff = service.getSchemaVersionsDiff(new SchemaId("reg", "a", null), 1L, 2L, REGION);
+ assertTrue(diff.contains("---"));
+ assertTrue(diff.contains("+++"));
+ }
+
+ @Test
+ void getSchemaVersionsDiffIdenticalReturnsEmpty() {
+ preCreateRegistry();
+ service.createSchema(new RegistryId("reg", null), "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION);
+
+ String diff = service.getSchemaVersionsDiff(new SchemaId("reg", "a", null), 1L, 1L, REGION);
+ assertEquals("", diff);
+ }
+
+ @Test
+ void checkSchemaVersionValidityForValidAvro() {
+ var r = service.checkSchemaVersionValidity("AVRO", AVRO_V1);
+ assertTrue(r.valid());
+ assertNull(r.error());
+ }
+
+ @Test
+ void checkSchemaVersionValidityForInvalidAvro() {
+ var r = service.checkSchemaVersionValidity("AVRO", "{not-valid-avro");
+ assertFalse(r.valid());
+ assertNotNull(r.error());
+ }
+
+ @Test
+ void checkSchemaVersionValidityRejectsUnknownDataFormat() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.checkSchemaVersionValidity("BOGUS", AVRO_V1));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ // ---- PR 4: metadata + tags ----
+
+ private String firstVersionId() {
+ preCreateRegistry();
+ return service.createSchema(new RegistryId("reg", null),
+ "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion().getSchemaVersionId();
+ }
+
+ @Test
+ void putSchemaVersionMetadataStoresKeyValue() {
+ String svId = firstVersionId();
+
+ var r = service.putSchemaVersionMetadata(svId, "team", "platform");
+
+ assertEquals("team", r.metadataKey());
+ assertEquals("platform", r.metadataValue());
+ var map = service.querySchemaVersionMetadata(svId, null);
+ assertEquals("platform", map.get("team").getMetadataValue());
+ }
+
+ @Test
+ void putSchemaVersionMetadataDuplicateKeyValueRejected() {
+ String svId = firstVersionId();
+ service.putSchemaVersionMetadata(svId, "team", "platform");
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.putSchemaVersionMetadata(svId, "team", "platform"));
+ assertEquals("AlreadyExistsException", ex.getErrorCode());
+ }
+
+ @Test
+ void putSchemaVersionMetadataSameKeyNewValueDemotesOld() {
+ String svId = firstVersionId();
+ service.putSchemaVersionMetadata(svId, "team", "platform");
+
+ service.putSchemaVersionMetadata(svId, "team", "data");
+
+ var map = service.querySchemaVersionMetadata(svId, null);
+ assertEquals("data", map.get("team").getMetadataValue());
+ assertEquals(1, map.get("team").getOtherMetadataValueList().size());
+ assertEquals("platform", map.get("team").getOtherMetadataValueList().get(0).getMetadataValue());
+ }
+
+ @Test
+ void putSchemaVersionMetadataForUnknownVersionThrows() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.putSchemaVersionMetadata("does-not-exist", "team", "platform"));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void removeSchemaVersionMetadataCurrentPromotesNext() {
+ String svId = firstVersionId();
+ service.putSchemaVersionMetadata(svId, "team", "platform");
+ service.putSchemaVersionMetadata(svId, "team", "data");
+
+ service.removeSchemaVersionMetadata(svId, "team", "data");
+
+ var map = service.querySchemaVersionMetadata(svId, null);
+ assertEquals("platform", map.get("team").getMetadataValue());
+ assertNull(map.get("team").getOtherMetadataValueList());
+ }
+
+ @Test
+ void removeSchemaVersionMetadataLastValueDeletesKey() {
+ String svId = firstVersionId();
+ service.putSchemaVersionMetadata(svId, "team", "platform");
+
+ service.removeSchemaVersionMetadata(svId, "team", "platform");
+
+ assertTrue(service.querySchemaVersionMetadata(svId, null).isEmpty());
+ }
+
+ @Test
+ void removeSchemaVersionMetadataNotFoundThrows() {
+ String svId = firstVersionId();
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.removeSchemaVersionMetadata(svId, "missing", "x"));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void querySchemaVersionMetadataWithFilterReturnsSubset() {
+ String svId = firstVersionId();
+ service.putSchemaVersionMetadata(svId, "team", "platform");
+ service.putSchemaVersionMetadata(svId, "owner", "alice");
+
+ var filtered = service.querySchemaVersionMetadata(svId,
+ List.of(new GlueSchemaRegistryService.MetadataKeyValueFilter("team", null)));
+
+ assertEquals(1, filtered.size());
+ assertTrue(filtered.containsKey("team"));
+ }
+
+ @Test
+ void deletingSchemaVersionRemovesMetadata() {
+ preCreateRegistry();
+ var first = service.createSchema(new RegistryId("reg", null),
+ "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).firstVersion();
+ service.registerSchemaVersion(new SchemaId("reg", "a", null), AVRO_V2_BACKWARD_OK, REGION);
+ service.updateSchema(new SchemaId("reg", "a", null), null, null, 2L, REGION);
+ service.putSchemaVersionMetadata(first.getSchemaVersionId(), "team", "platform");
+
+ service.deleteSchemaVersions(new SchemaId("reg", "a", null), "1", REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.querySchemaVersionMetadata(first.getSchemaVersionId(), null));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void tagAndGetTagsForRegistry() {
+ Registry r = service.createRegistry("reg", null, null, REGION);
+
+ service.tagResource(r.getRegistryArn(), Map.of("env", "prod", "team", "platform"));
+
+ Map<String, String> tags = service.getTags(r.getRegistryArn());
+ assertEquals(2, tags.size());
+ assertEquals("prod", tags.get("env"));
+ }
+
+ @Test
+ void tagAndGetTagsForSchema() {
+ preCreateRegistry();
+ var schema = service.createSchema(new RegistryId("reg", null),
+ "a", "AVRO", "BACKWARD", null, AVRO_V1, null, REGION).schema();
+
+ service.tagResource(schema.getSchemaArn(), Map.of("owner", "alice"));
+
+ Map<String, String> tags = service.getTags(schema.getSchemaArn());
+ assertEquals("alice", tags.get("owner"));
+ }
+
+ @Test
+ void untagResourceRemovesKeys() {
+ Registry r = service.createRegistry("reg", null, null, REGION);
+ service.tagResource(r.getRegistryArn(), Map.of("env", "prod", "team", "platform"));
+
+ service.untagResource(r.getRegistryArn(), List.of("env"));
+
+ Map<String, String> tags = service.getTags(r.getRegistryArn());
+ assertEquals(1, tags.size());
+ assertTrue(tags.containsKey("team"));
+ }
+
+ @Test
+ void tagResourceWithMalformedArnThrows() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.tagResource("not-an-arn", Map.of("a", "b")));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void tagResourceUnknownRegistryThrows() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ service.tagResource(
+ "arn:aws:glue:us-east-1:" + ACCOUNT_ID + ":registry/missing",
+ Map.of("a", "b")));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaCompatibilityCheckerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaCompatibilityCheckerTest.java
new file mode 100644
index 0000000000..c16454c661
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaCompatibilityCheckerTest.java
@@ -0,0 +1,161 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class SchemaCompatibilityCheckerTest {
+
+ private static final String AVRO_V1 =
+ "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"x\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"}]}";
+
+ private static final String AVRO_ADD_OPTIONAL =
+ "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"x\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":[\"null\",\"string\"],\"default\":null}]}";
+
+ private static final String AVRO_ADD_REQUIRED =
+ "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"x\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"email\",\"type\":\"string\"}]}";
+
+ private static final String PROTOBUF_REQUIRED_EMAIL =
+ "syntax = \"proto2\";\n"
+ + "package x;\n"
+ + "message User {\n"
+ + " required string name = 1;\n"
+ + " required string email = 2;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_REMOVE_REQUIRED_EMAIL =
+ "syntax = \"proto2\";\n"
+ + "package x;\n"
+ + "message User {\n"
+ + " required string name = 1;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_OPTIONAL_EMAIL =
+ "syntax = \"proto2\";\n"
+ + "package x;\n"
+ + "message User {\n"
+ + " required string name = 1;\n"
+ + " optional string email = 2;\n"
+ + "}\n";
+
+ private static final String PROTOBUF_ADD_REQUIRED_PHONE =
+ "syntax = \"proto2\";\n"
+ + "package x;\n"
+ + "message User {\n"
+ + " required string name = 1;\n"
+ + " optional string email = 2;\n"
+ + " required string phone = 3;\n"
+ + "}\n";
+
+ @Test
+ void noneAlwaysCompatible() {
+ var r = SchemaCompatibilityChecker.check("NONE", List.of(AVRO_V1), AVRO_ADD_REQUIRED, "AVRO");
+ assertTrue(r.compatible());
+ }
+
+ @Test
+ void disabledShortCircuits() {
+ var r = SchemaCompatibilityChecker.check("DISABLED", List.of(AVRO_V1), AVRO_ADD_REQUIRED, "AVRO");
+ assertTrue(r.compatible());
+ }
+
+ @Test
+ void emptyExistingIsCompatible() {
+ var r = SchemaCompatibilityChecker.check("BACKWARD", List.of(), AVRO_V1, "AVRO");
+ assertTrue(r.compatible());
+ }
+
+ @Test
+ void backwardAcceptsAddOptionalField() {
+ var r = SchemaCompatibilityChecker.check("BACKWARD", List.of(AVRO_V1), AVRO_ADD_OPTIONAL, "AVRO");
+ assertTrue(r.compatible(), () -> "expected compatible, got: " + r.reason());
+ }
+
+ @Test
+ void backwardRejectsAddRequiredField() {
+ var r = SchemaCompatibilityChecker.check("BACKWARD", List.of(AVRO_V1), AVRO_ADD_REQUIRED, "AVRO");
+ assertFalse(r.compatible());
+ assertNotNull(r.reason());
+ }
+
+ @Test
+ void backwardAllRejectsRequiredAddedAcrossAnyPriorVersion() {
+ var r = SchemaCompatibilityChecker.check("BACKWARD_ALL",
+ List.of(AVRO_V1, AVRO_ADD_OPTIONAL), AVRO_ADD_REQUIRED, "AVRO");
+ assertFalse(r.compatible());
+ }
+
+ @Test
+ void forwardAcceptsAddRequired() {
+ // FORWARD: latest reader can read new (writer) data. Adding a required field
+ // means new writers produce extra fields that old readers don't know about,
+ // which old readers ignore — so it is FORWARD-compatible.
+ var r = SchemaCompatibilityChecker.check("FORWARD", List.of(AVRO_V1), AVRO_ADD_REQUIRED, "AVRO");
+ assertTrue(r.compatible(), () -> "expected compatible, got: " + r.reason());
+ }
+
+ @Test
+ void protobufBackwardRejectsRemovingRequiredField() {
+ var r = SchemaCompatibilityChecker.check(
+ "BACKWARD",
+ List.of(PROTOBUF_REQUIRED_EMAIL),
+ PROTOBUF_REMOVE_REQUIRED_EMAIL,
+ "PROTOBUF");
+ assertFalse(r.compatible());
+ assertNotNull(r.reason());
+ }
+
+ @Test
+ void protobufForwardRejectsAddingRequiredField() {
+ var r = SchemaCompatibilityChecker.check(
+ "FORWARD",
+ List.of(PROTOBUF_OPTIONAL_EMAIL),
+ PROTOBUF_ADD_REQUIRED_PHONE,
+ "PROTOBUF");
+ assertFalse(r.compatible());
+ assertNotNull(r.reason());
+ }
+
+ @Test
+ void unknownModeThrows() {
+ assertThrows(IllegalArgumentException.class, () ->
+ SchemaCompatibilityChecker.check("WAT", List.of(AVRO_V1), AVRO_ADD_OPTIONAL, "AVRO"));
+ }
+
+ @Test
+ void unknownDataFormatThrows() {
+ assertThrows(IllegalArgumentException.class, () ->
+ SchemaCompatibilityChecker.check("BACKWARD", List.of(AVRO_V1), AVRO_ADD_OPTIONAL, "BOGUS"));
+ }
+
+ @Test
+ void canonicalizeNormalizesAvroWhitespace() {
+ String spaced = AVRO_V1.replace(",", " , ").replace(":", " : ");
+ String c1 = SchemaCompatibilityChecker.canonicalize(AVRO_V1, "AVRO");
+ String c2 = SchemaCompatibilityChecker.canonicalize(spaced, "AVRO");
+ assertEquals(c1, c2);
+ }
+
+ @Test
+ void validateDefinitionAcceptsValidAvro() {
+ assertNull(SchemaCompatibilityChecker.validateDefinition(AVRO_V1, "AVRO"));
+ }
+
+ @Test
+ void validateDefinitionRejectsInvalidAvro() {
+ String error = SchemaCompatibilityChecker.validateDefinition("{garbage", "AVRO");
+ assertNotNull(error);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaToColumnsConverterTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaToColumnsConverterTest.java
new file mode 100644
index 0000000000..b13ae421b6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/schemaregistry/SchemaToColumnsConverterTest.java
@@ -0,0 +1,277 @@
+package io.github.hectorvent.floci.services.glue.schemaregistry;
+
+import io.github.hectorvent.floci.services.glue.model.Column;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class SchemaToColumnsConverterTest {
+
+ // ---- Avro ----
+
+ @Test
+ void avroPrimitives() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"long\"},"
+ + "{\"name\":\"name\",\"type\":\"string\"},"
+ + "{\"name\":\"flag\",\"type\":\"boolean\"},"
+ + "{\"name\":\"ratio\",\"type\":\"double\"},"
+ + "{\"name\":\"count\",\"type\":\"int\"}"
+ + "]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ assertEquals(5, cols.size());
+ assertEquals("bigint", cols.get(0).getType());
+ assertEquals("string", cols.get(1).getType());
+ assertEquals("boolean", cols.get(2).getType());
+ assertEquals("double", cols.get(3).getType());
+ assertEquals("int", cols.get(4).getType());
+ }
+
+ @Test
+ void avroNullableUnionUnwrapsToInner() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"email\",\"type\":[\"null\",\"string\"],\"default\":null}]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ assertEquals("string", cols.get(0).getType());
+ }
+
+ @Test
+ void avroNestedRecordBecomesStruct() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"addr\",\"type\":{\"type\":\"record\",\"name\":\"Addr\",\"fields\":["
+ + "{\"name\":\"city\",\"type\":\"string\"},"
+ + "{\"name\":\"zip\",\"type\":\"int\"}]}}]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ assertEquals("struct<city:string,zip:int>", cols.get(0).getType());
+ }
+
+ @Test
+ void avroArrayBecomesArray() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"tags\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ assertEquals("array<string>", cols.get(0).getType());
+ }
+
+ @Test
+ void avroMapBecomesMap() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"meta\",\"type\":{\"type\":\"map\",\"values\":\"long\"}}]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ assertEquals("map<string,bigint>", cols.get(0).getType());
+ }
+
+ @Test
+ void avroEnumFallsBackToString() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"status\",\"type\":{\"type\":\"enum\",\"name\":\"S\",\"symbols\":[\"A\",\"B\"]}}]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ assertEquals("string", cols.get(0).getType());
+ }
+
+ @Test
+ void avroBytesBecomesBinary() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"blob\",\"type\":\"bytes\"}]}";
+
+ assertEquals("binary",
+ SchemaToColumnsConverter.toColumns("AVRO", def).get(0).getType());
+ }
+
+ @Test
+ void avroLogicalTypesMapToHiveSemanticTypes() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"created\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}},"
+ + "{\"name\":\"created_micros\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}},"
+ + "{\"name\":\"local_created\",\"type\":{\"type\":\"long\",\"logicalType\":\"local-timestamp-millis\"}},"
+ + "{\"name\":\"birth\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}},"
+ + "{\"name\":\"start\",\"type\":{\"type\":\"int\",\"logicalType\":\"time-millis\"}},"
+ + "{\"name\":\"id\",\"type\":{\"type\":\"string\",\"logicalType\":\"uuid\"}},"
+ + "{\"name\":\"price\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\","
+ + "\"precision\":12,\"scale\":4}}"
+ + "]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ Map<String, String> byName = cols.stream()
+ .collect(java.util.stream.Collectors.toMap(Column::getName, Column::getType));
+ assertEquals("timestamp", byName.get("created"));
+ assertEquals("timestamp", byName.get("created_micros"));
+ assertEquals("timestamp", byName.get("local_created"));
+ assertEquals("date", byName.get("birth"));
+ assertEquals("string", byName.get("start"));
+ assertEquals("string", byName.get("id"));
+ assertEquals("decimal(12,4)", byName.get("price"));
+ }
+
+ @Test
+ void avroNullableLogicalTypeStillUnwrapsToHiveType() {
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"created\",\"type\":[\"null\","
+ + "{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}],\"default\":null}]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ assertEquals("timestamp", cols.get(0).getType());
+ }
+
+ @Test
+ void avroUnknownLogicalTypeFallsBackToBaseType() {
+ // duration is a fixed(12) logical type with no Hive equivalent; falls through to base "binary".
+ String def = "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"d\",\"type\":{\"type\":\"fixed\",\"name\":\"D\",\"size\":12,"
+ + "\"logicalType\":\"duration\"}}]}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("AVRO", def);
+
+ assertEquals("binary", cols.get(0).getType());
+ }
+
+ @Test
+ void avroNonRecordRootReturnsEmpty() {
+ String def = "\"string\"";
+ assertTrue(SchemaToColumnsConverter.toColumns("AVRO", def).isEmpty());
+ }
+
+ @Test
+ void avroMalformedReturnsEmpty() {
+ assertTrue(SchemaToColumnsConverter.toColumns("AVRO", "{garbage").isEmpty());
+ }
+
+ // ---- JSON Schema ----
+
+ @Test
+ void jsonObjectWithMixedProperties() {
+ String def = "{\"type\":\"object\",\"properties\":{"
+ + "\"id\":{\"type\":\"integer\"},"
+ + "\"name\":{\"type\":\"string\"},"
+ + "\"flag\":{\"type\":\"boolean\"},"
+ + "\"ratio\":{\"type\":\"number\"}"
+ + "}}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("JSON", def);
+
+ Map<String, String> byName = cols.stream()
+ .collect(java.util.stream.Collectors.toMap(Column::getName, Column::getType));
+ assertEquals("bigint", byName.get("id"));
+ assertEquals("string", byName.get("name"));
+ assertEquals("boolean", byName.get("flag"));
+ assertEquals("double", byName.get("ratio"));
+ }
+
+ @Test
+ void jsonNestedObjectBecomesStruct() {
+ String def = "{\"type\":\"object\",\"properties\":{"
+ + "\"addr\":{\"type\":\"object\",\"properties\":{"
+ + "\"city\":{\"type\":\"string\"},\"zip\":{\"type\":\"integer\"}}}}}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("JSON", def);
+
+ assertEquals("struct<city:string,zip:bigint>", cols.get(0).getType());
+ }
+
+ @Test
+ void jsonArrayBecomesArray() {
+ String def = "{\"type\":\"object\",\"properties\":{"
+ + "\"tags\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}}}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("JSON", def);
+
+ assertEquals("array<string>", cols.get(0).getType());
+ }
+
+ @Test
+ void jsonNonObjectRootReturnsEmpty() {
+ String def = "{\"type\":\"string\"}";
+ assertTrue(SchemaToColumnsConverter.toColumns("JSON", def).isEmpty());
+ }
+
+ @Test
+ void jsonMalformedReturnsEmpty() {
+ assertTrue(SchemaToColumnsConverter.toColumns("JSON", "{not-json").isEmpty());
+ }
+
+ // ---- Protobuf ----
+
+ @Test
+ void protobufScalarFields() {
+ String def = "syntax = \"proto3\";\n"
+ + "message User {\n"
+ + " int64 id = 1;\n"
+ + " string name = 2;\n"
+ + " bool active = 3;\n"
+ + "}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("PROTOBUF", def);
+
+ java.util.Map<String, String> byName = cols.stream()
+ .collect(java.util.stream.Collectors.toMap(Column::getName, Column::getType));
+ assertEquals(3, cols.size(), () -> "got cols: " + cols.stream().map(c -> c.getName() + ":" + c.getType()).toList());
+ assertEquals("bigint", byName.get("id"));
+ assertEquals("string", byName.get("name"));
+ assertEquals("boolean", byName.get("active"));
+ }
+
+ @Test
+ void protobufRepeatedBecomesArray() {
+ String def = "syntax = \"proto3\";\n"
+ + "message User {\n"
+ + " repeated string tags = 1;\n"
+ + "}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("PROTOBUF", def);
+
+ assertEquals("array<string>", cols.get(0).getType());
+ }
+
+ @Test
+ void protobufNestedMessageBecomesStruct() {
+ String def = "syntax = \"proto3\";\n"
+ + "message User {\n"
+ + " message Addr { string city = 1; int32 zip = 2; }\n"
+ + " Addr addr = 1;\n"
+ + "}";
+
+ List<Column> cols = SchemaToColumnsConverter.toColumns("PROTOBUF", def);
+
+ assertEquals("struct<city:string,zip:int>", cols.get(0).getType());
+ }
+
+ @Test
+ void protobufMalformedReturnsEmpty() {
+ assertTrue(SchemaToColumnsConverter.toColumns("PROTOBUF", "not a proto file").isEmpty());
+ }
+
+ // ---- Generic ----
+
+ @Test
+ void unknownDataFormatReturnsEmpty() {
+ assertTrue(SchemaToColumnsConverter.toColumns("BOGUS", "anything").isEmpty());
+ }
+
+ @Test
+ void nullDefinitionReturnsEmpty() {
+ assertTrue(SchemaToColumnsConverter.toColumns("AVRO", null).isEmpty());
+ }
+
+ @Test
+ void blankDefinitionReturnsEmpty() {
+ assertTrue(SchemaToColumnsConverter.toColumns("AVRO", " ").isEmpty());
+ }
+}