Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(glue): jobs and crawlers (#2591)

NAMAN JAINde6bf8f599d6
diff --git a/docs/services/glue.md b/docs/services/glue.md
index 3fa94aef03..2f5743d119 100644
--- a/docs/services/glue.md
+++ b/docs/services/glue.md
@@ -44,6 +44,26 @@ Floci emulates the AWS Glue Data Catalog and Glue Schema Registry, allowing you
| UpdateUserDefinedFunction | Updates a stored user-defined function. |
| DeleteUserDefinedFunction | Deletes a user-defined function from a database. |
+#### Jobs
+
+| Action | Description |
+|--------|-------------|
+| CreateJob | Creates a new job definition. |
+| GetJob | Retrieves an existing job definition. |
+| GetJobs | Retrieves all current job definitions. |
+| UpdateJob | Updates an existing job definition. |
+| DeleteJob | Deletes a specified job definition. |
+
+#### Crawlers
+
+| Action | Description |
+|--------|-------------|
+| CreateCrawler | Creates a new crawler with specified targets, role, configuration, and optional schedule. |
+| GetCrawler | Retrieves metadata for a specified crawler. |
+| GetCrawlers | Retrieves metadata for all crawlers defined in the customer account. |
+| UpdateCrawler | Updates a crawler. |
+| DeleteCrawler | Removes a specified crawler from the AWS Glue Data Catalog. |
+
### Schema Registry
#### Registries
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 9ce4999f8b..e6a6bea468 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -4,10 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsException;
-import io.github.hectorvent.floci.services.glue.model.Database;
-import io.github.hectorvent.floci.services.glue.model.Partition;
-import io.github.hectorvent.floci.services.glue.model.Table;
-import io.github.hectorvent.floci.services.glue.model.UserDefinedFunction;
+import io.github.hectorvent.floci.services.glue.model.*;
import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
import io.github.hectorvent.floci.services.glue.schemaregistry.model.Registry;
import io.github.hectorvent.floci.services.glue.schemaregistry.model.RegistryId;
@@ -164,14 +161,67 @@ public class GlueJsonHandler {
case "PutSchemaVersionMetadata" -> handlePutSchemaVersionMetadata(request);
case "RemoveSchemaVersionMetadata" -> handleRemoveSchemaVersionMetadata(request);
case "QuerySchemaVersionMetadata" -> handleQuerySchemaVersionMetadata(request);
- case "TagResource" -> handleTagResource(request);
- case "UntagResource" -> handleUntagResource(request);
- case "GetTags" -> handleGetTags(request);
+ case "TagResource" -> handleTagResource(request, region);
+ case "UntagResource" -> handleUntagResource(request, region);
+ case "GetTags" -> handleGetTags(request, region);
+ case "CreateJob" -> {
+ CreateJobRequest req = mapper.treeToValue(request, CreateJobRequest.class);
+ Job job = toDomain(req);
+ glueService.createJob(job, req.getTags(), region);
+ yield Response.ok(new CreateJobResponse(job.getName())).build();
+ }
+ case "GetJob" -> {
+ GetJobRequest req = mapper.treeToValue(request, GetJobRequest.class);
+ yield Response.ok(new GetJobResponse(glueService.getJob(req.getJobName()))).build();
+ }
+ case "GetJobs" -> {
+ Integer maxResults = readMaxResults(request);
+ String nextToken = readNextToken(request);
+ GlueService.Page<Job> page = glueService.getJobs(maxResults, nextToken);
+ yield Response.ok(pageResponse("Jobs", page.items(), page.nextToken())).build();
+ }
+ case "UpdateJob" -> {
+ UpdateJobRequest req = mapper.treeToValue(request, UpdateJobRequest.class);
+ glueService.updateJob(req.getJobName(), req.getJobUpdate());
+ yield Response.ok(new UpdateJobResponse(req.getJobName())).build();
+ }
+ case "DeleteJob" -> {
+ DeleteJobRequest req = mapper.treeToValue(request, DeleteJobRequest.class);
+ glueService.deleteJob(req.getJobName(), region);
+ yield Response.ok(new DeleteJobResponse(req.getJobName())).build();
+ }
+ case "CreateCrawler" -> {
+ CreateCrawlerRequest req = mapper.treeToValue(request, CreateCrawlerRequest.class);
+ Crawler crawler = toDomain(req);
+ glueService.createCrawler(crawler, req.getTags(), region);
+ yield Response.ok().build();
+ }
+ case "GetCrawler" -> {
+ GetCrawlerRequest req = mapper.treeToValue(request, GetCrawlerRequest.class);
+ yield Response.ok(new GetCrawlerResponse(glueService.getCrawler(req.getName()))).build();
+ }
+ case "GetCrawlers" -> {
+ GetCrawlersRequest req = mapper.treeToValue(request, GetCrawlersRequest.class);
+ GlueService.Page<Crawler> page = glueService.getCrawlers(req.getMaxResults(), req.getNextToken());
+ GetCrawlersResponse res = new GetCrawlersResponse();
+ res.setCrawlers(page.items());
+ res.setNextToken(page.nextToken());
+ yield Response.ok(res).build();
+ }
+ case "UpdateCrawler" -> {
+ UpdateCrawlerRequest req = mapper.treeToValue(request, UpdateCrawlerRequest.class);
+ Crawler update = toDomain(req);
+ glueService.updateCrawler(update);
+ yield Response.ok().build();
+ }
+ case "DeleteCrawler" -> {
+ DeleteCrawlerRequest req = mapper.treeToValue(request, DeleteCrawlerRequest.class);
+ glueService.deleteCrawler(req.getName(), region);
+ yield Response.ok().build();
+ }
// Read-only Glue actions for resources the emulator does not model. The AWS SDK
// expects each to return a 200 with its result key present (empty), so we emit the
// documented empty shape rather than an InvalidAction 400 that callers can't read.
- case "GetJobs" -> Response.ok(Map.of("Jobs", List.of())).build();
- case "GetCrawlers" -> Response.ok(Map.of("Crawlers", List.of())).build();
case "ListDataQualityRulesets" -> Response.ok(Map.of("Rulesets", List.of())).build();
case "GetSecurityConfigurations" -> Response.ok(Map.of("SecurityConfigurations", List.of())).build();
default -> throw new AwsException("InvalidAction", "Action " + action + " is not supported", 400);
@@ -698,28 +748,104 @@ public class GlueJsonHandler {
}
@SuppressWarnings("unchecked")
- private Response handleTagResource(JsonNode request) {
+ private Response handleTagResource(JsonNode request, String region) {
String arn = request.path("ResourceArn").asText(null);
Map<String, String> tagsToAdd = request.has("TagsToAdd")
? mapper.convertValue(request.get("TagsToAdd"), Map.class)
: null;
- schemaRegistryService.tagResource(arn, tagsToAdd);
- return Response.ok(Map.of()).build();
+ glueService.tagResource(arn, tagsToAdd, region);
+ return Response.ok().build();
}
@SuppressWarnings("unchecked")
- private Response handleUntagResource(JsonNode request) {
+ private Response handleUntagResource(JsonNode request, String region) {
String arn = request.path("ResourceArn").asText(null);
List<String> tagsToRemove = request.has("TagsToRemove")
? mapper.convertValue(request.get("TagsToRemove"), List.class)
: null;
- schemaRegistryService.untagResource(arn, tagsToRemove);
- return Response.ok(Map.of()).build();
+ glueService.untagResource(arn, tagsToRemove, region);
+ return Response.ok().build();
}
- private Response handleGetTags(JsonNode request) {
+ private Response handleGetTags(JsonNode request, String region) {
String arn = request.path("ResourceArn").asText(null);
- Map<String, String> tags = schemaRegistryService.getTags(arn);
+ Map<String, String> tags = glueService.getTags(arn, region);
return Response.ok(Map.of("Tags", tags)).build();
}
+
+ private Job toDomain(CreateJobRequest req) {
+ Job job = new Job();
+ job.setName(req.getName());
+ job.setAllocatedCapacity(req.getAllocatedCapacity());
+ job.setCodeGenConfigurationNodes(req.getCodeGenConfigurationNodes());
+ job.setCommand(req.getCommand());
+ job.setConnections(req.getConnections());
+ job.setDefaultArguments(req.getDefaultArguments());
+ job.setDescription(req.getDescription());
+ job.setExecutionClass(req.getExecutionClass());
+ job.setExecutionProperty(req.getExecutionProperty());
+ job.setGlueVersion(req.getGlueVersion());
+ job.setJobMode(req.getJobMode());
+ job.setJobRunQueuingEnabled(req.getJobRunQueuingEnabled());
+ job.setLogUri(req.getLogUri());
+ job.setMaintenanceWindow(req.getMaintenanceWindow());
+ job.setMaxCapacity(req.getMaxCapacity());
+ job.setMaxRetries(req.getMaxRetries());
+ job.setNonOverridableArguments(req.getNonOverridableArguments());
+ job.setNotificationProperty(req.getNotificationProperty());
+ job.setNumberOfWorkers(req.getNumberOfWorkers());
+ job.setRole(req.getRole());
+ job.setSecurityConfiguration(req.getSecurityConfiguration());
+ job.setTimeout(req.getTimeout());
+ job.setWorkerType(req.getWorkerType());
+ return job;
+ }
+
+ private Crawler toDomain(CreateCrawlerRequest req) {
+ Crawler crawler = new Crawler();
+ crawler.setName(req.getName());
+ crawler.setClassifiers(req.getClassifiers());
+ crawler.setConfiguration(req.getConfiguration());
+ crawler.setCrawlerSecurityConfiguration(req.getCrawlerSecurityConfiguration());
+ crawler.setDatabaseName(req.getDatabaseName());
+ crawler.setDescription(req.getDescription());
+ crawler.setLakeFormationConfiguration(req.getLakeFormationConfiguration());
+ crawler.setLineageConfiguration(req.getLineageConfiguration());
+ crawler.setRecrawlPolicy(req.getRecrawlPolicy());
+ crawler.setRole(req.getRole());
+ crawler.setSchemaChangePolicy(req.getSchemaChangePolicy());
+ crawler.setTablePrefix(req.getTablePrefix());
+ crawler.setTargets(req.getTargets());
+ if (req.getSchedule() != null && !req.getSchedule().isBlank()) {
+ Schedule schedule = new Schedule();
+ schedule.setScheduleExpression(req.getSchedule());
+ schedule.setState("SCHEDULED");
+ crawler.setSchedule(schedule);
+ }
+ return crawler;
+ }
+
+ private Crawler toDomain(UpdateCrawlerRequest req) {
+ Crawler crawler = new Crawler();
+ crawler.setName(req.getName());
+ crawler.setClassifiers(req.getClassifiers());
+ crawler.setConfiguration(req.getConfiguration());
+ crawler.setCrawlerSecurityConfiguration(req.getCrawlerSecurityConfiguration());
+ crawler.setDatabaseName(req.getDatabaseName());
+ crawler.setDescription(req.getDescription());
+ crawler.setLakeFormationConfiguration(req.getLakeFormationConfiguration());
+ crawler.setLineageConfiguration(req.getLineageConfiguration());
+ crawler.setRecrawlPolicy(req.getRecrawlPolicy());
+ crawler.setRole(req.getRole());
+ crawler.setSchemaChangePolicy(req.getSchemaChangePolicy());
+ crawler.setTablePrefix(req.getTablePrefix());
+ crawler.setTargets(req.getTargets());
+ if (req.getSchedule() != null && !req.getSchedule().isBlank()) {
+ Schedule schedule = new Schedule();
+ schedule.setScheduleExpression(req.getSchedule());
+ schedule.setState("SCHEDULED");
+ crawler.setSchedule(schedule);
+ }
+ return crawler;
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 59b4ae5304..0c1b9796ae 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -7,7 +7,11 @@ import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Crawler;
+import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;
import io.github.hectorvent.floci.services.glue.model.Database;
+import io.github.hectorvent.floci.services.glue.model.Job;
+import io.github.hectorvent.floci.services.glue.model.JobUpdate;
import io.github.hectorvent.floci.services.glue.model.Partition;
import io.github.hectorvent.floci.services.glue.model.SchemaReference;
import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
@@ -62,6 +66,8 @@ public class GlueService {
private final StorageBackend<String, Partition> partitionStore;
private final StorageBackend<String, Map<String, Object>> partitionColumnStatisticsStore;
private final StorageBackend<String, UserDefinedFunction> functionStore;
+ private final StorageBackend<String, Job> jobStore;
+ private final StorageBackend<String, Crawler> crawlerStore;
private final GlueSchemaRegistryService schemaRegistryService;
private final RegionResolver regionResolver;
private final ResourceGroupsTaggingService resourceGroupsTaggingService;
@@ -79,6 +85,8 @@ public class GlueService {
this.partitionColumnStatisticsStore = storageFactory.create(
"glue", "partition_column_statistics.json", new TypeReference<>() {});
this.functionStore = storageFactory.create("glue", "functions.json", new TypeReference<>() {});
+ this.jobStore = storageFactory.create("glue", "jobs.json", new TypeReference<>() {});
+ this.crawlerStore = storageFactory.create("glue", "crawlers.json", new TypeReference<>() {});
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
this.resourceGroupsTaggingService = resourceGroupsTaggingService;
@@ -91,6 +99,8 @@ public class GlueService {
StorageBackend<String, Partition> partitionStore,
StorageBackend<String, Map<String, Object>> partitionColumnStatisticsStore,
StorageBackend<String, UserDefinedFunction> functionStore,
+ StorageBackend<String, Job> jobStore,
+ StorageBackend<String, Crawler> crawlerStore,
GlueSchemaRegistryService schemaRegistryService,
RegionResolver regionResolver,
ResourceGroupsTaggingService resourceGroupsTaggingService) {
@@ -101,6 +111,8 @@ public class GlueService {
this.partitionStore = partitionStore;
this.partitionColumnStatisticsStore = partitionColumnStatisticsStore;
this.functionStore = functionStore;
+ this.jobStore = jobStore;
+ this.crawlerStore = crawlerStore;
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
this.resourceGroupsTaggingService = resourceGroupsTaggingService;
@@ -840,6 +852,14 @@ public class GlueService {
return regionResolver.buildArn("glue", region, "database/" + databaseName);
}
+ private String jobArn(String region, String jobName) {
+ return regionResolver.buildArn("glue", region, "job/" + jobName);
+ }
+
+ private String crawlerArn(String region, String crawlerName) {
+ return regionResolver.buildArn("glue", region, "crawler/" + crawlerName);
+ }
+
private static Pattern compileFunctionPattern(String pattern) {
if (pattern == null) {
return Pattern.compile(".*");
@@ -1004,4 +1024,312 @@ public class GlueService {
return source != null ? new LinkedHashMap<>(source) : null;
}
+ private void validateRequired(String value, String fieldName) {
+ if (value == null || value.isBlank()) {
+ throw new AwsException("InvalidInputException", fieldName + " is required.", 400);
+ }
+ }
+
+ private void validateRequired(Object value, String fieldName) {
+ if (value == null) {
+ throw new AwsException("InvalidInputException", fieldName + " is required.", 400);
+ }
+ }
+
+ public void createJob(Job job) {
+ createJob(job, null, regionResolver.getDefaultRegion());
+ }
+
+ public void createJob(Job job, Map<String, String> tags, String region) {
+ validateRequired(job.getName(), "Name");
+ validateRequired(job.getRole(), "Role");
+ validateRequired(job.getCommand(), "Command");
+
+ // Jobs and crawlers skip normalizeName because AWS preserves their case
+ String name = job.getName();
+ if (jobStore.get(name).isPresent()) {
+ throw new AwsException("AlreadyExistsException", "Job " + name + " already exists.", 400);
+ }
+
+ if (job.getGlueVersion() == null) {
+ job.setGlueVersion("5.1");
+ }
+ if (job.getTimeout() == null) {
+ try {
+ double version = Double.parseDouble(job.getGlueVersion());
+ job.setTimeout(version >= 5.0 ? 480 : 2880);
+ } catch (NumberFormatException e) {
+ job.setTimeout(2880);
+ }
+ }
+
+ Instant now = Instant.now();
+ job.setCreatedOn(now);
+ job.setLastModifiedOn(now);
+ jobStore.put(name, job);
+ if (tags != null && !tags.isEmpty()) {
+ resourceGroupsTaggingService.tagResources(List.of(jobArn(region, name)), tags, region);
+ }
+ LOG.infov("Created Glue Job: {0}", name);
+ }
+
+ public Job getJob(String name) {
+ validateRequired(name, "JobName");
+ return jobStore.get(name)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Job " + name + " not found.", 400));
+ }
+
+ public List<Job> getJobs() {
+ return jobStore.scan(k -> true);
+ }
+
+ public Page<Job> getJobs(Integer maxResults, String nextToken) {
+ List<Job> all = jobStore.scan(k -> true);
+ all.sort(Comparator.comparing(Job::getName));
+ return paginate(all, maxResults, nextToken);
+ }
+
+ public void updateJob(String name, JobUpdate update) {
+ validateRequired(name, "JobName");
+ validateRequired(update, "JobUpdate");
+ // Jobs and crawlers skip normalizeName because AWS preserves their case
+ String normalizedName = name;
+ Job existing = jobStore.get(normalizedName)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Job " + name + " not found.", 400));
+
+ // UpdateJob replaces unspecified configuration with nulls (full replacement) rather than merging.
+ // This is deliberate and matches AWS behavior. We only copy fields that the user cannot specify.
+ Job updated = new Job();
+ updated.setName(existing.getName());
+ updated.setCreatedOn(existing.getCreatedOn());
+ updated.setProfileName(existing.getProfileName());
+ updated.setLastModifiedOn(Instant.now());
+
+ updated.setAllocatedCapacity(update.getAllocatedCapacity());
+ updated.setCodeGenConfigurationNodes(update.getCodeGenConfigurationNodes());
+ updated.setCommand(update.getCommand());
+ updated.setConnections(update.getConnections());
+ updated.setDefaultArguments(update.getDefaultArguments());
+ updated.setDescription(update.getDescription());
+ updated.setExecutionClass(update.getExecutionClass());
+ updated.setExecutionProperty(update.getExecutionProperty());
+ updated.setGlueVersion(update.getGlueVersion());
+ updated.setJobMode(update.getJobMode());
+ updated.setJobRunQueuingEnabled(update.getJobRunQueuingEnabled());
+ updated.setLogUri(update.getLogUri());
+ updated.setMaintenanceWindow(update.getMaintenanceWindow());
+ updated.setMaxCapacity(update.getMaxCapacity());
+ updated.setMaxRetries(update.getMaxRetries());
+ updated.setNonOverridableArguments(update.getNonOverridableArguments());
+ updated.setNotificationProperty(update.getNotificationProperty());
+ updated.setNumberOfWorkers(update.getNumberOfWorkers());
+ updated.setRole(update.getRole());
+ updated.setSecurityConfiguration(update.getSecurityConfiguration());
+ updated.setSourceControlDetails(update.getSourceControlDetails());
+ updated.setTimeout(update.getTimeout());
+ updated.setWorkerType(update.getWorkerType());
+
+ jobStore.put(normalizedName, updated);
+ LOG.infov("Updated Glue Job: {0}", normalizedName);
+ }
+
+ public void deleteJob(String name, String region) {
+ validateRequired(name, "JobName");
+ // Jobs and crawlers skip normalizeName because AWS preserves their case
+ String normalizedName = name;
+ if (jobStore.get(normalizedName).isEmpty()) {
+ throw new AwsException("EntityNotFoundException", "Job " + name + " not found.", 400);
+ }
+ jobStore.delete(normalizedName);
+ resourceGroupsTaggingService.deleteResources(List.of(jobArn(region, normalizedName)), region);
+ LOG.infov("Deleted Glue Job: {0}", name);
+ }
+
+ public void createCrawler(Crawler crawler) {
+ createCrawler(crawler, null, regionResolver.getDefaultRegion());
+ }
+
+ public void createCrawler(Crawler crawler, Map<String, String> tags, String region) {
+ validateRequired(crawler.getName(), "Name");
+ validateRequired(crawler.getRole(), "Role");
+ validateRequired(crawler.getTargets(), "Targets");
+
+ CrawlerTargets t = crawler.getTargets();
+ boolean hasTargets = (t.getS3Targets() != null && !t.getS3Targets().isEmpty()) ||
+ (t.getJdbcTargets() != null && !t.getJdbcTargets().isEmpty()) ||
+ (t.getDynamoDBTargets() != null && !t.getDynamoDBTargets().isEmpty()) ||
+ (t.getCatalogTargets() != null && !t.getCatalogTargets().isEmpty()) ||
+ (t.getDeltaTargets() != null && !t.getDeltaTargets().isEmpty()) ||
+ (t.getIcebergTargets() != null && !t.getIcebergTargets().isEmpty()) ||
+ (t.getMongoDBTargets() != null && !t.getMongoDBTargets().isEmpty()) ||
+ (t.getHudiTargets() != null && !t.getHudiTargets().isEmpty());
+ if (!hasTargets) {
+ throw new AwsException("InvalidInputException", "At least one crawl target must be specified.", 400);
+ }
+
+ // Jobs and crawlers skip normalizeName because AWS preserves their case
+ String name = crawler.getName();
+ if (crawlerStore.get(name).isPresent()) {
+ throw new AwsException("AlreadyExistsException", "Crawler " + name + " already exists.", 400);
+ }
+ Instant now = Instant.now();
+ crawler.setCreationTime(now);
+ crawler.setLastUpdated(now);
+ crawler.setState("READY");
+ crawler.setVersion(1L);
+ crawlerStore.put(name, crawler);
+ if (tags != null && !tags.isEmpty()) {
+ resourceGroupsTaggingService.tagResources(List.of(crawlerArn(region, name)), tags, region);
+ }
+ LOG.infov("Created Glue Crawler: {0}", name);
+ }
+
+ public Crawler getCrawler(String name) {
+ validateRequired(name, "Name");
+ return crawlerStore.get(name)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Crawler " + name + " not found.", 400));
+ }
+
+ public List<Crawler> getCrawlers() {
+ return crawlerStore.scan(k -> true);
+ }
+
+ public Page<Crawler> getCrawlers(Integer maxResults, String nextToken) {
+ List<Crawler> all = crawlerStore.scan(k -> true);
+ all.sort(Comparator.comparing(Crawler::getName));
+ return paginate(all, maxResults, nextToken);
+ }
+
+ public void updateCrawler(Crawler update) {
+ validateRequired(update.getName(), "Name");
+ // Jobs and crawlers skip normalizeName because AWS preserves their case
+ String name = update.getName();
+ Crawler existing = crawlerStore.get(name)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Crawler " + update.getName() + " not found.", 400));
+
+ Crawler updated = new Crawler();
+ updated.setName(existing.getName());
+ updated.setCreationTime(existing.getCreationTime());
+ updated.setLastUpdated(Instant.now());
+ updated.setState(existing.getState());
+ updated.setVersion((existing.getVersion() == null ? 1L : existing.getVersion()) + 1L);
+
+ updated.setClassifiers(update.getClassifiers() != null ? update.getClassifiers() : existing.getClassifiers());
+ updated.setConfiguration(update.getConfiguration() != null ? update.getConfiguration() : existing.getConfiguration());
+ updated.setCrawlerSecurityConfiguration(update.getCrawlerSecurityConfiguration() != null ? update.getCrawlerSecurityConfiguration() : existing.getCrawlerSecurityConfiguration());
+ updated.setDatabaseName(update.getDatabaseName() != null ? update.getDatabaseName() : existing.getDatabaseName());
+ updated.setDescription(update.getDescription() != null ? update.getDescription() : existing.getDescription());
+ updated.setLakeFormationConfiguration(update.getLakeFormationConfiguration() != null ? update.getLakeFormationConfiguration() : existing.getLakeFormationConfiguration());
+ updated.setLineageConfiguration(update.getLineageConfiguration() != null ? update.getLineageConfiguration() : existing.getLineageConfiguration());
+ updated.setRecrawlPolicy(update.getRecrawlPolicy() != null ? update.getRecrawlPolicy() : existing.getRecrawlPolicy());
+ updated.setRole(update.getRole() != null ? update.getRole() : existing.getRole());
+ updated.setSchedule(update.getSchedule() != null ? update.getSchedule() : existing.getSchedule());
+ updated.setSchemaChangePolicy(update.getSchemaChangePolicy() != null ? update.getSchemaChangePolicy() : existing.getSchemaChangePolicy());
+ updated.setTablePrefix(update.getTablePrefix() != null ? update.getTablePrefix() : existing.getTablePrefix());
+ updated.setTargets(update.getTargets() != null ? update.getTargets() : existing.getTargets());
+
+ crawlerStore.put(name, updated);
+ LOG.infov("Updated Glue Crawler: {0}", name);
+ }
+
+ public void deleteCrawler(String name, String region) {
+ validateRequired(name, "Name");
+ // Jobs and crawlers skip normalizeName because AWS preserves their case
+ String normalizedName = name;
+ if (crawlerStore.get(normalizedName).isEmpty()) {
+ throw new AwsException("EntityNotFoundException", "Crawler " + name + " not found.", 400);
+ }
+ crawlerStore.delete(normalizedName);
+ resourceGroupsTaggingService.deleteResources(List.of(crawlerArn(region, normalizedName)), region);
+ LOG.infov("Deleted Glue Crawler: {0}", name);
+ }
+
+ public void tagResource(String arn, Map<String, String> tags, String region) {
+ validateArn(arn);
+ if (arn.contains(":registry/") || arn.contains(":schema/")) {
+ schemaRegistryService.tagResource(arn, tags);
+ } else {
+ validateResourceExists(arn);
+ resourceGroupsTaggingService.tagResources(List.of(arn), tags, region);
+ }
+ }
+
+ public void untagResource(String arn, List<String> tagKeys, String region) {
+ validateArn(arn);
+ if (arn.contains(":registry/") || arn.contains(":schema/")) {
+ schemaRegistryService.untagResource(arn, tagKeys);
+ } else {
+ validateResourceExists(arn);
+ resourceGroupsTaggingService.untagResources(List.of(arn), tagKeys, region);
+ }
+ }
+
+ public Map<String, String> getTags(String arn, String region) {
+ validateArn(arn);
+ if (arn.contains(":registry/") || arn.contains(":schema/")) {
+ return schemaRegistryService.getTags(arn);
+ } else {
+ validateResourceExists(arn);
+ return resourceGroupsTaggingService.getTagsForResource(region, arn);
+ }
+ }
+
+ private void validateArn(String arn) {
+ if (arn == null || !arn.startsWith("arn:")) {
+ throw new AwsException("InvalidInputException", "Invalid ARN", 400);
+ }
+ }
+
+ private void validateResourceExists(String arn) {
+ String[] parts = arn.split(":");
+ if (parts.length >= 6) {
+ String resource = parts[5];
+ if (resource.startsWith("job/")) {
+ getJob(resource.substring(4));
+ return;
+ } else if (resource.startsWith("crawler/")) {
+ getCrawler(resource.substring(8));
+ return;
+ } else if (resource.startsWith("database/")) {
+ getDatabase(resource.substring(9));
+ return;
+ } else if (resource.startsWith("table/")) {
+ String[] tableParts = resource.substring(6).split("/");
+ if (tableParts.length >= 2) {
+ getTable(tableParts[0], tableParts[1]);
+ return;
+ }
+ } else if (resource.startsWith("userDefinedFunction/")) {
+ String[] funcParts = resource.substring(20).split("/");
+ if (funcParts.length >= 2) {
+ getUserDefinedFunction(funcParts[0], funcParts[1]);
+ return;
+ }
+ }
+ }
+ throw new AwsException("EntityNotFoundException", "Resource " + arn + " not found or not supported.", 400);
+ }
+
+ public record Page<T>(List<T> items, String nextToken) {}
+
+ private <T> Page<T> paginate(List<T> all, Integer maxResults, String nextToken) {
+ if (maxResults != null && (maxResults < 1 || maxResults > 1000)) {
+ throw new AwsException("InvalidInputException", "MaxResults must be between 1 and 1000", 400);
+ }
+ int limit = maxResults == null ? 100 : maxResults;
+ int start = 0;
+ if (nextToken != null && !nextToken.isBlank()) {
+ try {
+ start = Integer.parseInt(nextToken);
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidInputException", "Invalid NextToken", 400);
+ }
+ }
+ if (start < 0 || start > all.size()) {
+ throw new AwsException("InvalidInputException", "Invalid NextToken", 400);
+ }
+ int end = Math.min(start + limit, all.size());
+ String newToken = end < all.size() ? String.valueOf(end) : null;
+ return new Page<>(List.copyOf(all.subList(start, end)), newToken);
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CatalogTarget.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CatalogTarget.java
new file mode 100644
index 0000000000..f809a9d5ce
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CatalogTarget.java
@@ -0,0 +1,41 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class CatalogTarget {
+ @JsonProperty("ConnectionName")
+ private String connectionName;
+
+ @JsonProperty("DatabaseName")
+ private String databaseName;
+
+ @JsonProperty("DlqEventQueueArn")
+ private String dlqEventQueueArn;
+
+ @JsonProperty("EventQueueArn")
+ private String eventQueueArn;
+
+ @JsonProperty("Tables")
+ private List<String> tables;
+
+ public CatalogTarget() {}
+
+ public String getConnectionName() { return connectionName; }
+ public void setConnectionName(String connectionName) { this.connectionName = connectionName; }
+
+ public String getDatabaseName() { return databaseName; }
+ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
+
+ public String getDlqEventQueueArn() { return dlqEventQueueArn; }
+ public void setDlqEventQueueArn(String dlqEventQueueArn) { this.dlqEventQueueArn = dlqEventQueueArn; }
+
+ public String getEventQueueArn() { return eventQueueArn; }
+ public void setEventQueueArn(String eventQueueArn) { this.eventQueueArn = eventQueueArn; }
+
+ public List<String> getTables() { return tables; }
+ public void setTables(List<String> tables) { this.tables = tables; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionsList.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionsList.java
new file mode 100644
index 0000000000..11eb2f4f51
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionsList.java
@@ -0,0 +1,17 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class ConnectionsList {
+ @JsonProperty("Connections")
+ private List<String> connections;
+
+ public ConnectionsList() {}
+
+ public List<String> getConnections() { return connections; }
+ public void setConnections(List<String> connections) { this.connections = connections; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java
new file mode 100644
index 0000000000..85de429ebf
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java
@@ -0,0 +1,134 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
+import java.util.List;
+
+@RegisterForReflection
+public class Crawler {
+ @JsonProperty("Classifiers")
+ private List<String> classifiers;
+
+ @JsonProperty("Configuration")
+ private String configuration;
+
+ @JsonProperty("CrawlElapsedTime")
+ private Long crawlElapsedTime;
+
+ @JsonProperty("CrawlerSecurityConfiguration")
+ private String crawlerSecurityConfiguration;
+
+ @JsonProperty("CreationTime")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant creationTime;
+
+ @JsonProperty("DatabaseName")
+ private String databaseName;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("LakeFormationConfiguration")
+ private LakeFormationConfiguration lakeFormationConfiguration;
+
+ @JsonProperty("LastCrawl")
+ private LastCrawlInfo lastCrawl;
+
+ @JsonProperty("LastUpdated")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant lastUpdated;
+
+ @JsonProperty("LineageConfiguration")
+ private LineageConfiguration lineageConfiguration;
+
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("RecrawlPolicy")
+ private RecrawlPolicy recrawlPolicy;
+
+ @JsonProperty("Role")
+ private String role;
+
+ @JsonProperty("Schedule")
+ private Schedule schedule;
+
+ @JsonProperty("SchemaChangePolicy")
+ private SchemaChangePolicy schemaChangePolicy;
+
+ @JsonProperty("State")
+ private String state;
+
+ @JsonProperty("TablePrefix")
+ private String tablePrefix;
+
+ @JsonProperty("Targets")
+ private CrawlerTargets targets;
+
+ @JsonProperty("Version")
+ private Long version;
+
+ public Crawler() {}
+
+ public List<String> getClassifiers() { return classifiers; }
+ public void setClassifiers(List<String> classifiers) { this.classifiers = classifiers; }
+
+ public String getConfiguration() { return configuration; }
+ public void setConfiguration(String configuration) { this.configuration = configuration; }
+
+ public Long getCrawlElapsedTime() { return crawlElapsedTime; }
+ public void setCrawlElapsedTime(Long crawlElapsedTime) { this.crawlElapsedTime = crawlElapsedTime; }
+
+ public String getCrawlerSecurityConfiguration() { return crawlerSecurityConfiguration; }
+ public void setCrawlerSecurityConfiguration(String crawlerSecurityConfiguration) { this.crawlerSecurityConfiguration = crawlerSecurityConfiguration; }
+
+ public Instant getCreationTime() { return creationTime; }
+ public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }
+
+ public String getDatabaseName() { return databaseName; }
+ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public LakeFormationConfiguration getLakeFormationConfiguration() { return lakeFormationConfiguration; }
+ public void setLakeFormationConfiguration(LakeFormationConfiguration lakeFormationConfiguration) { this.lakeFormationConfiguration = lakeFormationConfiguration; }
+
+ public LastCrawlInfo getLastCrawl() { return lastCrawl; }
+ public void setLastCrawl(LastCrawlInfo lastCrawl) { this.lastCrawl = lastCrawl; }
+
+ public Instant getLastUpdated() { return lastUpdated; }
+ public void setLastUpdated(Instant lastUpdated) { this.lastUpdated = lastUpdated; }
+
+ public LineageConfiguration getLineageConfiguration() { return lineageConfiguration; }
+ public void setLineageConfiguration(LineageConfiguration lineageConfiguration) { this.lineageConfiguration = lineageConfiguration; }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public RecrawlPolicy getRecrawlPolicy() { return recrawlPolicy; }
+ public void setRecrawlPolicy(RecrawlPolicy recrawlPolicy) { this.recrawlPolicy = recrawlPolicy; }
+
+ public String getRole() { return role; }
+ public void setRole(String role) { this.role = role; }
+
+ public Schedule getSchedule() { return schedule; }
+ public void setSchedule(Schedule schedule) { this.schedule = schedule; }
+
+ public SchemaChangePolicy getSchemaChangePolicy() { return schemaChangePolicy; }
+ public void setSchemaChangePolicy(SchemaChangePolicy schemaChangePolicy) { this.schemaChangePolicy = schemaChangePolicy; }
+
+ public String getState() { return state; }
+ public void setState(String state) { this.state = state; }
+
+ public String getTablePrefix() { return tablePrefix; }
+ public void setTablePrefix(String tablePrefix) { this.tablePrefix = tablePrefix; }
+
+ public CrawlerTargets getTargets() { return targets; }
+ public void setTargets(CrawlerTargets targets) { this.targets = targets; }
+
+ public Long getVersion() { return version; }
+ public void setVersion(Long version) { this.version = version; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerTargets.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerTargets.java
new file mode 100644
index 0000000000..d3631c5b6a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerTargets.java
@@ -0,0 +1,59 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class CrawlerTargets {
+ @JsonProperty("CatalogTargets")
+ private List<CatalogTarget> catalogTargets;
+
+ @JsonProperty("DeltaTargets")
+ private List<DeltaTarget> deltaTargets;
+
+ @JsonProperty("DynamoDBTargets")
+ private List<DynamoDBTarget> dynamoDBTargets;
+
+ @JsonProperty("HudiTargets")
+ private List<HudiTarget> hudiTargets;
+
+ @JsonProperty("IcebergTargets")
+ private List<IcebergTarget> icebergTargets;
+
+ @JsonProperty("JdbcTargets")
+ private List<JdbcTarget> jdbcTargets;
+
+ @JsonProperty("MongoDBTargets")
+ private List<MongoDBTarget> mongoDBTargets;
+
+ @JsonProperty("S3Targets")
+ private List<S3Target> s3Targets;
+
+ public CrawlerTargets() {}
+
+ public List<CatalogTarget> getCatalogTargets() { return catalogTargets; }
+ public void setCatalogTargets(List<CatalogTarget> catalogTargets) { this.catalogTargets = catalogTargets; }
+
+ public List<DeltaTarget> getDeltaTargets() { return deltaTargets; }
+ public void setDeltaTargets(List<DeltaTarget> deltaTargets) { this.deltaTargets = deltaTargets; }
+
+ public List<DynamoDBTarget> getDynamoDBTargets() { return dynamoDBTargets; }
+ public void setDynamoDBTargets(List<DynamoDBTarget> dynamoDBTargets) { this.dynamoDBTargets = dynamoDBTargets; }
+
+ public List<HudiTarget> getHudiTargets() { return hudiTargets; }
+ public void setHudiTargets(List<HudiTarget> hudiTargets) { this.hudiTargets = hudiTargets; }
+
+ public List<IcebergTarget> getIcebergTargets() { return icebergTargets; }
+ public void setIcebergTargets(List<IcebergTarget> icebergTargets) { this.icebergTargets = icebergTargets; }
+
+ public List<JdbcTarget> getJdbcTargets() { return jdbcTargets; }
+ public void setJdbcTargets(List<JdbcTarget> jdbcTargets) { this.jdbcTargets = jdbcTargets; }
+
+ public List<MongoDBTarget> getMongoDBTargets() { return mongoDBTargets; }
+ public void setMongoDBTargets(List<MongoDBTarget> mongoDBTargets) { this.mongoDBTargets = mongoDBTargets; }
+
+ public List<S3Target> getS3Targets() { return s3Targets; }
+ public void setS3Targets(List<S3Target> s3Targets) { this.s3Targets = s3Targets; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateCrawlerRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateCrawlerRequest.java
new file mode 100644
index 0000000000..18ac318c02
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateCrawlerRequest.java
@@ -0,0 +1,102 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+import java.util.Map;
+
+@RegisterForReflection
+public class CreateCrawlerRequest {
+ @JsonProperty("Classifiers")
+ private List<String> classifiers;
+
+ @JsonProperty("Configuration")
+ private String configuration;
+
+ @JsonProperty("CrawlerSecurityConfiguration")
+ private String crawlerSecurityConfiguration;
+
+ @JsonProperty("DatabaseName")
+ private String databaseName;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("LakeFormationConfiguration")
+ private LakeFormationConfiguration lakeFormationConfiguration;
+
+ @JsonProperty("LineageConfiguration")
+ private LineageConfiguration lineageConfiguration;
+
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("RecrawlPolicy")
+ private RecrawlPolicy recrawlPolicy;
+
+ @JsonProperty("Role")
+ private String role;
+
+ @JsonProperty("Schedule")
+ private String schedule;
+
+ @JsonProperty("SchemaChangePolicy")
+ private SchemaChangePolicy schemaChangePolicy;
+
+ @JsonProperty("TablePrefix")
+ private String tablePrefix;
+
+ @JsonProperty("Tags")
+ private Map<String, String> tags;
+
+ @JsonProperty("Targets")
+ private CrawlerTargets targets;
+
+ public CreateCrawlerRequest() {}
+
+ public List<String> getClassifiers() { return classifiers; }
+ public void setClassifiers(List<String> classifiers) { this.classifiers = classifiers; }
+
+ public String getConfiguration() { return configuration; }
+ public void setConfiguration(String configuration) { this.configuration = configuration; }
+
+ public String getCrawlerSecurityConfiguration() { return crawlerSecurityConfiguration; }
+ public void setCrawlerSecurityConfiguration(String crawlerSecurityConfiguration) { this.crawlerSecurityConfiguration = crawlerSecurityConfiguration; }
+
+ public String getDatabaseName() { return databaseName; }
+ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public LakeFormationConfiguration getLakeFormationConfiguration() { return lakeFormationConfiguration; }
+ public void setLakeFormationConfiguration(LakeFormationConfiguration lakeFormationConfiguration) { this.lakeFormationConfiguration = lakeFormationConfiguration; }
+
+ public LineageConfiguration getLineageConfiguration() { return lineageConfiguration; }
+ public void setLineageConfiguration(LineageConfiguration lineageConfiguration) { this.lineageConfiguration = lineageConfiguration; }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public RecrawlPolicy getRecrawlPolicy() { return recrawlPolicy; }
+ public void setRecrawlPolicy(RecrawlPolicy recrawlPolicy) { this.recrawlPolicy = recrawlPolicy; }
+
+ public String getRole() { return role; }
+ public void setRole(String role) { this.role = role; }
+
+ public String getSchedule() { return schedule; }
+ public void setSchedule(String schedule) { this.schedule = schedule; }
+
+ public SchemaChangePolicy getSchemaChangePolicy() { return schemaChangePolicy; }
+ public void setSchemaChangePolicy(SchemaChangePolicy schemaChangePolicy) { this.schemaChangePolicy = schemaChangePolicy; }
+
+ public String getTablePrefix() { return tablePrefix; }
+ public void setTablePrefix(String tablePrefix) { this.tablePrefix = tablePrefix; }
+
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags; }
+
+ public CrawlerTargets getTargets() { return targets; }
+ public void setTargets(CrawlerTargets targets) { this.targets = targets; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateJobRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateJobRequest.java
new file mode 100644
index 0000000000..cf22aea72b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateJobRequest.java
@@ -0,0 +1,162 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.Map;
+
+@RegisterForReflection
+public class CreateJobRequest {
+ @JsonProperty("AllocatedCapacity")
+ private Integer allocatedCapacity;
+
+ @JsonProperty("CodeGenConfigurationNodes")
+ private Map<String, JsonNode> codeGenConfigurationNodes;
+
+ @JsonProperty("Command")
+ private JobCommand command;
+
+ @JsonProperty("Connections")
+ private ConnectionsList connections;
+
+ @JsonProperty("DefaultArguments")
+ private Map<String, String> defaultArguments;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("ExecutionClass")
+ private String executionClass;
+
+ @JsonProperty("ExecutionProperty")
+ private ExecutionProperty executionProperty;
+
+ @JsonProperty("GlueVersion")
+ private String glueVersion;
+
+ @JsonProperty("JobMode")
+ private String jobMode;
+
+ @JsonProperty("JobRunQueuingEnabled")
+ private Boolean jobRunQueuingEnabled;
+
+ @JsonProperty("LogUri")
+ private String logUri;
+
+ @JsonProperty("MaintenanceWindow")
+ private String maintenanceWindow;
+
+ @JsonProperty("MaxCapacity")
+ private Double maxCapacity;
+
+ @JsonProperty("MaxRetries")
+ private Integer maxRetries;
+
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("NonOverridableArguments")
+ private Map<String, String> nonOverridableArguments;
+
+ @JsonProperty("NotificationProperty")
+ private NotificationProperty notificationProperty;
+
+ @JsonProperty("NumberOfWorkers")
+ private Integer numberOfWorkers;
+
+ @JsonProperty("Role")
+ private String role;
+
+ @JsonProperty("SecurityConfiguration")
+ private String securityConfiguration;
+
+ @JsonProperty("SourceControlDetails")
+ private SourceControlDetails sourceControlDetails;
+
+ @JsonProperty("Tags")
+ private Map<String, String> tags;
+
+ @JsonProperty("Timeout")
+ private Integer timeout;
+
+ @JsonProperty("WorkerType")
+ private String workerType;
+
+ public CreateJobRequest() {}
+
+ public Integer getAllocatedCapacity() { return allocatedCapacity; }
+ public void setAllocatedCapacity(Integer allocatedCapacity) { this.allocatedCapacity = allocatedCapacity; }
+
+ public Map<String, JsonNode> getCodeGenConfigurationNodes() { return codeGenConfigurationNodes; }
+ public void setCodeGenConfigurationNodes(Map<String, JsonNode> codeGenConfigurationNodes) { this.codeGenConfigurationNodes = codeGenConfigurationNodes; }
+
+ public JobCommand getCommand() { return command; }
+ public void setCommand(JobCommand command) { this.command = command; }
+
+ public ConnectionsList getConnections() { return connections; }
+ public void setConnections(ConnectionsList connections) { this.connections = connections; }
+
+ public Map<String, String> getDefaultArguments() { return defaultArguments; }
+ public void setDefaultArguments(Map<String, String> defaultArguments) { this.defaultArguments = defaultArguments; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public String getExecutionClass() { return executionClass; }
+ public void setExecutionClass(String executionClass) { this.executionClass = executionClass; }
+
+ public ExecutionProperty getExecutionProperty() { return executionProperty; }
+ public void setExecutionProperty(ExecutionProperty executionProperty) { this.executionProperty = executionProperty; }
+
+ public String getGlueVersion() { return glueVersion; }
+ public void setGlueVersion(String glueVersion) { this.glueVersion = glueVersion; }
+
+ public String getJobMode() { return jobMode; }
+ public void setJobMode(String jobMode) { this.jobMode = jobMode; }
+
+ public Boolean getJobRunQueuingEnabled() { return jobRunQueuingEnabled; }
+ public void setJobRunQueuingEnabled(Boolean jobRunQueuingEnabled) { this.jobRunQueuingEnabled = jobRunQueuingEnabled; }
+
+ public String getLogUri() { return logUri; }
+ public void setLogUri(String logUri) { this.logUri = logUri; }
+
+ public String getMaintenanceWindow() { return maintenanceWindow; }
+ public void setMaintenanceWindow(String maintenanceWindow) { this.maintenanceWindow = maintenanceWindow; }
+
+ public Double getMaxCapacity() { return maxCapacity; }
+ public void setMaxCapacity(Double maxCapacity) { this.maxCapacity = maxCapacity; }
+
+ public Integer getMaxRetries() { return maxRetries; }
+ public void setMaxRetries(Integer maxRetries) { this.maxRetries = maxRetries; }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public Map<String, String> getNonOverridableArguments() { return nonOverridableArguments; }
+ public void setNonOverridableArguments(Map<String, String> nonOverridableArguments) { this.nonOverridableArguments = nonOverridableArguments; }
+
+ public NotificationProperty getNotificationProperty() { return notificationProperty; }
+ public void setNotificationProperty(NotificationProperty notificationProperty) { this.notificationProperty = notificationProperty; }
+
+ public Integer getNumberOfWorkers() { return numberOfWorkers; }
+ public void setNumberOfWorkers(Integer numberOfWorkers) { this.numberOfWorkers = numberOfWorkers; }
+
+ public String getRole() { return role; }
+ public void setRole(String role) { this.role = role; }
+
+ public String getSecurityConfiguration() { return securityConfiguration; }
+ public void setSecurityConfiguration(String securityConfiguration) { this.securityConfiguration = securityConfiguration; }
+
+ public SourceControlDetails getSourceControlDetails() { return sourceControlDetails; }
+ public void setSourceControlDetails(SourceControlDetails sourceControlDetails) { this.sourceControlDetails = sourceControlDetails; }
+
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags; }
+
+ public Integer getTimeout() { return timeout; }
+ public void setTimeout(Integer timeout) { this.timeout = timeout; }
+
+ public String getWorkerType() { return workerType; }
+ public void setWorkerType(String workerType) { this.workerType = workerType; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateJobResponse.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateJobResponse.java
new file mode 100644
index 0000000000..ec305d7ce4
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CreateJobResponse.java
@@ -0,0 +1,19 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class CreateJobResponse {
+ @JsonProperty("Name")
+ private String name;
+
+ public CreateJobResponse() {}
+
+ public CreateJobResponse(String name) {
+ this.name = name;
+ }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteCrawlerRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteCrawlerRequest.java
new file mode 100644
index 0000000000..8799bd53d3
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteCrawlerRequest.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class DeleteCrawlerRequest {
+ @JsonProperty("Name")
+ private String name;
+
+ public DeleteCrawlerRequest() {}
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteJobRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteJobRequest.java
new file mode 100644
index 0000000000..645664cb59
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteJobRequest.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class DeleteJobRequest {
+ @JsonProperty("JobName")
+ private String jobName;
+
+ public DeleteJobRequest() {}
+
+ public String getJobName() { return jobName; }
+ public void setJobName(String jobName) { this.jobName = jobName; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteJobResponse.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteJobResponse.java
new file mode 100644
index 0000000000..f576ae490f
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/DeleteJobResponse.java
@@ -0,0 +1,19 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class DeleteJobResponse {
+ @JsonProperty("JobName")
+ private String jobName;
+
+ public DeleteJobResponse() {}
+
+ public DeleteJobResponse(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public String getJobName() { return jobName; }
+ public void setJobName(String jobName) { this.jobName = jobName; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/DeltaTarget.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/DeltaTarget.java
new file mode 100644
index 0000000000..8a6ffe8abd
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/DeltaTarget.java
@@ -0,0 +1,35 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class DeltaTarget {
+ @JsonProperty("ConnectionName")
+ private String connectionName;
+
+ @JsonProperty("CreateNativeDeltaTable")
+ private Boolean createNativeDeltaTable;
+
+ @JsonProperty("DeltaTables")
+ private List<String> deltaTables;
+
+ @JsonProperty("WriteManifest")
+ private Boolean writeManifest;
+
+ public DeltaTarget() {}
+
+ public String getConnectionName() { return connectionName; }
+ public void setConnectionName(String connectionName) { this.connectionName = connectionName; }
+
+ public Boolean getCreateNativeDeltaTable() { return createNativeDeltaTable; }
+ public void setCreateNativeDeltaTable(Boolean createNativeDeltaTable) { this.createNativeDeltaTable = createNativeDeltaTable; }
+
+ public List<String> getDeltaTables() { return deltaTables; }
+ public void setDeltaTables(List<String> deltaTables) { this.deltaTables = deltaTables; }
+
+ public Boolean getWriteManifest() { return writeManifest; }
+ public void setWriteManifest(Boolean writeManifest) { this.writeManifest = writeManifest; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/DynamoDBTarget.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/DynamoDBTarget.java
new file mode 100644
index 0000000000..6d13a2c21d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/DynamoDBTarget.java
@@ -0,0 +1,27 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class DynamoDBTarget {
+ @JsonProperty("Path")
+ private String path;
+
+ @JsonProperty("ScanAll")
+ private Boolean scanAll;
+
+ @JsonProperty("ScanRate")
+ private Double scanRate;
+
+ public DynamoDBTarget() {}
+
+ public String getPath() { return path; }
+ public void setPath(String path) { this.path = path; }
+
+ public Boolean getScanAll() { return scanAll; }
+ public void setScanAll(Boolean scanAll) { this.scanAll = scanAll; }
+
+ public Double getScanRate() { return scanRate; }
+ public void setScanRate(Double scanRate) { this.scanRate = scanRate; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/ExecutionProperty.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/ExecutionProperty.java
new file mode 100644
index 0000000000..6429fa43d4
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/ExecutionProperty.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class ExecutionProperty {
+ @JsonProperty("MaxConcurrentRuns")
+ private Integer maxConcurrentRuns;
+
+ public ExecutionProperty() {}
+
+ public Integer getMaxConcurrentRuns() { return maxConcurrentRuns; }
+ public void setMaxConcurrentRuns(Integer maxConcurrentRuns) { this.maxConcurrentRuns = maxConcurrentRuns; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlerRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlerRequest.java
new file mode 100644
index 0000000000..f293eb3fb1
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlerRequest.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class GetCrawlerRequest {
+ @JsonProperty("Name")
+ private String name;
+
+ public GetCrawlerRequest() {}
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlerResponse.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlerResponse.java
new file mode 100644
index 0000000000..18f4f00ac9
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlerResponse.java
@@ -0,0 +1,19 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class GetCrawlerResponse {
+ @JsonProperty("Crawler")
+ private Crawler crawler;
+
+ public GetCrawlerResponse() {}
+
+ public GetCrawlerResponse(Crawler crawler) {
+ this.crawler = crawler;
+ }
+
+ public Crawler getCrawler() { return crawler; }
+ public void setCrawler(Crawler crawler) { this.crawler = crawler; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlersRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlersRequest.java
new file mode 100644
index 0000000000..5440d86289
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlersRequest.java
@@ -0,0 +1,21 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class GetCrawlersRequest {
+ @JsonProperty("MaxResults")
+ private Integer maxResults;
+
+ @JsonProperty("NextToken")
+ private String nextToken;
+
+ public GetCrawlersRequest() {}
+
+ public Integer getMaxResults() { return maxResults; }
+ public void setMaxResults(Integer maxResults) { this.maxResults = maxResults; }
+
+ public String getNextToken() { return nextToken; }
+ public void setNextToken(String nextToken) { this.nextToken = nextToken; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlersResponse.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlersResponse.java
new file mode 100644
index 0000000000..b38c5f9fe5
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetCrawlersResponse.java
@@ -0,0 +1,23 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class GetCrawlersResponse {
+ @JsonProperty("Crawlers")
+ private List<Crawler> crawlers;
+
+ @JsonProperty("NextToken")
+ private String nextToken;
+
+ public GetCrawlersResponse() {}
+
+ public List<Crawler> getCrawlers() { return crawlers; }
+ public void setCrawlers(List<Crawler> crawlers) { this.crawlers = crawlers; }
+
+ public String getNextToken() { return nextToken; }
+ public void setNextToken(String nextToken) { this.nextToken = nextToken; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GetJobRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetJobRequest.java
new file mode 100644
index 0000000000..5d35c6f577
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetJobRequest.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class GetJobRequest {
+ @JsonProperty("JobName")
+ private String jobName;
+
+ public GetJobRequest() {}
+
+ public String getJobName() { return jobName; }
+ public void setJobName(String jobName) { this.jobName = jobName; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GetJobResponse.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetJobResponse.java
new file mode 100644
index 0000000000..1cf983e7ac
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GetJobResponse.java
@@ -0,0 +1,19 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class GetJobResponse {
+ @JsonProperty("Job")
+ private Job job;
+
+ public GetJobResponse() {}
+
+ public GetJobResponse(Job job) {
+ this.job = job;
+ }
+
+ public Job getJob() { return job; }
+ public void setJob(Job job) { this.job = job; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/HudiTarget.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/HudiTarget.java
new file mode 100644
index 0000000000..f8b869808f
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/HudiTarget.java
@@ -0,0 +1,35 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class HudiTarget {
+ @JsonProperty("ConnectionName")
+ private String connectionName;
+
+ @JsonProperty("Exclusions")
+ private List<String> exclusions;
+
+ @JsonProperty("MaximumTraversalDepth")
+ private Integer maximumTraversalDepth;
+
+ @JsonProperty("Paths")
+ private List<String> paths;
+
+ public HudiTarget() {}
+
+ public String getConnectionName() { return connectionName; }
+ public void setConnectionName(String connectionName) { this.connectionName = connectionName; }
+
+ public List<String> getExclusions() { return exclusions; }
+ public void setExclusions(List<String> exclusions) { this.exclusions = exclusions; }
+
+ public Integer getMaximumTraversalDepth() { return maximumTraversalDepth; }
+ public void setMaximumTraversalDepth(Integer maximumTraversalDepth) { this.maximumTraversalDepth = maximumTraversalDepth; }
+
+ public List<String> getPaths() { return paths; }
+ public void setPaths(List<String> paths) { this.paths = paths; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/IcebergTarget.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/IcebergTarget.java
new file mode 100644
index 0000000000..4182c0e262
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/IcebergTarget.java
@@ -0,0 +1,35 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class IcebergTarget {
+ @JsonProperty("ConnectionName")
+ private String connectionName;
+
+ @JsonProperty("Exclusions")
+ private List<String> exclusions;
+
+ @JsonProperty("MaximumTraversalDepth")
+ private Integer maximumTraversalDepth;
+
+ @JsonProperty("Paths")
+ private List<String> paths;
+
+ public IcebergTarget() {}
+
+ public String getConnectionName() { return connectionName; }
+ public void setConnectionName(String connectionName) { this.connectionName = connectionName; }
+
+ public List<String> getExclusions() { return exclusions; }
+ public void setExclusions(List<String> exclusions) { this.exclusions = exclusions; }
+
+ public Integer getMaximumTraversalDepth() { return maximumTraversalDepth; }
+ public void setMaximumTraversalDepth(Integer maximumTraversalDepth) { this.maximumTraversalDepth = maximumTraversalDepth; }
+
+ public List<String> getPaths() { return paths; }
+ public void setPaths(List<String> paths) { this.paths = paths; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/JdbcTarget.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/JdbcTarget.java
new file mode 100644
index 0000000000..08ed36078f
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/JdbcTarget.java
@@ -0,0 +1,35 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class JdbcTarget {
+ @JsonProperty("ConnectionName")
+ private String connectionName;
+
+ @JsonProperty("EnableAdditionalMetadata")
+ private List<String> enableAdditionalMetadata;
+
+ @JsonProperty("Exclusions")
+ private List<String> exclusions;
+
+ @JsonProperty("Path")
+ private String path;
+
+ public JdbcTarget() {}
+
+ public String getConnectionName() { return connectionName; }
+ public void setConnectionName(String connectionName) { this.connectionName = connectionName; }
+
+ public List<String> getEnableAdditionalMetadata() { return enableAdditionalMetadata; }
+ public void setEnableAdditionalMetadata(List<String> enableAdditionalMetadata) { this.enableAdditionalMetadata = enableAdditionalMetadata; }
+
+ public List<String> getExclusions() { return exclusions; }
+ public void setExclusions(List<String> exclusions) { this.exclusions = exclusions; }
+
+ public String getPath() { return path; }
+ public void setPath(String path) { this.path = path; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java
new file mode 100644
index 0000000000..1353a4e482
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java
@@ -0,0 +1,178 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.Map;
+
+@RegisterForReflection
+public class Job {
+ @JsonProperty("AllocatedCapacity")
+ private Integer allocatedCapacity;
+
+ @JsonProperty("CodeGenConfigurationNodes")
+ private Map<String, JsonNode> codeGenConfigurationNodes;
+
+ @JsonProperty("Command")
+ private JobCommand command;
+
+ @JsonProperty("Connections")
+ private ConnectionsList connections;
+
+ @JsonProperty("CreatedOn")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant createdOn;
+
+ @JsonProperty("DefaultArguments")
+ private Map<String, String> defaultArguments;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("ExecutionClass")
+ private String executionClass;
+
+ @JsonProperty("ExecutionProperty")
+ private ExecutionProperty executionProperty;
+
+ @JsonProperty("GlueVersion")
+ private String glueVersion;
+
+ @JsonProperty("JobMode")
+ private String jobMode;
+
+ @JsonProperty("JobRunQueuingEnabled")
+ private Boolean jobRunQueuingEnabled;
+
+ @JsonProperty("LastModifiedOn")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant lastModifiedOn;
+
+ @JsonProperty("LogUri")
+ private String logUri;
+
+ @JsonProperty("MaintenanceWindow")
+ private String maintenanceWindow;
+
+ @JsonProperty("MaxCapacity")
+ private Double maxCapacity;
+
+ @JsonProperty("MaxRetries")
+ private Integer maxRetries;
+
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("NonOverridableArguments")
+ private Map<String, String> nonOverridableArguments;
+
+ @JsonProperty("NotificationProperty")
+ private NotificationProperty notificationProperty;
+
+ @JsonProperty("NumberOfWorkers")
+ private Integer numberOfWorkers;
+
+ @JsonProperty("ProfileName")
+ private String profileName;
+
+ @JsonProperty("Role")
+ private String role;
+
+ @JsonProperty("SecurityConfiguration")
+ private String securityConfiguration;
+
+ @JsonProperty("SourceControlDetails")
+ private SourceControlDetails sourceControlDetails;
+
+ @JsonProperty("Timeout")
+ private Integer timeout;
+
+ @JsonProperty("WorkerType")
+ private String workerType;
+
+ public Job() {}
+
+ public Integer getAllocatedCapacity() { return allocatedCapacity; }
+ public void setAllocatedCapacity(Integer allocatedCapacity) { this.allocatedCapacity = allocatedCapacity; }
+
+ public Map<String, JsonNode> getCodeGenConfigurationNodes() { return codeGenConfigurationNodes; }
+ public void setCodeGenConfigurationNodes(Map<String, JsonNode> codeGenConfigurationNodes) { this.codeGenConfigurationNodes = codeGenConfigurationNodes; }
+
+ public JobCommand getCommand() { return command; }
+ public void setCommand(JobCommand command) { this.command = command; }
+
+ public ConnectionsList getConnections() { return connections; }
+ public void setConnections(ConnectionsList connections) { this.connections = connections; }
+
+ public Instant getCreatedOn() { return createdOn; }
+ public void setCreatedOn(Instant createdOn) { this.createdOn = createdOn; }
+
+ public Map<String, String> getDefaultArguments() { return defaultArguments; }
+ public void setDefaultArguments(Map<String, String> defaultArguments) { this.defaultArguments = defaultArguments; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public String getExecutionClass() { return executionClass; }
+ public void setExecutionClass(String executionClass) { this.executionClass = executionClass; }
+
+ public ExecutionProperty getExecutionProperty() { return executionProperty; }
+ public void setExecutionProperty(ExecutionProperty executionProperty) { this.executionProperty = executionProperty; }
+
+ public String getGlueVersion() { return glueVersion; }
+ public void setGlueVersion(String glueVersion) { this.glueVersion = glueVersion; }
+
+ public String getJobMode() { return jobMode; }
+ public void setJobMode(String jobMode) { this.jobMode = jobMode; }
+
+ public Boolean getJobRunQueuingEnabled() { return jobRunQueuingEnabled; }
+ public void setJobRunQueuingEnabled(Boolean jobRunQueuingEnabled) { this.jobRunQueuingEnabled = jobRunQueuingEnabled; }
+
+ public Instant getLastModifiedOn() { return lastModifiedOn; }
+ public void setLastModifiedOn(Instant lastModifiedOn) { this.lastModifiedOn = lastModifiedOn; }
+
+ public String getLogUri() { return logUri; }
+ public void setLogUri(String logUri) { this.logUri = logUri; }
+
+ public String getMaintenanceWindow() { return maintenanceWindow; }
+ public void setMaintenanceWindow(String maintenanceWindow) { this.maintenanceWindow = maintenanceWindow; }
+
+ public Double getMaxCapacity() { return maxCapacity; }
+ public void setMaxCapacity(Double maxCapacity) { this.maxCapacity = maxCapacity; }
+
+ public Integer getMaxRetries() { return maxRetries; }
+ public void setMaxRetries(Integer maxRetries) { this.maxRetries = maxRetries; }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public Map<String, String> getNonOverridableArguments() { return nonOverridableArguments; }
+ public void setNonOverridableArguments(Map<String, String> nonOverridableArguments) { this.nonOverridableArguments = nonOverridableArguments; }
+
+ public NotificationProperty getNotificationProperty() { return notificationProperty; }
+ public void setNotificationProperty(NotificationProperty notificationProperty) { this.notificationProperty = notificationProperty; }
+
+ public Integer getNumberOfWorkers() { return numberOfWorkers; }
+ public void setNumberOfWorkers(Integer numberOfWorkers) { this.numberOfWorkers = numberOfWorkers; }
+
+ public String getProfileName() { return profileName; }
+ public void setProfileName(String profileName) { this.profileName = profileName; }
+
+ public String getRole() { return role; }
+ public void setRole(String role) { this.role = role; }
+
+ public String getSecurityConfiguration() { return securityConfiguration; }
+ public void setSecurityConfiguration(String securityConfiguration) { this.securityConfiguration = securityConfiguration; }
+
+ public SourceControlDetails getSourceControlDetails() { return sourceControlDetails; }
+ public void setSourceControlDetails(SourceControlDetails sourceControlDetails) { this.sourceControlDetails = sourceControlDetails; }
+
+ public Integer getTimeout() { return timeout; }
+ public void setTimeout(Integer timeout) { this.timeout = timeout; }
+
+ public String getWorkerType() { return workerType; }
+ public void setWorkerType(String workerType) { this.workerType = workerType; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/JobCommand.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/JobCommand.java
new file mode 100644
index 0000000000..7d5ae7cefd
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/JobCommand.java
@@ -0,0 +1,33 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class JobCommand {
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("PythonVersion")
+ private String pythonVersion;
+
+ @JsonProperty("Runtime")
+ private String runtime;
+
+ @JsonProperty("ScriptLocation")
+ private String scriptLocation;
+
+ public JobCommand() {}
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public String getPythonVersion() { return pythonVersion; }
+ public void setPythonVersion(String pythonVersion) { this.pythonVersion = pythonVersion; }
+
+ public String getRuntime() { return runtime; }
+ public void setRuntime(String runtime) { this.runtime = runtime; }
+
+ public String getScriptLocation() { return scriptLocation; }
+ public void setScriptLocation(String scriptLocation) { this.scriptLocation = scriptLocation; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/JobUpdate.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/JobUpdate.java
new file mode 100644
index 0000000000..ef3eb9b83a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/JobUpdate.java
@@ -0,0 +1,150 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.Map;
+
+@RegisterForReflection
+public class JobUpdate {
+ @JsonProperty("AllocatedCapacity")
+ private Integer allocatedCapacity;
+
+ @JsonProperty("CodeGenConfigurationNodes")
+ private Map<String, JsonNode> codeGenConfigurationNodes;
+
+ @JsonProperty("Command")
+ private JobCommand command;
+
+ @JsonProperty("Connections")
+ private ConnectionsList connections;
+
+ @JsonProperty("DefaultArguments")
+ private Map<String, String> defaultArguments;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("ExecutionClass")
+ private String executionClass;
+
+ @JsonProperty("ExecutionProperty")
+ private ExecutionProperty executionProperty;
+
+ @JsonProperty("GlueVersion")
+ private String glueVersion;
+
+ @JsonProperty("JobMode")
+ private String jobMode;
+
+ @JsonProperty("JobRunQueuingEnabled")
+ private Boolean jobRunQueuingEnabled;
+
+ @JsonProperty("LogUri")
+ private String logUri;
+
+ @JsonProperty("MaintenanceWindow")
+ private String maintenanceWindow;
+
+ @JsonProperty("MaxCapacity")
+ private Double maxCapacity;
+
+ @JsonProperty("MaxRetries")
+ private Integer maxRetries;
+
+ @JsonProperty("NonOverridableArguments")
+ private Map<String, String> nonOverridableArguments;
+
+ @JsonProperty("NotificationProperty")
+ private NotificationProperty notificationProperty;
+
+ @JsonProperty("NumberOfWorkers")
+ private Integer numberOfWorkers;
+
+ @JsonProperty("Role")
+ private String role;
+
+ @JsonProperty("SecurityConfiguration")
+ private String securityConfiguration;
+
+ @JsonProperty("SourceControlDetails")
+ private SourceControlDetails sourceControlDetails;
+
+ @JsonProperty("Timeout")
+ private Integer timeout;
+
+ @JsonProperty("WorkerType")
+ private String workerType;
+
+ public JobUpdate() {}
+
+ public Integer getAllocatedCapacity() { return allocatedCapacity; }
+ public void setAllocatedCapacity(Integer allocatedCapacity) { this.allocatedCapacity = allocatedCapacity; }
+
+ public Map<String, JsonNode> getCodeGenConfigurationNodes() { return codeGenConfigurationNodes; }
+ public void setCodeGenConfigurationNodes(Map<String, JsonNode> codeGenConfigurationNodes) { this.codeGenConfigurationNodes = codeGenConfigurationNodes; }
+
+ public JobCommand getCommand() { return command; }
+ public void setCommand(JobCommand command) { this.command = command; }
+
+ public ConnectionsList getConnections() { return connections; }
+ public void setConnections(ConnectionsList connections) { this.connections = connections; }
+
+ public Map<String, String> getDefaultArguments() { return defaultArguments; }
+ public void setDefaultArguments(Map<String, String> defaultArguments) { this.defaultArguments = defaultArguments; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public String getExecutionClass() { return executionClass; }
+ public void setExecutionClass(String executionClass) { this.executionClass = executionClass; }
+
+ public ExecutionProperty getExecutionProperty() { return executionProperty; }
+ public void setExecutionProperty(ExecutionProperty executionProperty) { this.executionProperty = executionProperty; }
+
+ public String getGlueVersion() { return glueVersion; }
+ public void setGlueVersion(String glueVersion) { this.glueVersion = glueVersion; }
+
+ public String getJobMode() { return jobMode; }
+ public void setJobMode(String jobMode) { this.jobMode = jobMode; }
+
+ public Boolean getJobRunQueuingEnabled() { return jobRunQueuingEnabled; }
+ public void setJobRunQueuingEnabled(Boolean jobRunQueuingEnabled) { this.jobRunQueuingEnabled = jobRunQueuingEnabled; }
+
+ public String getLogUri() { return logUri; }
+ public void setLogUri(String logUri) { this.logUri = logUri; }
+
+ public String getMaintenanceWindow() { return maintenanceWindow; }
+ public void setMaintenanceWindow(String maintenanceWindow) { this.maintenanceWindow = maintenanceWindow; }
+
+ public Double getMaxCapacity() { return maxCapacity; }
+ public void setMaxCapacity(Double maxCapacity) { this.maxCapacity = maxCapacity; }
+
+ public Integer getMaxRetries() { return maxRetries; }
+ public void setMaxRetries(Integer maxRetries) { this.maxRetries = maxRetries; }
+
+ public Map<String, String> getNonOverridableArguments() { return nonOverridableArguments; }
+ public void setNonOverridableArguments(Map<String, String> nonOverridableArguments) { this.nonOverridableArguments = nonOverridableArguments; }
+
+ public NotificationProperty getNotificationProperty() { return notificationProperty; }
+ public void setNotificationProperty(NotificationProperty notificationProperty) { this.notificationProperty = notificationProperty; }
+
+ public Integer getNumberOfWorkers() { return numberOfWorkers; }
+ public void setNumberOfWorkers(Integer numberOfWorkers) { this.numberOfWorkers = numberOfWorkers; }
+
+ public String getRole() { return role; }
+ public void setRole(String role) { this.role = role; }
+
+ public String getSecurityConfiguration() { return securityConfiguration; }
+ public void setSecurityConfiguration(String securityConfiguration) { this.securityConfiguration = securityConfiguration; }
+
+ public SourceControlDetails getSourceControlDetails() { return sourceControlDetails; }
+ public void setSourceControlDetails(SourceControlDetails sourceControlDetails) { this.sourceControlDetails = sourceControlDetails; }
+
+ public Integer getTimeout() { return timeout; }
+ public void setTimeout(Integer timeout) { this.timeout = timeout; }
+
+ public String getWorkerType() { return workerType; }
+ public void setWorkerType(String workerType) { this.workerType = workerType; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/LakeFormationConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/LakeFormationConfiguration.java
new file mode 100644
index 0000000000..994322b03b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/LakeFormationConfiguration.java
@@ -0,0 +1,21 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class LakeFormationConfiguration {
+ @JsonProperty("AccountId")
+ private String accountId;
+
+ @JsonProperty("UseLakeFormationCredentials")
+ private Boolean useLakeFormationCredentials;
+
+ public LakeFormationConfiguration() {}
+
+ public String getAccountId() { return accountId; }
+ public void setAccountId(String accountId) { this.accountId = accountId; }
+
+ public Boolean getUseLakeFormationCredentials() { return useLakeFormationCredentials; }
+ public void setUseLakeFormationCredentials(Boolean useLakeFormationCredentials) { this.useLakeFormationCredentials = useLakeFormationCredentials; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java
new file mode 100644
index 0000000000..7a18eea816
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java
@@ -0,0 +1,48 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
+
+@RegisterForReflection
+public class LastCrawlInfo {
+ @JsonProperty("ErrorMessage")
+ private String errorMessage;
+
+ @JsonProperty("LogGroup")
+ private String logGroup;
+
+ @JsonProperty("LogStream")
+ private String logStream;
+
+ @JsonProperty("MessagePrefix")
+ private String messagePrefix;
+
+ @JsonProperty("StartTime")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant startTime;
+
+ @JsonProperty("Status")
+ private String status;
+
+ public LastCrawlInfo() {}
+
+ public String getErrorMessage() { return errorMessage; }
+ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
+
+ public String getLogGroup() { return logGroup; }
+ public void setLogGroup(String logGroup) { this.logGroup = logGroup; }
+
+ public String getLogStream() { return logStream; }
+ public void setLogStream(String logStream) { this.logStream = logStream; }
+
+ public String getMessagePrefix() { return messagePrefix; }
+ public void setMessagePrefix(String messagePrefix) { this.messagePrefix = messagePrefix; }
+
+ public Instant getStartTime() { return startTime; }
+ public void setStartTime(Instant startTime) { this.startTime = startTime; }
+
+ public String getStatus() { return status; }
+ public void setStatus(String status) { this.status = status; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/LineageConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/LineageConfiguration.java
new file mode 100644
index 0000000000..7626496899
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/LineageConfiguration.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class LineageConfiguration {
+ @JsonProperty("CrawlerLineageSettings")
+ private String crawlerLineageSettings;
+
+ public LineageConfiguration() {}
+
+ public String getCrawlerLineageSettings() { return crawlerLineageSettings; }
+ public void setCrawlerLineageSettings(String crawlerLineageSettings) { this.crawlerLineageSettings = crawlerLineageSettings; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/MongoDBTarget.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/MongoDBTarget.java
new file mode 100644
index 0000000000..b460a3dae8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/MongoDBTarget.java
@@ -0,0 +1,27 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class MongoDBTarget {
+ @JsonProperty("ConnectionName")
+ private String connectionName;
+
+ @JsonProperty("Path")
+ private String path;
+
+ @JsonProperty("ScanAll")
+ private Boolean scanAll;
+
+ public MongoDBTarget() {}
+
+ public String getConnectionName() { return connectionName; }
+ public void setConnectionName(String connectionName) { this.connectionName = connectionName; }
+
+ public String getPath() { return path; }
+ public void setPath(String path) { this.path = path; }
+
+ public Boolean getScanAll() { return scanAll; }
+ public void setScanAll(Boolean scanAll) { this.scanAll = scanAll; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/NotificationProperty.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/NotificationProperty.java
new file mode 100644
index 0000000000..1408a681d3
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/NotificationProperty.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class NotificationProperty {
+ @JsonProperty("NotifyDelayAfter")
+ private Integer notifyDelayAfter;
+
+ public NotificationProperty() {}
+
+ public Integer getNotifyDelayAfter() { return notifyDelayAfter; }
+ public void setNotifyDelayAfter(Integer notifyDelayAfter) { this.notifyDelayAfter = notifyDelayAfter; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/RecrawlPolicy.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/RecrawlPolicy.java
new file mode 100644
index 0000000000..19627df831
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/RecrawlPolicy.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class RecrawlPolicy {
+ @JsonProperty("RecrawlBehavior")
+ private String recrawlBehavior;
+
+ public RecrawlPolicy() {}
+
+ public String getRecrawlBehavior() { return recrawlBehavior; }
+ public void setRecrawlBehavior(String recrawlBehavior) { this.recrawlBehavior = recrawlBehavior; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/S3Target.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/S3Target.java
new file mode 100644
index 0000000000..08e7adcd9d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/S3Target.java
@@ -0,0 +1,47 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class S3Target {
+ @JsonProperty("ConnectionName")
+ private String connectionName;
+
+ @JsonProperty("DlqEventQueueArn")
+ private String dlqEventQueueArn;
+
+ @JsonProperty("EventQueueArn")
+ private String eventQueueArn;
+
+ @JsonProperty("Exclusions")
+ private List<String> exclusions;
+
+ @JsonProperty("Path")
+ private String path;
+
+ @JsonProperty("SampleSize")
+ private Integer sampleSize;
+
+ public S3Target() {}
+
+ public String getConnectionName() { return connectionName; }
+ public void setConnectionName(String connectionName) { this.connectionName = connectionName; }
+
+ public String getDlqEventQueueArn() { return dlqEventQueueArn; }
+ public void setDlqEventQueueArn(String dlqEventQueueArn) { this.dlqEventQueueArn = dlqEventQueueArn; }
+
+ public String getEventQueueArn() { return eventQueueArn; }
+ public void setEventQueueArn(String eventQueueArn) { this.eventQueueArn = eventQueueArn; }
+
+ public List<String> getExclusions() { return exclusions; }
+ public void setExclusions(List<String> exclusions) { this.exclusions = exclusions; }
+
+ public String getPath() { return path; }
+ public void setPath(String path) { this.path = path; }
+
+ public Integer getSampleSize() { return sampleSize; }
+ public void setSampleSize(Integer sampleSize) { this.sampleSize = sampleSize; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Schedule.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Schedule.java
new file mode 100644
index 0000000000..ff03c2fe01
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Schedule.java
@@ -0,0 +1,21 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class Schedule {
+ @JsonProperty("ScheduleExpression")
+ private String scheduleExpression;
+
+ @JsonProperty("State")
+ private String state;
+
+ public Schedule() {}
+
+ public String getScheduleExpression() { return scheduleExpression; }
+ public void setScheduleExpression(String scheduleExpression) { this.scheduleExpression = scheduleExpression; }
+
+ public String getState() { return state; }
+ public void setState(String state) { this.state = state; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/SchemaChangePolicy.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/SchemaChangePolicy.java
new file mode 100644
index 0000000000..bdbc2f4cd1
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/SchemaChangePolicy.java
@@ -0,0 +1,21 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class SchemaChangePolicy {
+ @JsonProperty("DeleteBehavior")
+ private String deleteBehavior;
+
+ @JsonProperty("UpdateBehavior")
+ private String updateBehavior;
+
+ public SchemaChangePolicy() {}
+
+ public String getDeleteBehavior() { return deleteBehavior; }
+ public void setDeleteBehavior(String deleteBehavior) { this.deleteBehavior = deleteBehavior; }
+
+ public String getUpdateBehavior() { return updateBehavior; }
+ public void setUpdateBehavior(String updateBehavior) { this.updateBehavior = updateBehavior; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/SourceControlDetails.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/SourceControlDetails.java
new file mode 100644
index 0000000000..9294dfd520
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/SourceControlDetails.java
@@ -0,0 +1,57 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class SourceControlDetails {
+ @JsonProperty("AuthStrategy")
+ private String authStrategy;
+
+ @JsonProperty("AuthToken")
+ private String authToken;
+
+ @JsonProperty("Branch")
+ private String branch;
+
+ @JsonProperty("Folder")
+ private String folder;
+
+ @JsonProperty("LastCommitId")
+ private String lastCommitId;
+
+ @JsonProperty("Owner")
+ private String owner;
+
+ @JsonProperty("Provider")
+ private String provider;
+
+ @JsonProperty("Repository")
+ private String repository;
+
+ public SourceControlDetails() {}
+
+ public String getAuthStrategy() { return authStrategy; }
+ public void setAuthStrategy(String authStrategy) { this.authStrategy = authStrategy; }
+
+ public String getAuthToken() { return authToken; }
+ public void setAuthToken(String authToken) { this.authToken = authToken; }
+
+ public String getBranch() { return branch; }
+ public void setBranch(String branch) { this.branch = branch; }
+
+ public String getFolder() { return folder; }
+ public void setFolder(String folder) { this.folder = folder; }
+
+ public String getLastCommitId() { return lastCommitId; }
+ public void setLastCommitId(String lastCommitId) { this.lastCommitId = lastCommitId; }
+
+ public String getOwner() { return owner; }
+ public void setOwner(String owner) { this.owner = owner; }
+
+ public String getProvider() { return provider; }
+ public void setProvider(String provider) { this.provider = provider; }
+
+ public String getRepository() { return repository; }
+ public void setRepository(String repository) { this.repository = repository; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateCrawlerRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateCrawlerRequest.java
new file mode 100644
index 0000000000..17afc758ac
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateCrawlerRequest.java
@@ -0,0 +1,95 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+public class UpdateCrawlerRequest {
+ @JsonProperty("Classifiers")
+ private List<String> classifiers;
+
+ @JsonProperty("Configuration")
+ private String configuration;
+
+ @JsonProperty("CrawlerSecurityConfiguration")
+ private String crawlerSecurityConfiguration;
+
+ @JsonProperty("DatabaseName")
+ private String databaseName;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("LakeFormationConfiguration")
+ private LakeFormationConfiguration lakeFormationConfiguration;
+
+ @JsonProperty("LineageConfiguration")
+ private LineageConfiguration lineageConfiguration;
+
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("RecrawlPolicy")
+ private RecrawlPolicy recrawlPolicy;
+
+ @JsonProperty("Role")
+ private String role;
+
+ @JsonProperty("Schedule")
+ private String schedule;
+
+ @JsonProperty("SchemaChangePolicy")
+ private SchemaChangePolicy schemaChangePolicy;
+
+ @JsonProperty("TablePrefix")
+ private String tablePrefix;
+
+ @JsonProperty("Targets")
+ private CrawlerTargets targets;
+
+ public UpdateCrawlerRequest() {}
+
+ public List<String> getClassifiers() { return classifiers; }
+ public void setClassifiers(List<String> classifiers) { this.classifiers = classifiers; }
+
+ public String getConfiguration() { return configuration; }
+ public void setConfiguration(String configuration) { this.configuration = configuration; }
+
+ public String getCrawlerSecurityConfiguration() { return crawlerSecurityConfiguration; }
+ public void setCrawlerSecurityConfiguration(String crawlerSecurityConfiguration) { this.crawlerSecurityConfiguration = crawlerSecurityConfiguration; }
+
+ public String getDatabaseName() { return databaseName; }
+ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public LakeFormationConfiguration getLakeFormationConfiguration() { return lakeFormationConfiguration; }
+ public void setLakeFormationConfiguration(LakeFormationConfiguration lakeFormationConfiguration) { this.lakeFormationConfiguration = lakeFormationConfiguration; }
+
+ public LineageConfiguration getLineageConfiguration() { return lineageConfiguration; }
+ public void setLineageConfiguration(LineageConfiguration lineageConfiguration) { this.lineageConfiguration = lineageConfiguration; }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public RecrawlPolicy getRecrawlPolicy() { return recrawlPolicy; }
+ public void setRecrawlPolicy(RecrawlPolicy recrawlPolicy) { this.recrawlPolicy = recrawlPolicy; }
+
+ public String getRole() { return role; }
+ public void setRole(String role) { this.role = role; }
+
+ public String getSchedule() { return schedule; }
+ public void setSchedule(String schedule) { this.schedule = schedule; }
+
+ public SchemaChangePolicy getSchemaChangePolicy() { return schemaChangePolicy; }
+ public void setSchemaChangePolicy(SchemaChangePolicy schemaChangePolicy) { this.schemaChangePolicy = schemaChangePolicy; }
+
+ public String getTablePrefix() { return tablePrefix; }
+ public void setTablePrefix(String tablePrefix) { this.tablePrefix = tablePrefix; }
+
+ public CrawlerTargets getTargets() { return targets; }
+ public void setTargets(CrawlerTargets targets) { this.targets = targets; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateJobRequest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateJobRequest.java
new file mode 100644
index 0000000000..f5308031bb
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateJobRequest.java
@@ -0,0 +1,21 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class UpdateJobRequest {
+ @JsonProperty("JobName")
+ private String jobName;
+
+ @JsonProperty("JobUpdate")
+ private JobUpdate jobUpdate;
+
+ public UpdateJobRequest() {}
+
+ public String getJobName() { return jobName; }
+ public void setJobName(String jobName) { this.jobName = jobName; }
+
+ public JobUpdate getJobUpdate() { return jobUpdate; }
+ public void setJobUpdate(JobUpdate jobUpdate) { this.jobUpdate = jobUpdate; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateJobResponse.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateJobResponse.java
new file mode 100644
index 0000000000..0591cdd85c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/UpdateJobResponse.java
@@ -0,0 +1,19 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class UpdateJobResponse {
+ @JsonProperty("JobName")
+ private String jobName;
+
+ public UpdateJobResponse() {}
+
+ public UpdateJobResponse(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public String getJobName() { return jobName; }
+ public void setJobName(String jobName) { this.jobName = jobName; }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
index 47beb213b3..6188bff544 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
@@ -22,7 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Verifies the wire-accurate empty-list responses for the read-only Glue actions on resources
- * the emulator does not model (GetJobs, GetCrawlers, ListDataQualityRulesets, GetSecurityConfigurations).
+ * the emulator does not model (ListDataQualityRulesets, GetSecurityConfigurations).
* Each must return HTTP 200, an empty list under its result key, and omit NextToken.
*/
class GlueJsonHandlerEmptyListTest {
@@ -45,16 +45,6 @@ class GlueJsonHandlerEmptyListTest {
handler = new GlueJsonHandler(glueService, schemaRegistryService, mapper);
}
- @Test
- void getJobsReturnsEmptyJobsList() throws Exception {
- assertEmptyList("GetJobs", "Jobs");
- }
-
- @Test
- void getCrawlersReturnsEmptyCrawlersList() throws Exception {
- assertEmptyList("GetCrawlers", "Crawlers");
- }
-
@Test
void listDataQualityRulesetsReturnsEmptyRulesetsList() throws Exception {
assertEmptyList("ListDataQualityRulesets", "Rulesets");
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
new file mode 100644
index 0000000000..b045323554
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
@@ -0,0 +1,133 @@
+package io.github.hectorvent.floci.services.glue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;
+import jakarta.ws.rs.core.Response;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class GlueJsonHandlerTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT_ID = "000000000000";
+
+ private GlueJsonHandler handler;
+ private ObjectMapper mapper;
+
+ @BeforeEach
+ void setUp() {
+ mapper = new ObjectMapper();
+ mapper.registerModule(new com.fasterxml.jackson.datatype.jsr310.JavaTimeModule());
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);
+ StorageFactory storageFactory = new InMemoryStorageFactory();
+ GlueSchemaRegistryService schemaRegistryService =
+ new GlueSchemaRegistryService(storageFactory, regionResolver);
+ GlueService glueService = new GlueService(
+ storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory));
+ handler = new GlueJsonHandler(glueService, schemaRegistryService, mapper);
+ }
+
+ @Test
+ void createCrawlerWithScheduleSucceeds() throws Exception {
+ ObjectNode request = mapper.createObjectNode();
+ request.put("Name", "test-crawler");
+ request.put("Role", "arn:aws:iam::000000000000:role/role");
+ request.put("Schedule", "cron(15 12 * * ? *)");
+
+ ObjectNode targets = request.putObject("Targets");
+ targets.putArray("S3Targets").addObject().put("Path", "s3://bucket/path/");
+
+ Response response = handler.handle("CreateCrawler", request, REGION);
+ assertEquals(200, response.getStatus());
+
+ JsonNode body = mapper.valueToTree(response.getEntity());
+ assertNotNull(body);
+
+ Response getResponse = handler.handle("GetCrawler", mapper.createObjectNode().put("Name", "test-crawler"), REGION);
+ assertEquals(200, getResponse.getStatus());
+ JsonNode getBody = mapper.valueToTree(getResponse.getEntity());
+ assertTrue(getBody.has("Crawler"));
+ JsonNode crawler = getBody.get("Crawler");
+ assertEquals("test-crawler", crawler.get("Name").asText());
+ assertEquals("READY", crawler.get("State").asText());
+ assertEquals(1, crawler.get("Version").asInt());
+ assertTrue(crawler.has("Schedule"));
+ assertEquals("cron(15 12 * * ? *)", crawler.get("Schedule").get("ScheduleExpression").asText());
+ }
+
+ @Test
+ void updateCrawlerWithScheduleSucceeds() throws Exception {
+ ObjectNode request = mapper.createObjectNode();
+ request.put("Name", "test-crawler-update");
+ request.put("Role", "arn:aws:iam::000000000000:role/role");
+
+ ObjectNode targets = request.putObject("Targets");
+ targets.putArray("S3Targets").addObject().put("Path", "s3://bucket/path/");
+
+ assertEquals(200, handler.handle("CreateCrawler", request, REGION).getStatus());
+
+ ObjectNode updateRequest = mapper.createObjectNode();
+ updateRequest.put("Name", "test-crawler-update");
+ updateRequest.put("Schedule", "cron(0 0 * * ? *)");
+
+ Response response = handler.handle("UpdateCrawler", updateRequest, REGION);
+ assertEquals(200, response.getStatus());
+
+ Response getResponse = handler.handle("GetCrawler", mapper.createObjectNode().put("Name", "test-crawler-update"), REGION);
+ assertEquals(200, getResponse.getStatus());
+ JsonNode getBody = mapper.valueToTree(getResponse.getEntity());
+ JsonNode crawler = getBody.get("Crawler");
+ assertEquals("test-crawler-update", crawler.get("Name").asText());
+ assertEquals(2, crawler.get("Version").asInt());
+ assertTrue(crawler.has("Schedule"));
+ assertEquals("cron(0 0 * * ? *)", crawler.get("Schedule").get("ScheduleExpression").asText());
+ }
+
+ @Test
+ void createJobSucceeds() throws Exception {
+ ObjectNode request = mapper.createObjectNode();
+ request.put("Name", "test-job");
+ request.put("Role", "arn:aws:iam::000000000000:role/role");
+
+ ObjectNode command = request.putObject("Command");
+ command.put("Name", "glueetl");
+ command.put("ScriptLocation", "s3://bucket/script.py");
+
+ Response response = handler.handle("CreateJob", request, REGION);
+ assertEquals(200, response.getStatus());
+
+ Response getResponse = handler.handle("GetJob", mapper.createObjectNode().put("JobName", "test-job"), REGION);
+ assertEquals(200, getResponse.getStatus());
+ JsonNode getBody = mapper.valueToTree(getResponse.getEntity());
+ assertTrue(getBody.has("Job"));
+ JsonNode job = getBody.get("Job");
+ assertEquals("test-job", job.get("Name").asText());
+ assertEquals("glueetl", job.get("Command").get("Name").asText());
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return AccountAwareStorageBackend.inMemory("000000000000");
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index 865946c84c..d104c414e5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -9,7 +9,13 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.glue.model.Column;
import io.github.hectorvent.floci.services.glue.model.Database;
+import io.github.hectorvent.floci.services.glue.model.Crawler;
+import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;
+import io.github.hectorvent.floci.services.glue.model.Job;
+import io.github.hectorvent.floci.services.glue.model.JobCommand;
+import io.github.hectorvent.floci.services.glue.model.JobUpdate;
import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.S3Target;
import io.github.hectorvent.floci.services.glue.model.SchemaReference;
import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
import io.github.hectorvent.floci.services.glue.model.Table;
@@ -70,6 +76,8 @@ class GlueServiceTest {
partitionStore,
new InMemoryStorage<String, Map<String, Object>>(),
new InMemoryStorage<String, UserDefinedFunction>(),
+ new InMemoryStorage<String, Job>(),
+ new InMemoryStorage<String, Crawler>(),
schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null));
glueService.createDatabase(new Database("db1"));
}
@@ -1039,4 +1047,195 @@ class GlueServiceTest {
return AccountAwareStorageBackend.inMemory("000000000000");
}
}
+
+ private Job createValidJob(String name) {
+ Job job = new Job();
+ job.setName(name);
+ job.setRole("arn:aws:iam::000000000000:role/my-role");
+ job.setCommand(new JobCommand());
+ return job;
+ }
+
+ private Crawler createValidCrawler(String name) {
+ Crawler crawler = new Crawler();
+ crawler.setName(name);
+ crawler.setRole("arn:aws:iam::000000000000:role/my-role");
+ CrawlerTargets targets = new CrawlerTargets();
+ targets.setS3Targets(java.util.List.of(new S3Target()));
+ crawler.setTargets(targets);
+ return crawler;
+ }
+
+ @Test
+ void jobCanBeCreatedFetchedUpdatedAndDeleted() {
+ Job job = createValidJob("my-job");
+ job.setDescription("Original description");
+
+ glueService.createJob(job);
+
+ Job fetched = glueService.getJob("my-job");
+ assertEquals("my-job", fetched.getName());
+ assertEquals("Original description", fetched.getDescription());
+ assertNotNull(fetched.getCreatedOn());
+ assertEquals(fetched.getCreatedOn(), fetched.getLastModifiedOn());
+
+ assertEquals(1, glueService.getJobs().size());
+ assertEquals(1, glueService.getJobs(10, null).items().size());
+
+ JobUpdate update = new JobUpdate();
+ update.setDescription("Updated description");
+ update.setRole("arn:aws:iam::000000000000:role/new-role");
+
+ glueService.updateJob("my-job", update);
+
+ Job updated = glueService.getJob("my-job");
+ assertEquals("Updated description", updated.getDescription());
+ assertEquals("arn:aws:iam::000000000000:role/new-role", updated.getRole());
+ assertTrue(updated.getLastModifiedOn().isAfter(fetched.getCreatedOn()) || updated.getLastModifiedOn().equals(fetched.getCreatedOn()));
+
+ glueService.deleteJob("my-job", REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () -> glueService.getJob("my-job"));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ assertTrue(glueService.getJobs().isEmpty());
+ }
+
+ @Test
+ void crawlerCanBeCreatedFetchedUpdatedAndDeleted() {
+ Crawler crawler = createValidCrawler("my-crawler");
+ crawler.setDescription("Original crawler");
+ crawler.setDatabaseName("db1");
+
+ glueService.createCrawler(crawler);
+
+ Crawler fetched = glueService.getCrawler("my-crawler");
+ assertEquals("my-crawler", fetched.getName());
+ assertEquals("Original crawler", fetched.getDescription());
+ assertEquals("db1", fetched.getDatabaseName());
+ assertNotNull(fetched.getCreationTime());
+ assertEquals(fetched.getCreationTime(), fetched.getLastUpdated());
+
+ assertEquals(1, glueService.getCrawlers().size());
+ assertEquals(1, glueService.getCrawlers(10, null).items().size());
+
+ Crawler update = new Crawler();
+ update.setName("my-crawler");
+ update.setDescription("Updated crawler");
+ update.setDatabaseName("db2");
+
+ glueService.updateCrawler(update);
+
+ Crawler updated = glueService.getCrawler("my-crawler");
+ assertEquals("Updated crawler", updated.getDescription());
+ assertEquals("db2", updated.getDatabaseName());
+ assertTrue(updated.getLastUpdated().isAfter(fetched.getCreationTime()) || updated.getLastUpdated().equals(fetched.getCreationTime()));
+
+ glueService.deleteCrawler("my-crawler", REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () -> glueService.getCrawler("my-crawler"));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ assertTrue(glueService.getCrawlers().isEmpty());
+ }
+
+ @Test
+ void resourceTaggingWorksForJobsAndCrawlers() {
+ Job job = createValidJob("tagged-job");
+ glueService.createJob(job, Map.of("key1", "value1"), REGION);
+
+ String jobArn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":job/tagged-job";
+ Map<String, String> tags = glueService.getTags(jobArn, REGION);
+ assertEquals("value1", tags.get("key1"));
+
+ glueService.tagResource(jobArn, Map.of("key2", "value2"), REGION);
+ Map<String, String> updatedTags = glueService.getTags(jobArn, REGION);
+ assertEquals("value1", updatedTags.get("key1"));
+ assertEquals("value2", updatedTags.get("key2"));
+
+ glueService.untagResource(jobArn, List.of("key1"), REGION);
+ Map<String, String> finalTags = glueService.getTags(jobArn, REGION);
+ assertNull(finalTags.get("key1"));
+ assertEquals("value2", finalTags.get("key2"));
+
+ Crawler crawler = createValidCrawler("tagged-crawler");
+ glueService.createCrawler(crawler, Map.of("env", "prod"), REGION);
+
+ String crawlerArn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":crawler/tagged-crawler";
+ Map<String, String> crawlerTags = glueService.getTags(crawlerArn, REGION);
+ assertEquals("prod", crawlerTags.get("env"));
+ }
+
+ @Test
+ void jobAndCrawlerRejectsDuplicateCreation() {
+ Job job = createValidJob("dup-job");
+ glueService.createJob(job);
+
+ AwsException jobEx = assertThrows(AwsException.class, () -> glueService.createJob(job));
+ assertEquals("AlreadyExistsException", jobEx.getErrorCode());
+
+ Crawler crawler = createValidCrawler("dup-crawler");
+ glueService.createCrawler(crawler);
+
+ AwsException crawlerEx = assertThrows(AwsException.class, () -> glueService.createCrawler(crawler));
+ assertEquals("AlreadyExistsException", crawlerEx.getErrorCode());
+ }
+
+ @Test
+ void getJobsAndCrawlersPagination() {
+ for (int i = 0; i < 5; i++) {
+ Job job = createValidJob("job-" + i);
+ glueService.createJob(job);
+
+ Crawler crawler = createValidCrawler("crawler-" + i);
+ glueService.createCrawler(crawler);
+ }
+
+ GlueService.Page<Job> jobsPage = glueService.getJobs(2, null);
+ assertEquals(2, jobsPage.items().size());
+ assertNotNull(jobsPage.nextToken());
+
+ GlueService.Page<Job> jobsNextPage = glueService.getJobs(10, jobsPage.nextToken());
+ assertEquals(3, jobsNextPage.items().size());
+ assertNull(jobsNextPage.nextToken());
+
+ GlueService.Page<Crawler> crawlersPage = glueService.getCrawlers(3, null);
+ assertEquals(3, crawlersPage.items().size());
+ assertNotNull(crawlersPage.nextToken());
+
+ GlueService.Page<Crawler> crawlersNextPage = glueService.getCrawlers(10, crawlersPage.nextToken());
+ assertEquals(2, crawlersNextPage.items().size());
+ assertNull(crawlersNextPage.nextToken());
+
+ AwsException negativeMaxResultsJobs = assertThrows(AwsException.class, () -> glueService.getJobs(-1, null));
+ assertEquals("InvalidInputException", negativeMaxResultsJobs.getErrorCode());
+
+ AwsException zeroMaxResultsCrawlers = assertThrows(AwsException.class, () -> glueService.getCrawlers(0, null));
+ assertEquals("InvalidInputException", zeroMaxResultsCrawlers.getErrorCode());
+
+ AwsException hugeMaxResultsJobs = assertThrows(AwsException.class, () -> glueService.getJobs(1001, null));
+ assertEquals("InvalidInputException", hugeMaxResultsJobs.getErrorCode());
+ }
+
+ @Test
+ void taggingNonExistentResourceThrows() {
+ String fakeJobArn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":job/fake-job";
+ String fakeCrawlerArn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":crawler/fake-crawler";
+ String fakeDbArn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":database/fake-db";
+ String fakeTableArn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":table/fake-db/fake-table";
+ String fakeUdfArn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":userDefinedFunction/fake-db/fake-udf";
+
+ AwsException jobTagEx = assertThrows(AwsException.class, () -> glueService.tagResource(fakeJobArn, Map.of("k", "v"), REGION));
+ assertEquals("EntityNotFoundException", jobTagEx.getErrorCode());
+
+ AwsException crawlerGetTagsEx = assertThrows(AwsException.class, () -> glueService.getTags(fakeCrawlerArn, REGION));
+ assertEquals("EntityNotFoundException", crawlerGetTagsEx.getErrorCode());
+
+ AwsException dbTagEx = assertThrows(AwsException.class, () -> glueService.tagResource(fakeDbArn, Map.of("k", "v"), REGION));
+ assertEquals("EntityNotFoundException", dbTagEx.getErrorCode());
+
+ AwsException tableTagEx = assertThrows(AwsException.class, () -> glueService.tagResource(fakeTableArn, Map.of("k", "v"), REGION));
+ assertEquals("EntityNotFoundException", tableTagEx.getErrorCode());
+
+ AwsException udfTagEx = assertThrows(AwsException.class, () -> glueService.tagResource(fakeUdfArn, Map.of("k", "v"), REGION));
+ assertEquals("EntityNotFoundException", udfTagEx.getErrorCode());
+ }
}