Light, fluffy, and always free - The AWS Local Emulator alternative
feat(glue): start and stop crawls, crawler metrics, listing and schedules (#4427)
David93c7b1f93fd7
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCrawlerRunTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCrawlerRunTest.javanew file mode 100644index 0000000000..6932e3dbbf--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCrawlerRunTest.java@@ -0,0 +1,97 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.glue.GlueClient;+import software.amazon.awssdk.services.glue.model.BatchGetCrawlersResponse;+import software.amazon.awssdk.services.glue.model.Crawler;+import software.amazon.awssdk.services.glue.model.CrawlerMetrics;+import software.amazon.awssdk.services.glue.model.CrawlerNotRunningException;+import software.amazon.awssdk.services.glue.model.CrawlerState;+import software.amazon.awssdk.services.glue.model.LastCrawlStatus;+import software.amazon.awssdk.services.glue.model.ScheduleState;++import java.util.Map;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++@DisplayName("Glue crawler runs")+class GlueCrawlerRunTest {++ private static final String CRAWLER_NAME = TestFixtures.uniqueName("run_crawler");+ private static final String ROLE = "arn:aws:iam::000000000000:role/glue";++ private static GlueClient glue;++ @BeforeAll+ static void setup() {+ glue = TestFixtures.glueClient();+ glue.createCrawler(r -> r+ .name(CRAWLER_NAME)+ .role(ROLE)+ .targets(t -> t.s3Targets(s -> s.path("s3://raw/events")))+ .schedule("cron(0 2 * * ? *)")+ .tags(Map.of("team", "data")));+ }++ @AfterAll+ static void cleanup() {+ if (glue == null) {+ return;+ }+ try {+ glue.deleteCrawler(r -> r.name(CRAWLER_NAME));+ }+ catch (Exception ignored) {+ // Cleanup is best effort: the crawler name is unique to this run and the emulator is disposable.+ }+ glue.close();+ }++ @Test+ @DisplayName("StartCrawler completes a crawl that GetCrawler and GetCrawlerMetrics parse")+ void startAndReadACrawl() {+ glue.startCrawler(r -> r.name(CRAWLER_NAME));++ Crawler crawler = glue.getCrawler(r -> r.name(CRAWLER_NAME)).crawler();+ assertThat(crawler.state()).isEqualTo(CrawlerState.READY);+ assertThat(crawler.lastCrawl().status()).isEqualTo(LastCrawlStatus.SUCCEEDED);+ assertThat(crawler.lastCrawl().startTime()).isNotNull();++ CrawlerMetrics metrics = glue.getCrawlerMetrics(r -> r.crawlerNameList(CRAWLER_NAME))+ .crawlerMetricsList().get(0);+ assertThat(metrics.crawlerName()).isEqualTo(CRAWLER_NAME);+ assertThat(metrics.timeLeftSeconds()).isZero();+ assertThat(metrics.stillEstimating()).isFalse();++ assertThatThrownBy(() -> glue.stopCrawler(r -> r.name(CRAWLER_NAME)))+ .isInstanceOf(CrawlerNotRunningException.class);+ }++ @Test+ @DisplayName("ListCrawlers filters on tags and BatchGetCrawlers reports missing names")+ void listAndBatchGetCrawlers() {+ assertThat(glue.listCrawlers(r -> r.tags(Map.of("team", "data"))).crawlerNames()).contains(CRAWLER_NAME);++ BatchGetCrawlersResponse response = glue.batchGetCrawlers(r -> r.crawlerNames(CRAWLER_NAME, "absent_crawler"));+ assertThat(response.crawlers()).extracting(Crawler::name).containsExactly(CRAWLER_NAME);+ assertThat(response.crawlersNotFound()).containsExactly("absent_crawler");+ }++ @Test+ @DisplayName("The crawler schedule can be stopped, started and replaced")+ void schedule() {+ glue.stopCrawlerSchedule(r -> r.crawlerName(CRAWLER_NAME));+ assertThat(glue.getCrawler(r -> r.name(CRAWLER_NAME)).crawler().schedule().state())+ .isEqualTo(ScheduleState.NOT_SCHEDULED);++ glue.startCrawlerSchedule(r -> r.crawlerName(CRAWLER_NAME));+ glue.updateCrawlerSchedule(r -> r.crawlerName(CRAWLER_NAME).schedule("cron(30 3 * * ? *)"));+ Crawler crawler = glue.getCrawler(r -> r.name(CRAWLER_NAME)).crawler();+ assertThat(crawler.schedule().state()).isEqualTo(ScheduleState.SCHEDULED);+ assertThat(crawler.schedule().scheduleExpression()).isEqualTo("cron(30 3 * * ? *)");+ }+}diff --git a/docs/services/glue.md b/docs/services/glue.mdindex 9c49f183e9..9716181c07 100644--- a/docs/services/glue.md+++ b/docs/services/glue.md@@ -165,8 +165,23 @@ waiting in `WAITING`.| CreateCrawler | Creates a new crawler with specified targets, role, configuration, and optional schedule. || GetCrawler | Retrieves metadata for a specified crawler. || GetCrawlers | Retrieves metadata for all crawlers defined in the customer account. |-| UpdateCrawler | Updates a crawler. |-| DeleteCrawler | Removes a specified crawler from the AWS Glue Data Catalog. |+| ListCrawlers | Lists crawler names, optionally only those carrying every given tag. |+| BatchGetCrawlers | Retrieves several crawlers by name and reports the names not found. |+| UpdateCrawler | Updates a crawler; refused with `CrawlerRunningException` while a crawl runs. |+| DeleteCrawler | Removes a crawler and its crawl history; refused with `CrawlerRunningException` while a crawl runs. |+| StartCrawler | Starts a crawl. |+| StopCrawler | Stops the running crawl, which is then recorded as `CANCELLED`. |+| GetCrawlerMetrics | Reports time left, last and median runtime for the named crawlers, or for all of them. |+| UpdateCrawlerSchedule | Replaces a crawler's cron schedule. |+| StartCrawlerSchedule | Sets a crawler's schedule to `SCHEDULED`. |+| StopCrawlerSchedule | Sets a crawler's schedule to `NOT_SCHEDULED`. |++Crawls do not read the data store or write tables. A crawl follows the crawler's state machine:+`GetCrawler` reports `RUNNING` with `CrawlElapsedTime` while it runs, then `READY` with a `LastCrawl`+of `SUCCEEDED` (or `CANCELLED` after `StopCrawler`). With the default `crawler-run-duration-seconds` of+0 a crawl finishes as soon as it starts; a positive value keeps the crawler `RUNNING` that long.+`GetCrawlerMetrics` reports zero tables created, updated and deleted. Schedules are stored and their+state can be switched, but a schedule never starts a crawl on its own.#### Classifiers@@ -231,6 +246,7 @@ Supported schema formats are `AVRO`, `JSON`, and `PROTOBUF`. Compatibility modes|---|---|---|| `FLOCI_SERVICES_GLUE_ENABLED` | `true` | Enable or disable the service || `FLOCI_SERVICES_GLUE_JOB_RUN_DURATION_SECONDS` | `0` | Seconds a job run stays `RUNNING` before it succeeds; `0` finishes it as soon as it starts |+| `FLOCI_SERVICES_GLUE_CRAWLER_RUN_DURATION_SECONDS` | `0` | Seconds a crawl keeps the crawler `RUNNING` before it succeeds; `0` finishes it as soon as it starts |## Integration with Athenadiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex d2842ea2d7..3d3cfe5a58 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1961,6 +1961,10 @@ public interface EmulatorConfig {/** How long a job run stays RUNNING before it succeeds; 0 = a run succeeds as soon as it starts. */@WithDefault("0")int jobRunDurationSeconds();++ /** How long a crawl keeps the crawler RUNNING before it succeeds; 0 = a crawl finishes as soon as it starts. */+ @WithDefault("0")+ int crawlerRunDurationSeconds();}interface SesServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.javanew file mode 100644index 0000000000..47743793fe--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java@@ -0,0 +1,245 @@+package io.github.hectorvent.floci.services.glue;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.glue.model.Crawler;+import io.github.hectorvent.floci.services.glue.model.CrawlerRunRecord;+import io.github.hectorvent.floci.services.glue.model.LastCrawlInfo;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Clock;+import java.time.Duration;+import java.time.Instant;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.UUID;++/**+ * Glue crawler runs. No data store is crawled and no table is written: a crawl follows the crawler+ * state machine (READY, RUNNING, back to READY with a LastCrawl status) and settles lazily when+ * read. With the default {@code crawler-run-duration-seconds} of 0 a crawl finishes as soon as it+ * starts; a positive duration keeps the crawler RUNNING for that long.+ *+ * <p>Every public method is synchronized. Starting and stopping a crawl, and the crawler mutations+ * AWS refuses while a crawl runs (UpdateCrawler, DeleteCrawler), check the crawl state and act on+ * it; the store has no atomic compare and set, so those steps must not interleave.+ */+@ApplicationScoped+public class GlueCrawlerRunService {++ private static final Logger LOG = Logger.getLogger(GlueCrawlerRunService.class);++ static final String STATE_READY = "READY";+ static final String STATE_RUNNING = "RUNNING";+ static final String STATUS_SUCCEEDED = "SUCCEEDED";+ static final String STATUS_CANCELLED = "CANCELLED";++ private static final String LOG_GROUP = "/aws-glue/crawlers";+ // Enough history for a stable median without letting a long-lived crawler grow without bound.+ private static final int MAX_RUNTIME_HISTORY = 100;++ private final StorageBackend<String, CrawlerRunRecord> recordStore;+ private final GlueService glueService;+ private final int runDurationSeconds;+ private final Clock clock;++ @Inject+ public GlueCrawlerRunService(StorageFactory storageFactory, GlueService glueService, EmulatorConfig config) {+ this(storageFactory.create("glue", "crawler_runs.json", new TypeReference<>() {}),+ glueService, config.services().glue().crawlerRunDurationSeconds(), Clock.systemUTC());+ }++ GlueCrawlerRunService(StorageBackend<String, CrawlerRunRecord> recordStore, GlueService glueService,+ int runDurationSeconds, Clock clock) {+ if (runDurationSeconds < 0) {+ throw new IllegalArgumentException(+ "floci.services.glue.crawler-run-duration-seconds must not be negative: " + runDurationSeconds);+ }+ this.recordStore = recordStore;+ this.glueService = glueService;+ this.runDurationSeconds = runDurationSeconds;+ this.clock = clock;+ }++ public synchronized void startCrawler(String name) {+ glueService.getCrawler(name);+ CrawlerRunRecord record = settledRecord(name);+ if (record.getCurrentStart() != null) {+ throw new AwsException("CrawlerRunningException", "Crawler with name " + name + " has already started", 400);+ }+ record.setCurrentStart(clock.instant());+ record.setCurrentMessagePrefix(UUID.randomUUID().toString());+ recordStore.put(name, record);+ LOG.infov("Started Glue crawler {0}", name);+ settle(record);+ }++ public synchronized void stopCrawler(String name) {+ glueService.getCrawler(name);+ CrawlerRunRecord record = settledRecord(name);+ if (record.getCurrentStart() == null) {+ throw new AwsException("CrawlerNotRunningException", "Crawler with name " + name + " isn't running", 400);+ }+ finish(record, clock.instant(), STATUS_CANCELLED);+ LOG.infov("Stopped Glue crawler {0}", name);+ }++ /** Fills in the crawl state of a crawler read from the store: State, CrawlElapsedTime, LastCrawl. */+ public synchronized Crawler withRunState(Crawler crawler) {+ CrawlerRunRecord record = settledRecord(crawler.getName());+ Instant now = clock.instant();+ if (record.getCurrentStart() != null) {+ crawler.setState(STATE_RUNNING);+ crawler.setCrawlElapsedTime(Duration.between(record.getCurrentStart(), now).toMillis());+ } else {+ crawler.setState(STATE_READY);+ crawler.setCrawlElapsedTime(null);+ }+ if (record.getLastStatus() != null) {+ LastCrawlInfo lastCrawl = new LastCrawlInfo();+ lastCrawl.setStatus(record.getLastStatus());+ lastCrawl.setStartTime(record.getLastStart());+ lastCrawl.setLogGroup(LOG_GROUP);+ lastCrawl.setLogStream(crawler.getName());+ lastCrawl.setMessagePrefix(record.getLastMessagePrefix());+ crawler.setLastCrawl(lastCrawl);+ } else {+ crawler.setLastCrawl(null);+ }+ return crawler;+ }++ public synchronized List<Crawler> withRunState(List<Crawler> crawlers) {+ for (Crawler crawler : crawlers) {+ withRunState(crawler);+ }+ return crawlers;+ }++ /**+ * Metrics for the named crawlers, or for every crawler when no names are given. A name that+ * matches no crawler is skipped: the operation declares no EntityNotFoundException.+ */+ public synchronized GlueService.Page<Map<String, Object>> getCrawlerMetrics(List<String> names,+ Integer maxResults,+ String nextToken) {+ List<String> selected = new ArrayList<>();+ if (names == null || names.isEmpty()) {+ for (Crawler crawler : glueService.getCrawlers()) {+ selected.add(crawler.getName());+ }+ } else {+ for (Crawler crawler : glueService.batchGetCrawlers(names).crawlers()) {+ selected.add(crawler.getName());+ }+ }+ selected.sort(Comparator.naturalOrder());+ List<Map<String, Object>> metrics = new ArrayList<>();+ for (String name : selected) {+ metrics.add(metricsFor(settledRecord(name)));+ }+ return glueService.paginate(metrics, maxResults, nextToken);+ }++ public synchronized void updateCrawler(Crawler update) {+ requireNotRunning(update.getName());+ glueService.updateCrawler(update);+ }++ /** Crawl records belong to their crawler: deleting it removes them, and a running crawler cannot be deleted. */+ public synchronized void deleteCrawler(String name, String region) {+ requireNotRunning(name);+ glueService.deleteCrawler(name, region);+ recordStore.delete(name);+ }++ public synchronized void updateCrawlerSchedule(String name, String expression) {+ glueService.updateCrawlerSchedule(name, expression);+ }++ public synchronized void startCrawlerSchedule(String name) {+ glueService.startCrawlerSchedule(name);+ }++ public synchronized void stopCrawlerSchedule(String name) {+ glueService.stopCrawlerSchedule(name);+ }++ private void requireNotRunning(String name) {+ if (name != null && settledRecord(name).getCurrentStart() != null) {+ throw new AwsException("CrawlerRunningException", "Crawler with name " + name + " is running", 400);+ }+ }++ private Map<String, Object> metricsFor(CrawlerRunRecord record) {+ Map<String, Object> metrics = new LinkedHashMap<>();+ metrics.put("CrawlerName", record.getCrawlerName());+ double timeLeft = 0;+ if (record.getCurrentStart() != null) {+ Instant finishesAt = record.getCurrentStart().plusSeconds(runDurationSeconds);+ timeLeft = Math.max(0, Duration.between(clock.instant(), finishesAt).toMillis() / 1000.0);+ }+ metrics.put("TimeLeftSeconds", timeLeft);+ metrics.put("StillEstimating", false);+ List<Double> runtimes = record.getRuntimeSeconds();+ metrics.put("LastRuntimeSeconds", runtimes.isEmpty() ? 0.0 : runtimes.getLast());+ metrics.put("MedianRuntimeSeconds", median(runtimes));+ metrics.put("TablesCreated", 0);+ metrics.put("TablesUpdated", 0);+ metrics.put("TablesDeleted", 0);+ return metrics;+ }++ private static double median(List<Double> values) {+ if (values.isEmpty()) {+ return 0.0;+ }+ List<Double> sorted = new ArrayList<>(values);+ sorted.sort(Comparator.naturalOrder());+ int middle = sorted.size() / 2;+ return sorted.size() % 2 == 1 ? sorted.get(middle) : (sorted.get(middle - 1) + sorted.get(middle)) / 2.0;+ }++ private CrawlerRunRecord settledRecord(String name) {+ CrawlerRunRecord record = recordStore.get(name).orElseGet(() -> {+ CrawlerRunRecord fresh = new CrawlerRunRecord();+ fresh.setCrawlerName(name);+ return fresh;+ });+ settle(record);+ return record;+ }++ private void settle(CrawlerRunRecord record) {+ if (record.getCurrentStart() == null) {+ return;+ }+ Instant finishesAt = record.getCurrentStart().plusSeconds(runDurationSeconds);+ if (!clock.instant().isBefore(finishesAt)) {+ finish(record, finishesAt, STATUS_SUCCEEDED);+ }+ }++ private void finish(CrawlerRunRecord record, Instant finishedAt, String status) {+ List<Double> runtimes = new ArrayList<>(record.getRuntimeSeconds());+ runtimes.add(Duration.between(record.getCurrentStart(), finishedAt).toMillis() / 1000.0);+ if (runtimes.size() > MAX_RUNTIME_HISTORY) {+ runtimes.removeFirst();+ }+ record.setRuntimeSeconds(runtimes);+ record.setLastStart(record.getCurrentStart());+ record.setLastStatus(status);+ record.setLastMessagePrefix(record.getCurrentMessagePrefix());+ record.setCurrentStart(null);+ record.setCurrentMessagePrefix(null);+ recordStore.put(record.getCrawlerName(), record);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex 670b124714..5dce251376 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -32,16 +32,19 @@ public class GlueJsonHandler {private final GlueService glueService;private final GlueJobRunService jobRunService;+ private final GlueCrawlerRunService crawlerRunService;private final GlueSchemaRegistryService schemaRegistryService;private final ObjectMapper mapper;@Injectpublic GlueJsonHandler(GlueService glueService,GlueJobRunService jobRunService,+ GlueCrawlerRunService crawlerRunService,GlueSchemaRegistryService schemaRegistryService,ObjectMapper mapper) {this.glueService = glueService;this.jobRunService = jobRunService;+ this.crawlerRunService = crawlerRunService;this.schemaRegistryService = schemaRegistryService;this.mapper = mapper;}@@ -302,27 +305,75 @@ public class GlueJsonHandler {}case "GetCrawler" -> {GetCrawlerRequest req = mapper.treeToValue(request, GetCrawlerRequest.class);- yield Response.ok(new GetCrawlerResponse(glueService.getCrawler(req.getName()))).build();+ yield Response.ok(new GetCrawlerResponse(+ crawlerRunService.withRunState(glueService.getCrawler(req.getName())))).build();}case "GetCrawlers" -> {GetCrawlersRequest req = mapper.treeToValue(request, GetCrawlersRequest.class);GlueService.Page<Crawler> page = glueService.getCrawlers(req.getMaxResults(), req.getNextToken());GetCrawlersResponse res = new GetCrawlersResponse();- res.setCrawlers(page.items());+ res.setCrawlers(crawlerRunService.withRunState(page.items()));res.setNextToken(page.nextToken());yield Response.ok(res).build();}case "UpdateCrawler" -> {UpdateCrawlerRequest req = mapper.treeToValue(request, UpdateCrawlerRequest.class);Crawler update = toDomain(req);- glueService.updateCrawler(update);+ crawlerRunService.updateCrawler(update);yield Response.ok().build();}case "DeleteCrawler" -> {DeleteCrawlerRequest req = mapper.treeToValue(request, DeleteCrawlerRequest.class);- glueService.deleteCrawler(req.getName(), region);+ crawlerRunService.deleteCrawler(req.getName(), region);yield Response.ok().build();}+ case "ListCrawlers" -> {+ Map<String, String> tags = request.hasNonNull("Tags")+ ? mapper.convertValue(request.get("Tags"), STRING_MAP)+ : null;+ GlueService.Page<String> page = glueService.listCrawlers(+ readMaxResults(request), readNextToken(request), tags, region);+ yield Response.ok(pageResponse("CrawlerNames", page.items(), page.nextToken())).build();+ }+ case "BatchGetCrawlers" -> {+ List<String> names = request.hasNonNull("CrawlerNames")+ ? mapper.convertValue(request.get("CrawlerNames"), STRING_LIST)+ : null;+ GlueService.BatchGetCrawlersResult result = glueService.batchGetCrawlers(names);+ Map<String, Object> response = new LinkedHashMap<>();+ response.put("Crawlers", crawlerRunService.withRunState(result.crawlers()));+ response.put("CrawlersNotFound", result.crawlersNotFound());+ yield Response.ok(response).build();+ }+ case "StartCrawler" -> {+ crawlerRunService.startCrawler(request.path("Name").asText(null));+ yield Response.ok(Map.of()).build();+ }+ case "StopCrawler" -> {+ crawlerRunService.stopCrawler(request.path("Name").asText(null));+ yield Response.ok(Map.of()).build();+ }+ case "GetCrawlerMetrics" -> {+ List<String> names = request.hasNonNull("CrawlerNameList")+ ? mapper.convertValue(request.get("CrawlerNameList"), STRING_LIST)+ : null;+ GlueService.Page<Map<String, Object>> page = crawlerRunService.getCrawlerMetrics(+ names, readMaxResults(request), readNextToken(request));+ yield Response.ok(pageResponse("CrawlerMetricsList", page.items(), page.nextToken())).build();+ }+ case "UpdateCrawlerSchedule" -> {+ crawlerRunService.updateCrawlerSchedule(request.path("CrawlerName").asText(null),+ request.path("Schedule").asText(null));+ yield Response.ok(Map.of()).build();+ }+ case "StartCrawlerSchedule" -> {+ crawlerRunService.startCrawlerSchedule(request.path("CrawlerName").asText(null));+ yield Response.ok(Map.of()).build();+ }+ case "StopCrawlerSchedule" -> {+ crawlerRunService.stopCrawlerSchedule(request.path("CrawlerName").asText(null));+ yield Response.ok(Map.of()).build();+ }case "CreateConnection" -> handleCreateConnection(request, region);case "GetConnection" -> handleGetConnection(request);case "GetConnections" -> handleGetConnections(request);diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 14277b5803..c751c56316 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -26,6 +26,7 @@ import io.github.hectorvent.floci.services.glue.model.KeySchemaElement;import io.github.hectorvent.floci.services.glue.model.Partition;import io.github.hectorvent.floci.services.glue.model.PartitionIndex;import io.github.hectorvent.floci.services.glue.model.PartitionIndexDescriptor;+import io.github.hectorvent.floci.services.glue.model.Schedule;import io.github.hectorvent.floci.services.glue.model.SchemaReference;import io.github.hectorvent.floci.services.glue.model.SecurityConfiguration;import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;@@ -149,6 +150,12 @@ public class GlueService {private static final String INDEX_STATUS_ACTIVE = "ACTIVE";private static final String INDEX_STATUS_DELETING = "DELETING";+ private static final String SCHEDULE_SCHEDULED = "SCHEDULED";+ private static final String SCHEDULE_NOT_SCHEDULED = "NOT_SCHEDULED";+ private static final Pattern CRON_FIELD = Pattern.compile("[0-9A-Za-z*?/,#-]+");+ // BatchGetCrawlers takes a CrawlerNameList, which the API model caps at 100 names.+ private static final int MAX_BATCH_GET_CRAWLERS = 100;+private final StorageBackend<String, Database> databaseStore;private final StorageBackend<String, Table> tableStore;private final StorageBackend<String, Table> tableVersionStore;@@ -2037,6 +2044,110 @@ public class GlueService {return paginate(all, maxResults, nextToken);}+ public Page<String> listCrawlers(Integer maxResults, String nextToken, Map<String, String> tags,+ String region) {+ List<String> names = new ArrayList<>();+ for (Crawler crawler : crawlerStore.scan(k -> true)) {+ if (tags == null || tags.isEmpty() || hasTags(crawlerArn(region, crawler.getName()), tags, region)) {+ names.add(crawler.getName());+ }+ }+ names.sort(Comparator.naturalOrder());+ return paginate(names, maxResults, nextToken);+ }++ public BatchGetCrawlersResult batchGetCrawlers(List<String> names) {+ if (names == null) {+ throw new AwsException("InvalidInputException", "CrawlerNames is required.", 400);+ }+ if (names.size() > MAX_BATCH_GET_CRAWLERS) {+ throw new AwsException("InvalidInputException",+ "CrawlerNames must contain at most " + MAX_BATCH_GET_CRAWLERS + " names.", 400);+ }+ List<Crawler> crawlers = new ArrayList<>();+ List<String> notFound = new ArrayList<>();+ for (String name : names) {+ Optional<Crawler> crawler = name == null ? Optional.empty() : crawlerStore.get(name);+ if (crawler.isPresent()) {+ crawlers.add(crawler.get());+ } else {+ notFound.add(name);+ }+ }+ return new BatchGetCrawlersResult(crawlers, notFound);+ }++ public record BatchGetCrawlersResult(List<Crawler> crawlers, List<String> crawlersNotFound) {}++ public void updateCrawlerSchedule(String name, String expression) {+ validateRequired(name, "CrawlerName");+ Crawler crawler = getCrawler(name);+ if (expression == null || expression.isBlank()) {+ crawler.setSchedule(null);+ } else {+ validateCronExpression(expression);+ Schedule schedule = crawler.getSchedule() != null ? crawler.getSchedule() : new Schedule();+ schedule.setScheduleExpression(expression);+ if (schedule.getState() == null) {+ schedule.setState(SCHEDULE_SCHEDULED);+ }+ crawler.setSchedule(schedule);+ }+ crawler.setLastUpdated(Instant.now());+ crawlerStore.put(name, crawler);+ }++ /**+ * Glue schedules are AWS cron expressions: cron(Minutes Hours Day-of-month Month Day-of-week Year),+ * with {@code ?} in exactly one of the two day fields.+ */+ static void validateCronExpression(String expression) {+ String message = "Schedule must be a cron expression, for example cron(15 12 * * ? *).";+ if (expression == null || !expression.startsWith("cron(") || !expression.endsWith(")")) {+ throw new AwsException("InvalidInputException", message, 400);+ }+ String[] fields = expression.substring(5, expression.length() - 1).trim().split("\\s+");+ if (fields.length != 6) {+ throw new AwsException("InvalidInputException", message, 400);+ }+ for (String field : fields) {+ if (!CRON_FIELD.matcher(field).matches()) {+ throw new AwsException("InvalidInputException", message, 400);+ }+ }+ if ("?".equals(fields[2]) == "?".equals(fields[4])) {+ throw new AwsException("InvalidInputException",+ "Exactly one of day-of-month and day-of-week must be '?' in " + expression, 400);+ }+ }++ public void startCrawlerSchedule(String name) {+ validateRequired(name, "CrawlerName");+ Crawler crawler = getCrawler(name);+ Schedule schedule = crawler.getSchedule();+ if (schedule == null || schedule.getScheduleExpression() == null) {+ throw new AwsException("NoScheduleException", "Crawler " + name + " has no schedule.", 400);+ }+ if (SCHEDULE_SCHEDULED.equals(schedule.getState())) {+ throw new AwsException("SchedulerRunningException",+ "The schedule of crawler " + name + " is already running.", 400);+ }+ schedule.setState(SCHEDULE_SCHEDULED);+ crawlerStore.put(name, crawler);+ }++ public void stopCrawlerSchedule(String name) {+ validateRequired(name, "CrawlerName");+ Crawler crawler = getCrawler(name);+ Schedule schedule = crawler.getSchedule();+ if (schedule == null || !SCHEDULE_SCHEDULED.equals(schedule.getState())) {+ throw new AwsException("SchedulerNotRunningException",+ "The schedule of crawler " + name + " is not running.", 400);+ }+ schedule.setState(SCHEDULE_NOT_SCHEDULED);+ crawlerStore.put(name, crawler);+ }+public void updateCrawler(Crawler update) {validateRequired(update.getName(), "Name");// Jobs and crawlers skip normalizeName because AWS preserves their casediff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.javanew file mode 100644index 0000000000..8fc5e96c68--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java@@ -0,0 +1,45 @@+package io.github.hectorvent.floci.services.glue.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;+import java.util.ArrayList;+import java.util.List;++/**+ * Floci's own bookkeeping for a crawler's crawls; never sent on the wire. The crawl in progress (if+ * any), the last finished crawl, and the runtimes GetCrawlerMetrics summarises.+ */+@RegisterForReflection+public class CrawlerRunRecord {+ private String crawlerName;+ private Instant currentStart;+ private String currentMessagePrefix;+ private Instant lastStart;+ private String lastStatus;+ private String lastMessagePrefix;+ private List<Double> runtimeSeconds = new ArrayList<>();++ public CrawlerRunRecord() {}++ public String getCrawlerName() { return crawlerName; }+ public void setCrawlerName(String crawlerName) { this.crawlerName = crawlerName; }++ public Instant getCurrentStart() { return currentStart; }+ public void setCurrentStart(Instant currentStart) { this.currentStart = currentStart; }++ public String getCurrentMessagePrefix() { return currentMessagePrefix; }+ public void setCurrentMessagePrefix(String currentMessagePrefix) { this.currentMessagePrefix = currentMessagePrefix; }++ public Instant getLastStart() { return lastStart; }+ public void setLastStart(Instant lastStart) { this.lastStart = lastStart; }++ public String getLastStatus() { return lastStatus; }+ public void setLastStatus(String lastStatus) { this.lastStatus = lastStatus; }++ public String getLastMessagePrefix() { return lastMessagePrefix; }+ public void setLastMessagePrefix(String lastMessagePrefix) { this.lastMessagePrefix = lastMessagePrefix; }++ public List<Double> getRuntimeSeconds() { return runtimeSeconds; }+ public void setRuntimeSeconds(List<Double> runtimeSeconds) { this.runtimeSeconds = runtimeSeconds; }+}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 882d7bcd02..ee4f541e27 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -510,6 +510,7 @@ floci:glue:enabled: truejob-run-duration-seconds: 0 # 0 = job runs succeed as soon as they start+ crawler-run-duration-seconds: 0 # 0 = crawls finish as soon as they startses:enabled: true# smtp-host: mailpit # SMTP server for email relay (empty = store only)diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunIntegrationTest.javanew file mode 100644index 0000000000..e28c24b872--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunIntegrationTest.java@@ -0,0 +1,63 @@+package io.github.hectorvent.floci.services.glue;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.ValidatableResponse;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasItem;++@QuarkusTest+class GlueCrawlerRunIntegrationTest {++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String ROLE = "arn:aws:iam::000000000000:role/my-role";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ private static ValidatableResponse call(String action, String body) {+ return given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue." + action)+ .body(body)+ .when().post("/")+ .then();+ }++ @Test+ void crawlRunsToCompletionAndItsHistoryGoesWithTheCrawler() {+ String name = "crawl-" + UUID.randomUUID().toString().substring(0, 8);+ String byName = "{ \"Name\": \"" + name + "\" }";+ call("CreateCrawler", """+ { "Name": "%s", "Role": "%s", "Targets": { "S3Targets": [ { "Path": "s3://raw/data" } ] } }+ """.formatted(name, ROLE)).statusCode(200);++ call("StartCrawler", byName).statusCode(200);++ call("GetCrawler", byName)+ .statusCode(200)+ .body("Crawler.State", equalTo("READY"))+ .body("Crawler.LastCrawl.Status", equalTo("SUCCEEDED"));++ call("ListCrawlers", "{}")+ .statusCode(200)+ .body("CrawlerNames", hasItem(name));++ call("StopCrawler", byName)+ .statusCode(400)+ .body("__type", equalTo("CrawlerNotRunningException"));++ call("DeleteCrawler", byName).statusCode(200);++ call("StartCrawler", byName)+ .statusCode(400)+ .body("__type", equalTo("EntityNotFoundException"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.javanew file mode 100644index 0000000000..0b94ca1413--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java@@ -0,0 +1,334 @@+package io.github.hectorvent.floci.services.glue;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.glue.model.Crawler;+import io.github.hectorvent.floci.services.glue.model.CrawlerRunRecord;+import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;+import io.github.hectorvent.floci.services.glue.model.S3Target;+import io.github.hectorvent.floci.services.glue.model.Schedule;+import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;+import io.github.hectorvent.floci.services.kms.KmsService;+import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++class GlueCrawlerRunServiceTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT_ID = "000000000000";+ private static final String ROLE = "arn:aws:iam::000000000000:role/glue";+ private static final long CLOCK_TICKS_MILLIS = 100;++ private GlueService glueService;+ private InMemoryStorage<String, CrawlerRunRecord> recordStore;+ private MutableClock clock;++ @BeforeEach+ void setUp() {+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);+ StorageFactory storageFactory = new InMemoryStorageFactory();+ glueService = new GlueService(storageFactory, new GlueSchemaRegistryService(storageFactory, regionResolver),+ regionResolver, new ResourceGroupsTaggingService(storageFactory),+ new KmsService(storageFactory, regionResolver));+ recordStore = new InMemoryStorage<>();+ clock = new MutableClock();+ }++ /**+ * The shared test clock moves 1 ms on every read, so a stop "now" lands a few milliseconds after+ * the whole seconds the test advanced.+ */+ private static void assertSeconds(double expected, Object actual) {+ assertEquals(expected, (Double) actual, CLOCK_TICKS_MILLIS / 1000.0);+ }++ private GlueCrawlerRunService service(int runDurationSeconds) {+ return new GlueCrawlerRunService(recordStore, glueService, runDurationSeconds, clock);+ }++ private void createCrawler(String name, String scheduleExpression) {+ S3Target target = new S3Target();+ target.setPath("s3://raw/" + name);+ CrawlerTargets targets = new CrawlerTargets();+ targets.setS3Targets(List.of(target));+ Crawler crawler = new Crawler();+ crawler.setName(name);+ crawler.setRole(ROLE);+ crawler.setTargets(targets);+ if (scheduleExpression != null) {+ Schedule schedule = new Schedule();+ schedule.setScheduleExpression(scheduleExpression);+ schedule.setState("SCHEDULED");+ crawler.setSchedule(schedule);+ }+ glueService.createCrawler(crawler, null, REGION);+ }++ private Crawler read(GlueCrawlerRunService service, String name) {+ return service.withRunState(glueService.getCrawler(name));+ }++ private Map<String, Object> metrics(GlueCrawlerRunService service, String name) {+ return service.getCrawlerMetrics(List.of(name), null, null).items().getFirst();+ }++ @Test+ void neverStartedCrawlerIsReadyWithNoLastCrawl() {+ createCrawler("raw", null);++ Crawler crawler = read(service(0), "raw");++ assertEquals("READY", crawler.getState());+ assertNull(crawler.getLastCrawl());+ assertNull(crawler.getCrawlElapsedTime());+ }++ @Test+ void crawlFinishesAsSoonAsItStartsByDefault() {+ createCrawler("raw", null);+ GlueCrawlerRunService service = service(0);++ service.startCrawler("raw");+ Crawler crawler = read(service, "raw");++ assertEquals("READY", crawler.getState());+ assertEquals("SUCCEEDED", crawler.getLastCrawl().getStatus());+ assertNotNull(crawler.getLastCrawl().getStartTime());+ assertEquals("/aws-glue/crawlers", crawler.getLastCrawl().getLogGroup());+ assertEquals("raw", crawler.getLastCrawl().getLogStream());+ assertEquals(0.0, metrics(service, "raw").get("LastRuntimeSeconds"));+ }++ @Test+ void crawlerStaysRunningUntilTheConfiguredDurationHasPassed() {+ createCrawler("raw", null);+ GlueCrawlerRunService service = service(60);+ service.startCrawler("raw");++ clock.advance(Duration.ofSeconds(30));+ Crawler running = read(service, "raw");+ assertEquals("RUNNING", running.getState());+ assertEquals(30_000L, running.getCrawlElapsedTime(), CLOCK_TICKS_MILLIS);+ assertNull(running.getLastCrawl());+ assertSeconds(30.0, metrics(service, "raw").get("TimeLeftSeconds"));++ clock.advance(Duration.ofSeconds(45));+ Crawler finished = read(service, "raw");+ assertEquals("READY", finished.getState());+ assertEquals("SUCCEEDED", finished.getLastCrawl().getStatus());+ assertEquals(60.0, metrics(service, "raw").get("LastRuntimeSeconds"));+ assertEquals(0.0, metrics(service, "raw").get("TimeLeftSeconds"));+ }++ @Test+ void startWhileRunningAndStopWhileIdleAreRefused() {+ createCrawler("raw", null);+ GlueCrawlerRunService service = service(60);++ assertEquals("CrawlerNotRunningException",+ assertThrows(AwsException.class, () -> service.stopCrawler("raw")).getErrorCode());+ service.startCrawler("raw");+ assertEquals("CrawlerRunningException",+ assertThrows(AwsException.class, () -> service.startCrawler("raw")).getErrorCode());+ assertEquals("EntityNotFoundException",+ assertThrows(AwsException.class, () -> service.startCrawler("missing")).getErrorCode());+ }++ @Test+ void stoppingACrawlRecordsItAsCancelled() {+ createCrawler("raw", null);+ GlueCrawlerRunService service = service(60);+ service.startCrawler("raw");++ clock.advance(Duration.ofSeconds(10));+ service.stopCrawler("raw");+ Crawler crawler = read(service, "raw");++ assertEquals("READY", crawler.getState());+ assertEquals("CANCELLED", crawler.getLastCrawl().getStatus());+ assertSeconds(10.0, metrics(service, "raw").get("LastRuntimeSeconds"));+ }++ @Test+ void metricsReportTheLastAndMedianRuntime() {+ createCrawler("raw", null);+ GlueCrawlerRunService service = service(600);+ for (int seconds : new int[] {10, 30, 20}) {+ service.startCrawler("raw");+ clock.advance(Duration.ofSeconds(seconds));+ service.stopCrawler("raw");+ clock.advance(Duration.ofSeconds(1));+ }++ Map<String, Object> metrics = metrics(service, "raw");++ assertEquals("raw", metrics.get("CrawlerName"));+ assertSeconds(20.0, metrics.get("LastRuntimeSeconds"));+ assertSeconds(20.0, metrics.get("MedianRuntimeSeconds"));+ assertEquals(false, metrics.get("StillEstimating"));+ assertEquals(0, metrics.get("TablesCreated"));+ }++ @Test+ void metricsCoverEveryCrawlerWhenNoNamesAreGivenAndSkipUnknownNames() {+ createCrawler("b", null);+ createCrawler("a", null);+ GlueCrawlerRunService service = service(0);++ List<Map<String, Object>> all = service.getCrawlerMetrics(null, null, null).items();+ assertEquals(List.of("a", "b"), all.stream().map(m -> m.get("CrawlerName")).toList());++ List<Map<String, Object>> named = service.getCrawlerMetrics(List.of("b", "missing"), null, null).items();+ assertEquals(List.of("b"), named.stream().map(m -> m.get("CrawlerName")).toList());+ }++ @Test+ void runningCrawlerCannotBeUpdatedOrDeleted() {+ createCrawler("raw", null);+ GlueCrawlerRunService service = service(60);+ service.startCrawler("raw");+ Crawler update = new Crawler();+ update.setName("raw");+ update.setDescription("changed");++ assertEquals("CrawlerRunningException",+ assertThrows(AwsException.class, () -> service.updateCrawler(update)).getErrorCode());+ assertEquals("CrawlerRunningException",+ assertThrows(AwsException.class, () -> service.deleteCrawler("raw", REGION)).getErrorCode());++ clock.advance(Duration.ofSeconds(60));+ service.updateCrawler(update);+ assertEquals("changed", glueService.getCrawler("raw").getDescription());+ }++ @Test+ void deletingACrawlerRemovesItsCrawlHistory() {+ createCrawler("raw", null);+ GlueCrawlerRunService service = service(0);+ service.startCrawler("raw");++ service.deleteCrawler("raw", REGION);+ createCrawler("raw", null);++ assertTrue(recordStore.scan(key -> true).isEmpty());+ assertNull(read(service, "raw").getLastCrawl());+ }++ @Test+ void scheduleCanBeStoppedStartedAndReplaced() {+ createCrawler("nightly", "cron(0 2 * * ? *)");+ createCrawler("adhoc", null);+ GlueCrawlerRunService service = service(0);++ service.stopCrawlerSchedule("nightly");+ assertEquals("NOT_SCHEDULED", glueService.getCrawler("nightly").getSchedule().getState());+ assertEquals("SchedulerNotRunningException",+ assertThrows(AwsException.class, () -> service.stopCrawlerSchedule("nightly")).getErrorCode());++ service.startCrawlerSchedule("nightly");+ assertEquals("SCHEDULED", glueService.getCrawler("nightly").getSchedule().getState());+ assertEquals("SchedulerRunningException",+ assertThrows(AwsException.class, () -> service.startCrawlerSchedule("nightly")).getErrorCode());++ service.updateCrawlerSchedule("nightly", "cron(30 3 * * ? *)");+ assertEquals("cron(30 3 * * ? *)", glueService.getCrawler("nightly").getSchedule().getScheduleExpression());+ assertEquals("InvalidInputException", assertThrows(AwsException.class,+ () -> service.updateCrawlerSchedule("nightly", "every day")).getErrorCode());++ assertEquals("NoScheduleException",+ assertThrows(AwsException.class, () -> service.startCrawlerSchedule("adhoc")).getErrorCode());+ }++ @Test+ void scheduleMustBeAWellFormedAwsCronExpression() {+ createCrawler("nightly", "cron(0 2 * * ? *)");+ GlueCrawlerRunService service = service(0);++ for (String malformed : List.of("cron(foo)", "cron(0 2 * * *)", "cron(0 2 * * * *)",+ "cron(0 2 ? * ? *)", "cron(0 2 * * ? *) ", "rate(1 day)", "cron(0 2 * * ? * $)")) {+ assertEquals("InvalidInputException", assertThrows(AwsException.class,+ () -> service.updateCrawlerSchedule("nightly", malformed)).getErrorCode(), malformed);+ }+ for (String valid : List.of("cron(15 12 * * ? *)", "cron(0/5 8-17 ? * MON-FRI *)", "cron(0 0 L * ? 2026)")) {+ service.updateCrawlerSchedule("nightly", valid);+ assertEquals(valid, glueService.getCrawler("nightly").getSchedule().getScheduleExpression());+ }+ }++ @Test+ void concurrentStartsLetExactlyOneCrawlRun() throws Exception {+ createCrawler("raw", null);+ GlueCrawlerRunService service = service(60);+ int threads = 16;+ CountDownLatch go = new CountDownLatch(1);+ AtomicInteger started = new AtomicInteger();+ AtomicInteger refused = new AtomicInteger();+ ExecutorService pool = Executors.newFixedThreadPool(threads);+ try {+ List<Future<?>> futures = new ArrayList<>();+ for (int i = 0; i < threads; i++) {+ futures.add(pool.submit(() -> {+ go.await();+ try {+ service.startCrawler("raw");+ started.incrementAndGet();+ } catch (AwsException expected) {+ // CrawlerRunningException is the outcome under test for all but one start.+ refused.incrementAndGet();+ }+ return null;+ }));+ }+ go.countDown();+ for (Future<?> future : futures) {+ future.get(10, TimeUnit.SECONDS);+ }+ } finally {+ pool.shutdownNow();+ }++ assertEquals(1, started.get());+ assertEquals(threads - 1, refused.get());+ }++ @Test+ void negativeRunDurationIsRefusedAtConstruction() {+ assertThrows(IllegalArgumentException.class, () -> service(-1));+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return AccountAwareStorageBackend.inMemory(ACCOUNT_ID);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJobRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJobRunServiceTest.javaindex 178af64361..d7e8f048bf 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJobRunServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJobRunServiceTest.java@@ -14,13 +14,12 @@ import io.github.hectorvent.floci.services.glue.model.JobRun;import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;import io.github.hectorvent.floci.services.kms.KmsService;import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;+import io.github.hectorvent.floci.testing.MutableClock;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;-import java.time.Clock;+import java.time.Duration;import java.time.Instant;-import java.time.ZoneId;-import java.time.ZoneOffset;import java.util.ArrayList;import java.util.List;import java.util.Map;@@ -45,7 +44,6 @@ class GlueJobRunServiceTest {private static final String REGION = "us-east-1";private static final String ACCOUNT_ID = "000000000000";private static final String ROLE = "arn:aws:iam::000000000000:role/glue";- private static final Instant T0 = Instant.parse("2026-09-25T12:00:00Z");private GlueService glueService;private InMemoryStorage<String, JobRun> runStore;@@ -59,7 +57,7 @@ class GlueJobRunServiceTest {regionResolver, new ResourceGroupsTaggingService(storageFactory),new KmsService(storageFactory, regionResolver));runStore = new InMemoryStorage<>();- clock = new MutableClock(T0);+ clock = new MutableClock();}private GlueJobRunService service(int runDurationSeconds) {@@ -87,12 +85,15 @@ class GlueJobRunServiceTest {void runSucceedsAsSoonAsItStartsByDefault() {createJob("etl", null, null);+ Instant before = clock.instant();JobRun run = service(0).startJobRun("etl", null, new JobRun());+ Instant after = clock.instant();assertTrue(run.getId().matches("jr_[0-9a-f]{64}"), run.getId());assertEquals("SUCCEEDED", run.getJobRunState());- assertEquals(T0, run.getStartedOn());- assertEquals(T0, run.getCompletedOn());+ assertTrue(run.getStartedOn().isAfter(before) && run.getStartedOn().isBefore(after),+ "StartedOn " + run.getStartedOn() + " should be read from the clock during StartJobRun");+ assertEquals(run.getStartedOn(), run.getCompletedOn());assertEquals(0, run.getExecutionTime());assertEquals(0, run.getAttempt());assertEquals("G.1X", run.getWorkerType());@@ -124,17 +125,17 @@ class GlueJobRunServiceTest {void runStaysRunningUntilTheConfiguredDurationHasPassed() {createJob("etl", null, null);GlueJobRunService service = service(60);- String runId = service.startJobRun("etl", null, new JobRun()).getId();+ JobRun started = service.startJobRun("etl", null, new JobRun());- clock.set(T0.plusSeconds(59));- JobRun running = service.getJobRun("etl", runId);+ clock.advance(Duration.ofSeconds(59));+ JobRun running = service.getJobRun("etl", started.getId());assertEquals("RUNNING", running.getJobRunState());assertNull(running.getCompletedOn());- clock.set(T0.plusSeconds(90));- JobRun finished = service.getJobRun("etl", runId);+ clock.advance(Duration.ofSeconds(31));+ JobRun finished = service.getJobRun("etl", started.getId());assertEquals("SUCCEEDED", finished.getJobRunState());- assertEquals(T0.plusSeconds(60), finished.getCompletedOn());+ assertEquals(started.getStartedOn().plusSeconds(60), finished.getCompletedOn());assertEquals(60, finished.getExecutionTime());}@@ -142,13 +143,13 @@ class GlueJobRunServiceTest {void runLongerThanTheJobTimeoutEndsInTimeout() {createJob("etl", null, 1);GlueJobRunService service = service(120);- String runId = service.startJobRun("etl", null, new JobRun()).getId();+ JobRun started = service.startJobRun("etl", null, new JobRun());- clock.set(T0.plusSeconds(61));- JobRun run = service.getJobRun("etl", runId);+ clock.advance(Duration.ofSeconds(61));+ JobRun run = service.getJobRun("etl", started.getId());assertEquals("TIMEOUT", run.getJobRunState());- assertEquals(T0.plusSeconds(60), run.getCompletedOn());+ assertEquals(started.getStartedOn().plusSeconds(60), run.getCompletedOn());assertEquals(60, run.getExecutionTime());}@@ -162,7 +163,7 @@ class GlueJobRunServiceTest {() -> service.startJobRun("etl", null, new JobRun()));assertEquals("ConcurrentRunsExceededException", error.getErrorCode());- clock.set(T0.plusSeconds(60));+ clock.advance(Duration.ofSeconds(60));assertEquals("RUNNING", service.startJobRun("etl", null, new JobRun()).getJobRunState());}@@ -187,9 +188,9 @@ class GlueJobRunServiceTest {createJob("etl", 5, null);GlueJobRunService service = service(60);String finished = service.startJobRun("etl", null, new JobRun()).getId();- clock.set(T0.plusSeconds(60));+ clock.advance(Duration.ofSeconds(60));String stillRunning = service.startJobRun("etl", null, new JobRun()).getId();- clock.set(T0.plusSeconds(70));+ clock.advance(Duration.ofSeconds(10));GlueJobRunService.StopResult result =service.batchStopJobRun("etl", List.of(stillRunning, finished, "jr_missing"));@@ -209,7 +210,7 @@ class GlueJobRunServiceTest {createJob("etl", 10, null);GlueJobRunService service = service(0);String first = service.startJobRun("etl", null, new JobRun()).getId();- clock.set(T0.plusSeconds(1));+ clock.advance(Duration.ofSeconds(1));String second = service.startJobRun("etl", null, new JobRun()).getId();GlueService.Page<JobRun> page = service.getJobRuns("etl", 1, null);@@ -435,31 +436,4 @@ class GlueJobRunServiceTest {return AccountAwareStorageBackend.inMemory(ACCOUNT_ID);}}-- private static final class MutableClock extends Clock {- private Instant now;-- private MutableClock(Instant now) {- this.now = now;- }-- void set(Instant now) {- this.now = now;- }-- @Override- public ZoneOffset getZone() {- return ZoneOffset.UTC;- }-- @Override- public Clock withZone(ZoneId zone) {- return this;- }-- @Override- public Instant instant() {- return now;- }- }}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.javaindex b28c93ec7d..462d94d4d1 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java@@ -47,6 +47,7 @@ class GlueJsonHandlerEmptyListTest {new ResourceGroupsTaggingService(storageFactory), new KmsService(storageFactory, regionResolver));handler = new GlueJsonHandler(glueService,new GlueJobRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC()),+ new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC()),schemaRegistryService, mapper);}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.javaindex 88ab00f401..9c93db2d33 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java@@ -48,6 +48,7 @@ class GlueJsonHandlerTest {new KmsService(storageFactory, regionResolver));handler = new GlueJsonHandler(glueService,new GlueJobRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC()),+ new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC()),schemaRegistryService, mapper);}@@ -875,4 +876,75 @@ class GlueJsonHandlerTest {assertEquals("tagged", got.get("Jobs").get(0).get("Name").asText());assertEquals("absent", got.get("JobsNotFound").get(0).asText());}++ private void createCrawlerWithTags(String name, Map<String, String> tags) throws Exception {+ ObjectNode create = mapper.createObjectNode();+ create.put("Name", name);+ create.put("Role", "arn:aws:iam::000000000000:role/glue");+ create.putObject("Targets").putArray("S3Targets").addObject().put("Path", "s3://raw/" + name);+ create.put("Schedule", "cron(0 2 * * ? *)");+ ObjectNode tagNode = create.putObject("Tags");+ tags.forEach(tagNode::put);+ assertEquals(200, handler.handle("CreateCrawler", create, REGION).getStatus());+ }++ /**+ * The crawl lifecycle as a client polls it: StartCrawler and StopCrawler answer with empty+ * bodies, GetCrawler carries State and a LastCrawl with a numeric StartTime, and+ * GetCrawlerMetrics reports numbers, never nulls.+ */+ @Test+ void crawlerRunOperationsAnswerWithTheDocumentedBodies() throws Exception {+ createCrawlerWithTags("raw", Map.of());+ ObjectNode byName = mapper.createObjectNode().put("Name", "raw");++ JsonNode before = mapper.valueToTree(handler.handle("GetCrawler", byName, REGION).getEntity()).get("Crawler");+ assertEquals("READY", before.get("State").asText());++ Response started = handler.handle("StartCrawler", byName, REGION);+ assertEquals(0, mapper.valueToTree(started.getEntity()).size());++ JsonNode crawler = mapper.valueToTree(handler.handle("GetCrawler", byName, REGION).getEntity()).get("Crawler");+ assertEquals("READY", crawler.get("State").asText());+ assertEquals("SUCCEEDED", crawler.get("LastCrawl").get("Status").asText());+ assertTrue(crawler.get("LastCrawl").get("StartTime").isNumber());++ AwsException idle = assertThrows(AwsException.class, () -> handler.handle("StopCrawler", byName, REGION));+ assertEquals("CrawlerNotRunningException", idle.getErrorCode());++ ObjectNode metricsRequest = mapper.createObjectNode();+ metricsRequest.putArray("CrawlerNameList").add("raw");+ JsonNode metrics = mapper.valueToTree(handler.handle("GetCrawlerMetrics", metricsRequest, REGION).getEntity())+ .get("CrawlerMetricsList").get(0);+ assertEquals("raw", metrics.get("CrawlerName").asText());+ assertTrue(metrics.get("TimeLeftSeconds").isNumber());+ assertTrue(metrics.get("MedianRuntimeSeconds").isNumber());+ assertFalse(metrics.get("StillEstimating").asBoolean());++ ObjectNode schedule = mapper.createObjectNode().put("CrawlerName", "raw");+ assertEquals(0, mapper.valueToTree(handler.handle("StopCrawlerSchedule", schedule, REGION).getEntity()).size());+ JsonNode stopped = mapper.valueToTree(handler.handle("GetCrawler", byName, REGION).getEntity()).get("Crawler");+ assertEquals("NOT_SCHEDULED", stopped.get("Schedule").get("State").asText());+ }++ @Test+ void listCrawlersFiltersOnTagsAndBatchGetCrawlersReportsMissingNames() throws Exception {+ createCrawlerWithTags("tagged", Map.of("team", "data"));+ createCrawlerWithTags("plain", Map.of());++ JsonNode all = mapper.valueToTree(handler.handle("ListCrawlers", mapper.createObjectNode(), REGION).getEntity());+ assertEquals(List.of("plain", "tagged"), mapper.convertValue(all.get("CrawlerNames"), List.class));++ ObjectNode filtered = mapper.createObjectNode();+ filtered.putObject("Tags").put("team", "data");+ JsonNode byTag = mapper.valueToTree(handler.handle("ListCrawlers", filtered, REGION).getEntity());+ assertEquals(List.of("tagged"), mapper.convertValue(byTag.get("CrawlerNames"), List.class));++ ObjectNode batch = mapper.createObjectNode();+ batch.putArray("CrawlerNames").add("tagged").add("absent");+ JsonNode got = mapper.valueToTree(handler.handle("BatchGetCrawlers", batch, REGION).getEntity());+ assertEquals("tagged", got.get("Crawlers").get(0).get("Name").asText());+ assertEquals("READY", got.get("Crawlers").get(0).get("State").asText());+ assertEquals("absent", got.get("CrawlersNotFound").get(0).asText());+ }}