Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(glue): start and stop crawls, crawler metrics, listing and schedules (#4427)

David93c7b1f93fd7
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCrawlerRunTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCrawlerRunTest.java
new file mode 100644
index 0000000000..6932e3dbbf
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCrawlerRunTest.java
@@ -0,0 +1,97 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.glue.model.BatchGetCrawlersResponse;
+import software.amazon.awssdk.services.glue.model.Crawler;
+import software.amazon.awssdk.services.glue.model.CrawlerMetrics;
+import software.amazon.awssdk.services.glue.model.CrawlerNotRunningException;
+import software.amazon.awssdk.services.glue.model.CrawlerState;
+import software.amazon.awssdk.services.glue.model.LastCrawlStatus;
+import software.amazon.awssdk.services.glue.model.ScheduleState;
+
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@DisplayName("Glue crawler runs")
+class GlueCrawlerRunTest {
+
+ private static final String CRAWLER_NAME = TestFixtures.uniqueName("run_crawler");
+ private static final String ROLE = "arn:aws:iam::000000000000:role/glue";
+
+ private static GlueClient glue;
+
+ @BeforeAll
+ static void setup() {
+ glue = TestFixtures.glueClient();
+ glue.createCrawler(r -> r
+ .name(CRAWLER_NAME)
+ .role(ROLE)
+ .targets(t -> t.s3Targets(s -> s.path("s3://raw/events")))
+ .schedule("cron(0 2 * * ? *)")
+ .tags(Map.of("team", "data")));
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (glue == null) {
+ return;
+ }
+ try {
+ glue.deleteCrawler(r -> r.name(CRAWLER_NAME));
+ }
+ catch (Exception ignored) {
+ // Cleanup is best effort: the crawler name is unique to this run and the emulator is disposable.
+ }
+ glue.close();
+ }
+
+ @Test
+ @DisplayName("StartCrawler completes a crawl that GetCrawler and GetCrawlerMetrics parse")
+ void startAndReadACrawl() {
+ glue.startCrawler(r -> r.name(CRAWLER_NAME));
+
+ Crawler crawler = glue.getCrawler(r -> r.name(CRAWLER_NAME)).crawler();
+ assertThat(crawler.state()).isEqualTo(CrawlerState.READY);
+ assertThat(crawler.lastCrawl().status()).isEqualTo(LastCrawlStatus.SUCCEEDED);
+ assertThat(crawler.lastCrawl().startTime()).isNotNull();
+
+ CrawlerMetrics metrics = glue.getCrawlerMetrics(r -> r.crawlerNameList(CRAWLER_NAME))
+ .crawlerMetricsList().get(0);
+ assertThat(metrics.crawlerName()).isEqualTo(CRAWLER_NAME);
+ assertThat(metrics.timeLeftSeconds()).isZero();
+ assertThat(metrics.stillEstimating()).isFalse();
+
+ assertThatThrownBy(() -> glue.stopCrawler(r -> r.name(CRAWLER_NAME)))
+ .isInstanceOf(CrawlerNotRunningException.class);
+ }
+
+ @Test
+ @DisplayName("ListCrawlers filters on tags and BatchGetCrawlers reports missing names")
+ void listAndBatchGetCrawlers() {
+ assertThat(glue.listCrawlers(r -> r.tags(Map.of("team", "data"))).crawlerNames()).contains(CRAWLER_NAME);
+
+ BatchGetCrawlersResponse response = glue.batchGetCrawlers(r -> r.crawlerNames(CRAWLER_NAME, "absent_crawler"));
+ assertThat(response.crawlers()).extracting(Crawler::name).containsExactly(CRAWLER_NAME);
+ assertThat(response.crawlersNotFound()).containsExactly("absent_crawler");
+ }
+
+ @Test
+ @DisplayName("The crawler schedule can be stopped, started and replaced")
+ void schedule() {
+ glue.stopCrawlerSchedule(r -> r.crawlerName(CRAWLER_NAME));
+ assertThat(glue.getCrawler(r -> r.name(CRAWLER_NAME)).crawler().schedule().state())
+ .isEqualTo(ScheduleState.NOT_SCHEDULED);
+
+ glue.startCrawlerSchedule(r -> r.crawlerName(CRAWLER_NAME));
+ glue.updateCrawlerSchedule(r -> r.crawlerName(CRAWLER_NAME).schedule("cron(30 3 * * ? *)"));
+ Crawler crawler = glue.getCrawler(r -> r.name(CRAWLER_NAME)).crawler();
+ assertThat(crawler.schedule().state()).isEqualTo(ScheduleState.SCHEDULED);
+ assertThat(crawler.schedule().scheduleExpression()).isEqualTo("cron(30 3 * * ? *)");
+ }
+}
diff --git a/docs/services/glue.md b/docs/services/glue.md
index 9c49f183e9..9716181c07 100644
--- a/docs/services/glue.md
+++ b/docs/services/glue.md
@@ -165,8 +165,23 @@ waiting in `WAITING`.
| CreateCrawler | Creates a new crawler with specified targets, role, configuration, and optional schedule. |
| GetCrawler | Retrieves metadata for a specified crawler. |
| GetCrawlers | Retrieves metadata for all crawlers defined in the customer account. |
-| UpdateCrawler | Updates a crawler. |
-| DeleteCrawler | Removes a specified crawler from the AWS Glue Data Catalog. |
+| ListCrawlers | Lists crawler names, optionally only those carrying every given tag. |
+| BatchGetCrawlers | Retrieves several crawlers by name and reports the names not found. |
+| UpdateCrawler | Updates a crawler; refused with `CrawlerRunningException` while a crawl runs. |
+| DeleteCrawler | Removes a crawler and its crawl history; refused with `CrawlerRunningException` while a crawl runs. |
+| StartCrawler | Starts a crawl. |
+| StopCrawler | Stops the running crawl, which is then recorded as `CANCELLED`. |
+| GetCrawlerMetrics | Reports time left, last and median runtime for the named crawlers, or for all of them. |
+| UpdateCrawlerSchedule | Replaces a crawler's cron schedule. |
+| StartCrawlerSchedule | Sets a crawler's schedule to `SCHEDULED`. |
+| StopCrawlerSchedule | Sets a crawler's schedule to `NOT_SCHEDULED`. |
+
+Crawls do not read the data store or write tables. A crawl follows the crawler's state machine:
+`GetCrawler` reports `RUNNING` with `CrawlElapsedTime` while it runs, then `READY` with a `LastCrawl`
+of `SUCCEEDED` (or `CANCELLED` after `StopCrawler`). With the default `crawler-run-duration-seconds` of
+0 a crawl finishes as soon as it starts; a positive value keeps the crawler `RUNNING` that long.
+`GetCrawlerMetrics` reports zero tables created, updated and deleted. Schedules are stored and their
+state can be switched, but a schedule never starts a crawl on its own.
#### Classifiers
@@ -231,6 +246,7 @@ Supported schema formats are `AVRO`, `JSON`, and `PROTOBUF`. Compatibility modes
|---|---|---|
| `FLOCI_SERVICES_GLUE_ENABLED` | `true` | Enable or disable the service |
| `FLOCI_SERVICES_GLUE_JOB_RUN_DURATION_SECONDS` | `0` | Seconds a job run stays `RUNNING` before it succeeds; `0` finishes it as soon as it starts |
+| `FLOCI_SERVICES_GLUE_CRAWLER_RUN_DURATION_SECONDS` | `0` | Seconds a crawl keeps the crawler `RUNNING` before it succeeds; `0` finishes it as soon as it starts |
## Integration with Athena
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index d2842ea2d7..3d3cfe5a58 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1961,6 +1961,10 @@ public interface EmulatorConfig {
/** How long a job run stays RUNNING before it succeeds; 0 = a run succeeds as soon as it starts. */
@WithDefault("0")
int jobRunDurationSeconds();
+
+ /** How long a crawl keeps the crawler RUNNING before it succeeds; 0 = a crawl finishes as soon as it starts. */
+ @WithDefault("0")
+ int crawlerRunDurationSeconds();
}
interface SesServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java
new file mode 100644
index 0000000000..47743793fe
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java
@@ -0,0 +1,245 @@
+package io.github.hectorvent.floci.services.glue;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.glue.model.Crawler;
+import io.github.hectorvent.floci.services.glue.model.CrawlerRunRecord;
+import io.github.hectorvent.floci.services.glue.model.LastCrawlInfo;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Glue crawler runs. No data store is crawled and no table is written: a crawl follows the crawler
+ * state machine (READY, RUNNING, back to READY with a LastCrawl status) and settles lazily when
+ * read. With the default {@code crawler-run-duration-seconds} of 0 a crawl finishes as soon as it
+ * starts; a positive duration keeps the crawler RUNNING for that long.
+ *
+ * <p>Every public method is synchronized. Starting and stopping a crawl, and the crawler mutations
+ * AWS refuses while a crawl runs (UpdateCrawler, DeleteCrawler), check the crawl state and act on
+ * it; the store has no atomic compare and set, so those steps must not interleave.
+ */
+@ApplicationScoped
+public class GlueCrawlerRunService {
+
+ private static final Logger LOG = Logger.getLogger(GlueCrawlerRunService.class);
+
+ static final String STATE_READY = "READY";
+ static final String STATE_RUNNING = "RUNNING";
+ static final String STATUS_SUCCEEDED = "SUCCEEDED";
+ static final String STATUS_CANCELLED = "CANCELLED";
+
+ private static final String LOG_GROUP = "/aws-glue/crawlers";
+ // Enough history for a stable median without letting a long-lived crawler grow without bound.
+ private static final int MAX_RUNTIME_HISTORY = 100;
+
+ private final StorageBackend<String, CrawlerRunRecord> recordStore;
+ private final GlueService glueService;
+ private final int runDurationSeconds;
+ private final Clock clock;
+
+ @Inject
+ public GlueCrawlerRunService(StorageFactory storageFactory, GlueService glueService, EmulatorConfig config) {
+ this(storageFactory.create("glue", "crawler_runs.json", new TypeReference<>() {}),
+ glueService, config.services().glue().crawlerRunDurationSeconds(), Clock.systemUTC());
+ }
+
+ GlueCrawlerRunService(StorageBackend<String, CrawlerRunRecord> recordStore, GlueService glueService,
+ int runDurationSeconds, Clock clock) {
+ if (runDurationSeconds < 0) {
+ throw new IllegalArgumentException(
+ "floci.services.glue.crawler-run-duration-seconds must not be negative: " + runDurationSeconds);
+ }
+ this.recordStore = recordStore;
+ this.glueService = glueService;
+ this.runDurationSeconds = runDurationSeconds;
+ this.clock = clock;
+ }
+
+ public synchronized void startCrawler(String name) {
+ glueService.getCrawler(name);
+ CrawlerRunRecord record = settledRecord(name);
+ if (record.getCurrentStart() != null) {
+ throw new AwsException("CrawlerRunningException", "Crawler with name " + name + " has already started", 400);
+ }
+ record.setCurrentStart(clock.instant());
+ record.setCurrentMessagePrefix(UUID.randomUUID().toString());
+ recordStore.put(name, record);
+ LOG.infov("Started Glue crawler {0}", name);
+ settle(record);
+ }
+
+ public synchronized void stopCrawler(String name) {
+ glueService.getCrawler(name);
+ CrawlerRunRecord record = settledRecord(name);
+ if (record.getCurrentStart() == null) {
+ throw new AwsException("CrawlerNotRunningException", "Crawler with name " + name + " isn't running", 400);
+ }
+ finish(record, clock.instant(), STATUS_CANCELLED);
+ LOG.infov("Stopped Glue crawler {0}", name);
+ }
+
+ /** Fills in the crawl state of a crawler read from the store: State, CrawlElapsedTime, LastCrawl. */
+ public synchronized Crawler withRunState(Crawler crawler) {
+ CrawlerRunRecord record = settledRecord(crawler.getName());
+ Instant now = clock.instant();
+ if (record.getCurrentStart() != null) {
+ crawler.setState(STATE_RUNNING);
+ crawler.setCrawlElapsedTime(Duration.between(record.getCurrentStart(), now).toMillis());
+ } else {
+ crawler.setState(STATE_READY);
+ crawler.setCrawlElapsedTime(null);
+ }
+ if (record.getLastStatus() != null) {
+ LastCrawlInfo lastCrawl = new LastCrawlInfo();
+ lastCrawl.setStatus(record.getLastStatus());
+ lastCrawl.setStartTime(record.getLastStart());
+ lastCrawl.setLogGroup(LOG_GROUP);
+ lastCrawl.setLogStream(crawler.getName());
+ lastCrawl.setMessagePrefix(record.getLastMessagePrefix());
+ crawler.setLastCrawl(lastCrawl);
+ } else {
+ crawler.setLastCrawl(null);
+ }
+ return crawler;
+ }
+
+ public synchronized List<Crawler> withRunState(List<Crawler> crawlers) {
+ for (Crawler crawler : crawlers) {
+ withRunState(crawler);
+ }
+ return crawlers;
+ }
+
+ /**
+ * Metrics for the named crawlers, or for every crawler when no names are given. A name that
+ * matches no crawler is skipped: the operation declares no EntityNotFoundException.
+ */
+ public synchronized GlueService.Page<Map<String, Object>> getCrawlerMetrics(List<String> names,
+ Integer maxResults,
+ String nextToken) {
+ List<String> selected = new ArrayList<>();
+ if (names == null || names.isEmpty()) {
+ for (Crawler crawler : glueService.getCrawlers()) {
+ selected.add(crawler.getName());
+ }
+ } else {
+ for (Crawler crawler : glueService.batchGetCrawlers(names).crawlers()) {
+ selected.add(crawler.getName());
+ }
+ }
+ selected.sort(Comparator.naturalOrder());
+ List<Map<String, Object>> metrics = new ArrayList<>();
+ for (String name : selected) {
+ metrics.add(metricsFor(settledRecord(name)));
+ }
+ return glueService.paginate(metrics, maxResults, nextToken);
+ }
+
+ public synchronized void updateCrawler(Crawler update) {
+ requireNotRunning(update.getName());
+ glueService.updateCrawler(update);
+ }
+
+ /** Crawl records belong to their crawler: deleting it removes them, and a running crawler cannot be deleted. */
+ public synchronized void deleteCrawler(String name, String region) {
+ requireNotRunning(name);
+ glueService.deleteCrawler(name, region);
+ recordStore.delete(name);
+ }
+
+ public synchronized void updateCrawlerSchedule(String name, String expression) {
+ glueService.updateCrawlerSchedule(name, expression);
+ }
+
+ public synchronized void startCrawlerSchedule(String name) {
+ glueService.startCrawlerSchedule(name);
+ }
+
+ public synchronized void stopCrawlerSchedule(String name) {
+ glueService.stopCrawlerSchedule(name);
+ }
+
+ private void requireNotRunning(String name) {
+ if (name != null && settledRecord(name).getCurrentStart() != null) {
+ throw new AwsException("CrawlerRunningException", "Crawler with name " + name + " is running", 400);
+ }
+ }
+
+ private Map<String, Object> metricsFor(CrawlerRunRecord record) {
+ Map<String, Object> metrics = new LinkedHashMap<>();
+ metrics.put("CrawlerName", record.getCrawlerName());
+ double timeLeft = 0;
+ if (record.getCurrentStart() != null) {
+ Instant finishesAt = record.getCurrentStart().plusSeconds(runDurationSeconds);
+ timeLeft = Math.max(0, Duration.between(clock.instant(), finishesAt).toMillis() / 1000.0);
+ }
+ metrics.put("TimeLeftSeconds", timeLeft);
+ metrics.put("StillEstimating", false);
+ List<Double> runtimes = record.getRuntimeSeconds();
+ metrics.put("LastRuntimeSeconds", runtimes.isEmpty() ? 0.0 : runtimes.getLast());
+ metrics.put("MedianRuntimeSeconds", median(runtimes));
+ metrics.put("TablesCreated", 0);
+ metrics.put("TablesUpdated", 0);
+ metrics.put("TablesDeleted", 0);
+ return metrics;
+ }
+
+ private static double median(List<Double> values) {
+ if (values.isEmpty()) {
+ return 0.0;
+ }
+ List<Double> sorted = new ArrayList<>(values);
+ sorted.sort(Comparator.naturalOrder());
+ int middle = sorted.size() / 2;
+ return sorted.size() % 2 == 1 ? sorted.get(middle) : (sorted.get(middle - 1) + sorted.get(middle)) / 2.0;
+ }
+
+ private CrawlerRunRecord settledRecord(String name) {
+ CrawlerRunRecord record = recordStore.get(name).orElseGet(() -> {
+ CrawlerRunRecord fresh = new CrawlerRunRecord();
+ fresh.setCrawlerName(name);
+ return fresh;
+ });
+ settle(record);
+ return record;
+ }
+
+ private void settle(CrawlerRunRecord record) {
+ if (record.getCurrentStart() == null) {
+ return;
+ }
+ Instant finishesAt = record.getCurrentStart().plusSeconds(runDurationSeconds);
+ if (!clock.instant().isBefore(finishesAt)) {
+ finish(record, finishesAt, STATUS_SUCCEEDED);
+ }
+ }
+
+ private void finish(CrawlerRunRecord record, Instant finishedAt, String status) {
+ List<Double> runtimes = new ArrayList<>(record.getRuntimeSeconds());
+ runtimes.add(Duration.between(record.getCurrentStart(), finishedAt).toMillis() / 1000.0);
+ if (runtimes.size() > MAX_RUNTIME_HISTORY) {
+ runtimes.removeFirst();
+ }
+ record.setRuntimeSeconds(runtimes);
+ record.setLastStart(record.getCurrentStart());
+ record.setLastStatus(status);
+ record.setLastMessagePrefix(record.getCurrentMessagePrefix());
+ record.setCurrentStart(null);
+ record.setCurrentMessagePrefix(null);
+ recordStore.put(record.getCrawlerName(), record);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 670b124714..5dce251376 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -32,16 +32,19 @@ public class GlueJsonHandler {
private final GlueService glueService;
private final GlueJobRunService jobRunService;
+ private final GlueCrawlerRunService crawlerRunService;
private final GlueSchemaRegistryService schemaRegistryService;
private final ObjectMapper mapper;
@Inject
public GlueJsonHandler(GlueService glueService,
GlueJobRunService jobRunService,
+ GlueCrawlerRunService crawlerRunService,
GlueSchemaRegistryService schemaRegistryService,
ObjectMapper mapper) {
this.glueService = glueService;
this.jobRunService = jobRunService;
+ this.crawlerRunService = crawlerRunService;
this.schemaRegistryService = schemaRegistryService;
this.mapper = mapper;
}
@@ -302,27 +305,75 @@ public class GlueJsonHandler {
}
case "GetCrawler" -> {
GetCrawlerRequest req = mapper.treeToValue(request, GetCrawlerRequest.class);
- yield Response.ok(new GetCrawlerResponse(glueService.getCrawler(req.getName()))).build();
+ yield Response.ok(new GetCrawlerResponse(
+ crawlerRunService.withRunState(glueService.getCrawler(req.getName())))).build();
}
case "GetCrawlers" -> {
GetCrawlersRequest req = mapper.treeToValue(request, GetCrawlersRequest.class);
GlueService.Page<Crawler> page = glueService.getCrawlers(req.getMaxResults(), req.getNextToken());
GetCrawlersResponse res = new GetCrawlersResponse();
- res.setCrawlers(page.items());
+ res.setCrawlers(crawlerRunService.withRunState(page.items()));
res.setNextToken(page.nextToken());
yield Response.ok(res).build();
}
case "UpdateCrawler" -> {
UpdateCrawlerRequest req = mapper.treeToValue(request, UpdateCrawlerRequest.class);
Crawler update = toDomain(req);
- glueService.updateCrawler(update);
+ crawlerRunService.updateCrawler(update);
yield Response.ok().build();
}
case "DeleteCrawler" -> {
DeleteCrawlerRequest req = mapper.treeToValue(request, DeleteCrawlerRequest.class);
- glueService.deleteCrawler(req.getName(), region);
+ crawlerRunService.deleteCrawler(req.getName(), region);
yield Response.ok().build();
}
+ case "ListCrawlers" -> {
+ Map<String, String> tags = request.hasNonNull("Tags")
+ ? mapper.convertValue(request.get("Tags"), STRING_MAP)
+ : null;
+ GlueService.Page<String> page = glueService.listCrawlers(
+ readMaxResults(request), readNextToken(request), tags, region);
+ yield Response.ok(pageResponse("CrawlerNames", page.items(), page.nextToken())).build();
+ }
+ case "BatchGetCrawlers" -> {
+ List<String> names = request.hasNonNull("CrawlerNames")
+ ? mapper.convertValue(request.get("CrawlerNames"), STRING_LIST)
+ : null;
+ GlueService.BatchGetCrawlersResult result = glueService.batchGetCrawlers(names);
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("Crawlers", crawlerRunService.withRunState(result.crawlers()));
+ response.put("CrawlersNotFound", result.crawlersNotFound());
+ yield Response.ok(response).build();
+ }
+ case "StartCrawler" -> {
+ crawlerRunService.startCrawler(request.path("Name").asText(null));
+ yield Response.ok(Map.of()).build();
+ }
+ case "StopCrawler" -> {
+ crawlerRunService.stopCrawler(request.path("Name").asText(null));
+ yield Response.ok(Map.of()).build();
+ }
+ case "GetCrawlerMetrics" -> {
+ List<String> names = request.hasNonNull("CrawlerNameList")
+ ? mapper.convertValue(request.get("CrawlerNameList"), STRING_LIST)
+ : null;
+ GlueService.Page<Map<String, Object>> page = crawlerRunService.getCrawlerMetrics(
+ names, readMaxResults(request), readNextToken(request));
+ yield Response.ok(pageResponse("CrawlerMetricsList", page.items(), page.nextToken())).build();
+ }
+ case "UpdateCrawlerSchedule" -> {
+ crawlerRunService.updateCrawlerSchedule(request.path("CrawlerName").asText(null),
+ request.path("Schedule").asText(null));
+ yield Response.ok(Map.of()).build();
+ }
+ case "StartCrawlerSchedule" -> {
+ crawlerRunService.startCrawlerSchedule(request.path("CrawlerName").asText(null));
+ yield Response.ok(Map.of()).build();
+ }
+ case "StopCrawlerSchedule" -> {
+ crawlerRunService.stopCrawlerSchedule(request.path("CrawlerName").asText(null));
+ yield Response.ok(Map.of()).build();
+ }
case "CreateConnection" -> handleCreateConnection(request, region);
case "GetConnection" -> handleGetConnection(request);
case "GetConnections" -> handleGetConnections(request);
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 14277b5803..c751c56316 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -26,6 +26,7 @@ import io.github.hectorvent.floci.services.glue.model.KeySchemaElement;
import io.github.hectorvent.floci.services.glue.model.Partition;
import io.github.hectorvent.floci.services.glue.model.PartitionIndex;
import io.github.hectorvent.floci.services.glue.model.PartitionIndexDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Schedule;
import io.github.hectorvent.floci.services.glue.model.SchemaReference;
import io.github.hectorvent.floci.services.glue.model.SecurityConfiguration;
import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
@@ -149,6 +150,12 @@ public class GlueService {
private static final String INDEX_STATUS_ACTIVE = "ACTIVE";
private static final String INDEX_STATUS_DELETING = "DELETING";
+ private static final String SCHEDULE_SCHEDULED = "SCHEDULED";
+ private static final String SCHEDULE_NOT_SCHEDULED = "NOT_SCHEDULED";
+ private static final Pattern CRON_FIELD = Pattern.compile("[0-9A-Za-z*?/,#-]+");
+ // BatchGetCrawlers takes a CrawlerNameList, which the API model caps at 100 names.
+ private static final int MAX_BATCH_GET_CRAWLERS = 100;
+
private final StorageBackend<String, Database> databaseStore;
private final StorageBackend<String, Table> tableStore;
private final StorageBackend<String, Table> tableVersionStore;
@@ -2037,6 +2044,110 @@ public class GlueService {
return paginate(all, maxResults, nextToken);
}
+ public Page<String> listCrawlers(Integer maxResults, String nextToken, Map<String, String> tags,
+ String region) {
+ List<String> names = new ArrayList<>();
+ for (Crawler crawler : crawlerStore.scan(k -> true)) {
+ if (tags == null || tags.isEmpty() || hasTags(crawlerArn(region, crawler.getName()), tags, region)) {
+ names.add(crawler.getName());
+ }
+ }
+ names.sort(Comparator.naturalOrder());
+ return paginate(names, maxResults, nextToken);
+ }
+
+ public BatchGetCrawlersResult batchGetCrawlers(List<String> names) {
+ if (names == null) {
+ throw new AwsException("InvalidInputException", "CrawlerNames is required.", 400);
+ }
+ if (names.size() > MAX_BATCH_GET_CRAWLERS) {
+ throw new AwsException("InvalidInputException",
+ "CrawlerNames must contain at most " + MAX_BATCH_GET_CRAWLERS + " names.", 400);
+ }
+ List<Crawler> crawlers = new ArrayList<>();
+ List<String> notFound = new ArrayList<>();
+ for (String name : names) {
+ Optional<Crawler> crawler = name == null ? Optional.empty() : crawlerStore.get(name);
+ if (crawler.isPresent()) {
+ crawlers.add(crawler.get());
+ } else {
+ notFound.add(name);
+ }
+ }
+ return new BatchGetCrawlersResult(crawlers, notFound);
+ }
+
+ public record BatchGetCrawlersResult(List<Crawler> crawlers, List<String> crawlersNotFound) {}
+
+ public void updateCrawlerSchedule(String name, String expression) {
+ validateRequired(name, "CrawlerName");
+ Crawler crawler = getCrawler(name);
+ if (expression == null || expression.isBlank()) {
+ crawler.setSchedule(null);
+ } else {
+ validateCronExpression(expression);
+ Schedule schedule = crawler.getSchedule() != null ? crawler.getSchedule() : new Schedule();
+ schedule.setScheduleExpression(expression);
+ if (schedule.getState() == null) {
+ schedule.setState(SCHEDULE_SCHEDULED);
+ }
+ crawler.setSchedule(schedule);
+ }
+ crawler.setLastUpdated(Instant.now());
+ crawlerStore.put(name, crawler);
+ }
+
+ /**
+ * Glue schedules are AWS cron expressions: cron(Minutes Hours Day-of-month Month Day-of-week Year),
+ * with {@code ?} in exactly one of the two day fields.
+ */
+ static void validateCronExpression(String expression) {
+ String message = "Schedule must be a cron expression, for example cron(15 12 * * ? *).";
+ if (expression == null || !expression.startsWith("cron(") || !expression.endsWith(")")) {
+ throw new AwsException("InvalidInputException", message, 400);
+ }
+ String[] fields = expression.substring(5, expression.length() - 1).trim().split("\\s+");
+ if (fields.length != 6) {
+ throw new AwsException("InvalidInputException", message, 400);
+ }
+ for (String field : fields) {
+ if (!CRON_FIELD.matcher(field).matches()) {
+ throw new AwsException("InvalidInputException", message, 400);
+ }
+ }
+ if ("?".equals(fields[2]) == "?".equals(fields[4])) {
+ throw new AwsException("InvalidInputException",
+ "Exactly one of day-of-month and day-of-week must be '?' in " + expression, 400);
+ }
+ }
+
+ public void startCrawlerSchedule(String name) {
+ validateRequired(name, "CrawlerName");
+ Crawler crawler = getCrawler(name);
+ Schedule schedule = crawler.getSchedule();
+ if (schedule == null || schedule.getScheduleExpression() == null) {
+ throw new AwsException("NoScheduleException", "Crawler " + name + " has no schedule.", 400);
+ }
+ if (SCHEDULE_SCHEDULED.equals(schedule.getState())) {
+ throw new AwsException("SchedulerRunningException",
+ "The schedule of crawler " + name + " is already running.", 400);
+ }
+ schedule.setState(SCHEDULE_SCHEDULED);
+ crawlerStore.put(name, crawler);
+ }
+
+ public void stopCrawlerSchedule(String name) {
+ validateRequired(name, "CrawlerName");
+ Crawler crawler = getCrawler(name);
+ Schedule schedule = crawler.getSchedule();
+ if (schedule == null || !SCHEDULE_SCHEDULED.equals(schedule.getState())) {
+ throw new AwsException("SchedulerNotRunningException",
+ "The schedule of crawler " + name + " is not running.", 400);
+ }
+ schedule.setState(SCHEDULE_NOT_SCHEDULED);
+ crawlerStore.put(name, crawler);
+ }
+
public void updateCrawler(Crawler update) {
validateRequired(update.getName(), "Name");
// Jobs and crawlers skip normalizeName because AWS preserves their case
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java
new file mode 100644
index 0000000000..8fc5e96c68
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java
@@ -0,0 +1,45 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Floci's own bookkeeping for a crawler's crawls; never sent on the wire. The crawl in progress (if
+ * any), the last finished crawl, and the runtimes GetCrawlerMetrics summarises.
+ */
+@RegisterForReflection
+public class CrawlerRunRecord {
+ private String crawlerName;
+ private Instant currentStart;
+ private String currentMessagePrefix;
+ private Instant lastStart;
+ private String lastStatus;
+ private String lastMessagePrefix;
+ private List<Double> runtimeSeconds = new ArrayList<>();
+
+ public CrawlerRunRecord() {}
+
+ public String getCrawlerName() { return crawlerName; }
+ public void setCrawlerName(String crawlerName) { this.crawlerName = crawlerName; }
+
+ public Instant getCurrentStart() { return currentStart; }
+ public void setCurrentStart(Instant currentStart) { this.currentStart = currentStart; }
+
+ public String getCurrentMessagePrefix() { return currentMessagePrefix; }
+ public void setCurrentMessagePrefix(String currentMessagePrefix) { this.currentMessagePrefix = currentMessagePrefix; }
+
+ public Instant getLastStart() { return lastStart; }
+ public void setLastStart(Instant lastStart) { this.lastStart = lastStart; }
+
+ public String getLastStatus() { return lastStatus; }
+ public void setLastStatus(String lastStatus) { this.lastStatus = lastStatus; }
+
+ public String getLastMessagePrefix() { return lastMessagePrefix; }
+ public void setLastMessagePrefix(String lastMessagePrefix) { this.lastMessagePrefix = lastMessagePrefix; }
+
+ public List<Double> getRuntimeSeconds() { return runtimeSeconds; }
+ public void setRuntimeSeconds(List<Double> runtimeSeconds) { this.runtimeSeconds = runtimeSeconds; }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 882d7bcd02..ee4f541e27 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -510,6 +510,7 @@ floci:
glue:
enabled: true
job-run-duration-seconds: 0 # 0 = job runs succeed as soon as they start
+ crawler-run-duration-seconds: 0 # 0 = crawls finish as soon as they start
ses:
enabled: true
# smtp-host: mailpit # SMTP server for email relay (empty = store only)
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunIntegrationTest.java
new file mode 100644
index 0000000000..e28c24b872
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunIntegrationTest.java
@@ -0,0 +1,63 @@
+package io.github.hectorvent.floci.services.glue;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+
+@QuarkusTest
+class GlueCrawlerRunIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String ROLE = "arn:aws:iam::000000000000:role/my-role";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ private static ValidatableResponse call(String action, String body) {
+ return given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue." + action)
+ .body(body)
+ .when().post("/")
+ .then();
+ }
+
+ @Test
+ void crawlRunsToCompletionAndItsHistoryGoesWithTheCrawler() {
+ String name = "crawl-" + UUID.randomUUID().toString().substring(0, 8);
+ String byName = "{ \"Name\": \"" + name + "\" }";
+ call("CreateCrawler", """
+ { "Name": "%s", "Role": "%s", "Targets": { "S3Targets": [ { "Path": "s3://raw/data" } ] } }
+ """.formatted(name, ROLE)).statusCode(200);
+
+ call("StartCrawler", byName).statusCode(200);
+
+ call("GetCrawler", byName)
+ .statusCode(200)
+ .body("Crawler.State", equalTo("READY"))
+ .body("Crawler.LastCrawl.Status", equalTo("SUCCEEDED"));
+
+ call("ListCrawlers", "{}")
+ .statusCode(200)
+ .body("CrawlerNames", hasItem(name));
+
+ call("StopCrawler", byName)
+ .statusCode(400)
+ .body("__type", equalTo("CrawlerNotRunningException"));
+
+ call("DeleteCrawler", byName).statusCode(200);
+
+ call("StartCrawler", byName)
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java
new file mode 100644
index 0000000000..0b94ca1413
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java
@@ -0,0 +1,334 @@
+package io.github.hectorvent.floci.services.glue;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.glue.model.Crawler;
+import io.github.hectorvent.floci.services.glue.model.CrawlerRunRecord;
+import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;
+import io.github.hectorvent.floci.services.glue.model.S3Target;
+import io.github.hectorvent.floci.services.glue.model.Schedule;
+import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.kms.KmsService;
+import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class GlueCrawlerRunServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT_ID = "000000000000";
+ private static final String ROLE = "arn:aws:iam::000000000000:role/glue";
+ private static final long CLOCK_TICKS_MILLIS = 100;
+
+ private GlueService glueService;
+ private InMemoryStorage<String, CrawlerRunRecord> recordStore;
+ private MutableClock clock;
+
+ @BeforeEach
+ void setUp() {
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);
+ StorageFactory storageFactory = new InMemoryStorageFactory();
+ glueService = new GlueService(storageFactory, new GlueSchemaRegistryService(storageFactory, regionResolver),
+ regionResolver, new ResourceGroupsTaggingService(storageFactory),
+ new KmsService(storageFactory, regionResolver));
+ recordStore = new InMemoryStorage<>();
+ clock = new MutableClock();
+ }
+
+ /**
+ * The shared test clock moves 1 ms on every read, so a stop "now" lands a few milliseconds after
+ * the whole seconds the test advanced.
+ */
+ private static void assertSeconds(double expected, Object actual) {
+ assertEquals(expected, (Double) actual, CLOCK_TICKS_MILLIS / 1000.0);
+ }
+
+ private GlueCrawlerRunService service(int runDurationSeconds) {
+ return new GlueCrawlerRunService(recordStore, glueService, runDurationSeconds, clock);
+ }
+
+ private void createCrawler(String name, String scheduleExpression) {
+ S3Target target = new S3Target();
+ target.setPath("s3://raw/" + name);
+ CrawlerTargets targets = new CrawlerTargets();
+ targets.setS3Targets(List.of(target));
+ Crawler crawler = new Crawler();
+ crawler.setName(name);
+ crawler.setRole(ROLE);
+ crawler.setTargets(targets);
+ if (scheduleExpression != null) {
+ Schedule schedule = new Schedule();
+ schedule.setScheduleExpression(scheduleExpression);
+ schedule.setState("SCHEDULED");
+ crawler.setSchedule(schedule);
+ }
+ glueService.createCrawler(crawler, null, REGION);
+ }
+
+ private Crawler read(GlueCrawlerRunService service, String name) {
+ return service.withRunState(glueService.getCrawler(name));
+ }
+
+ private Map<String, Object> metrics(GlueCrawlerRunService service, String name) {
+ return service.getCrawlerMetrics(List.of(name), null, null).items().getFirst();
+ }
+
+ @Test
+ void neverStartedCrawlerIsReadyWithNoLastCrawl() {
+ createCrawler("raw", null);
+
+ Crawler crawler = read(service(0), "raw");
+
+ assertEquals("READY", crawler.getState());
+ assertNull(crawler.getLastCrawl());
+ assertNull(crawler.getCrawlElapsedTime());
+ }
+
+ @Test
+ void crawlFinishesAsSoonAsItStartsByDefault() {
+ createCrawler("raw", null);
+ GlueCrawlerRunService service = service(0);
+
+ service.startCrawler("raw");
+ Crawler crawler = read(service, "raw");
+
+ assertEquals("READY", crawler.getState());
+ assertEquals("SUCCEEDED", crawler.getLastCrawl().getStatus());
+ assertNotNull(crawler.getLastCrawl().getStartTime());
+ assertEquals("/aws-glue/crawlers", crawler.getLastCrawl().getLogGroup());
+ assertEquals("raw", crawler.getLastCrawl().getLogStream());
+ assertEquals(0.0, metrics(service, "raw").get("LastRuntimeSeconds"));
+ }
+
+ @Test
+ void crawlerStaysRunningUntilTheConfiguredDurationHasPassed() {
+ createCrawler("raw", null);
+ GlueCrawlerRunService service = service(60);
+ service.startCrawler("raw");
+
+ clock.advance(Duration.ofSeconds(30));
+ Crawler running = read(service, "raw");
+ assertEquals("RUNNING", running.getState());
+ assertEquals(30_000L, running.getCrawlElapsedTime(), CLOCK_TICKS_MILLIS);
+ assertNull(running.getLastCrawl());
+ assertSeconds(30.0, metrics(service, "raw").get("TimeLeftSeconds"));
+
+ clock.advance(Duration.ofSeconds(45));
+ Crawler finished = read(service, "raw");
+ assertEquals("READY", finished.getState());
+ assertEquals("SUCCEEDED", finished.getLastCrawl().getStatus());
+ assertEquals(60.0, metrics(service, "raw").get("LastRuntimeSeconds"));
+ assertEquals(0.0, metrics(service, "raw").get("TimeLeftSeconds"));
+ }
+
+ @Test
+ void startWhileRunningAndStopWhileIdleAreRefused() {
+ createCrawler("raw", null);
+ GlueCrawlerRunService service = service(60);
+
+ assertEquals("CrawlerNotRunningException",
+ assertThrows(AwsException.class, () -> service.stopCrawler("raw")).getErrorCode());
+ service.startCrawler("raw");
+ assertEquals("CrawlerRunningException",
+ assertThrows(AwsException.class, () -> service.startCrawler("raw")).getErrorCode());
+ assertEquals("EntityNotFoundException",
+ assertThrows(AwsException.class, () -> service.startCrawler("missing")).getErrorCode());
+ }
+
+ @Test
+ void stoppingACrawlRecordsItAsCancelled() {
+ createCrawler("raw", null);
+ GlueCrawlerRunService service = service(60);
+ service.startCrawler("raw");
+
+ clock.advance(Duration.ofSeconds(10));
+ service.stopCrawler("raw");
+ Crawler crawler = read(service, "raw");
+
+ assertEquals("READY", crawler.getState());
+ assertEquals("CANCELLED", crawler.getLastCrawl().getStatus());
+ assertSeconds(10.0, metrics(service, "raw").get("LastRuntimeSeconds"));
+ }
+
+ @Test
+ void metricsReportTheLastAndMedianRuntime() {
+ createCrawler("raw", null);
+ GlueCrawlerRunService service = service(600);
+ for (int seconds : new int[] {10, 30, 20}) {
+ service.startCrawler("raw");
+ clock.advance(Duration.ofSeconds(seconds));
+ service.stopCrawler("raw");
+ clock.advance(Duration.ofSeconds(1));
+ }
+
+ Map<String, Object> metrics = metrics(service, "raw");
+
+ assertEquals("raw", metrics.get("CrawlerName"));
+ assertSeconds(20.0, metrics.get("LastRuntimeSeconds"));
+ assertSeconds(20.0, metrics.get("MedianRuntimeSeconds"));
+ assertEquals(false, metrics.get("StillEstimating"));
+ assertEquals(0, metrics.get("TablesCreated"));
+ }
+
+ @Test
+ void metricsCoverEveryCrawlerWhenNoNamesAreGivenAndSkipUnknownNames() {
+ createCrawler("b", null);
+ createCrawler("a", null);
+ GlueCrawlerRunService service = service(0);
+
+ List<Map<String, Object>> all = service.getCrawlerMetrics(null, null, null).items();
+ assertEquals(List.of("a", "b"), all.stream().map(m -> m.get("CrawlerName")).toList());
+
+ List<Map<String, Object>> named = service.getCrawlerMetrics(List.of("b", "missing"), null, null).items();
+ assertEquals(List.of("b"), named.stream().map(m -> m.get("CrawlerName")).toList());
+ }
+
+ @Test
+ void runningCrawlerCannotBeUpdatedOrDeleted() {
+ createCrawler("raw", null);
+ GlueCrawlerRunService service = service(60);
+ service.startCrawler("raw");
+ Crawler update = new Crawler();
+ update.setName("raw");
+ update.setDescription("changed");
+
+ assertEquals("CrawlerRunningException",
+ assertThrows(AwsException.class, () -> service.updateCrawler(update)).getErrorCode());
+ assertEquals("CrawlerRunningException",
+ assertThrows(AwsException.class, () -> service.deleteCrawler("raw", REGION)).getErrorCode());
+
+ clock.advance(Duration.ofSeconds(60));
+ service.updateCrawler(update);
+ assertEquals("changed", glueService.getCrawler("raw").getDescription());
+ }
+
+ @Test
+ void deletingACrawlerRemovesItsCrawlHistory() {
+ createCrawler("raw", null);
+ GlueCrawlerRunService service = service(0);
+ service.startCrawler("raw");
+
+ service.deleteCrawler("raw", REGION);
+ createCrawler("raw", null);
+
+ assertTrue(recordStore.scan(key -> true).isEmpty());
+ assertNull(read(service, "raw").getLastCrawl());
+ }
+
+ @Test
+ void scheduleCanBeStoppedStartedAndReplaced() {
+ createCrawler("nightly", "cron(0 2 * * ? *)");
+ createCrawler("adhoc", null);
+ GlueCrawlerRunService service = service(0);
+
+ service.stopCrawlerSchedule("nightly");
+ assertEquals("NOT_SCHEDULED", glueService.getCrawler("nightly").getSchedule().getState());
+ assertEquals("SchedulerNotRunningException",
+ assertThrows(AwsException.class, () -> service.stopCrawlerSchedule("nightly")).getErrorCode());
+
+ service.startCrawlerSchedule("nightly");
+ assertEquals("SCHEDULED", glueService.getCrawler("nightly").getSchedule().getState());
+ assertEquals("SchedulerRunningException",
+ assertThrows(AwsException.class, () -> service.startCrawlerSchedule("nightly")).getErrorCode());
+
+ service.updateCrawlerSchedule("nightly", "cron(30 3 * * ? *)");
+ assertEquals("cron(30 3 * * ? *)", glueService.getCrawler("nightly").getSchedule().getScheduleExpression());
+ assertEquals("InvalidInputException", assertThrows(AwsException.class,
+ () -> service.updateCrawlerSchedule("nightly", "every day")).getErrorCode());
+
+ assertEquals("NoScheduleException",
+ assertThrows(AwsException.class, () -> service.startCrawlerSchedule("adhoc")).getErrorCode());
+ }
+
+ @Test
+ void scheduleMustBeAWellFormedAwsCronExpression() {
+ createCrawler("nightly", "cron(0 2 * * ? *)");
+ GlueCrawlerRunService service = service(0);
+
+ for (String malformed : List.of("cron(foo)", "cron(0 2 * * *)", "cron(0 2 * * * *)",
+ "cron(0 2 ? * ? *)", "cron(0 2 * * ? *) ", "rate(1 day)", "cron(0 2 * * ? * $)")) {
+ assertEquals("InvalidInputException", assertThrows(AwsException.class,
+ () -> service.updateCrawlerSchedule("nightly", malformed)).getErrorCode(), malformed);
+ }
+ for (String valid : List.of("cron(15 12 * * ? *)", "cron(0/5 8-17 ? * MON-FRI *)", "cron(0 0 L * ? 2026)")) {
+ service.updateCrawlerSchedule("nightly", valid);
+ assertEquals(valid, glueService.getCrawler("nightly").getSchedule().getScheduleExpression());
+ }
+ }
+
+ @Test
+ void concurrentStartsLetExactlyOneCrawlRun() throws Exception {
+ createCrawler("raw", null);
+ GlueCrawlerRunService service = service(60);
+ int threads = 16;
+ CountDownLatch go = new CountDownLatch(1);
+ AtomicInteger started = new AtomicInteger();
+ AtomicInteger refused = new AtomicInteger();
+ ExecutorService pool = Executors.newFixedThreadPool(threads);
+ try {
+ List<Future<?>> futures = new ArrayList<>();
+ for (int i = 0; i < threads; i++) {
+ futures.add(pool.submit(() -> {
+ go.await();
+ try {
+ service.startCrawler("raw");
+ started.incrementAndGet();
+ } catch (AwsException expected) {
+ // CrawlerRunningException is the outcome under test for all but one start.
+ refused.incrementAndGet();
+ }
+ return null;
+ }));
+ }
+ go.countDown();
+ for (Future<?> future : futures) {
+ future.get(10, TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+
+ assertEquals(1, started.get());
+ assertEquals(threads - 1, refused.get());
+ }
+
+ @Test
+ void negativeRunDurationIsRefusedAtConstruction() {
+ assertThrows(IllegalArgumentException.class, () -> service(-1));
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return AccountAwareStorageBackend.inMemory(ACCOUNT_ID);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJobRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJobRunServiceTest.java
index 178af64361..d7e8f048bf 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJobRunServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJobRunServiceTest.java
@@ -14,13 +14,12 @@ import io.github.hectorvent.floci.services.glue.model.JobRun;
import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
import io.github.hectorvent.floci.services.kms.KmsService;
import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;
+import io.github.hectorvent.floci.testing.MutableClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import java.time.Clock;
+import java.time.Duration;
import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -45,7 +44,6 @@ class GlueJobRunServiceTest {
private static final String REGION = "us-east-1";
private static final String ACCOUNT_ID = "000000000000";
private static final String ROLE = "arn:aws:iam::000000000000:role/glue";
- private static final Instant T0 = Instant.parse("2026-09-25T12:00:00Z");
private GlueService glueService;
private InMemoryStorage<String, JobRun> runStore;
@@ -59,7 +57,7 @@ class GlueJobRunServiceTest {
regionResolver, new ResourceGroupsTaggingService(storageFactory),
new KmsService(storageFactory, regionResolver));
runStore = new InMemoryStorage<>();
- clock = new MutableClock(T0);
+ clock = new MutableClock();
}
private GlueJobRunService service(int runDurationSeconds) {
@@ -87,12 +85,15 @@ class GlueJobRunServiceTest {
void runSucceedsAsSoonAsItStartsByDefault() {
createJob("etl", null, null);
+ Instant before = clock.instant();
JobRun run = service(0).startJobRun("etl", null, new JobRun());
+ Instant after = clock.instant();
assertTrue(run.getId().matches("jr_[0-9a-f]{64}"), run.getId());
assertEquals("SUCCEEDED", run.getJobRunState());
- assertEquals(T0, run.getStartedOn());
- assertEquals(T0, run.getCompletedOn());
+ assertTrue(run.getStartedOn().isAfter(before) && run.getStartedOn().isBefore(after),
+ "StartedOn " + run.getStartedOn() + " should be read from the clock during StartJobRun");
+ assertEquals(run.getStartedOn(), run.getCompletedOn());
assertEquals(0, run.getExecutionTime());
assertEquals(0, run.getAttempt());
assertEquals("G.1X", run.getWorkerType());
@@ -124,17 +125,17 @@ class GlueJobRunServiceTest {
void runStaysRunningUntilTheConfiguredDurationHasPassed() {
createJob("etl", null, null);
GlueJobRunService service = service(60);
- String runId = service.startJobRun("etl", null, new JobRun()).getId();
+ JobRun started = service.startJobRun("etl", null, new JobRun());
- clock.set(T0.plusSeconds(59));
- JobRun running = service.getJobRun("etl", runId);
+ clock.advance(Duration.ofSeconds(59));
+ JobRun running = service.getJobRun("etl", started.getId());
assertEquals("RUNNING", running.getJobRunState());
assertNull(running.getCompletedOn());
- clock.set(T0.plusSeconds(90));
- JobRun finished = service.getJobRun("etl", runId);
+ clock.advance(Duration.ofSeconds(31));
+ JobRun finished = service.getJobRun("etl", started.getId());
assertEquals("SUCCEEDED", finished.getJobRunState());
- assertEquals(T0.plusSeconds(60), finished.getCompletedOn());
+ assertEquals(started.getStartedOn().plusSeconds(60), finished.getCompletedOn());
assertEquals(60, finished.getExecutionTime());
}
@@ -142,13 +143,13 @@ class GlueJobRunServiceTest {
void runLongerThanTheJobTimeoutEndsInTimeout() {
createJob("etl", null, 1);
GlueJobRunService service = service(120);
- String runId = service.startJobRun("etl", null, new JobRun()).getId();
+ JobRun started = service.startJobRun("etl", null, new JobRun());
- clock.set(T0.plusSeconds(61));
- JobRun run = service.getJobRun("etl", runId);
+ clock.advance(Duration.ofSeconds(61));
+ JobRun run = service.getJobRun("etl", started.getId());
assertEquals("TIMEOUT", run.getJobRunState());
- assertEquals(T0.plusSeconds(60), run.getCompletedOn());
+ assertEquals(started.getStartedOn().plusSeconds(60), run.getCompletedOn());
assertEquals(60, run.getExecutionTime());
}
@@ -162,7 +163,7 @@ class GlueJobRunServiceTest {
() -> service.startJobRun("etl", null, new JobRun()));
assertEquals("ConcurrentRunsExceededException", error.getErrorCode());
- clock.set(T0.plusSeconds(60));
+ clock.advance(Duration.ofSeconds(60));
assertEquals("RUNNING", service.startJobRun("etl", null, new JobRun()).getJobRunState());
}
@@ -187,9 +188,9 @@ class GlueJobRunServiceTest {
createJob("etl", 5, null);
GlueJobRunService service = service(60);
String finished = service.startJobRun("etl", null, new JobRun()).getId();
- clock.set(T0.plusSeconds(60));
+ clock.advance(Duration.ofSeconds(60));
String stillRunning = service.startJobRun("etl", null, new JobRun()).getId();
- clock.set(T0.plusSeconds(70));
+ clock.advance(Duration.ofSeconds(10));
GlueJobRunService.StopResult result =
service.batchStopJobRun("etl", List.of(stillRunning, finished, "jr_missing"));
@@ -209,7 +210,7 @@ class GlueJobRunServiceTest {
createJob("etl", 10, null);
GlueJobRunService service = service(0);
String first = service.startJobRun("etl", null, new JobRun()).getId();
- clock.set(T0.plusSeconds(1));
+ clock.advance(Duration.ofSeconds(1));
String second = service.startJobRun("etl", null, new JobRun()).getId();
GlueService.Page<JobRun> page = service.getJobRuns("etl", 1, null);
@@ -435,31 +436,4 @@ class GlueJobRunServiceTest {
return AccountAwareStorageBackend.inMemory(ACCOUNT_ID);
}
}
-
- private static final class MutableClock extends Clock {
- private Instant now;
-
- private MutableClock(Instant now) {
- this.now = now;
- }
-
- void set(Instant now) {
- this.now = now;
- }
-
- @Override
- public ZoneOffset getZone() {
- return ZoneOffset.UTC;
- }
-
- @Override
- public Clock withZone(ZoneId zone) {
- return this;
- }
-
- @Override
- public Instant instant() {
- return now;
- }
- }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
index b28c93ec7d..462d94d4d1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
@@ -47,6 +47,7 @@ class GlueJsonHandlerEmptyListTest {
new ResourceGroupsTaggingService(storageFactory), new KmsService(storageFactory, regionResolver));
handler = new GlueJsonHandler(glueService,
new GlueJobRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC()),
+ new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC()),
schemaRegistryService, mapper);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
index 88ab00f401..9c93db2d33 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
@@ -48,6 +48,7 @@ class GlueJsonHandlerTest {
new KmsService(storageFactory, regionResolver));
handler = new GlueJsonHandler(glueService,
new GlueJobRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC()),
+ new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC()),
schemaRegistryService, mapper);
}
@@ -875,4 +876,75 @@ class GlueJsonHandlerTest {
assertEquals("tagged", got.get("Jobs").get(0).get("Name").asText());
assertEquals("absent", got.get("JobsNotFound").get(0).asText());
}
+
+ private void createCrawlerWithTags(String name, Map<String, String> tags) throws Exception {
+ ObjectNode create = mapper.createObjectNode();
+ create.put("Name", name);
+ create.put("Role", "arn:aws:iam::000000000000:role/glue");
+ create.putObject("Targets").putArray("S3Targets").addObject().put("Path", "s3://raw/" + name);
+ create.put("Schedule", "cron(0 2 * * ? *)");
+ ObjectNode tagNode = create.putObject("Tags");
+ tags.forEach(tagNode::put);
+ assertEquals(200, handler.handle("CreateCrawler", create, REGION).getStatus());
+ }
+
+ /**
+ * The crawl lifecycle as a client polls it: StartCrawler and StopCrawler answer with empty
+ * bodies, GetCrawler carries State and a LastCrawl with a numeric StartTime, and
+ * GetCrawlerMetrics reports numbers, never nulls.
+ */
+ @Test
+ void crawlerRunOperationsAnswerWithTheDocumentedBodies() throws Exception {
+ createCrawlerWithTags("raw", Map.of());
+ ObjectNode byName = mapper.createObjectNode().put("Name", "raw");
+
+ JsonNode before = mapper.valueToTree(handler.handle("GetCrawler", byName, REGION).getEntity()).get("Crawler");
+ assertEquals("READY", before.get("State").asText());
+
+ Response started = handler.handle("StartCrawler", byName, REGION);
+ assertEquals(0, mapper.valueToTree(started.getEntity()).size());
+
+ JsonNode crawler = mapper.valueToTree(handler.handle("GetCrawler", byName, REGION).getEntity()).get("Crawler");
+ assertEquals("READY", crawler.get("State").asText());
+ assertEquals("SUCCEEDED", crawler.get("LastCrawl").get("Status").asText());
+ assertTrue(crawler.get("LastCrawl").get("StartTime").isNumber());
+
+ AwsException idle = assertThrows(AwsException.class, () -> handler.handle("StopCrawler", byName, REGION));
+ assertEquals("CrawlerNotRunningException", idle.getErrorCode());
+
+ ObjectNode metricsRequest = mapper.createObjectNode();
+ metricsRequest.putArray("CrawlerNameList").add("raw");
+ JsonNode metrics = mapper.valueToTree(handler.handle("GetCrawlerMetrics", metricsRequest, REGION).getEntity())
+ .get("CrawlerMetricsList").get(0);
+ assertEquals("raw", metrics.get("CrawlerName").asText());
+ assertTrue(metrics.get("TimeLeftSeconds").isNumber());
+ assertTrue(metrics.get("MedianRuntimeSeconds").isNumber());
+ assertFalse(metrics.get("StillEstimating").asBoolean());
+
+ ObjectNode schedule = mapper.createObjectNode().put("CrawlerName", "raw");
+ assertEquals(0, mapper.valueToTree(handler.handle("StopCrawlerSchedule", schedule, REGION).getEntity()).size());
+ JsonNode stopped = mapper.valueToTree(handler.handle("GetCrawler", byName, REGION).getEntity()).get("Crawler");
+ assertEquals("NOT_SCHEDULED", stopped.get("Schedule").get("State").asText());
+ }
+
+ @Test
+ void listCrawlersFiltersOnTagsAndBatchGetCrawlersReportsMissingNames() throws Exception {
+ createCrawlerWithTags("tagged", Map.of("team", "data"));
+ createCrawlerWithTags("plain", Map.of());
+
+ JsonNode all = mapper.valueToTree(handler.handle("ListCrawlers", mapper.createObjectNode(), REGION).getEntity());
+ assertEquals(List.of("plain", "tagged"), mapper.convertValue(all.get("CrawlerNames"), List.class));
+
+ ObjectNode filtered = mapper.createObjectNode();
+ filtered.putObject("Tags").put("team", "data");
+ JsonNode byTag = mapper.valueToTree(handler.handle("ListCrawlers", filtered, REGION).getEntity());
+ assertEquals(List.of("tagged"), mapper.convertValue(byTag.get("CrawlerNames"), List.class));
+
+ ObjectNode batch = mapper.createObjectNode();
+ batch.putArray("CrawlerNames").add("tagged").add("absent");
+ JsonNode got = mapper.valueToTree(handler.handle("BatchGetCrawlers", batch, REGION).getEntity());
+ assertEquals("tagged", got.get("Crawlers").get(0).get("Name").asText());
+ assertEquals("READY", got.get("Crawlers").get(0).get("State").asText());
+ assertEquals("absent", got.get("CrawlersNotFound").get(0).asText());
+ }
}