Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(glue): create and run workflows (#4471)

Davidbc15516cc0af
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueWorkflowTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueWorkflowTest.java
new file mode 100644
index 0000000000..a5e0e94764
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueWorkflowTest.java
@@ -0,0 +1,104 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.glue.model.JobRunState;
+import software.amazon.awssdk.services.glue.model.LogicalOperator;
+import software.amazon.awssdk.services.glue.model.Node;
+import software.amazon.awssdk.services.glue.model.NodeType;
+import software.amazon.awssdk.services.glue.model.TriggerType;
+import software.amazon.awssdk.services.glue.model.Workflow;
+import software.amazon.awssdk.services.glue.model.WorkflowRun;
+import software.amazon.awssdk.services.glue.model.WorkflowRunStatus;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@DisplayName("Glue workflows")
+class GlueWorkflowTest {
+
+ private static final String WORKFLOW = TestFixtures.uniqueName("workflow");
+ private static final String EXTRACT = TestFixtures.uniqueName("workflow_extract");
+ private static final String LOAD = TestFixtures.uniqueName("workflow_load");
+ private static final String START = TestFixtures.uniqueName("workflow_start");
+ private static final String THEN = TestFixtures.uniqueName("workflow_then");
+ private static final String ROLE = "arn:aws:iam::000000000000:role/glue";
+
+ private static GlueClient glue;
+
+ @BeforeAll
+ static void setup() {
+ glue = TestFixtures.glueClient();
+ for (String job : List.of(EXTRACT, LOAD)) {
+ glue.createJob(r -> r.name(job).role(ROLE).command(c -> c.name("glueetl")));
+ }
+ glue.createWorkflow(r -> r.name(WORKFLOW).defaultRunProperties(Map.of("env", "dev")));
+ glue.createTrigger(r -> r.name(START).workflowName(WORKFLOW).type(TriggerType.ON_DEMAND)
+ .actions(a -> a.jobName(EXTRACT)));
+ glue.createTrigger(r -> r.name(THEN).workflowName(WORKFLOW).type(TriggerType.CONDITIONAL)
+ .startOnCreation(true).actions(a -> a.jobName(LOAD))
+ .predicate(p -> p.conditions(c -> c.logicalOperator(LogicalOperator.EQUALS)
+ .jobName(EXTRACT).state(JobRunState.SUCCEEDED))));
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (glue == null) {
+ return;
+ }
+ for (String trigger : List.of(START, THEN)) {
+ try {
+ glue.deleteTrigger(r -> r.name(trigger));
+ }
+ catch (Exception ignored) {
+ // Cleanup is best effort: the names are unique to this run and the emulator is disposable.
+ }
+ }
+ try {
+ glue.deleteWorkflow(r -> r.name(WORKFLOW));
+ }
+ catch (Exception ignored) {
+ // Cleanup is best effort, as above.
+ }
+ for (String job : List.of(EXTRACT, LOAD)) {
+ try {
+ glue.deleteJob(r -> r.jobName(job));
+ }
+ catch (Exception ignored) {
+ // Cleanup is best effort, as above.
+ }
+ }
+ glue.close();
+ }
+
+ @Test
+ @DisplayName("StartWorkflowRun runs the triggers to completion and GetWorkflowRun parses")
+ void runToCompletion() {
+ String runId = glue.startWorkflowRun(r -> r.name(WORKFLOW).runProperties(Map.of("day", "2026-09-26"))).runId();
+
+ WorkflowRun run = glue.getWorkflowRun(r -> r.name(WORKFLOW).runId(runId).includeGraph(true)).run();
+ assertThat(run.status()).isEqualTo(WorkflowRunStatus.COMPLETED);
+ assertThat(run.statistics().succeededActions()).isEqualTo(2);
+ assertThat(run.workflowRunProperties()).containsEntry("env", "dev").containsEntry("day", "2026-09-26");
+ Node load = run.graph().nodes().stream().filter(n -> LOAD.equals(n.name())).findFirst().orElseThrow();
+ assertThat(load.jobDetails().jobRuns()).hasSize(1);
+ assertThat(load.jobDetails().jobRuns().get(0).jobRunState()).isEqualTo(JobRunState.SUCCEEDED);
+ }
+
+ @Test
+ @DisplayName("GetWorkflow with IncludeGraph and ListWorkflows parse")
+ void readWorkflow() {
+ Workflow workflow = glue.getWorkflow(r -> r.name(WORKFLOW).includeGraph(true)).workflow();
+ assertThat(workflow.name()).isEqualTo(WORKFLOW);
+ assertThat(workflow.createdOn()).isNotNull();
+ assertThat(workflow.graph().nodes()).extracting(Node::type)
+ .containsExactlyInAnyOrder(NodeType.TRIGGER, NodeType.TRIGGER, NodeType.JOB, NodeType.JOB);
+ assertThat(workflow.graph().edges()).hasSize(3);
+ assertThat(glue.listWorkflows(r -> r.maxResults(25)).workflows()).contains(WORKFLOW);
+ }
+}
diff --git a/docs/services/glue.md b/docs/services/glue.md
index b3eea476b0..5b877f7c18 100644
--- a/docs/services/glue.md
+++ b/docs/services/glue.md
@@ -207,8 +207,38 @@ finishes. One request runs at most 25 firing rounds; a longer chain continues on
As an emulator safeguard (AWS has no such limit), triggers start at most 100 runs on behalf of the run
that set off a chain, the one no trigger started, so a loop of triggers of any shape stops there.
A run a trigger starts carries its `TriggerName`. `SCHEDULED` triggers are stored and can
-be activated, but no timer fires them. `EVENT` triggers and `WorkflowName` need workflows, which are
-not emulated yet.
+be activated, but no timer fires them. A trigger can belong to a workflow through `WorkflowName`, and an
+`EVENT` trigger must; `EVENT` triggers are stored, but no EventBridge event starts them.
+
+#### Workflows
+
+| Action | Description |
+|--------|-------------|
+| CreateWorkflow | Creates a workflow, with default run properties and an optional `MaxConcurrentRuns`. |
+| GetWorkflow | Retrieves a workflow with its `LastRun`, and with `IncludeGraph` its graph of trigger, job and crawler nodes. |
+| ListWorkflows | Lists workflow names. |
+| BatchGetWorkflows | Retrieves several workflows by name and reports the names not found. |
+| UpdateWorkflow | Applies the members the request sets. |
+| DeleteWorkflow | Deletes a workflow and its runs; its triggers are kept. Deleting one that does not exist succeeds. |
+| StartWorkflowRun | Starts a run: the workflow's starting triggers fire inside it. |
+| StopWorkflowRun | Stops a running workflow run and what is running in it. |
+| ResumeWorkflowRun | Starts chosen job and crawler nodes of a finished run again in a new run, and what follows them. |
+| GetWorkflowRun | Retrieves a run with its `Statistics`, and with `IncludeGraph` the job runs and crawls of each node. |
+| GetWorkflowRuns | Lists a workflow's runs, newest first. |
+| GetWorkflowRunProperties | Retrieves a run's properties. |
+| PutWorkflowRunProperties | Adds properties to a run, replacing any with the same name. |
+
+`StartWorkflowRun` fires the workflow's starting triggers (every trigger of the workflow that is not
+`CONDITIONAL`) inside a new run, with the default run properties overlaid by the request's. A
+workflow's activated `CONDITIONAL` triggers fire only on job runs and crawls of a running run of that
+workflow, evaluate `AND` within that run, and start their actions inside it; job runs in a workflow run
+get the `--WORKFLOW_NAME` and `--WORKFLOW_RUN_ID` arguments. A run is `COMPLETED` once nothing is
+running in it and nothing more fires. An action that cannot be launched (its job at its concurrency limit,
+its crawler already running) is reported in the run's `ErrorMessage`, and a run none of whose starting
+actions could launch ends `ERROR`. `ResumeWorkflowRun` starts each chosen job with the arguments it had in
+the original run; if a node cannot be launched (its job at its concurrency limit, its crawler running) the
+request fails before anything starts.
+Blueprints are not emulated.
#### Classifiers
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java
index 1b27493e3b..f2bc32c0c0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java
@@ -49,6 +49,9 @@ public class GlueCrawlerRunService {
private static final int MAX_RUNTIME_HISTORY = 100;
private final StorageBackend<String, CrawlerRunRecord> recordStore;
+ // Workflow run id to the crawls that ran in it. Kept apart from each crawler's recent history, which
+ // only holds its latest crawls, so an older workflow run keeps its crawls; cleared with the workflow.
+ private final StorageBackend<String, List<FinishedCrawl>> workflowCrawlStore;
private final GlueService glueService;
private final int runDurationSeconds;
private final Clock clock;
@@ -56,31 +59,34 @@ public class GlueCrawlerRunService {
@Inject
public GlueCrawlerRunService(StorageFactory storageFactory, GlueService glueService, EmulatorConfig config) {
this(storageFactory.create("glue", "crawler_runs.json", new TypeReference<>() {}),
+ storageFactory.create("glue", "workflow_run_crawls.json", new TypeReference<>() {}),
glueService, config.services().glue().crawlerRunDurationSeconds(), Clock.systemUTC());
}
- GlueCrawlerRunService(StorageBackend<String, CrawlerRunRecord> recordStore, GlueService glueService,
+ GlueCrawlerRunService(StorageBackend<String, CrawlerRunRecord> recordStore,
+ StorageBackend<String, List<FinishedCrawl>> workflowCrawlStore, GlueService glueService,
int runDurationSeconds, Clock clock) {
if (runDurationSeconds < 0) {
throw new IllegalArgumentException(
"floci.services.glue.crawler-run-duration-seconds must not be negative: " + runDurationSeconds);
}
this.recordStore = recordStore;
+ this.workflowCrawlStore = workflowCrawlStore;
this.glueService = glueService;
this.runDurationSeconds = runDurationSeconds;
this.clock = clock;
}
public synchronized String startCrawler(String name) {
- return startCrawler(name, null);
+ return startCrawler(name, null, null);
}
/**
* Starts a crawl and returns its crawl id ({@code crawl:} and a UUID, so ids never repeat, even for
- * a crawler deleted and created again). A trigger passes the origin of its chain; a null origin
- * makes the crawl its own origin.
+ * a crawler deleted and created again). A trigger passes the origin of its chain (a null origin
+ * makes the crawl its own origin) and, when it belongs to a workflow, the workflow run.
*/
- public synchronized String startCrawler(String name, String originRunId) {
+ public synchronized String startCrawler(String name, String originRunId, String workflowRunId) {
glueService.getCrawler(name);
CrawlerRunRecord record = settledRecord(name);
if (record.getCurrentStart() != null) {
@@ -91,6 +97,7 @@ public class GlueCrawlerRunService {
record.setCurrentMessagePrefix(UUID.randomUUID().toString());
record.setCurrentCrawlId(crawlId);
record.setCurrentOriginRunId(originRunId != null ? originRunId : crawlId);
+ record.setCurrentWorkflowRunId(workflowRunId);
recordStore.put(name, record);
LOG.infov("Started Glue crawler {0}", name);
settle(record);
@@ -175,7 +182,7 @@ public class GlueCrawlerRunService {
for (FinishedCrawl crawl : record.getRecentCrawls()) {
if (crawl.getSequence() > after) {
completions.add(new GlueRunCompletion(Long.toString(crawl.getSequence()), crawl.getFinishedAt(),
- crawl.getStatus(), originOf(name, crawl)));
+ crawl.getStatus(), originOf(name, crawl), crawl.getWorkflowRunId()));
}
}
return completions;
@@ -207,6 +214,48 @@ public class GlueCrawlerRunService {
return CRAWL_ID_PREFIX + crawlerName + ":" + crawl.getSequence();
}
+ /** A crawl of a workflow run as the workflow graph reports it: the {@code Crawl} structure. */
+ public record WorkflowCrawl(String crawlerName, String state, Instant startedOn, Instant completedOn) {}
+
+ /** The crawls, settled, that belong to a workflow run: finished ones and a crawl still running. */
+ public synchronized List<WorkflowCrawl> crawlsInWorkflowRun(String workflowRunId) {
+ List<WorkflowCrawl> running = new ArrayList<>();
+ for (CrawlerRunRecord stored : recordStore.scan(key -> true)) {
+ CrawlerRunRecord record = settledRecord(stored.getCrawlerName());
+ if (record.getCurrentStart() != null && workflowRunId.equals(record.getCurrentWorkflowRunId())) {
+ running.add(new WorkflowCrawl(record.getCrawlerName(), STATE_RUNNING, record.getCurrentStart(), null));
+ }
+ }
+ List<WorkflowCrawl> crawls = new ArrayList<>();
+ for (FinishedCrawl crawl : workflowCrawlStore.get(workflowRunId).orElse(List.of())) {
+ crawls.add(new WorkflowCrawl(crawl.getCrawlerName(), crawl.getStatus(), crawl.getStartedAt(),
+ crawl.getFinishedAt()));
+ }
+ crawls.addAll(running);
+ crawls.sort(Comparator.comparing(WorkflowCrawl::startedOn));
+ return crawls;
+ }
+
+ /**
+ * Forgets the crawls of a workflow run that no longer exists, and detaches a crawl still running in
+ * it, so that crawl does not record history under the removed run when it finishes.
+ */
+ public synchronized void forgetWorkflowRun(String workflowRunId) {
+ workflowCrawlStore.delete(workflowRunId);
+ for (CrawlerRunRecord record : recordStore.scan(key -> true)) {
+ if (workflowRunId.equals(record.getCurrentWorkflowRunId())) {
+ record.setCurrentWorkflowRunId(null);
+ recordStore.put(record.getCrawlerName(), record);
+ }
+ }
+ }
+
+ /** Whether the crawler is crawling now, after settling a crawl that has run its time. */
+ public synchronized boolean isRunning(String name) {
+ glueService.getCrawler(name);
+ return settledRecord(name).getCurrentStart() != null;
+ }
+
public synchronized void updateCrawler(Crawler update) {
requireNotRunning(update.getName());
glueService.updateCrawler(update);
@@ -300,6 +349,14 @@ public class GlueCrawlerRunService {
finished.setStatus(status);
finished.setCrawlId(record.getCurrentCrawlId());
finished.setOriginRunId(record.getCurrentOriginRunId());
+ finished.setWorkflowRunId(record.getCurrentWorkflowRunId());
+ finished.setCrawlerName(record.getCrawlerName());
+ if (finished.getWorkflowRunId() != null) {
+ List<FinishedCrawl> ofRun = new ArrayList<>(
+ workflowCrawlStore.get(finished.getWorkflowRunId()).orElse(List.of()));
+ ofRun.add(finished);
+ workflowCrawlStore.put(finished.getWorkflowRunId(), ofRun);
+ }
List<FinishedCrawl> recent = new ArrayList<>(record.getRecentCrawls());
recent.add(finished);
if (recent.size() > MAX_RUNTIME_HISTORY) {
@@ -314,6 +371,7 @@ public class GlueCrawlerRunService {
record.setCurrentMessagePrefix(null);
record.setCurrentCrawlId(null);
record.setCurrentOriginRunId(null);
+ record.setCurrentWorkflowRunId(null);
recordStore.put(record.getCrawlerName(), record);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJobRunService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJobRunService.java
index 1832c09f3f..b301b4e463 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJobRunService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJobRunService.java
@@ -86,15 +86,16 @@ public class GlueJobRunService {
* unset field falls back to the job definition.
*/
public synchronized JobRun startJobRun(String jobName, String previousRunId, JobRun overrides) {
- return startJobRun(jobName, previousRunId, overrides, null);
+ return startJobRun(jobName, previousRunId, overrides, null, null);
}
/**
* As {@link #startJobRun(String, String, JobRun)}, for a run a trigger starts on behalf of
- * {@code originRunId}; a null origin makes the run its own origin.
+ * {@code originRunId} (a null origin makes the run its own origin), inside {@code workflowRunId}
+ * when the trigger belongs to a workflow.
*/
public synchronized JobRun startJobRun(String jobName, String previousRunId, JobRun overrides,
- String originRunId) {
+ String originRunId, String workflowRunId) {
Job job = glueService.getJob(jobName);
// The job's own Timeout is stored as given by CreateJob/UpdateJob, so check what the run inherits.
Integer timeout = firstNonNull(overrides.getTimeout(), job.getTimeout());
@@ -142,6 +143,7 @@ public class GlueJobRunService {
runStore.put(run.getId(), run);
JobRunBookkeeping bookkeeping = new JobRunBookkeeping();
bookkeeping.setOriginRunId(originRunId != null ? originRunId : run.getId());
+ bookkeeping.setWorkflowRunId(workflowRunId);
bookkeepingStore.put(run.getId(), bookkeeping);
LOG.infov("Started Glue job run {0} for job {1}", run.getId(), jobName);
return settle(run);
@@ -212,8 +214,9 @@ public class GlueJobRunService {
if (isTerminal(run)) {
String position = positionOf(run);
if (afterPosition == null || afterPosition.isEmpty() || position.compareTo(afterPosition) > 0) {
+ JobRunBookkeeping bookkeeping = bookkeepingOf(run.getId());
completions.add(new GlueRunCompletion(position, run.getCompletedOn(), run.getJobRunState(),
- bookkeepingOf(run.getId()).getOriginRunId()));
+ bookkeeping.getOriginRunId(), bookkeeping.getWorkflowRunId()));
}
}
}
@@ -231,6 +234,31 @@ public class GlueJobRunService {
return bookkeepingStore.get(originRunId).map(JobRunBookkeeping::getLegacyTriggeredRuns).orElse(0);
}
+ /**
+ * Whether a run of the job could start now without {@code ConcurrentRunsExceededException}, as
+ * {@link #startJobRun} decides it for a run that does not override JobRunQueuingEnabled.
+ */
+ public synchronized boolean hasCapacity(String jobName) {
+ Job job = glueService.getJob(jobName);
+ if (Boolean.TRUE.equals(job.getJobRunQueuingEnabled())) {
+ return true;
+ }
+ long active = runsOf(jobName).stream().filter(run -> !isTerminal(settle(run))).count();
+ return active < maxConcurrentRuns(job);
+ }
+
+ /** The runs, settled, that belong to a workflow run, oldest first. */
+ public synchronized List<JobRun> runsInWorkflowRun(String workflowRunId) {
+ List<JobRun> runs = new ArrayList<>();
+ for (JobRun run : runStore.scan(key -> true)) {
+ if (workflowRunId.equals(bookkeepingOf(run.getId()).getWorkflowRunId())) {
+ runs.add(settle(run));
+ }
+ }
+ runs.sort(Comparator.comparing(JobRun::getStartedOn).thenComparing(JobRun::getId));
+ return runs;
+ }
+
/** Runs belong to their job: deleting the job removes them, as it does on AWS. */
public synchronized void deleteRuns(String jobName) {
for (JobRun run : runsOf(jobName)) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 3a940f2577..22b6637cba 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -35,6 +35,8 @@ public class GlueJsonHandler {
"StartJobRun", "GetJobRun", "GetJobRuns", "BatchStopJobRun",
"StartCrawler", "StopCrawler", "GetCrawler", "GetCrawlers", "BatchGetCrawlers", "GetCrawlerMetrics",
"UpdateCrawler",
+ "StartWorkflowRun", "StopWorkflowRun", "ResumeWorkflowRun", "GetWorkflowRun", "GetWorkflowRuns",
+ "GetWorkflow", "BatchGetWorkflows",
"StartTrigger");
private final GlueService glueService;
@@ -515,10 +517,91 @@ public class GlueJsonHandler {
triggerService.stopTrigger(name);
yield Response.ok(Map.of("Name", name)).build();
}
+ case "CreateWorkflow" -> {
+ Workflow workflow = new Workflow();
+ workflow.setName(request.path("Name").asText(null));
+ workflow.setDescription(request.path("Description").asText(null));
+ workflow.setDefaultRunProperties(readStringMap(request, "DefaultRunProperties"));
+ if (request.hasNonNull("MaxConcurrentRuns")) {
+ workflow.setMaxConcurrentRuns(request.get("MaxConcurrentRuns").asInt());
+ }
+ glueService.createWorkflow(workflow, readStringMap(request, "Tags"), region);
+ yield Response.ok(Map.of("Name", workflow.getName())).build();
+ }
+ case "GetWorkflow" -> Response.ok(Map.of("Workflow", triggerService.workflowView(
+ glueService.getWorkflow(request.path("Name").asText(null)),
+ request.path("IncludeGraph").asBoolean(false)))).build();
+ case "ListWorkflows" -> {
+ GlueService.Page<String> page = glueService.listWorkflows(readMaxResults(request), readNextToken(request));
+ yield Response.ok(pageResponse("Workflows", page.items(), page.nextToken())).build();
+ }
+ case "BatchGetWorkflows" -> {
+ List<String> names = request.hasNonNull("Names")
+ ? mapper.convertValue(request.get("Names"), STRING_LIST)
+ : null;
+ GlueService.BatchGetWorkflowsResult result = glueService.batchGetWorkflows(names);
+ List<Workflow> workflows = new ArrayList<>();
+ for (Workflow workflow : result.workflows()) {
+ workflows.add(triggerService.workflowView(workflow, request.path("IncludeGraph").asBoolean(false)));
+ }
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("Workflows", workflows);
+ response.put("MissingWorkflows", result.missingWorkflows());
+ yield Response.ok(response).build();
+ }
+ case "UpdateWorkflow" -> {
+ String name = request.path("Name").asText(null);
+ glueService.updateWorkflow(name, request.path("Description").asText(null),
+ readStringMap(request, "DefaultRunProperties"),
+ request.hasNonNull("MaxConcurrentRuns") ? request.get("MaxConcurrentRuns").asInt() : null);
+ yield Response.ok(Map.of("Name", name)).build();
+ }
+ case "DeleteWorkflow" -> {
+ String name = request.path("Name").asText(null);
+ triggerService.deleteWorkflow(name, region);
+ yield Response.ok(Map.of("Name", name)).build();
+ }
+ case "StartWorkflowRun" -> Response.ok(Map.of("RunId", triggerService.startWorkflowRun(
+ request.path("Name").asText(null), readStringMap(request, "RunProperties")))).build();
+ case "StopWorkflowRun" -> {
+ triggerService.stopWorkflowRun(request.path("Name").asText(null), request.path("RunId").asText(null));
+ yield Response.ok(Map.of()).build();
+ }
+ case "ResumeWorkflowRun" -> {
+ List<String> nodeIds = request.hasNonNull("NodeIds")
+ ? mapper.convertValue(request.get("NodeIds"), STRING_LIST)
+ : null;
+ GlueTriggerService.ResumedRun resumed = triggerService.resumeWorkflowRun(
+ request.path("Name").asText(null), request.path("RunId").asText(null), nodeIds);
+ Map<String, Object> response = new LinkedHashMap<>();
+ response.put("RunId", resumed.runId());
+ response.put("NodeIds", resumed.nodeIds());
+ yield Response.ok(response).build();
+ }
+ case "GetWorkflowRun" -> Response.ok(Map.of("Run", triggerService.getWorkflowRun(
+ request.path("Name").asText(null), request.path("RunId").asText(null),
+ request.path("IncludeGraph").asBoolean(false)))).build();
+ case "GetWorkflowRuns" -> {
+ GlueService.Page<WorkflowRun> page = triggerService.getWorkflowRuns(request.path("Name").asText(null),
+ request.path("IncludeGraph").asBoolean(false), readMaxResults(request), readNextToken(request));
+ yield Response.ok(pageResponse("Runs", page.items(), page.nextToken())).build();
+ }
+ case "GetWorkflowRunProperties" -> Response.ok(Map.of("RunProperties",
+ triggerService.getWorkflowRunProperties(request.path("Name").asText(null),
+ request.path("RunId").asText(null)))).build();
+ case "PutWorkflowRunProperties" -> {
+ triggerService.putWorkflowRunProperties(request.path("Name").asText(null),
+ request.path("RunId").asText(null), readStringMap(request, "RunProperties"));
+ yield Response.ok(Map.of()).build();
+ }
default -> throw new AwsException("InvalidAction", "Action " + action + " is not supported", 400);
};
}
+ private Map<String, String> readStringMap(JsonNode request, String field) {
+ return request.hasNonNull(field) ? mapper.convertValue(request.get(field), STRING_MAP) : null;
+ }
+
/** The definition members CreateTrigger and TriggerUpdate share. */
private Trigger toTrigger(JsonNode node) throws Exception {
Trigger trigger = new Trigger();
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueRunCompletion.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueRunCompletion.java
index 137353de6b..ffaa34ac73 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueRunCompletion.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueRunCompletion.java
@@ -6,6 +6,8 @@ import java.time.Instant;
* One finished job run or crawl, as a conditional trigger sees it. {@code position} orders the
* completions of one job or crawler; a trigger remembers the last position it has processed.
* {@code originRunId} is the run that set off the trigger chain this run belongs to (the run itself
- * when no trigger started it): a job run id, or a crawl id beginning {@code crawl:}.
+ * when no trigger started it): a job run id, or a crawl id beginning {@code crawl:}. {@code workflowRunId}
+ * is the workflow run the run or crawl belongs to, or null.
*/
-public record GlueRunCompletion(String position, Instant finishedAt, String state, String originRunId) {}
+public record GlueRunCompletion(String position, Instant finishedAt, String state, String originRunId,
+ String workflowRunId) {}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 72ca799c75..ce4134ca55 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -36,6 +36,7 @@ import io.github.hectorvent.floci.services.glue.model.Trigger;
import io.github.hectorvent.floci.services.glue.model.TriggerAction;
import io.github.hectorvent.floci.services.glue.model.TriggerCondition;
import io.github.hectorvent.floci.services.glue.model.UserDefinedFunction;
+import io.github.hectorvent.floci.services.glue.model.Workflow;
import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
import io.github.hectorvent.floci.services.glue.schemaregistry.SchemaToColumnsConverter;
import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
@@ -163,6 +164,8 @@ public class GlueService {
private static final Set<String> CONDITION_JOB_STATES = Set.of("SUCCEEDED", "STOPPED", "FAILED", "TIMEOUT");
private static final Set<String> CONDITION_CRAWL_STATES = Set.of("SUCCEEDED", "CANCELLED", "FAILED");
private static final int MAX_TRIGGERS_PAGE_SIZE = 200;
+ private static final int MAX_WORKFLOWS_PAGE_SIZE = 25;
+ private static final int MAX_BATCH_GET_WORKFLOWS = 25;
// BatchGetCrawlers takes a CrawlerNameList, which the API model caps at 100 names.
private static final int MAX_BATCH_GET_CRAWLERS = 100;
@@ -182,6 +185,7 @@ public class GlueService {
private final StorageBackend<String, DataCatalogEncryptionSettings> encryptionSettingsStore;
private final StorageBackend<String, SecurityConfiguration> securityConfigurationStore;
private final StorageBackend<String, Trigger> triggerStore;
+ private final StorageBackend<String, Workflow> workflowStore;
private final GlueSchemaRegistryService schemaRegistryService;
private final RegionResolver regionResolver;
private final ResourceGroupsTaggingService resourceGroupsTaggingService;
@@ -213,6 +217,7 @@ public class GlueService {
this.securityConfigurationStore = storageFactory.create(
"glue", "security_configurations.json", new TypeReference<>() {});
this.triggerStore = storageFactory.create("glue", "triggers.json", new TypeReference<>() {});
+ this.workflowStore = storageFactory.create("glue", "workflows.json", new TypeReference<>() {});
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
this.resourceGroupsTaggingService = resourceGroupsTaggingService;
@@ -235,6 +240,7 @@ public class GlueService {
StorageBackend<String, DataCatalogEncryptionSettings> encryptionSettingsStore,
StorageBackend<String, SecurityConfiguration> securityConfigurationStore,
StorageBackend<String, Trigger> triggerStore,
+ StorageBackend<String, Workflow> workflowStore,
GlueSchemaRegistryService schemaRegistryService,
RegionResolver regionResolver,
ResourceGroupsTaggingService resourceGroupsTaggingService,
@@ -255,6 +261,7 @@ public class GlueService {
this.encryptionSettingsStore = encryptionSettingsStore;
this.securityConfigurationStore = securityConfigurationStore;
this.triggerStore = triggerStore;
+ this.workflowStore = workflowStore;
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
this.resourceGroupsTaggingService = resourceGroupsTaggingService;
@@ -1519,6 +1526,10 @@ public class GlueService {
return regionResolver.buildArn("glue", region, "trigger/" + triggerName);
}
+ private String workflowArn(String region, String workflowName) {
+ return regionResolver.buildArn("glue", region, "workflow/" + workflowName);
+ }
+
private static Pattern compileFunctionPattern(String pattern) {
if (pattern == null) {
return Pattern.compile(".*");
@@ -2166,6 +2177,108 @@ public class GlueService {
crawlerStore.put(name, crawler);
}
+ // ---- Workflows --------------------------------------------------------------------------
+
+ public void createWorkflow(Workflow workflow, Map<String, String> tags, String region) {
+ validateRequired(workflow.getName(), "Name");
+ validateWorkflowConcurrency(workflow.getMaxConcurrentRuns());
+ String name = workflow.getName();
+ if (workflowStore.get(name).isPresent()) {
+ throw new AwsException("AlreadyExistsException", "Workflow " + name + " already exists.", 400);
+ }
+ Instant now = Instant.now();
+ workflow.setCreatedOn(now);
+ workflow.setLastModifiedOn(now);
+ workflowStore.put(name, workflow);
+ if (tags != null && !tags.isEmpty()) {
+ resourceGroupsTaggingService.tagResources(List.of(workflowArn(region, name)), tags, region);
+ }
+ LOG.infov("Created Glue workflow {0}", name);
+ }
+
+ public Workflow getWorkflow(String name) {
+ validateRequired(name, "Name");
+ return workflowStore.get(name)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Workflow " + name + " not found.", 400));
+ }
+
+ public Page<String> listWorkflows(Integer maxResults, String nextToken) {
+ if (maxResults != null && (maxResults < 1 || maxResults > MAX_WORKFLOWS_PAGE_SIZE)) {
+ throw new AwsException("InvalidInputException",
+ "MaxResults must be between 1 and " + MAX_WORKFLOWS_PAGE_SIZE, 400);
+ }
+ List<String> names = new ArrayList<>();
+ for (Workflow workflow : workflowStore.scan(k -> true)) {
+ names.add(workflow.getName());
+ }
+ names.sort(Comparator.naturalOrder());
+ return paginate(names, maxResults, nextToken);
+ }
+
+ public BatchGetWorkflowsResult batchGetWorkflows(List<String> names) {
+ if (names == null || names.isEmpty() || names.size() > MAX_BATCH_GET_WORKFLOWS) {
+ throw new AwsException("InvalidInputException",
+ "Names must contain between 1 and " + MAX_BATCH_GET_WORKFLOWS + " names.", 400);
+ }
+ List<Workflow> workflows = new ArrayList<>();
+ List<String> missing = new ArrayList<>();
+ for (String name : names) {
+ Optional<Workflow> workflow = name == null ? Optional.empty() : workflowStore.get(name);
+ if (workflow.isPresent()) {
+ workflows.add(workflow.get());
+ } else {
+ missing.add(name);
+ }
+ }
+ return new BatchGetWorkflowsResult(workflows, missing);
+ }
+
+ public record BatchGetWorkflowsResult(List<Workflow> workflows, List<String> missingWorkflows) {}
+
+ /** Applies the members an UpdateWorkflow request sets; the rest of the workflow is kept. */
+ public void updateWorkflow(String name, String description, Map<String, String> defaultRunProperties,
+ Integer maxConcurrentRuns) {
+ Workflow workflow = getWorkflow(name);
+ validateWorkflowConcurrency(maxConcurrentRuns);
+ if (description != null) {
+ workflow.setDescription(description);
+ }
+ if (defaultRunProperties != null) {
+ workflow.setDefaultRunProperties(defaultRunProperties);
+ }
+ if (maxConcurrentRuns != null) {
+ workflow.setMaxConcurrentRuns(maxConcurrentRuns);
+ }
+ workflow.setLastModifiedOn(Instant.now());
+ workflowStore.put(name, workflow);
+ }
+
+ /**
+ * DeleteWorkflow declares no EntityNotFoundException: deleting a missing workflow succeeds. The
+ * workflow's triggers are left in place, as the API does not say they are removed.
+ */
+ public void deleteWorkflow(String name, String region) {
+ validateRequired(name, "Name");
+ workflowStore.delete(name);
+ resourceGroupsTaggingService.deleteResources(List.of(workflowArn(region, name)), region);
+ }
+
+ public List<Trigger> triggersOfWorkflow(String workflowName) {
+ List<Trigger> triggers = new ArrayList<>();
+ for (Trigger trigger : allTriggers()) {
+ if (workflowName.equals(trigger.getWorkflowName())) {
+ triggers.add(trigger);
+ }
+ }
+ return triggers;
+ }
+
+ private static void validateWorkflowConcurrency(Integer maxConcurrentRuns) {
+ if (maxConcurrentRuns != null && maxConcurrentRuns < 1) {
+ throw new AwsException("InvalidInputException", "MaxConcurrentRuns must be at least 1.", 400);
+ }
+ }
+
// ---- Triggers ---------------------------------------------------------------------------
public void createTrigger(Trigger trigger, Map<String, String> tags, String region) {
@@ -2175,10 +2288,8 @@ public class GlueService {
throw new AwsException("InvalidInputException", "Invalid trigger type: " + trigger.getType(), 400);
}
if (trigger.getWorkflowName() != null) {
- throw new AwsException("EntityNotFoundException",
- "Workflow " + trigger.getWorkflowName() + " not found.", 400);
- }
- if ("EVENT".equals(trigger.getType())) {
+ getWorkflow(trigger.getWorkflowName());
+ } else if ("EVENT".equals(trigger.getType())) {
throw new AwsException("InvalidInputException", "An EVENT trigger must belong to a workflow.", 400);
}
validateTriggerDefinition(trigger);
@@ -2830,6 +2941,9 @@ public class GlueService {
} else if (resource.startsWith("trigger/")) {
getTrigger(resource.substring(8));
return;
+ } else if (resource.startsWith("workflow/")) {
+ getWorkflow(resource.substring(9));
+ return;
} else if (resource.startsWith("connection/")) {
getConnection(resource.substring(11), false);
return;
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueTriggerService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueTriggerService.java
index b045e23608..227bfdd32c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueTriggerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueTriggerService.java
@@ -10,18 +10,24 @@ import io.github.hectorvent.floci.services.glue.model.Trigger;
import io.github.hectorvent.floci.services.glue.model.TriggerAction;
import io.github.hectorvent.floci.services.glue.model.TriggerChainBudget;
import io.github.hectorvent.floci.services.glue.model.TriggerCondition;
+import io.github.hectorvent.floci.services.glue.model.Workflow;
+import io.github.hectorvent.floci.services.glue.model.WorkflowRun;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
/**
* Glue trigger lifecycle and firing. An ON_DEMAND trigger fires when started. A CONDITIONAL trigger,
@@ -31,6 +37,12 @@ import java.util.Set;
* trigger has not seen yet, in finishing order. A SCHEDULED trigger is stored and can be activated,
* but no timer fires it.
*
+ * <p>This service also runs workflows. StartWorkflowRun fires the workflow's starting (non
+ * CONDITIONAL) triggers inside a new workflow run; a workflow's CONDITIONAL triggers fire only on
+ * completions of runs and crawls that belong to a RUNNING run of that workflow, evaluate AND within
+ * that run, and start their actions inside it. A workflow run is COMPLETED once an evaluation ends
+ * with nothing left running in it and nothing more to fire.
+ *
* <p>Every public method is synchronized. This service calls the job run and crawler run services
* and they never call back, so the lock order is always this service first.
*/
@@ -44,6 +56,11 @@ public class GlueTriggerService {
static final String STATE_CREATED = "CREATED";
static final String STATE_ACTIVATED = "ACTIVATED";
static final String STATE_DEACTIVATED = "DEACTIVATED";
+ static final String STATUS_RUNNING = "RUNNING";
+ static final String STATUS_COMPLETED = "COMPLETED";
+ static final String STATUS_STOPPED = "STOPPED";
+ static final String STATUS_ERROR = "ERROR";
+ private static final String STATE_JOB_RUNNING = "RUNNING";
// Work done within one request; a longer chain continues on the next run-related request.
private static final int MAX_FIRING_ROUNDS = 25;
@@ -65,6 +82,8 @@ public class GlueTriggerService {
private final StorageBackend<String, Map<String, String>> seenStore;
private final StorageBackend<String, TriggerChainBudget> budgetStore;
private final int maxTrackedChains;
+ private final Clock clock;
+ private final StorageBackend<String, WorkflowRun> workflowRunStore;
private final GlueService glueService;
private final GlueJobRunService jobRunService;
private final GlueCrawlerRunService crawlerRunService;
@@ -77,22 +96,28 @@ public class GlueTriggerService {
// than rewritten in full on each claim.
storageFactory.create("glue", "trigger_chain_budgets.json", new TypeReference<>() {},
WriteProfile.APPEND_HEAVY),
+ storageFactory.create("glue", "workflow_runs.json", new TypeReference<>() {}),
glueService, jobRunService, crawlerRunService);
}
GlueTriggerService(StorageBackend<String, Map<String, String>> seenStore,
- StorageBackend<String, TriggerChainBudget> budgetStore, GlueService glueService,
+ StorageBackend<String, TriggerChainBudget> budgetStore,
+ StorageBackend<String, WorkflowRun> workflowRunStore, GlueService glueService,
GlueJobRunService jobRunService, GlueCrawlerRunService crawlerRunService) {
- this(seenStore, budgetStore, glueService, jobRunService, crawlerRunService, MAX_TRACKED_CHAINS);
+ this(seenStore, budgetStore, workflowRunStore, glueService, jobRunService, crawlerRunService,
+ MAX_TRACKED_CHAINS, Clock.systemUTC());
}
GlueTriggerService(StorageBackend<String, Map<String, String>> seenStore,
- StorageBackend<String, TriggerChainBudget> budgetStore, GlueService glueService,
+ StorageBackend<String, TriggerChainBudget> budgetStore,
+ StorageBackend<String, WorkflowRun> workflowRunStore, GlueService glueService,
GlueJobRunService jobRunService, GlueCrawlerRunService crawlerRunService,
- int maxTrackedChains) {
+ int maxTrackedChains, Clock clock) {
this.seenStore = seenStore;
this.budgetStore = budgetStore;
this.maxTrackedChains = maxTrackedChains;
+ this.clock = clock;
+ this.workflowRunStore = workflowRunStore;
this.glueService = glueService;
this.jobRunService = jobRunService;
this.crawlerRunService = crawlerRunService;
@@ -130,7 +155,7 @@ public class GlueTriggerService {
public synchronized void startTrigger(String name) {
Trigger trigger = glueService.getTrigger(name);
if (TYPE_ON_DEMAND.equals(trigger.getType())) {
- fire(trigger, null, true);
+ fire(trigger, null, null, true);
return;
}
boolean wasActive = STATE_ACTIVATED.equals(trigger.getState());
@@ -160,27 +185,505 @@ public class GlueTriggerService {
public synchronized void fireConditionalTriggers() {
for (int round = 0; round < MAX_FIRING_ROUNDS; round++) {
boolean anyFired = false;
+ Set<String> runningWorkflowRuns = runningWorkflowRunIds();
for (Trigger trigger : glueService.allTriggers()) {
if (!isActiveConditional(trigger)) {
continue;
}
- for (GlueRunCompletion cause : newFirings(trigger)) {
- anyFired |= fire(trigger, cause.originRunId(), false) > 0;
+ for (GlueRunCompletion cause : newFirings(trigger, runningWorkflowRuns)) {
+ anyFired |= fire(trigger, cause.originRunId(),
+ trigger.getWorkflowName() != null ? cause.workflowRunId() : null, false) > 0;
}
}
if (!anyFired) {
+ // Nothing more fires, so a workflow run with nothing left running is finished. Not
+ // checked when the round limit cut the evaluation short: more may still fire then.
+ completeFinishedWorkflowRuns();
return;
}
}
}
+ // ---- Workflow runs ----------------------------------------------------------------------
+
+ /** Starts a run of the workflow: its starting (non CONDITIONAL) triggers fire inside the new run. */
+ public synchronized String startWorkflowRun(String workflowName, Map<String, String> runProperties) {
+ Workflow workflow = glueService.getWorkflow(workflowName);
+ requireCapacity(workflow);
+ Map<String, String> properties = new LinkedHashMap<>();
+ if (workflow.getDefaultRunProperties() != null) {
+ properties.putAll(workflow.getDefaultRunProperties());
+ }
+ if (runProperties != null) {
+ properties.putAll(runProperties);
+ }
+ WorkflowRun run = newWorkflowRun(workflowName, null, properties);
+ int actions = 0;
+ int started = 0;
+ for (Trigger trigger : glueService.triggersOfWorkflow(workflowName)) {
+ if (!TYPE_CONDITIONAL.equals(trigger.getType())) {
+ actions += trigger.getActions().size();
+ started += fire(trigger, null, run.getWorkflowRunId(), false);
+ }
+ }
+ if (actions > 0 && started == 0) {
+ // Nothing the workflow starts with could be launched; the reasons are in ErrorMessage.
+ finishWorkflowRun(workflowRunStore.get(run.getWorkflowRunId()).orElse(run), STATUS_ERROR, clock.instant());
+ return run.getWorkflowRunId();
+ }
+ fireConditionalTriggers();
+ return run.getWorkflowRunId();
+ }
+
+ public synchronized void stopWorkflowRun(String workflowName, String runId) {
+ findWorkflowRun(workflowName, runId);
+ // A run whose last action finished since the previous evaluation is COMPLETED, not stoppable.
+ fireConditionalTriggers();
+ WorkflowRun run = findWorkflowRun(workflowName, runId);
+ if (!STATUS_RUNNING.equals(run.getStatus())) {
+ throw new AwsException("IllegalWorkflowStateException",
+ "Workflow run " + runId + " is not running; its status is " + run.getStatus(), 400);
+ }
+ stopWhatIsRunning(runId);
+ finishWorkflowRun(run, STATUS_STOPPED, clock.instant());
+ }
+
+ /**
+ * Starts the given job and crawler nodes of a finished run again in a new run of the workflow, whose
+ * conditional triggers then carry on downstream. Each node must have run in the original run.
+ */
+ public synchronized ResumedRun resumeWorkflowRun(String workflowName, String runId, List<String> requestedNodeIds) {
+ WorkflowRun original = findWorkflowRun(workflowName, runId);
+ if (STATUS_RUNNING.equals(original.getStatus())) {
+ throw new AwsException("IllegalWorkflowStateException",
+ "Workflow run " + runId + " is still running.", 400);
+ }
+ if (requestedNodeIds == null || requestedNodeIds.isEmpty()) {
+ throw new AwsException("InvalidInputException", "NodeIds is required.", 400);
+ }
+ // A node named twice is restarted once: launching it a second time would collide with the first.
+ List<String> nodeIds = new ArrayList<>(new LinkedHashSet<>(requestedNodeIds));
+ Map<String, Map<String, Object>> nodes = new HashMap<>();
+ for (Map<String, Object> node : graphNodes(workflowName)) {
+ nodes.put((String) node.get("UniqueId"), node);
+ }
+ Set<String> ranJobs = new LinkedHashSet<>();
+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(runId)) {
+ ranJobs.add(jobRun.getJobName());
+ }
+ Set<String> ranCrawlers = new LinkedHashSet<>();
+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawlerRunService.crawlsInWorkflowRun(runId)) {
+ ranCrawlers.add(crawl.crawlerName());
+ }
+ for (String nodeId : nodeIds) {
+ Map<String, Object> node = nodes.get(nodeId);
+ boolean ran = node != null && ("JOB".equals(node.get("Type")) ? ranJobs.contains((String) node.get("Name"))
+ : "CRAWLER".equals(node.get("Type")) && ranCrawlers.contains((String) node.get("Name")));
+ if (!ran) {
+ throw new AwsException("InvalidInputException",
+ "Node " + nodeId + " is not a job or crawler node that ran in workflow run " + runId, 400);
+ }
+ }
+ requireCapacity(glueService.getWorkflow(workflowName));
+ // Refuse before anything starts when a node cannot launch, so a failed resume normally leaves
+ // no run and no action behind.
+ for (String nodeId : nodeIds) {
+ Map<String, Object> node = nodes.get(nodeId);
+ String name = (String) node.get("Name");
+ if ("JOB".equals(node.get("Type")) && !jobRunService.hasCapacity(name)) {
+ throw new AwsException("ConcurrentRunsExceededException", "Concurrent runs exceeded for " + name, 400);
+ }
+ if ("CRAWLER".equals(node.get("Type")) && crawlerRunService.isRunning(name)) {
+ throw new AwsException("CrawlerRunningException", "Crawler with name " + name + " has already started", 400);
+ }
+ }
+ Map<String, JobRun> originalJobRuns = new HashMap<>();
+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(runId)) {
+ originalJobRuns.put(jobRun.getJobName(), jobRun);
+ }
+ WorkflowRun resumed = newWorkflowRun(workflowName, runId, original.getWorkflowRunProperties());
+ String resumedId = resumed.getWorkflowRunId();
+ try {
+ for (String nodeId : nodeIds) {
+ Map<String, Object> node = nodes.get(nodeId);
+ String name = (String) node.get("Name");
+ if ("JOB".equals(node.get("Type"))) {
+ // The job runs again with the arguments it had in the original run, now in the new run.
+ JobRun before = originalJobRuns.get(name);
+ JobRun overrides = new JobRun();
+ overrides.setArguments(withWorkflowArguments(before != null ? before.getArguments() : null, resumedId));
+ if (before != null) {
+ overrides.setTimeout(before.getTimeout());
+ overrides.setSecurityConfiguration(before.getSecurityConfiguration());
+ overrides.setNotificationProperty(before.getNotificationProperty());
+ overrides.setTriggerName(before.getTriggerName());
+ }
+ jobRunService.startJobRun(name, null, overrides, null, resumedId);
+ } else {
+ crawlerRunService.startCrawler(name, null, resumedId);
+ }
+ }
+ } catch (AwsException e) {
+ // Only reachable if a node became unable to launch after the check above. What already
+ // started keeps its run, so the run is kept too, stopped and marked ERROR with the reason.
+ noteLaunchFailure(resumedId, "Resume could not start every node: " + e.getMessage());
+ stopWhatIsRunning(resumedId);
+ finishWorkflowRun(workflowRunStore.get(resumedId).orElse(resumed), STATUS_ERROR, clock.instant());
+ throw e;
+ }
+ fireConditionalTriggers();
+ return new ResumedRun(resumedId, nodeIds);
+ }
+
+ public synchronized WorkflowRun getWorkflowRun(String workflowName, String runId, boolean includeGraph) {
+ fireConditionalTriggers();
+ return view(findWorkflowRun(workflowName, runId), includeGraph);
+ }
+
+ public synchronized GlueService.Page<WorkflowRun> getWorkflowRuns(String workflowName, boolean includeGraph,
+ Integer maxResults, String nextToken) {
+ glueService.getWorkflow(workflowName);
+ fireConditionalTriggers();
+ List<WorkflowRun> runs = new ArrayList<>();
+ for (WorkflowRun run : runsOfWorkflow(workflowName)) {
+ runs.add(view(run, includeGraph));
+ }
+ runs.sort(Comparator.comparing(WorkflowRun::getStartedOn).reversed()
+ .thenComparing(WorkflowRun::getWorkflowRunId));
+ return glueService.paginate(runs, maxResults, nextToken);
+ }
+
+ public synchronized Map<String, String> getWorkflowRunProperties(String workflowName, String runId) {
+ WorkflowRun run = findWorkflowRun(workflowName, runId);
+ return run.getWorkflowRunProperties() != null ? run.getWorkflowRunProperties() : Map.of();
+ }
+
+ /** Adds the properties to the run's, replacing any with the same name. */
+ public synchronized void putWorkflowRunProperties(String workflowName, String runId, Map<String, String> properties) {
+ if (properties == null) {
+ throw new AwsException("InvalidInputException", "RunProperties is required.", 400);
+ }
+ WorkflowRun run = findWorkflowRun(workflowName, runId);
+ Map<String, String> merged = new LinkedHashMap<>();
+ if (run.getWorkflowRunProperties() != null) {
+ merged.putAll(run.getWorkflowRunProperties());
+ }
+ merged.putAll(properties);
+ run.setWorkflowRunProperties(merged);
+ workflowRunStore.put(run.getWorkflowRunId(), run);
+ }
+
+ /** The workflow as GetWorkflow and BatchGetWorkflows report it: with LastRun, and the graph if asked. */
+ public synchronized Workflow workflowView(Workflow stored, boolean includeGraph) {
+ fireConditionalTriggers();
+ Workflow workflow = new Workflow();
+ workflow.setName(stored.getName());
+ workflow.setDescription(stored.getDescription());
+ workflow.setDefaultRunProperties(stored.getDefaultRunProperties());
+ workflow.setCreatedOn(stored.getCreatedOn());
+ workflow.setLastModifiedOn(stored.getLastModifiedOn());
+ workflow.setMaxConcurrentRuns(stored.getMaxConcurrentRuns());
+ runsOfWorkflow(stored.getName()).stream()
+ .max(Comparator.comparing(WorkflowRun::getStartedOn))
+ .ifPresent(last -> workflow.setLastRun(view(last, false)));
+ if (includeGraph) {
+ workflow.setGraph(graph(stored.getName(), null));
+ }
+ return workflow;
+ }
+
+ /** Deletes the workflow and its runs; its triggers stay, as they do not belong to the workflow's storage. */
+ public synchronized void deleteWorkflow(String workflowName, String region) {
+ for (WorkflowRun run : runsOfWorkflow(workflowName)) {
+ workflowRunStore.delete(run.getWorkflowRunId());
+ forgetWorkflowRunState(run.getWorkflowRunId());
+ crawlerRunService.forgetWorkflowRun(run.getWorkflowRunId());
+ }
+ glueService.deleteWorkflow(workflowName, region);
+ }
+
+ private void stopWhatIsRunning(String workflowRunId) {
+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(workflowRunId)) {
+ if (STATE_JOB_RUNNING.equals(jobRun.getJobRunState())) {
+ jobRunService.batchStopJobRun(jobRun.getJobName(), List.of(jobRun.getId()));
+ }
+ }
+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawlerRunService.crawlsInWorkflowRun(workflowRunId)) {
+ if (STATE_JOB_RUNNING.equals(crawl.state())) {
+ crawlerRunService.stopCrawler(crawl.crawlerName());
+ }
+ }
+ }
+
+ private void requireCapacity(Workflow workflow) {
+ // Settles runs, fires what is due and completes finished workflow runs before counting.
+ fireConditionalTriggers();
+ if (workflow.getMaxConcurrentRuns() == null) {
+ return;
+ }
+ long running = runsOfWorkflow(workflow.getName()).stream()
+ .filter(run -> STATUS_RUNNING.equals(run.getStatus())).count();
+ if (running >= workflow.getMaxConcurrentRuns()) {
+ throw new AwsException("ConcurrentRunsExceededException",
+ "Concurrent runs exceeded for workflow " + workflow.getName(), 400);
+ }
+ }
+
+ private WorkflowRun newWorkflowRun(String workflowName, String previousRunId, Map<String, String> properties) {
+ WorkflowRun run = new WorkflowRun();
+ run.setName(workflowName);
+ run.setWorkflowRunId("wr_" + UUID.randomUUID().toString().replace("-", "")
+ + UUID.randomUUID().toString().replace("-", ""));
+ run.setPreviousRunId(previousRunId);
+ run.setWorkflowRunProperties(properties != null ? new LinkedHashMap<>(properties) : new LinkedHashMap<>());
+ run.setStartedOn(clock.instant());
+ run.setStatus(STATUS_RUNNING);
+ workflowRunStore.put(run.getWorkflowRunId(), run);
+ LOG.infov("Started Glue workflow run {0} of {1}", run.getWorkflowRunId(), workflowName);
+ return run;
+ }
+
+ private WorkflowRun findWorkflowRun(String workflowName, String runId) {
+ glueService.getWorkflow(workflowName);
+ if (runId == null) {
+ throw new AwsException("InvalidInputException", "RunId is required.", 400);
+ }
+ return workflowRunStore.get(runId)
+ .filter(run -> workflowName.equals(run.getName()))
+ .orElseThrow(() -> new AwsException("EntityNotFoundException",
+ "Workflow run " + runId + " not found for workflow " + workflowName, 400));
+ }
+
+ private List<WorkflowRun> runsOfWorkflow(String workflowName) {
+ List<WorkflowRun> runs = new ArrayList<>();
+ for (WorkflowRun run : workflowRunStore.scan(key -> true)) {
+ if (workflowName.equals(run.getName())) {
+ runs.add(run);
+ }
+ }
+ return runs;
+ }
+
+ private Set<String> runningWorkflowRunIds() {
+ Set<String> running = new LinkedHashSet<>();
+ for (WorkflowRun run : workflowRunStore.scan(key -> true)) {
+ if (STATUS_RUNNING.equals(run.getStatus())) {
+ running.add(run.getWorkflowRunId());
+ }
+ }
+ return running;
+ }
+
+ private String workflowNameOf(String workflowRunId) {
+ return workflowRunStore.get(workflowRunId).map(WorkflowRun::getName).orElse(null);
+ }
+
+ private void completeFinishedWorkflowRuns() {
+ for (WorkflowRun run : workflowRunStore.scan(key -> true)) {
+ if (!STATUS_RUNNING.equals(run.getStatus())) {
+ continue;
+ }
+ Instant lastFinished = run.getStartedOn();
+ boolean anyRunning = false;
+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(run.getWorkflowRunId())) {
+ if (jobRun.getCompletedOn() == null) {
+ anyRunning = true;
+ } else if (jobRun.getCompletedOn().isAfter(lastFinished)) {
+ lastFinished = jobRun.getCompletedOn();
+ }
+ }
+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawlerRunService.crawlsInWorkflowRun(run.getWorkflowRunId())) {
+ if (crawl.completedOn() == null) {
+ anyRunning = true;
+ } else if (crawl.completedOn().isAfter(lastFinished)) {
+ lastFinished = crawl.completedOn();
+ }
+ }
+ if (!anyRunning) {
+ finishWorkflowRun(run, STATUS_COMPLETED, lastFinished);
+ }
+ }
+ }
+
+ private void finishWorkflowRun(WorkflowRun run, String status, Instant completedOn) {
+ run.setStatus(status);
+ run.setCompletedOn(completedOn);
+ workflowRunStore.put(run.getWorkflowRunId(), run);
+ forgetWorkflowRunState(run.getWorkflowRunId());
+ LOG.infov("Glue workflow run {0} is {1}", run.getWorkflowRunId(), status);
+ }
+
+ /** A finished workflow run's triggers no longer need the outcomes they kept for it. */
+ private void forgetWorkflowRunState(String workflowRunId) {
+ String prefix = STATE + workflowRunId + "|";
+ for (String triggerName : seenStore.keys()) {
+ Map<String, String> seen = seenStore.get(triggerName).orElse(Map.of());
+ if (seen.keySet().stream().anyMatch(key -> key.startsWith(prefix))) {
+ Map<String, String> kept = new HashMap<>(seen);
+ kept.keySet().removeIf(key -> key.startsWith(prefix));
+ seenStore.put(triggerName, kept);
+ }
+ }
+ }
+
+ private WorkflowRun view(WorkflowRun stored, boolean includeGraph) {
+ WorkflowRun run = new WorkflowRun();
+ run.setName(stored.getName());
+ run.setWorkflowRunId(stored.getWorkflowRunId());
+ run.setPreviousRunId(stored.getPreviousRunId());
+ run.setWorkflowRunProperties(stored.getWorkflowRunProperties());
+ run.setStartedOn(stored.getStartedOn());
+ run.setCompletedOn(stored.getCompletedOn());
+ run.setStatus(stored.getStatus());
+ run.setErrorMessage(stored.getErrorMessage());
+ run.setStatistics(statistics(stored.getWorkflowRunId()));
+ if (includeGraph) {
+ run.setGraph(graph(stored.getName(), stored.getWorkflowRunId()));
+ }
+ return run;
+ }
+
+ private Map<String, Integer> statistics(String workflowRunId) {
+ Map<String, Integer> counts = new LinkedHashMap<>();
+ for (String key : List.of("TotalActions", "TimeoutActions", "FailedActions", "StoppedActions",
+ "SucceededActions", "RunningActions", "ErroredActions", "WaitingActions")) {
+ counts.put(key, 0);
+ }
+ List<String> states = new ArrayList<>();
+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(workflowRunId)) {
+ states.add(jobRun.getJobRunState());
+ }
+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawlerRunService.crawlsInWorkflowRun(workflowRunId)) {
+ states.add("CANCELLED".equals(crawl.state()) ? "STOPPED" : crawl.state());
+ }
+ for (String state : states) {
+ counts.merge("TotalActions", 1, Integer::sum);
+ String bucket = switch (state) {
+ case "SUCCEEDED" -> "SucceededActions";
+ case "FAILED" -> "FailedActions";
+ case "STOPPED", "STOPPING" -> "StoppedActions";
+ case "TIMEOUT" -> "TimeoutActions";
+ case "ERROR" -> "ErroredActions";
+ case "WAITING" -> "WaitingActions";
+ default -> "RunningActions";
+ };
+ counts.merge(bucket, 1, Integer::sum);
+ }
+ return counts;
+ }
+
+ /**
+ * The workflow graph: a TRIGGER node per trigger of the workflow, a JOB or CRAWLER node per job or
+ * crawler its triggers start or watch, an edge from each watched node to the trigger and from the
+ * trigger to each node it starts. With a workflow run, job nodes list the run's job runs and crawler
+ * nodes its crawls.
+ */
+ private Map<String, Object> graph(String workflowName, String workflowRunId) {
+ List<Map<String, Object>> nodes = new ArrayList<>(graphNodes(workflowName));
+ List<Map<String, Object>> edges = new ArrayList<>();
+ for (Trigger trigger : glueService.triggersOfWorkflow(workflowName)) {
+ String triggerId = nodeId(workflowName, "TRIGGER", trigger.getName());
+ if (trigger.getPredicate() != null && trigger.getPredicate().getConditions() != null) {
+ for (TriggerCondition condition : trigger.getPredicate().getConditions()) {
+ edges.add(edge(nodeId(workflowName, condition.getJobName() != null ? "JOB" : "CRAWLER",
+ condition.getJobName() != null ? condition.getJobName() : condition.getCrawlerName()), triggerId));
+ }
+ }
+ for (TriggerAction action : trigger.getActions()) {
+ edges.add(edge(triggerId, nodeId(workflowName, action.getJobName() != null ? "JOB" : "CRAWLER",
+ action.getJobName() != null ? action.getJobName() : action.getCrawlerName())));
+ }
+ }
+ if (workflowRunId != null) {
+ List<JobRun> jobRuns = jobRunService.runsInWorkflowRun(workflowRunId);
+ List<GlueCrawlerRunService.WorkflowCrawl> crawls = crawlerRunService.crawlsInWorkflowRun(workflowRunId);
+ for (Map<String, Object> node : nodes) {
+ String name = (String) node.get("Name");
+ if ("JOB".equals(node.get("Type"))) {
+ node.put("JobDetails", Map.of("JobRuns",
+ jobRuns.stream().filter(run -> name.equals(run.getJobName())).toList()));
+ } else if ("CRAWLER".equals(node.get("Type"))) {
+ List<Map<String, Object>> crawlList = new ArrayList<>();
+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawls) {
+ if (name.equals(crawl.crawlerName())) {
+ Map<String, Object> entry = new LinkedHashMap<>();
+ entry.put("State", crawl.state());
+ entry.put("StartedOn", epochSeconds(crawl.startedOn()));
+ if (crawl.completedOn() != null) {
+ entry.put("CompletedOn", epochSeconds(crawl.completedOn()));
+ }
+ entry.put("LogGroup", "/aws-glue/crawlers");
+ entry.put("LogStream", name);
+ crawlList.add(entry);
+ }
+ }
+ node.put("CrawlerDetails", Map.of("Crawls", crawlList));
+ }
+ }
+ }
+ Map<String, Object> graph = new LinkedHashMap<>();
+ graph.put("Nodes", nodes);
+ graph.put("Edges", edges);
+ return graph;
+ }
+
+ private List<Map<String, Object>> graphNodes(String workflowName) {
+ Map<String, Map<String, Object>> nodes = new LinkedHashMap<>();
+ for (Trigger trigger : glueService.triggersOfWorkflow(workflowName)) {
+ Map<String, Object> triggerNode = node(workflowName, "TRIGGER", trigger.getName());
+ triggerNode.put("TriggerDetails", Map.of("Trigger", trigger));
+ nodes.putIfAbsent((String) triggerNode.get("UniqueId"), triggerNode);
+ if (trigger.getPredicate() != null && trigger.getPredicate().getConditions() != null) {
+ for (TriggerCondition condition : trigger.getPredicate().getConditions()) {
+ Map<String, Object> watched = condition.getJobName() != null
+ ? node(workflowName, "JOB", condition.getJobName())
+ : node(workflowName, "CRAWLER", condition.getCrawlerName());
+ nodes.putIfAbsent((String) watched.get("UniqueId"), watched);
+ }
+ }
+ for (TriggerAction action : trigger.getActions()) {
+ Map<String, Object> started = action.getJobName() != null
+ ? node(workflowName, "JOB", action.getJobName())
+ : node(workflowName, "CRAWLER", action.getCrawlerName());
+ nodes.putIfAbsent((String) started.get("UniqueId"), started);
+ }
+ }
+ return new ArrayList<>(nodes.values());
+ }
+
+ private static Map<String, Object> node(String workflowName, String type, String name) {
+ Map<String, Object> node = new LinkedHashMap<>();
+ node.put("Type", type);
+ node.put("Name", name);
+ node.put("UniqueId", nodeId(workflowName, type, name));
+ return node;
+ }
+
+ private static Map<String, Object> edge(String sourceId, String destinationId) {
+ Map<String, Object> edge = new LinkedHashMap<>();
+ edge.put("SourceId", sourceId);
+ edge.put("DestinationId", destinationId);
+ return edge;
+ }
+
+ /** Stable per workflow, type and name, so a node keeps its id across reads and resumes. */
+ private static String nodeId(String workflowName, String type, String name) {
+ return "wnode_" + UUID.nameUUIDFromBytes((workflowName + "\u0000" + type + "\u0000" + name)
+ .getBytes(StandardCharsets.UTF_8)).toString().replace("-", "");
+ }
+
+ private static double epochSeconds(Instant instant) {
+ return instant.toEpochMilli() / 1000.0;
+ }
+
/**
* Processes the completions this trigger has not seen yet, in finishing order, and returns the
- * completions that fire it. ANY fires on
- * each completion that matches one of its conditions; AND fires on a matching completion once
- * every condition's latest outcome matches.
+ * completions that fire it. ANY fires on each completion that matches one of its conditions; AND
+ * fires on a matching completion once every condition's latest outcome matches. A workflow's
+ * trigger only counts completions from a running run of its workflow and keeps its latest outcomes
+ * per workflow run, so AND is met within one run.
*/
- private List<GlueRunCompletion> newFirings(Trigger trigger) {
+ private List<GlueRunCompletion> newFirings(Trigger trigger, Set<String> runningWorkflowRuns) {
Map<String, String> before = seenStore.get(trigger.getName()).orElse(Map.of());
Map<String, String> seen = new HashMap<>(before);
List<KeyedCompletion> fresh = new ArrayList<>();
@@ -200,11 +703,20 @@ public class GlueTriggerService {
if (processed == null || event.completion().position().compareTo(processed) > 0) {
seen.put(POSITION + event.key(), event.completion().position());
}
- seen.put(STATE + event.key(), event.completion().state());
+ String scope = "";
+ if (trigger.getWorkflowName() != null) {
+ String workflowRunId = event.completion().workflowRunId();
+ if (workflowRunId == null || !runningWorkflowRuns.contains(workflowRunId)
+ || !trigger.getWorkflowName().equals(workflowNameOf(workflowRunId))) {
+ continue;
+ }
+ scope = workflowRunId + "|";
+ }
+ seen.put(STATE + scope + event.key(), event.completion().state());
boolean eventMatches = false;
boolean allMatch = true;
for (TriggerCondition condition : trigger.getPredicate().getConditions()) {
- boolean matches = expectedState(condition).equals(seen.get(STATE + keyOf(condition)));
+ boolean matches = expectedState(condition).equals(seen.get(STATE + scope + keyOf(condition)));
allMatch &= matches;
eventMatches |= matches && keyOf(condition).equals(event.key())
&& expectedState(condition).equals(event.completion().state());
@@ -282,7 +794,7 @@ public class GlueTriggerService {
forgetLeastRecentlyUsedChainIfFull();
}
budget.setTriggeredRuns(budget.getTriggeredRuns() + 1);
- budget.setLastUsed(Instant.now());
+ budget.setLastUsed(clock.instant());
budgetStore.put(originRunId, budget);
return true;
}
@@ -320,12 +832,14 @@ public class GlueTriggerService {
}
/**
- * Starts the trigger's actions on behalf of {@code originRunId} and returns how many started; with a
- * null origin (StartTrigger on an ON_DEMAND trigger) each run or crawl started is its own origin and
- * no budget applies. With an origin, each action claims one run of the chain's budget before it
- * starts and gives it back if it cannot start, so only runs that actually start are counted.
+ * Starts the trigger's actions on behalf of {@code originRunId} and returns how many started, inside
+ * {@code workflowRunId} when it is not null. With a null origin (StartTrigger on an ON_DEMAND trigger,
+ * or a workflow's starting trigger) each run or crawl started is its own origin and no budget
+ * applies. With an origin, each action claims one run of the chain's budget before it starts and
+ * gives it back if it cannot start, so only runs that actually start are counted. A job run in a
+ * workflow run gets the {@code --WORKFLOW_NAME} and {@code --WORKFLOW_RUN_ID} arguments, as on AWS.
*/
- private int fire(Trigger trigger, String originRunId, boolean propagateFailures) {
+ private int fire(Trigger trigger, String originRunId, String workflowRunId, boolean propagateFailures) {
LOG.infov("Firing Glue trigger {0}", trigger.getName());
int started = 0;
for (TriggerAction action : trigger.getActions()) {
@@ -337,14 +851,14 @@ public class GlueTriggerService {
try {
if (action.getJobName() != null) {
JobRun overrides = new JobRun();
- overrides.setArguments(action.getArguments());
+ overrides.setArguments(withWorkflowArguments(action.getArguments(), workflowRunId));
overrides.setTimeout(action.getTimeout());
overrides.setSecurityConfiguration(action.getSecurityConfiguration());
overrides.setNotificationProperty(action.getNotificationProperty());
overrides.setTriggerName(trigger.getName());
- jobRunService.startJobRun(action.getJobName(), null, overrides, originRunId);
+ jobRunService.startJobRun(action.getJobName(), null, overrides, originRunId, workflowRunId);
} else {
- crawlerRunService.startCrawler(action.getCrawlerName(), originRunId);
+ crawlerRunService.startCrawler(action.getCrawlerName(), originRunId, workflowRunId);
}
started++;
} catch (AwsException e) {
@@ -354,13 +868,38 @@ public class GlueTriggerService {
if (propagateFailures) {
throw e;
}
- LOG.warnv("Glue trigger {0} could not start {1}: {2}", trigger.getName(),
- action.getJobName() != null ? action.getJobName() : action.getCrawlerName(), e.getMessage());
+ String target = action.getJobName() != null ? action.getJobName() : action.getCrawlerName();
+ LOG.warnv("Glue trigger {0} could not start {1}: {2}", trigger.getName(), target, e.getMessage());
+ if (workflowRunId != null) {
+ noteLaunchFailure(workflowRunId, "Trigger " + trigger.getName() + " could not start " + target
+ + ": " + e.getMessage());
+ }
}
}
return started;
}
+ /** Records on the workflow run that one of its actions could not be launched. */
+ private void noteLaunchFailure(String workflowRunId, String message) {
+ workflowRunStore.get(workflowRunId).ifPresent(run -> {
+ run.setErrorMessage(run.getErrorMessage() == null ? message : run.getErrorMessage() + "; " + message);
+ workflowRunStore.put(workflowRunId, run);
+ });
+ }
+
+ private Map<String, String> withWorkflowArguments(Map<String, String> arguments, String workflowRunId) {
+ if (workflowRunId == null) {
+ return arguments;
+ }
+ Map<String, String> merged = new LinkedHashMap<>();
+ if (arguments != null) {
+ merged.putAll(arguments);
+ }
+ merged.put("--WORKFLOW_NAME", workflowNameOf(workflowRunId));
+ merged.put("--WORKFLOW_RUN_ID", workflowRunId);
+ return merged;
+ }
+
private static String expectedState(TriggerCondition condition) {
return condition.getJobName() != null ? condition.getState() : condition.getCrawlState();
}
@@ -374,4 +913,7 @@ public class GlueTriggerService {
}
private record KeyedCompletion(String key, GlueRunCompletion completion) {}
+
+ /** ResumeWorkflowRun's answer: the new run and the nodes restarted in it. */
+ public record ResumedRun(String runId, List<String> nodeIds) {}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java
index 9a3ee679ab..06141701cf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java
@@ -17,6 +17,7 @@ public class CrawlerRunRecord {
private String currentMessagePrefix;
private String currentCrawlId;
private String currentOriginRunId;
+ private String currentWorkflowRunId;
private Instant lastStart;
private String lastStatus;
private String lastMessagePrefix;
@@ -44,6 +45,9 @@ public class CrawlerRunRecord {
public String getCurrentOriginRunId() { return currentOriginRunId; }
public void setCurrentOriginRunId(String currentOriginRunId) { this.currentOriginRunId = currentOriginRunId; }
+ public String getCurrentWorkflowRunId() { return currentWorkflowRunId; }
+ public void setCurrentWorkflowRunId(String currentWorkflowRunId) { this.currentWorkflowRunId = currentWorkflowRunId; }
+
public String getLastStatus() { return lastStatus; }
public void setLastStatus(String lastStatus) { this.lastStatus = lastStatus; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/FinishedCrawl.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/FinishedCrawl.java
index 24f6f198a4..edd957e46c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/model/FinishedCrawl.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/FinishedCrawl.java
@@ -13,11 +13,13 @@ import java.time.Instant;
@JsonIgnoreProperties(ignoreUnknown = true)
public class FinishedCrawl {
private long sequence;
+ private String crawlerName;
private Instant startedAt;
private Instant finishedAt;
private String status;
private String crawlId;
private String originRunId;
+ private String workflowRunId;
@JsonProperty(value = "triggeredRuns", access = JsonProperty.Access.WRITE_ONLY)
private int legacyTriggeredRuns;
@@ -27,6 +29,9 @@ public class FinishedCrawl {
public int getLegacyTriggeredRuns() { return legacyTriggeredRuns; }
public void setLegacyTriggeredRuns(int legacyTriggeredRuns) { this.legacyTriggeredRuns = legacyTriggeredRuns; }
+ public String getCrawlerName() { return crawlerName; }
+ public void setCrawlerName(String crawlerName) { this.crawlerName = crawlerName; }
+
public long getSequence() { return sequence; }
public void setSequence(long sequence) { this.sequence = sequence; }
@@ -45,4 +50,6 @@ public class FinishedCrawl {
public String getOriginRunId() { return originRunId; }
public void setOriginRunId(String originRunId) { this.originRunId = originRunId; }
+ public String getWorkflowRunId() { return workflowRunId; }
+ public void setWorkflowRunId(String workflowRunId) { this.workflowRunId = workflowRunId; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/JobRunBookkeeping.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/JobRunBookkeeping.java
index 8956652584..6bc9f4c522 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/model/JobRunBookkeeping.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/JobRunBookkeeping.java
@@ -6,7 +6,8 @@ import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Floci's own bookkeeping for one job run, never sent on the wire, and deleted with the run: the order
- * in which Floci saw it finish, and the run that set off its trigger chain.
+ * in which Floci saw it finish, the run that set off its trigger chain, and the workflow run it belongs
+ * to.
*/
@RegisterForReflection
// Earlier versions stored the chain's run count here as triggeredRuns: it is still read, so a chain in
@@ -15,6 +16,7 @@ import io.quarkus.runtime.annotations.RegisterForReflection;
public class JobRunBookkeeping {
private long completionOrder;
private String originRunId;
+ private String workflowRunId;
@JsonProperty(value = "triggeredRuns", access = JsonProperty.Access.WRITE_ONLY)
private int legacyTriggeredRuns;
@@ -30,4 +32,6 @@ public class JobRunBookkeeping {
public String getOriginRunId() { return originRunId; }
public void setOriginRunId(String originRunId) { this.originRunId = originRunId; }
+ public String getWorkflowRunId() { return workflowRunId; }
+ public void setWorkflowRunId(String workflowRunId) { this.workflowRunId = workflowRunId; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Workflow.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Workflow.java
new file mode 100644
index 0000000000..245859ccd8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Workflow.java
@@ -0,0 +1,69 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.Map;
+
+/**
+ * A Glue workflow. LastRun and Graph are filled in when the workflow is read; the stored copy
+ * leaves them unset.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Workflow {
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("DefaultRunProperties")
+ private Map<String, String> defaultRunProperties;
+
+ @JsonProperty("CreatedOn")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant createdOn;
+
+ @JsonProperty("LastModifiedOn")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant lastModifiedOn;
+
+ @JsonProperty("LastRun")
+ private WorkflowRun lastRun;
+
+ @JsonProperty("Graph")
+ private Map<String, Object> graph;
+
+ @JsonProperty("MaxConcurrentRuns")
+ private Integer maxConcurrentRuns;
+
+ public Workflow() {}
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public Map<String, String> getDefaultRunProperties() { return defaultRunProperties; }
+ public void setDefaultRunProperties(Map<String, String> defaultRunProperties) { this.defaultRunProperties = defaultRunProperties; }
+
+ public Instant getCreatedOn() { return createdOn; }
+ public void setCreatedOn(Instant createdOn) { this.createdOn = createdOn; }
+
+ public Instant getLastModifiedOn() { return lastModifiedOn; }
+ public void setLastModifiedOn(Instant lastModifiedOn) { this.lastModifiedOn = lastModifiedOn; }
+
+ public WorkflowRun getLastRun() { return lastRun; }
+ public void setLastRun(WorkflowRun lastRun) { this.lastRun = lastRun; }
+
+ public Map<String, Object> getGraph() { return graph; }
+ public void setGraph(Map<String, Object> graph) { this.graph = graph; }
+
+ public Integer getMaxConcurrentRuns() { return maxConcurrentRuns; }
+ public void setMaxConcurrentRuns(Integer maxConcurrentRuns) { this.maxConcurrentRuns = maxConcurrentRuns; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/WorkflowRun.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/WorkflowRun.java
new file mode 100644
index 0000000000..5d2cf866e8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/WorkflowRun.java
@@ -0,0 +1,81 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.Map;
+
+/**
+ * One run of a Glue workflow. Statistics and Graph are computed from the job runs and crawls of
+ * the run when it is read; the stored copy leaves them unset.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class WorkflowRun {
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("WorkflowRunId")
+ private String workflowRunId;
+
+ @JsonProperty("PreviousRunId")
+ private String previousRunId;
+
+ @JsonProperty("WorkflowRunProperties")
+ private Map<String, String> workflowRunProperties;
+
+ @JsonProperty("StartedOn")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant startedOn;
+
+ @JsonProperty("CompletedOn")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant completedOn;
+
+ @JsonProperty("Status")
+ private String status;
+
+ @JsonProperty("ErrorMessage")
+ private String errorMessage;
+
+ @JsonProperty("Statistics")
+ private Map<String, Integer> statistics;
+
+ @JsonProperty("Graph")
+ private Map<String, Object> graph;
+
+ public WorkflowRun() {}
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public String getWorkflowRunId() { return workflowRunId; }
+ public void setWorkflowRunId(String workflowRunId) { this.workflowRunId = workflowRunId; }
+
+ public String getPreviousRunId() { return previousRunId; }
+ public void setPreviousRunId(String previousRunId) { this.previousRunId = previousRunId; }
+
+ public Map<String, String> getWorkflowRunProperties() { return workflowRunProperties; }
+ public void setWorkflowRunProperties(Map<String, String> workflowRunProperties) { this.workflowRunProperties = workflowRunProperties; }
+
+ public Instant getStartedOn() { return startedOn; }
+ public void setStartedOn(Instant startedOn) { this.startedOn = startedOn; }
+
+ public Instant getCompletedOn() { return completedOn; }
+ public void setCompletedOn(Instant completedOn) { this.completedOn = completedOn; }
+
+ public String getStatus() { return status; }
+ public void setStatus(String status) { this.status = status; }
+
+ public String getErrorMessage() { return errorMessage; }
+ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
+
+ public Map<String, Integer> getStatistics() { return statistics; }
+ public void setStatistics(Map<String, Integer> statistics) { this.statistics = statistics; }
+
+ public Map<String, Object> getGraph() { return graph; }
+ public void setGraph(Map<String, Object> graph) { this.graph = graph; }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java
index 0b94ca1413..ff4d4fc1aa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java
@@ -66,7 +66,7 @@ class GlueCrawlerRunServiceTest {
}
private GlueCrawlerRunService service(int runDurationSeconds) {
- return new GlueCrawlerRunService(recordStore, glueService, runDurationSeconds, clock);
+ return new GlueCrawlerRunService(recordStore, new InMemoryStorage<>(), glueService, runDurationSeconds, clock);
}
private void createCrawler(String name, String scheduleExpression) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
index 7b227578e2..800ea218ae 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
@@ -48,9 +48,9 @@ class GlueJsonHandlerEmptyListTest {
GlueJobRunService jobRunService =
new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
GlueCrawlerRunService crawlerRunService =
- new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
+ new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
handler = new GlueJsonHandler(glueService, jobRunService, crawlerRunService,
- new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService,
+ new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService,
jobRunService, crawlerRunService),
schemaRegistryService, mapper);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
index c370eb5d13..c27ede91d3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
@@ -49,9 +49,9 @@ class GlueJsonHandlerTest {
GlueJobRunService jobRunService =
new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
GlueCrawlerRunService crawlerRunService =
- new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
+ new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
handler = new GlueJsonHandler(glueService, jobRunService, crawlerRunService,
- new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService,
+ new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService,
jobRunService, crawlerRunService),
schemaRegistryService, mapper);
}
@@ -1021,4 +1021,84 @@ class GlueJsonHandlerTest {
assertEquals(1, runs.size());
assertEquals("after-extract", runs.get(0).get("TriggerName").asText());
}
+
+ /**
+ * Workflow operations on the wire: CreateWorkflow answers with the name, StartWorkflowRun with the
+ * run id, GetWorkflowRun nests the run under "Run" with numeric timestamps and Statistics, and
+ * GetWorkflow with IncludeGraph carries Nodes and Edges.
+ */
+ @Test
+ void workflowOperationsAnswerWithTheDocumentedBodies() throws Exception {
+ createJobWithTags("extract", Map.of());
+ createJobWithTags("load", Map.of());
+ ObjectNode create = mapper.createObjectNode();
+ create.put("Name", "etl");
+ create.putObject("DefaultRunProperties").put("env", "dev");
+ create.putObject("Tags").put("team", "data");
+ assertEquals("etl", mapper.valueToTree(handler.handle("CreateWorkflow", create, REGION).getEntity())
+ .get("Name").asText());
+ ObjectNode start = mapper.createObjectNode();
+ start.put("Name", "start");
+ start.put("Type", "ON_DEMAND");
+ start.put("WorkflowName", "etl");
+ start.putArray("Actions").addObject().put("JobName", "extract");
+ handler.handle("CreateTrigger", start, REGION);
+ ObjectNode then = mapper.createObjectNode();
+ then.put("Name", "then-load");
+ then.put("Type", "CONDITIONAL");
+ then.put("WorkflowName", "etl");
+ then.put("StartOnCreation", true);
+ then.putArray("Actions").addObject().put("JobName", "load");
+ then.putObject("Predicate").putArray("Conditions").addObject()
+ .put("LogicalOperator", "EQUALS").put("JobName", "extract").put("State", "SUCCEEDED");
+ handler.handle("CreateTrigger", then, REGION);
+
+ ObjectNode startRun = mapper.createObjectNode().put("Name", "etl");
+ String runId = mapper.valueToTree(handler.handle("StartWorkflowRun", startRun, REGION).getEntity())
+ .get("RunId").asText();
+
+ ObjectNode getRun = mapper.createObjectNode().put("Name", "etl").put("RunId", runId);
+ JsonNode run = mapper.valueToTree(handler.handle("GetWorkflowRun", getRun, REGION).getEntity()).get("Run");
+ assertEquals("COMPLETED", run.get("Status").asText());
+ assertEquals("etl", run.get("Name").asText());
+ assertTrue(run.get("StartedOn").isNumber());
+ assertTrue(run.get("CompletedOn").isNumber());
+ assertEquals(2, run.get("Statistics").get("SucceededActions").asInt());
+ assertEquals("dev", run.get("WorkflowRunProperties").get("env").asText());
+ assertFalse(run.has("Graph"));
+
+ ObjectNode getWorkflow = mapper.createObjectNode().put("Name", "etl").put("IncludeGraph", true);
+ JsonNode workflow = mapper.valueToTree(handler.handle("GetWorkflow", getWorkflow, REGION).getEntity())
+ .get("Workflow");
+ assertEquals(runId, workflow.get("LastRun").get("WorkflowRunId").asText());
+ assertEquals(4, workflow.get("Graph").get("Nodes").size());
+ assertEquals(3, workflow.get("Graph").get("Edges").size());
+ assertTrue(workflow.get("CreatedOn").isNumber());
+
+ JsonNode runs = mapper.valueToTree(handler.handle(
+ "GetWorkflowRuns", mapper.createObjectNode().put("Name", "etl"), REGION).getEntity());
+ assertEquals(1, runs.get("Runs").size());
+ JsonNode names = mapper.valueToTree(handler.handle("ListWorkflows", mapper.createObjectNode(), REGION).getEntity());
+ assertEquals("etl", names.get("Workflows").get(0).asText());
+
+ ObjectNode put = mapper.createObjectNode().put("Name", "etl").put("RunId", runId);
+ put.putObject("RunProperties").put("rows", "42");
+ assertEquals(0, mapper.valueToTree(handler.handle("PutWorkflowRunProperties", put, REGION).getEntity()).size());
+ JsonNode properties = mapper.valueToTree(handler.handle("GetWorkflowRunProperties", getRun, REGION).getEntity());
+ assertEquals("42", properties.get("RunProperties").get("rows").asText());
+
+ ObjectNode tags = mapper.createObjectNode();
+ tags.put("ResourceArn", "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":workflow/etl");
+ assertEquals("data", mapper.valueToTree(handler.handle("GetTags", tags, REGION).getEntity())
+ .get("Tags").get("team").asText());
+
+ ObjectNode batch = mapper.createObjectNode();
+ batch.putArray("Names").add("etl").add("absent");
+ JsonNode got = mapper.valueToTree(handler.handle("BatchGetWorkflows", batch, REGION).getEntity());
+ assertEquals("etl", got.get("Workflows").get(0).get("Name").asText());
+ assertEquals("absent", got.get("MissingWorkflows").get(0).asText());
+
+ assertEquals("etl", mapper.valueToTree(handler.handle(
+ "DeleteWorkflow", mapper.createObjectNode().put("Name", "etl"), REGION).getEntity()).get("Name").asText());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index 484165bdb2..6b5a438754 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -32,6 +32,7 @@ import io.github.hectorvent.floci.services.glue.model.S3Target;
import io.github.hectorvent.floci.services.glue.model.SchemaReference;
import io.github.hectorvent.floci.services.glue.model.SecurityConfiguration;
import io.github.hectorvent.floci.services.glue.model.Trigger;
+import io.github.hectorvent.floci.services.glue.model.Workflow;
import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
import io.github.hectorvent.floci.services.glue.model.Table;
import io.github.hectorvent.floci.services.glue.model.UserDefinedFunction;
@@ -112,6 +113,7 @@ class GlueServiceTest {
new InMemoryStorage<String, DataCatalogEncryptionSettings>(),
new InMemoryStorage<String, SecurityConfiguration>(),
new InMemoryStorage<String, Trigger>(),
+ new InMemoryStorage<String, Workflow>(),
schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null), kmsService);
glueService.createDatabase(new Database("db1"));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueTriggerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueTriggerServiceTest.java
index 6ed40a69ba..839d501b5e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueTriggerServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueTriggerServiceTest.java
@@ -67,8 +67,8 @@ class GlueTriggerServiceTest {
regionResolver, new ResourceGroupsTaggingService(storageFactory),
new KmsService(storageFactory, regionResolver));
jobRuns = new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
- crawls = new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
- triggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, crawls);
+ crawls = new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());
+ triggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, crawls);
for (String job : List.of("extract", "load", "report")) {
createJob(job);
}
@@ -326,10 +326,9 @@ class GlueTriggerServiceTest {
/** A run that could not start gives its claim back, so a failing action does not shorten the chain. */
@Test
void actionsThatFailToStartDoNotUseUpTheBudget() {
- GlueCrawlerRunService slowCrawls = new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 600,
+ GlueCrawlerRunService slowCrawls = new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 600,
new MutableClock());
- GlueTriggerService loopTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(),
- glueService, jobRuns, slowCrawls);
+ GlueTriggerService loopTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, slowCrawls);
slowCrawls.startCrawler("raw");
Trigger loop = conditional("loop", null, List.of(jobIs("extract", "SUCCEEDED")), "extract");
TriggerAction crawl = new TriggerAction();
@@ -351,8 +350,7 @@ class GlueTriggerServiceTest {
MutableClock clock = new MutableClock();
GlueJobRunService slowRuns = new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(),
glueService, 60, clock);
- GlueTriggerService chain = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(),
- glueService, slowRuns, crawls);
+ GlueTriggerService chain = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, crawls);
chain.createTrigger(conditional("after-crawl", "ANY", List.of(crawlIs("raw", "SUCCEEDED")), "extract"),
true, null, REGION);
chain.createTrigger(conditional("after-extract", null, List.of(jobIs("extract", "SUCCEEDED")), "load"),
@@ -376,9 +374,8 @@ class GlueTriggerServiceTest {
InMemoryStorage<String, JobRunBookkeeping> bookkeeping = new InMemoryStorage<>();
GlueJobRunService slowRuns = new GlueJobRunService(new InMemoryStorage<>(), bookkeeping, glueService, 60, clock);
InMemoryStorage<String, CrawlerRunRecord> crawlRecords = new InMemoryStorage<>();
- GlueCrawlerRunService slowCrawls = new GlueCrawlerRunService(crawlRecords, glueService, 60, clock);
- GlueTriggerService legacy = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(),
- glueService, slowRuns, slowCrawls);
+ GlueCrawlerRunService slowCrawls = new GlueCrawlerRunService(crawlRecords, new InMemoryStorage<>(), glueService, 60, clock);
+ GlueTriggerService legacy = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, slowCrawls);
legacy.createTrigger(conditional("after-extract", null, List.of(jobIs("extract", "SUCCEEDED")), "load"),
true, null, REGION);
legacy.createTrigger(conditional("after-crawl", "ANY", List.of(crawlIs("raw", "SUCCEEDED")), "report"),
@@ -419,8 +416,7 @@ class GlueTriggerServiceTest {
MutableClock clock = new MutableClock();
InMemoryStorage<String, JobRunBookkeeping> bookkeeping = new InMemoryStorage<>();
GlueJobRunService slowRuns = new GlueJobRunService(new InMemoryStorage<>(), bookkeeping, glueService, 60, clock);
- GlueTriggerService upgraded = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(),
- glueService, slowRuns, crawls);
+ GlueTriggerService upgraded = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, crawls);
upgraded.createTrigger(conditional("loop", null, List.of(jobIs("extract", "SUCCEEDED")), "extract"),
true, null, REGION);
String origin = slowRuns.startJobRun("extract", null, new JobRun()).getId();
@@ -440,8 +436,8 @@ class GlueTriggerServiceTest {
void theBudgetStoreKeepsAtMostTheTrackedChains() {
int cap = 20;
InMemoryStorage<String, TriggerChainBudget> budgets = new InMemoryStorage<>();
- GlueTriggerService bounded = new GlueTriggerService(new InMemoryStorage<>(), budgets, glueService, jobRuns,
- crawls, cap);
+ GlueTriggerService bounded = new GlueTriggerService(new InMemoryStorage<>(), budgets, new InMemoryStorage<>(), glueService, jobRuns,
+ crawls, cap, Clock.systemUTC());
bounded.createTrigger(conditional("after", null, List.of(jobIs("extract", "SUCCEEDED")), "load"),
true, null, REGION);
@@ -633,7 +629,7 @@ class GlueTriggerServiceTest {
void aSuccessIsNotLostWhenALaterRunOfTheSameJobIsStoppedFirst() {
MutableClock clock = new MutableClock();
GlueJobRunService slowRuns = new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 60, clock);
- GlueTriggerService slowTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, crawls);
+ GlueTriggerService slowTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, crawls);
slowTriggers.createTrigger(conditional("after", null, List.of(jobIs("extract", "SUCCEEDED")), "load"),
true, null, REGION);
slowRuns.startJobRun("extract", null, new JobRun());
@@ -652,7 +648,7 @@ class GlueTriggerServiceTest {
Clock frozen = Clock.fixed(Instant.parse("2026-09-25T12:00:00Z"), ZoneOffset.UTC);
GlueJobRunService frozenRuns =
new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, frozen);
- GlueTriggerService frozenTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, frozenRuns, crawls);
+ GlueTriggerService frozenTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, frozenRuns, crawls);
frozenTriggers.createTrigger(conditional("each", "ANY", List.of(jobIs("extract", "SUCCEEDED")), "load"),
true, null, REGION);
@@ -667,9 +663,9 @@ class GlueTriggerServiceTest {
@Test
void crawlsFinishingInTheSameClockTickEachCount() {
Clock frozen = Clock.fixed(Instant.parse("2026-09-25T12:00:00Z"), ZoneOffset.UTC);
- GlueCrawlerRunService frozenCrawls = new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, frozen);
+ GlueCrawlerRunService frozenCrawls = new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, frozen);
GlueTriggerService frozenTriggers =
- new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, frozenCrawls);
+ new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, frozenCrawls);
frozenTriggers.createTrigger(conditional("each", "ANY", List.of(crawlIs("raw", "SUCCEEDED")), "load"),
true, null, REGION);
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowIntegrationTest.java
new file mode 100644
index 0000000000..42e22e32f4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowIntegrationTest.java
@@ -0,0 +1,71 @@
+package io.github.hectorvent.floci.services.glue;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+
+@QuarkusTest
+class GlueWorkflowIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String ROLE = "arn:aws:iam::000000000000:role/my-role";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ private static ValidatableResponse call(String action, String body) {
+ return given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue." + action)
+ .body(body)
+ .when().post("/")
+ .then();
+ }
+
+ @Test
+ void aWorkflowRunRunsItsTriggersToCompletion() {
+ String suffix = UUID.randomUUID().toString().substring(0, 8);
+ String workflow = "wf-" + suffix;
+ String extract = "extract-" + suffix;
+ String load = "load-" + suffix;
+ for (String job : new String[] {extract, load}) {
+ call("CreateJob", """
+ { "Name": "%s", "Role": "%s", "Command": { "Name": "glueetl" } }
+ """.formatted(job, ROLE)).statusCode(200);
+ }
+ call("CreateWorkflow", "{ \"Name\": \"" + workflow + "\" }")
+ .statusCode(200)
+ .body("Name", equalTo(workflow));
+ call("CreateTrigger", """
+ { "Name": "start-%s", "Type": "ON_DEMAND", "WorkflowName": "%s", "Actions": [ { "JobName": "%s" } ] }
+ """.formatted(suffix, workflow, extract)).statusCode(200);
+ call("CreateTrigger", """
+ { "Name": "then-%s", "Type": "CONDITIONAL", "WorkflowName": "%s", "StartOnCreation": true,
+ "Actions": [ { "JobName": "%s" } ],
+ "Predicate": { "Conditions": [
+ { "LogicalOperator": "EQUALS", "JobName": "%s", "State": "SUCCEEDED" } ] } }
+ """.formatted(suffix, workflow, load, extract)).statusCode(200);
+
+ String runId = call("StartWorkflowRun", "{ \"Name\": \"" + workflow + "\" }")
+ .statusCode(200)
+ .extract().path("RunId");
+
+ call("GetWorkflowRun", "{ \"Name\": \"" + workflow + "\", \"RunId\": \"" + runId + "\" }")
+ .statusCode(200)
+ .body("Run.Status", equalTo("COMPLETED"))
+ .body("Run.Statistics.SucceededActions", equalTo(2));
+
+ call("DeleteWorkflow", "{ \"Name\": \"" + workflow + "\" }").statusCode(200);
+ call("GetWorkflow", "{ \"Name\": \"" + workflow + "\" }")
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowServiceTest.java
new file mode 100644
index 0000000000..76dbd49ffe
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowServiceTest.java
@@ -0,0 +1,641 @@
+package io.github.hectorvent.floci.services.glue;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.glue.model.Crawler;
+import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;
+import io.github.hectorvent.floci.services.glue.model.ExecutionProperty;
+import io.github.hectorvent.floci.services.glue.model.Job;
+import io.github.hectorvent.floci.services.glue.model.JobCommand;
+import io.github.hectorvent.floci.services.glue.model.JobRun;
+import io.github.hectorvent.floci.services.glue.model.Predicate;
+import io.github.hectorvent.floci.services.glue.model.S3Target;
+import io.github.hectorvent.floci.services.glue.model.Trigger;
+import io.github.hectorvent.floci.services.glue.model.TriggerAction;
+import io.github.hectorvent.floci.services.glue.model.TriggerCondition;
+import io.github.hectorvent.floci.services.glue.model.Workflow;
+import io.github.hectorvent.floci.services.glue.model.WorkflowRun;
+import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.kms.KmsService;
+import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class GlueWorkflowServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT_ID = "000000000000";
+ private static final String ROLE = "arn:aws:iam::000000000000:role/glue";
+
+ private GlueService glueService;
+ private GlueJobRunService jobRuns;
+ private GlueCrawlerRunService crawls;
+ private GlueTriggerService workflows;
+ private MutableClock clock;
+
+ @BeforeEach
+ void setUp() {
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);
+ StorageFactory storageFactory = new InMemoryStorageFactory();
+ glueService = new GlueService(storageFactory, new GlueSchemaRegistryService(storageFactory, regionResolver),
+ regionResolver, new ResourceGroupsTaggingService(storageFactory),
+ new KmsService(storageFactory, regionResolver));
+ clock = new MutableClock();
+ useRunDuration(0);
+ for (String job : List.of("extract", "load")) {
+ Job definition = new Job();
+ definition.setName(job);
+ definition.setRole(ROLE);
+ definition.setCommand(new JobCommand());
+ ExecutionProperty property = new ExecutionProperty();
+ property.setMaxConcurrentRuns(10);
+ definition.setExecutionProperty(property);
+ glueService.createJob(definition, null, REGION);
+ }
+ Crawler crawler = new Crawler();
+ crawler.setName("raw");
+ crawler.setRole(ROLE);
+ S3Target target = new S3Target();
+ target.setPath("s3://raw/data");
+ CrawlerTargets targets = new CrawlerTargets();
+ targets.setS3Targets(List.of(target));
+ crawler.setTargets(targets);
+ glueService.createCrawler(crawler, null, REGION);
+ }
+
+ private void useRunDuration(int seconds) {
+ useDurations(seconds, seconds);
+ }
+
+ private void useDurations(int jobSeconds, int crawlSeconds) {
+ jobRuns = new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobSeconds, clock);
+ crawls = new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, crawlSeconds,
+ clock);
+ workflows = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),
+ glueService, jobRuns, crawls, GlueTriggerService.MAX_TRACKED_CHAINS, clock);
+ }
+
+ private void createWorkflow(String name, Integer maxConcurrentRuns, Map<String, String> defaults) {
+ Workflow workflow = new Workflow();
+ workflow.setName(name);
+ workflow.setMaxConcurrentRuns(maxConcurrentRuns);
+ workflow.setDefaultRunProperties(defaults);
+ glueService.createWorkflow(workflow, null, REGION);
+ }
+
+ private static TriggerAction startJob(String job) {
+ TriggerAction action = new TriggerAction();
+ action.setJobName(job);
+ return action;
+ }
+
+ private static TriggerAction startCrawler(String crawler) {
+ TriggerAction action = new TriggerAction();
+ action.setCrawlerName(crawler);
+ return action;
+ }
+
+ private static TriggerCondition jobIs(String job, String state) {
+ TriggerCondition condition = new TriggerCondition();
+ condition.setLogicalOperator("EQUALS");
+ condition.setJobName(job);
+ condition.setState(state);
+ return condition;
+ }
+
+ private static TriggerCondition crawlIs(String crawler, String state) {
+ TriggerCondition condition = new TriggerCondition();
+ condition.setLogicalOperator("EQUALS");
+ condition.setCrawlerName(crawler);
+ condition.setCrawlState(state);
+ return condition;
+ }
+
+ private void onDemand(String name, String workflow, TriggerAction... actions) {
+ Trigger trigger = new Trigger();
+ trigger.setName(name);
+ trigger.setType("ON_DEMAND");
+ trigger.setWorkflowName(workflow);
+ trigger.setActions(List.of(actions));
+ workflows.createTrigger(trigger, false, null, REGION);
+ }
+
+ private void conditional(String name, String workflow, String logical, List<TriggerCondition> conditions,
+ String startsJob) {
+ Trigger trigger = new Trigger();
+ trigger.setName(name);
+ trigger.setType("CONDITIONAL");
+ trigger.setWorkflowName(workflow);
+ trigger.setActions(List.of(startJob(startsJob)));
+ Predicate predicate = new Predicate();
+ predicate.setLogical(logical);
+ predicate.setConditions(conditions);
+ trigger.setPredicate(predicate);
+ workflows.createTrigger(trigger, true, null, REGION);
+ }
+
+ /** extract, then load when extract succeeds: the smallest two-step workflow. */
+ private void twoStepPipeline(String workflow) {
+ onDemand("start", workflow, startJob("extract"));
+ conditional("then-load", workflow, null, List.of(jobIs("extract", "SUCCEEDED")), "load");
+ }
+
+ private List<JobRun> runsOf(String job) {
+ return jobRuns.getJobRuns(job, null, null).items();
+ }
+
+ @Test
+ void definitionsFollowTheApiRules() {
+ createWorkflow("etl", null, null);
+ assertEquals("AlreadyExistsException", assertThrows(AwsException.class,
+ () -> createWorkflow("etl", null, null)).getErrorCode());
+ assertEquals("InvalidInputException", assertThrows(AwsException.class,
+ () -> createWorkflow("zero", 0, null)).getErrorCode());
+
+ glueService.updateWorkflow("etl", "nightly", null, 2);
+ Workflow updated = glueService.getWorkflow("etl");
+ assertEquals("nightly", updated.getDescription());
+ assertEquals(2, updated.getMaxConcurrentRuns());
+
+ assertEquals("EntityNotFoundException", assertThrows(AwsException.class,
+ () -> onDemand("orphan", "missing", startJob("extract"))).getErrorCode());
+ Trigger event = new Trigger();
+ event.setName("on-event");
+ event.setType("EVENT");
+ event.setActions(List.of(startJob("extract")));
+ assertEquals("InvalidInputException", assertThrows(AwsException.class,
+ () -> workflows.createTrigger(event, false, null, REGION)).getErrorCode());
+ event.setWorkflowName("etl");
+ workflows.createTrigger(event, false, null, REGION);
+ assertEquals("etl", glueService.getTrigger("on-event").getWorkflowName());
+
+ workflows.deleteWorkflow("etl", REGION);
+ workflows.deleteWorkflow("etl", REGION);
+ assertEquals("EntityNotFoundException",
+ assertThrows(AwsException.class, () -> glueService.getWorkflow("etl")).getErrorCode());
+ }
+
+ @Test
+ void aWorkflowRunRunsItsPipelineAndCompletes() {
+ createWorkflow("etl", null, Map.of("env", "dev", "day", "default"));
+ twoStepPipeline("etl");
+
+ String runId = workflows.startWorkflowRun("etl", Map.of("day", "2026-09-26"));
+
+ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);
+ assertTrue(runId.matches("wr_[0-9a-f]{64}"), runId);
+ assertEquals("COMPLETED", run.getStatus());
+ assertNotNull(run.getCompletedOn());
+ assertEquals(Map.of("env", "dev", "day", "2026-09-26"), run.getWorkflowRunProperties());
+ assertEquals(2, run.getStatistics().get("TotalActions"));
+ assertEquals(2, run.getStatistics().get("SucceededActions"));
+ assertEquals(0, run.getStatistics().get("RunningActions"));
+ JobRun load = runsOf("load").getFirst();
+ assertEquals("then-load", load.getTriggerName());
+ assertEquals("etl", load.getArguments().get("--WORKFLOW_NAME"));
+ assertEquals(runId, load.getArguments().get("--WORKFLOW_RUN_ID"));
+ }
+
+ /** Run times come from the service clock, so a run's timestamps follow the actions in it. */
+ @Test
+ void aRunIsTimedByTheActionsInIt() {
+ useRunDuration(60);
+ createWorkflow("etl", null, null);
+ twoStepPipeline("etl");
+
+ Instant before = clock.instant();
+ String runId = workflows.startWorkflowRun("etl", null);
+ Instant after = clock.instant();
+ clock.advance(Duration.ofSeconds(61));
+ assertEquals("RUNNING", workflows.getWorkflowRun("etl", runId, false).getStatus());
+ clock.advance(Duration.ofSeconds(61));
+ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);
+
+ assertEquals("COMPLETED", run.getStatus());
+ assertTrue(run.getStartedOn().isAfter(before) && run.getStartedOn().isBefore(after), run.getStartedOn().toString());
+ JobRun load = runsOf("load").getFirst();
+ assertEquals(load.getCompletedOn(), run.getCompletedOn());
+ assertTrue(run.getCompletedOn().isAfter(runsOf("extract").getFirst().getCompletedOn()));
+ long seconds = Duration.between(run.getStartedOn(), run.getCompletedOn()).toSeconds();
+ assertTrue(seconds >= 120 && seconds < 125, seconds + " seconds");
+ }
+
+ @Test
+ void aStoppedRunCompletesWhenItIsStopped() {
+ useRunDuration(60);
+ createWorkflow("etl", null, null);
+ twoStepPipeline("etl");
+ String runId = workflows.startWorkflowRun("etl", null);
+ clock.advance(Duration.ofSeconds(30));
+
+ Instant before = clock.instant();
+ workflows.stopWorkflowRun("etl", runId);
+ Instant after = clock.instant();
+
+ Instant completedOn = workflows.getWorkflowRun("etl", runId, false).getCompletedOn();
+ assertTrue(completedOn.isAfter(before) && completedOn.isBefore(after), completedOn.toString());
+ }
+
+ @Test
+ void workflowTriggersIgnoreRunsOutsideTheirWorkflow() {
+ createWorkflow("etl", null, null);
+ twoStepPipeline("etl");
+
+ jobRuns.startJobRun("extract", null, new JobRun());
+ workflows.fireConditionalTriggers();
+
+ assertTrue(runsOf("load").isEmpty());
+ }
+
+ @Test
+ void andIsMetWithinOneWorkflowRunOnly() {
+ createWorkflow("etl", null, null);
+ onDemand("start", "etl", startJob("extract"));
+ conditional("both", "etl", "AND", List.of(jobIs("extract", "SUCCEEDED"), crawlIs("raw", "SUCCEEDED")), "load");
+ crawls.startCrawler("raw");
+
+ String runId = workflows.startWorkflowRun("etl", null);
+
+ assertTrue(runsOf("load").isEmpty());
+ assertEquals("COMPLETED", workflows.getWorkflowRun("etl", runId, false).getStatus());
+
+ onDemand("start-both", "etl", startCrawler("raw"));
+ workflows.startWorkflowRun("etl", null);
+ assertEquals(1, runsOf("load").size());
+ }
+
+ @Test
+ void maxConcurrentRunsRefusesAnotherRunWhileOneIsRunning() {
+ useRunDuration(60);
+ createWorkflow("etl", 1, null);
+ twoStepPipeline("etl");
+
+ String first = workflows.startWorkflowRun("etl", null);
+ assertEquals("ConcurrentRunsExceededException", assertThrows(AwsException.class,
+ () -> workflows.startWorkflowRun("etl", null)).getErrorCode());
+ assertEquals("RUNNING", workflows.getWorkflowRun("etl", first, false).getStatus());
+
+ clock.advance(Duration.ofSeconds(61));
+ assertEquals("RUNNING", workflows.getWorkflowRun("etl", first, false).getStatus());
+ clock.advance(Duration.ofSeconds(61));
+ assertEquals("COMPLETED", workflows.getWorkflowRun("etl", first, false).getStatus());
+ assertNotNull(workflows.startWorkflowRun("etl", null));
+ }
+
+ @Test
+ void stoppingARunStopsWhatIsRunningInIt() {
+ useRunDuration(60);
+ createWorkflow("etl", null, null);
+ twoStepPipeline("etl");
+ String runId = workflows.startWorkflowRun("etl", null);
+
+ workflows.stopWorkflowRun("etl", runId);
+
+ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);
+ assertEquals("STOPPED", run.getStatus());
+ assertEquals(1, run.getStatistics().get("StoppedActions"));
+ assertEquals("STOPPED", runsOf("extract").getFirst().getJobRunState());
+ assertEquals("IllegalWorkflowStateException", assertThrows(AwsException.class,
+ () -> workflows.stopWorkflowRun("etl", runId)).getErrorCode());
+ clock.advance(Duration.ofSeconds(120));
+ workflows.fireConditionalTriggers();
+ assertTrue(runsOf("load").isEmpty());
+ }
+
+ @Test
+ void resumingRestartsTheChosenNodesAndWhatFollowsInANewRun() {
+ createWorkflow("etl", null, Map.of("env", "dev"));
+ twoStepPipeline("etl");
+ String first = workflows.startWorkflowRun("etl", null);
+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();
+ String extractNode = nodeId(graph, "JOB", "extract");
+ String triggerNode = nodeId(graph, "TRIGGER", "start");
+
+ String resumed = workflows.resumeWorkflowRun("etl", first, List.of(extractNode)).runId();
+
+ WorkflowRun run = workflows.getWorkflowRun("etl", resumed, false);
+ assertEquals(first, run.getPreviousRunId());
+ assertEquals("COMPLETED", run.getStatus());
+ assertEquals(Map.of("env", "dev"), run.getWorkflowRunProperties());
+ assertEquals(2, run.getStatistics().get("TotalActions"));
+ assertEquals(2, runsOf("load").size());
+ // The downstream node ran because of the resume, inside the resumed run, started by its trigger.
+ JobRun downstream = runsOf("load").stream()
+ .filter(load -> resumed.equals(load.getArguments().get("--WORKFLOW_RUN_ID")))
+ .findFirst().orElseThrow();
+ assertEquals("then-load", downstream.getTriggerName());
+ assertEquals("SUCCEEDED", downstream.getJobRunState());
+ assertEquals(List.of(downstream.getId()), jobRunIdsOnNode(workflows.getWorkflowRun("etl", resumed, true), "load"));
+ assertEquals("InvalidInputException", assertThrows(AwsException.class,
+ () -> workflows.resumeWorkflowRun("etl", first, List.of(triggerNode))).getErrorCode());
+ }
+
+ /**
+ * Resume runs "the selected nodes and all nodes that are downstream from the selected nodes": in
+ * extract, then load, then report, resuming load runs load and report in the new run, and not extract.
+ */
+ @Test
+ void resumingAMiddleNodeRunsItAndWhatIsDownstreamButNotWhatIsUpstream() {
+ Job report = new Job();
+ report.setName("report");
+ report.setRole(ROLE);
+ report.setCommand(new JobCommand());
+ glueService.createJob(report, null, REGION);
+ createWorkflow("etl", null, null);
+ twoStepPipeline("etl");
+ conditional("then-report", "etl", null, List.of(jobIs("load", "SUCCEEDED")), "report");
+ String first = workflows.startWorkflowRun("etl", null);
+ assertEquals(3, workflows.getWorkflowRun("etl", first, false).getStatistics().get("SucceededActions"));
+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();
+
+ String resumed = workflows.resumeWorkflowRun("etl", first, List.of(nodeId(graph, "JOB", "load"))).runId();
+
+ WorkflowRun run = workflows.getWorkflowRun("etl", resumed, true);
+ assertEquals("COMPLETED", run.getStatus());
+ assertEquals(2, run.getStatistics().get("TotalActions"));
+ assertEquals(2, run.getStatistics().get("SucceededActions"));
+ assertEquals(1, jobRunIdsOnNode(run, "load").size());
+ assertEquals(1, jobRunIdsOnNode(run, "report").size());
+ assertTrue(jobRunIdsOnNode(run, "extract").isEmpty());
+ assertEquals(1, runsOf("extract").size());
+ JobRun downstream = runsOf("report").stream()
+ .filter(r -> resumed.equals(r.getArguments().get("--WORKFLOW_RUN_ID")))
+ .findFirst().orElseThrow();
+ assertEquals("then-report", downstream.getTriggerName());
+ }
+
+ @SuppressWarnings("unchecked")
+ private static List<String> jobRunIdsOnNode(WorkflowRun run, String jobName) {
+ Map<String, Object> node = ((List<Map<String, Object>>) run.getGraph().get("Nodes")).stream()
+ .filter(n -> "JOB".equals(n.get("Type")) && jobName.equals(n.get("Name")))
+ .findFirst().orElseThrow();
+ return ((List<JobRun>) ((Map<String, Object>) node.get("JobDetails")).get("JobRuns")).stream()
+ .map(JobRun::getId).toList();
+ }
+
+ @Test
+ void aNodeNamedTwiceIsResumedOnce() {
+ useDurations(0, 60);
+ createWorkflow("etl", null, null);
+ onDemand("start", "etl", startCrawler("raw"));
+ String first = workflows.startWorkflowRun("etl", null);
+ clock.advance(Duration.ofSeconds(61));
+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();
+ String raw = nodeId(graph, "CRAWLER", "raw");
+
+ GlueTriggerService.ResumedRun resumed = workflows.resumeWorkflowRun("etl", first, List.of(raw, raw));
+
+ assertEquals(List.of(raw), resumed.nodeIds());
+ assertEquals("RUNNING", workflows.getWorkflowRun("etl", resumed.runId(), false).getStatus());
+ assertNull(workflows.getWorkflowRun("etl", resumed.runId(), false).getErrorMessage());
+ }
+
+ @Test
+ void aRunningWorkflowRunCannotBeResumed() {
+ useRunDuration(60);
+ createWorkflow("etl", null, null);
+ twoStepPipeline("etl");
+ String runId = workflows.startWorkflowRun("etl", null);
+
+ assertEquals("IllegalWorkflowStateException", assertThrows(AwsException.class,
+ () -> workflows.resumeWorkflowRun("etl", runId, List.of("wnode_any"))).getErrorCode());
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void theGraphLinksTriggersToTheJobsTheyWatchAndStart() {
+ createWorkflow("etl", null, null);
+ twoStepPipeline("etl");
+ String runId = workflows.startWorkflowRun("etl", null);
+
+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();
+ List<Map<String, Object>> nodes = (List<Map<String, Object>>) graph.get("Nodes");
+ List<Map<String, Object>> edges = (List<Map<String, Object>>) graph.get("Edges");
+ assertEquals(4, nodes.size());
+ assertEquals(3, edges.size());
+ assertTrue(edges.contains(Map.of("SourceId", nodeId(graph, "TRIGGER", "start"),
+ "DestinationId", nodeId(graph, "JOB", "extract"))));
+ assertTrue(edges.contains(Map.of("SourceId", nodeId(graph, "JOB", "extract"),
+ "DestinationId", nodeId(graph, "TRIGGER", "then-load"))));
+
+ Map<String, Object> runGraph = workflows.getWorkflowRun("etl", runId, true).getGraph();
+ Map<String, Object> loadNode = ((List<Map<String, Object>>) runGraph.get("Nodes")).stream()
+ .filter(node -> "load".equals(node.get("Name"))).findFirst().orElseThrow();
+ assertEquals(1, ((List<JobRun>) ((Map<String, Object>) loadNode.get("JobDetails")).get("JobRuns")).size());
+ assertNull(workflows.getWorkflowRun("etl", runId, false).getGraph());
+ }
+
+ @Test
+ void runPropertiesAreMergedAndLastRunIsReported() {
+ createWorkflow("etl", null, Map.of("env", "dev"));
+ twoStepPipeline("etl");
+ String runId = workflows.startWorkflowRun("etl", null);
+
+ workflows.putWorkflowRunProperties("etl", runId, Map.of("rows", "42", "env", "prod"));
+
+ assertEquals(Map.of("env", "prod", "rows", "42"), workflows.getWorkflowRunProperties("etl", runId));
+ Workflow workflow = workflows.workflowView(glueService.getWorkflow("etl"), false);
+ assertEquals(runId, workflow.getLastRun().getWorkflowRunId());
+ assertNull(workflow.getGraph());
+ assertEquals("EntityNotFoundException", assertThrows(AwsException.class,
+ () -> workflows.getWorkflowRunProperties("etl", "wr_missing")).getErrorCode());
+ }
+
+ @Test
+ void deletingAWorkflowRemovesItsRunsButKeepsItsTriggers() {
+ createWorkflow("etl", null, null);
+ twoStepPipeline("etl");
+ workflows.startWorkflowRun("etl", null);
+
+ workflows.deleteWorkflow("etl", REGION);
+ createWorkflow("etl", null, null);
+
+ assertTrue(workflows.getWorkflowRuns("etl", false, null, null).items().isEmpty());
+ assertEquals("etl", glueService.getTrigger("start").getWorkflowName());
+ }
+
+ @Test
+ void concurrentStartsRespectMaxConcurrentRuns() throws Exception {
+ useRunDuration(60);
+ createWorkflow("etl", 1, null);
+ twoStepPipeline("etl");
+ int threads = 8;
+ CountDownLatch go = new CountDownLatch(1);
+ AtomicInteger started = new AtomicInteger();
+ ExecutorService pool = Executors.newFixedThreadPool(threads);
+ try {
+ List<Future<?>> futures = new ArrayList<>();
+ for (int i = 0; i < threads; i++) {
+ futures.add(pool.submit(() -> {
+ go.await();
+ try {
+ workflows.startWorkflowRun("etl", null);
+ started.incrementAndGet();
+ } catch (AwsException expected) {
+ // ConcurrentRunsExceededException is the outcome under test for all but one start.
+ }
+ return null;
+ }));
+ }
+ go.countDown();
+ for (Future<?> future : futures) {
+ future.get(10, TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+
+ assertEquals(1, started.get());
+ assertEquals(1, workflows.getWorkflowRuns("etl", false, null, null).items().size());
+ }
+
+ @Test
+ void aRunWhoseStartingActionsCannotLaunchEndsInError() {
+ useRunDuration(60);
+ createWorkflow("etl", null, null);
+ onDemand("start", "etl", startCrawler("raw"));
+ crawls.startCrawler("raw");
+
+ String runId = workflows.startWorkflowRun("etl", null);
+
+ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);
+ assertEquals("ERROR", run.getStatus());
+ assertTrue(run.getErrorMessage().contains("could not start raw"), run.getErrorMessage());
+ assertNotNull(run.getCompletedOn());
+ }
+
+ @Test
+ void aLaunchFailureBesideAStartedActionIsReportedOnTheRun() {
+ useRunDuration(60);
+ createWorkflow("etl", null, null);
+ onDemand("start", "etl", startCrawler("raw"), startJob("extract"));
+ crawls.startCrawler("raw");
+
+ String runId = workflows.startWorkflowRun("etl", null);
+
+ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);
+ assertEquals("RUNNING", run.getStatus());
+ assertTrue(run.getErrorMessage().contains("could not start raw"), run.getErrorMessage());
+ }
+
+ @Test
+ void aResumeThatCannotLaunchANodeLeavesNoRunBehind() {
+ useDurations(0, 60);
+ createWorkflow("etl", null, null);
+ onDemand("start", "etl", startJob("extract"), startCrawler("raw"));
+ String first = workflows.startWorkflowRun("etl", null);
+ clock.advance(Duration.ofSeconds(61));
+ assertEquals("COMPLETED", workflows.getWorkflowRun("etl", first, false).getStatus());
+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();
+ crawls.startCrawler("raw");
+
+ assertEquals("CrawlerRunningException", assertThrows(AwsException.class, () -> workflows.resumeWorkflowRun(
+ "etl", first, List.of(nodeId(graph, "JOB", "extract"), nodeId(graph, "CRAWLER", "raw")))).getErrorCode());
+
+ assertEquals(1, workflows.getWorkflowRuns("etl", false, null, null).items().size());
+ assertEquals(1, runsOf("extract").size());
+ }
+
+ @Test
+ void deletingAWorkflowWhileItsCrawlRunsLeavesNoCrawlHistoryBehind() {
+ useDurations(0, 60);
+ createWorkflow("etl", null, null);
+ onDemand("start", "etl", startCrawler("raw"));
+ String runId = workflows.startWorkflowRun("etl", null);
+
+ workflows.deleteWorkflow("etl", REGION);
+ clock.advance(Duration.ofSeconds(61));
+ crawls.startCrawler("raw");
+
+ assertTrue(crawls.crawlsInWorkflowRun(runId).isEmpty());
+ }
+
+ @Test
+ void aResumedJobRunsWithItsOriginalArguments() {
+ createWorkflow("etl", null, null);
+ TriggerAction extract = startJob("extract");
+ extract.setArguments(Map.of("--mode", "full"));
+ onDemand("start", "etl", extract);
+ String first = workflows.startWorkflowRun("etl", null);
+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();
+
+ String resumed = workflows.resumeWorkflowRun("etl", first, List.of(nodeId(graph, "JOB", "extract"))).runId();
+
+ JobRun rerun = runsOf("extract").getFirst();
+ assertEquals("full", rerun.getArguments().get("--mode"));
+ assertEquals(resumed, rerun.getArguments().get("--WORKFLOW_RUN_ID"));
+ assertEquals("start", rerun.getTriggerName());
+ }
+
+ /** A workflow run keeps its crawls however many newer crawls the crawler has run since. */
+ @Test
+ void anOlderRunKeepsItsCrawlsAfterTheCrawlerHistoryMovesOn() {
+ createWorkflow("etl", null, null);
+ onDemand("start", "etl", startCrawler("raw"));
+ String runId = workflows.startWorkflowRun("etl", null);
+ for (int i = 0; i < 110; i++) {
+ crawls.startCrawler("raw");
+ }
+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();
+
+ assertEquals(1, workflows.getWorkflowRun("etl", runId, false).getStatistics().get("TotalActions"));
+ assertNotNull(workflows.resumeWorkflowRun("etl", runId, List.of(nodeId(graph, "CRAWLER", "raw"))).runId());
+ }
+
+ @Test
+ void aRunThatHasFinishedCannotBeStopped() {
+ useRunDuration(60);
+ createWorkflow("etl", null, null);
+ onDemand("start", "etl", startJob("extract"));
+ String runId = workflows.startWorkflowRun("etl", null);
+ clock.advance(Duration.ofSeconds(61));
+
+ assertEquals("IllegalWorkflowStateException", assertThrows(AwsException.class,
+ () -> workflows.stopWorkflowRun("etl", runId)).getErrorCode());
+ assertEquals("COMPLETED", workflows.getWorkflowRun("etl", runId, false).getStatus());
+ }
+
+ @SuppressWarnings("unchecked")
+ private static String nodeId(Map<String, Object> graph, String type, String name) {
+ return ((List<Map<String, Object>>) graph.get("Nodes")).stream()
+ .filter(node -> type.equals(node.get("Type")) && name.equals(node.get("Name")))
+ .map(node -> (String) node.get("UniqueId"))
+ .findFirst().orElseThrow();
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return AccountAwareStorageBackend.inMemory(ACCOUNT_ID);
+ }
+ }
+}