Light, fluffy, and always free - The AWS Local Emulator alternative
feat(glue): create and run workflows (#4471)
Davidbc15516cc0af
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueWorkflowTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueWorkflowTest.javanew file mode 100644index 0000000000..a5e0e94764--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueWorkflowTest.java@@ -0,0 +1,104 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.glue.GlueClient;+import software.amazon.awssdk.services.glue.model.JobRunState;+import software.amazon.awssdk.services.glue.model.LogicalOperator;+import software.amazon.awssdk.services.glue.model.Node;+import software.amazon.awssdk.services.glue.model.NodeType;+import software.amazon.awssdk.services.glue.model.TriggerType;+import software.amazon.awssdk.services.glue.model.Workflow;+import software.amazon.awssdk.services.glue.model.WorkflowRun;+import software.amazon.awssdk.services.glue.model.WorkflowRunStatus;++import java.util.List;+import java.util.Map;++import static org.assertj.core.api.Assertions.assertThat;++@DisplayName("Glue workflows")+class GlueWorkflowTest {++ private static final String WORKFLOW = TestFixtures.uniqueName("workflow");+ private static final String EXTRACT = TestFixtures.uniqueName("workflow_extract");+ private static final String LOAD = TestFixtures.uniqueName("workflow_load");+ private static final String START = TestFixtures.uniqueName("workflow_start");+ private static final String THEN = TestFixtures.uniqueName("workflow_then");+ private static final String ROLE = "arn:aws:iam::000000000000:role/glue";++ private static GlueClient glue;++ @BeforeAll+ static void setup() {+ glue = TestFixtures.glueClient();+ for (String job : List.of(EXTRACT, LOAD)) {+ glue.createJob(r -> r.name(job).role(ROLE).command(c -> c.name("glueetl")));+ }+ glue.createWorkflow(r -> r.name(WORKFLOW).defaultRunProperties(Map.of("env", "dev")));+ glue.createTrigger(r -> r.name(START).workflowName(WORKFLOW).type(TriggerType.ON_DEMAND)+ .actions(a -> a.jobName(EXTRACT)));+ glue.createTrigger(r -> r.name(THEN).workflowName(WORKFLOW).type(TriggerType.CONDITIONAL)+ .startOnCreation(true).actions(a -> a.jobName(LOAD))+ .predicate(p -> p.conditions(c -> c.logicalOperator(LogicalOperator.EQUALS)+ .jobName(EXTRACT).state(JobRunState.SUCCEEDED))));+ }++ @AfterAll+ static void cleanup() {+ if (glue == null) {+ return;+ }+ for (String trigger : List.of(START, THEN)) {+ try {+ glue.deleteTrigger(r -> r.name(trigger));+ }+ catch (Exception ignored) {+ // Cleanup is best effort: the names are unique to this run and the emulator is disposable.+ }+ }+ try {+ glue.deleteWorkflow(r -> r.name(WORKFLOW));+ }+ catch (Exception ignored) {+ // Cleanup is best effort, as above.+ }+ for (String job : List.of(EXTRACT, LOAD)) {+ try {+ glue.deleteJob(r -> r.jobName(job));+ }+ catch (Exception ignored) {+ // Cleanup is best effort, as above.+ }+ }+ glue.close();+ }++ @Test+ @DisplayName("StartWorkflowRun runs the triggers to completion and GetWorkflowRun parses")+ void runToCompletion() {+ String runId = glue.startWorkflowRun(r -> r.name(WORKFLOW).runProperties(Map.of("day", "2026-09-26"))).runId();++ WorkflowRun run = glue.getWorkflowRun(r -> r.name(WORKFLOW).runId(runId).includeGraph(true)).run();+ assertThat(run.status()).isEqualTo(WorkflowRunStatus.COMPLETED);+ assertThat(run.statistics().succeededActions()).isEqualTo(2);+ assertThat(run.workflowRunProperties()).containsEntry("env", "dev").containsEntry("day", "2026-09-26");+ Node load = run.graph().nodes().stream().filter(n -> LOAD.equals(n.name())).findFirst().orElseThrow();+ assertThat(load.jobDetails().jobRuns()).hasSize(1);+ assertThat(load.jobDetails().jobRuns().get(0).jobRunState()).isEqualTo(JobRunState.SUCCEEDED);+ }++ @Test+ @DisplayName("GetWorkflow with IncludeGraph and ListWorkflows parse")+ void readWorkflow() {+ Workflow workflow = glue.getWorkflow(r -> r.name(WORKFLOW).includeGraph(true)).workflow();+ assertThat(workflow.name()).isEqualTo(WORKFLOW);+ assertThat(workflow.createdOn()).isNotNull();+ assertThat(workflow.graph().nodes()).extracting(Node::type)+ .containsExactlyInAnyOrder(NodeType.TRIGGER, NodeType.TRIGGER, NodeType.JOB, NodeType.JOB);+ assertThat(workflow.graph().edges()).hasSize(3);+ assertThat(glue.listWorkflows(r -> r.maxResults(25)).workflows()).contains(WORKFLOW);+ }+}diff --git a/docs/services/glue.md b/docs/services/glue.mdindex b3eea476b0..5b877f7c18 100644--- a/docs/services/glue.md+++ b/docs/services/glue.md@@ -207,8 +207,38 @@ finishes. One request runs at most 25 firing rounds; a longer chain continues onAs an emulator safeguard (AWS has no such limit), triggers start at most 100 runs on behalf of the runthat set off a chain, the one no trigger started, so a loop of triggers of any shape stops there.A run a trigger starts carries its `TriggerName`. `SCHEDULED` triggers are stored and can-be activated, but no timer fires them. `EVENT` triggers and `WorkflowName` need workflows, which are-not emulated yet.+be activated, but no timer fires them. A trigger can belong to a workflow through `WorkflowName`, and an+`EVENT` trigger must; `EVENT` triggers are stored, but no EventBridge event starts them.++#### Workflows++| Action | Description |+|--------|-------------|+| CreateWorkflow | Creates a workflow, with default run properties and an optional `MaxConcurrentRuns`. |+| GetWorkflow | Retrieves a workflow with its `LastRun`, and with `IncludeGraph` its graph of trigger, job and crawler nodes. |+| ListWorkflows | Lists workflow names. |+| BatchGetWorkflows | Retrieves several workflows by name and reports the names not found. |+| UpdateWorkflow | Applies the members the request sets. |+| DeleteWorkflow | Deletes a workflow and its runs; its triggers are kept. Deleting one that does not exist succeeds. |+| StartWorkflowRun | Starts a run: the workflow's starting triggers fire inside it. |+| StopWorkflowRun | Stops a running workflow run and what is running in it. |+| ResumeWorkflowRun | Starts chosen job and crawler nodes of a finished run again in a new run, and what follows them. |+| GetWorkflowRun | Retrieves a run with its `Statistics`, and with `IncludeGraph` the job runs and crawls of each node. |+| GetWorkflowRuns | Lists a workflow's runs, newest first. |+| GetWorkflowRunProperties | Retrieves a run's properties. |+| PutWorkflowRunProperties | Adds properties to a run, replacing any with the same name. |++`StartWorkflowRun` fires the workflow's starting triggers (every trigger of the workflow that is not+`CONDITIONAL`) inside a new run, with the default run properties overlaid by the request's. A+workflow's activated `CONDITIONAL` triggers fire only on job runs and crawls of a running run of that+workflow, evaluate `AND` within that run, and start their actions inside it; job runs in a workflow run+get the `--WORKFLOW_NAME` and `--WORKFLOW_RUN_ID` arguments. A run is `COMPLETED` once nothing is+running in it and nothing more fires. An action that cannot be launched (its job at its concurrency limit,+its crawler already running) is reported in the run's `ErrorMessage`, and a run none of whose starting+actions could launch ends `ERROR`. `ResumeWorkflowRun` starts each chosen job with the arguments it had in+the original run; if a node cannot be launched (its job at its concurrency limit, its crawler running) the+request fails before anything starts.+Blueprints are not emulated.#### Classifiersdiff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.javaindex 1b27493e3b..f2bc32c0c0 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunService.java@@ -49,6 +49,9 @@ public class GlueCrawlerRunService {private static final int MAX_RUNTIME_HISTORY = 100;private final StorageBackend<String, CrawlerRunRecord> recordStore;+ // Workflow run id to the crawls that ran in it. Kept apart from each crawler's recent history, which+ // only holds its latest crawls, so an older workflow run keeps its crawls; cleared with the workflow.+ private final StorageBackend<String, List<FinishedCrawl>> workflowCrawlStore;private final GlueService glueService;private final int runDurationSeconds;private final Clock clock;@@ -56,31 +59,34 @@ public class GlueCrawlerRunService {@Injectpublic GlueCrawlerRunService(StorageFactory storageFactory, GlueService glueService, EmulatorConfig config) {this(storageFactory.create("glue", "crawler_runs.json", new TypeReference<>() {}),+ storageFactory.create("glue", "workflow_run_crawls.json", new TypeReference<>() {}),glueService, config.services().glue().crawlerRunDurationSeconds(), Clock.systemUTC());}- GlueCrawlerRunService(StorageBackend<String, CrawlerRunRecord> recordStore, GlueService glueService,+ GlueCrawlerRunService(StorageBackend<String, CrawlerRunRecord> recordStore,+ StorageBackend<String, List<FinishedCrawl>> workflowCrawlStore, GlueService glueService,int runDurationSeconds, Clock clock) {if (runDurationSeconds < 0) {throw new IllegalArgumentException("floci.services.glue.crawler-run-duration-seconds must not be negative: " + runDurationSeconds);}this.recordStore = recordStore;+ this.workflowCrawlStore = workflowCrawlStore;this.glueService = glueService;this.runDurationSeconds = runDurationSeconds;this.clock = clock;}public synchronized String startCrawler(String name) {- return startCrawler(name, null);+ return startCrawler(name, null, null);}/*** Starts a crawl and returns its crawl id ({@code crawl:} and a UUID, so ids never repeat, even for- * a crawler deleted and created again). A trigger passes the origin of its chain; a null origin- * makes the crawl its own origin.+ * a crawler deleted and created again). A trigger passes the origin of its chain (a null origin+ * makes the crawl its own origin) and, when it belongs to a workflow, the workflow run.*/- public synchronized String startCrawler(String name, String originRunId) {+ public synchronized String startCrawler(String name, String originRunId, String workflowRunId) {glueService.getCrawler(name);CrawlerRunRecord record = settledRecord(name);if (record.getCurrentStart() != null) {@@ -91,6 +97,7 @@ public class GlueCrawlerRunService {record.setCurrentMessagePrefix(UUID.randomUUID().toString());record.setCurrentCrawlId(crawlId);record.setCurrentOriginRunId(originRunId != null ? originRunId : crawlId);+ record.setCurrentWorkflowRunId(workflowRunId);recordStore.put(name, record);LOG.infov("Started Glue crawler {0}", name);settle(record);@@ -175,7 +182,7 @@ public class GlueCrawlerRunService {for (FinishedCrawl crawl : record.getRecentCrawls()) {if (crawl.getSequence() > after) {completions.add(new GlueRunCompletion(Long.toString(crawl.getSequence()), crawl.getFinishedAt(),- crawl.getStatus(), originOf(name, crawl)));+ crawl.getStatus(), originOf(name, crawl), crawl.getWorkflowRunId()));}}return completions;@@ -207,6 +214,48 @@ public class GlueCrawlerRunService {return CRAWL_ID_PREFIX + crawlerName + ":" + crawl.getSequence();}+ /** A crawl of a workflow run as the workflow graph reports it: the {@code Crawl} structure. */+ public record WorkflowCrawl(String crawlerName, String state, Instant startedOn, Instant completedOn) {}++ /** The crawls, settled, that belong to a workflow run: finished ones and a crawl still running. */+ public synchronized List<WorkflowCrawl> crawlsInWorkflowRun(String workflowRunId) {+ List<WorkflowCrawl> running = new ArrayList<>();+ for (CrawlerRunRecord stored : recordStore.scan(key -> true)) {+ CrawlerRunRecord record = settledRecord(stored.getCrawlerName());+ if (record.getCurrentStart() != null && workflowRunId.equals(record.getCurrentWorkflowRunId())) {+ running.add(new WorkflowCrawl(record.getCrawlerName(), STATE_RUNNING, record.getCurrentStart(), null));+ }+ }+ List<WorkflowCrawl> crawls = new ArrayList<>();+ for (FinishedCrawl crawl : workflowCrawlStore.get(workflowRunId).orElse(List.of())) {+ crawls.add(new WorkflowCrawl(crawl.getCrawlerName(), crawl.getStatus(), crawl.getStartedAt(),+ crawl.getFinishedAt()));+ }+ crawls.addAll(running);+ crawls.sort(Comparator.comparing(WorkflowCrawl::startedOn));+ return crawls;+ }++ /**+ * Forgets the crawls of a workflow run that no longer exists, and detaches a crawl still running in+ * it, so that crawl does not record history under the removed run when it finishes.+ */+ public synchronized void forgetWorkflowRun(String workflowRunId) {+ workflowCrawlStore.delete(workflowRunId);+ for (CrawlerRunRecord record : recordStore.scan(key -> true)) {+ if (workflowRunId.equals(record.getCurrentWorkflowRunId())) {+ record.setCurrentWorkflowRunId(null);+ recordStore.put(record.getCrawlerName(), record);+ }+ }+ }++ /** Whether the crawler is crawling now, after settling a crawl that has run its time. */+ public synchronized boolean isRunning(String name) {+ glueService.getCrawler(name);+ return settledRecord(name).getCurrentStart() != null;+ }+public synchronized void updateCrawler(Crawler update) {requireNotRunning(update.getName());glueService.updateCrawler(update);@@ -300,6 +349,14 @@ public class GlueCrawlerRunService {finished.setStatus(status);finished.setCrawlId(record.getCurrentCrawlId());finished.setOriginRunId(record.getCurrentOriginRunId());+ finished.setWorkflowRunId(record.getCurrentWorkflowRunId());+ finished.setCrawlerName(record.getCrawlerName());+ if (finished.getWorkflowRunId() != null) {+ List<FinishedCrawl> ofRun = new ArrayList<>(+ workflowCrawlStore.get(finished.getWorkflowRunId()).orElse(List.of()));+ ofRun.add(finished);+ workflowCrawlStore.put(finished.getWorkflowRunId(), ofRun);+ }List<FinishedCrawl> recent = new ArrayList<>(record.getRecentCrawls());recent.add(finished);if (recent.size() > MAX_RUNTIME_HISTORY) {@@ -314,6 +371,7 @@ public class GlueCrawlerRunService {record.setCurrentMessagePrefix(null);record.setCurrentCrawlId(null);record.setCurrentOriginRunId(null);+ record.setCurrentWorkflowRunId(null);recordStore.put(record.getCrawlerName(), record);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJobRunService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJobRunService.javaindex 1832c09f3f..b301b4e463 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJobRunService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJobRunService.java@@ -86,15 +86,16 @@ public class GlueJobRunService {* unset field falls back to the job definition.*/public synchronized JobRun startJobRun(String jobName, String previousRunId, JobRun overrides) {- return startJobRun(jobName, previousRunId, overrides, null);+ return startJobRun(jobName, previousRunId, overrides, null, null);}/*** As {@link #startJobRun(String, String, JobRun)}, for a run a trigger starts on behalf of- * {@code originRunId}; a null origin makes the run its own origin.+ * {@code originRunId} (a null origin makes the run its own origin), inside {@code workflowRunId}+ * when the trigger belongs to a workflow.*/public synchronized JobRun startJobRun(String jobName, String previousRunId, JobRun overrides,- String originRunId) {+ String originRunId, String workflowRunId) {Job job = glueService.getJob(jobName);// The job's own Timeout is stored as given by CreateJob/UpdateJob, so check what the run inherits.Integer timeout = firstNonNull(overrides.getTimeout(), job.getTimeout());@@ -142,6 +143,7 @@ public class GlueJobRunService {runStore.put(run.getId(), run);JobRunBookkeeping bookkeeping = new JobRunBookkeeping();bookkeeping.setOriginRunId(originRunId != null ? originRunId : run.getId());+ bookkeeping.setWorkflowRunId(workflowRunId);bookkeepingStore.put(run.getId(), bookkeeping);LOG.infov("Started Glue job run {0} for job {1}", run.getId(), jobName);return settle(run);@@ -212,8 +214,9 @@ public class GlueJobRunService {if (isTerminal(run)) {String position = positionOf(run);if (afterPosition == null || afterPosition.isEmpty() || position.compareTo(afterPosition) > 0) {+ JobRunBookkeeping bookkeeping = bookkeepingOf(run.getId());completions.add(new GlueRunCompletion(position, run.getCompletedOn(), run.getJobRunState(),- bookkeepingOf(run.getId()).getOriginRunId()));+ bookkeeping.getOriginRunId(), bookkeeping.getWorkflowRunId()));}}}@@ -231,6 +234,31 @@ public class GlueJobRunService {return bookkeepingStore.get(originRunId).map(JobRunBookkeeping::getLegacyTriggeredRuns).orElse(0);}+ /**+ * Whether a run of the job could start now without {@code ConcurrentRunsExceededException}, as+ * {@link #startJobRun} decides it for a run that does not override JobRunQueuingEnabled.+ */+ public synchronized boolean hasCapacity(String jobName) {+ Job job = glueService.getJob(jobName);+ if (Boolean.TRUE.equals(job.getJobRunQueuingEnabled())) {+ return true;+ }+ long active = runsOf(jobName).stream().filter(run -> !isTerminal(settle(run))).count();+ return active < maxConcurrentRuns(job);+ }++ /** The runs, settled, that belong to a workflow run, oldest first. */+ public synchronized List<JobRun> runsInWorkflowRun(String workflowRunId) {+ List<JobRun> runs = new ArrayList<>();+ for (JobRun run : runStore.scan(key -> true)) {+ if (workflowRunId.equals(bookkeepingOf(run.getId()).getWorkflowRunId())) {+ runs.add(settle(run));+ }+ }+ runs.sort(Comparator.comparing(JobRun::getStartedOn).thenComparing(JobRun::getId));+ return runs;+ }+/** Runs belong to their job: deleting the job removes them, as it does on AWS. */public synchronized void deleteRuns(String jobName) {for (JobRun run : runsOf(jobName)) {diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex 3a940f2577..22b6637cba 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -35,6 +35,8 @@ public class GlueJsonHandler {"StartJobRun", "GetJobRun", "GetJobRuns", "BatchStopJobRun","StartCrawler", "StopCrawler", "GetCrawler", "GetCrawlers", "BatchGetCrawlers", "GetCrawlerMetrics","UpdateCrawler",+ "StartWorkflowRun", "StopWorkflowRun", "ResumeWorkflowRun", "GetWorkflowRun", "GetWorkflowRuns",+ "GetWorkflow", "BatchGetWorkflows","StartTrigger");private final GlueService glueService;@@ -515,10 +517,91 @@ public class GlueJsonHandler {triggerService.stopTrigger(name);yield Response.ok(Map.of("Name", name)).build();}+ case "CreateWorkflow" -> {+ Workflow workflow = new Workflow();+ workflow.setName(request.path("Name").asText(null));+ workflow.setDescription(request.path("Description").asText(null));+ workflow.setDefaultRunProperties(readStringMap(request, "DefaultRunProperties"));+ if (request.hasNonNull("MaxConcurrentRuns")) {+ workflow.setMaxConcurrentRuns(request.get("MaxConcurrentRuns").asInt());+ }+ glueService.createWorkflow(workflow, readStringMap(request, "Tags"), region);+ yield Response.ok(Map.of("Name", workflow.getName())).build();+ }+ case "GetWorkflow" -> Response.ok(Map.of("Workflow", triggerService.workflowView(+ glueService.getWorkflow(request.path("Name").asText(null)),+ request.path("IncludeGraph").asBoolean(false)))).build();+ case "ListWorkflows" -> {+ GlueService.Page<String> page = glueService.listWorkflows(readMaxResults(request), readNextToken(request));+ yield Response.ok(pageResponse("Workflows", page.items(), page.nextToken())).build();+ }+ case "BatchGetWorkflows" -> {+ List<String> names = request.hasNonNull("Names")+ ? mapper.convertValue(request.get("Names"), STRING_LIST)+ : null;+ GlueService.BatchGetWorkflowsResult result = glueService.batchGetWorkflows(names);+ List<Workflow> workflows = new ArrayList<>();+ for (Workflow workflow : result.workflows()) {+ workflows.add(triggerService.workflowView(workflow, request.path("IncludeGraph").asBoolean(false)));+ }+ Map<String, Object> response = new LinkedHashMap<>();+ response.put("Workflows", workflows);+ response.put("MissingWorkflows", result.missingWorkflows());+ yield Response.ok(response).build();+ }+ case "UpdateWorkflow" -> {+ String name = request.path("Name").asText(null);+ glueService.updateWorkflow(name, request.path("Description").asText(null),+ readStringMap(request, "DefaultRunProperties"),+ request.hasNonNull("MaxConcurrentRuns") ? request.get("MaxConcurrentRuns").asInt() : null);+ yield Response.ok(Map.of("Name", name)).build();+ }+ case "DeleteWorkflow" -> {+ String name = request.path("Name").asText(null);+ triggerService.deleteWorkflow(name, region);+ yield Response.ok(Map.of("Name", name)).build();+ }+ case "StartWorkflowRun" -> Response.ok(Map.of("RunId", triggerService.startWorkflowRun(+ request.path("Name").asText(null), readStringMap(request, "RunProperties")))).build();+ case "StopWorkflowRun" -> {+ triggerService.stopWorkflowRun(request.path("Name").asText(null), request.path("RunId").asText(null));+ yield Response.ok(Map.of()).build();+ }+ case "ResumeWorkflowRun" -> {+ List<String> nodeIds = request.hasNonNull("NodeIds")+ ? mapper.convertValue(request.get("NodeIds"), STRING_LIST)+ : null;+ GlueTriggerService.ResumedRun resumed = triggerService.resumeWorkflowRun(+ request.path("Name").asText(null), request.path("RunId").asText(null), nodeIds);+ Map<String, Object> response = new LinkedHashMap<>();+ response.put("RunId", resumed.runId());+ response.put("NodeIds", resumed.nodeIds());+ yield Response.ok(response).build();+ }+ case "GetWorkflowRun" -> Response.ok(Map.of("Run", triggerService.getWorkflowRun(+ request.path("Name").asText(null), request.path("RunId").asText(null),+ request.path("IncludeGraph").asBoolean(false)))).build();+ case "GetWorkflowRuns" -> {+ GlueService.Page<WorkflowRun> page = triggerService.getWorkflowRuns(request.path("Name").asText(null),+ request.path("IncludeGraph").asBoolean(false), readMaxResults(request), readNextToken(request));+ yield Response.ok(pageResponse("Runs", page.items(), page.nextToken())).build();+ }+ case "GetWorkflowRunProperties" -> Response.ok(Map.of("RunProperties",+ triggerService.getWorkflowRunProperties(request.path("Name").asText(null),+ request.path("RunId").asText(null)))).build();+ case "PutWorkflowRunProperties" -> {+ triggerService.putWorkflowRunProperties(request.path("Name").asText(null),+ request.path("RunId").asText(null), readStringMap(request, "RunProperties"));+ yield Response.ok(Map.of()).build();+ }default -> throw new AwsException("InvalidAction", "Action " + action + " is not supported", 400);};}+ private Map<String, String> readStringMap(JsonNode request, String field) {+ return request.hasNonNull(field) ? mapper.convertValue(request.get(field), STRING_MAP) : null;+ }+/** The definition members CreateTrigger and TriggerUpdate share. */private Trigger toTrigger(JsonNode node) throws Exception {Trigger trigger = new Trigger();diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueRunCompletion.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueRunCompletion.javaindex 137353de6b..ffaa34ac73 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueRunCompletion.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueRunCompletion.java@@ -6,6 +6,8 @@ import java.time.Instant;* One finished job run or crawl, as a conditional trigger sees it. {@code position} orders the* completions of one job or crawler; a trigger remembers the last position it has processed.* {@code originRunId} is the run that set off the trigger chain this run belongs to (the run itself- * when no trigger started it): a job run id, or a crawl id beginning {@code crawl:}.+ * when no trigger started it): a job run id, or a crawl id beginning {@code crawl:}. {@code workflowRunId}+ * is the workflow run the run or crawl belongs to, or null.*/-public record GlueRunCompletion(String position, Instant finishedAt, String state, String originRunId) {}+public record GlueRunCompletion(String position, Instant finishedAt, String state, String originRunId,+ String workflowRunId) {}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 72ca799c75..ce4134ca55 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -36,6 +36,7 @@ import io.github.hectorvent.floci.services.glue.model.Trigger;import io.github.hectorvent.floci.services.glue.model.TriggerAction;import io.github.hectorvent.floci.services.glue.model.TriggerCondition;import io.github.hectorvent.floci.services.glue.model.UserDefinedFunction;+import io.github.hectorvent.floci.services.glue.model.Workflow;import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;import io.github.hectorvent.floci.services.glue.schemaregistry.SchemaToColumnsConverter;import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;@@ -163,6 +164,8 @@ public class GlueService {private static final Set<String> CONDITION_JOB_STATES = Set.of("SUCCEEDED", "STOPPED", "FAILED", "TIMEOUT");private static final Set<String> CONDITION_CRAWL_STATES = Set.of("SUCCEEDED", "CANCELLED", "FAILED");private static final int MAX_TRIGGERS_PAGE_SIZE = 200;+ private static final int MAX_WORKFLOWS_PAGE_SIZE = 25;+ private static final int MAX_BATCH_GET_WORKFLOWS = 25;// BatchGetCrawlers takes a CrawlerNameList, which the API model caps at 100 names.private static final int MAX_BATCH_GET_CRAWLERS = 100;@@ -182,6 +185,7 @@ public class GlueService {private final StorageBackend<String, DataCatalogEncryptionSettings> encryptionSettingsStore;private final StorageBackend<String, SecurityConfiguration> securityConfigurationStore;private final StorageBackend<String, Trigger> triggerStore;+ private final StorageBackend<String, Workflow> workflowStore;private final GlueSchemaRegistryService schemaRegistryService;private final RegionResolver regionResolver;private final ResourceGroupsTaggingService resourceGroupsTaggingService;@@ -213,6 +217,7 @@ public class GlueService {this.securityConfigurationStore = storageFactory.create("glue", "security_configurations.json", new TypeReference<>() {});this.triggerStore = storageFactory.create("glue", "triggers.json", new TypeReference<>() {});+ this.workflowStore = storageFactory.create("glue", "workflows.json", new TypeReference<>() {});this.schemaRegistryService = schemaRegistryService;this.regionResolver = regionResolver;this.resourceGroupsTaggingService = resourceGroupsTaggingService;@@ -235,6 +240,7 @@ public class GlueService {StorageBackend<String, DataCatalogEncryptionSettings> encryptionSettingsStore,StorageBackend<String, SecurityConfiguration> securityConfigurationStore,StorageBackend<String, Trigger> triggerStore,+ StorageBackend<String, Workflow> workflowStore,GlueSchemaRegistryService schemaRegistryService,RegionResolver regionResolver,ResourceGroupsTaggingService resourceGroupsTaggingService,@@ -255,6 +261,7 @@ public class GlueService {this.encryptionSettingsStore = encryptionSettingsStore;this.securityConfigurationStore = securityConfigurationStore;this.triggerStore = triggerStore;+ this.workflowStore = workflowStore;this.schemaRegistryService = schemaRegistryService;this.regionResolver = regionResolver;this.resourceGroupsTaggingService = resourceGroupsTaggingService;@@ -1519,6 +1526,10 @@ public class GlueService {return regionResolver.buildArn("glue", region, "trigger/" + triggerName);}+ private String workflowArn(String region, String workflowName) {+ return regionResolver.buildArn("glue", region, "workflow/" + workflowName);+ }+private static Pattern compileFunctionPattern(String pattern) {if (pattern == null) {return Pattern.compile(".*");@@ -2166,6 +2177,108 @@ public class GlueService {crawlerStore.put(name, crawler);}+ // ---- Workflows --------------------------------------------------------------------------++ public void createWorkflow(Workflow workflow, Map<String, String> tags, String region) {+ validateRequired(workflow.getName(), "Name");+ validateWorkflowConcurrency(workflow.getMaxConcurrentRuns());+ String name = workflow.getName();+ if (workflowStore.get(name).isPresent()) {+ throw new AwsException("AlreadyExistsException", "Workflow " + name + " already exists.", 400);+ }+ Instant now = Instant.now();+ workflow.setCreatedOn(now);+ workflow.setLastModifiedOn(now);+ workflowStore.put(name, workflow);+ if (tags != null && !tags.isEmpty()) {+ resourceGroupsTaggingService.tagResources(List.of(workflowArn(region, name)), tags, region);+ }+ LOG.infov("Created Glue workflow {0}", name);+ }++ public Workflow getWorkflow(String name) {+ validateRequired(name, "Name");+ return workflowStore.get(name)+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Workflow " + name + " not found.", 400));+ }++ public Page<String> listWorkflows(Integer maxResults, String nextToken) {+ if (maxResults != null && (maxResults < 1 || maxResults > MAX_WORKFLOWS_PAGE_SIZE)) {+ throw new AwsException("InvalidInputException",+ "MaxResults must be between 1 and " + MAX_WORKFLOWS_PAGE_SIZE, 400);+ }+ List<String> names = new ArrayList<>();+ for (Workflow workflow : workflowStore.scan(k -> true)) {+ names.add(workflow.getName());+ }+ names.sort(Comparator.naturalOrder());+ return paginate(names, maxResults, nextToken);+ }++ public BatchGetWorkflowsResult batchGetWorkflows(List<String> names) {+ if (names == null || names.isEmpty() || names.size() > MAX_BATCH_GET_WORKFLOWS) {+ throw new AwsException("InvalidInputException",+ "Names must contain between 1 and " + MAX_BATCH_GET_WORKFLOWS + " names.", 400);+ }+ List<Workflow> workflows = new ArrayList<>();+ List<String> missing = new ArrayList<>();+ for (String name : names) {+ Optional<Workflow> workflow = name == null ? Optional.empty() : workflowStore.get(name);+ if (workflow.isPresent()) {+ workflows.add(workflow.get());+ } else {+ missing.add(name);+ }+ }+ return new BatchGetWorkflowsResult(workflows, missing);+ }++ public record BatchGetWorkflowsResult(List<Workflow> workflows, List<String> missingWorkflows) {}++ /** Applies the members an UpdateWorkflow request sets; the rest of the workflow is kept. */+ public void updateWorkflow(String name, String description, Map<String, String> defaultRunProperties,+ Integer maxConcurrentRuns) {+ Workflow workflow = getWorkflow(name);+ validateWorkflowConcurrency(maxConcurrentRuns);+ if (description != null) {+ workflow.setDescription(description);+ }+ if (defaultRunProperties != null) {+ workflow.setDefaultRunProperties(defaultRunProperties);+ }+ if (maxConcurrentRuns != null) {+ workflow.setMaxConcurrentRuns(maxConcurrentRuns);+ }+ workflow.setLastModifiedOn(Instant.now());+ workflowStore.put(name, workflow);+ }++ /**+ * DeleteWorkflow declares no EntityNotFoundException: deleting a missing workflow succeeds. The+ * workflow's triggers are left in place, as the API does not say they are removed.+ */+ public void deleteWorkflow(String name, String region) {+ validateRequired(name, "Name");+ workflowStore.delete(name);+ resourceGroupsTaggingService.deleteResources(List.of(workflowArn(region, name)), region);+ }++ public List<Trigger> triggersOfWorkflow(String workflowName) {+ List<Trigger> triggers = new ArrayList<>();+ for (Trigger trigger : allTriggers()) {+ if (workflowName.equals(trigger.getWorkflowName())) {+ triggers.add(trigger);+ }+ }+ return triggers;+ }++ private static void validateWorkflowConcurrency(Integer maxConcurrentRuns) {+ if (maxConcurrentRuns != null && maxConcurrentRuns < 1) {+ throw new AwsException("InvalidInputException", "MaxConcurrentRuns must be at least 1.", 400);+ }+ }+// ---- Triggers ---------------------------------------------------------------------------public void createTrigger(Trigger trigger, Map<String, String> tags, String region) {@@ -2175,10 +2288,8 @@ public class GlueService {throw new AwsException("InvalidInputException", "Invalid trigger type: " + trigger.getType(), 400);}if (trigger.getWorkflowName() != null) {- throw new AwsException("EntityNotFoundException",- "Workflow " + trigger.getWorkflowName() + " not found.", 400);- }- if ("EVENT".equals(trigger.getType())) {+ getWorkflow(trigger.getWorkflowName());+ } else if ("EVENT".equals(trigger.getType())) {throw new AwsException("InvalidInputException", "An EVENT trigger must belong to a workflow.", 400);}validateTriggerDefinition(trigger);@@ -2830,6 +2941,9 @@ public class GlueService {} else if (resource.startsWith("trigger/")) {getTrigger(resource.substring(8));return;+ } else if (resource.startsWith("workflow/")) {+ getWorkflow(resource.substring(9));+ return;} else if (resource.startsWith("connection/")) {getConnection(resource.substring(11), false);return;diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueTriggerService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueTriggerService.javaindex b045e23608..227bfdd32c 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueTriggerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueTriggerService.java@@ -10,18 +10,24 @@ import io.github.hectorvent.floci.services.glue.model.Trigger;import io.github.hectorvent.floci.services.glue.model.TriggerAction;import io.github.hectorvent.floci.services.glue.model.TriggerChainBudget;import io.github.hectorvent.floci.services.glue.model.TriggerCondition;+import io.github.hectorvent.floci.services.glue.model.Workflow;+import io.github.hectorvent.floci.services.glue.model.WorkflowRun;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.nio.charset.StandardCharsets;+import java.time.Clock;import java.time.Instant;import java.util.ArrayList;import java.util.Comparator;import java.util.HashMap;+import java.util.LinkedHashMap;import java.util.LinkedHashSet;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.UUID;/*** Glue trigger lifecycle and firing. An ON_DEMAND trigger fires when started. A CONDITIONAL trigger,@@ -31,6 +37,12 @@ import java.util.Set;* trigger has not seen yet, in finishing order. A SCHEDULED trigger is stored and can be activated,* but no timer fires it.*+ * <p>This service also runs workflows. StartWorkflowRun fires the workflow's starting (non+ * CONDITIONAL) triggers inside a new workflow run; a workflow's CONDITIONAL triggers fire only on+ * completions of runs and crawls that belong to a RUNNING run of that workflow, evaluate AND within+ * that run, and start their actions inside it. A workflow run is COMPLETED once an evaluation ends+ * with nothing left running in it and nothing more to fire.+ ** <p>Every public method is synchronized. This service calls the job run and crawler run services* and they never call back, so the lock order is always this service first.*/@@ -44,6 +56,11 @@ public class GlueTriggerService {static final String STATE_CREATED = "CREATED";static final String STATE_ACTIVATED = "ACTIVATED";static final String STATE_DEACTIVATED = "DEACTIVATED";+ static final String STATUS_RUNNING = "RUNNING";+ static final String STATUS_COMPLETED = "COMPLETED";+ static final String STATUS_STOPPED = "STOPPED";+ static final String STATUS_ERROR = "ERROR";+ private static final String STATE_JOB_RUNNING = "RUNNING";// Work done within one request; a longer chain continues on the next run-related request.private static final int MAX_FIRING_ROUNDS = 25;@@ -65,6 +82,8 @@ public class GlueTriggerService {private final StorageBackend<String, Map<String, String>> seenStore;private final StorageBackend<String, TriggerChainBudget> budgetStore;private final int maxTrackedChains;+ private final Clock clock;+ private final StorageBackend<String, WorkflowRun> workflowRunStore;private final GlueService glueService;private final GlueJobRunService jobRunService;private final GlueCrawlerRunService crawlerRunService;@@ -77,22 +96,28 @@ public class GlueTriggerService {// than rewritten in full on each claim.storageFactory.create("glue", "trigger_chain_budgets.json", new TypeReference<>() {},WriteProfile.APPEND_HEAVY),+ storageFactory.create("glue", "workflow_runs.json", new TypeReference<>() {}),glueService, jobRunService, crawlerRunService);}GlueTriggerService(StorageBackend<String, Map<String, String>> seenStore,- StorageBackend<String, TriggerChainBudget> budgetStore, GlueService glueService,+ StorageBackend<String, TriggerChainBudget> budgetStore,+ StorageBackend<String, WorkflowRun> workflowRunStore, GlueService glueService,GlueJobRunService jobRunService, GlueCrawlerRunService crawlerRunService) {- this(seenStore, budgetStore, glueService, jobRunService, crawlerRunService, MAX_TRACKED_CHAINS);+ this(seenStore, budgetStore, workflowRunStore, glueService, jobRunService, crawlerRunService,+ MAX_TRACKED_CHAINS, Clock.systemUTC());}GlueTriggerService(StorageBackend<String, Map<String, String>> seenStore,- StorageBackend<String, TriggerChainBudget> budgetStore, GlueService glueService,+ StorageBackend<String, TriggerChainBudget> budgetStore,+ StorageBackend<String, WorkflowRun> workflowRunStore, GlueService glueService,GlueJobRunService jobRunService, GlueCrawlerRunService crawlerRunService,- int maxTrackedChains) {+ int maxTrackedChains, Clock clock) {this.seenStore = seenStore;this.budgetStore = budgetStore;this.maxTrackedChains = maxTrackedChains;+ this.clock = clock;+ this.workflowRunStore = workflowRunStore;this.glueService = glueService;this.jobRunService = jobRunService;this.crawlerRunService = crawlerRunService;@@ -130,7 +155,7 @@ public class GlueTriggerService {public synchronized void startTrigger(String name) {Trigger trigger = glueService.getTrigger(name);if (TYPE_ON_DEMAND.equals(trigger.getType())) {- fire(trigger, null, true);+ fire(trigger, null, null, true);return;}boolean wasActive = STATE_ACTIVATED.equals(trigger.getState());@@ -160,27 +185,505 @@ public class GlueTriggerService {public synchronized void fireConditionalTriggers() {for (int round = 0; round < MAX_FIRING_ROUNDS; round++) {boolean anyFired = false;+ Set<String> runningWorkflowRuns = runningWorkflowRunIds();for (Trigger trigger : glueService.allTriggers()) {if (!isActiveConditional(trigger)) {continue;}- for (GlueRunCompletion cause : newFirings(trigger)) {- anyFired |= fire(trigger, cause.originRunId(), false) > 0;+ for (GlueRunCompletion cause : newFirings(trigger, runningWorkflowRuns)) {+ anyFired |= fire(trigger, cause.originRunId(),+ trigger.getWorkflowName() != null ? cause.workflowRunId() : null, false) > 0;}}if (!anyFired) {+ // Nothing more fires, so a workflow run with nothing left running is finished. Not+ // checked when the round limit cut the evaluation short: more may still fire then.+ completeFinishedWorkflowRuns();return;}}}+ // ---- Workflow runs ----------------------------------------------------------------------++ /** Starts a run of the workflow: its starting (non CONDITIONAL) triggers fire inside the new run. */+ public synchronized String startWorkflowRun(String workflowName, Map<String, String> runProperties) {+ Workflow workflow = glueService.getWorkflow(workflowName);+ requireCapacity(workflow);+ Map<String, String> properties = new LinkedHashMap<>();+ if (workflow.getDefaultRunProperties() != null) {+ properties.putAll(workflow.getDefaultRunProperties());+ }+ if (runProperties != null) {+ properties.putAll(runProperties);+ }+ WorkflowRun run = newWorkflowRun(workflowName, null, properties);+ int actions = 0;+ int started = 0;+ for (Trigger trigger : glueService.triggersOfWorkflow(workflowName)) {+ if (!TYPE_CONDITIONAL.equals(trigger.getType())) {+ actions += trigger.getActions().size();+ started += fire(trigger, null, run.getWorkflowRunId(), false);+ }+ }+ if (actions > 0 && started == 0) {+ // Nothing the workflow starts with could be launched; the reasons are in ErrorMessage.+ finishWorkflowRun(workflowRunStore.get(run.getWorkflowRunId()).orElse(run), STATUS_ERROR, clock.instant());+ return run.getWorkflowRunId();+ }+ fireConditionalTriggers();+ return run.getWorkflowRunId();+ }++ public synchronized void stopWorkflowRun(String workflowName, String runId) {+ findWorkflowRun(workflowName, runId);+ // A run whose last action finished since the previous evaluation is COMPLETED, not stoppable.+ fireConditionalTriggers();+ WorkflowRun run = findWorkflowRun(workflowName, runId);+ if (!STATUS_RUNNING.equals(run.getStatus())) {+ throw new AwsException("IllegalWorkflowStateException",+ "Workflow run " + runId + " is not running; its status is " + run.getStatus(), 400);+ }+ stopWhatIsRunning(runId);+ finishWorkflowRun(run, STATUS_STOPPED, clock.instant());+ }++ /**+ * Starts the given job and crawler nodes of a finished run again in a new run of the workflow, whose+ * conditional triggers then carry on downstream. Each node must have run in the original run.+ */+ public synchronized ResumedRun resumeWorkflowRun(String workflowName, String runId, List<String> requestedNodeIds) {+ WorkflowRun original = findWorkflowRun(workflowName, runId);+ if (STATUS_RUNNING.equals(original.getStatus())) {+ throw new AwsException("IllegalWorkflowStateException",+ "Workflow run " + runId + " is still running.", 400);+ }+ if (requestedNodeIds == null || requestedNodeIds.isEmpty()) {+ throw new AwsException("InvalidInputException", "NodeIds is required.", 400);+ }+ // A node named twice is restarted once: launching it a second time would collide with the first.+ List<String> nodeIds = new ArrayList<>(new LinkedHashSet<>(requestedNodeIds));+ Map<String, Map<String, Object>> nodes = new HashMap<>();+ for (Map<String, Object> node : graphNodes(workflowName)) {+ nodes.put((String) node.get("UniqueId"), node);+ }+ Set<String> ranJobs = new LinkedHashSet<>();+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(runId)) {+ ranJobs.add(jobRun.getJobName());+ }+ Set<String> ranCrawlers = new LinkedHashSet<>();+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawlerRunService.crawlsInWorkflowRun(runId)) {+ ranCrawlers.add(crawl.crawlerName());+ }+ for (String nodeId : nodeIds) {+ Map<String, Object> node = nodes.get(nodeId);+ boolean ran = node != null && ("JOB".equals(node.get("Type")) ? ranJobs.contains((String) node.get("Name"))+ : "CRAWLER".equals(node.get("Type")) && ranCrawlers.contains((String) node.get("Name")));+ if (!ran) {+ throw new AwsException("InvalidInputException",+ "Node " + nodeId + " is not a job or crawler node that ran in workflow run " + runId, 400);+ }+ }+ requireCapacity(glueService.getWorkflow(workflowName));+ // Refuse before anything starts when a node cannot launch, so a failed resume normally leaves+ // no run and no action behind.+ for (String nodeId : nodeIds) {+ Map<String, Object> node = nodes.get(nodeId);+ String name = (String) node.get("Name");+ if ("JOB".equals(node.get("Type")) && !jobRunService.hasCapacity(name)) {+ throw new AwsException("ConcurrentRunsExceededException", "Concurrent runs exceeded for " + name, 400);+ }+ if ("CRAWLER".equals(node.get("Type")) && crawlerRunService.isRunning(name)) {+ throw new AwsException("CrawlerRunningException", "Crawler with name " + name + " has already started", 400);+ }+ }+ Map<String, JobRun> originalJobRuns = new HashMap<>();+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(runId)) {+ originalJobRuns.put(jobRun.getJobName(), jobRun);+ }+ WorkflowRun resumed = newWorkflowRun(workflowName, runId, original.getWorkflowRunProperties());+ String resumedId = resumed.getWorkflowRunId();+ try {+ for (String nodeId : nodeIds) {+ Map<String, Object> node = nodes.get(nodeId);+ String name = (String) node.get("Name");+ if ("JOB".equals(node.get("Type"))) {+ // The job runs again with the arguments it had in the original run, now in the new run.+ JobRun before = originalJobRuns.get(name);+ JobRun overrides = new JobRun();+ overrides.setArguments(withWorkflowArguments(before != null ? before.getArguments() : null, resumedId));+ if (before != null) {+ overrides.setTimeout(before.getTimeout());+ overrides.setSecurityConfiguration(before.getSecurityConfiguration());+ overrides.setNotificationProperty(before.getNotificationProperty());+ overrides.setTriggerName(before.getTriggerName());+ }+ jobRunService.startJobRun(name, null, overrides, null, resumedId);+ } else {+ crawlerRunService.startCrawler(name, null, resumedId);+ }+ }+ } catch (AwsException e) {+ // Only reachable if a node became unable to launch after the check above. What already+ // started keeps its run, so the run is kept too, stopped and marked ERROR with the reason.+ noteLaunchFailure(resumedId, "Resume could not start every node: " + e.getMessage());+ stopWhatIsRunning(resumedId);+ finishWorkflowRun(workflowRunStore.get(resumedId).orElse(resumed), STATUS_ERROR, clock.instant());+ throw e;+ }+ fireConditionalTriggers();+ return new ResumedRun(resumedId, nodeIds);+ }++ public synchronized WorkflowRun getWorkflowRun(String workflowName, String runId, boolean includeGraph) {+ fireConditionalTriggers();+ return view(findWorkflowRun(workflowName, runId), includeGraph);+ }++ public synchronized GlueService.Page<WorkflowRun> getWorkflowRuns(String workflowName, boolean includeGraph,+ Integer maxResults, String nextToken) {+ glueService.getWorkflow(workflowName);+ fireConditionalTriggers();+ List<WorkflowRun> runs = new ArrayList<>();+ for (WorkflowRun run : runsOfWorkflow(workflowName)) {+ runs.add(view(run, includeGraph));+ }+ runs.sort(Comparator.comparing(WorkflowRun::getStartedOn).reversed()+ .thenComparing(WorkflowRun::getWorkflowRunId));+ return glueService.paginate(runs, maxResults, nextToken);+ }++ public synchronized Map<String, String> getWorkflowRunProperties(String workflowName, String runId) {+ WorkflowRun run = findWorkflowRun(workflowName, runId);+ return run.getWorkflowRunProperties() != null ? run.getWorkflowRunProperties() : Map.of();+ }++ /** Adds the properties to the run's, replacing any with the same name. */+ public synchronized void putWorkflowRunProperties(String workflowName, String runId, Map<String, String> properties) {+ if (properties == null) {+ throw new AwsException("InvalidInputException", "RunProperties is required.", 400);+ }+ WorkflowRun run = findWorkflowRun(workflowName, runId);+ Map<String, String> merged = new LinkedHashMap<>();+ if (run.getWorkflowRunProperties() != null) {+ merged.putAll(run.getWorkflowRunProperties());+ }+ merged.putAll(properties);+ run.setWorkflowRunProperties(merged);+ workflowRunStore.put(run.getWorkflowRunId(), run);+ }++ /** The workflow as GetWorkflow and BatchGetWorkflows report it: with LastRun, and the graph if asked. */+ public synchronized Workflow workflowView(Workflow stored, boolean includeGraph) {+ fireConditionalTriggers();+ Workflow workflow = new Workflow();+ workflow.setName(stored.getName());+ workflow.setDescription(stored.getDescription());+ workflow.setDefaultRunProperties(stored.getDefaultRunProperties());+ workflow.setCreatedOn(stored.getCreatedOn());+ workflow.setLastModifiedOn(stored.getLastModifiedOn());+ workflow.setMaxConcurrentRuns(stored.getMaxConcurrentRuns());+ runsOfWorkflow(stored.getName()).stream()+ .max(Comparator.comparing(WorkflowRun::getStartedOn))+ .ifPresent(last -> workflow.setLastRun(view(last, false)));+ if (includeGraph) {+ workflow.setGraph(graph(stored.getName(), null));+ }+ return workflow;+ }++ /** Deletes the workflow and its runs; its triggers stay, as they do not belong to the workflow's storage. */+ public synchronized void deleteWorkflow(String workflowName, String region) {+ for (WorkflowRun run : runsOfWorkflow(workflowName)) {+ workflowRunStore.delete(run.getWorkflowRunId());+ forgetWorkflowRunState(run.getWorkflowRunId());+ crawlerRunService.forgetWorkflowRun(run.getWorkflowRunId());+ }+ glueService.deleteWorkflow(workflowName, region);+ }++ private void stopWhatIsRunning(String workflowRunId) {+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(workflowRunId)) {+ if (STATE_JOB_RUNNING.equals(jobRun.getJobRunState())) {+ jobRunService.batchStopJobRun(jobRun.getJobName(), List.of(jobRun.getId()));+ }+ }+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawlerRunService.crawlsInWorkflowRun(workflowRunId)) {+ if (STATE_JOB_RUNNING.equals(crawl.state())) {+ crawlerRunService.stopCrawler(crawl.crawlerName());+ }+ }+ }++ private void requireCapacity(Workflow workflow) {+ // Settles runs, fires what is due and completes finished workflow runs before counting.+ fireConditionalTriggers();+ if (workflow.getMaxConcurrentRuns() == null) {+ return;+ }+ long running = runsOfWorkflow(workflow.getName()).stream()+ .filter(run -> STATUS_RUNNING.equals(run.getStatus())).count();+ if (running >= workflow.getMaxConcurrentRuns()) {+ throw new AwsException("ConcurrentRunsExceededException",+ "Concurrent runs exceeded for workflow " + workflow.getName(), 400);+ }+ }++ private WorkflowRun newWorkflowRun(String workflowName, String previousRunId, Map<String, String> properties) {+ WorkflowRun run = new WorkflowRun();+ run.setName(workflowName);+ run.setWorkflowRunId("wr_" + UUID.randomUUID().toString().replace("-", "")+ + UUID.randomUUID().toString().replace("-", ""));+ run.setPreviousRunId(previousRunId);+ run.setWorkflowRunProperties(properties != null ? new LinkedHashMap<>(properties) : new LinkedHashMap<>());+ run.setStartedOn(clock.instant());+ run.setStatus(STATUS_RUNNING);+ workflowRunStore.put(run.getWorkflowRunId(), run);+ LOG.infov("Started Glue workflow run {0} of {1}", run.getWorkflowRunId(), workflowName);+ return run;+ }++ private WorkflowRun findWorkflowRun(String workflowName, String runId) {+ glueService.getWorkflow(workflowName);+ if (runId == null) {+ throw new AwsException("InvalidInputException", "RunId is required.", 400);+ }+ return workflowRunStore.get(runId)+ .filter(run -> workflowName.equals(run.getName()))+ .orElseThrow(() -> new AwsException("EntityNotFoundException",+ "Workflow run " + runId + " not found for workflow " + workflowName, 400));+ }++ private List<WorkflowRun> runsOfWorkflow(String workflowName) {+ List<WorkflowRun> runs = new ArrayList<>();+ for (WorkflowRun run : workflowRunStore.scan(key -> true)) {+ if (workflowName.equals(run.getName())) {+ runs.add(run);+ }+ }+ return runs;+ }++ private Set<String> runningWorkflowRunIds() {+ Set<String> running = new LinkedHashSet<>();+ for (WorkflowRun run : workflowRunStore.scan(key -> true)) {+ if (STATUS_RUNNING.equals(run.getStatus())) {+ running.add(run.getWorkflowRunId());+ }+ }+ return running;+ }++ private String workflowNameOf(String workflowRunId) {+ return workflowRunStore.get(workflowRunId).map(WorkflowRun::getName).orElse(null);+ }++ private void completeFinishedWorkflowRuns() {+ for (WorkflowRun run : workflowRunStore.scan(key -> true)) {+ if (!STATUS_RUNNING.equals(run.getStatus())) {+ continue;+ }+ Instant lastFinished = run.getStartedOn();+ boolean anyRunning = false;+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(run.getWorkflowRunId())) {+ if (jobRun.getCompletedOn() == null) {+ anyRunning = true;+ } else if (jobRun.getCompletedOn().isAfter(lastFinished)) {+ lastFinished = jobRun.getCompletedOn();+ }+ }+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawlerRunService.crawlsInWorkflowRun(run.getWorkflowRunId())) {+ if (crawl.completedOn() == null) {+ anyRunning = true;+ } else if (crawl.completedOn().isAfter(lastFinished)) {+ lastFinished = crawl.completedOn();+ }+ }+ if (!anyRunning) {+ finishWorkflowRun(run, STATUS_COMPLETED, lastFinished);+ }+ }+ }++ private void finishWorkflowRun(WorkflowRun run, String status, Instant completedOn) {+ run.setStatus(status);+ run.setCompletedOn(completedOn);+ workflowRunStore.put(run.getWorkflowRunId(), run);+ forgetWorkflowRunState(run.getWorkflowRunId());+ LOG.infov("Glue workflow run {0} is {1}", run.getWorkflowRunId(), status);+ }++ /** A finished workflow run's triggers no longer need the outcomes they kept for it. */+ private void forgetWorkflowRunState(String workflowRunId) {+ String prefix = STATE + workflowRunId + "|";+ for (String triggerName : seenStore.keys()) {+ Map<String, String> seen = seenStore.get(triggerName).orElse(Map.of());+ if (seen.keySet().stream().anyMatch(key -> key.startsWith(prefix))) {+ Map<String, String> kept = new HashMap<>(seen);+ kept.keySet().removeIf(key -> key.startsWith(prefix));+ seenStore.put(triggerName, kept);+ }+ }+ }++ private WorkflowRun view(WorkflowRun stored, boolean includeGraph) {+ WorkflowRun run = new WorkflowRun();+ run.setName(stored.getName());+ run.setWorkflowRunId(stored.getWorkflowRunId());+ run.setPreviousRunId(stored.getPreviousRunId());+ run.setWorkflowRunProperties(stored.getWorkflowRunProperties());+ run.setStartedOn(stored.getStartedOn());+ run.setCompletedOn(stored.getCompletedOn());+ run.setStatus(stored.getStatus());+ run.setErrorMessage(stored.getErrorMessage());+ run.setStatistics(statistics(stored.getWorkflowRunId()));+ if (includeGraph) {+ run.setGraph(graph(stored.getName(), stored.getWorkflowRunId()));+ }+ return run;+ }++ private Map<String, Integer> statistics(String workflowRunId) {+ Map<String, Integer> counts = new LinkedHashMap<>();+ for (String key : List.of("TotalActions", "TimeoutActions", "FailedActions", "StoppedActions",+ "SucceededActions", "RunningActions", "ErroredActions", "WaitingActions")) {+ counts.put(key, 0);+ }+ List<String> states = new ArrayList<>();+ for (JobRun jobRun : jobRunService.runsInWorkflowRun(workflowRunId)) {+ states.add(jobRun.getJobRunState());+ }+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawlerRunService.crawlsInWorkflowRun(workflowRunId)) {+ states.add("CANCELLED".equals(crawl.state()) ? "STOPPED" : crawl.state());+ }+ for (String state : states) {+ counts.merge("TotalActions", 1, Integer::sum);+ String bucket = switch (state) {+ case "SUCCEEDED" -> "SucceededActions";+ case "FAILED" -> "FailedActions";+ case "STOPPED", "STOPPING" -> "StoppedActions";+ case "TIMEOUT" -> "TimeoutActions";+ case "ERROR" -> "ErroredActions";+ case "WAITING" -> "WaitingActions";+ default -> "RunningActions";+ };+ counts.merge(bucket, 1, Integer::sum);+ }+ return counts;+ }++ /**+ * The workflow graph: a TRIGGER node per trigger of the workflow, a JOB or CRAWLER node per job or+ * crawler its triggers start or watch, an edge from each watched node to the trigger and from the+ * trigger to each node it starts. With a workflow run, job nodes list the run's job runs and crawler+ * nodes its crawls.+ */+ private Map<String, Object> graph(String workflowName, String workflowRunId) {+ List<Map<String, Object>> nodes = new ArrayList<>(graphNodes(workflowName));+ List<Map<String, Object>> edges = new ArrayList<>();+ for (Trigger trigger : glueService.triggersOfWorkflow(workflowName)) {+ String triggerId = nodeId(workflowName, "TRIGGER", trigger.getName());+ if (trigger.getPredicate() != null && trigger.getPredicate().getConditions() != null) {+ for (TriggerCondition condition : trigger.getPredicate().getConditions()) {+ edges.add(edge(nodeId(workflowName, condition.getJobName() != null ? "JOB" : "CRAWLER",+ condition.getJobName() != null ? condition.getJobName() : condition.getCrawlerName()), triggerId));+ }+ }+ for (TriggerAction action : trigger.getActions()) {+ edges.add(edge(triggerId, nodeId(workflowName, action.getJobName() != null ? "JOB" : "CRAWLER",+ action.getJobName() != null ? action.getJobName() : action.getCrawlerName())));+ }+ }+ if (workflowRunId != null) {+ List<JobRun> jobRuns = jobRunService.runsInWorkflowRun(workflowRunId);+ List<GlueCrawlerRunService.WorkflowCrawl> crawls = crawlerRunService.crawlsInWorkflowRun(workflowRunId);+ for (Map<String, Object> node : nodes) {+ String name = (String) node.get("Name");+ if ("JOB".equals(node.get("Type"))) {+ node.put("JobDetails", Map.of("JobRuns",+ jobRuns.stream().filter(run -> name.equals(run.getJobName())).toList()));+ } else if ("CRAWLER".equals(node.get("Type"))) {+ List<Map<String, Object>> crawlList = new ArrayList<>();+ for (GlueCrawlerRunService.WorkflowCrawl crawl : crawls) {+ if (name.equals(crawl.crawlerName())) {+ Map<String, Object> entry = new LinkedHashMap<>();+ entry.put("State", crawl.state());+ entry.put("StartedOn", epochSeconds(crawl.startedOn()));+ if (crawl.completedOn() != null) {+ entry.put("CompletedOn", epochSeconds(crawl.completedOn()));+ }+ entry.put("LogGroup", "/aws-glue/crawlers");+ entry.put("LogStream", name);+ crawlList.add(entry);+ }+ }+ node.put("CrawlerDetails", Map.of("Crawls", crawlList));+ }+ }+ }+ Map<String, Object> graph = new LinkedHashMap<>();+ graph.put("Nodes", nodes);+ graph.put("Edges", edges);+ return graph;+ }++ private List<Map<String, Object>> graphNodes(String workflowName) {+ Map<String, Map<String, Object>> nodes = new LinkedHashMap<>();+ for (Trigger trigger : glueService.triggersOfWorkflow(workflowName)) {+ Map<String, Object> triggerNode = node(workflowName, "TRIGGER", trigger.getName());+ triggerNode.put("TriggerDetails", Map.of("Trigger", trigger));+ nodes.putIfAbsent((String) triggerNode.get("UniqueId"), triggerNode);+ if (trigger.getPredicate() != null && trigger.getPredicate().getConditions() != null) {+ for (TriggerCondition condition : trigger.getPredicate().getConditions()) {+ Map<String, Object> watched = condition.getJobName() != null+ ? node(workflowName, "JOB", condition.getJobName())+ : node(workflowName, "CRAWLER", condition.getCrawlerName());+ nodes.putIfAbsent((String) watched.get("UniqueId"), watched);+ }+ }+ for (TriggerAction action : trigger.getActions()) {+ Map<String, Object> started = action.getJobName() != null+ ? node(workflowName, "JOB", action.getJobName())+ : node(workflowName, "CRAWLER", action.getCrawlerName());+ nodes.putIfAbsent((String) started.get("UniqueId"), started);+ }+ }+ return new ArrayList<>(nodes.values());+ }++ private static Map<String, Object> node(String workflowName, String type, String name) {+ Map<String, Object> node = new LinkedHashMap<>();+ node.put("Type", type);+ node.put("Name", name);+ node.put("UniqueId", nodeId(workflowName, type, name));+ return node;+ }++ private static Map<String, Object> edge(String sourceId, String destinationId) {+ Map<String, Object> edge = new LinkedHashMap<>();+ edge.put("SourceId", sourceId);+ edge.put("DestinationId", destinationId);+ return edge;+ }++ /** Stable per workflow, type and name, so a node keeps its id across reads and resumes. */+ private static String nodeId(String workflowName, String type, String name) {+ return "wnode_" + UUID.nameUUIDFromBytes((workflowName + "\u0000" + type + "\u0000" + name)+ .getBytes(StandardCharsets.UTF_8)).toString().replace("-", "");+ }++ private static double epochSeconds(Instant instant) {+ return instant.toEpochMilli() / 1000.0;+ }+/*** Processes the completions this trigger has not seen yet, in finishing order, and returns the- * completions that fire it. ANY fires on- * each completion that matches one of its conditions; AND fires on a matching completion once- * every condition's latest outcome matches.+ * completions that fire it. ANY fires on each completion that matches one of its conditions; AND+ * fires on a matching completion once every condition's latest outcome matches. A workflow's+ * trigger only counts completions from a running run of its workflow and keeps its latest outcomes+ * per workflow run, so AND is met within one run.*/- private List<GlueRunCompletion> newFirings(Trigger trigger) {+ private List<GlueRunCompletion> newFirings(Trigger trigger, Set<String> runningWorkflowRuns) {Map<String, String> before = seenStore.get(trigger.getName()).orElse(Map.of());Map<String, String> seen = new HashMap<>(before);List<KeyedCompletion> fresh = new ArrayList<>();@@ -200,11 +703,20 @@ public class GlueTriggerService {if (processed == null || event.completion().position().compareTo(processed) > 0) {seen.put(POSITION + event.key(), event.completion().position());}- seen.put(STATE + event.key(), event.completion().state());+ String scope = "";+ if (trigger.getWorkflowName() != null) {+ String workflowRunId = event.completion().workflowRunId();+ if (workflowRunId == null || !runningWorkflowRuns.contains(workflowRunId)+ || !trigger.getWorkflowName().equals(workflowNameOf(workflowRunId))) {+ continue;+ }+ scope = workflowRunId + "|";+ }+ seen.put(STATE + scope + event.key(), event.completion().state());boolean eventMatches = false;boolean allMatch = true;for (TriggerCondition condition : trigger.getPredicate().getConditions()) {- boolean matches = expectedState(condition).equals(seen.get(STATE + keyOf(condition)));+ boolean matches = expectedState(condition).equals(seen.get(STATE + scope + keyOf(condition)));allMatch &= matches;eventMatches |= matches && keyOf(condition).equals(event.key())&& expectedState(condition).equals(event.completion().state());@@ -282,7 +794,7 @@ public class GlueTriggerService {forgetLeastRecentlyUsedChainIfFull();}budget.setTriggeredRuns(budget.getTriggeredRuns() + 1);- budget.setLastUsed(Instant.now());+ budget.setLastUsed(clock.instant());budgetStore.put(originRunId, budget);return true;}@@ -320,12 +832,14 @@ public class GlueTriggerService {}/**- * Starts the trigger's actions on behalf of {@code originRunId} and returns how many started; with a- * null origin (StartTrigger on an ON_DEMAND trigger) each run or crawl started is its own origin and- * no budget applies. With an origin, each action claims one run of the chain's budget before it- * starts and gives it back if it cannot start, so only runs that actually start are counted.+ * Starts the trigger's actions on behalf of {@code originRunId} and returns how many started, inside+ * {@code workflowRunId} when it is not null. With a null origin (StartTrigger on an ON_DEMAND trigger,+ * or a workflow's starting trigger) each run or crawl started is its own origin and no budget+ * applies. With an origin, each action claims one run of the chain's budget before it starts and+ * gives it back if it cannot start, so only runs that actually start are counted. A job run in a+ * workflow run gets the {@code --WORKFLOW_NAME} and {@code --WORKFLOW_RUN_ID} arguments, as on AWS.*/- private int fire(Trigger trigger, String originRunId, boolean propagateFailures) {+ private int fire(Trigger trigger, String originRunId, String workflowRunId, boolean propagateFailures) {LOG.infov("Firing Glue trigger {0}", trigger.getName());int started = 0;for (TriggerAction action : trigger.getActions()) {@@ -337,14 +851,14 @@ public class GlueTriggerService {try {if (action.getJobName() != null) {JobRun overrides = new JobRun();- overrides.setArguments(action.getArguments());+ overrides.setArguments(withWorkflowArguments(action.getArguments(), workflowRunId));overrides.setTimeout(action.getTimeout());overrides.setSecurityConfiguration(action.getSecurityConfiguration());overrides.setNotificationProperty(action.getNotificationProperty());overrides.setTriggerName(trigger.getName());- jobRunService.startJobRun(action.getJobName(), null, overrides, originRunId);+ jobRunService.startJobRun(action.getJobName(), null, overrides, originRunId, workflowRunId);} else {- crawlerRunService.startCrawler(action.getCrawlerName(), originRunId);+ crawlerRunService.startCrawler(action.getCrawlerName(), originRunId, workflowRunId);}started++;} catch (AwsException e) {@@ -354,13 +868,38 @@ public class GlueTriggerService {if (propagateFailures) {throw e;}- LOG.warnv("Glue trigger {0} could not start {1}: {2}", trigger.getName(),- action.getJobName() != null ? action.getJobName() : action.getCrawlerName(), e.getMessage());+ String target = action.getJobName() != null ? action.getJobName() : action.getCrawlerName();+ LOG.warnv("Glue trigger {0} could not start {1}: {2}", trigger.getName(), target, e.getMessage());+ if (workflowRunId != null) {+ noteLaunchFailure(workflowRunId, "Trigger " + trigger.getName() + " could not start " + target+ + ": " + e.getMessage());+ }}}return started;}+ /** Records on the workflow run that one of its actions could not be launched. */+ private void noteLaunchFailure(String workflowRunId, String message) {+ workflowRunStore.get(workflowRunId).ifPresent(run -> {+ run.setErrorMessage(run.getErrorMessage() == null ? message : run.getErrorMessage() + "; " + message);+ workflowRunStore.put(workflowRunId, run);+ });+ }++ private Map<String, String> withWorkflowArguments(Map<String, String> arguments, String workflowRunId) {+ if (workflowRunId == null) {+ return arguments;+ }+ Map<String, String> merged = new LinkedHashMap<>();+ if (arguments != null) {+ merged.putAll(arguments);+ }+ merged.put("--WORKFLOW_NAME", workflowNameOf(workflowRunId));+ merged.put("--WORKFLOW_RUN_ID", workflowRunId);+ return merged;+ }+private static String expectedState(TriggerCondition condition) {return condition.getJobName() != null ? condition.getState() : condition.getCrawlState();}@@ -374,4 +913,7 @@ public class GlueTriggerService {}private record KeyedCompletion(String key, GlueRunCompletion completion) {}++ /** ResumeWorkflowRun's answer: the new run and the nodes restarted in it. */+ public record ResumedRun(String runId, List<String> nodeIds) {}}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.javaindex 9a3ee679ab..06141701cf 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/CrawlerRunRecord.java@@ -17,6 +17,7 @@ public class CrawlerRunRecord {private String currentMessagePrefix;private String currentCrawlId;private String currentOriginRunId;+ private String currentWorkflowRunId;private Instant lastStart;private String lastStatus;private String lastMessagePrefix;@@ -44,6 +45,9 @@ public class CrawlerRunRecord {public String getCurrentOriginRunId() { return currentOriginRunId; }public void setCurrentOriginRunId(String currentOriginRunId) { this.currentOriginRunId = currentOriginRunId; }+ public String getCurrentWorkflowRunId() { return currentWorkflowRunId; }+ public void setCurrentWorkflowRunId(String currentWorkflowRunId) { this.currentWorkflowRunId = currentWorkflowRunId; }+public String getLastStatus() { return lastStatus; }public void setLastStatus(String lastStatus) { this.lastStatus = lastStatus; }diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/FinishedCrawl.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/FinishedCrawl.javaindex 24f6f198a4..edd957e46c 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/model/FinishedCrawl.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/FinishedCrawl.java@@ -13,11 +13,13 @@ import java.time.Instant;@JsonIgnoreProperties(ignoreUnknown = true)public class FinishedCrawl {private long sequence;+ private String crawlerName;private Instant startedAt;private Instant finishedAt;private String status;private String crawlId;private String originRunId;+ private String workflowRunId;@JsonProperty(value = "triggeredRuns", access = JsonProperty.Access.WRITE_ONLY)private int legacyTriggeredRuns;@@ -27,6 +29,9 @@ public class FinishedCrawl {public int getLegacyTriggeredRuns() { return legacyTriggeredRuns; }public void setLegacyTriggeredRuns(int legacyTriggeredRuns) { this.legacyTriggeredRuns = legacyTriggeredRuns; }+ public String getCrawlerName() { return crawlerName; }+ public void setCrawlerName(String crawlerName) { this.crawlerName = crawlerName; }+public long getSequence() { return sequence; }public void setSequence(long sequence) { this.sequence = sequence; }@@ -45,4 +50,6 @@ public class FinishedCrawl {public String getOriginRunId() { return originRunId; }public void setOriginRunId(String originRunId) { this.originRunId = originRunId; }+ public String getWorkflowRunId() { return workflowRunId; }+ public void setWorkflowRunId(String workflowRunId) { this.workflowRunId = workflowRunId; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/JobRunBookkeeping.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/JobRunBookkeeping.javaindex 8956652584..6bc9f4c522 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/model/JobRunBookkeeping.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/JobRunBookkeeping.java@@ -6,7 +6,8 @@ import io.quarkus.runtime.annotations.RegisterForReflection;/*** Floci's own bookkeeping for one job run, never sent on the wire, and deleted with the run: the order- * in which Floci saw it finish, and the run that set off its trigger chain.+ * in which Floci saw it finish, the run that set off its trigger chain, and the workflow run it belongs+ * to.*/@RegisterForReflection// Earlier versions stored the chain's run count here as triggeredRuns: it is still read, so a chain in@@ -15,6 +16,7 @@ import io.quarkus.runtime.annotations.RegisterForReflection;public class JobRunBookkeeping {private long completionOrder;private String originRunId;+ private String workflowRunId;@JsonProperty(value = "triggeredRuns", access = JsonProperty.Access.WRITE_ONLY)private int legacyTriggeredRuns;@@ -30,4 +32,6 @@ public class JobRunBookkeeping {public String getOriginRunId() { return originRunId; }public void setOriginRunId(String originRunId) { this.originRunId = originRunId; }+ public String getWorkflowRunId() { return workflowRunId; }+ public void setWorkflowRunId(String workflowRunId) { this.workflowRunId = workflowRunId; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Workflow.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Workflow.javanew file mode 100644index 0000000000..245859ccd8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Workflow.java@@ -0,0 +1,69 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;+import java.util.Map;++/**+ * A Glue workflow. LastRun and Graph are filled in when the workflow is read; the stored copy+ * leaves them unset.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class Workflow {+ @JsonProperty("Name")+ private String name;++ @JsonProperty("Description")+ private String description;++ @JsonProperty("DefaultRunProperties")+ private Map<String, String> defaultRunProperties;++ @JsonProperty("CreatedOn")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant createdOn;++ @JsonProperty("LastModifiedOn")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant lastModifiedOn;++ @JsonProperty("LastRun")+ private WorkflowRun lastRun;++ @JsonProperty("Graph")+ private Map<String, Object> graph;++ @JsonProperty("MaxConcurrentRuns")+ private Integer maxConcurrentRuns;++ public Workflow() {}++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }++ public String getDescription() { return description; }+ public void setDescription(String description) { this.description = description; }++ public Map<String, String> getDefaultRunProperties() { return defaultRunProperties; }+ public void setDefaultRunProperties(Map<String, String> defaultRunProperties) { this.defaultRunProperties = defaultRunProperties; }++ public Instant getCreatedOn() { return createdOn; }+ public void setCreatedOn(Instant createdOn) { this.createdOn = createdOn; }++ public Instant getLastModifiedOn() { return lastModifiedOn; }+ public void setLastModifiedOn(Instant lastModifiedOn) { this.lastModifiedOn = lastModifiedOn; }++ public WorkflowRun getLastRun() { return lastRun; }+ public void setLastRun(WorkflowRun lastRun) { this.lastRun = lastRun; }++ public Map<String, Object> getGraph() { return graph; }+ public void setGraph(Map<String, Object> graph) { this.graph = graph; }++ public Integer getMaxConcurrentRuns() { return maxConcurrentRuns; }+ public void setMaxConcurrentRuns(Integer maxConcurrentRuns) { this.maxConcurrentRuns = maxConcurrentRuns; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/WorkflowRun.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/WorkflowRun.javanew file mode 100644index 0000000000..5d2cf866e8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/WorkflowRun.java@@ -0,0 +1,81 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;+import java.util.Map;++/**+ * One run of a Glue workflow. Statistics and Graph are computed from the job runs and crawls of+ * the run when it is read; the stored copy leaves them unset.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class WorkflowRun {+ @JsonProperty("Name")+ private String name;++ @JsonProperty("WorkflowRunId")+ private String workflowRunId;++ @JsonProperty("PreviousRunId")+ private String previousRunId;++ @JsonProperty("WorkflowRunProperties")+ private Map<String, String> workflowRunProperties;++ @JsonProperty("StartedOn")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant startedOn;++ @JsonProperty("CompletedOn")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant completedOn;++ @JsonProperty("Status")+ private String status;++ @JsonProperty("ErrorMessage")+ private String errorMessage;++ @JsonProperty("Statistics")+ private Map<String, Integer> statistics;++ @JsonProperty("Graph")+ private Map<String, Object> graph;++ public WorkflowRun() {}++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }++ public String getWorkflowRunId() { return workflowRunId; }+ public void setWorkflowRunId(String workflowRunId) { this.workflowRunId = workflowRunId; }++ public String getPreviousRunId() { return previousRunId; }+ public void setPreviousRunId(String previousRunId) { this.previousRunId = previousRunId; }++ public Map<String, String> getWorkflowRunProperties() { return workflowRunProperties; }+ public void setWorkflowRunProperties(Map<String, String> workflowRunProperties) { this.workflowRunProperties = workflowRunProperties; }++ public Instant getStartedOn() { return startedOn; }+ public void setStartedOn(Instant startedOn) { this.startedOn = startedOn; }++ public Instant getCompletedOn() { return completedOn; }+ public void setCompletedOn(Instant completedOn) { this.completedOn = completedOn; }++ public String getStatus() { return status; }+ public void setStatus(String status) { this.status = status; }++ public String getErrorMessage() { return errorMessage; }+ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }++ public Map<String, Integer> getStatistics() { return statistics; }+ public void setStatistics(Map<String, Integer> statistics) { this.statistics = statistics; }++ public Map<String, Object> getGraph() { return graph; }+ public void setGraph(Map<String, Object> graph) { this.graph = graph; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.javaindex 0b94ca1413..ff4d4fc1aa 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCrawlerRunServiceTest.java@@ -66,7 +66,7 @@ class GlueCrawlerRunServiceTest {}private GlueCrawlerRunService service(int runDurationSeconds) {- return new GlueCrawlerRunService(recordStore, glueService, runDurationSeconds, clock);+ return new GlueCrawlerRunService(recordStore, new InMemoryStorage<>(), glueService, runDurationSeconds, clock);}private void createCrawler(String name, String scheduleExpression) {diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.javaindex 7b227578e2..800ea218ae 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java@@ -48,9 +48,9 @@ class GlueJsonHandlerEmptyListTest {GlueJobRunService jobRunService =new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());GlueCrawlerRunService crawlerRunService =- new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());+ new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());handler = new GlueJsonHandler(glueService, jobRunService, crawlerRunService,- new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService,+ new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService,jobRunService, crawlerRunService),schemaRegistryService, mapper);}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.javaindex c370eb5d13..c27ede91d3 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java@@ -49,9 +49,9 @@ class GlueJsonHandlerTest {GlueJobRunService jobRunService =new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());GlueCrawlerRunService crawlerRunService =- new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());+ new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());handler = new GlueJsonHandler(glueService, jobRunService, crawlerRunService,- new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService,+ new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService,jobRunService, crawlerRunService),schemaRegistryService, mapper);}@@ -1021,4 +1021,84 @@ class GlueJsonHandlerTest {assertEquals(1, runs.size());assertEquals("after-extract", runs.get(0).get("TriggerName").asText());}++ /**+ * Workflow operations on the wire: CreateWorkflow answers with the name, StartWorkflowRun with the+ * run id, GetWorkflowRun nests the run under "Run" with numeric timestamps and Statistics, and+ * GetWorkflow with IncludeGraph carries Nodes and Edges.+ */+ @Test+ void workflowOperationsAnswerWithTheDocumentedBodies() throws Exception {+ createJobWithTags("extract", Map.of());+ createJobWithTags("load", Map.of());+ ObjectNode create = mapper.createObjectNode();+ create.put("Name", "etl");+ create.putObject("DefaultRunProperties").put("env", "dev");+ create.putObject("Tags").put("team", "data");+ assertEquals("etl", mapper.valueToTree(handler.handle("CreateWorkflow", create, REGION).getEntity())+ .get("Name").asText());+ ObjectNode start = mapper.createObjectNode();+ start.put("Name", "start");+ start.put("Type", "ON_DEMAND");+ start.put("WorkflowName", "etl");+ start.putArray("Actions").addObject().put("JobName", "extract");+ handler.handle("CreateTrigger", start, REGION);+ ObjectNode then = mapper.createObjectNode();+ then.put("Name", "then-load");+ then.put("Type", "CONDITIONAL");+ then.put("WorkflowName", "etl");+ then.put("StartOnCreation", true);+ then.putArray("Actions").addObject().put("JobName", "load");+ then.putObject("Predicate").putArray("Conditions").addObject()+ .put("LogicalOperator", "EQUALS").put("JobName", "extract").put("State", "SUCCEEDED");+ handler.handle("CreateTrigger", then, REGION);++ ObjectNode startRun = mapper.createObjectNode().put("Name", "etl");+ String runId = mapper.valueToTree(handler.handle("StartWorkflowRun", startRun, REGION).getEntity())+ .get("RunId").asText();++ ObjectNode getRun = mapper.createObjectNode().put("Name", "etl").put("RunId", runId);+ JsonNode run = mapper.valueToTree(handler.handle("GetWorkflowRun", getRun, REGION).getEntity()).get("Run");+ assertEquals("COMPLETED", run.get("Status").asText());+ assertEquals("etl", run.get("Name").asText());+ assertTrue(run.get("StartedOn").isNumber());+ assertTrue(run.get("CompletedOn").isNumber());+ assertEquals(2, run.get("Statistics").get("SucceededActions").asInt());+ assertEquals("dev", run.get("WorkflowRunProperties").get("env").asText());+ assertFalse(run.has("Graph"));++ ObjectNode getWorkflow = mapper.createObjectNode().put("Name", "etl").put("IncludeGraph", true);+ JsonNode workflow = mapper.valueToTree(handler.handle("GetWorkflow", getWorkflow, REGION).getEntity())+ .get("Workflow");+ assertEquals(runId, workflow.get("LastRun").get("WorkflowRunId").asText());+ assertEquals(4, workflow.get("Graph").get("Nodes").size());+ assertEquals(3, workflow.get("Graph").get("Edges").size());+ assertTrue(workflow.get("CreatedOn").isNumber());++ JsonNode runs = mapper.valueToTree(handler.handle(+ "GetWorkflowRuns", mapper.createObjectNode().put("Name", "etl"), REGION).getEntity());+ assertEquals(1, runs.get("Runs").size());+ JsonNode names = mapper.valueToTree(handler.handle("ListWorkflows", mapper.createObjectNode(), REGION).getEntity());+ assertEquals("etl", names.get("Workflows").get(0).asText());++ ObjectNode put = mapper.createObjectNode().put("Name", "etl").put("RunId", runId);+ put.putObject("RunProperties").put("rows", "42");+ assertEquals(0, mapper.valueToTree(handler.handle("PutWorkflowRunProperties", put, REGION).getEntity()).size());+ JsonNode properties = mapper.valueToTree(handler.handle("GetWorkflowRunProperties", getRun, REGION).getEntity());+ assertEquals("42", properties.get("RunProperties").get("rows").asText());++ ObjectNode tags = mapper.createObjectNode();+ tags.put("ResourceArn", "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":workflow/etl");+ assertEquals("data", mapper.valueToTree(handler.handle("GetTags", tags, REGION).getEntity())+ .get("Tags").get("team").asText());++ ObjectNode batch = mapper.createObjectNode();+ batch.putArray("Names").add("etl").add("absent");+ JsonNode got = mapper.valueToTree(handler.handle("BatchGetWorkflows", batch, REGION).getEntity());+ assertEquals("etl", got.get("Workflows").get(0).get("Name").asText());+ assertEquals("absent", got.get("MissingWorkflows").get(0).asText());++ assertEquals("etl", mapper.valueToTree(handler.handle(+ "DeleteWorkflow", mapper.createObjectNode().put("Name", "etl"), REGION).getEntity()).get("Name").asText());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex 484165bdb2..6b5a438754 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -32,6 +32,7 @@ import io.github.hectorvent.floci.services.glue.model.S3Target;import io.github.hectorvent.floci.services.glue.model.SchemaReference;import io.github.hectorvent.floci.services.glue.model.SecurityConfiguration;import io.github.hectorvent.floci.services.glue.model.Trigger;+import io.github.hectorvent.floci.services.glue.model.Workflow;import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;import io.github.hectorvent.floci.services.glue.model.Table;import io.github.hectorvent.floci.services.glue.model.UserDefinedFunction;@@ -112,6 +113,7 @@ class GlueServiceTest {new InMemoryStorage<String, DataCatalogEncryptionSettings>(),new InMemoryStorage<String, SecurityConfiguration>(),new InMemoryStorage<String, Trigger>(),+ new InMemoryStorage<String, Workflow>(),schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null), kmsService);glueService.createDatabase(new Database("db1"));}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueTriggerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueTriggerServiceTest.javaindex 6ed40a69ba..839d501b5e 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueTriggerServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueTriggerServiceTest.java@@ -67,8 +67,8 @@ class GlueTriggerServiceTest {regionResolver, new ResourceGroupsTaggingService(storageFactory),new KmsService(storageFactory, regionResolver));jobRuns = new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());- crawls = new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());- triggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, crawls);+ crawls = new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, Clock.systemUTC());+ triggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, crawls);for (String job : List.of("extract", "load", "report")) {createJob(job);}@@ -326,10 +326,9 @@ class GlueTriggerServiceTest {/** A run that could not start gives its claim back, so a failing action does not shorten the chain. */@Testvoid actionsThatFailToStartDoNotUseUpTheBudget() {- GlueCrawlerRunService slowCrawls = new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 600,+ GlueCrawlerRunService slowCrawls = new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 600,new MutableClock());- GlueTriggerService loopTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(),- glueService, jobRuns, slowCrawls);+ GlueTriggerService loopTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, slowCrawls);slowCrawls.startCrawler("raw");Trigger loop = conditional("loop", null, List.of(jobIs("extract", "SUCCEEDED")), "extract");TriggerAction crawl = new TriggerAction();@@ -351,8 +350,7 @@ class GlueTriggerServiceTest {MutableClock clock = new MutableClock();GlueJobRunService slowRuns = new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(),glueService, 60, clock);- GlueTriggerService chain = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(),- glueService, slowRuns, crawls);+ GlueTriggerService chain = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, crawls);chain.createTrigger(conditional("after-crawl", "ANY", List.of(crawlIs("raw", "SUCCEEDED")), "extract"),true, null, REGION);chain.createTrigger(conditional("after-extract", null, List.of(jobIs("extract", "SUCCEEDED")), "load"),@@ -376,9 +374,8 @@ class GlueTriggerServiceTest {InMemoryStorage<String, JobRunBookkeeping> bookkeeping = new InMemoryStorage<>();GlueJobRunService slowRuns = new GlueJobRunService(new InMemoryStorage<>(), bookkeeping, glueService, 60, clock);InMemoryStorage<String, CrawlerRunRecord> crawlRecords = new InMemoryStorage<>();- GlueCrawlerRunService slowCrawls = new GlueCrawlerRunService(crawlRecords, glueService, 60, clock);- GlueTriggerService legacy = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(),- glueService, slowRuns, slowCrawls);+ GlueCrawlerRunService slowCrawls = new GlueCrawlerRunService(crawlRecords, new InMemoryStorage<>(), glueService, 60, clock);+ GlueTriggerService legacy = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, slowCrawls);legacy.createTrigger(conditional("after-extract", null, List.of(jobIs("extract", "SUCCEEDED")), "load"),true, null, REGION);legacy.createTrigger(conditional("after-crawl", "ANY", List.of(crawlIs("raw", "SUCCEEDED")), "report"),@@ -419,8 +416,7 @@ class GlueTriggerServiceTest {MutableClock clock = new MutableClock();InMemoryStorage<String, JobRunBookkeeping> bookkeeping = new InMemoryStorage<>();GlueJobRunService slowRuns = new GlueJobRunService(new InMemoryStorage<>(), bookkeeping, glueService, 60, clock);- GlueTriggerService upgraded = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(),- glueService, slowRuns, crawls);+ GlueTriggerService upgraded = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, crawls);upgraded.createTrigger(conditional("loop", null, List.of(jobIs("extract", "SUCCEEDED")), "extract"),true, null, REGION);String origin = slowRuns.startJobRun("extract", null, new JobRun()).getId();@@ -440,8 +436,8 @@ class GlueTriggerServiceTest {void theBudgetStoreKeepsAtMostTheTrackedChains() {int cap = 20;InMemoryStorage<String, TriggerChainBudget> budgets = new InMemoryStorage<>();- GlueTriggerService bounded = new GlueTriggerService(new InMemoryStorage<>(), budgets, glueService, jobRuns,- crawls, cap);+ GlueTriggerService bounded = new GlueTriggerService(new InMemoryStorage<>(), budgets, new InMemoryStorage<>(), glueService, jobRuns,+ crawls, cap, Clock.systemUTC());bounded.createTrigger(conditional("after", null, List.of(jobIs("extract", "SUCCEEDED")), "load"),true, null, REGION);@@ -633,7 +629,7 @@ class GlueTriggerServiceTest {void aSuccessIsNotLostWhenALaterRunOfTheSameJobIsStoppedFirst() {MutableClock clock = new MutableClock();GlueJobRunService slowRuns = new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 60, clock);- GlueTriggerService slowTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, crawls);+ GlueTriggerService slowTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, slowRuns, crawls);slowTriggers.createTrigger(conditional("after", null, List.of(jobIs("extract", "SUCCEEDED")), "load"),true, null, REGION);slowRuns.startJobRun("extract", null, new JobRun());@@ -652,7 +648,7 @@ class GlueTriggerServiceTest {Clock frozen = Clock.fixed(Instant.parse("2026-09-25T12:00:00Z"), ZoneOffset.UTC);GlueJobRunService frozenRuns =new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, frozen);- GlueTriggerService frozenTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, frozenRuns, crawls);+ GlueTriggerService frozenTriggers = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, frozenRuns, crawls);frozenTriggers.createTrigger(conditional("each", "ANY", List.of(jobIs("extract", "SUCCEEDED")), "load"),true, null, REGION);@@ -667,9 +663,9 @@ class GlueTriggerServiceTest {@Testvoid crawlsFinishingInTheSameClockTickEachCount() {Clock frozen = Clock.fixed(Instant.parse("2026-09-25T12:00:00Z"), ZoneOffset.UTC);- GlueCrawlerRunService frozenCrawls = new GlueCrawlerRunService(new InMemoryStorage<>(), glueService, 0, frozen);+ GlueCrawlerRunService frozenCrawls = new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, 0, frozen);GlueTriggerService frozenTriggers =- new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, frozenCrawls);+ new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobRuns, frozenCrawls);frozenTriggers.createTrigger(conditional("each", "ANY", List.of(crawlIs("raw", "SUCCEEDED")), "load"),true, null, REGION);diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowIntegrationTest.javanew file mode 100644index 0000000000..42e22e32f4--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowIntegrationTest.java@@ -0,0 +1,71 @@+package io.github.hectorvent.floci.services.glue;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.ValidatableResponse;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;++@QuarkusTest+class GlueWorkflowIntegrationTest {++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String ROLE = "arn:aws:iam::000000000000:role/my-role";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ private static ValidatableResponse call(String action, String body) {+ return given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue." + action)+ .body(body)+ .when().post("/")+ .then();+ }++ @Test+ void aWorkflowRunRunsItsTriggersToCompletion() {+ String suffix = UUID.randomUUID().toString().substring(0, 8);+ String workflow = "wf-" + suffix;+ String extract = "extract-" + suffix;+ String load = "load-" + suffix;+ for (String job : new String[] {extract, load}) {+ call("CreateJob", """+ { "Name": "%s", "Role": "%s", "Command": { "Name": "glueetl" } }+ """.formatted(job, ROLE)).statusCode(200);+ }+ call("CreateWorkflow", "{ \"Name\": \"" + workflow + "\" }")+ .statusCode(200)+ .body("Name", equalTo(workflow));+ call("CreateTrigger", """+ { "Name": "start-%s", "Type": "ON_DEMAND", "WorkflowName": "%s", "Actions": [ { "JobName": "%s" } ] }+ """.formatted(suffix, workflow, extract)).statusCode(200);+ call("CreateTrigger", """+ { "Name": "then-%s", "Type": "CONDITIONAL", "WorkflowName": "%s", "StartOnCreation": true,+ "Actions": [ { "JobName": "%s" } ],+ "Predicate": { "Conditions": [+ { "LogicalOperator": "EQUALS", "JobName": "%s", "State": "SUCCEEDED" } ] } }+ """.formatted(suffix, workflow, load, extract)).statusCode(200);++ String runId = call("StartWorkflowRun", "{ \"Name\": \"" + workflow + "\" }")+ .statusCode(200)+ .extract().path("RunId");++ call("GetWorkflowRun", "{ \"Name\": \"" + workflow + "\", \"RunId\": \"" + runId + "\" }")+ .statusCode(200)+ .body("Run.Status", equalTo("COMPLETED"))+ .body("Run.Statistics.SucceededActions", equalTo(2));++ call("DeleteWorkflow", "{ \"Name\": \"" + workflow + "\" }").statusCode(200);+ call("GetWorkflow", "{ \"Name\": \"" + workflow + "\" }")+ .statusCode(400)+ .body("__type", equalTo("EntityNotFoundException"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowServiceTest.javanew file mode 100644index 0000000000..76dbd49ffe--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueWorkflowServiceTest.java@@ -0,0 +1,641 @@+package io.github.hectorvent.floci.services.glue;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.glue.model.Crawler;+import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;+import io.github.hectorvent.floci.services.glue.model.ExecutionProperty;+import io.github.hectorvent.floci.services.glue.model.Job;+import io.github.hectorvent.floci.services.glue.model.JobCommand;+import io.github.hectorvent.floci.services.glue.model.JobRun;+import io.github.hectorvent.floci.services.glue.model.Predicate;+import io.github.hectorvent.floci.services.glue.model.S3Target;+import io.github.hectorvent.floci.services.glue.model.Trigger;+import io.github.hectorvent.floci.services.glue.model.TriggerAction;+import io.github.hectorvent.floci.services.glue.model.TriggerCondition;+import io.github.hectorvent.floci.services.glue.model.Workflow;+import io.github.hectorvent.floci.services.glue.model.WorkflowRun;+import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;+import io.github.hectorvent.floci.services.kms.KmsService;+import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.time.Instant;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++class GlueWorkflowServiceTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT_ID = "000000000000";+ private static final String ROLE = "arn:aws:iam::000000000000:role/glue";++ private GlueService glueService;+ private GlueJobRunService jobRuns;+ private GlueCrawlerRunService crawls;+ private GlueTriggerService workflows;+ private MutableClock clock;++ @BeforeEach+ void setUp() {+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);+ StorageFactory storageFactory = new InMemoryStorageFactory();+ glueService = new GlueService(storageFactory, new GlueSchemaRegistryService(storageFactory, regionResolver),+ regionResolver, new ResourceGroupsTaggingService(storageFactory),+ new KmsService(storageFactory, regionResolver));+ clock = new MutableClock();+ useRunDuration(0);+ for (String job : List.of("extract", "load")) {+ Job definition = new Job();+ definition.setName(job);+ definition.setRole(ROLE);+ definition.setCommand(new JobCommand());+ ExecutionProperty property = new ExecutionProperty();+ property.setMaxConcurrentRuns(10);+ definition.setExecutionProperty(property);+ glueService.createJob(definition, null, REGION);+ }+ Crawler crawler = new Crawler();+ crawler.setName("raw");+ crawler.setRole(ROLE);+ S3Target target = new S3Target();+ target.setPath("s3://raw/data");+ CrawlerTargets targets = new CrawlerTargets();+ targets.setS3Targets(List.of(target));+ crawler.setTargets(targets);+ glueService.createCrawler(crawler, null, REGION);+ }++ private void useRunDuration(int seconds) {+ useDurations(seconds, seconds);+ }++ private void useDurations(int jobSeconds, int crawlSeconds) {+ jobRuns = new GlueJobRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, jobSeconds, clock);+ crawls = new GlueCrawlerRunService(new InMemoryStorage<>(), new InMemoryStorage<>(), glueService, crawlSeconds,+ clock);+ workflows = new GlueTriggerService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),+ glueService, jobRuns, crawls, GlueTriggerService.MAX_TRACKED_CHAINS, clock);+ }++ private void createWorkflow(String name, Integer maxConcurrentRuns, Map<String, String> defaults) {+ Workflow workflow = new Workflow();+ workflow.setName(name);+ workflow.setMaxConcurrentRuns(maxConcurrentRuns);+ workflow.setDefaultRunProperties(defaults);+ glueService.createWorkflow(workflow, null, REGION);+ }++ private static TriggerAction startJob(String job) {+ TriggerAction action = new TriggerAction();+ action.setJobName(job);+ return action;+ }++ private static TriggerAction startCrawler(String crawler) {+ TriggerAction action = new TriggerAction();+ action.setCrawlerName(crawler);+ return action;+ }++ private static TriggerCondition jobIs(String job, String state) {+ TriggerCondition condition = new TriggerCondition();+ condition.setLogicalOperator("EQUALS");+ condition.setJobName(job);+ condition.setState(state);+ return condition;+ }++ private static TriggerCondition crawlIs(String crawler, String state) {+ TriggerCondition condition = new TriggerCondition();+ condition.setLogicalOperator("EQUALS");+ condition.setCrawlerName(crawler);+ condition.setCrawlState(state);+ return condition;+ }++ private void onDemand(String name, String workflow, TriggerAction... actions) {+ Trigger trigger = new Trigger();+ trigger.setName(name);+ trigger.setType("ON_DEMAND");+ trigger.setWorkflowName(workflow);+ trigger.setActions(List.of(actions));+ workflows.createTrigger(trigger, false, null, REGION);+ }++ private void conditional(String name, String workflow, String logical, List<TriggerCondition> conditions,+ String startsJob) {+ Trigger trigger = new Trigger();+ trigger.setName(name);+ trigger.setType("CONDITIONAL");+ trigger.setWorkflowName(workflow);+ trigger.setActions(List.of(startJob(startsJob)));+ Predicate predicate = new Predicate();+ predicate.setLogical(logical);+ predicate.setConditions(conditions);+ trigger.setPredicate(predicate);+ workflows.createTrigger(trigger, true, null, REGION);+ }++ /** extract, then load when extract succeeds: the smallest two-step workflow. */+ private void twoStepPipeline(String workflow) {+ onDemand("start", workflow, startJob("extract"));+ conditional("then-load", workflow, null, List.of(jobIs("extract", "SUCCEEDED")), "load");+ }++ private List<JobRun> runsOf(String job) {+ return jobRuns.getJobRuns(job, null, null).items();+ }++ @Test+ void definitionsFollowTheApiRules() {+ createWorkflow("etl", null, null);+ assertEquals("AlreadyExistsException", assertThrows(AwsException.class,+ () -> createWorkflow("etl", null, null)).getErrorCode());+ assertEquals("InvalidInputException", assertThrows(AwsException.class,+ () -> createWorkflow("zero", 0, null)).getErrorCode());++ glueService.updateWorkflow("etl", "nightly", null, 2);+ Workflow updated = glueService.getWorkflow("etl");+ assertEquals("nightly", updated.getDescription());+ assertEquals(2, updated.getMaxConcurrentRuns());++ assertEquals("EntityNotFoundException", assertThrows(AwsException.class,+ () -> onDemand("orphan", "missing", startJob("extract"))).getErrorCode());+ Trigger event = new Trigger();+ event.setName("on-event");+ event.setType("EVENT");+ event.setActions(List.of(startJob("extract")));+ assertEquals("InvalidInputException", assertThrows(AwsException.class,+ () -> workflows.createTrigger(event, false, null, REGION)).getErrorCode());+ event.setWorkflowName("etl");+ workflows.createTrigger(event, false, null, REGION);+ assertEquals("etl", glueService.getTrigger("on-event").getWorkflowName());++ workflows.deleteWorkflow("etl", REGION);+ workflows.deleteWorkflow("etl", REGION);+ assertEquals("EntityNotFoundException",+ assertThrows(AwsException.class, () -> glueService.getWorkflow("etl")).getErrorCode());+ }++ @Test+ void aWorkflowRunRunsItsPipelineAndCompletes() {+ createWorkflow("etl", null, Map.of("env", "dev", "day", "default"));+ twoStepPipeline("etl");++ String runId = workflows.startWorkflowRun("etl", Map.of("day", "2026-09-26"));++ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);+ assertTrue(runId.matches("wr_[0-9a-f]{64}"), runId);+ assertEquals("COMPLETED", run.getStatus());+ assertNotNull(run.getCompletedOn());+ assertEquals(Map.of("env", "dev", "day", "2026-09-26"), run.getWorkflowRunProperties());+ assertEquals(2, run.getStatistics().get("TotalActions"));+ assertEquals(2, run.getStatistics().get("SucceededActions"));+ assertEquals(0, run.getStatistics().get("RunningActions"));+ JobRun load = runsOf("load").getFirst();+ assertEquals("then-load", load.getTriggerName());+ assertEquals("etl", load.getArguments().get("--WORKFLOW_NAME"));+ assertEquals(runId, load.getArguments().get("--WORKFLOW_RUN_ID"));+ }++ /** Run times come from the service clock, so a run's timestamps follow the actions in it. */+ @Test+ void aRunIsTimedByTheActionsInIt() {+ useRunDuration(60);+ createWorkflow("etl", null, null);+ twoStepPipeline("etl");++ Instant before = clock.instant();+ String runId = workflows.startWorkflowRun("etl", null);+ Instant after = clock.instant();+ clock.advance(Duration.ofSeconds(61));+ assertEquals("RUNNING", workflows.getWorkflowRun("etl", runId, false).getStatus());+ clock.advance(Duration.ofSeconds(61));+ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);++ assertEquals("COMPLETED", run.getStatus());+ assertTrue(run.getStartedOn().isAfter(before) && run.getStartedOn().isBefore(after), run.getStartedOn().toString());+ JobRun load = runsOf("load").getFirst();+ assertEquals(load.getCompletedOn(), run.getCompletedOn());+ assertTrue(run.getCompletedOn().isAfter(runsOf("extract").getFirst().getCompletedOn()));+ long seconds = Duration.between(run.getStartedOn(), run.getCompletedOn()).toSeconds();+ assertTrue(seconds >= 120 && seconds < 125, seconds + " seconds");+ }++ @Test+ void aStoppedRunCompletesWhenItIsStopped() {+ useRunDuration(60);+ createWorkflow("etl", null, null);+ twoStepPipeline("etl");+ String runId = workflows.startWorkflowRun("etl", null);+ clock.advance(Duration.ofSeconds(30));++ Instant before = clock.instant();+ workflows.stopWorkflowRun("etl", runId);+ Instant after = clock.instant();++ Instant completedOn = workflows.getWorkflowRun("etl", runId, false).getCompletedOn();+ assertTrue(completedOn.isAfter(before) && completedOn.isBefore(after), completedOn.toString());+ }++ @Test+ void workflowTriggersIgnoreRunsOutsideTheirWorkflow() {+ createWorkflow("etl", null, null);+ twoStepPipeline("etl");++ jobRuns.startJobRun("extract", null, new JobRun());+ workflows.fireConditionalTriggers();++ assertTrue(runsOf("load").isEmpty());+ }++ @Test+ void andIsMetWithinOneWorkflowRunOnly() {+ createWorkflow("etl", null, null);+ onDemand("start", "etl", startJob("extract"));+ conditional("both", "etl", "AND", List.of(jobIs("extract", "SUCCEEDED"), crawlIs("raw", "SUCCEEDED")), "load");+ crawls.startCrawler("raw");++ String runId = workflows.startWorkflowRun("etl", null);++ assertTrue(runsOf("load").isEmpty());+ assertEquals("COMPLETED", workflows.getWorkflowRun("etl", runId, false).getStatus());++ onDemand("start-both", "etl", startCrawler("raw"));+ workflows.startWorkflowRun("etl", null);+ assertEquals(1, runsOf("load").size());+ }++ @Test+ void maxConcurrentRunsRefusesAnotherRunWhileOneIsRunning() {+ useRunDuration(60);+ createWorkflow("etl", 1, null);+ twoStepPipeline("etl");++ String first = workflows.startWorkflowRun("etl", null);+ assertEquals("ConcurrentRunsExceededException", assertThrows(AwsException.class,+ () -> workflows.startWorkflowRun("etl", null)).getErrorCode());+ assertEquals("RUNNING", workflows.getWorkflowRun("etl", first, false).getStatus());++ clock.advance(Duration.ofSeconds(61));+ assertEquals("RUNNING", workflows.getWorkflowRun("etl", first, false).getStatus());+ clock.advance(Duration.ofSeconds(61));+ assertEquals("COMPLETED", workflows.getWorkflowRun("etl", first, false).getStatus());+ assertNotNull(workflows.startWorkflowRun("etl", null));+ }++ @Test+ void stoppingARunStopsWhatIsRunningInIt() {+ useRunDuration(60);+ createWorkflow("etl", null, null);+ twoStepPipeline("etl");+ String runId = workflows.startWorkflowRun("etl", null);++ workflows.stopWorkflowRun("etl", runId);++ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);+ assertEquals("STOPPED", run.getStatus());+ assertEquals(1, run.getStatistics().get("StoppedActions"));+ assertEquals("STOPPED", runsOf("extract").getFirst().getJobRunState());+ assertEquals("IllegalWorkflowStateException", assertThrows(AwsException.class,+ () -> workflows.stopWorkflowRun("etl", runId)).getErrorCode());+ clock.advance(Duration.ofSeconds(120));+ workflows.fireConditionalTriggers();+ assertTrue(runsOf("load").isEmpty());+ }++ @Test+ void resumingRestartsTheChosenNodesAndWhatFollowsInANewRun() {+ createWorkflow("etl", null, Map.of("env", "dev"));+ twoStepPipeline("etl");+ String first = workflows.startWorkflowRun("etl", null);+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();+ String extractNode = nodeId(graph, "JOB", "extract");+ String triggerNode = nodeId(graph, "TRIGGER", "start");++ String resumed = workflows.resumeWorkflowRun("etl", first, List.of(extractNode)).runId();++ WorkflowRun run = workflows.getWorkflowRun("etl", resumed, false);+ assertEquals(first, run.getPreviousRunId());+ assertEquals("COMPLETED", run.getStatus());+ assertEquals(Map.of("env", "dev"), run.getWorkflowRunProperties());+ assertEquals(2, run.getStatistics().get("TotalActions"));+ assertEquals(2, runsOf("load").size());+ // The downstream node ran because of the resume, inside the resumed run, started by its trigger.+ JobRun downstream = runsOf("load").stream()+ .filter(load -> resumed.equals(load.getArguments().get("--WORKFLOW_RUN_ID")))+ .findFirst().orElseThrow();+ assertEquals("then-load", downstream.getTriggerName());+ assertEquals("SUCCEEDED", downstream.getJobRunState());+ assertEquals(List.of(downstream.getId()), jobRunIdsOnNode(workflows.getWorkflowRun("etl", resumed, true), "load"));+ assertEquals("InvalidInputException", assertThrows(AwsException.class,+ () -> workflows.resumeWorkflowRun("etl", first, List.of(triggerNode))).getErrorCode());+ }++ /**+ * Resume runs "the selected nodes and all nodes that are downstream from the selected nodes": in+ * extract, then load, then report, resuming load runs load and report in the new run, and not extract.+ */+ @Test+ void resumingAMiddleNodeRunsItAndWhatIsDownstreamButNotWhatIsUpstream() {+ Job report = new Job();+ report.setName("report");+ report.setRole(ROLE);+ report.setCommand(new JobCommand());+ glueService.createJob(report, null, REGION);+ createWorkflow("etl", null, null);+ twoStepPipeline("etl");+ conditional("then-report", "etl", null, List.of(jobIs("load", "SUCCEEDED")), "report");+ String first = workflows.startWorkflowRun("etl", null);+ assertEquals(3, workflows.getWorkflowRun("etl", first, false).getStatistics().get("SucceededActions"));+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();++ String resumed = workflows.resumeWorkflowRun("etl", first, List.of(nodeId(graph, "JOB", "load"))).runId();++ WorkflowRun run = workflows.getWorkflowRun("etl", resumed, true);+ assertEquals("COMPLETED", run.getStatus());+ assertEquals(2, run.getStatistics().get("TotalActions"));+ assertEquals(2, run.getStatistics().get("SucceededActions"));+ assertEquals(1, jobRunIdsOnNode(run, "load").size());+ assertEquals(1, jobRunIdsOnNode(run, "report").size());+ assertTrue(jobRunIdsOnNode(run, "extract").isEmpty());+ assertEquals(1, runsOf("extract").size());+ JobRun downstream = runsOf("report").stream()+ .filter(r -> resumed.equals(r.getArguments().get("--WORKFLOW_RUN_ID")))+ .findFirst().orElseThrow();+ assertEquals("then-report", downstream.getTriggerName());+ }++ @SuppressWarnings("unchecked")+ private static List<String> jobRunIdsOnNode(WorkflowRun run, String jobName) {+ Map<String, Object> node = ((List<Map<String, Object>>) run.getGraph().get("Nodes")).stream()+ .filter(n -> "JOB".equals(n.get("Type")) && jobName.equals(n.get("Name")))+ .findFirst().orElseThrow();+ return ((List<JobRun>) ((Map<String, Object>) node.get("JobDetails")).get("JobRuns")).stream()+ .map(JobRun::getId).toList();+ }++ @Test+ void aNodeNamedTwiceIsResumedOnce() {+ useDurations(0, 60);+ createWorkflow("etl", null, null);+ onDemand("start", "etl", startCrawler("raw"));+ String first = workflows.startWorkflowRun("etl", null);+ clock.advance(Duration.ofSeconds(61));+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();+ String raw = nodeId(graph, "CRAWLER", "raw");++ GlueTriggerService.ResumedRun resumed = workflows.resumeWorkflowRun("etl", first, List.of(raw, raw));++ assertEquals(List.of(raw), resumed.nodeIds());+ assertEquals("RUNNING", workflows.getWorkflowRun("etl", resumed.runId(), false).getStatus());+ assertNull(workflows.getWorkflowRun("etl", resumed.runId(), false).getErrorMessage());+ }++ @Test+ void aRunningWorkflowRunCannotBeResumed() {+ useRunDuration(60);+ createWorkflow("etl", null, null);+ twoStepPipeline("etl");+ String runId = workflows.startWorkflowRun("etl", null);++ assertEquals("IllegalWorkflowStateException", assertThrows(AwsException.class,+ () -> workflows.resumeWorkflowRun("etl", runId, List.of("wnode_any"))).getErrorCode());+ }++ @Test+ @SuppressWarnings("unchecked")+ void theGraphLinksTriggersToTheJobsTheyWatchAndStart() {+ createWorkflow("etl", null, null);+ twoStepPipeline("etl");+ String runId = workflows.startWorkflowRun("etl", null);++ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();+ List<Map<String, Object>> nodes = (List<Map<String, Object>>) graph.get("Nodes");+ List<Map<String, Object>> edges = (List<Map<String, Object>>) graph.get("Edges");+ assertEquals(4, nodes.size());+ assertEquals(3, edges.size());+ assertTrue(edges.contains(Map.of("SourceId", nodeId(graph, "TRIGGER", "start"),+ "DestinationId", nodeId(graph, "JOB", "extract"))));+ assertTrue(edges.contains(Map.of("SourceId", nodeId(graph, "JOB", "extract"),+ "DestinationId", nodeId(graph, "TRIGGER", "then-load"))));++ Map<String, Object> runGraph = workflows.getWorkflowRun("etl", runId, true).getGraph();+ Map<String, Object> loadNode = ((List<Map<String, Object>>) runGraph.get("Nodes")).stream()+ .filter(node -> "load".equals(node.get("Name"))).findFirst().orElseThrow();+ assertEquals(1, ((List<JobRun>) ((Map<String, Object>) loadNode.get("JobDetails")).get("JobRuns")).size());+ assertNull(workflows.getWorkflowRun("etl", runId, false).getGraph());+ }++ @Test+ void runPropertiesAreMergedAndLastRunIsReported() {+ createWorkflow("etl", null, Map.of("env", "dev"));+ twoStepPipeline("etl");+ String runId = workflows.startWorkflowRun("etl", null);++ workflows.putWorkflowRunProperties("etl", runId, Map.of("rows", "42", "env", "prod"));++ assertEquals(Map.of("env", "prod", "rows", "42"), workflows.getWorkflowRunProperties("etl", runId));+ Workflow workflow = workflows.workflowView(glueService.getWorkflow("etl"), false);+ assertEquals(runId, workflow.getLastRun().getWorkflowRunId());+ assertNull(workflow.getGraph());+ assertEquals("EntityNotFoundException", assertThrows(AwsException.class,+ () -> workflows.getWorkflowRunProperties("etl", "wr_missing")).getErrorCode());+ }++ @Test+ void deletingAWorkflowRemovesItsRunsButKeepsItsTriggers() {+ createWorkflow("etl", null, null);+ twoStepPipeline("etl");+ workflows.startWorkflowRun("etl", null);++ workflows.deleteWorkflow("etl", REGION);+ createWorkflow("etl", null, null);++ assertTrue(workflows.getWorkflowRuns("etl", false, null, null).items().isEmpty());+ assertEquals("etl", glueService.getTrigger("start").getWorkflowName());+ }++ @Test+ void concurrentStartsRespectMaxConcurrentRuns() throws Exception {+ useRunDuration(60);+ createWorkflow("etl", 1, null);+ twoStepPipeline("etl");+ int threads = 8;+ CountDownLatch go = new CountDownLatch(1);+ AtomicInteger started = new AtomicInteger();+ ExecutorService pool = Executors.newFixedThreadPool(threads);+ try {+ List<Future<?>> futures = new ArrayList<>();+ for (int i = 0; i < threads; i++) {+ futures.add(pool.submit(() -> {+ go.await();+ try {+ workflows.startWorkflowRun("etl", null);+ started.incrementAndGet();+ } catch (AwsException expected) {+ // ConcurrentRunsExceededException is the outcome under test for all but one start.+ }+ return null;+ }));+ }+ go.countDown();+ for (Future<?> future : futures) {+ future.get(10, TimeUnit.SECONDS);+ }+ } finally {+ pool.shutdownNow();+ }++ assertEquals(1, started.get());+ assertEquals(1, workflows.getWorkflowRuns("etl", false, null, null).items().size());+ }++ @Test+ void aRunWhoseStartingActionsCannotLaunchEndsInError() {+ useRunDuration(60);+ createWorkflow("etl", null, null);+ onDemand("start", "etl", startCrawler("raw"));+ crawls.startCrawler("raw");++ String runId = workflows.startWorkflowRun("etl", null);++ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);+ assertEquals("ERROR", run.getStatus());+ assertTrue(run.getErrorMessage().contains("could not start raw"), run.getErrorMessage());+ assertNotNull(run.getCompletedOn());+ }++ @Test+ void aLaunchFailureBesideAStartedActionIsReportedOnTheRun() {+ useRunDuration(60);+ createWorkflow("etl", null, null);+ onDemand("start", "etl", startCrawler("raw"), startJob("extract"));+ crawls.startCrawler("raw");++ String runId = workflows.startWorkflowRun("etl", null);++ WorkflowRun run = workflows.getWorkflowRun("etl", runId, false);+ assertEquals("RUNNING", run.getStatus());+ assertTrue(run.getErrorMessage().contains("could not start raw"), run.getErrorMessage());+ }++ @Test+ void aResumeThatCannotLaunchANodeLeavesNoRunBehind() {+ useDurations(0, 60);+ createWorkflow("etl", null, null);+ onDemand("start", "etl", startJob("extract"), startCrawler("raw"));+ String first = workflows.startWorkflowRun("etl", null);+ clock.advance(Duration.ofSeconds(61));+ assertEquals("COMPLETED", workflows.getWorkflowRun("etl", first, false).getStatus());+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();+ crawls.startCrawler("raw");++ assertEquals("CrawlerRunningException", assertThrows(AwsException.class, () -> workflows.resumeWorkflowRun(+ "etl", first, List.of(nodeId(graph, "JOB", "extract"), nodeId(graph, "CRAWLER", "raw")))).getErrorCode());++ assertEquals(1, workflows.getWorkflowRuns("etl", false, null, null).items().size());+ assertEquals(1, runsOf("extract").size());+ }++ @Test+ void deletingAWorkflowWhileItsCrawlRunsLeavesNoCrawlHistoryBehind() {+ useDurations(0, 60);+ createWorkflow("etl", null, null);+ onDemand("start", "etl", startCrawler("raw"));+ String runId = workflows.startWorkflowRun("etl", null);++ workflows.deleteWorkflow("etl", REGION);+ clock.advance(Duration.ofSeconds(61));+ crawls.startCrawler("raw");++ assertTrue(crawls.crawlsInWorkflowRun(runId).isEmpty());+ }++ @Test+ void aResumedJobRunsWithItsOriginalArguments() {+ createWorkflow("etl", null, null);+ TriggerAction extract = startJob("extract");+ extract.setArguments(Map.of("--mode", "full"));+ onDemand("start", "etl", extract);+ String first = workflows.startWorkflowRun("etl", null);+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();++ String resumed = workflows.resumeWorkflowRun("etl", first, List.of(nodeId(graph, "JOB", "extract"))).runId();++ JobRun rerun = runsOf("extract").getFirst();+ assertEquals("full", rerun.getArguments().get("--mode"));+ assertEquals(resumed, rerun.getArguments().get("--WORKFLOW_RUN_ID"));+ assertEquals("start", rerun.getTriggerName());+ }++ /** A workflow run keeps its crawls however many newer crawls the crawler has run since. */+ @Test+ void anOlderRunKeepsItsCrawlsAfterTheCrawlerHistoryMovesOn() {+ createWorkflow("etl", null, null);+ onDemand("start", "etl", startCrawler("raw"));+ String runId = workflows.startWorkflowRun("etl", null);+ for (int i = 0; i < 110; i++) {+ crawls.startCrawler("raw");+ }+ Map<String, Object> graph = workflows.workflowView(glueService.getWorkflow("etl"), true).getGraph();++ assertEquals(1, workflows.getWorkflowRun("etl", runId, false).getStatistics().get("TotalActions"));+ assertNotNull(workflows.resumeWorkflowRun("etl", runId, List.of(nodeId(graph, "CRAWLER", "raw"))).runId());+ }++ @Test+ void aRunThatHasFinishedCannotBeStopped() {+ useRunDuration(60);+ createWorkflow("etl", null, null);+ onDemand("start", "etl", startJob("extract"));+ String runId = workflows.startWorkflowRun("etl", null);+ clock.advance(Duration.ofSeconds(61));++ assertEquals("IllegalWorkflowStateException", assertThrows(AwsException.class,+ () -> workflows.stopWorkflowRun("etl", runId)).getErrorCode());+ assertEquals("COMPLETED", workflows.getWorkflowRun("etl", runId, false).getStatus());+ }++ @SuppressWarnings("unchecked")+ private static String nodeId(Map<String, Object> graph, String type, String name) {+ return ((List<Map<String, Object>>) graph.get("Nodes")).stream()+ .filter(node -> type.equals(node.get("Type")) && name.equals(node.get("Name")))+ .map(node -> (String) node.get("UniqueId"))+ .findFirst().orElseThrow();+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return AccountAwareStorageBackend.inMemory(ACCOUNT_ID);+ }+ }+}