Light, fluffy, and always free - The AWS Local Emulator alternative
ci: abort hung tests with a thread dump and time out a stuck surefire fork (#4940)
Niels Kootstra546d7e894e14
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex fe09c6913d..11f7958d37 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -36,10 +36,10 @@ jobs:test:name: Build and Test (shard ${{ matrix.shard }})runs-on: ubuntu-latest- # Shards run 10m49s to 13m47s, so 15 left the slowest about a minute of margin and a slow- # runner spent it. This is a runaway-job kill switch, not a budget: Actions bills the minutes a- # job actually uses, so a higher ceiling costs nothing until something hangs.- timeout-minutes: 20+ # Shards now run up to 16m50s (about 15 min of it in "Run tests"), so 20 left little margin.+ # This is a runaway-job kill switch, not a budget: Actions bills the minutes a job actually+ # uses, so a higher ceiling costs nothing until something hangs.+ timeout-minutes: 25# Drafts skip the suite to spare runners; it runs once the PR is ready for review.# Written on the job rather than as a gate job so it costs no runner. Pushes to main# always run; `closed` and `edited` are excluded so widening the trigger types cannot@@ -115,11 +115,15 @@ jobs:cmp -s all-classes.txt "$RUNNER_TEMP/compiled-classes.txt" \|| { echo "::error::test class list from src/test/java differs from target/test-classes:"; diff all-classes.txt "$RUNNER_TEMP/compiled-classes.txt" | head -50; exit 1; }+ # surefire.timeout kills a fork whose test ignores JUnit's interrupt (deadlock, blocking+ # I/O). It caps the whole shard (one fork runs all its tests), so 20 min leaves room above+ # the slowest shard's ~15 min and still lets surefire report before the 25-minute job limit.- name: Run testsrun: >./mvnw surefire:test -B-Dsurefire.includesFile="$GITHUB_WORKSPACE/shard.txt"-Dsurefire.failIfNoTests=true+ -Dsurefire.timeout=1200# The prefetch runs in the background past its own step, so its per-image# results are only complete once the tests are done. A pull still runningdiff --git a/pom.xml b/pom.xmlindex 8a8c740473..2e88d7813d 100644--- a/pom.xml+++ b/pom.xml@@ -452,6 +452,16 @@<systemPropertyVariables><java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager></systemPropertyVariables>+ <!-- Abort a hung test with a thread dump instead of letting CI's job timeout kill+ the shard silently. Set here rather than in junit-platform.properties, which+ would shadow the one quarkus-junit ships (its profile-aware class orderer). -->+ <properties>+ <configurationParameters>+ junit.jupiter.execution.timeout.testable.method.default = 2m+ junit.jupiter.execution.timeout.lifecycle.method.default = 2m+ junit.jupiter.execution.timeout.threaddump.enabled = true+ </configurationParameters>+ </properties></configuration></plugin><plugin>diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksRegistryHostsDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksRegistryHostsDockerIntegrationTest.javaindex a8da7ef3cf..e1c11aea1e 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksRegistryHostsDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksRegistryHostsDockerIntegrationTest.java@@ -23,6 +23,7 @@ import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.Assumptions;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import java.io.ByteArrayOutputStream;import java.nio.ByteBuffer;@@ -53,6 +54,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;*/@QuarkusTest@TestProfile(EksRegistryHostsDockerIntegrationTest.Profile.class)+// Three 60 s cluster-ready waits plus 30 s, 30 s and 45 s polls: past the suite's 2-minute default.+@Timeout(value = 6, unit = TimeUnit.MINUTES)class EksRegistryHostsDockerIntegrationTest {private static final Logger LOG = Logger.getLogger(EksRegistryHostsDockerIntegrationTest.class);diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.javaindex 8d51c1ba13..d8ecfa11ca 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java@@ -13,6 +13,7 @@ import org.junit.jupiter.api.Assumptions;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import java.io.IOException;import java.nio.charset.StandardCharsets;@@ -32,6 +33,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTest+// Real containers may take up to awaitTerminalStatus's 150 s, past the suite's 2-minute default.+@Timeout(value = 3, unit = TimeUnit.MINUTES)class SageMakerDockerIntegrationTest {private static final String AUTH ="AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/sagemaker/aws4_request";diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerGpuDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerGpuDockerIntegrationTest.javaindex 95ebfbedff..cc7494c821 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerGpuDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerGpuDockerIntegrationTest.java@@ -13,10 +13,12 @@ import org.junit.jupiter.api.Assumptions;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import java.time.Duration;import java.util.HashMap;import java.util.Map;+import java.util.concurrent.TimeUnit;import static io.restassured.RestAssured.given;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -52,6 +54,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;* {@code mode} defaults to {@code cdi}.*/@QuarkusTest+// The training job may poll for up to 300 s plus the image pull, past the suite's 2-minute default.+@Timeout(value = 10, unit = TimeUnit.MINUTES)@TestProfile(SageMakerGpuDockerIntegrationTest.GpuHardwareProfile.class)class SageMakerGpuDockerIntegrationTest {diff --git a/src/test/java/io/github/hectorvent/floci/services/timestreaminfluxdb/TimestreamInfluxDbDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/timestreaminfluxdb/TimestreamInfluxDbDockerIntegrationTest.javaindex e52ba7c1da..b34eed7890 100644--- a/src/test/java/io/github/hectorvent/floci/services/timestreaminfluxdb/TimestreamInfluxDbDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/timestreaminfluxdb/TimestreamInfluxDbDockerIntegrationTest.java@@ -11,6 +11,7 @@ import org.junit.jupiter.api.Assumptions;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Tag;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import java.net.URI;import java.net.http.HttpClient;@@ -20,6 +21,7 @@ import java.nio.charset.StandardCharsets;import java.time.Duration;import java.util.Base64;import java.util.Map;+import java.util.concurrent.TimeUnit;import static io.restassured.RestAssured.given;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -29,6 +31,8 @@ import static org.junit.jupiter.api.Assertions.fail;@Tag("docker")@QuarkusTest@TestProfile(TimestreamInfluxDbDockerIntegrationTest.RealContainerProfile.class)+// Three 180 s status waits on real containers, plus the requests and cleanup around them.+@Timeout(value = 15, unit = TimeUnit.MINUTES)class TimestreamInfluxDbDockerIntegrationTest {private static final String CONTENT_TYPE = "application/x-amz-json-1.0";