Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

test: run isolated suites one at a time, after the shared ones

Claudea18963a16063
diff --git a/.github/workflows/run-blackbox-tests.yml b/.github/workflows/run-blackbox-tests.yml
index 3208b46ef5..64c0b26cb3 100644
--- a/.github/workflows/run-blackbox-tests.yml
+++ b/.github/workflows/run-blackbox-tests.yml
@@ -90,6 +90,25 @@ jobs:
env:
INFISICAL_TEST_IMAGE: infisical-test:ci
+ # The shared stack is gone once the step above ends (its make target takes it
+ # down). Prune what it left so the isolated run starts with the disk it needs.
+ - name: Free disk before isolated suites
+ if: ${{ !cancelled() }}
+ run: |
+ docker volume prune -af
+ docker builder prune -af
+ df -h /
+
+ # Each isolated package boots its own Postgres, Redis and Infisical, so they run
+ # one at a time. Runs even when the shared suites failed, so one red run does
+ # not hide the other.
+ - name: Run isolated suites
+ if: ${{ !cancelled() }}
+ run: make test-isolated
+ working-directory: tests
+ env:
+ INFISICAL_TEST_IMAGE: infisical-test:ci
+
- name: Upload container logs
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
diff --git a/tests/AGENTS.md b/tests/AGENTS.md
index 58562972a7..dbb9ebc919 100644
--- a/tests/AGENTS.md
+++ b/tests/AGENTS.md
@@ -29,9 +29,10 @@ Not here:
## Commands
```bash
-make test-suites # product tests; starts and stops the stack
-make test-harness # harness tests
-make test-all # both
+make test-suites # shared suites; starts and stops the stack
+make test-isolated # isolated suites, one package at a time
+make test-harness # harness tests on the shared stack
+make test-all # all three, the isolated run going ahead when the others fail
make test-unit # no containers
make lint # gofmt, vet, staticcheck
```
@@ -51,8 +52,12 @@ configuration, or the old container is reused.
```
suites/ behaviour, grouped by product (suites/secretmanager/secrets) or
- platform area (suites/machineidentity, suites/user, suites/group)
+ platform area (suites/machineidentity, suites/user, suites/group);
+ harness.Shared only
+suitesisolated/ packages on harness.Isolated, which write instance-wide state; each
+ boots its own stack, so they run one at a time (`make test-isolated`)
harnesstest/ harness behaviour; only for failures the harness alone can cause
+ (harnesstest/isolated is the harness's own Isolated package)
harness/ Stack, Profile, Tenant, Principal
fixture/ platform resources: projects, App Connections
fixture/<product>/ one package per product: secretmanager, pki, pam
@@ -72,6 +77,9 @@ clients/api/ generated client; never edit by hand
## Rules
- `t.Parallel()` on every test and subtest under `Shared`; never under `Isolated`.
+- A package on `harness.Isolated` lives under `suitesisolated/` (or `harnesstest/isolated/`),
+ and nothing else uses it. CI cannot hold several isolated stacks beside the shared one, so
+ it runs these after the shared suites with `-p 1`; a layout test in `harness/` enforces it.
- Every test body is `// Setup`, `// Action`, `// Assert`.
- Assert with `testify/require` only.
- Every test creates its own tenant.
@@ -95,7 +103,8 @@ and assert the exact status and message.
```bash
make lint
make test-all
-make up && go test -p 4 ./suites/... ./harnesstest/... -count=3; make down
+make up && go test -p 4 ./suites/... ./harnesstest/shared/... -count=3; make down
+go test -p 1 ./suitesisolated/... ./harnesstest/isolated/... -count=3; make down
```
The last command is the flake gate. A failure there is shared state leaking across
diff --git a/tests/Makefile b/tests/Makefile
index 5220690e9e..3f94db8d57 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -8,7 +8,7 @@ GOFMT := $(shell go env GOROOT)/bin/gofmt
# are missing and no test can reach them.
INFISICAL_OPENAPI_URL ?= http://localhost:8080
-.PHONY: fmt lint test test-unit test-docker test-suites test-harness test-all up down status generate-client
+.PHONY: fmt lint test test-unit test-docker test-suites test-isolated test-harness test-all up down status generate-client
fmt:
$(GOFMT) -w .
@@ -21,11 +21,11 @@ lint:
# Unit tests only: no Docker, no containers, runs in under a second.
#
-# Everything except ./suites/..., which needs a stack. Derived rather than listed,
+# Everything except the suites and the harness tests, which need a stack. Derived rather than listed,
# because a hand-written list silently stops covering a package the day someone adds
# one -- which is exactly what happened when fixture/, provider/ and stub/ appeared.
test-unit:
- go test $$(go list ./... | grep -vE '/(suites|harnesstest)/') -count=1
+ go test $$(go list ./... | grep -vE '/(suites|suitesisolated|harnesstest)/') -count=1
test: lint test-unit
@@ -50,17 +50,33 @@ test-docker:
# Infisical is broken; `test-harness` failing means the harness is. One target running
# both would report them the same way.
test-suites: PKGS = ./suites/...
-test-harness: PKGS = ./harnesstest/...
-test-all: PKGS = ./suites/... ./harnesstest/...
+test-harness: PKGS = ./harnesstest/shared/...
-# Extra `go test` flags. CI passes `-skip KnownIssues`: those tests assert the intended
-# behaviour for open backend bugs and fail until the bugs are fixed.
+# Extra `go test` flags, such as `-run` or `-skip`.
TEST_FLAGS ?=
-test-suites test-harness test-all:
+test-suites test-harness:
@trap '$(MAKE) --no-print-directory down' EXIT; \
go run ./cmd/inf up && go test $(PKGS) -count=1 $(TEST_FLAGS)
+# Packages on harness.Isolated, which each boot their own Postgres, Redis and Infisical.
+# `-p 1` runs one package at a time, and each removes its containers and volumes before
+# the next starts, so at most one isolated stack exists. No `inf up`: these packages do
+# not use the shared stack, and fakenet is started by the first package that needs it.
+# A layout test in ./harness keeps every isolated package under these two roots.
+test-isolated:
+ @trap '$(MAKE) --no-print-directory down' EXIT; \
+ go test -p 1 ./suitesisolated/... ./harnesstest/isolated/... -count=1 $(TEST_FLAGS)
+
+# The shared suites and harness tests, then the isolated ones. The isolated run goes
+# ahead when the shared one fails, so one red run does not hide the other.
+test-all:
+ @status=0; \
+ $(MAKE) --no-print-directory test-suites || status=$$?; \
+ $(MAKE) --no-print-directory test-harness || status=$$?; \
+ $(MAKE) --no-print-directory test-isolated || status=$$?; \
+ exit $$status
+
up:
go run ./cmd/inf up
diff --git a/tests/docs/writing-tests.md b/tests/docs/writing-tests.md
index 8dc42df92a..f2c4cc2489 100644
--- a/tests/docs/writing-tests.md
+++ b/tests/docs/writing-tests.md
@@ -46,7 +46,9 @@ func TestSecret_Create(t *testing.T) {
| `harness.Shared` | one instance for the run, a fresh organization per test | almost always |
| `harness.Isolated` | the package's own Postgres, Redis, and Infisical | the test writes instance-wide state: super-admin config, login methods, encryption strategy, run modes |
-`Isolated` costs a full boot per package. Containers the test talks to through Infisical,
+`Isolated` costs a full boot per package, so an `Isolated` package lives under
+`suitesisolated/`, where `make test-isolated` and CI run one package at a time after the
+shared suites; a layout test in `harness/` fails a misplaced one. Containers the test talks to through Infisical,
such as a rotation target database, are added with `harness.WithInfra(...)`. Instance-wide
environment, such as `RELAY_AUTH_SECRET`, is set with `harness.WithInfisicalEnv(k, v)`,
which only an `Isolated` package may use.
diff --git a/tests/harness/layout_test.go b/tests/harness/layout_test.go
new file mode 100644
index 0000000000..453eaff994
--- /dev/null
+++ b/tests/harness/layout_test.go
@@ -0,0 +1,65 @@
+package harness_test
+
+import (
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/Infisical/infisical/tests/infra"
+)
+
+// isolatedRoots are the only directories whose packages may use harness.Isolated. CI
+// runs them one package at a time, after the shared suites, because each boots its own
+// Postgres, Redis and Infisical; an Isolated package anywhere else would run alongside
+// the shared stack and every other package, which is more than a CI runner can hold.
+var isolatedRoots = []string{"suitesisolated", filepath.Join("harnesstest", "isolated")}
+
+func TestLayout_IsolatedPackagesLiveInIsolatedRoots(t *testing.T) {
+ root, err := infra.RepoRoot()
+ if err != nil {
+ t.Fatalf("finding the repository root: %v", err)
+ }
+ testsDir := filepath.Join(root, "tests")
+
+ var isolated []string
+ for _, dir := range []string{"suites", "suitesisolated", "harnesstest"} {
+ err := filepath.WalkDir(filepath.Join(testsDir, dir), func(path string, d fs.DirEntry, err error) error {
+ if err != nil || d.IsDir() || d.Name() != "main_test.go" {
+ return err
+ }
+ src, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ rel, _ := filepath.Rel(testsDir, filepath.Dir(path))
+ inRoot := underAny(rel, isolatedRoots)
+ usesIsolated := strings.Contains(string(src), "harness.Isolated")
+ switch {
+ case usesIsolated && !inRoot:
+ t.Errorf("%s uses harness.Isolated; move it under tests/suitesisolated/", rel)
+ case !usesIsolated && inRoot:
+ t.Errorf("%s is under an isolated root but does not use harness.Isolated; move it under tests/suites/", rel)
+ case usesIsolated:
+ isolated = append(isolated, rel)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("walking %s: %v", dir, err)
+ }
+ }
+ if len(isolated) == 0 {
+ t.Fatal("found no harness.Isolated package; the walk is not reading the suites")
+ }
+}
+
+func underAny(rel string, roots []string) bool {
+ for _, r := range roots {
+ if rel == r || strings.HasPrefix(rel, r+string(filepath.Separator)) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/tests/suites/admin/admin_test.go b/tests/suitesisolated/admin/admin_test.go
similarity index 100%
rename from tests/suites/admin/admin_test.go
rename to tests/suitesisolated/admin/admin_test.go
diff --git a/tests/suites/admin/main_test.go b/tests/suitesisolated/admin/main_test.go
similarity index 100%
rename from tests/suites/admin/main_test.go
rename to tests/suitesisolated/admin/main_test.go
diff --git a/tests/suites/gateway/instancerelay/instance_relay_test.go b/tests/suitesisolated/instancerelay/instance_relay_test.go
similarity index 100%
rename from tests/suites/gateway/instancerelay/instance_relay_test.go
rename to tests/suitesisolated/instancerelay/instance_relay_test.go
diff --git a/tests/suites/gateway/instancerelay/main_test.go b/tests/suitesisolated/instancerelay/main_test.go
similarity index 100%
rename from tests/suites/gateway/instancerelay/main_test.go
rename to tests/suitesisolated/instancerelay/main_test.go
diff --git a/tests/suites/user/isolated/main_test.go b/tests/suitesisolated/oauth/main_test.go
similarity index 91%
rename from tests/suites/user/isolated/main_test.go
rename to tests/suitesisolated/oauth/main_test.go
index 76f6d4b289..79561290a9 100644
--- a/tests/suites/user/isolated/main_test.go
+++ b/tests/suitesisolated/oauth/main_test.go
@@ -1,4 +1,4 @@
-package isolated_test
+package oauth_test
import (
"testing"
diff --git a/tests/suites/user/isolated/oauth_test.go b/tests/suitesisolated/oauth/oauth_test.go
similarity index 99%
rename from tests/suites/user/isolated/oauth_test.go
rename to tests/suitesisolated/oauth/oauth_test.go
index 5444704809..25d91c079d 100644
--- a/tests/suites/user/isolated/oauth_test.go
+++ b/tests/suitesisolated/oauth/oauth_test.go
@@ -1,4 +1,4 @@
-package isolated_test
+package oauth_test
import (
"bytes"