Skip to content

🎧 Your Personal Streaming Service

ci: run the plugins test suite in parallel processes (#6051)

Deluan Quintãoa2de8e61efe8
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 012e3a9c3..9da99e864 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -131,24 +131,14 @@ jobs:
uses: actions/checkout@v7
- uses: actions/setup-go@v6
- id: setup-go
with:
go-version-file: go.mod
- name: Download dependencies
run: go mod download
- # Without this, the plugins suite recompiles every test plugin WASM module,
- # which dominates the job runtime under -race.
- - name: Cache the plugins test suite WASM compilation cache
- uses: actions/cache@v6
- with:
- path: plugins/testdata/.wazero-cache
- key: wazero-${{ runner.os }}-go${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('plugins/testdata/*/*.go', 'plugins/testdata/*/go.*', 'plugins/pdk/go/**/*.go', 'plugins/pdk/go/go.*') }}
- restore-keys: wazero-${{ runner.os }}-
-
- name: Test
- run: go test -shuffle=on -tags netgo,sqlite_fts5 -race ./... -v
+ run: go test -shuffle=on -tags netgo,sqlite_fts5 -race -v $(go list ./... | grep -v '/plugins$')
- name: Test ndpgen
run: |
@@ -157,6 +147,30 @@ jobs:
go build -o ndpgen .
./ndpgen --help
+ go-plugins:
+ name: Test Go plugins
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v7
+
+ - uses: actions/setup-go@v6
+ id: setup-go
+ with:
+ go-version-file: go.mod
+
+ # Without this, the suite recompiles every test plugin WASM module,
+ # which dominates its runtime under -race.
+ - name: Cache the WASM compilation cache
+ uses: actions/cache@v6
+ with:
+ path: plugins/testdata/.wazero-cache
+ key: wazero-${{ runner.os }}-go${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('plugins/testdata/*/*.go', 'plugins/testdata/*/go.*', 'plugins/pdk/go/**/*.go', 'plugins/pdk/go/go.*') }}
+ restore-keys: wazero-${{ runner.os }}-
+
+ - name: Test plugins
+ run: go tool ginkgo -p -race -tags netgo,sqlite_fts5 ./plugins/
+
go-windows:
name: Test Go code (Windows)
runs-on: windows-2022
@@ -294,7 +308,7 @@ jobs:
build:
name: Build
- needs: [js, go, go-windows, go-lint, i18n-lint, git-version, check-push-enabled, validate-migrations]
+ needs: [js, go, go-plugins, go-windows, go-lint, i18n-lint, git-version, check-push-enabled, validate-migrations]
strategy:
matrix:
platform: [ linux/amd64, linux/arm64, linux/arm/v5, linux/arm/v6, linux/arm/v7, linux/386, linux/riscv64, darwin/amd64, darwin/arm64, windows/amd64, windows/386 ]
diff --git a/.gitignore b/.gitignore
index 810074e72..4ed971491 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,3 +47,4 @@ zz_*_test.go
# wazero compilation cache for the plugins test suite
/plugins/testdata/.wazero-cache/
+/plugins/testdata/*.stage/
diff --git a/plugins/plugins_suite_test.go b/plugins/plugins_suite_test.go
index c8585f2c1..2dae65d00 100644
--- a/plugins/plugins_suite_test.go
+++ b/plugins/plugins_suite_test.go
@@ -7,6 +7,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
+ "fmt"
"net/http"
"os"
"os/exec"
@@ -39,7 +40,6 @@ var (
func TestPlugins(t *testing.T) {
tests.Init(t, false)
- buildTestPlugins(t, testDataDir)
// Set globally so tests using configtest.SetupConfig inherit it. The cache
// persists between runs; entries are content-addressed, so a stale one only misses.
@@ -51,16 +51,11 @@ func TestPlugins(t *testing.T) {
RunSpecs(t, "Plugins Suite")
}
-func buildTestPlugins(t *testing.T, path string) {
- t.Helper()
+func buildTestPlugins(path string) {
start := time.Now()
- t.Logf("[BeforeSuite] Current working directory: %s", path)
- cmd := exec.Command("make", "-C", path)
- out, err := cmd.CombinedOutput()
- t.Logf("[BeforeSuite] Make output: %s elapsed: %s", string(out), time.Since(start))
- if err != nil {
- t.Fatalf("Failed to build test plugins: %v", err)
- }
+ out, err := exec.Command("make", "-C", path).CombinedOutput()
+ fmt.Fprintf(GinkgoWriter, "[BeforeSuite] built test plugins in %s:\n%s", time.Since(start), out)
+ Expect(err).ToNot(HaveOccurred(), "failed to build test plugins")
}
// createTestManager creates a new plugin Manager with the given plugin config.
@@ -146,7 +141,10 @@ func createTestManagerWithPluginsAndMetrics(pluginConfig map[string]map[string]s
return manager, tmpDir
}
-var _ = BeforeSuite(func() {
+var _ = SynchronizedBeforeSuite(func() {
+ // Build once: the testdata Makefile is not safe to run concurrently.
+ buildTestPlugins(testDataDir)
+}, func() {
// Get testdata directory (where test plugin .ndp packages live)
_, currentFile, _, ok := runtime.Caller(0)
Expect(ok).To(BeTrue())
diff --git a/plugins/testdata/Makefile b/plugins/testdata/Makefile
index 758497095..de948a2bb 100644
--- a/plugins/testdata/Makefile
+++ b/plugins/testdata/Makefile
@@ -10,17 +10,19 @@ all: $(PLUGINS:%=%.ndp)
clean:
rm -f $(PLUGINS:%=%.ndp) $(PLUGINS:%=%.wasm)
- rm -rf .wazero-cache
+ rm -rf .wazero-cache $(PLUGINS:%=%.stage)
# PDK source files that trigger rebuild when changed (recursive)
PDK_SOURCES := $(shell find ../pdk/go -name '*.go' 2>/dev/null)
# Build the .ndp package (zip containing manifest.json + plugin.wasm)
+# Stage under a per-target name: a shared plugin.wasm breaks concurrent builds.
%.ndp: %.wasm %/manifest.json
@rm -f $@
- @cp $< plugin.wasm
- zip -j $@ $*/manifest.json plugin.wasm
- @rm -f plugin.wasm
+ @rm -rf $*.stage && mkdir -p $*.stage
+ @cp $< $*.stage/plugin.wasm
+ zip -j $@ $*/manifest.json $*.stage/plugin.wasm
+ @rm -rf $*.stage
@mv $< $<.tmp && mv $<.tmp $< # Touch wasm to ensure it's older than ndp
# Build the wasm binary. -buildvcs=false keeps the bytes stable across commits, so