Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

feat(khaos): the pooled credential and the secret file, on a helper that needed two capabilities

CarterPerez-dev4b24c69bbbed
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/dbgateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/dbgateway.go
new file mode 100644
index 00000000..0614b089
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/dbgateway.go
@@ -0,0 +1,314 @@
+// ©AngelaMos | 2026
+// dbgateway.go
+
+package main
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+
+ _ "github.com/lib/pq"
+)
+
+const (
+ routeReady = "/ready"
+
+ // routeAlive is PROCESS liveness and nothing else. The container's own
+ // healthcheck has to use it, because /health is the endpoint under test:
+ // pointing a healthcheck at it makes the fixture unstartable until its
+ // database credential exists, and the credential is seeded after the
+ // stack is up.
+ routeAlive = "/alive"
+
+ // PoolUnbounded is the DEFAULT configuration of a Go service, not a
+ // deliberately broken one: `database/sql` closes a connection for age only
+ // when SetConnMaxLifetime is positive, and the zero value is zero. Every
+ // connection authenticated before a rotation keeps working until the
+ // process restarts.
+ PoolUnbounded = "unbounded"
+
+ // PoolBounded is the knob that converts a rotation into a revocation. A
+ // background cleaner reaps idle connections past the lifetime with no reuse
+ // attempt, so the pool redials and the stale secret surfaces.
+ PoolBounded = "bounded"
+
+ // PoolHeld is the sharpest shape and the one research 07 section 5.3 did
+ // not have: a connection CHECKED OUT and never returned is age-unbounded,
+ // because the cleaner only ever walks the idle list.
+ PoolHeld = "held"
+
+ boundedLifetime = 2 * time.Second
+ boundedIdle = time.Second
+
+ dbQuery = "SELECT 1"
+ dbTimeout = 3 * time.Second
+ readyPoolSize = 1
+
+ headerReason = "X-Khaos-Reason"
+
+ reasonNoCredential = "no bearer credential was presented"
+ reasonNotAuthorized = "the identity provider refused the credential"
+ reasonDatabaseDown = "the database refused or could not be reached"
+ reasonPooled = "answered on a pooled connection"
+ reasonFresh = "answered on a connection dialled for this request"
+)
+
+// dbGateway is the fixture the pooled-credential experiments measure. It holds
+// a database credential of its own, which is the one khaos rotates, and it
+// answers three questions that a real service usually conflates:
+//
+// - /protected serves data, and needs both a caller credential and the
+// database;
+// - /health answers on a POOLED connection, which is what almost every
+// liveness probe does and what keeps reporting green through a total
+// credential failure;
+// - /ready forces a FRESH connection, which is what readiness should do and
+// almost none do.
+type dbGateway struct {
+ db *sql.DB
+ dsn string
+ mode string
+
+ upstream string
+ failMode string
+ client *http.Client
+
+ mu sync.Mutex
+ held *sql.Conn
+}
+
+func newDBGateway(dsn, mode, upstream, failMode string) *dbGateway {
+ pool, err := sql.Open("postgres", dsn)
+ if err != nil {
+ pool = nil
+ }
+
+ gateway := &dbGateway{
+ db: pool,
+ dsn: dsn,
+ mode: mode,
+ upstream: upstream,
+ failMode: failMode,
+ client: &http.Client{Timeout: upstreamTimeout},
+ }
+
+ gateway.configure()
+
+ return gateway
+}
+
+// configure is where the fixture's whole behaviour lives, and every value is
+// the real default rather than an exaggeration.
+func (g *dbGateway) configure() {
+ if g.db == nil {
+ return
+ }
+
+ switch g.mode {
+ case PoolBounded:
+ g.db.SetConnMaxLifetime(boundedLifetime)
+ g.db.SetConnMaxIdleTime(boundedIdle)
+
+ case PoolHeld:
+ // The SAME lifetime the bounded mode sets, which is the point: the
+ // background cleaner only ever walks the idle list, so a connection
+ // that is checked out rather than idle is not bounded by it at all.
+ // Without the lifetime this mode would just be the unbounded one with
+ // extra steps.
+ //
+ // The connection is acquired on first use rather than at startup,
+ // because the engine may not hold the fixture's role yet when the
+ // container starts and a checkout that failed once would leave this
+ // mode measuring the unbounded one.
+ g.db.SetConnMaxLifetime(boundedLifetime)
+ g.db.SetConnMaxIdleTime(boundedIdle)
+
+ default:
+ // PoolUnbounded. Nothing is set, deliberately: this is the shape a
+ // service has when nobody called SetConnMaxLifetime at all.
+ }
+}
+
+func (g *dbGateway) routes() *http.ServeMux {
+ mux := http.NewServeMux()
+ mux.HandleFunc(routeAlive, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+ mux.HandleFunc(routeHealth, g.health)
+ mux.HandleFunc(routeReady, g.ready)
+ mux.HandleFunc(routeProtected, g.protected)
+ mux.HandleFunc(issuePath, g.forward(issuePath))
+ mux.HandleFunc(revokePath, g.forward(revokePath))
+
+ return mux
+}
+
+// health is the trap. It asks the database over whatever connection the pool
+// hands it, which after a rotation is one that authenticated before the
+// rotation, so it answers green while no new connection can be made at all.
+func (g *dbGateway) health(w http.ResponseWriter, r *http.Request) {
+ if err := g.queryPooled(r.Context()); err != nil {
+ w.Header().Set(headerReason, reasonDatabaseDown)
+ w.WriteHeader(http.StatusServiceUnavailable)
+
+ return
+ }
+
+ w.Header().Set(headerReason, reasonPooled)
+ w.WriteHeader(http.StatusOK)
+}
+
+// ready forces a connection of its own, which is the only way to answer the
+// question readiness is asking.
+func (g *dbGateway) ready(w http.ResponseWriter, r *http.Request) {
+ if err := g.queryFresh(r.Context()); err != nil {
+ w.Header().Set(headerReason, reasonDatabaseDown)
+ w.WriteHeader(http.StatusServiceUnavailable)
+
+ return
+ }
+
+ w.Header().Set(headerReason, reasonFresh)
+ w.WriteHeader(http.StatusOK)
+}
+
+func (g *dbGateway) protected(w http.ResponseWriter, r *http.Request) {
+ token, scheme := strings.CutPrefix(
+ r.Header.Get(headerAuthorization),
+ bearerPrefix,
+ )
+
+ if !scheme || token == "" {
+ w.Header().Set(headerReason, reasonNoCredential)
+ w.WriteHeader(http.StatusUnauthorized)
+
+ return
+ }
+
+ active, reachable := g.introspect(token)
+
+ if !reachable && g.failMode != FailOpen {
+ w.Header().Set(headerReason, reasonNotAuthorized)
+ w.WriteHeader(http.StatusForbidden)
+
+ return
+ }
+
+ if reachable && !active {
+ w.Header().Set(headerReason, reasonNotAuthorized)
+ w.WriteHeader(http.StatusForbidden)
+
+ return
+ }
+
+ // The credential is good, so whether this request is served is now purely
+ // a question about the database.
+ if err := g.queryPooled(r.Context()); err != nil {
+ w.Header().Set(headerReason, reasonDatabaseDown)
+ w.WriteHeader(http.StatusServiceUnavailable)
+
+ return
+ }
+
+ w.Header().Set(headerReason, reasonPooled)
+ w.WriteHeader(http.StatusOK)
+}
+
+// queryPooled prefers the held connection when there is one, because that is
+// the configuration under test, and otherwise lets the pool decide.
+func (g *dbGateway) queryPooled(ctx context.Context) error {
+ if g.db == nil {
+ return sql.ErrConnDone
+ }
+
+ ctx, cancel := context.WithTimeout(ctx, dbTimeout)
+ defer cancel()
+
+ var one int
+
+ if held := g.heldConn(ctx); held != nil {
+ return held.QueryRowContext(ctx, dbQuery).Scan(&one)
+ }
+
+ return g.db.QueryRowContext(ctx, dbQuery).Scan(&one)
+}
+
+// heldConn is the checked-out connection this fixture keeps for the life of
+// the process, acquired on first use. It is never returned to the pool, which
+// is what makes it age-unbounded.
+func (g *dbGateway) heldConn(ctx context.Context) *sql.Conn {
+ if g.mode != PoolHeld {
+ return nil
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ if g.held != nil {
+ return g.held
+ }
+
+ conn, err := g.db.Conn(ctx)
+ if err != nil {
+ return nil
+ }
+
+ g.held = conn
+
+ return g.held
+}
+
+// queryFresh opens its own pool for one question so the dial cannot be
+// satisfied from anything that authenticated earlier.
+func (g *dbGateway) queryFresh(ctx context.Context) error {
+ fresh, err := sql.Open("postgres", g.dsn)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = fresh.Close() }()
+
+ fresh.SetMaxIdleConns(0)
+ fresh.SetMaxOpenConns(readyPoolSize)
+
+ ctx, cancel := context.WithTimeout(ctx, dbTimeout)
+ defer cancel()
+
+ var one int
+
+ return fresh.QueryRowContext(ctx, dbQuery).Scan(&one)
+}
+
+func (g *dbGateway) introspect(token string) (bool, bool) {
+ response, err := g.client.Get(g.upstream + introspectPath + token)
+ if err != nil {
+ return false, false
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ if response.StatusCode == http.StatusUnauthorized ||
+ response.StatusCode == http.StatusForbidden {
+ return false, true
+ }
+
+ if response.StatusCode != http.StatusOK {
+ return false, false
+ }
+
+ var body introspection
+ if err := json.NewDecoder(response.Body).Decode(&body); err != nil {
+ return false, false
+ }
+
+ return body.Active, true
+}
+
+func (g *dbGateway) forward(path string) http.HandlerFunc {
+ plain := &gateway{upstream: g.upstream, client: g.client}
+
+ return plain.forward(path)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
index cb38be45..5a22fb27 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"net/http"
+ "net/url"
"os"
"os/signal"
"syscall"
@@ -25,7 +26,10 @@ const (
roleTLS = "tls"
roleRogue = "rogue"
roleResolver = "resolver"
+ roleDB = "db"
+ roleSecret = "secret"
roleHealth = "healthcheck"
+ roleAlive = "alivecheck"
envListen = "KHAOS_RANGE_LISTEN"
envUpstream = "KHAOS_RANGE_UPSTREAM"
@@ -34,7 +38,26 @@ const (
envReresolve = "KHAOS_RANGE_RERESOLVE"
trueValue = "true"
envFailMode = "KHAOS_RANGE_FAIL_MODE"
- envCacheTTL = "KHAOS_RANGE_CACHE_TTL"
+
+ // The pooled-credential fixture's own database credential. khaos reads the
+ // password out of this container's environment, because the container is
+ // the only authority on what it authenticated with, and rotates that role
+ // at the engine.
+ envDBHost = "KHAOS_RANGE_DB_HOST"
+ envDBUser = "KHAOS_RANGE_DB_USER"
+ envDBPassword = "KHAOS_RANGE_DB_PASSWORD"
+ envDBName = "KHAOS_RANGE_DB_NAME"
+ envPoolMode = "KHAOS_RANGE_POOL_MODE"
+
+ defaultDBHost = "postgres:5432"
+ defaultDBUser = "appuser"
+ defaultDBName = "appdb"
+
+ envSecretPath = "KHAOS_RANGE_SECRET_PATH"
+ envSecretMode = "KHAOS_RANGE_SECRET_MODE"
+
+ defaultSecretPath = "/run/khaos-secrets/hmac"
+ envCacheTTL = "KHAOS_RANGE_CACHE_TTL"
defaultListen = ":8080"
defaultUpstream = "http://idp:8080"
@@ -51,11 +74,13 @@ const (
// look like the fault's.
readHeaderTimeout = 120 * time.Second
- usage = "usage: khaos-range <idp|gateway|verifier|tls|rogue|resolver|healthcheck>"
- selfURL = "http://127.0.0.1:8080/health"
- healthWait = 2 * time.Second
- errorFormat = "khaos-range: %v\n"
- exitError = 1
+ usage = "usage: khaos-range <idp|gateway|verifier|tls|rogue|resolver|db|secret|healthcheck|alivecheck>"
+ selfBase = "http://127.0.0.1:8080"
+ selfHealthPath = "/health"
+ selfAlivePath = "/alive"
+ healthWait = 2 * time.Second
+ errorFormat = "khaos-range: %v\n"
+ exitError = 1
)
func main() {
@@ -71,7 +96,11 @@ func run() error {
}
if os.Args[1] == roleHealth {
- return selfCheck()
+ return selfCheck(selfHealthPath)
+ }
+
+ if os.Args[1] == roleAlive {
+ return selfCheck(selfAlivePath)
}
// The resolver binds UDP and TCP, so it never reaches the server below.
@@ -135,6 +164,18 @@ func roleHandler(role string) (http.Handler, error) {
).routes(), nil
case roleRogue:
return newRogueProvider().routes(), nil
+ case roleSecret:
+ return newSecretGateway(
+ envOr(envSecretPath, defaultSecretPath),
+ envOr(envSecretMode, SecretUnchecked),
+ ).routes(), nil
+ case roleDB:
+ return newDBGateway(
+ databaseDSN(),
+ envOr(envPoolMode, PoolUnbounded),
+ envOr(envUpstream, defaultUpstream),
+ envOr(envFailMode, FailClosed),
+ ).routes(), nil
case roleTLS:
return newTLSGateway(
envOr(envUpstream, defaultUpstream),
@@ -150,10 +191,28 @@ func roleHandler(role string) (http.Handler, error) {
}
}
-func selfCheck() error {
+// databaseDSN is assembled from separate variables rather than taken as one
+// string, so khaos can read the PASSWORD on its own out of this container's
+// environment and put the same value back after the run.
+func databaseDSN() string {
+ target := url.URL{
+ Scheme: "postgres",
+ User: url.UserPassword(
+ envOr(envDBUser, defaultDBUser),
+ os.Getenv(envDBPassword),
+ ),
+ Host: envOr(envDBHost, defaultDBHost),
+ Path: "/" + envOr(envDBName, defaultDBName),
+ RawQuery: "sslmode=disable&connect_timeout=3",
+ }
+
+ return target.String()
+}
+
+func selfCheck(path string) error {
client := &http.Client{Timeout: healthWait}
- response, err := client.Get(selfURL)
+ response, err := client.Get(selfBase + path)
if err != nil {
return err
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/secretgateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/secretgateway.go
new file mode 100644
index 00000000..e273f793
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/secretgateway.go
@@ -0,0 +1,127 @@
+// ©AngelaMos | 2026
+// secretgateway.go
+
+package main
+
+import (
+ "crypto/rand"
+ "net/http"
+ "os"
+ "strings"
+
+ "github.com/CarterPerez-dev/khaos/internal/rangelab"
+ "github.com/CarterPerez-dev/khaos/internal/secretfile"
+)
+
+const (
+ // SecretValidated is the correct handling research 07 part 8 prescribes:
+ // explicit length validation after every read. An empty secret is not "no
+ // secret", it is "a secret equal to the empty string", and HMAC will
+ // happily compute with it.
+ SecretValidated = rangelab.SecretValidated
+
+ // SecretUnchecked is the CVE-2019-20933 shape: the secret is read, no
+ // length check is made, and the empty value flows straight into the
+ // crypto. The upstream fix for that CVE is literally
+ // `if h.Config.SharedSecret == "" { 401 }`.
+ SecretUnchecked = rangelab.SecretUnchecked
+
+ routeSecretToken = "/secret-token"
+
+ reasonSecretUnreadable = "the secret file could not be read"
+ reasonSecretEmpty = "the secret file is empty, and this mode refuses to verify with an empty key"
+ reasonSecretBad = "the credential does not verify against the secret this service holds"
+ reasonSecretOK = "verified against the secret read for this request"
+)
+
+// secretGateway re-reads its secret on EVERY request, which is what makes the
+// fault observable at all: a service that read it once at startup holds the
+// original for the life of the process and khaos would be measuring nothing.
+// That is the same property the DNS gateway needs for re-resolution, and it is
+// a deliberate choice of fixture rather than a claim about real services.
+type secretGateway struct {
+ path string
+ mode string
+}
+
+func newSecretGateway(path, mode string) *secretGateway {
+ return &secretGateway{path: path, mode: mode}
+}
+
+func (g *secretGateway) routes() *http.ServeMux {
+ mux := http.NewServeMux()
+
+ mux.HandleFunc(routeAlive, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+
+ // Liveness does NOT depend on the secret, deliberately. A healthcheck that
+ // did would make the container unstartable the moment khaos broke the
+ // secret, and compose would restart the target mid-measurement.
+ mux.HandleFunc(routeHealth, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+
+ mux.HandleFunc(routeSecretToken, g.issue)
+ mux.HandleFunc(routeProtected, g.protected)
+
+ return mux
+}
+
+// issue hands out a credential signed with whatever secret the service holds
+// right now, which is how khaos gets a valid one without reading the file.
+func (g *secretGateway) issue(w http.ResponseWriter, _ *http.Request) {
+ secret, err := os.ReadFile(g.path)
+ if err != nil {
+ w.Header().Set(headerReason, reasonSecretUnreadable)
+ w.WriteHeader(http.StatusServiceUnavailable)
+
+ return
+ }
+
+ w.Header().Set(contentType, mimeJSON)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(
+ `{"token":"` + secretfile.Token(secret, rand.Text()) + `"}`,
+ ))
+}
+
+func (g *secretGateway) protected(w http.ResponseWriter, r *http.Request) {
+ token, scheme := strings.CutPrefix(
+ r.Header.Get(headerAuthorization),
+ bearerPrefix,
+ )
+
+ if !scheme || token == "" {
+ w.Header().Set(headerReason, reasonNoCredential)
+ w.WriteHeader(http.StatusUnauthorized)
+
+ return
+ }
+
+ secret, err := os.ReadFile(g.path)
+ if err != nil {
+ w.Header().Set(headerReason, reasonSecretUnreadable)
+ w.WriteHeader(http.StatusServiceUnavailable)
+
+ return
+ }
+
+ // The one line that separates the two modes, and the whole fixture.
+ if g.mode == SecretValidated && len(secret) == 0 {
+ w.Header().Set(headerReason, reasonSecretEmpty)
+ w.WriteHeader(http.StatusForbidden)
+
+ return
+ }
+
+ if !secretfile.Verify(token, secret) {
+ w.Header().Set(headerReason, reasonSecretBad)
+ w.WriteHeader(http.StatusForbidden)
+
+ return
+ }
+
+ w.Header().Set(headerReason, reasonSecretOK)
+ w.WriteHeader(http.StatusOK)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
index eb71a28d..cd6baa24 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
@@ -121,6 +121,8 @@ enforcing in the first place.`
errNoAdminSecret = "%s does not carry %s in its environment, so khaos has no administrative credential for the %s and will not guess one. khaos reads it from the container rather than taking it on a flag, because a flag is world-readable in /proc and a journal entry would outlive the run. Point --%s at the variable that holds it, or run `khaos range up --with-db` for a stack that is already wired"
+ errNoPooledPath = "%w: this experiment rotates the credential a service's own connection pool authenticated with, and the running range has no %s; bring the range down and up again with `khaos range up --with-db`"
+
errNoSubjectSecret = "%s does not carry %s in its environment, so khaos cannot read the credential its own connection pool authenticated with, and a rotation it cannot undo is not one khaos will perform"
errTargetOffNetwork = "%s is not attached to %s, so it cannot be using the chaos resolver that serves it and khaos could not count its queries: %w"
@@ -510,7 +512,15 @@ func runExperiment(
return err
}
- env, closeTiers, err := standUpTiers(ctx, out, work, &spec, accepted)
+ env, closeTiers, err := standUpTiers(
+ ctx,
+ out,
+ work,
+ &spec,
+ accepted,
+ dockerx.Filter{Project: project},
+ guard,
+ )
if err != nil {
return err
}
@@ -749,6 +759,12 @@ func rangeSupplies() []string {
experiment.ParamResolverNetwork,
experiment.ParamDNSAddress,
experiment.ParamEngine,
+ experiment.ParamEngineTarget,
+ experiment.ParamSubjectRole,
+ experiment.ParamSubjectVar,
+ experiment.ParamHealthURL,
+ experiment.ParamReadyURL,
+ experiment.ParamSecretPath,
}
}
@@ -763,6 +779,8 @@ const (
tierTLS
tierDNS
tierCred
+ tierPool
+ tierSecret
)
func tierOf(kind string) tier {
@@ -775,6 +793,10 @@ func tierOf(kind string) tier {
return tierTLS
case inject.NeedsResolver(kind):
return tierDNS
+ case inject.NeedsSecret(kind):
+ return tierSecret
+ case inject.IsPoolFault(kind):
+ return tierPool
case inject.NeedsCred(kind):
return tierCred
default:
@@ -799,48 +821,92 @@ type rangeAnswers struct {
CertHost string
}
-func rangeDefaults(
- lab rangelab.Lab,
- answers rangeAnswers,
- at tier,
-) map[string]string {
- url := lab.ProtectedURL()
- target := rangelab.IDPName
- engine := ""
+// tierAnswers is what one tier's own gateway and fixtures answer. Split out of
+// rangeDefaults so a seventh tier adds a case rather than another ten lines to
+// one function.
+type tierAnswers struct {
+ url string
+ target string
+ engine string
+ engineTarget string
+ subjectRole string
+ subjectVar string
+ healthURL string
+ readyURL string
+ secretPath string
+}
+
+func answersFor(lab rangelab.Lab, at tier) tierAnswers {
+ answers := tierAnswers{
+ url: lab.ProtectedURL(),
+ target: rangelab.IDPName,
+ }
switch at {
case tierProxy:
- url = lab.ProxiedProtectedURL()
+ answers.url = lab.ProxiedProtectedURL()
+
+ // The target is the gateway, not the provider: the fault lands on khaos's
+ // own provider, so the gateway is the only container to ask consent for.
case tierIdP:
- // The target is the gateway, not the provider: the fault lands on khaos's
- // own provider, so the gateway is the only container to ask consent for.
- url = lab.VerifierProtectedURL()
- target = rangelab.VerifierGatewayName
+ answers.url = lab.VerifierProtectedURL()
+ answers.target = rangelab.VerifierGatewayName
case tierTLS:
- url = lab.TLSProtectedURL()
- target = rangelab.TLSGatewayName
+ answers.url = lab.TLSProtectedURL()
+ answers.target = rangelab.TLSGatewayName
case tierDNS:
- url = lab.DNSProtectedURL()
- target = rangelab.DNSGatewayName
+ answers.url = lab.DNSProtectedURL()
+ answers.target = rangelab.DNSGatewayName
// The target IS the engine here. Nothing of the operator's is mutated: the
// fault acts on a role khaos mints on that engine for the run. PostgreSQL
// is the zero-flag default because its own image carries the loopback
- // `trust` rule that research 07 section 6.1 measured a probe through, so it
- // is the engine where khaos's negative control earns its keep.
+ // `trust` rule that research 07 section 6.1 measured a probe through, so
+ // it is the engine where khaos's negative control earns its keep.
case tierCred:
- url = ""
- target = rangelab.PostgresName
- engine = string(dbx.Postgres)
+ answers.url = ""
+ answers.target = rangelab.PostgresName
+ answers.engine = string(dbx.Postgres)
+
+ // The pooled faults measure the SERVICE and perform the rotation at the
+ // engine, so they name both containers. The role is the fixture's own,
+ // never khaos's administrative one.
+ case tierPool:
+ answers.url = lab.DBProtectedURL()
+ answers.target = rangelab.DBGatewayName
+ answers.engine = string(dbx.Postgres)
+ answers.engineTarget = rangelab.PostgresName
+ answers.subjectRole = rangelab.AppRole
+ answers.subjectVar = rangelab.EnvDBPassword
+ answers.healthURL = lab.DBHealthURL()
+ answers.readyURL = lab.DBReadyURL()
+
+ // The one family that breaks a FILE rather than a credential, so the
+ // target is the service holding it and the path is read off the fixture
+ // rather than guessed.
+ case tierSecret:
+ answers.url = lab.SecretProtectedURL()
+ answers.target = rangelab.SecretGatewayName
+ answers.secretPath = rangelab.SecretPath
default:
// The direct tier's URL and target are the values set above.
}
+ return answers
+}
+
+func rangeDefaults(
+ lab rangelab.Lab,
+ answers rangeAnswers,
+ at tier,
+) map[string]string {
+ tiered := answersFor(lab, at)
+
defaults := map[string]string{
- experiment.ParamTarget: target,
+ experiment.ParamTarget: tiered.target,
experiment.ParamToken: answers.Token,
experiment.ParamRevocable: answers.Revocable,
experiment.ParamAuthority: answers.Authority,
@@ -857,22 +923,30 @@ func rangeDefaults(
experiment.ParamDNSHost: rangelab.DNSDependencyName,
experiment.ParamResolverNetwork: rangelab.NetworkName,
- // The redirect faults point at the range's ROGUE provider, which approves
- // every credential; an unroutable address would only be another outage.
+ // The redirect faults point at the range's ROGUE provider, which
+ // approves every credential; an unroutable address would only be
+ // another outage.
experiment.ParamDNSAddress: rangelab.RogueAddress,
}
- // Two answers are TIER-CONDITIONAL and are left out rather than filled with
- // an empty string. The credential tier has no url because its probes are
- // database connections, and no other tier has an engine. An empty answer
- // would read as "the range claims to answer this and cannot", which is the
- // shape that sends an operator looking for a flag that does nothing.
- if url != "" {
- defaults[experiment.ParamURL] = url
- }
-
- if engine != "" {
- defaults[experiment.ParamEngine] = engine
+ // TIER-CONDITIONAL answers are left out rather than filled with an empty
+ // string. The credential tier has no url because its probes are database
+ // connections, and no other tier has an engine. An empty answer would read
+ // as "the range claims to answer this and cannot", which is the shape that
+ // sends an operator looking for a flag that does nothing.
+ for name, value := range map[string]string{
+ experiment.ParamURL: tiered.url,
+ experiment.ParamEngine: tiered.engine,
+ experiment.ParamEngineTarget: tiered.engineTarget,
+ experiment.ParamSubjectRole: tiered.subjectRole,
+ experiment.ParamSubjectVar: tiered.subjectVar,
+ experiment.ParamHealthURL: tiered.healthURL,
+ experiment.ParamReadyURL: tiered.readyURL,
+ experiment.ParamSecretPath: tiered.secretPath,
+ } {
+ if value != "" {
+ defaults[name] = value
+ }
}
return defaults
@@ -912,26 +986,8 @@ func fillFromRange(
// Refuse before announcing, or the note names a URL resolved to port 0.
at := tierOf(definition.FaultKind)
- if at == tierProxy && lab.ProxiedPort == 0 {
- return nil, fmt.Errorf(
- errNoProxiedPath,
- rangelab.ErrRangeIncomplete,
- rangelab.ProxiedGatewayName,
- )
- }
-
- if at == tierIdP && (lab.VerifierPort == 0 || lab.Issuer == "") {
- return nil, fmt.Errorf(
- errNoVerifierPath,
- rangelab.ErrRangeIncomplete,
- rangelab.VerifierGatewayName,
- )
- }
-
- if at == tierCred {
- if err := lab.RequireStatefulTier(); err != nil {
- return nil, err
- }
+ if err := requireTier(lab, at); err != nil {
+ return nil, err
}
answers, err := askTheRange(
@@ -939,6 +995,7 @@ func fillFromRange(
reader,
lab,
fillable(definition, supplied),
+ at,
)
if err != nil {
return nil, err
@@ -970,6 +1027,56 @@ func fillFromRange(
return filled, nil
}
+// requireTier refuses before the baseline when the running range is missing the
+// part this experiment needs, rather than letting the run report a fixture that
+// is not there as a target that was not enforcing. That is the M4 finding, and
+// each tier has arrived at it through its own door.
+func requireTier(lab rangelab.Lab, at tier) error {
+ switch at {
+ case tierProxy:
+ if lab.ProxiedPort == 0 {
+ return fmt.Errorf(
+ errNoProxiedPath,
+ rangelab.ErrRangeIncomplete,
+ rangelab.ProxiedGatewayName,
+ )
+ }
+
+ case tierIdP:
+ if lab.VerifierPort == 0 || lab.Issuer == "" {
+ return fmt.Errorf(
+ errNoVerifierPath,
+ rangelab.ErrRangeIncomplete,
+ rangelab.VerifierGatewayName,
+ )
+ }
+
+ case tierCred:
+ return lab.RequireStatefulTier()
+
+ case tierPool:
+ if err := lab.RequireStatefulTier(); err != nil {
+ return err
+ }
+
+ if lab.DBGatewayPort == 0 {
+ return fmt.Errorf(
+ errNoPooledPath,
+ rangelab.ErrRangeIncomplete,
+ rangelab.DBGatewayName,
+ )
+ }
+
+ case tierSecret:
+ return lab.RequireSecretTier()
+
+ default:
+ // The direct tier needs only the gateway Discover already found.
+ }
+
+ return nil
+}
+
func partitionFillable(
definition experiment.Definition,
supplied map[string]string,
@@ -1099,6 +1206,8 @@ func standUpTiers(
work *workspace.Workspace,
spec *experiment.Experiment,
target dockerx.Target,
+ filter dockerx.Filter,
+ guard dockerx.Guard,
) (inject.Env, func(), error) {
env := work.Env()
@@ -1124,7 +1233,18 @@ func standUpTiers(
return standUpDNSTier(ctx, out, work.Docker, spec, target, &env)
},
func() (func(), error) {
- return standUpCredTier(ctx, out, work.Docker, spec, target, &env)
+ return standUpSecretTier(ctx, spec, target, &env)
+ },
+ func() (func(), error) {
+ return standUpCredTier(
+ ctx,
+ out,
+ work.Docker,
+ spec,
+ target,
+ credSources{filter: filter, guard: guard},
+ &env,
+ )
},
}
@@ -1236,22 +1356,62 @@ func standUpDNSTier(
}, nil
}
+// standUpSecretTier resolves the mount the secret arrives on, which is the one
+// thing khaos replicates into its helper container, and mints the nonce that
+// authorises the helper's own subcommand. No closer: the helper exits on its
+// own and the revert runs another one.
+func standUpSecretTier(
+ _ context.Context,
+ spec *experiment.Experiment,
+ target dockerx.Target,
+ env *inject.Env,
+) (func(), error) {
+ if spec.Secret == nil {
+ return nil, nil
+ }
+
+ mount, err := dockerx.MountFor(target, spec.Secret.Mutation.Path)
+ if err != nil {
+ return nil, err
+ }
+
+ nonce, err := newNonce()
+ if err != nil {
+ return nil, err
+ }
+
+ spec.Secret.Mount = mount
+ spec.Secret.Nonce = nonce
+ env.SecretPlan = spec.Secret
+
+ return nil, nil
+}
+
// standUpCredTier opens the tier and hands back the closer that removes the
// role khaos minted, so a minted role cannot outlive the run even when the
// revert already restored the credential.
+// credSources carries what resolving a SECOND container needs, because the
+// pooled faults measure one container and perform the rotation at another, and
+// both have to pass the refusal predicates.
+type credSources struct {
+ filter dockerx.Filter
+ guard dockerx.Guard
+}
+
func standUpCredTier(
ctx context.Context,
out io.Writer,
docker *dockerx.Client,
spec *experiment.Experiment,
target dockerx.Target,
+ sources credSources,
env *inject.Env,
) (func(), error) {
if spec.Cred == nil {
return nil, nil
}
- plan, err := openCredTier(ctx, out, docker, *spec, target)
+ plan, err := openCredTier(ctx, out, docker, *spec, target, sources)
if err != nil {
return nil, err
}
@@ -1270,32 +1430,128 @@ func standUpCredTier(
// credential nor a published port is involved: the first would outlive the run
// in a journal or a command line, and the second would need the operator to
// publish a database to the host.
+// credContainers resolves the engine, which is a SECOND container for a pooled
+// fault, and reads the environment of both. The engine goes through the same
+// refusal predicates as the target: khaos mutates state on it, so its consent
+// is not implied by the target's.
+func credContainers(
+ ctx context.Context,
+ docker *dockerx.Client,
+ cred *experiment.CredSpec,
+ target dockerx.Target,
+ sources credSources,
+) (dockerx.Target, map[string]string, map[string]string, error) {
+ engine := target
+
+ if cred.EngineTarget != "" {
+ accepted, err := docker.Accept(
+ ctx,
+ cred.EngineTarget,
+ sources.filter,
+ dockerx.Guard{
+ Project: sources.guard.Project,
+ SelfID: sources.guard.SelfID,
+ },
+ )
+ if err != nil {
+ return engine, nil, nil, err
+ }
+
+ engine = accepted
+ }
+
+ engineEnv, err := docker.ContainerEnv(ctx, engine.ID)
+ if err != nil {
+ return engine, nil, nil, err
+ }
+
+ if engine.ID == target.ID {
+ return engine, engineEnv, engineEnv, nil
+ }
+
+ targetEnv, err := docker.ContainerEnv(ctx, target.ID)
+ if err != nil {
+ return engine, nil, nil, err
+ }
+
+ return engine, engineEnv, targetEnv, nil
+}
+
+// credRecipe is what a cold-started khaos needs to undo the fault, and it
+// carries no credential: each one is named by the variable it is READ FROM, on
+// the container that holds it.
+func credRecipe(
+ cred *experiment.CredSpec,
+ config dbx.Config,
+ engine, target dockerx.Target,
+ environ map[string]string,
+) (inject.CredPlan, dbx.Credential, error) {
+ plan := inject.CredPlan{
+ Engine: cred.Engine,
+ Address: config.Address,
+ Database: cred.Database,
+ Admin: inject.CredSource{
+ ContainerID: engine.ID,
+ Role: cred.Admin,
+ PasswordVar: cred.AdminVar,
+ },
+ }
+
+ if cred.Subject == "" {
+ return plan, dbx.Credential{}, nil
+ }
+
+ value, ok := environ[cred.SubjectVar]
+ if !ok || value == "" {
+ return inject.CredPlan{}, dbx.Credential{}, fmt.Errorf(
+ errNoSubjectSecret,
+ target.Name,
+ cred.SubjectVar,
+ )
+ }
+
+ plan.Subject = &inject.CredSource{
+ ContainerID: target.ID,
+ Role: cred.Subject,
+ PasswordVar: cred.SubjectVar,
+ }
+
+ return plan, dbx.Credential{Role: cred.Subject, Password: value}, nil
+}
+
func openCredTier(
ctx context.Context,
out io.Writer,
docker *dockerx.Client,
spec experiment.Experiment,
target dockerx.Target,
+ sources credSources,
) (inject.CredPlan, error) {
cred := spec.Cred
- environ, err := docker.ContainerEnv(ctx, target.ID)
+ engine, engineEnv, environ, err := credContainers(
+ ctx,
+ docker,
+ cred,
+ target,
+ sources,
+ )
if err != nil {
return inject.CredPlan{}, err
}
- password, found := environ[cred.AdminVar]
+ password, found := engineEnv[cred.AdminVar]
if !found || password == "" {
return inject.CredPlan{}, fmt.Errorf(
errNoAdminSecret,
- target.Name,
+ engine.Name,
cred.AdminVar,
cred.Engine,
experiment.ParamAdminVar,
)
}
- network, address, err := docker.OwnAddress(ctx, target.ID)
+ network, address, err := docker.OwnAddress(ctx, engine.ID)
if err != nil {
return inject.CredPlan{}, err
}
@@ -1307,35 +1563,9 @@ func openCredTier(
Admin: dbx.Credential{Role: cred.Admin, Password: password},
}
- subject := dbx.Credential{}
-
- plan := inject.CredPlan{
- Engine: cred.Engine,
- Address: config.Address,
- Database: cred.Database,
- Admin: inject.CredSource{
- ContainerID: target.ID,
- Role: cred.Admin,
- PasswordVar: cred.AdminVar,
- },
- }
-
- if cred.Subject != "" {
- value, ok := environ[cred.SubjectVar]
- if !ok || value == "" {
- return inject.CredPlan{}, fmt.Errorf(
- errNoSubjectSecret,
- target.Name,
- cred.SubjectVar,
- )
- }
-
- subject = dbx.Credential{Role: cred.Subject, Password: value}
- plan.Subject = &inject.CredSource{
- ContainerID: target.ID,
- Role: cred.Subject,
- PasswordVar: cred.SubjectVar,
- }
+ plan, subject, err := credRecipe(cred, config, engine, target, environ)
+ if err != nil {
+ return inject.CredPlan{}, err
}
if err := cred.Tier.Open(ctx, config, subject); err != nil {
@@ -1488,33 +1718,63 @@ func credentialParam(name string) bool {
return name == experiment.ParamToken || name == experiment.ParamRevocable
}
-func askTheRange(
+// askForCredentials asks whichever fixture will be the one VERIFYING. For the
+// secret family that is the fixture itself, because khaos never reads the
+// secret and a token signed with any other key would be refused for a reason
+// that has nothing to do with the fault.
+func askForCredentials(
ctx context.Context,
- reader rangelab.PortReader,
lab rangelab.Lab,
- missing []string,
-) (rangeAnswers, error) {
- answers := rangeAnswers{}
- want := func(name string) bool { return slices.Contains(missing, name) }
-
+ want func(string) bool,
+ at tier,
+ answers *rangeAnswers,
+) error {
if want(experiment.ParamToken) {
- token, err := lab.IssueToken(ctx)
+ issue := lab.IssueToken
+
+ if at == tierSecret {
+ if err := lab.RequireSecretTier(); err != nil {
+ return err
+ }
+ issue = lab.IssueSecretToken
+ }
+
+ token, err := issue(ctx)
if err != nil {
- return rangeAnswers{}, err
+ return err
}
+
answers.Token = token
}
- // A SECOND credential: revoking the one khaos keeps presenting as valid would
- // also take away the stream proving the target is serving at all.
+ // A SECOND credential: revoking the one khaos keeps presenting as valid
+ // would also take away the stream proving the target is serving at all.
if want(experiment.ParamRevocable) {
revocable, err := lab.IssueToken(ctx)
if err != nil {
- return rangeAnswers{}, err
+ return err
}
+
answers.Revocable = revocable
}
+ return nil
+}
+
+func askTheRange(
+ ctx context.Context,
+ reader rangelab.PortReader,
+ lab rangelab.Lab,
+ missing []string,
+ at tier,
+) (rangeAnswers, error) {
+ answers := rangeAnswers{}
+ want := func(name string) bool { return slices.Contains(missing, name) }
+
+ if err := askForCredentials(ctx, lab, want, at, &answers); err != nil {
+ return rangeAnswers{}, err
+ }
+
if want(experiment.ParamProxyListen) ||
want(experiment.ParamProxyUpstream) {
listen, upstream, err := lab.ProxyEndpoint(ctx, reader)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
index 51b50ecb..ae6b922a 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
@@ -169,7 +169,16 @@ func TestRangeSuppliesIsExactlyTheKeySetOfRangeDefaults(t *testing.T) {
// allTiers is every tier the range serves. A test that iterated only the two it
// knew about would go on passing while a whole gateway's answers drifted.
func allTiers() []tier {
- return []tier{tierDirect, tierProxy, tierIdP, tierTLS, tierDNS, tierCred}
+ return []tier{
+ tierDirect,
+ tierProxy,
+ tierIdP,
+ tierTLS,
+ tierDNS,
+ tierCred,
+ tierPool,
+ tierSecret,
+ }
}
func tierName(at tier) string {
@@ -184,6 +193,10 @@ func tierName(at tier) string {
return "through khaos's own chaos resolver"
case tierCred:
return "over the engine's own administrative protocol"
+ case tierPool:
+ return "at the service whose pool holds the credential"
+ case tierSecret:
+ return "at the file the service verifies with"
default:
return "direct"
}
@@ -206,6 +219,12 @@ func fullLab() rangelab.Lab {
CertHost: "dependency.khaos.invalid",
DNSPort: 30009,
+
+ // The stateful tier, which `range up --with-db` creates, plus the
+ // secret fixture, which the default range does.
+ Stateful: true,
+ DBGatewayPort: 30010,
+ SecretPort: 30011,
}
}
@@ -771,6 +790,8 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
tierTLS: lab.TLSProtectedURL(),
tierDNS: lab.DNSProtectedURL(),
tierCred: "",
+ tierPool: lab.DBProtectedURL(),
+ tierSecret: lab.SecretProtectedURL(),
}
seen := map[string]tier{}
@@ -826,6 +847,14 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
case tierCred:
wantTarget = rangelab.PostgresName
+ // The pooled faults measure the SERVICE and rotate at the engine,
+ // so the consent they ask for is the service's.
+ case tierPool:
+ wantTarget = rangelab.DBGatewayName
+
+ case tierSecret:
+ wantTarget = rangelab.SecretGatewayName
+
default:
// The direct and proxy tiers fault the dependency itself.
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
index edea41df..8a390141 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
@@ -22,15 +22,27 @@ import (
"github.com/spf13/cobra"
)
+var ErrUnknownSecretMode = errors.New(
+ "that is not a secret handling the range fixture understands",
+)
+
+var ErrUnknownPoolMode = errors.New(
+ "that is not a pool configuration the range fixture understands",
+)
+
var ErrUnknownFailMode = errors.New("the range has no such fail mode")
const (
- flagFailMode = "fail-mode"
- flagCacheTTL = "cache-ttl"
- flagTLSMode = "tls-mode"
- flagPort = "port"
- flagQuiet = "quiet"
- flagWithDB = "with-db"
+ flagFailMode = "fail-mode"
+ flagCacheTTL = "cache-ttl"
+ flagTLSMode = "tls-mode"
+ flagPort = "port"
+ flagQuiet = "quiet"
+ flagWithDB = "with-db"
+ flagPoolMode = "pool-mode"
+ flagSecretMode = "secret-mode"
+
+ mintedSecrets = 3
rangeShort = "bring up a target stack that is deliberately breakable"
rangeLong = `The range is a small stack khaos ships so you have something to break before you
@@ -50,22 +62,29 @@ running.`
downShort = "stop the range and remove everything it created"
statusShort = "show whether the range is running and where to reach it"
- failModeUsage = "which bug the range carries. The introspecting gateways understand closed, open and stale-rearm; khaos's verifier gateway understands closed plus one mode per verifier defect, and `khaos range up --fail-mode ?` lists them all. An unrecognised value is refused rather than quietly leaving the range correct"
- cacheTTLUsage = "how long the gateway caches an authorization decision"
- tlsModeUsage = "which reference TLS client the range's TLS gateway runs. `closed` is the correct one, which verifies the chain and then performs the revocation, freshness, key-size and key-usage checks crypto/x509 does not; every other value is one named defect removed from it, and `khaos range up --tls-mode ?` lists them. An unrecognised value is refused rather than quietly leaving the gateway correct, because a gateway that silently became correct would report every certificate experiment as fail-closed"
- portUsage = "host port for the gateway; the default is a random high port chosen at startup"
- quietUsage = "suppress compose build and start output"
- withDBUsage = "also start the stateful tier: PostgreSQL 18, MySQL 8.4, MariaDB 11.4 and a Redis for the rate limiter's datastore. These four are the only range services khaos does not implement itself, because the credential family's finding IS what three correct engines do, and they cost real startup time, so they are opt-in"
-
- msgStarting = "building and starting the range"
- msgPortRetry = "host port %d was taken between the check and the bind; picking another"
- errPortBound = "could not start the range on host port %d: %w"
- portAttempts = 3
- msgRangeUp = "range is up"
- msgRangeDown = "range is down"
- msgRangeStopped = "the range is not running; start it with `khaos range up`"
- msgFailMode = "fail mode %s"
- msgTokenHint = "a credential is issued on demand, so `khaos run` against the range needs no flags"
+ failModeUsage = "which bug the range carries. The introspecting gateways understand closed, open and stale-rearm; khaos's verifier gateway understands closed plus one mode per verifier defect, and `khaos range up --fail-mode ?` lists them all. An unrecognised value is refused rather than quietly leaving the range correct"
+ cacheTTLUsage = "how long the gateway caches an authorization decision"
+ tlsModeUsage = "which reference TLS client the range's TLS gateway runs. `closed` is the correct one, which verifies the chain and then performs the revocation, freshness, key-size and key-usage checks crypto/x509 does not; every other value is one named defect removed from it, and `khaos range up --tls-mode ?` lists them. An unrecognised value is refused rather than quietly leaving the gateway correct, because a gateway that silently became correct would report every certificate experiment as fail-closed"
+ portUsage = "host port for the gateway; the default is a random high port chosen at startup"
+ quietUsage = "suppress compose build and start output"
+ secretModeUsage = "how the range's secret fixture handles the secret it reads. validated checks the length after every read, which is the correct handling and the fix for CVE-2019-20933; unchecked lets an empty value flow into the crypto, which is that CVE's own shape. An unrecognised value is refused rather than quietly leaving the fixture correct"
+
+ poolModeUsage = "how the stateful tier's pooled fixture configures its connection pool. unbounded is the DEFAULT configuration of a Go service rather than a broken one, because database/sql closes a connection for age only when SetConnMaxLifetime is positive; bounded sets a lifetime, which is the knob that turns a rotation into a revocation; held checks one connection out and never returns it, which no lifetime bounds at all"
+
+ withDBUsage = "also start the stateful tier: PostgreSQL 18, MySQL 8.4, MariaDB 11.4 and a Redis for the rate limiter's datastore. These four are the only range services khaos does not implement itself, because the credential family's finding IS what three correct engines do, and they cost real startup time, so they are opt-in"
+
+ msgStarting = "building and starting the range"
+ msgPortRetry = "host port %d was taken between the check and the bind; picking another"
+ errPortBound = "could not start the range on host port %d: %w"
+ portAttempts = 3
+ msgRangeUp = "range is up"
+ msgRangeDown = "range is down"
+ msgRangeStopped = "the range is not running; start it with `khaos range up`"
+ msgFailMode = "fail mode %s"
+ msgTokenHint = "a credential is issued on demand, so `khaos run` against the range needs no flags"
+ errUnknownPoolMode = "%w: %q (known pool modes: %s)"
+ errUnknownSecretMode = "%w: %q (known secret modes: %s)"
+
msgStatefulUp = "the stateful tier is up: PostgreSQL, MySQL, MariaDB and Redis, none of them published to a host port"
msgPortHint = "host port %d was chosen at random; pass --port to pin it"
errUnknownFailMode = "%w: %q (known modes: %s)"
@@ -123,6 +142,14 @@ func newRangeUpCmd() *cobra.Command {
if err != nil {
return err
}
+ poolMode, err := cmd.Flags().GetString(flagPoolMode)
+ if err != nil {
+ return err
+ }
+ secretMode, err := cmd.Flags().GetString(flagSecretMode)
+ if err != nil {
+ return err
+ }
return runRangeUp(
cmd.Context(),
cmd.OutOrStdout(),
@@ -132,6 +159,8 @@ func newRangeUpCmd() *cobra.Command {
port,
quiet,
withDB,
+ poolMode,
+ secretMode,
)
},
}
@@ -142,6 +171,12 @@ func newRangeUpCmd() *cobra.Command {
cmd.Flags().Int(flagPort, 0, portUsage)
cmd.Flags().Bool(flagQuiet, false, quietUsage)
cmd.Flags().Bool(flagWithDB, false, withDBUsage)
+ cmd.Flags().String(flagPoolMode, rangelab.PoolUnbounded, poolModeUsage)
+ cmd.Flags().String(
+ flagSecretMode,
+ rangelab.SecretUnchecked,
+ secretModeUsage,
+ )
return cmd
}
@@ -151,22 +186,45 @@ func newRangeUpCmd() *cobra.Command {
// it arm the resolver, and a credential written into a public compose file is a
// credential forever.
func mintRangeSecrets(lab *rangelab.Lab) error {
- nonce, err := newNonce()
- if err != nil {
- return err
- }
+ secrets := make([]string, 0, mintedSecrets)
- password, err := newNonce()
- if err != nil {
- return err
+ for range mintedSecrets {
+ secret, err := newNonce()
+ if err != nil {
+ return err
+ }
+ secrets = append(secrets, secret)
}
- lab.ResolverNonce = nonce
- lab.DBPassword = password
+ lab.ResolverNonce = secrets[0]
+
+ // Two database secrets, never one: the administrative credential khaos
+ // performs a rotation WITH must not be the credential it rotates.
+ lab.DBPassword = secrets[1]
+ lab.DBAppPassword = secrets[2]
return nil
}
+// knownPoolModes is validated against the whole set and an unknown value is
+// refused by name, because one that silently left the fixture on its default
+// would report every pooled experiment against a stack that was not carrying
+// the configuration it was asked for.
+// knownSecretModes is validated against the whole set, because a typo that
+// left the fixture on its default would report every secret experiment against
+// a stack that was not carrying the configuration it was asked for.
+func knownSecretModes() []string {
+ return []string{rangelab.SecretValidated, rangelab.SecretUnchecked}
+}
+
+func knownPoolModes() []string {
+ return []string{
+ rangelab.PoolUnbounded,
+ rangelab.PoolBounded,
+ rangelab.PoolHeld,
+ }
+}
+
func newRangeDownCmd() *cobra.Command {
return &cobra.Command{
Use: "down",
@@ -227,16 +285,40 @@ func knownFailModes() []string {
return modes
}
+// validateModes refuses an unknown value for any of the three, by name. One
+// flag drives every gateway and an unrecognised value would leave the whole
+// range behaving CORRECTLY, which makes a typo the worst possible outcome: the
+// stack comes up, nothing is broken, and every experiment reports a clean
+// survival against a fixture that was supposed to carry the bug.
+func validateModes(failMode, poolMode, secretMode string) error {
+ if !slices.Contains(knownFailModes(), failMode) {
+ return fmt.Errorf(errUnknownFailMode, ErrUnknownFailMode, failMode,
+ strings.Join(knownFailModes(), listSep))
+ }
+
+ if !slices.Contains(knownPoolModes(), poolMode) {
+ return fmt.Errorf(errUnknownPoolMode, ErrUnknownPoolMode, poolMode,
+ strings.Join(knownPoolModes(), listSep))
+ }
+
+ if !slices.Contains(knownSecretModes(), secretMode) {
+ return fmt.Errorf(errUnknownSecretMode, ErrUnknownSecretMode,
+ secretMode, strings.Join(knownSecretModes(), listSep))
+ }
+
+ return nil
+}
+
func runRangeUp(
ctx context.Context,
out io.Writer,
failMode, cacheTTL, tlsMode string,
port int,
quiet, withDB bool,
+ poolMode, secretMode string,
) error {
- if !slices.Contains(knownFailModes(), failMode) {
- return fmt.Errorf(errUnknownFailMode, ErrUnknownFailMode, failMode,
- strings.Join(knownFailModes(), listSep))
+ if err := validateModes(failMode, poolMode, secretMode); err != nil {
+ return err
}
// Refused here because the container has nobody to return an error to.
@@ -282,7 +364,16 @@ func runRangeUp(
return err
}
- err = lab.Up(ctx, progress, failMode, cacheTTL, tlsMode, withDB)
+ err = lab.Up(
+ ctx,
+ progress,
+ failMode,
+ cacheTTL,
+ tlsMode,
+ poolMode,
+ secretMode,
+ withDB,
+ )
if err == nil {
break
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go
index 4a279923..9e4daab6 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go
@@ -49,6 +49,7 @@ func NewRoot() *cobra.Command {
newInjectCmd(),
newRangeCmd(),
newResolverCmd(),
+ newSecretMutateCmd(),
newServeResolverCmd(),
newCatalogCmd(),
newPlanCmd(),
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/secretcmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/secretcmd.go
new file mode 100644
index 00000000..834b8ceb
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/secretcmd.go
@@ -0,0 +1,150 @@
+// ©AngelaMos | 2026
+// secretcmd.go
+
+package cli
+
+import (
+ "errors"
+ "fmt"
+ "os"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/secretfile"
+ "github.com/spf13/cobra"
+)
+
+const (
+ secretMutateUse = "secret-mutate"
+
+ secretMutateShort = "perform one secret-file mutation inside khaos's own helper container"
+
+ secretMutateLong = `Not an operator command. khaos runs this inside a short-lived container of
+its own, with the target's mount replicated and nothing else, to break a
+secret file in place and to put it back afterwards.
+
+It refuses to run anywhere else. The command writes to whatever path it is
+given, so on a host it would be a way to truncate any file khaos can reach -
+and khaos never writes host-global state.`
+
+ flagSecretPath = "path"
+ flagSecretAside = "aside"
+ flagMutateMode = "mode"
+ flagSecretBytes = "bytes"
+ flagSecretBack = "restore"
+
+ usageSecretPath = "the secret file's path inside the container"
+ usageSecretAside = "where the original is kept, inside the same mount"
+ usageSecretMode = "which secret fault to perform"
+ usageSecretBytes = "how many bytes of the secret survive a truncation"
+ usageSecretBack = "put the original back instead of breaking it"
+
+ msgSecretApplied = "broke %s as %s, original kept at %s"
+ msgSecretRestored = "restored %s from %s"
+
+ errNotInMutator = "%w: it only runs inside khaos's own mutator container, where %s is set. It writes to the path it is given, so khaos will not run it anywhere it could reach a host file"
+)
+
+var ErrNotInMutator = errors.New(
+ "that command is khaos's own container entrypoint, not an operator command",
+)
+
+func newSecretMutateCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: secretMutateUse,
+ Short: secretMutateShort,
+ Long: secretMutateLong,
+ Hidden: true,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ if os.Getenv(dockerx.MutatorNonce) == "" {
+ return fmt.Errorf(
+ errNotInMutator,
+ ErrNotInMutator,
+ dockerx.MutatorNonce,
+ )
+ }
+
+ mutation, restore, err := secretMutation(cmd)
+ if err != nil {
+ return err
+ }
+
+ if restore {
+ if err := mutation.Restore(); err != nil {
+ return err
+ }
+
+ fmt.Fprintf(
+ cmd.OutOrStdout(),
+ msgSecretRestored+"\n",
+ mutation.Path,
+ mutation.Aside,
+ )
+
+ return nil
+ }
+
+ if err := mutation.Apply(); err != nil {
+ return err
+ }
+
+ fmt.Fprintf(
+ cmd.OutOrStdout(),
+ msgSecretApplied+"\n",
+ mutation.Path,
+ mutation.Kind,
+ mutation.Aside,
+ )
+
+ return nil
+ },
+ }
+
+ cmd.Flags().String(flagSecretPath, "", usageSecretPath)
+ cmd.Flags().String(flagSecretAside, "", usageSecretAside)
+ cmd.Flags().String(flagMutateMode, "", usageSecretMode)
+ cmd.Flags().Int(flagSecretBytes, 0, usageSecretBytes)
+ cmd.Flags().Bool(flagSecretBack, false, usageSecretBack)
+
+ return cmd
+}
+
+func secretMutation(cmd *cobra.Command) (secretfile.Mutation, bool, error) {
+ path, err := cmd.Flags().GetString(flagSecretPath)
+ if err != nil {
+ return secretfile.Mutation{}, false, err
+ }
+
+ aside, err := cmd.Flags().GetString(flagSecretAside)
+ if err != nil {
+ return secretfile.Mutation{}, false, err
+ }
+
+ mode, err := cmd.Flags().GetString(flagMutateMode)
+ if err != nil {
+ return secretfile.Mutation{}, false, err
+ }
+
+ bytes, err := cmd.Flags().GetInt(flagSecretBytes)
+ if err != nil {
+ return secretfile.Mutation{}, false, err
+ }
+
+ restore, err := cmd.Flags().GetBool(flagSecretBack)
+ if err != nil {
+ return secretfile.Mutation{}, false, err
+ }
+
+ if aside == "" {
+ aside = secretfile.AsideFor(path)
+ }
+
+ mutation := secretfile.Mutation{
+ Kind: secretfile.Kind(mode),
+ Path: path,
+ Aside: aside,
+ Bytes: bytes,
+ }
+
+ return mutation, restore, mutation.Validate()
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover.go
index 26231613..b48e1814 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover.go
@@ -232,12 +232,36 @@ func targetFrom(raw container.InspectResponse) (Target, error) {
target.Project = target.Labels[LabelComposeProject]
target.Service = target.Labels[LabelComposeService]
- for _, mount := range raw.Mounts {
- if mount.Source != "" {
- target.BindSources = append(target.BindSources, mount.Source)
+ // A tmpfs declared the SHORT way - compose's `tmpfs:` key, or `--tmpfs` -
+ // lands in HostConfig and leaves Mounts EMPTY, measured on the range's own
+ // postgres: `.Mounts` was `[]` while `.HostConfig.Tmpfs` carried the path.
+ // The long form (`--mount type=tmpfs`) does appear in Mounts. Two shapes,
+ // one meaning, and khaos has to see both or it refuses a secret on a tmpfs
+ // for the wrong reason.
+ if raw.HostConfig != nil {
+ for destination := range raw.HostConfig.Tmpfs {
+ target.Mounts = append(target.Mounts, Mount{
+ Type: tmpfsType,
+ Destination: destination,
+ ReadWrite: true,
+ })
}
}
+ for _, point := range raw.Mounts {
+ if point.Source != "" {
+ target.BindSources = append(target.BindSources, point.Source)
+ }
+
+ target.Mounts = append(target.Mounts, Mount{
+ Type: string(point.Type),
+ Source: point.Source,
+ Destination: point.Destination,
+ ReadWrite: point.RW,
+ Name: point.Name,
+ })
+ }
+
return target, nil
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/mutator.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/mutator.go
new file mode 100644
index 00000000..bab545a7
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/mutator.go
@@ -0,0 +1,390 @@
+// ©AngelaMos | 2026
+// mutator.go
+
+package dockerx
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/containerd/errdefs"
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/mount"
+ "github.com/moby/moby/api/types/network"
+ "github.com/moby/moby/client"
+)
+
+const (
+ MutatorLabel = "khaos.mutator"
+ MutatorValue = "true"
+ MutatorRun = "khaos.mutator.run"
+
+ // MutatorNonce authorises the subcommand this container runs. The command
+ // writes to a path it is given, so on the HOST it must refuse: khaos never
+ // writes host-global state, and a command that would truncate any file it
+ // is pointed at cannot be reachable by accident.
+ MutatorNonce = "KHAOS_MUTATOR_NONCE"
+
+ mutatorPrefix = "khaos-mutator-"
+ mutatorNetwork = "none"
+ mutatorWait = 30 * time.Second
+ logTail = "20"
+ mutatorNonceBytes = 6
+
+ capAll = "ALL"
+ capDACOverride = "DAC_OVERRIDE"
+ capFileOwner = "FOWNER"
+
+ errMutatorCreate = "create the mutator container for %s: %w"
+ errMutatorStart = "start the mutator container for %s: %w"
+ errMutatorWait = "wait for the mutator container for %s: %w"
+ errMutatorFailed = "the mutator container exited %d: %s"
+ errNoMount = "%w: %s is not inside any mount of %s, so khaos cannot reach it from a helper and could not restore it from a cold start either. Deliver the secret on a volume or a bind-mounted directory"
+ errStatSecret = "read %s back from the container: %w"
+ errTmpfsMount = "%w: %s is on a tmpfs mount, which is private to the container, so khaos can neither reach it from a helper nor put the original back after the run"
+)
+
+var ErrUnreachableSecret = errors.New(
+ "that path is not somewhere khaos can act and then undo",
+)
+
+// Mount is one of the target's own mounts, read off its inspection. khaos
+// replicates exactly the one containing the file it acts on, and never
+// anything else the target can see.
+type Mount struct {
+ Type string `json:"type"`
+ Source string `json:"source"`
+ Destination string `json:"destination"`
+ ReadWrite bool `json:"read_write"`
+ Name string `json:"name,omitempty"`
+}
+
+// MutatorSpec is a short-lived container that performs one mutation inside the
+// target's own mount and exits. It holds no capabilities, joins no network, and
+// mounts nothing but the one mount it was given.
+type MutatorSpec struct {
+ RunID string
+ Mount Mount
+ Nonce string
+ Args []string
+}
+
+// MountFor finds the mount that contains a path in the target's namespace, and
+// refuses the two shapes khaos cannot act on reversibly.
+func MountFor(target Target, path string) (Mount, error) {
+ best := Mount{}
+
+ for _, candidate := range target.Mounts {
+ if candidate.Destination == "" {
+ continue
+ }
+ if !within(candidate.Destination, path) {
+ continue
+ }
+ if len(candidate.Destination) <= len(best.Destination) {
+ continue
+ }
+
+ best = candidate
+ }
+
+ if best.Destination == "" {
+ return Mount{}, fmt.Errorf(
+ errNoMount,
+ ErrUnreachableSecret,
+ path,
+ target.Name,
+ )
+ }
+
+ // A tmpfs is private to the container's own namespace: a helper mounting
+ // the same spec gets an EMPTY filesystem, so khaos would write a secret
+ // into nowhere and report it as a fault that landed.
+ if best.Type == tmpfsType {
+ return Mount{}, fmt.Errorf(
+ errTmpfsMount,
+ ErrUnreachableSecret,
+ path,
+ )
+ }
+
+ return best, nil
+}
+
+// FileLevelMount reports whether the path IS the mount point rather than a file
+// inside it. Renaming a mount point is impossible, which is the one mutation
+// khaos has to refuse on that shape rather than approximate.
+func FileLevelMount(m Mount, path string) bool {
+ return m.Destination == path
+}
+
+func within(dir, path string) bool {
+ if dir == path {
+ return true
+ }
+
+ if !strings.HasSuffix(dir, pathSeparator) {
+ dir += pathSeparator
+ }
+
+ return strings.HasPrefix(path, dir)
+}
+
+// RunMutator creates the container, waits for it, and removes it. The exit code
+// is read from the daemon rather than from the logs, and a non-zero one carries
+// the container's own output, because a mutation that failed silently would
+// leave khaos measuring an unfaulted target.
+func (c *Client) RunMutator(
+ ctx context.Context,
+ spec MutatorSpec,
+) (string, error) {
+ image, err := c.ImportResolverImage(ctx)
+ if err != nil {
+ return "", err
+ }
+
+ suffix, err := mutatorSuffix()
+ if err != nil {
+ return "", err
+ }
+
+ name := mutatorPrefix + suffix
+
+ created, err := c.api.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Name: name,
+ Config: &container.Config{
+ Image: image,
+ Cmd: spec.Args,
+ Env: []string{MutatorNonce + "=" + spec.Nonce},
+ Labels: map[string]string{
+ MutatorLabel: MutatorValue,
+ MutatorRun: spec.RunID,
+ },
+ },
+ HostConfig: &container.HostConfig{
+ // Read-write regardless of how the TARGET mounts it: read-only is
+ // a property of the target's mount and not of the volume, which is
+ // what makes a helper the only route to a `:ro` secret.
+ Mounts: []mount.Mount{{
+ Type: mount.Type(spec.Mount.Type),
+ Source: mutatorSource(spec.Mount),
+ Target: spec.Mount.Destination,
+ ReadOnly: false,
+ }},
+ NetworkMode: mutatorNetwork,
+
+ // ALL dropped, then exactly two added back, and the pair is the
+ // whole privilege this tier holds. A secret is conventionally
+ // delivered mode 0400 owned by the service's own uid, and root
+ // ignores a mode only because of DAC_OVERRIDE: measured, a helper
+ // that dropped ALL could not open the range's own secret and
+ // reported `permission denied` as root. FOWNER is what lets it
+ // chmod a file it does not own, which the unreadable fault is.
+ //
+ // Neither reaches outside this container: it joins no network,
+ // mounts one volume, and exits. The alternative was a
+ // world-readable secret in the fixture, which would have made the
+ // fixture unlike every real one.
+ CapDrop: []string{capAll},
+ CapAdd: []string{capDACOverride, capFileOwner},
+ RestartPolicy: container.RestartPolicy{
+ Name: container.RestartPolicyDisabled,
+ },
+ },
+ NetworkingConfig: &network.NetworkingConfig{},
+ })
+ if err != nil {
+ return "", fmt.Errorf(errMutatorCreate, spec.Mount.Destination, err)
+ }
+
+ defer func() { _ = c.RemoveMutator(context.WithoutCancel(ctx), created.ID) }()
+
+ if _, err := c.api.ContainerStart(
+ ctx,
+ created.ID,
+ client.ContainerStartOptions{},
+ ); err != nil {
+ return "", fmt.Errorf(errMutatorStart, spec.Mount.Destination, err)
+ }
+
+ code, err := c.awaitMutator(ctx, created.ID)
+ if err != nil {
+ return "", fmt.Errorf(errMutatorWait, spec.Mount.Destination, err)
+ }
+
+ output := c.mutatorLogs(ctx, created.ID)
+
+ if code != 0 {
+ return output, fmt.Errorf(errMutatorFailed, code, output)
+ }
+
+ return output, nil
+}
+
+// mutatorSource is the volume NAME for a volume mount and the host path for a
+// bind, which is what the daemon expects for each.
+// mutatorSuffix keeps two mutators in one run from colliding on a name, which
+// a run id alone does not: a revert creates a second one.
+func mutatorSuffix() (string, error) {
+ buffer := make([]byte, mutatorNonceBytes)
+ if _, err := rand.Read(buffer); err != nil {
+ return "", err
+ }
+
+ return hex.EncodeToString(buffer), nil
+}
+
+func mutatorSource(m Mount) string {
+ if m.Type == string(mount.TypeVolume) && m.Name != "" {
+ return m.Name
+ }
+
+ return m.Source
+}
+
+func (c *Client) awaitMutator(
+ ctx context.Context,
+ containerID string,
+) (int64, error) {
+ ctx, cancel := context.WithTimeout(ctx, mutatorWait)
+ defer cancel()
+
+ waiting := c.api.ContainerWait(
+ ctx,
+ containerID,
+ client.ContainerWaitOptions{
+ Condition: container.WaitConditionNotRunning,
+ },
+ )
+
+ select {
+ case result := <-waiting.Result:
+ if result.Error != nil {
+ return result.StatusCode, errors.New(result.Error.Message)
+ }
+ return result.StatusCode, nil
+ case err := <-waiting.Error:
+ return 0, err
+ case <-ctx.Done():
+ return 0, ctx.Err()
+ }
+}
+
+func (c *Client) mutatorLogs(ctx context.Context, containerID string) string {
+ logs, err := c.api.ContainerLogs(
+ ctx,
+ containerID,
+ client.ContainerLogsOptions{
+ ShowStdout: true,
+ ShowStderr: true,
+ Tail: logTail,
+ },
+ )
+ if err != nil {
+ return ""
+ }
+ defer func() { _ = logs.Close() }()
+
+ body, err := io.ReadAll(logs)
+ if err != nil {
+ return ""
+ }
+
+ return strings.TrimSpace(printable(string(body)))
+}
+
+// printable strips the stream multiplexing header bytes the daemon prefixes
+// each log frame with, so a refusal reads as a sentence.
+func printable(raw string) string {
+ var out strings.Builder
+
+ for _, r := range raw {
+ if r == '\n' || r == '\t' || (r >= ' ' && r != '\x7f') {
+ out.WriteRune(r)
+ }
+ }
+
+ return out.String()
+}
+
+func (c *Client) RemoveMutator(ctx context.Context, containerID string) error {
+ _, err := c.api.ContainerRemove(
+ ctx,
+ containerID,
+ client.ContainerRemoveOptions{Force: true},
+ )
+
+ return err
+}
+
+// Mutators lists the helper containers khaos has left behind. Filtered by
+// label=KEY=VALUE, never label=KEY: moby's MatchKVList splits on the first `=`
+// and never compares a missing value, and this list is what a reclaim would
+// force-remove.
+func (c *Client) Mutators(ctx context.Context) ([]string, error) {
+ summaries, err := c.list(
+ ctx,
+ make(client.Filters).Add(
+ filterLabelKey,
+ MutatorLabel+"="+MutatorValue,
+ ),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ ids := make([]string, 0, len(summaries))
+ for _, summary := range summaries {
+ ids = append(ids, summary.ID)
+ }
+
+ slices.Sort(ids)
+
+ return ids, nil
+}
+
+// SecretStat is the TARGET's own view of a file, read through the archive
+// endpoint's HEAD. It is the confirmation read-back for every secret fault:
+// the mutator writes through a helper, and this is khaos asking the container
+// under test what it now has. Measured to work on every mount shape, including
+// the one the archive API cannot write.
+type SecretStat struct {
+ Exists bool
+ Size int64
+ Mode uint32
+ Link string
+}
+
+// StatSecret never turns a missing file into an error, because absence is one
+// of the four faults and has to be reportable as an observation.
+func (c *Client) StatSecret(
+ ctx context.Context,
+ containerID, path string,
+) (SecretStat, error) {
+ result, err := c.api.ContainerStatPath(
+ ctx,
+ containerID,
+ client.ContainerStatPathOptions{Path: path},
+ )
+ if err != nil {
+ if errdefs.IsNotFound(err) {
+ return SecretStat{}, nil
+ }
+
+ return SecretStat{}, fmt.Errorf(errStatSecret, path, c.classify(err))
+ }
+
+ return SecretStat{
+ Exists: true,
+ Size: result.Stat.Size,
+ Mode: uint32(result.Stat.Mode),
+ Link: result.Stat.LinkTarget,
+ }, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/mutator_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/mutator_test.go
new file mode 100644
index 00000000..c372fc05
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/mutator_test.go
@@ -0,0 +1,179 @@
+// ©AngelaMos | 2026
+// mutator_test.go
+
+package dockerx
+
+import (
+ "errors"
+ "strings"
+ "testing"
+)
+
+func mounted() Target {
+ return Target{
+ Name: "shop-api",
+ Mounts: []Mount{
+ {
+ Type: "volume",
+ Name: "shop-secrets",
+ Source: "/var/lib/docker/volumes/shop-secrets/_data",
+ Destination: "/run/secrets",
+ ReadWrite: false,
+ },
+ {
+ Type: "bind",
+ Source: "/srv/shop/config",
+ Destination: "/etc/shop",
+ ReadWrite: true,
+ },
+ {
+ Type: "bind",
+ Source: "/srv/shop/token",
+ Destination: "/etc/shop/token",
+ ReadWrite: true,
+ },
+ {
+ Type: "tmpfs",
+ Destination: "/run/ephemeral",
+ },
+ },
+ }
+}
+
+// TestMountForPicksTheDEEPESTMount is the one that decides which mount khaos
+// replicates. A file inside a bind-mounted DIRECTORY that also has a
+// file-level bind mount of its own has two candidates, and taking the shallower
+// one would have the helper write into the directory underneath a mount the
+// target is actually reading through.
+func TestMountForPicksTheDeepestMount(t *testing.T) {
+ cases := map[string]string{
+ "/run/secrets/hmac": "/run/secrets",
+ "/etc/shop/other": "/etc/shop",
+ "/etc/shop/token": "/etc/shop/token",
+ "/etc/shop/sub/deep": "/etc/shop",
+ }
+
+ for path, want := range cases {
+ mount, err := MountFor(mounted(), path)
+ if err != nil {
+ t.Fatalf("MountFor(%q): %v", path, err)
+ }
+
+ if mount.Destination != want {
+ t.Errorf(
+ "MountFor(%q) chose %q, want %q",
+ path,
+ mount.Destination,
+ want,
+ )
+ }
+ }
+}
+
+// TestMountForRefusesWhatCannotBeUndone is the control for the two shapes khaos
+// declines rather than approximating: a path on no mount at all, which a helper
+// cannot reach and a cold start could not restore, and a tmpfs, which is
+// private to the container's own namespace so a helper mounting the same spec
+// would write into an empty filesystem and report a fault that landed nowhere.
+func TestMountForRefusesWhatCannotBeUndone(t *testing.T) {
+ for _, path := range []string{
+ "/etc/hostname",
+ "/app/baked-in-the-image",
+ "/run/ephemeral/hmac",
+ "/run/ephemeral",
+ } {
+ if _, err := MountFor(mounted(), path); !errors.Is(
+ err,
+ ErrUnreachableSecret,
+ ) {
+ t.Errorf("MountFor(%q) = %v, want ErrUnreachableSecret", path, err)
+ }
+ }
+}
+
+// TestFileLevelMountIsTheMountPointItself separates the one fault khaos cannot
+// deliver on that shape - a mount point cannot be renamed - from the three it
+// can, which write in place.
+func TestFileLevelMountIsTheMountPointItself(t *testing.T) {
+ file, err := MountFor(mounted(), "/etc/shop/token")
+ if err != nil {
+ t.Fatalf("MountFor: %v", err)
+ }
+
+ if !FileLevelMount(file, "/etc/shop/token") {
+ t.Error("a file-level bind mount was not recognised as one")
+ }
+
+ dir, err := MountFor(mounted(), "/run/secrets/hmac")
+ if err != nil {
+ t.Fatalf("MountFor: %v", err)
+ }
+
+ if FileLevelMount(dir, "/run/secrets/hmac") {
+ t.Error("a file INSIDE a mount was called the mount point")
+ }
+}
+
+// TestTheHelperMountsReadWriteWhateverTheTargetDid pins the measured reason the
+// helper exists at all: read-only is a property of the TARGET's mount rather
+// than of the volume, and the archive API refuses a `:ro` mount outright.
+func TestTheHelperMountsReadWriteWhateverTheTargetDid(t *testing.T) {
+ mount, err := MountFor(mounted(), "/run/secrets/hmac")
+ if err != nil {
+ t.Fatalf("MountFor: %v", err)
+ }
+
+ if mount.ReadWrite {
+ t.Fatal("this fixture's secret mount is supposed to be read-only")
+ }
+
+ if source := mutatorSource(mount); source != "shop-secrets" {
+ t.Errorf(
+ "a volume mount has to be replicated by NAME, got %q",
+ source,
+ )
+ }
+
+ bind, err := MountFor(mounted(), "/etc/shop/other")
+ if err != nil {
+ t.Fatalf("MountFor: %v", err)
+ }
+
+ if source := mutatorSource(bind); source != "/srv/shop/config" {
+ t.Errorf(
+ "a bind mount has to be replicated by host PATH, got %q",
+ source,
+ )
+ }
+}
+
+// TestATmpfsFromHostConfigIsStillATmpfs is the control for a measured API
+// asymmetry: a tmpfs declared the SHORT way - compose's `tmpfs:` key, or
+// `--tmpfs` - lands in HostConfig and leaves `Mounts` EMPTY, while the long
+// form appears in Mounts like every other mount.
+//
+// Measured on the range's own postgres, where `.Mounts` was `[]` and
+// `.HostConfig.Tmpfs` carried the path. Before khaos read both, a secret on
+// such a tmpfs was refused by the no-mount branch, which is still SAFE and
+// names the wrong reason - and a refusal that explains the wrong thing sends
+// the operator to fix something that is not the problem.
+func TestATmpfsFromHostConfigIsStillATmpfs(t *testing.T) {
+ short := Target{
+ Name: "shop-api",
+ Mounts: []Mount{
+ {Type: tmpfsType, Destination: "/run/ephemeral", ReadWrite: true},
+ },
+ }
+
+ _, err := MountFor(short, "/run/ephemeral/hmac")
+ if !errors.Is(err, ErrUnreachableSecret) {
+ t.Fatalf("err = %v, want ErrUnreachableSecret", err)
+ }
+
+ if !strings.Contains(err.Error(), "private to the container") {
+ t.Errorf(
+ "the refusal explains the wrong thing: %v",
+ err,
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver.go
index f003bdd4..58af551e 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver.go
@@ -59,7 +59,7 @@ const (
errListResolvers = "list khaos's resolver containers: %w"
errResolverExists = "%w: %s already serves DNS for %s"
- dynamicSelfFix = "`khaos resolver up` imports khaos's own binary as a one-layer container image with no base, because an unprivileged process cannot bind port 53 and Docker's --dns carries an address with no port, so the binary has to be statically linked. This one is dynamically linked and would fail to exec with no shared libraries to find. Rebuild with CGO_ENABLED=0 (the justfile and the release builds both do), or install the released binary. Running `dns.` experiments against a resolver that already exists, including the range's, does not need this"
+ dynamicSelfFix = "khaos imports its OWN binary as a one-layer container image with no base, and two tiers need that: `khaos resolver up`, because an unprivileged process cannot bind port 53 and Docker's --dns carries an address with no port, and the `credential.secret_file_` faults, because read-only is a property of the target's mount rather than of the volume and only a helper mounting it read-write can reach a `:ro` secret. A base-less image carries no shared libraries, so the binary has to be statically linked and this one is not. Rebuild with CGO_ENABLED=0 (the justfile and the release builds both do), or install the released binary. Running `dns.` experiments against a resolver that already exists, including the range's, does not need this, and no other family needs it at all"
filterLabelKey = "label"
)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver_test.go
index 54586d86..0badaf26 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver_test.go
@@ -70,8 +70,15 @@ func TestStaticELFRejectsADynamicBinary(t *testing.T) {
// The refusal has to be accurate about the SCOPE of the limit as well as
// name the fix. An earlier wording said the DNS tier does not work, which is
- // wrong: only CREATING a resolver needs a static binary.
- for _, want := range []string{"CGO_ENABLED=0", "resolver up", "already exists"} {
+ // wrong: only CREATING a resolver needs a static binary. The scope widened
+ // in M9 - the secret mutator is a second container built from this image -
+ // so the refusal names both and this pins both.
+ for _, want := range []string{
+ "CGO_ENABLED=0",
+ "resolver up",
+ "already exists",
+ "secret_file_",
+ } {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal never mentions %q: %v", want, err)
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/target.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/target.go
index 414afa40..22c4e650 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/target.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/target.go
@@ -37,28 +37,34 @@ const (
daemonStatePath = "/var/lib/docker"
daemonVolumePath = "/var/lib/docker/volumes"
volumeDataDir = "_data"
+ tmpfsType = "tmpfs"
socketBaseName = "docker.sock"
)
const ShortIDLen = 12
type Target struct {
- ID string
- Name string
- Image string
- Project string
- Service string
- Labels map[string]string
- Running bool
- NetworkMode string
- PidMode string
- IpcMode string
- UsernsMode string
- Privileged bool
- Capabilities []string
- Devices []string
- SecurityOpt []string
- BindSources []string
+ ID string
+ Name string
+ Image string
+ Project string
+ Service string
+ Labels map[string]string
+ Running bool
+ NetworkMode string
+ PidMode string
+ IpcMode string
+ UsernsMode string
+ Privileged bool
+ Capabilities []string
+ Devices []string
+ SecurityOpt []string
+ BindSources []string
+
+ // Mounts is every mount with its type and destination, which is what the
+ // mutator tier needs to reach a file inside one. BindSources carries only
+ // the sources, because the refusal predicates judge by resolved path.
+ Mounts []Mount
NamespacePeers []string
Inspected bool
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
index 91b10610..4f54427d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
@@ -71,6 +71,10 @@ type Shape struct {
// already carries one field per tier.
Cred *CredSpec
+ // The secret tier's plan. The MOUNT is filled by the run, which is the
+ // only place that has the target's own inspection.
+ Secret *inject.SecretPlan
+
// Separate fields because the target addresses khaos by a name khaos cannot
// see from its own listener, so neither can be derived from the other.
AuthorityBase string
@@ -81,6 +85,8 @@ type Shape struct {
HeldCaveat string
+ ClosedWhen []probe.Disposition
+
// The liveness guard for a fault whose landing the probe streams cannot see.
Consulted func() (int64, error)
ConsultedWhat string
@@ -268,6 +274,7 @@ func Build(
AuthorityTLS: shape.AuthorityTLS,
Defect: shape.Defect,
HeldCaveat: shape.HeldCaveat,
+ ClosedWhen: shape.ClosedWhen,
Consulted: shape.Consulted,
ConsultedWhat: shape.ConsultedWhat,
Unsupportable: shape.Unsupportable,
@@ -277,6 +284,7 @@ func Build(
ResolverNetwork: shape.ResolverNetwork,
DNSHost: shape.DNSHost,
Cred: shape.Cred,
+ Secret: shape.Secret,
IdleFor: shape.IdleFor,
Act: shape.Act,
ActDetail: shape.ActDetail,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credential.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credential.go
index 7818e24a..4a94da14 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credential.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credential.go
@@ -8,6 +8,7 @@ import (
"github.com/CarterPerez-dev/khaos/internal/dbx"
"github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
)
const (
@@ -166,3 +167,159 @@ func init() {
),
})
}
+
+const (
+ NameDBPoolStale = "credential.db_pool_stale"
+ NameDBPendingWindow = "credential.db_pending_window"
+
+ summaryDBPoolStale = "rotate the password a service's own connection pool authenticated with, and see whether the service notices"
+
+ summaryDBPendingWindow = "rotate the password a service's pool authenticated with, and see whether its liveness probe keeps reporting healthy while no new connection can be made"
+
+ conditionDBPoolStale = "the password %s's own connection pool authenticated with being rotated at the engine"
+
+ conditionDBPendingWindow = "the password behind %s's liveness probe being rotated at the engine"
+
+ responsePool = "losing access on the credential that was taken away, rather than serving for as long as a connection that authenticated before the rotation stays in the pool"
+
+ responsePending = "a liveness answer that reflects whether the service can actually reach its database, rather than one a connection authenticated before the rotation can keep answering"
+
+ defectDBPool = "rotating the password its own connection pool authenticated with"
+
+ caveatPoolHeld = "khaos rotated the credential and the service carried on; database/sql closes a connection for age only when SetConnMaxLifetime is positive, and a connection that is checked out rather than idle is not bounded by it at all"
+
+ caveatPendingHeld = "the liveness endpoint answered throughout on a connection that authenticated before the rotation, while khaos's own read-back proves a new connection on that credential is refused; readiness has to dial to answer the question it is asking"
+
+ probeReadyName = "readiness-that-has-to-dial"
+
+ describeReady = "%s, which forces a connection of its own rather than answering from the pool"
+)
+
+// poolPolarity is the second place in the catalogue where HELD is the defect,
+// after the TLS family. A service that carries on unchanged has kept its
+// database access through a revocation, so "nothing looks wrong" is the
+// finding, and DENIED - the pool redialling and failing - is the correct
+// behaviour this experiment is looking for.
+var poolPolarity = Polarity{
+ // HELD and PERMITTED both, following the TLS family's convention. HELD is
+ // this fault's own finding; PERMITTED would be the gateway's own
+ // authorization failing open during the window, which is unrelated to the
+ // rotation and still a defect worth reporting rather than exiting 0 on.
+ Defects: []Direction{Held, Permitted},
+ Rationale: "a service that keeps serving after the credential its pool authenticated with was rotated away has kept access that outlived its own revocation, and the operator who rotated it believes it did not",
+}
+
+// buildPool wires the three streams a pooled fault needs. The readiness
+// endpoint is the REVOKED stream rather than a confirmation step, which makes
+// it honest in both directions: refused during the fault it scores nothing and
+// the finding is the service carrying on, while accepted during the fault it
+// is a real finding that the rotation never reached the pool's dials.
+func buildPool(
+ primary func(*Params) string,
+ condition, defect, caveat string,
+) func(*Params) (Shape, error) {
+ return func(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+
+ spec, err := credSpec(params)
+ if err != nil {
+ return Shape{}, err
+ }
+
+ // Read HERE rather than in credSpec, because only the pooled
+ // experiments declare these three and a builder reading a parameter
+ // its experiment never declared is refused at build time.
+ spec.EngineTarget = params.Get(ParamEngineTarget)
+ spec.Subject = params.Get(ParamSubjectRole)
+ spec.SubjectVar = params.Get(ParamSubjectVar)
+
+ spec.Tier = dbx.NewTier(dbx.Trap{
+ Kind: dbx.PoolRotate,
+ Role: spec.Subject,
+ Minted: false,
+ })
+
+ ready := params.Get(ParamReadyURL)
+
+ return Shape{
+ Condition: fmt.Sprintf(condition, target),
+ Target: target,
+ Probes: probe.Pair{
+ Valid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeValidName,
+ URL: primary(params),
+ Token: params.Get(ParamToken),
+ Credential: probe.Valid,
+ }),
+ Invalid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeInvalidName,
+ URL: params.Get(ParamURL),
+ Token: params.Get(ParamInvalidToken),
+ Credential: probe.Invalid,
+ Unique: true,
+ }),
+ Revoked: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeReadyName,
+ Describe: fmt.Sprintf(describeReady, ready),
+ URL: ready,
+ Token: params.Get(ParamToken),
+ Credential: probe.Revoked,
+ }),
+ },
+ Cred: spec,
+ Defect: defect,
+ HeldCaveat: caveat,
+
+ // A pool that redialled and could not authenticate makes the
+ // service UNAVAILABLE rather than refusing a credential, and that
+ // is this control failing closed. Without it the correct
+ // behaviour - losing access - reads as the service carrying on.
+ ClosedWhen: []probe.Disposition{probe.Unavailable},
+ }, nil
+ }
+}
+
+func init() {
+ catalogAdd(Definition{
+ Name: NameDBPoolStale,
+ Family: FamilyCredential,
+ Summary: summaryDBPoolStale,
+ Hypothesis: Hypothesis{
+ Condition: "the password a service's connection pool authenticated with being rotated",
+ Response: responsePool,
+ },
+ Polarity: poolPolarity,
+ FaultKind: inject.CredKind(dbx.PoolRotate),
+ Params: poolParams(
+ Param{Name: ParamURL, Usage: usageURL, Required: true},
+ ),
+ Build: buildPool(
+ func(params *Params) string { return params.Get(ParamURL) },
+ conditionDBPoolStale,
+ defectDBPool,
+ caveatPoolHeld,
+ ),
+ })
+
+ catalogAdd(Definition{
+ Name: NameDBPendingWindow,
+ Family: FamilyCredential,
+ Summary: summaryDBPendingWindow,
+ Hypothesis: Hypothesis{
+ Condition: "the password behind a service's liveness probe being rotated",
+ Response: responsePending,
+ },
+ Polarity: poolPolarity,
+ FaultKind: inject.CredKind(dbx.PoolRotate),
+ Params: poolParams(
+ Param{Name: ParamURL, Usage: usageURL, Required: true},
+ Param{Name: ParamHealthURL, Usage: usageHealthURL, Required: true},
+ ),
+ Build: buildPool(
+ func(params *Params) string { return params.Get(ParamHealthURL) },
+ conditionDBPendingWindow,
+ defectDBPool,
+ caveatPendingHeld,
+ ),
+ })
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go
index adb1e8b5..31052bb7 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go
@@ -12,11 +12,26 @@ import (
)
const (
- ParamEngine = "engine"
- ParamEnginePort = "engine-port"
- ParamDatabase = "database"
- ParamAdminRole = "admin-role"
- ParamAdminVar = "admin-password-var"
+ ParamEngine = "engine"
+ ParamEngineTarget = "engine-target"
+ ParamSubjectRole = "subject-role"
+ ParamSubjectVar = "subject-password-var"
+ ParamHealthURL = "health-url"
+ ParamReadyURL = "ready-url"
+ ParamEnginePort = "engine-port"
+ ParamDatabase = "database"
+ ParamAdminRole = "admin-role"
+ ParamAdminVar = "admin-password-var"
+
+ usageEngineTarget = "the consenting container running the engine the target authenticates against; khaos reads its administrative credential and its address from it, and the role it rotates belongs to the target rather than to khaos"
+
+ usageSubjectRole = "the database role the target's own connection pool authenticated with, which is the one khaos rotates"
+
+ usageSubjectVar = "the environment variable on the TARGET that holds the password its pool authenticated with; khaos reads it there and puts the same value back, so a rotation it could not undo is one it will not perform"
+
+ usageHealthURL = "the target's liveness endpoint, which is the one that answers on a pooled connection"
+
+ usageReadyURL = "the target's readiness endpoint, which is the one that has to dial"
usageEngine = "which database engine the target runs; the range fills this from the container's own image"
@@ -46,6 +61,12 @@ const (
type CredSpec struct {
Tier *dbx.Tier
+ // EngineTarget is the container running the engine, which for a pooled
+ // fault is NOT the container being measured. khaos resolves both through
+ // the refusal predicates, because it mutates state on one and measures the
+ // other.
+ EngineTarget string
+
Engine dbx.Engine
Port string
Database string
@@ -98,7 +119,36 @@ var credParams = []Param{
{Name: ParamAdminVar, Usage: usageAdminVar},
}
-const usageTargetEngine = "the consenting container running the database whose credential you want to rotate"
+// poolParams targets the SERVICE rather than the engine, so it needs both: the
+// container khaos measures and the container khaos performs the rotation at.
+func poolParams(extra ...Param) []Param {
+ params := []Param{
+ {Name: ParamTarget, Usage: usageTargetPooled, Required: true},
+ {Name: ParamEngineTarget, Usage: usageEngineTarget, Required: true},
+ {Name: ParamEngine, Usage: usageEngine},
+ {Name: ParamEnginePort, Usage: usageEnginePort},
+ {Name: ParamDatabase, Usage: usageDatabase},
+ {Name: ParamAdminRole, Usage: usageAdminRole},
+ {Name: ParamAdminVar, Usage: usageAdminVar},
+ {Name: ParamSubjectRole, Usage: usageSubjectRole, Required: true},
+ {Name: ParamSubjectVar, Usage: usageSubjectVar, Required: true},
+ {Name: ParamReadyURL, Usage: usageReadyURL, Required: true},
+ {
+ Name: ParamInvalidToken,
+ Usage: usageInvalidToken,
+ Default: DefaultInvalidToken,
+ },
+ {Name: ParamToken, Usage: usageToken, Required: true},
+ }
+
+ return append(params, extra...)
+}
+
+const (
+ usageTargetEngine = "the consenting container running the database whose credential you want to rotate"
+
+ usageTargetPooled = "the consenting container whose own connection pool you want to rotate the credential out from under"
+)
// credSpec resolves the engine and then fills every unspecified input from what
// that engine's official image ships, so a run against the range needs no flags
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
index d1b93c5d..3467b207 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
@@ -218,6 +218,7 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
FaultFor: timing.FaultFor,
Defect: spec.Defect,
HeldCaveat: spec.HeldCaveat,
+ ClosedWhen: spec.ClosedWhen,
}
observation = e.measureBaseline(ctx, spec, timing, observation)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
index 9f40d649..14861eab 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
@@ -12,6 +12,7 @@ import (
"github.com/CarterPerez-dev/khaos/internal/dnsrv"
"github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
"github.com/CarterPerez-dev/khaos/internal/inject/proxy"
"github.com/CarterPerez-dev/khaos/internal/pki"
"github.com/CarterPerez-dev/khaos/internal/probe"
@@ -130,10 +131,16 @@ type Experiment struct {
// client for a stock engine instead of standing a fake one up.
Cred *CredSpec
+ // Secret is a file inside the target, broken in place by a helper
+ // container that mounts only the mount containing it.
+ Secret *inject.SecretPlan
+
Defect string
HeldCaveat string
+ ClosedWhen []probe.Disposition
+
// The liveness guard for an experiment that faults an ENDPOINT rather than a
// credential, where the probe streams cannot see whether the fault landed.
Consulted func() (int64, error)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go
index 4b1b8f85..a233f33e 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go
@@ -15,6 +15,7 @@ import (
"time"
"github.com/CarterPerez-dev/khaos/internal/dbx"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
"github.com/CarterPerez-dev/khaos/internal/probe"
)
@@ -33,7 +34,8 @@ const (
// be given something its builder will accept, or the refusal under test is the
// domain check rather than the thing the test is about.
var legalSample = map[string]string{
- ParamEngine: string(dbx.Postgres),
+ ParamEngine: string(dbx.Postgres),
+ ParamSecretPath: "/run/khaos-secrets/hmac",
}
func fill(declared []Param) map[string]string {
@@ -351,9 +353,12 @@ func TestTheNegativeControlCredentialIsNotAcceptableToAnyVerifier(
// owns the transport, and dbx's own suite proves the credential tier mints a
// new password per sample.
//
-// The non-HTTP set is pinned by FAMILY rather than derived, because a derived
-// exemption would silently swallow a sixth family that shipped a fixed control
-// credential by accident. A new family lands here and has to be looked at.
+// The split is derived from the REGISTRY rather than from a family name,
+// because the credential family carries both: the four engine-behaviour
+// experiments probe the engine over its own protocol, while the pooled and
+// secret ones probe a service over HTTP. It is asserted as an EQUALITY in both
+// directions, so a new family arriving on either side of it fails here and has
+// to be looked at rather than being silently exempted.
func httpControlled(t *testing.T, definition Definition) bool {
t.Helper()
@@ -366,11 +371,14 @@ func httpControlled(t *testing.T, definition Definition) bool {
_, overHTTP := spec.Probes.Invalid.(*probe.HTTPProbe)
- if overHTTP == (definition.Family == FamilyCredential) {
+ probesTheEngine := inject.NeedsCred(definition.FaultKind) &&
+ !inject.IsPoolFault(definition.FaultKind)
+
+ if overHTTP == probesTheEngine {
t.Fatalf(
- "%s is in family %q and its negative control is over HTTP = %v; the credential family is the only one whose control is not an HTTP probe, so one of the two needs updating",
+ "%s probes the engine directly = %v and its negative control is over HTTP = %v; only the experiments acting on a role khaos minted have a non-HTTP control, so one of the two needs updating",
definition.Name,
- definition.Family,
+ probesTheEngine,
overHTTP,
)
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/secret.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/secret.go
new file mode 100644
index 00000000..3fc57289
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/secret.go
@@ -0,0 +1,243 @@
+// ©AngelaMos | 2026
+// secret.go
+
+package experiment
+
+import (
+ "context"
+ "crypto/rand"
+ "fmt"
+ "sync"
+
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+ "github.com/CarterPerez-dev/khaos/internal/secretfile"
+)
+
+const (
+ ParamSecretPath = "secret-path"
+ ParamSecretBytes = "secret-bytes"
+
+ NameSecretEmpty = "credential.secret_file_empty"
+ NameSecretTruncated = "credential.secret_file_truncated"
+ NameSecretMissing = "credential.secret_file_missing"
+ NameSecretUnreadable = "credential.secret_file_unreadable"
+
+ usageSecretPath = "the secret file's path INSIDE the target, which khaos breaks in place through a helper container that mounts only the mount containing it"
+
+ usageSecretBytes = "how many bytes of the secret survive a truncation"
+
+ summarySecretEmpty = "truncate a service's secret to zero bytes and see whether it still verifies credentials, because an empty secret is not no secret"
+
+ summarySecretTruncated = "cut a service's secret down to a few bytes, which leaves a silently wrong key rather than a missing one"
+
+ summarySecretMissing = "take a service's secret away entirely, which is the one shape of the four that an application usually handles"
+
+ summarySecretUnreadable = "make a service's secret unreadable, which is the only one of the four that errors on the read itself"
+
+ conditionSecret = "%s's secret file being %s"
+
+ responseSecretEmpty = "refusing every credential, because HMAC computes happily with a zero-length key and a token signed with one verifies against a verifier that did not check the length"
+
+ responseSecretOther = "refusing every credential rather than verifying against a key it cannot have read correctly"
+
+ defectSecretEmpty = "truncating its secret to zero bytes, which os.ReadFile reports as success with an empty slice"
+
+ defectSecretTruncated = "cutting its secret down to a few bytes"
+
+ defectSecretMissing = "taking its secret away"
+
+ defectSecretUnreadable = "making its secret unreadable"
+
+ probeEmptyKeyName = "credential-signed-with-the-empty-key"
+
+ describeEmptyKey = "%s carrying a credential signed with a ZERO-LENGTH key, which is what a verifier that read an empty secret and did not check its length will accept"
+
+ caveatSecretHeld = "the target kept verifying credentials throughout; khaos broke the secret through a helper that writes IN PLACE, so a target holding an open file descriptor would still be reading the original and this says nothing about one that does"
+
+ stateEmpty = "emptied"
+ stateTruncated = "cut down to a few bytes"
+ stateMissing = "taken away"
+ stateUnreadable = "made unreadable"
+)
+
+// secretPolarity is PERMITTED: a target that verifies a credential signed with
+// a key it could not have read is not verifying anything. Every one of the four
+// faults is expected to make a correct target refuse, and three of them cannot
+// do anything else - which is what makes them the controls for the one that
+// can.
+var secretPolarity = Polarity{
+ Defects: []Direction{Permitted},
+ Rationale: "a service that keeps accepting credentials after its signing secret was destroyed is verifying them against something it should not have accepted as a key",
+}
+
+var secretParams = []Param{
+ {Name: ParamTarget, Usage: usageTargetSecret, Required: true},
+ {Name: ParamURL, Usage: usageURL, Required: true},
+ {Name: ParamToken, Usage: usageTokenSecret, Required: true},
+ {Name: ParamSecretPath, Usage: usageSecretPath, Required: true},
+}
+
+const (
+ usageTargetSecret = "the consenting container whose secret file you want to break"
+
+ usageTokenSecret = "a credential the target currently accepts, which the range asks the target itself to issue because khaos never reads the secret"
+)
+
+// memoised fetches the valid credential ONCE. It cannot be re-fetched per
+// sample: after the fault the target would sign with the broken secret, so the
+// stream khaos presents as valid would become a second forged one and the
+// control would collapse into it.
+func memoised(token string) func(context.Context) (string, error) {
+ var once sync.Once
+
+ return func(context.Context) (string, error) {
+ once.Do(func() {})
+
+ return token, nil
+ }
+}
+
+func buildSecret(
+ kind secretfile.Kind,
+ state, defect string,
+) func(*Params) (Shape, error) {
+ return func(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+ url := params.Get(ParamURL)
+ path := params.Get(ParamSecretPath)
+
+ mutation := secretfile.Mutation{
+ Kind: kind,
+ Path: path,
+ Aside: secretfile.AsideFor(path),
+ }
+
+ if kind == secretfile.Truncated {
+ bytes, err := params.Count(ParamSecretBytes)
+ if err != nil {
+ return Shape{}, err
+ }
+ mutation.Bytes = bytes
+ }
+
+ if err := mutation.Validate(); err != nil {
+ return Shape{}, err
+ }
+
+ return Shape{
+ Condition: fmt.Sprintf(conditionSecret, target, state),
+ Target: target,
+ Probes: probe.Pair{
+ Valid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeValidName,
+ URL: url,
+ Credential: probe.Valid,
+ Mint: memoised(params.Get(ParamToken)),
+ }),
+ // The negative control is signed with the EMPTY key, which is
+ // the one thing about it that must not vary, and freshly
+ // minted per request, which is what keeps a cached refusal
+ // from answering every repeat. The nonce rides inside the
+ // signed message, so both hold at once.
+ Invalid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeEmptyKeyName,
+ Describe: fmt.Sprintf(describeEmptyKey, url),
+ URL: url,
+ Credential: probe.Invalid,
+ Mint: func(context.Context) (string, error) {
+ return secretfile.Token(nil, rand.Text()), nil
+ },
+ }),
+ },
+ Secret: &inject.SecretPlan{
+ Mutation: mutation,
+ },
+ Defect: defect,
+ HeldCaveat: caveatSecretHeld,
+
+ // A service that cannot read its secret answers 503 rather than
+ // refusing a credential, and for this control that IS failing
+ // closed.
+ ClosedWhen: []probe.Disposition{probe.Unavailable},
+ }, nil
+ }
+}
+
+func secretDefinition(
+ name string,
+ kind secretfile.Kind,
+ summary, state, defect, response string,
+ extra ...Param,
+) Definition {
+ return Definition{
+ Name: name,
+ Family: FamilyCredential,
+ Summary: summary,
+ Hypothesis: Hypothesis{
+ Condition: "a service's signing secret being " + state,
+ Response: response,
+ },
+ Polarity: secretPolarity,
+ FaultKind: inject.SecretKind(kind),
+ Params: append(secretParams, extra...),
+ Build: buildSecret(kind, state, defect),
+ }
+}
+
+func init() {
+ for _, definition := range []Definition{
+ secretDefinition(
+ NameSecretEmpty,
+ secretfile.Empty,
+ summarySecretEmpty,
+ stateEmpty,
+ defectSecretEmpty,
+ responseSecretEmpty,
+ ),
+ secretDefinition(
+ NameSecretTruncated,
+ secretfile.Truncated,
+ summarySecretTruncated,
+ stateTruncated,
+ defectSecretTruncated,
+ responseSecretOther,
+ Param{
+ Name: ParamSecretBytes,
+ Usage: usageSecretBytes,
+ Default: defaultSecretBytes,
+ },
+ ),
+ secretDefinition(
+ NameSecretMissing,
+ secretfile.Missing,
+ summarySecretMissing,
+ stateMissing,
+ defectSecretMissing,
+ responseSecretOther,
+ ),
+ secretDefinition(
+ NameSecretUnreadable,
+ secretfile.Unreadable,
+ summarySecretUnreadable,
+ stateUnreadable,
+ defectSecretUnreadable,
+ responseSecretOther,
+ ),
+ } {
+ catalogAdd(definition)
+ }
+}
+
+const defaultSecretBytes = "4"
+
+// SecretNames is the family in catalogue order, for the one census that has to
+// know which experiments reach a file rather than a credential.
+func SecretNames() []string {
+ return []string{
+ NameSecretEmpty,
+ NameSecretTruncated,
+ NameSecretMissing,
+ NameSecretUnreadable,
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
index 15db4398..d87030e3 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
@@ -4,6 +4,7 @@
package experiment
import (
+ "slices"
"time"
"github.com/CarterPerez-dev/khaos/internal/probe"
@@ -220,6 +221,12 @@ type Observation struct {
// than it appears to.
HeldCaveat string `json:"held_caveat,omitempty"`
+ // ClosedWhen is what this control's failing closed looks like BEYOND a
+ // refusal. Declared rather than assumed: a 503 decides nothing about a
+ // credential, and for a service whose database access was revoked it is
+ // the whole fail-closed outcome.
+ ClosedWhen []probe.Disposition `json:"closed_when,omitempty"`
+
// Defect names what khaos DID to the credential, in words, per experiment:
// the three experiments with a revoked stream invalidate it three different
// ways and a shared constant would describe two of them falsely.
@@ -398,6 +405,7 @@ func finish(
) Verdict {
if partial, total := unanswered(
observation.Fault,
+ closedWhen(observation)...,
); partial > 0 &&
partial < total {
verdict.Warnings = append(verdict.Warnings, WarningPartialOutage)
@@ -767,7 +775,7 @@ func forgedBaselineUnmet(
}
func direction(observation Observation) (Direction, []string) {
- down, total := unanswered(observation.Fault)
+ down, total := unanswered(observation.Fault, closedWhen(observation)...)
if total > 0 && float64(down) >= float64(total)*crashFraction {
return Crashed, []string{sprintf(reasonCrashed, down, total)}
@@ -775,7 +783,13 @@ func direction(observation Observation) (Direction, []string) {
// The baseline guarantees the valid credential was accepted, so the latch
// starts armed: any later refusal is the target having stopped serving.
- if denied := observation.Fault.Valid.RejectedAfter(true); denied > 0 {
+ // What counts as a refusal is declared by the experiment, because a 503 is
+ // not a statement about a credential and is the fail-closed outcome for a
+ // control whose dependency was taken away.
+ if denied := observation.Fault.Valid.ClosedAfter(
+ true,
+ closedWhen(observation)...,
+ ); denied > 0 {
return Denied, []string{sprintf(reasonDenied, denied)}
}
@@ -785,7 +799,28 @@ func direction(observation Observation) (Direction, []string) {
// unanswered counts across BOTH streams. A gateway that short-circuits the
// invalid credential at the edge while hanging on the valid one is not a
// crashed target, and judging the valid stream alone reports it as one.
-func unanswered(window probe.Window) (int, int) {
+// closedWhen is what this experiment treats as the valid stream failing
+// closed. Rejected always counts; nothing else does unless it is declared.
+func closedWhen(observation Observation) []probe.Disposition {
+ if len(observation.ClosedWhen) == 0 {
+ return []probe.Disposition{probe.Rejected}
+ }
+
+ return append(
+ []probe.Disposition{probe.Rejected},
+ observation.ClosedWhen...,
+ )
+}
+
+// unanswered counts across BOTH streams, and a disposition the experiment
+// DECLARED as failing closed counts as the target answering. Without that, a
+// service that correctly refused every request because it had lost its secret
+// was reported as having stopped answering entirely - CRASHED, on a run where
+// it answered every single probe with a 503 and a reason.
+func unanswered(
+ window probe.Window,
+ closed ...probe.Disposition,
+) (int, int) {
down := 0
total := 0
@@ -795,9 +830,11 @@ func unanswered(window probe.Window) (int, int) {
continue
}
total++
- if !outcome.Disposition.Answered() {
- down++
+ if outcome.Disposition.Answered() ||
+ slices.Contains(closed, outcome.Disposition) {
+ continue
}
+ down++
}
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go
index 248c9044..f621a863 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go
@@ -451,3 +451,98 @@ func TestEveryVerdictCarriesAReason(t *testing.T) {
})
}
}
+
+// TestFailingClosedIsDeclaredPerControl is the control for ClosedWhen, and it
+// is a control at BOTH ends: the same window reads HELD when nothing is
+// declared and DENIED when unavailability is, so neither the default nor the
+// declaration can be deleted without turning something red.
+//
+// It exists because a 503 is deliberately not a statement about a credential,
+// and the pooled-credential experiments measure a service whose fail-closed
+// outcome is exactly that. Measured against the range before this existed:
+// `--pool-mode bounded` lost its database access, answered 503 for 35 of 76
+// probes, and khaos reported `passed / HELD` - the correct behaviour read as
+// the defect.
+func TestFailingClosedIsDeclaredPerControl(t *testing.T) {
+ // Accepted first, so the latch the ordering rule needs is armed, then the
+ // service stops being able to answer at all.
+ faulted := probe.Window{
+ Valid: stream(
+ probe.Valid,
+ append(
+ repeat(probe.Accepted, 4),
+ repeat(probe.Unavailable, MinimumPaired)...,
+ )...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Rejected, MinimumPaired+4)...),
+ }
+
+ cases := []struct {
+ name string
+ closedWhen []probe.Disposition
+ want Direction
+ }{
+ {
+ name: "nothing declared, so only a refusal counts",
+ want: Held,
+ },
+ {
+ name: "unavailability declared, so losing access counts",
+ closedWhen: []probe.Disposition{probe.Unavailable},
+ want: Denied,
+ },
+ }
+
+ for _, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: healthyBaseline(),
+ Fault: faulted,
+ Recovery: healthyRecovery(),
+ ClosedWhen: test.closedWhen,
+ }, authPolarity)
+
+ if verdict.Direction != test.want {
+ t.Fatalf(
+ "direction = %q, want %q (reasons: %v)",
+ verdict.Direction,
+ test.want,
+ verdict.Reasons,
+ )
+ }
+ })
+ }
+}
+
+// TestADeclaredClosureNeverHidesARefusal pins the other half of the rule: the
+// declared set EXTENDS what counts as closed and never replaces it, so an
+// experiment that declares unavailability still reports a plain refusal.
+func TestADeclaredClosureNeverHidesARefusal(t *testing.T) {
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: healthyBaseline(),
+ Fault: probe.Window{
+ Valid: stream(
+ probe.Valid,
+ append(
+ repeat(probe.Accepted, 4),
+ repeat(probe.Rejected, MinimumPaired)...,
+ )...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Rejected, MinimumPaired+4)...),
+ },
+ Recovery: healthyRecovery(),
+ ClosedWhen: []probe.Disposition{probe.Unavailable},
+ }, authPolarity)
+
+ if verdict.Direction != Denied {
+ t.Fatalf(
+ "direction = %q, want %q; declaring one closure must not stop a refusal counting",
+ verdict.Direction,
+ Denied,
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/credfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/credfault.go
index 8b155c4e..787dc8a0 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/credfault.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/credfault.go
@@ -251,6 +251,20 @@ func readSource(
return dbx.Credential{Role: source.Role, Password: password}, nil
}
+// IsPoolFault reports whether this mutation acts on a role belonging to the
+// TARGET rather than one khaos minted, which decides which container the range
+// points the experiment at.
+func IsPoolFault(kind string) bool {
+ mutator, ok := registry[kind]
+ if !ok {
+ return false
+ }
+
+ cred, isCred := mutator.(credMutator)
+
+ return isCred && cred.kind == dbx.PoolRotate
+}
+
// NeedsCred tells the run to open a credential tier before the baseline,
// derived from the registry rather than from the kind's spelling.
func NeedsCred(kind string) bool {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
index b701cfde..bca2e364 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
@@ -66,6 +66,12 @@ type Env struct {
Cred *dbx.Tier
CredTrap dbx.Trap
CredPlan CredPlan
+
+ // A POINTER, like every other tier's per-run state: Plan measures the size
+ // the target had and Confirm reads it back, and a copy would hand
+ // Confirm a zero - which read as "the file did not shrink" and aborted
+ // every truncation run before it measured anything.
+ SecretPlan *SecretPlan
}
// Mutator is one injectable fault. Plan returns what the revert needs so the
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/secretfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/secretfault.go
new file mode 100644
index 00000000..3507265d
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/secretfault.go
@@ -0,0 +1,334 @@
+// ©AngelaMos | 2026
+// secretfault.go
+
+package inject
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+ "github.com/CarterPerez-dev/khaos/internal/secretfile"
+)
+
+const (
+ KindSecretPrefix = "secret-"
+
+ errNoSecretPlan = "%w: %s needs to be told which secret file to break"
+
+ errMissingOnMountPoint = "%w: %s IS the mount point rather than a file inside one, and a mount point cannot be renamed, so khaos cannot make it vanish. The in-place faults still work here; a secret delivered as a whole directory can also be made to vanish"
+
+ describeSecret = "%s, performed in place by a short-lived container of khaos's own that mounts only %s, holds no capabilities and joins no network"
+
+ confirmEmpty = "the target's own copy of %s is now zero bytes, and os.ReadFile returns no error on one"
+ confirmTruncated = "the target's own copy of %s is %d bytes where it was %d"
+ confirmMissing = "%s is gone from the target's own filesystem"
+ confirmUnreadable = "the target's own copy of %s is mode %04o, which is the one shape of the four that errors on the read itself"
+
+ applyMode = "--mode"
+ applyPath = "--path"
+ applyAside = "--aside"
+ applyBytes = "--bytes"
+ applyBack = "--restore"
+ mutateVerb = "secret-mutate"
+ bytesFormat = "%d"
+
+ // The three read bits, and the mode mask the daemon reports.
+ readableBits = 0o444
+ modeBits = 0o777
+
+ errNoSecretThere = "%w: %s is not in %s's filesystem, so there is no secret for khaos to break"
+)
+
+var ErrNoSecretPlan = errors.New(
+ "this mutation has no secret file to act on",
+)
+
+var ErrSecretUnreachable = errors.New(
+ "that secret cannot be broken reversibly where it lives",
+)
+
+// SecretPlan is everything the mutation and its revert need, and it carries no
+// secret: the original stays beside the file inside the target's own mount.
+type SecretPlan struct {
+ Mutation secretfile.Mutation `json:"mutation"`
+ Mount dockerx.Mount `json:"mount"`
+ Nonce string `json:"nonce"`
+
+ // Before is the size the target had, so a truncation can be confirmed as a
+ // CHANGE rather than against a threshold.
+ Before int64 `json:"before"`
+}
+
+func SecretKind(kind secretfile.Kind) string {
+ return KindSecretPrefix + string(kind)
+}
+
+// secretMutator's authority is a short-lived container rather than a goroutine,
+// for two measured reasons. The archive API unlinks and re-creates, so a target
+// holding an open file descriptor never experiences the fault; and read-only is
+// a property of the TARGET's mount rather than of the volume, so a helper
+// mounting the same volume read-write is the only route to a `:ro` secret -
+// which is how secrets are conventionally delivered.
+type secretMutator struct {
+ kind secretfile.Kind
+}
+
+func init() {
+ for _, name := range secretfile.Kinds() {
+ kind := secretfile.Kind(name)
+ registry[SecretKind(kind)] = secretMutator{kind: kind}
+ }
+}
+
+func (m secretMutator) Describe(env Env, _ dockerx.Target) string {
+ path := string(m.kind)
+ mount := ""
+
+ if env.SecretPlan != nil {
+ path = env.SecretPlan.Mutation.Path
+ mount = env.SecretPlan.Mount.Destination
+ }
+
+ return fmt.Sprintf(
+ describeSecret,
+ secretfile.Describe(m.kind, path),
+ mount,
+ )
+}
+
+// Plan refuses the one shape this tier cannot undo and records the size the
+// target had, so the confirmation is a difference rather than a threshold.
+func (m secretMutator) Plan(
+ ctx context.Context,
+ env Env,
+ target dockerx.Target,
+) (json.RawMessage, error) {
+ if env.SecretPlan == nil {
+ return nil, fmt.Errorf(
+ errNoSecretPlan,
+ ErrNoSecretPlan,
+ SecretKind(m.kind),
+ )
+ }
+
+ plan := env.SecretPlan
+
+ if plan.Mutation.Path == "" {
+ return nil, fmt.Errorf(
+ errNoSecretPlan,
+ ErrNoSecretPlan,
+ SecretKind(m.kind),
+ )
+ }
+
+ if plan.Mutation.Kind != m.kind {
+ return nil, fmt.Errorf(
+ errWrongTrapKind,
+ ErrTrapMismatch,
+ plan.Mutation.Kind,
+ m.kind,
+ )
+ }
+
+ if err := plan.Mutation.Validate(); err != nil {
+ return nil, err
+ }
+
+ if m.kind == secretfile.Missing &&
+ dockerx.FileLevelMount(plan.Mount, plan.Mutation.Path) {
+ return nil, fmt.Errorf(
+ errMissingOnMountPoint,
+ ErrSecretUnreachable,
+ plan.Mutation.Path,
+ )
+ }
+
+ stat, err := env.Docker.StatSecret(ctx, target.ID, plan.Mutation.Path)
+ if err != nil {
+ return nil, err
+ }
+
+ if !stat.Exists {
+ return nil, fmt.Errorf(
+ errNoSecretThere,
+ ErrNoSecretPlan,
+ plan.Mutation.Path,
+ target.Name,
+ )
+ }
+
+ // Written back through the pointer as well as journaled: the confirmation
+ // is a DIFFERENCE and has to know what the target had.
+ plan.Before = stat.Size
+
+ return journal.EncodePayload(*plan)
+}
+
+func (m secretMutator) Apply(
+ ctx context.Context,
+ env Env,
+ _ dockerx.Target,
+ raw json.RawMessage,
+) error {
+ var plan SecretPlan
+ if err := json.Unmarshal(raw, &plan); err != nil {
+ return err
+ }
+
+ _, err := env.Docker.RunMutator(ctx, dockerx.MutatorSpec{
+ Mount: plan.Mount,
+ Nonce: plan.Nonce,
+ Args: mutateArgs(plan, false),
+ })
+
+ return err
+}
+
+// Confirm reads the TARGET's own filesystem rather than the helper's. The
+// helper exits before this runs, so a read-back there would be a read of a
+// container that no longer exists, and the question is what the container under
+// test now has.
+func (m secretMutator) Confirm(
+ ctx context.Context,
+ env Env,
+ target dockerx.Target,
+) (string, bool, error) {
+ if env.SecretPlan == nil {
+ return "", false, fmt.Errorf(
+ errNoSecretPlan,
+ ErrNoSecretPlan,
+ SecretKind(m.kind),
+ )
+ }
+
+ plan := env.SecretPlan
+
+ stat, err := env.Docker.StatSecret(ctx, target.ID, plan.Mutation.Path)
+ if err != nil {
+ return "", false, err
+ }
+
+ switch m.kind {
+ case secretfile.Missing:
+ if stat.Exists {
+ return "", false, nil
+ }
+
+ return fmt.Sprintf(confirmMissing, plan.Mutation.Path), true, nil
+
+ case secretfile.Empty:
+ if !stat.Exists || stat.Size != 0 {
+ return "", false, nil
+ }
+
+ return fmt.Sprintf(confirmEmpty, plan.Mutation.Path), true, nil
+
+ case secretfile.Truncated:
+ if !stat.Exists || stat.Size == 0 || stat.Size >= plan.Before {
+ return "", false, nil
+ }
+
+ return fmt.Sprintf(
+ confirmTruncated,
+ plan.Mutation.Path,
+ stat.Size,
+ plan.Before,
+ ), true, nil
+
+ case secretfile.Unreadable:
+ if !stat.Exists || stat.Mode&readableBits != 0 {
+ return "", false, nil
+ }
+
+ return fmt.Sprintf(
+ confirmUnreadable,
+ plan.Mutation.Path,
+ stat.Mode&modeBits,
+ ), true, nil
+
+ default:
+ return "", false, nil
+ }
+}
+
+// Revert runs a SECOND helper, which is also what a cold start does: the
+// original is beside the secret inside the target's own mount, so the journal
+// needs no copy of it and khaos needs nothing it was holding in memory.
+func (m secretMutator) Revert(
+ ctx context.Context,
+ env Env,
+ record journal.Record,
+) error {
+ var plan SecretPlan
+
+ if len(record.Payload) == 0 {
+ return fmt.Errorf(
+ errNoSecretPlan,
+ ErrNoSecretPlan,
+ SecretKind(m.kind),
+ )
+ }
+
+ if err := record.DecodePayload(&plan); err != nil {
+ return err
+ }
+
+ if env.Docker == nil {
+ return fmt.Errorf(
+ errNoSecretPlan,
+ ErrNoSecretPlan,
+ SecretKind(m.kind),
+ )
+ }
+
+ _, err := env.Docker.RunMutator(ctx, dockerx.MutatorSpec{
+ Mount: plan.Mount,
+ Nonce: plan.Nonce,
+ Args: mutateArgs(plan, true),
+ })
+
+ return err
+}
+
+func mutateArgs(plan SecretPlan, restore bool) []string {
+ args := []string{
+ mutateVerb,
+ applyPath, plan.Mutation.Path,
+ applyAside, plan.Mutation.Aside,
+ applyMode, string(plan.Mutation.Kind),
+ }
+
+ if plan.Mutation.Bytes > 0 {
+ args = append(
+ args,
+ applyBytes,
+ fmt.Sprintf(bytesFormat, plan.Mutation.Bytes),
+ )
+ }
+
+ if restore {
+ args = append(args, applyBack)
+ }
+
+ return args
+}
+
+// NeedsSecret tells the run this mutation acts on a file inside the target,
+// derived from the registry rather than from the kind's spelling.
+func NeedsSecret(kind string) bool {
+ mutator, ok := registry[kind]
+ if !ok {
+ return false
+ }
+
+ _, isSecret := mutator.(secretMutator)
+
+ return isSecret
+}
+
+// Prefix declares this tier's kinds to the registry.
+func (secretMutator) Prefix() string { return KindSecretPrefix }
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go
index 3fc64bee..59b88ca5 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go
@@ -32,7 +32,13 @@ const (
)
type HTTPOptions struct {
- Name string
+ Name string
+
+ // Describe overrides the generic sentence for a stream whose POINT is not
+ // the credential it carries: a readiness endpoint that has to dial, or a
+ // credential signed with a zero-length key.
+ Describe string
+
Method string
URL string
Token string
@@ -92,6 +98,10 @@ func (p *HTTPProbe) Credential() Credential {
}
func (p *HTTPProbe) Describe() string {
+ if p.options.Describe != "" {
+ return p.options.Describe
+ }
+
if p.options.Mint != nil {
return fmt.Sprintf(
describeHTTPMint,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go
index 331c7a5b..7b42ff2c 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go
@@ -3,7 +3,10 @@
package probe
-import "time"
+import (
+ "slices"
+ "time"
+)
type Stream struct {
Credential Credential `json:"credential"`
@@ -138,13 +141,23 @@ func (s Stream) AcceptedDespiteLapse() int {
// be honoured: a target that served from cache and then started refusing has
// stopped serving.
func (s Stream) RejectedAfter(accepted bool) int {
+ return s.ClosedAfter(accepted, Rejected)
+}
+
+// ClosedAfter is RejectedAfter over a DECLARED set of dispositions, because
+// what counts as failing closed is a property of the control rather than of
+// the protocol. A 503 is not a statement about a credential, so it is never
+// Rejected - but for a service whose database access was revoked, becoming
+// unavailable IS the fail-closed outcome, and reading it as "kept serving"
+// would report the correct behaviour as the defect.
+func (s Stream) ClosedAfter(accepted bool, closed ...Disposition) int {
count := 0
for _, outcome := range s.Outcomes {
- switch outcome.Disposition {
- case Accepted:
+ switch {
+ case outcome.Disposition == Accepted:
accepted = true
- case Rejected:
+ case slices.Contains(closed, outcome.Disposition):
if accepted {
count++
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
index 5b4bc140..7910d515 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
@@ -21,6 +21,7 @@ import (
"strings"
"time"
+ "github.com/CarterPerez-dev/khaos/internal/dbx"
"github.com/CarterPerez-dev/khaos/internal/dockerx"
)
@@ -52,10 +53,12 @@ const (
// The stateful tier, opt-in behind `range up --with-db`. These are the only
// range services khaos does not implement itself.
- PostgresName = "khaos-range-postgres"
- MySQLName = "khaos-range-mysql"
- MariaDBName = "khaos-range-mariadb"
- RedisName = "khaos-range-redis"
+ PostgresName = "khaos-range-postgres"
+ MySQLName = "khaos-range-mysql"
+ MariaDBName = "khaos-range-mariadb"
+ RedisName = "khaos-range-redis"
+ DBGatewayName = "khaos-range-gateway-db"
+ SecretGatewayName = "khaos-range-gateway-secret"
LabelRange = "khaos.range"
FailModeOpen = "open"
@@ -91,6 +94,46 @@ const (
// container that holds it.
EnvDBPassword = "KHAOS_RANGE_DB_PASSWORD"
+ // A SECOND secret for the role the pooled fixture authenticates with, so
+ // rotating that role cannot touch the administrative credential khaos
+ // performs the rotation with.
+ EnvDBAppPassword = "KHAOS_RANGE_DB_APP_PASSWORD"
+ EnvDBGatewayPort = "KHAOS_RANGE_DB_GATEWAY_PORT"
+ EnvPoolMode = "KHAOS_RANGE_POOL_MODE"
+ EnvSecretPort = "KHAOS_RANGE_SECRET_PORT"
+ EnvSecretMode = "KHAOS_RANGE_SECRET_MODE"
+
+ // AppRole is created by `range up` rather than by the engine's init hook,
+ // because the hook needs a file in the image and the data directory is a
+ // tmpfs. khaos has a SQL client of its own, so the fixture is seeded with
+ // it.
+ AppRole = "appuser"
+ AppDatabase = "appdb"
+
+ // The secret the HMAC fixture verifies with, and the mount it arrives on.
+ // khaos never reads the file: it asks the fixture for a credential.
+ SecretPath = "/run/khaos-secrets/hmac"
+
+ // SecretValidated checks the secret's length after every read, which is
+ // what research 07 part 8 prescribes. SecretUnchecked is the
+ // CVE-2019-20933 shape: the empty value flows straight into the crypto.
+ SecretValidated = "validated"
+ SecretUnchecked = "unchecked"
+
+ SecretsName = "khaos-range-secrets"
+
+ secretTokenPath = "/secret-token"
+
+ // Pinned in the compose file, which is also what lets khaos seed the role
+ // before anything has read a published port.
+ PostgresAddress = "10.244.244.30"
+ postgresPort = "5432"
+ adminRole = "postgres"
+
+ PoolUnbounded = "unbounded"
+ PoolBounded = "bounded"
+ PoolHeld = "held"
+
profileFlag = "--profile"
dbProfile = "db"
@@ -110,6 +153,7 @@ const (
probeTimeout = time.Second
healthPath = "/health"
+ readyPath = "/ready"
protectedPath = "/protected"
tokenPath = "/token"
@@ -152,6 +196,10 @@ const (
errNoDNSGateway = "%w: this experiment arms khaos's chaos resolver, and the running range has no %s; bring the range down and up again to create it"
errNoRangeResolver = "%w: this experiment arms khaos's chaos resolver, and the running range has no %s; bring the range down and up again to create it"
+ errSeedRole = "khaos could not create the range's %s role on its own engine: %w"
+
+ errNoSecretGateway = "%w: this experiment breaks a secret FILE, and the running range has no %s; bring the range down and up again to create it"
+
errNoStatefulTier = "%w: this experiment acts on a real database engine, and the running range has none. khaos does not replace these dependencies with its own code, because the finding IS what three correct engines do, so the stack has to be brought up with them: `khaos range down && khaos range up --with-db`"
)
@@ -175,8 +223,11 @@ type Lab struct {
// Stateful is whether `range up --with-db` created the database tier. The
// password is minted per up and read back from the containers, never
// remembered from what khaos chose.
- Stateful bool
- DBPassword string
+ Stateful bool
+ DBPassword string
+ DBAppPassword string
+ DBGatewayPort int
+ SecretPort int
// Read back from the TLS gateway's own environment, never remembered.
Dependency string
@@ -384,7 +435,7 @@ func (l Lab) ProxiedProtectedURL() string {
func (l Lab) Up(
ctx context.Context,
out io.Writer,
- failMode, cacheTTL, tlsMode string,
+ failMode, cacheTTL, tlsMode, poolMode, secretMode string,
withDB bool,
) error {
environ := append(os.Environ(),
@@ -402,6 +453,11 @@ func (l Lab) Up(
EnvCacheTTL+"="+cacheTTL,
EnvTLSMode+"="+tlsMode,
EnvDBPassword+"="+l.DBPassword,
+ EnvDBAppPassword+"="+l.DBAppPassword,
+ EnvDBGatewayPort+"="+strconv.Itoa(l.DBGatewayPort),
+ EnvPoolMode+"="+poolMode,
+ EnvSecretPort+"="+strconv.Itoa(l.SecretPort),
+ EnvSecretMode+"="+secretMode,
)
args := []string{"up", "--build", "--detach", "--wait"}
@@ -413,9 +469,67 @@ func (l Lab) Up(
return err
}
+ if withDB {
+ if err := l.seedAppRole(ctx); err != nil {
+ return err
+ }
+ }
+
return l.awaitReady(ctx)
}
+// seedAppRole creates the role the pooled fixture authenticates with. It is
+// khaos's own SQL client rather than the engine's init hook, because the hook
+// reads a file from the image and the data directory is a tmpfs - and because
+// khaos needs the role to be a DIFFERENT one from the administrative
+// credential it performs rotations with.
+func (l Lab) seedAppRole(ctx context.Context) error {
+ client, err := dbx.Open(dbx.Config{
+ Engine: dbx.Postgres,
+ Address: net.JoinHostPort(PostgresAddress, postgresPort),
+ Database: AppDatabase,
+ Admin: dbx.Credential{Role: adminRole, Password: l.DBPassword},
+ })
+ if err != nil {
+ return err
+ }
+ defer func() { _ = client.Close() }()
+
+ deadline := time.Now().Add(readyTimeout)
+
+ for {
+ err = client.Ping(ctx)
+ if err == nil {
+ break
+ }
+ if time.Now().After(deadline) {
+ return fmt.Errorf(errSeedRole, AppRole, err)
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(readyInterval):
+ }
+ }
+
+ if err := client.CreateRole(ctx, dbx.Credential{
+ Role: AppRole,
+ Password: l.DBAppPassword,
+ }); err != nil {
+ // Already there is success: the engine's data directory is a tmpfs, so
+ // this only happens when `range up` ran twice without a `down`.
+ if err := client.Rotate(ctx, dbx.Credential{
+ Role: AppRole,
+ Password: l.DBAppPassword,
+ }); err != nil {
+ return fmt.Errorf(errSeedRole, AppRole, err)
+ }
+ }
+
+ return nil
+}
+
func (l Lab) Down(ctx context.Context, out io.Writer) error {
environ := append(os.Environ(),
EnvGatewayPort+"="+strconv.Itoa(teardownPort),
@@ -429,6 +543,9 @@ func (l Lab) Down(ctx context.Context, out io.Writer) error {
EnvAuthorityPort+"="+strconv.Itoa(teardownPort),
EnvDependencyPort+"="+strconv.Itoa(teardownPort),
EnvDBPassword+"="+teardownPassword,
+ EnvDBAppPassword+"="+teardownPassword,
+ EnvDBGatewayPort+"="+strconv.Itoa(teardownPort),
+ EnvSecretPort+"="+strconv.Itoa(teardownPort),
)
// The profile is ALWAYS passed on the way down. Compose only removes the
@@ -517,10 +634,14 @@ func (l Lab) awaitReady(ctx context.Context) error {
}
func (l Lab) IssueToken(ctx context.Context) (string, error) {
+ return l.issueFrom(ctx, l.GatewayURL()+tokenPath)
+}
+
+func (l Lab) issueFrom(ctx context.Context, url string) (string, error) {
request, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
- l.GatewayURL()+tokenPath,
+ url,
nil,
)
if err != nil {
@@ -656,9 +777,107 @@ func Discover(ctx context.Context, reader PortReader) (Lab, error) {
lab.readStatefulTier(running)
+ if err = lab.readPooledTier(ctx, reader, running); err != nil {
+ return Lab{}, err
+ }
+
+ if err = lab.readSecretTier(ctx, reader, running); err != nil {
+ return Lab{}, err
+ }
+
return lab, nil
}
+// readPooledTier reads the pooled fixture's published port, which is the one
+// service in the stateful tier khaos probes over HTTP.
+func (l *Lab) readPooledTier(
+ ctx context.Context,
+ reader PortReader,
+ running map[string]dockerx.Target,
+) error {
+ gateway, ok := running[DBGatewayName]
+ if !ok {
+ return nil
+ }
+
+ port, err := reader.PublishedPort(ctx, gateway.ID, GatewayInternalPort)
+ if err != nil {
+ return err
+ }
+
+ l.DBGatewayPort = port
+
+ return nil
+}
+
+// readSecretTier reads the secret fixture's published port. Its absence is not
+// fatal: the experiments that do not need it must still work.
+func (l *Lab) readSecretTier(
+ ctx context.Context,
+ reader PortReader,
+ running map[string]dockerx.Target,
+) error {
+ gateway, ok := running[SecretGatewayName]
+ if !ok {
+ return nil
+ }
+
+ port, err := reader.PublishedPort(ctx, gateway.ID, GatewayInternalPort)
+ if err != nil {
+ return err
+ }
+
+ l.SecretPort = port
+
+ return nil
+}
+
+func (l Lab) SecretGatewayURL() string {
+ return "http://" + net.JoinHostPort(
+ loopback,
+ strconv.Itoa(l.SecretPort),
+ )
+}
+
+func (l Lab) SecretProtectedURL() string {
+ return l.SecretGatewayURL() + protectedPath
+}
+
+// IssueSecretToken asks the HMAC fixture for a credential signed with whatever
+// secret it currently holds, so khaos never has to read the file - and so the
+// credential it presents as valid is the one the target would accept.
+func (l Lab) IssueSecretToken(ctx context.Context) (string, error) {
+ return l.issueFrom(ctx, l.SecretGatewayURL()+secretTokenPath)
+}
+
+func (l Lab) RequireSecretTier() error {
+ if l.SecretPort != 0 {
+ return nil
+ }
+
+ return fmt.Errorf(errNoSecretGateway, ErrRangeIncomplete, SecretGatewayName)
+}
+
+// DBGatewayURL is the pooled fixture's entry point.
+func (l Lab) DBGatewayURL() string {
+ return "http://" + net.JoinHostPort(
+ loopback,
+ strconv.Itoa(l.DBGatewayPort),
+ )
+}
+
+func (l Lab) DBProtectedURL() string {
+ return l.DBGatewayURL() + protectedPath
+}
+
+func (l Lab) DBHealthURL() string {
+ return l.DBGatewayURL() + healthPath
+}
+
+func (l Lab) DBReadyURL() string {
+ return l.DBGatewayURL() + readyPath
+}
+
// readTLSTier reports whether the TLS gateway was there to read, because its
// absence is not fatal: the experiments that do not need it must still work.
func (l *Lab) readTLSTier(
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go
new file mode 100644
index 00000000..dd51e4cb
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go
@@ -0,0 +1,295 @@
+// ©AngelaMos | 2026
+// secretfile.go
+
+package secretfile
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+)
+
+type Kind string
+
+const (
+ // Empty is the nastiest of the four and the only one that is not an error
+ // at the syscall level: os.ReadFile returns ([]byte{}, nil), so every
+ // error-handling path in the target is bypassed and the empty value flows
+ // into the crypto. A guard written `if b == nil` does not catch it.
+ Empty Kind = "empty"
+
+ // Truncated is a silently WRONG key rather than a missing one.
+ Truncated Kind = "truncated"
+
+ // Missing is the one an application usually handles, because it is the one
+ // that errors.
+ Missing Kind = "missing"
+
+ // Unreadable is the natural negative control for Empty: it is the only
+ // shape of the four that returns an error from the read itself.
+ Unreadable Kind = "unreadable"
+)
+
+const (
+ // AsideSuffix keeps the original beside the secret, inside the target's own
+ // mount. khaos never carries the content out: a snapshot in the journal or
+ // in khaos's state directory would leave the secret on disk after the run.
+ AsideSuffix = ".khaos-held"
+
+ unreadableMode = 0o000
+ restoredMode = 0o600
+
+ // A secret is conventionally delivered read-only, and the OWNER of a 0400
+ // file cannot write it either. Making it writable first means the
+ // same-uid case needs no capability at all; the helper's DAC_OVERRIDE and
+ // FOWNER are only for the cross-uid case, which is the usual one.
+ writableMode = 0o600
+
+ defaultTruncate = 4
+
+ describeEmpty = "truncate %s to zero bytes in place, which is the one secret fault that is not an error at the syscall level: os.ReadFile returns no error and an empty slice, so every error-handling path in the target is bypassed"
+
+ describeTruncated = "cut %s down to a few bytes in place, which leaves a silently WRONG key rather than a missing one"
+
+ describeMissing = "move %s aside so it is gone from the target's own filesystem"
+
+ describeUnreadable = "make %s unreadable in place, which is the one shape of the four that errors on the read itself and is the natural negative control for the empty one"
+
+ errUnknownKind = "%w: %q (known secret faults: %s)"
+ errNoPath = "%w: a secret fault needs the path of the file it acts on"
+ errRelative = "%w: %q is not an absolute path inside the container"
+ errSelfAside = "%w: the path khaos would keep the original at is the secret itself"
+ errNoSecret = "%s does not exist in the container, so there is no secret for khaos to break"
+ errHeldExists = "%s already exists, so khaos will not overwrite what may be another run's original"
+ errNoHeld = "%s does not exist, so khaos has nothing to restore the secret from"
+)
+
+var ErrUnknownKind = errors.New(
+ "no secret fault is registered under that name",
+)
+
+var ErrBadPath = errors.New(
+ "that is not a path khaos will act on",
+)
+
+// Mutation is one secret-file fault, performed INSIDE the target's own mount by
+// a short-lived container. Every field is a path in the container's namespace.
+type Mutation struct {
+ Kind Kind `json:"kind"`
+ Path string `json:"path"`
+ Aside string `json:"aside"`
+ Bytes int `json:"bytes,omitempty"`
+}
+
+func Kinds() []string {
+ return []string{
+ string(Empty),
+ string(Missing),
+ string(Truncated),
+ string(Unreadable),
+ }
+}
+
+// Describe names what the fault does to the file, in the operator's terms
+// rather than in the syscall's.
+func Describe(kind Kind, path string) string {
+ switch kind {
+ case Empty:
+ return fmt.Sprintf(describeEmpty, path)
+ case Truncated:
+ return fmt.Sprintf(describeTruncated, path)
+ case Missing:
+ return fmt.Sprintf(describeMissing, path)
+ case Unreadable:
+ return fmt.Sprintf(describeUnreadable, path)
+ default:
+ return string(kind)
+ }
+}
+
+func AsideFor(path string) string {
+ return path + AsideSuffix
+}
+
+func (m Mutation) Validate() error {
+ if !slices.Contains(Kinds(), string(m.Kind)) {
+ return fmt.Errorf(
+ errUnknownKind,
+ ErrUnknownKind,
+ m.Kind,
+ strings.Join(Kinds(), ", "),
+ )
+ }
+
+ if m.Path == "" {
+ return fmt.Errorf(errNoPath, ErrBadPath)
+ }
+
+ if !filepath.IsAbs(m.Path) {
+ return fmt.Errorf(errRelative, ErrBadPath, m.Path)
+ }
+
+ if m.Aside == "" || m.Aside == m.Path {
+ return fmt.Errorf(errSelfAside, ErrBadPath)
+ }
+
+ if !filepath.IsAbs(m.Aside) {
+ return fmt.Errorf(errRelative, ErrBadPath, m.Aside)
+ }
+
+ return nil
+}
+
+// Apply performs the fault. Everything that can be done IN PLACE is, because
+// the archive API replaces the inode and a target holding an open file
+// descriptor then never experiences the fault at all - measured. Only Missing
+// has to move the file, which is also why it is the one khaos cannot deliver
+// against a file-level bind mount.
+func (m Mutation) Apply() error {
+ if err := m.Validate(); err != nil {
+ return err
+ }
+
+ info, err := os.Stat(m.Path)
+ if err != nil {
+ return fmt.Errorf(errNoSecret, m.Path)
+ }
+
+ if _, err := os.Stat(m.Aside); err == nil {
+ return fmt.Errorf(errHeldExists, m.Aside)
+ }
+
+ switch m.Kind {
+ case Missing:
+ return os.Rename(m.Path, m.Aside)
+
+ case Empty:
+ if err := m.hold(info); err != nil {
+ return err
+ }
+
+ return os.Truncate(m.Path, 0)
+
+ case Truncated:
+ if err := m.hold(info); err != nil {
+ return err
+ }
+
+ return os.Truncate(m.Path, m.keep(info.Size()))
+
+ case Unreadable:
+ if err := copyFile(m.Path, m.Aside, info.Mode()); err != nil {
+ return err
+ }
+
+ return os.Chmod(m.Path, unreadableMode)
+
+ default:
+ return fmt.Errorf(
+ errUnknownKind,
+ ErrUnknownKind,
+ m.Kind,
+ strings.Join(Kinds(), ", "),
+ )
+ }
+}
+
+// hold copies the original aside and then makes the secret writable, in that
+// order: the copy has to happen while the file can still be read.
+func (m Mutation) hold(info os.FileInfo) error {
+ if err := copyFile(m.Path, m.Aside, info.Mode()); err != nil {
+ return err
+ }
+
+ return os.Chmod(m.Path, writableMode)
+}
+
+// keep is how much of the secret survives truncation: enough to be a plausible
+// key and not enough to be the right one.
+func (m Mutation) keep(size int64) int64 {
+ want := int64(m.Bytes)
+ if want <= 0 {
+ want = defaultTruncate
+ }
+
+ if want >= size {
+ want = size / 2
+ }
+
+ return want
+}
+
+// Restore puts the original back. It is idempotent and reports a missing
+// original rather than guessing, because a secret khaos cannot restore is one
+// it should never have broken.
+func (m Mutation) Restore() error {
+ if err := m.Validate(); err != nil {
+ return err
+ }
+
+ held, err := os.Stat(m.Aside)
+ if err != nil {
+ // Already restored is success: a revert runs again after a reclaim.
+ if _, err := os.Stat(m.Path); err == nil && m.Kind != Unreadable {
+ return nil
+ }
+
+ return fmt.Errorf(errNoHeld, m.Aside)
+ }
+
+ if m.Kind == Missing {
+ return os.Rename(m.Aside, m.Path)
+ }
+
+ // Writable first, for the same reason Apply needed it: the mode khaos is
+ // about to restore is usually one that forbids the write restoring it.
+ if err := os.Chmod(m.Path, writableMode); err != nil {
+ return err
+ }
+
+ if err := copyFile(m.Aside, m.Path, held.Mode()); err != nil {
+ return err
+ }
+
+ if err := os.Chmod(m.Path, held.Mode().Perm()); err != nil {
+ return err
+ }
+
+ return os.Remove(m.Aside)
+}
+
+func copyFile(from, to string, mode os.FileMode) error {
+ //nolint:gosec // the path is the fault's subject, guarded by the nonce
+ source, err := os.Open(from)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = source.Close() }()
+
+ if mode.Perm() == 0 {
+ mode = restoredMode
+ }
+
+ // Truncated rather than appended, so a restore over a longer file leaves
+ // no tail of the fault behind.
+ //nolint:gosec // same: this command exists to write the path it is given
+ target, err := os.OpenFile(
+ to,
+ os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
+ mode.Perm(),
+ )
+ if err != nil {
+ return err
+ }
+
+ if _, err := io.Copy(target, source); err != nil {
+ _ = target.Close()
+ return err
+ }
+
+ return target.Close()
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go
new file mode 100644
index 00000000..12629d9e
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go
@@ -0,0 +1,344 @@
+// ©AngelaMos | 2026
+// secretfile_test.go
+
+package secretfile
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+const (
+ original = "a-real-secret-that-is-long-enough-to-truncate"
+ secretPerm = 0o400
+)
+
+func seeded(t *testing.T) (string, string) {
+ t.Helper()
+
+ dir := t.TempDir()
+ path := filepath.Join(dir, "hmac")
+
+ if err := os.WriteFile(path, []byte(original), secretPerm); err != nil {
+ t.Fatalf("seeding the secret: %v", err)
+ }
+
+ return path, AsideFor(path)
+}
+
+// TestEveryFaultIsReversible is the control for the whole tier's contract: the
+// original stays beside the secret inside the target's own mount, so khaos
+// needs no copy of it in a journal or in memory, and a cold-started revert can
+// put it back. Every one of the four has to round-trip.
+func TestEveryFaultIsReversible(t *testing.T) {
+ for _, kind := range []Kind{Empty, Truncated, Missing, Unreadable} {
+ t.Run(string(kind), func(t *testing.T) {
+ path, aside := seeded(t)
+
+ mutation := Mutation{Kind: kind, Path: path, Aside: aside}
+
+ if err := mutation.Apply(); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+
+ if err := mutation.Restore(); err != nil {
+ t.Fatalf("restore: %v", err)
+ }
+
+ restored, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("reading the restored secret: %v", err)
+ }
+
+ if string(restored) != original {
+ t.Fatalf(
+ "the restored secret is %q, want %q",
+ restored,
+ original,
+ )
+ }
+
+ info, err := os.Stat(path)
+ if err != nil {
+ t.Fatalf("stat: %v", err)
+ }
+
+ if info.Mode().Perm() != secretPerm {
+ t.Errorf(
+ "the restored secret is mode %04o, want %04o",
+ info.Mode().Perm(),
+ secretPerm,
+ )
+ }
+
+ if _, err := os.Stat(aside); !errors.Is(err, fs.ErrNotExist) {
+ t.Errorf(
+ "khaos left %s behind, which is the original sitting beside the secret",
+ aside,
+ )
+ }
+ })
+ }
+}
+
+// TestEachFaultIsTheSHAPE research 07 part 8 describes, measured rather than
+// asserted. The four shapes are what make them distinguishable from outside: an
+// empty read is not an error, a truncated one is not either, a missing one is
+// ErrNotExist and an unreadable one is ErrPermission.
+func TestEachFaultIsTheShapeTheResearchDescribes(t *testing.T) {
+ t.Run("empty reads as success with an empty slice", func(t *testing.T) {
+ path, aside := seeded(t)
+
+ if err := (Mutation{
+ Kind: Empty,
+ Path: path,
+ Aside: aside,
+ }).Apply(); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("an empty secret must not error: %v", err)
+ }
+
+ if len(body) != 0 {
+ t.Fatalf("len = %d, want 0", len(body))
+ }
+
+ // The sharpening that matters: the slice is NOT nil, so a guard
+ // written `if b == nil` does not catch an empty secret.
+ if body == nil {
+ t.Error(
+ "os.ReadFile returned a nil slice, which would make `if b == nil` a working guard",
+ )
+ }
+ })
+
+ t.Run("truncated reads as success with a wrong key", func(t *testing.T) {
+ path, aside := seeded(t)
+
+ if err := (Mutation{
+ Kind: Truncated,
+ Path: path,
+ Aside: aside,
+ Bytes: 4,
+ }).Apply(); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("a truncated secret must not error: %v", err)
+ }
+
+ if len(body) != 4 {
+ t.Fatalf("len = %d, want 4", len(body))
+ }
+
+ if string(body) == original {
+ t.Fatal("the secret was not truncated at all")
+ }
+ })
+
+ t.Run("missing reads as ErrNotExist", func(t *testing.T) {
+ path, aside := seeded(t)
+
+ if err := (Mutation{
+ Kind: Missing,
+ Path: path,
+ Aside: aside,
+ }).Apply(); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+
+ if _, err := os.ReadFile(path); !errors.Is(err, fs.ErrNotExist) {
+ t.Fatalf("err = %v, want fs.ErrNotExist", err)
+ }
+ })
+
+ t.Run("unreadable reads as ErrPermission", func(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("running as root, which ignores the mode this fault sets")
+ }
+
+ path, aside := seeded(t)
+
+ if err := (Mutation{
+ Kind: Unreadable,
+ Path: path,
+ Aside: aside,
+ }).Apply(); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+
+ if _, err := os.ReadFile(path); !errors.Is(err, fs.ErrPermission) {
+ t.Fatalf("err = %v, want fs.ErrPermission", err)
+ }
+ })
+}
+
+// TestApplyRefusesToOverwriteAnOriginal is the guard that keeps a second run
+// from destroying the first one's copy, which would leave a secret khaos can
+// never put back.
+func TestApplyRefusesToOverwriteAnOriginal(t *testing.T) {
+ path, aside := seeded(t)
+
+ mutation := Mutation{Kind: Empty, Path: path, Aside: aside}
+
+ if err := mutation.Apply(); err != nil {
+ t.Fatalf("first apply: %v", err)
+ }
+
+ if err := mutation.Apply(); err == nil {
+ t.Fatal("a second apply overwrote the original khaos was keeping")
+ }
+
+ if err := mutation.Restore(); err != nil {
+ t.Fatalf("restore: %v", err)
+ }
+
+ body, err := os.ReadFile(path)
+ if err != nil || string(body) != original {
+ t.Fatalf("the original did not survive: %q, %v", body, err)
+ }
+}
+
+func TestValidateRefusesWhatCannotBeUndone(t *testing.T) {
+ cases := []struct {
+ name string
+ mutation Mutation
+ want error
+ }{
+ {
+ name: "no kind",
+ mutation: Mutation{Path: "/x", Aside: "/x.held"},
+ want: ErrUnknownKind,
+ },
+ {
+ name: "an invented kind",
+ mutation: Mutation{Kind: "shred", Path: "/x", Aside: "/x.held"},
+ want: ErrUnknownKind,
+ },
+ {
+ name: "no path",
+ mutation: Mutation{Kind: Empty, Aside: "/x.held"},
+ want: ErrBadPath,
+ },
+ {
+ name: "a relative path",
+ mutation: Mutation{Kind: Empty, Path: "hmac", Aside: "/x.held"},
+ want: ErrBadPath,
+ },
+ {
+ name: "the original kept at the secret itself",
+ mutation: Mutation{Kind: Empty, Path: "/x", Aside: "/x"},
+ want: ErrBadPath,
+ },
+ {
+ name: "no original at all",
+ mutation: Mutation{Kind: Empty, Path: "/x"},
+ want: ErrBadPath,
+ },
+ }
+
+ for _, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ if err := test.mutation.Validate(); !errors.Is(err, test.want) {
+ t.Fatalf("err = %v, want %v", err, test.want)
+ }
+ })
+ }
+}
+
+func TestApplyRefusesASecretThatIsNotThere(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "absent")
+
+ err := (Mutation{
+ Kind: Empty,
+ Path: path,
+ Aside: AsideFor(path),
+ }).Apply()
+
+ if err == nil {
+ t.Fatal("khaos broke a secret that did not exist")
+ }
+}
+
+// TestTheEmptyKeyVerifies is the mechanism behind the whole empty-secret
+// fault, and the reason it is a finding rather than an outage: crypto/hmac
+// computes with a zero-length key and the result verifies.
+func TestTheEmptyKeyVerifies(t *testing.T) {
+ token := Token(nil, "a-nonce")
+
+ if !Verify(token, nil) {
+ t.Fatal("a token signed with the empty key does not verify against it")
+ }
+
+ if !Verify(token, []byte{}) {
+ t.Fatal("nil and an empty slice are not the same key")
+ }
+
+ if Verify(token, []byte(original)) {
+ t.Fatal("a token signed with the empty key verified against a real one")
+ }
+
+ if Verify(Token([]byte(original), "a-nonce"), nil) {
+ t.Fatal("a token signed with a real key verified against the empty one")
+ }
+}
+
+// TestEveryTokenIsDifferentAndStillVerifies is the control for the nonce: the
+// catalogue census requires the negative control to vary per request, and this
+// experiment requires it to stay signed with the same key. Both at once is
+// what the nonce inside the signed message buys.
+func TestEveryTokenIsDifferentAndStillVerifies(t *testing.T) {
+ seen := map[string]bool{}
+
+ const rounds = 8
+
+ for i := range rounds {
+ nonce := fmt.Sprintf("nonce-%d", i)
+ token := Token(nil, nonce)
+
+ seen[token] = true
+
+ if !Verify(token, nil) {
+ t.Fatalf("%q does not verify against the empty key", token)
+ }
+
+ if Verify(token, []byte(original)) {
+ t.Fatalf("%q verified against a real key", token)
+ }
+ }
+
+ if len(seen) != rounds {
+ t.Fatalf(
+ "%d calls produced %d distinct credentials",
+ rounds,
+ len(seen),
+ )
+ }
+}
+
+func TestVerifyRefusesAMalformedCredential(t *testing.T) {
+ for _, token := range []string{
+ "",
+ "no-separator",
+ Payload,
+ Payload + ".nonce",
+ Payload + ".nonce.not-hex",
+ Payload + ".00",
+ "wrong-payload.nonce.00",
+ Token([]byte(original), "one") + ".extra",
+ } {
+ if Verify(token, []byte(original)) {
+ t.Errorf("%q verified", token)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/token.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/token.go
new file mode 100644
index 00000000..d215c09b
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/token.go
@@ -0,0 +1,64 @@
+// ©AngelaMos | 2026
+// token.go
+
+package secretfile
+
+import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "crypto/subtle"
+ "encoding/hex"
+ "strings"
+)
+
+const (
+ // Payload is the fixed part of the message. These experiments are about the
+ // KEY the target verifies with, so the key is the only thing that varies
+ // between a valid credential and the control.
+ Payload = "khaos-range-secret-probe"
+
+ // The message carries a NONCE as well, which is what lets the control be
+ // freshly minted per request while staying signed with the same key. A
+ // fixed credential can be answered from one cached refusal, and the
+ // catalogue census refuses an experiment whose control repeats - measured
+ // against a gateway that failed open with a cache TTL longer than the
+ // fault window and reported a clean pass.
+ separator = "."
+ parts = 3
+)
+
+// Token is HMAC-SHA256 over a fixed payload, shared by khaos and the range
+// fixture so that neither can drift from the other about what a valid
+// credential is.
+//
+// An EMPTY key is the whole point of the empty-secret fault: crypto/hmac
+// computes happily with a zero-length key, and a token signed with it verifies
+// against a verifier that read an empty secret and did not check its length.
+func Token(secret []byte, nonce string) string {
+ return Payload + separator + nonce + separator +
+ hex.EncodeToString(sign(secret, nonce))
+}
+
+func sign(secret []byte, nonce string) []byte {
+ mac := hmac.New(sha256.New, secret)
+ mac.Write([]byte(Payload + separator + nonce))
+
+ return mac.Sum(nil)
+}
+
+// Verify is constant-time and does NOT check the key's length, which is the
+// defect the `unchecked` fixture mode carries deliberately. The caller decides
+// whether an empty key is acceptable.
+func Verify(token string, secret []byte) bool {
+ fields := strings.Split(token, separator)
+ if len(fields) != parts || fields[0] != Payload {
+ return false
+ }
+
+ presented, err := hex.DecodeString(fields[parts-1])
+ if err != nil {
+ return false
+ }
+
+ return subtle.ConstantTimeCompare(presented, sign(secret, fields[1])) == 1
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/Dockerfile b/PROJECTS/intermediate/chaos-engineering-tool/range/Dockerfile
index 171bc4d5..69a0bce4 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/range/Dockerfile
+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/Dockerfile
@@ -16,5 +16,17 @@ FROM alpine:3.22
RUN adduser -D -u 10001 range
COPY --from=build /out/khaos-range /usr/local/bin/khaos-range
+# The secret the HMAC fixture verifies with, baked at BUILD time so a fresh
+# named volume is seeded from the image when it is first mounted. That is what
+# lets the fixture mount it READ-ONLY, which is how secrets are conventionally
+# delivered and the shape khaos's archive-API route cannot write at all - the
+# mutator container mounting the same volume read-write is the only route to
+# it. Not a secret in any real sense: this range exists to be broken.
+RUN mkdir -p /run/khaos-secrets \
+ && head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' \
+ > /run/khaos-secrets/hmac \
+ && chown -R range:range /run/khaos-secrets \
+ && chmod 0400 /run/khaos-secrets/hmac
+
USER range
ENTRYPOINT ["/usr/local/bin/khaos-range"]
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
index 26de8498..00eb0b6d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
@@ -441,6 +441,88 @@ services:
range:
ipv4_address: 10.244.244.33
restart: "no"
+ # The pooled-credential fixture, and the only range service that holds a
+ # database credential of its own. It is in the stateful profile because it
+ # cannot start without the engine.
+ #
+ # The credential is passed as separate variables rather than as one DSN, so
+ # khaos can read the PASSWORD on its own out of this container's environment,
+ # rotate that role at the engine, and put the same value back afterwards
+ # without ever journaling it.
+ gateway-db:
+ container_name: khaos-range-gateway-db
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["db"]
+ profiles: ["db"]
+ environment:
+ KHAOS_RANGE_UPSTREAM: "http://idp:8080"
+ KHAOS_RANGE_DB_HOST: "postgres:5432"
+ KHAOS_RANGE_DB_USER: "appuser"
+ KHAOS_RANGE_DB_PASSWORD: "${KHAOS_RANGE_DB_APP_PASSWORD:?khaos mints this; run `khaos range up --with-db`}"
+ KHAOS_RANGE_DB_NAME: "appdb"
+ KHAOS_RANGE_POOL_MODE: "${KHAOS_RANGE_POOL_MODE:-unbounded}"
+ KHAOS_RANGE_FAIL_MODE: "${KHAOS_RANGE_FAIL_MODE:-closed}"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ ports:
+ - "127.0.0.1:${KHAOS_RANGE_DB_GATEWAY_PORT:?khaos chooses this port; run `khaos range up --with-db`}:8080"
+ depends_on:
+ postgres:
+ condition: service_healthy
+ idp:
+ condition: service_started
+ networks:
+ range:
+ ipv4_address: 10.244.244.34
+ # PROCESS liveness, not database liveness: /health is the endpoint these
+ # experiments measure, and a healthcheck pointed at it would make the
+ # fixture unstartable until its credential exists.
+ healthcheck:
+ test: ["CMD", "/usr/local/bin/khaos-range", "alivecheck"]
+ interval: 2s
+ timeout: 2s
+ retries: 15
+ restart: "no"
+
+ # The secret-delivery fixture. It re-reads its secret on EVERY request, which
+ # is what makes the fault observable: a service that read it once at startup
+ # holds the original for the life of the process.
+ #
+ # The volume is mounted READ-ONLY, which is how secrets are conventionally
+ # delivered and the whole reason khaos breaks them through a helper container:
+ # the archive API refuses a `:ro` mount with HTTP 400, while read-only-ness is
+ # a property of THIS mount rather than of the volume, so a helper mounting the
+ # same volume read-write writes straight through it.
+ gateway-secret:
+ container_name: khaos-range-gateway-secret
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["secret"]
+ environment:
+ KHAOS_RANGE_SECRET_MODE: "${KHAOS_RANGE_SECRET_MODE:-unchecked}"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ ports:
+ - "127.0.0.1:${KHAOS_RANGE_SECRET_PORT:?khaos chooses this port; run `khaos range up`}:8080"
+ volumes:
+ - secrets:/run/khaos-secrets:ro
+ networks:
+ range:
+ ipv4_address: 10.244.244.16
+ healthcheck:
+ test: ["CMD", "/usr/local/bin/khaos-range", "alivecheck"]
+ interval: 2s
+ timeout: 2s
+ retries: 15
+ restart: "no"
+
networks:
range:
@@ -460,3 +542,10 @@ networks:
ipam:
config:
- subnet: 10.244.244.0/24
+
+volumes:
+ # Seeded from the range image, which is what makes the read-only mount above
+ # possible: Docker copies the image's content at that path into a fresh named
+ # volume the first time it is mounted.
+ secrets:
+ name: khaos-range-secrets
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
index 07cbc9c9..8fee56de 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
@@ -182,15 +182,22 @@ assert_control_held() {
"the invalid credential was never rejected"
}
+# EVERY helper below that greps ends in `|| true`, and that is not defensive
+# noise: with `set -e`, a grep inside a command substitution that legitimately
+# matches nothing kills the script, and `grep -c` does it even when printing a
+# perfectly good `0`. The symptom is a gate that exits 1 with no FAIL line at
+# all, which reads like a mystery rather than like a bug. It has fired three
+# times in this one file: on an unmatched confirmation line, on a count of
+# leftovers that was correctly zero, and on an absent environment variable.
observable_of() {
grep -E '^ plan confirm fault confirmed by ' <<<"$report" |
- sed -E 's/^.*fault confirmed by //'
+ sed -E 's/^.*fault confirmed by //' || true
}
admin_password() {
docker inspect "${CONTAINER[$1]}" \
--format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null |
- grep "^${ADMIN_VAR[$1]}=" | cut -d= -f2-
+ grep "^${ADMIN_VAR[$1]}=" | cut -d= -f2- || true
}
engine_address() {
@@ -219,15 +226,33 @@ leftover_roles() {
esac
}
+# The pool configuration is fixed at container creation, deliberately, so a
+# gate spanning all three has to restart the range. A fixture that could be
+# reconfigured mid-run would let khaos change the target it is measuring.
start_range() {
+ local pool="${1:-unbounded}"
+ local secret="${2:-unchecked}"
+
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --with-db --no-color --quiet >/dev/null 2>&1; then
- fail "range up --with-db"
+ if ! "$BIN" range up --with-db --pool-mode "$pool" \
+ --secret-mode "$secret" --no-color --quiet >/dev/null 2>&1; then
+ fail "range up --with-db --pool-mode $pool --secret-mode $secret"
return 1
fi
return 0
}
+secret_leftovers() {
+ docker ps -a --filter "label=khaos.mutator=true" --format '{{.Names}}' |
+ wc -l | tr -d '[:space:]'
+}
+
+held_originals() {
+ docker exec khaos-range-gateway-secret \
+ sh -c 'ls /run/khaos-secrets/ 2>/dev/null' 2>/dev/null |
+ grep -c 'khaos-held' | tr -d '[:space:]' || true
+}
+
echo "== the stateful tier is refused by name when it is not running =="
# The refusal has to arrive with a range UP and the database tier DOWN, which is
@@ -255,7 +280,7 @@ fi
echo "== THE PAIR: rotation is not revocation, and termination is =="
-if start_range; then
+if start_range unbounded; then
for engine in "${ENGINES[@]}"; do
container="${CONTAINER[$engine]}"
@@ -303,6 +328,250 @@ if start_range; then
done
fi
+
+echo "== the pooled fixture: the fail-open is the DEFAULT configuration =="
+
+# Three pool configurations, and the verdict has to follow the configuration
+# rather than the fault. The fault is identical in all three runs.
+#
+# unbounded the shape a Go service has when nobody called
+# SetConnMaxLifetime. HELD is the defect: the service carried on.
+# bounded the knob that converts a rotation into a revocation. DENIED is
+# correct, and it arrives as UNAVAILABILITY rather than a refusal,
+# which is why the experiment declares what failing closed means.
+# held the same lifetime as bounded, plus one connection checked out
+# and never returned. The cleaner only walks the IDLE list, so the
+# lifetime does not bound it and HELD is the defect again. This is
+# the row research 07 section 5.3 did not have.
+declare -A WANT_VERDICT=(
+ [unbounded]="passed / HELD"
+ [bounded]="deviated / DENIED"
+ [held]="passed / HELD"
+)
+
+declare -A WANT_EXIT=(
+ [unbounded]=2
+ [bounded]=0
+ [held]=2
+)
+
+# What khaos DID is carried by the HELD CAVEAT for these two, not by the defect
+# phrase: that phrase is part of a leak reason and a leak is exactly what a
+# HELD finding does not have. Asserting the defect phrase here failed six times
+# against a product that was reporting correctly.
+declare -A WANT_CAVEAT=(
+ [credential.db_pool_stale]="khaos rotated the credential and the service carried on"
+ [credential.db_pending_window]="the liveness endpoint answered throughout on a connection that authenticated before the rotation"
+)
+
+for pool in unbounded bounded held; do
+ if ! start_range "$pool"; then
+ continue
+ fi
+
+ for experiment in credential.db_pool_stale credential.db_pending_window; do
+ run_experiment "$experiment"
+
+ label="${experiment#credential.} on a ${pool} pool"
+
+ assert_verdict "$label" "${WANT_VERDICT[$pool]}"
+ assert_exit "$label" "${WANT_EXIT[$pool]}"
+ assert_confirmed "$label"
+ assert_control_held "$label"
+
+ # A HELD verdict has to say what khaos did and what HELD means here.
+ # A DENIED one carries no caveat at all, deliberately: the caveat is
+ # the explanation of a survival.
+ if [[ "${WANT_VERDICT[$pool]}" == "passed / HELD" ]]; then
+ assert_report_has \
+ "${label} says what khaos did and what surviving it means" \
+ "${WANT_CAVEAT[$experiment]}"
+ else
+ assert_report_lacks \
+ "${label} does not explain a survival that did not happen" \
+ "${WANT_CAVEAT[$experiment]}"
+ fi
+
+ # The sharpest row, asserted per experiment rather than after the loop,
+ # where $report held the LAST experiment's output: a lifetime IS set in
+ # held mode and the service still carried on.
+ if [[ "$pool" == "held" &&
+ "$experiment" == "credential.db_pool_stale" ]]; then
+ assert_report_has \
+ "a checked-out connection is not bounded by the lifetime" \
+ "a connection that is checked out rather than idle is not bounded by it at all"
+ fi
+
+ # The correct configuration has to lose access rather than merely slow
+ # down, or DENIED would be an artefact of the probe timing out.
+ if [[ "$pool" == "bounded" ]]; then
+ assert_report_has \
+ "${label} stopped being able to serve" \
+ "the valid credential stopped being accepted"
+ fi
+ done
+done
+
+echo "== the secret file: one fault of four is a finding, and the other three say why =="
+
+# The four shapes are what make the one that matters discriminating. Research 07
+# part 8 is explicit that the empty file is the nastiest because it is the ONLY
+# one that is not an error at the syscall level: os.ReadFile returns no error
+# and an empty slice, so every error-handling path in the target is bypassed and
+# the empty value reaches the crypto. crypto/hmac then computes happily with a
+# zero-length key, which is CVE-2019-20933's exact shape.
+#
+# The other three are expected to make a correct target refuse and CANNOT do
+# anything else, which is what makes them the controls: if all four read the
+# same, khaos is not measuring the key at all.
+declare -A WANT_SECRET_UNCHECKED=(
+ [credential.secret_file_empty]="deviated / PERMITTED"
+ [credential.secret_file_truncated]="deviated / DENIED"
+ [credential.secret_file_missing]="deviated / DENIED"
+ [credential.secret_file_unreadable]="deviated / DENIED"
+)
+
+declare -A WANT_SECRET_EXIT=(
+ [credential.secret_file_empty]=2
+ [credential.secret_file_truncated]=0
+ [credential.secret_file_missing]=0
+ [credential.secret_file_unreadable]=0
+)
+
+if start_range unbounded unchecked; then
+ declare -A SECRET_OBSERVABLE=()
+
+ for experiment in "${!WANT_SECRET_UNCHECKED[@]}"; do
+ run_experiment "$experiment"
+
+ label="${experiment#credential.} against a verifier that does not check the length"
+
+ assert_verdict "$label" "${WANT_SECRET_UNCHECKED[$experiment]}"
+ assert_exit "$label" "${WANT_SECRET_EXIT[$experiment]}"
+ assert_confirmed "$label"
+ assert_control_held "$label"
+
+ SECRET_OBSERVABLE[$experiment]="$(observable_of)"
+ done
+
+ # The confirmation is read off the TARGET's own filesystem, so each fault
+ # has to name what the target now has rather than what khaos asked for.
+ assert_report_has \
+ "the empty-secret confirmation names the target's own copy" \
+ "is now zero bytes, and os.ReadFile returns no error on one"
+
+ distinct=$(printf '%s\n' "${SECRET_OBSERVABLE[@]}" | sort -u | wc -l)
+ if [[ "$distinct" == "${#SECRET_OBSERVABLE[@]}" ]]; then
+ pass "all four secret faults produce DISTINCT observables"
+ else
+ fail "the four secret faults produced ${distinct} distinct observables, so some of them are not evidence"
+ fi
+
+ # khaos's helper is short-lived by construction, and an orphan would be a
+ # container holding two file-permission capabilities over the operator's
+ # own volume.
+ left="$(secret_leftovers)"
+ if [[ "$left" == "0" ]]; then
+ pass "no mutator container is left behind"
+ else
+ fail "${left} mutator container(s) left behind"
+ fi
+
+ # The original is kept BESIDE the secret, inside the target's own mount, so
+ # khaos never carries it out - and the revert has to remove it again.
+ held="$(held_originals)"
+ if [[ "$held" == "0" ]]; then
+ pass "the original khaos kept beside the secret is gone after the revert"
+ else
+ fail "khaos left ${held} copy of the original inside the target's mount"
+ fi
+fi
+
+echo "== the length check is the fix, and khaos says so =="
+
+# The same four faults against a verifier that validates the length after every
+# read, which is what research 07 part 8 prescribes and what the upstream fix
+# for CVE-2019-20933 literally added. The one finding has to disappear.
+if start_range unbounded validated; then
+ run_experiment credential.secret_file_empty
+
+ assert_verdict \
+ "secret_file_empty against a verifier that checks the length" \
+ "deviated / DENIED"
+ assert_exit \
+ "secret_file_empty against a verifier that checks the length" 0
+ assert_confirmed \
+ "secret_file_empty against a verifier that checks the length"
+ assert_report_lacks \
+ "no credential was honoured once the length was checked" \
+ "was accepted"
+fi
+
+echo "== a secret khaos cannot reach reversibly is refused by name =="
+
+if start_range unbounded unchecked; then
+ # A path on no mount at all. A helper cannot reach it, and a cold-started
+ # revert could not put it back, so khaos declines rather than breaking
+ # something it could not undo.
+ run_unmeasured credential.secret_file_empty \
+ --target khaos-range-gateway-secret \
+ --url "http://127.0.0.1:1/protected" \
+ --token nothing \
+ --secret-path /etc/hostname
+
+ if [[ "$last_code" != 0 ]]; then
+ pass "a secret on no mount at all is refused"
+ else
+ fail "khaos broke a file it could not have restored"
+ fi
+
+ assert_report_has \
+ "the refusal names the reason and the fix" \
+ "Deliver the secret on a volume or a bind-mounted directory"
+
+ # A tmpfs is private to the container's own namespace, so a helper mounting
+ # the same spec gets an EMPTY filesystem: khaos would write a secret into
+ # nowhere and report a fault that landed.
+ run_unmeasured credential.secret_file_empty \
+ --target khaos-range-postgres \
+ --url "http://127.0.0.1:1/protected" \
+ --token nothing \
+ --secret-path /var/lib/postgresql/khaos-probe
+
+ if [[ "$last_code" != 0 ]]; then
+ pass "a secret on a tmpfs mount is refused"
+ else
+ fail "khaos accepted a secret on a tmpfs, which a helper cannot reach"
+ fi
+
+ assert_report_has \
+ "the tmpfs refusal names why a helper cannot reach it" \
+ "private to the container"
+fi
+
+echo "== the pooled faults put BOTH containers to the refusal predicates =="
+
+# khaos measures one container and performs the rotation at another, so the
+# engine's consent is not implied by the target's. A rotation at a container
+# that never consented would be khaos mutating something nobody approved.
+if start_range unbounded; then
+ run_unmeasured credential.db_pool_stale \
+ --target khaos-range-gateway-db \
+ --engine-target khaos-range-idp \
+ --engine postgres \
+ --subject-role appuser \
+ --subject-password-var KHAOS_RANGE_DB_PASSWORD \
+ --ready-url "http://127.0.0.1:1/ready" \
+ --url "http://127.0.0.1:1/protected" \
+ --token nothing
+
+ if [[ "$last_code" != 0 ]]; then
+ pass "an engine container with no administrative credential is refused"
+ else
+ fail "khaos accepted an engine container it could not read a credential from"
+ fi
+fi
+
echo "== the two other controls an operator reaches for do not revoke either =="
for engine in "${ENGINES[@]}"; do
@@ -434,7 +703,7 @@ fi
echo "== census: every catalogued credential.db_ experiment was run here =="
catalogued="$(NO_COLOR=1 "$BIN" run --help |
- grep -oE '^ credential\.[a-z_]+' | tr -d ' ' | sort -u)"
+ grep -oE '^ credential\.[a-z_]+' | tr -d ' ' | sort -u || true)"
if [[ -z "$catalogued" ]]; then
fail "the census read no credential experiments at all, so it cannot fail for the right reason"