Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

feat(khaos): the control family, and the first refusal that is not a rejection

CarterPerez-dev7abfb9aae5c0
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controld.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controld.go
new file mode 100644
index 00000000..db8ffc57
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controld.go
@@ -0,0 +1,64 @@
+// ©AngelaMos | 2026
+// controld.go
+
+package main
+
+import (
+ "net/http"
+)
+
+const (
+ routePolicyDocument = "/policy"
+ routeRuleSet = "/rules"
+
+ // The marker all three controls key on: the policy denies the subject,
+ // the rule set matches the pattern, the limiter grants it no budget.
+ flaggedMarker = "flagged"
+)
+
+// controlBackend serves the policy document and the rule set over HTTP,
+// which is what both anchors are: OPA bundles are fetched over HTTP and
+// ModSecurity's SecRemoteRules failure is a problem downloading a URL.
+//
+// One container, two dependencies: khaos faults the proxy in front of one
+// address and not the other.
+type controlBackend struct{}
+
+func newControlBackend() *controlBackend {
+ return &controlBackend{}
+}
+
+type policyDocument struct {
+ Deny []string `json:"deny"`
+}
+
+type ruleSet struct {
+ Match []string `json:"match"`
+}
+
+func (b *controlBackend) routes() *http.ServeMux {
+ mux := http.NewServeMux()
+
+ mux.HandleFunc(routeAlive, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+ mux.HandleFunc(routeHealth, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+
+ mux.HandleFunc(routePolicyDocument, func(
+ w http.ResponseWriter,
+ _ *http.Request,
+ ) {
+ writeJSON(w, policyDocument{Deny: []string{flaggedMarker}})
+ })
+
+ mux.HandleFunc(routeRuleSet, func(
+ w http.ResponseWriter,
+ _ *http.Request,
+ ) {
+ writeJSON(w, ruleSet{Match: []string{flaggedMarker}})
+ })
+
+ return mux
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controlgateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controlgateway.go
new file mode 100644
index 00000000..a79b6b10
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controlgateway.go
@@ -0,0 +1,340 @@
+// ©AngelaMos | 2026
+// controlgateway.go
+
+package main
+
+import (
+ "crypto/rand"
+ "encoding/json"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/secretfile"
+)
+
+const (
+ routeLimited = "/limited"
+ routeGoverned = "/governed"
+ routeFiltered = "/filtered"
+
+ routeFlaggedToken = "/flagged-token"
+
+ headerControlReason = "X-Khaos-Control"
+
+ limitWindow = 10 * time.Second
+
+ // Short enough that a severed dependency is felt inside one fault window.
+ controlRefresh = 500 * time.Millisecond
+
+ reasonWithinBudget = "the limiter counted this request and it is within the budget"
+
+ reasonOverBudget = "the limiter counted this request and it is over the budget"
+
+ reasonLimiterFellOpen = "the limiter could not reach its datastore, so this mode proxied the request anyway and the limit is not being enforced"
+
+ reasonLimiterHeld = "the limiter could not reach its datastore, and this mode refuses rather than stop counting"
+
+ reasonPolicyAllows = "the loaded policy does not deny this subject"
+
+ reasonPolicyDenies = "the loaded policy denies this subject"
+
+ reasonPolicyStale = "the policy store is unreachable, and this mode keeps enforcing the last policy it loaded"
+
+ reasonPolicyDropped = "the policy store is unreachable, so this mode is enforcing no policy at all"
+
+ reasonPolicyNeverLoaded = "this service started with its policy store unreachable and has never held a policy, so its policy set is empty and an empty policy set denies nothing"
+
+ reasonPolicyCannotEvaluate = "this service has never loaded a policy and refuses rather than serve a request it cannot evaluate"
+
+ reasonRulesClean = "no rule in the loaded set matches this request"
+
+ reasonRulesMatched = "a rule in the loaded set matches this request"
+
+ reasonRulesStale = "the rules backend is unreachable, and this mode keeps matching against the last set it loaded"
+
+ reasonRulesDropped = "the rules backend is unreachable, so this mode is matching against no rules at all"
+
+ reasonRulesNeverLoaded = "this service started with its rules backend unreachable and has never held a rule set, so nothing is being filtered"
+
+ reasonRulesCannotFilter = "this service has never loaded a rule set and refuses rather than pass traffic it cannot inspect"
+)
+
+// controlGateway carries three security controls, each dialling its own
+// dependency address so a fault on one leaves the other two answering.
+//
+// Authentication is this service's own job and is correct in every mode: a
+// service with no other check refuses nothing, and khaos voids a run against
+// a target it never observed refusing anything.
+type controlGateway struct {
+ store string
+ policy string
+ rules string
+
+ failMode string
+ limit int64
+
+ // secret is what this service authenticates WITH, rather than a set of
+ // credentials it remembers issuing. The cold-start experiment restarts
+ // this container, and a remembered set is empty afterwards: every
+ // credential khaos holds reads as unknown, the valid stream goes 401, and
+ // the run measures the restart instead of the policy.
+ secret []byte
+
+ client *http.Client
+
+ mu sync.Mutex
+ denied []string
+ matched []string
+
+ // Separates "lost what it had" from "never had any". A cold start is only
+ // a distinct observation while those are two states.
+ loadedPolicy bool
+ loadedRules bool
+
+ freshPolicy bool
+ freshRules bool
+}
+
+func newControlGateway(
+ store, policy, rules, failMode, secret string,
+ limit int64,
+) *controlGateway {
+ return &controlGateway{
+ store: store,
+ policy: policy,
+ rules: rules,
+ failMode: failMode,
+ secret: []byte(secret),
+ limit: limit,
+ client: &http.Client{Timeout: upstreamTimeout},
+ }
+}
+
+func (g *controlGateway) routes() *http.ServeMux {
+ go g.refreshing()
+
+ mux := http.NewServeMux()
+
+ mux.HandleFunc(routeAlive, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+ mux.HandleFunc(routeHealth, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+
+ mux.HandleFunc(issuePath, g.issue(false))
+ mux.HandleFunc(routeFlaggedToken, g.issue(true))
+
+ mux.HandleFunc(routeLimited, g.guard(g.limited))
+ mux.HandleFunc(routeGoverned, g.guard(g.governed))
+ mux.HandleFunc(routeFiltered, g.guard(g.filtered))
+
+ return mux
+}
+
+// issue hands out a credential this service authenticates. The marker rides
+// in the NONCE, so all three controls can read it off a signed credential and
+// the signature still verifies after a restart.
+func (g *controlGateway) issue(flagged bool) http.HandlerFunc {
+ return func(w http.ResponseWriter, _ *http.Request) {
+ nonce := rand.Text()
+ if flagged {
+ nonce = flaggedMarker + "-" + nonce
+ }
+
+ w.Header().Set(contentType, mimeJSON)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(
+ `{"token":"` + secretfile.Token(g.secret, nonce) + `"}`,
+ ))
+ }
+}
+
+type decision struct {
+ status int
+ reason string
+}
+
+// guard authenticates, so a refusal from a control is never confused with
+// this service failing to identify the caller.
+func (g *controlGateway) guard(
+ control func(string) decision,
+) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ credential, scheme := strings.CutPrefix(
+ r.Header.Get(headerAuthorization),
+ bearerPrefix,
+ )
+
+ if !scheme || credential == "" {
+ w.Header().Set(headerControlReason, reasonNoCredential)
+ w.WriteHeader(http.StatusUnauthorized)
+
+ return
+ }
+
+ if !secretfile.Verify(credential, g.secret) {
+ w.Header().Set(headerControlReason, reasonNotAuthenticated)
+ w.WriteHeader(http.StatusUnauthorized)
+
+ return
+ }
+
+ outcome := control(credential)
+
+ w.Header().Set(headerControlReason, outcome.reason)
+ w.WriteHeader(outcome.status)
+ }
+}
+
+func (g *controlGateway) failsOpen() bool {
+ return g.failMode == FailOpen
+}
+
+// limited is F8. The budget is local and the counter is remote, so a severed
+// datastore leaves the limiter knowing the limit and unable to apply it.
+func (g *controlGateway) limited(credential string) decision {
+ budget := g.limit
+ if matches(credential, []string{flaggedMarker}) {
+ budget = 0
+ }
+
+ window := time.Now().Unix() / int64(limitWindow.Seconds())
+ key := credential + ":" + strconv.FormatInt(window, 10)
+
+ count, err := respCount(g.store, key, limitWindow)
+ if err != nil {
+ if g.failsOpen() {
+ return decision{http.StatusOK, reasonLimiterFellOpen}
+ }
+
+ return decision{http.StatusTooManyRequests, reasonLimiterHeld}
+ }
+
+ if count > budget {
+ return decision{http.StatusTooManyRequests, reasonOverBudget}
+ }
+
+ return decision{http.StatusOK, reasonWithinBudget}
+}
+
+// governed is F11 and F12: losing a loaded policy, and never loading one.
+func (g *controlGateway) governed(credential string) decision {
+ g.mu.Lock()
+ loaded, fresh := g.loadedPolicy, g.freshPolicy
+ denied := append([]string(nil), g.denied...)
+ g.mu.Unlock()
+
+ if !loaded {
+ if g.failsOpen() {
+ return decision{http.StatusOK, reasonPolicyNeverLoaded}
+ }
+
+ return decision{http.StatusForbidden, reasonPolicyCannotEvaluate}
+ }
+
+ if !fresh && g.failsOpen() {
+ return decision{http.StatusOK, reasonPolicyDropped}
+ }
+
+ if matches(credential, denied) {
+ if fresh {
+ return decision{http.StatusForbidden, reasonPolicyDenies}
+ }
+
+ return decision{http.StatusForbidden, reasonPolicyStale}
+ }
+
+ return decision{http.StatusOK, reasonPolicyAllows}
+}
+
+// filtered is F21. It hunts the operator's own fallback, not a vendor
+// default: of the vendors that document one, both deny.
+func (g *controlGateway) filtered(credential string) decision {
+ g.mu.Lock()
+ loaded, fresh := g.loadedRules, g.freshRules
+ matched := append([]string(nil), g.matched...)
+ g.mu.Unlock()
+
+ if !loaded {
+ if g.failsOpen() {
+ return decision{http.StatusOK, reasonRulesNeverLoaded}
+ }
+
+ return decision{http.StatusForbidden, reasonRulesCannotFilter}
+ }
+
+ if !fresh && g.failsOpen() {
+ return decision{http.StatusOK, reasonRulesDropped}
+ }
+
+ if matches(credential, matched) {
+ if fresh {
+ return decision{http.StatusForbidden, reasonRulesMatched}
+ }
+
+ return decision{http.StatusForbidden, reasonRulesStale}
+ }
+
+ return decision{http.StatusOK, reasonRulesClean}
+}
+
+func matches(credential string, patterns []string) bool {
+ for _, pattern := range patterns {
+ if pattern != "" && strings.Contains(credential, pattern) {
+ return true
+ }
+ }
+
+ return false
+}
+
+// refreshing re-fetches on an interval rather than per request: holding a
+// last-known-good is what makes "lost it" and "never had it" two states.
+func (g *controlGateway) refreshing() {
+ g.refreshOnce()
+
+ for range time.Tick(controlRefresh) {
+ g.refreshOnce()
+ }
+}
+
+func (g *controlGateway) refreshOnce() {
+ var document policyDocument
+ policyOK := g.fetch(g.policy+routePolicyDocument, &document)
+
+ var set ruleSet
+ rulesOK := g.fetch(g.rules+routeRuleSet, &set)
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ g.freshPolicy = policyOK
+ if policyOK {
+ g.loadedPolicy = true
+ g.denied = document.Deny
+ }
+
+ g.freshRules = rulesOK
+ if rulesOK {
+ g.loadedRules = true
+ g.matched = set.Match
+ }
+}
+
+func (g *controlGateway) fetch(url string, into any) bool {
+ response, err := g.client.Get(url)
+ if err != nil {
+ return false
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ if response.StatusCode != http.StatusOK {
+ return false
+ }
+
+ return json.NewDecoder(response.Body).Decode(into) == nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controlgateway_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controlgateway_test.go
new file mode 100644
index 00000000..16ea6fa8
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/controlgateway_test.go
@@ -0,0 +1,709 @@
+// ©AngelaMos | 2026
+// controlgateway_test.go
+
+package main
+
+import (
+ "bufio"
+ "encoding/json"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+const controlTestSecret = "a-signing-key-for-this-test"
+
+// stubStore is a datastore that speaks enough RESP to answer the two commands
+// a fixed-window counter sends, and that can be taken away mid-test the way
+// khaos's proxy takes the real one away.
+type stubStore struct {
+ listener net.Listener
+
+ // resetting accepts the connection and then resets it, which is what
+ // khaos's proxy.Reset fault does. Closing the listener instead produces a
+ // refused DIAL, a different code path: the client fails before it ever
+ // writes a command, so the mid-protocol failure production actually hits
+ // goes unexercised.
+ resetting atomic.Bool
+
+ mu sync.Mutex
+ counts map[string]int64
+}
+
+func newStubStore(t *testing.T) *stubStore {
+ t.Helper()
+
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen for the stub datastore: %v", err)
+ }
+
+ store := &stubStore{listener: listener, counts: map[string]int64{}}
+ t.Cleanup(store.close)
+
+ go store.serve()
+
+ return store
+}
+
+func (s *stubStore) address() string {
+ return s.listener.Addr().String()
+}
+
+func (s *stubStore) close() {
+ _ = s.listener.Close()
+}
+
+func (s *stubStore) serve() {
+ for {
+ conn, err := s.listener.Accept()
+ if err != nil {
+ return
+ }
+
+ go s.handle(conn)
+ }
+}
+
+func (s *stubStore) reset() {
+ s.resetting.Store(true)
+}
+
+func (s *stubStore) handle(conn net.Conn) {
+ defer func() { _ = conn.Close() }()
+
+ if s.resetting.Load() {
+ if tcp, ok := conn.(*net.TCPConn); ok {
+ _ = tcp.SetLinger(0)
+ }
+
+ return
+ }
+
+ reader := bufio.NewReader(conn)
+
+ for {
+ parts, err := readCommand(reader)
+ if err != nil {
+ return
+ }
+
+ if len(parts) == 0 {
+ return
+ }
+
+ reply := ":1\r\n"
+
+ if strings.EqualFold(parts[0], respINCR) && len(parts) > 1 {
+ s.mu.Lock()
+ s.counts[parts[1]]++
+ reply = ":" + strconv.FormatInt(s.counts[parts[1]], 10) + "\r\n"
+ s.mu.Unlock()
+ }
+
+ if _, err = conn.Write([]byte(reply)); err != nil {
+ return
+ }
+ }
+}
+
+// readCommand parses the array-of-bulk-strings encoding respEncode writes,
+// which is the only shape the fixture ever sends.
+func readCommand(reader *bufio.Reader) ([]string, error) {
+ header, err := reader.ReadString('\n')
+ if err != nil {
+ return nil, err
+ }
+
+ count, err := strconv.Atoi(strings.TrimRight(header[1:], "\r\n"))
+ if err != nil {
+ return nil, err
+ }
+
+ parts := make([]string, 0, count)
+
+ for range count {
+ if _, err = reader.ReadString('\n'); err != nil {
+ return nil, err
+ }
+
+ value, err := reader.ReadString('\n')
+ if err != nil {
+ return nil, err
+ }
+
+ parts = append(parts, strings.TrimRight(value, "\r\n"))
+ }
+
+ return parts, nil
+}
+
+// controlFixture wires the gateway to a live datastore and a live backend, so
+// every test below measures the real client paths rather than a stub decision.
+func controlFixture(
+ t *testing.T,
+ failMode string,
+) (*controlGateway, *stubStore, *httptest.Server) {
+ t.Helper()
+
+ store := newStubStore(t)
+
+ backend := httptest.NewServer(newControlBackend().routes())
+ t.Cleanup(backend.Close)
+
+ gateway := newControlGateway(
+ store.address(),
+ backend.URL,
+ backend.URL,
+ failMode,
+ controlTestSecret,
+ defaultLimit,
+ )
+
+ return gateway, store, backend
+}
+
+// loaded waits for the first refresh: a control holding a last-known-good
+// starts without one, and that state is a different test.
+func loaded(t *testing.T, gateway *controlGateway) {
+ t.Helper()
+
+ deadline := time.Now().Add(5 * time.Second)
+
+ for time.Now().Before(deadline) {
+ gateway.mu.Lock()
+ ready := gateway.loadedPolicy && gateway.loadedRules
+ gateway.mu.Unlock()
+
+ if ready {
+ return
+ }
+
+ time.Sleep(10 * time.Millisecond)
+ }
+
+ t.Fatal("the gateway never loaded a policy or a rule set")
+}
+
+// awaitStale waits for the gateway to NOTICE its policy store is gone. It
+// fails rather than falling through: a timeout here means the state never
+// arrived, and reading that as a content failure sends the reader to the
+// wrong line.
+func awaitStale(t *testing.T, gateway *controlGateway) {
+ t.Helper()
+
+ deadline := time.Now().Add(5 * time.Second)
+
+ for time.Now().Before(deadline) {
+ gateway.mu.Lock()
+ stale := !gateway.freshPolicy
+ gateway.mu.Unlock()
+
+ if stale {
+ return
+ }
+
+ time.Sleep(10 * time.Millisecond)
+ }
+
+ t.Fatal("the gateway never noticed its policy store was unreachable")
+}
+
+func token(t *testing.T, mux *http.ServeMux, path string) string {
+ t.Helper()
+
+ recorder := httptest.NewRecorder()
+ mux.ServeHTTP(
+ recorder,
+ httptest.NewRequest(http.MethodPost, path, nil),
+ )
+
+ var body struct {
+ Token string `json:"token"`
+ }
+
+ if err := json.NewDecoder(recorder.Body).Decode(&body); err != nil {
+ t.Fatalf("decode the issued credential: %v", err)
+ }
+
+ if body.Token == "" {
+ t.Fatal("the fixture issued an empty credential")
+ }
+
+ return body.Token
+}
+
+func call(
+ t *testing.T,
+ mux *http.ServeMux,
+ path, credential string,
+) (int, string) {
+ t.Helper()
+
+ request := httptest.NewRequest(http.MethodGet, path, nil)
+ request.Header.Set(headerAuthorization, bearerPrefix+credential)
+
+ recorder := httptest.NewRecorder()
+ mux.ServeHTTP(recorder, request)
+
+ return recorder.Code, recorder.Header().Get(headerControlReason)
+}
+
+// Every control sits behind this service's own authentication, so a refusal
+// from one is never read as the service failing to identify the caller.
+func TestEveryControlAuthenticatesFirst(t *testing.T) {
+ gateway, _, _ := controlFixture(t, FailClosed)
+ mux := gateway.routes()
+
+ for _, path := range []string{
+ routeLimited,
+ routeGoverned,
+ routeFiltered,
+ } {
+ status, _ := call(t, mux, path, "a-credential-nobody-issued")
+ if status != http.StatusUnauthorized {
+ t.Errorf(
+ "%s answered %d to a credential this service never issued, want 401",
+ path,
+ status,
+ )
+ }
+ }
+}
+
+// While the limiter can count, the flagged credential has no budget and must
+// be throttled. That refusal is this control's negative control.
+func TestTheLimiterThrottlesWhatHasNoBudget(t *testing.T) {
+ gateway, _, _ := controlFixture(t, FailClosed)
+ mux := gateway.routes()
+
+ status, reason := call(t, mux, routeLimited, token(t, mux, issuePath))
+ if status != http.StatusOK {
+ t.Errorf(
+ "an ordinary credential within its budget got %d (%s), want 200",
+ status,
+ reason,
+ )
+ }
+
+ status, reason = call(
+ t,
+ mux,
+ routeLimited,
+ token(t, mux, routeFlaggedToken),
+ )
+ if status != http.StatusTooManyRequests {
+ t.Errorf(
+ "a credential with no budget got %d (%s), want 429",
+ status,
+ reason,
+ )
+ }
+}
+
+// F8, both directions: the datastore is severed while the limit itself is
+// still known, which is where Kong's fault_tolerant decides.
+func TestTheLimiterSplitsOnItsFailModeWhenTheDatastoreIsGone(t *testing.T) {
+ // The REASON, not only the status: a flagged credential is refused 429
+ // whether the limiter is over budget or cannot count at all, so a row
+ // asserting the status alone survives the severance being deleted.
+ for _, row := range []struct {
+ mode string
+ status int
+ reason string
+ }{
+ {FailOpen, http.StatusOK, reasonLimiterFellOpen},
+ {FailClosed, http.StatusTooManyRequests, reasonLimiterHeld},
+ } {
+ for _, sever := range []struct {
+ name string
+ how func(*stubStore)
+ }{
+ // Both shapes, because they take different paths through the
+ // client: a refused dial fails before a command is written, and
+ // a reset fails mid-protocol. khaos's proxy applies the second.
+ {"the datastore refuses the dial", (*stubStore).close},
+ {"the datastore resets the connection", (*stubStore).reset},
+ } {
+ t.Run(row.mode+", "+sever.name, func(t *testing.T) {
+ gateway, store, _ := controlFixture(t, row.mode)
+ mux := gateway.routes()
+
+ flagged := token(t, mux, routeFlaggedToken)
+
+ if status, _ := call(
+ t,
+ mux,
+ routeLimited,
+ flagged,
+ ); status != http.StatusTooManyRequests {
+ t.Fatalf(
+ "the limiter was not enforcing before the fault, got %d",
+ status,
+ )
+ }
+
+ sever.how(store)
+
+ status, reason := call(t, mux, routeLimited, flagged)
+ if status != row.status {
+ t.Errorf(
+ "with the datastore severed, %s answered %d (%s), want %d",
+ row.mode,
+ status,
+ reason,
+ row.status,
+ )
+ }
+
+ if reason != row.reason {
+ t.Errorf(
+ "%s refused for %q, want %q; the status alone cannot tell "+
+ "'over budget' from 'cannot count'",
+ row.mode,
+ reason,
+ row.reason,
+ )
+ }
+ })
+ }
+ }
+}
+
+// F21's negative control, which had none: the rule matcher has to REFUSE a
+// request a loaded rule matches. Without this the matcher can be gutted to
+// always allow and the whole package stays green, because the only other
+// test that calls it does so after its backend is already severed.
+func TestTheRuleMatcherRefusesWhatARuleMatches(t *testing.T) {
+ gateway, _, _ := controlFixture(t, FailClosed)
+ mux := gateway.routes()
+
+ loaded(t, gateway)
+
+ status, reason := call(
+ t,
+ mux,
+ routeFiltered,
+ token(t, mux, routeFlaggedToken),
+ )
+ if status != http.StatusForbidden || reason != reasonRulesMatched {
+ t.Errorf(
+ "a request a loaded rule matches got %d (%s), want 403 and %q",
+ status,
+ reason,
+ reasonRulesMatched,
+ )
+ }
+
+ status, reason = call(t, mux, routeFiltered, token(t, mux, issuePath))
+ if status != http.StatusOK || reason != reasonRulesClean {
+ t.Errorf(
+ "a request no rule matches got %d (%s), want 200 and %q",
+ status,
+ reason,
+ reasonRulesClean,
+ )
+ }
+}
+
+// The fail-CLOSED half of policy_store_down, which is the gate's HELD row and
+// had no unit coverage: a target that loses its store keeps enforcing the
+// last policy it loaded.
+func TestAFailClosedPolicyControlKeepsEnforcingWhatItLoaded(t *testing.T) {
+ gateway, _, backend := controlFixture(t, FailClosed)
+ mux := gateway.routes()
+
+ loaded(t, gateway)
+
+ flagged := token(t, mux, routeFlaggedToken)
+
+ backend.Close()
+ awaitStale(t, gateway)
+
+ status, reason := call(t, mux, routeGoverned, flagged)
+ if status != http.StatusForbidden || reason != reasonPolicyStale {
+ t.Errorf(
+ "a fail-closed target that lost its store answered %d (%s), want 403 and %q",
+ status,
+ reason,
+ reasonPolicyStale,
+ )
+ }
+
+ // And the legitimate subject is still served, which is what makes this
+ // HELD rather than DENIED on the live range.
+ if status, reason := call(
+ t,
+ mux,
+ routeGoverned,
+ token(t, mux, issuePath),
+ ); status != http.StatusOK {
+ t.Errorf(
+ "the legitimate subject stopped being served: %d (%s)",
+ status,
+ reason,
+ )
+ }
+}
+
+// A credential that is absent or not a bearer token is refused by this
+// service's own authentication, never by a control.
+func TestAMissingOrNonBearerCredentialIsRefusedBeforeAnyControl(t *testing.T) {
+ gateway, _, _ := controlFixture(t, FailClosed)
+ mux := gateway.routes()
+
+ for _, row := range []struct{ name, header string }{
+ {"no header at all", ""},
+ {"a non-bearer scheme", "Basic bm90LWEtYmVhcmVy"},
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ request := httptest.NewRequest(http.MethodGet, routeGoverned, nil)
+ if row.header != "" {
+ request.Header.Set(headerAuthorization, row.header)
+ }
+
+ recorder := httptest.NewRecorder()
+ mux.ServeHTTP(recorder, request)
+
+ if recorder.Code != http.StatusUnauthorized {
+ t.Errorf("%s got %d, want 401", row.name, recorder.Code)
+ }
+
+ if got := recorder.Header().Get(
+ headerControlReason,
+ ); got != reasonNoCredential {
+ t.Errorf("reason was %q, want %q", got, reasonNoCredential)
+ }
+ })
+ }
+}
+
+// F11 and F12 are two bugs reporting the same direction on the same
+// observable, so the fixture has to keep them apart.
+func TestThePolicyControlSeparatesLostFromNeverLoaded(t *testing.T) {
+ gateway, _, backend := controlFixture(t, FailOpen)
+ mux := gateway.routes()
+
+ loaded(t, gateway)
+
+ flagged := token(t, mux, routeFlaggedToken)
+
+ if status, _ := call(
+ t,
+ mux,
+ routeGoverned,
+ flagged,
+ ); status != http.StatusForbidden {
+ t.Fatalf(
+ "the policy was not being enforced before the fault, got %d",
+ status,
+ )
+ }
+
+ backend.Close()
+ awaitStale(t, gateway)
+
+ status, reason := call(t, mux, routeGoverned, flagged)
+ if status != http.StatusOK {
+ t.Errorf(
+ "a fail-open target that lost its policy answered %d (%s), want 200",
+ status,
+ reason,
+ )
+ }
+
+ if reason != reasonPolicyDropped {
+ t.Errorf(
+ "a target that LOST a loaded policy reported %q, which is the never-loaded wording",
+ reason,
+ )
+ }
+}
+
+// The cold start, which is the reading the F12 row is about: an empty policy
+// set usually means nothing is denied.
+func TestAColdStartedPolicyControlSplitsOnItsFailMode(t *testing.T) {
+ for _, row := range []struct {
+ mode string
+ status int
+ reason string
+ }{
+ {FailOpen, http.StatusOK, reasonPolicyNeverLoaded},
+ {FailClosed, http.StatusForbidden, reasonPolicyCannotEvaluate},
+ } {
+ t.Run(row.mode, func(t *testing.T) {
+ store := newStubStore(t)
+
+ gateway := newControlGateway(
+ store.address(),
+ "http://127.0.0.1:0",
+ "http://127.0.0.1:0",
+ row.mode,
+ controlTestSecret,
+ defaultLimit,
+ )
+ mux := gateway.routes()
+
+ status, reason := call(
+ t,
+ mux,
+ routeGoverned,
+ token(t, mux, routeFlaggedToken),
+ )
+
+ if status != row.status {
+ t.Errorf(
+ "a cold-started %s target answered %d, want %d",
+ row.mode,
+ status,
+ row.status,
+ )
+ }
+
+ if reason != row.reason {
+ t.Errorf("reason was %q, want %q", reason, row.reason)
+ }
+
+ // The STATE, not the first sample. Both rows above would hold
+ // for the first millisecond even against a reachable store,
+ // because the refresh goroutine has not run yet: this is what
+ // proves the gateway can never load rather than has not yet.
+ time.Sleep(3 * controlRefresh)
+
+ gateway.mu.Lock()
+ everLoaded := gateway.loadedPolicy
+ gateway.mu.Unlock()
+
+ if everLoaded {
+ t.Fatal(
+ "the gateway loaded a policy after all, so this test was measuring a race rather than a cold start",
+ )
+ }
+ })
+ }
+}
+
+// The cold-start experiment restarts this container mid-run, so a credential
+// khaos was issued before the restart has to still authenticate after it.
+//
+// Measured on a live range: with the credentials held in memory, the restart
+// left every one of them unknown, the valid stream went 401 for the whole
+// fault window, and khaos reported `deviated / DENIED` on a run that had
+// measured its own restart rather than the policy.
+func TestACredentialSurvivesARestartOfTheGateway(t *testing.T) {
+ before, store, backend := controlFixture(t, FailOpen)
+ credential := token(t, before.routes(), issuePath)
+
+ after := newControlGateway(
+ store.address(),
+ backend.URL,
+ backend.URL,
+ FailOpen,
+ controlTestSecret,
+ defaultLimit,
+ )
+ mux := after.routes()
+
+ loaded(t, after)
+
+ status, reason := call(t, mux, routeGoverned, credential)
+ if status == http.StatusUnauthorized {
+ t.Fatalf(
+ "a credential issued before the restart no longer authenticates (%s), so a cold-start run measures the restart rather than the control",
+ reason,
+ )
+ }
+}
+
+// F21. The rule path must be severable on its own even though it terminates
+// at the same backend as the policy path.
+func TestTheRuleAndPolicyPathsAreSeveredIndependently(t *testing.T) {
+ store := newStubStore(t)
+
+ policy := httptest.NewServer(newControlBackend().routes())
+ t.Cleanup(policy.Close)
+
+ rules := httptest.NewServer(newControlBackend().routes())
+ t.Cleanup(rules.Close)
+
+ gateway := newControlGateway(
+ store.address(),
+ policy.URL,
+ rules.URL,
+ FailOpen,
+ controlTestSecret,
+ defaultLimit,
+ )
+ mux := gateway.routes()
+
+ loaded(t, gateway)
+
+ flagged := token(t, mux, routeFlaggedToken)
+
+ // The rule path must be ENFORCING before it is severed, or this test
+ // passes against a matcher that never refused anything.
+ if status, reason := call(
+ t,
+ mux,
+ routeFiltered,
+ flagged,
+ ); status != http.StatusForbidden {
+ t.Fatalf(
+ "the rule matcher was not enforcing before the fault: %d (%s)",
+ status,
+ reason,
+ )
+ }
+
+ rules.Close()
+
+ deadline := time.Now().Add(5 * time.Second)
+ split := false
+ for time.Now().Before(deadline) && !split {
+ gateway.mu.Lock()
+ split = !gateway.freshRules && gateway.freshPolicy
+ gateway.mu.Unlock()
+
+ if !split {
+ time.Sleep(10 * time.Millisecond)
+ }
+ }
+
+ if !split {
+ t.Fatal(
+ "the gateway never reached the state this test needs: rules stale and policy still fresh",
+ )
+ }
+
+ if status, reason := call(
+ t,
+ mux,
+ routeFiltered,
+ flagged,
+ ); status != http.StatusOK {
+ t.Errorf(
+ "the severed rule path answered %d (%s), want a fail-open 200",
+ status,
+ reason,
+ )
+ }
+
+ if status, reason := call(
+ t,
+ mux,
+ routeGoverned,
+ flagged,
+ ); status != http.StatusForbidden {
+ t.Errorf(
+ "severing the rule path also stopped the policy path enforcing: %d (%s), want 403",
+ status,
+ reason,
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
index 5d2e2dbc..e407e3ce 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
@@ -11,6 +11,7 @@ import (
"net/url"
"os"
"os/signal"
+ "strconv"
"syscall"
"time"
@@ -29,6 +30,8 @@ const (
roleDB = "db"
roleSecret = "secret"
roleAuthz = "authz"
+ roleControl = "control"
+ roleControld = "controld"
roleHealth = "healthcheck"
roleAlive = "alivecheck"
@@ -63,6 +66,30 @@ const (
envAuthzMode = "KHAOS_RANGE_AUTHZ_MODE"
+ // One address per control, so khaos's proxy severs exactly one of the
+ // three. The policy store and the rule set terminate at the same backend
+ // and are still two dependencies, because the fault is in the path.
+ envLimitStore = "KHAOS_RANGE_LIMIT_STORE"
+ envPolicyStore = "KHAOS_RANGE_POLICY_STORE"
+ envRulesStore = "KHAOS_RANGE_RULES_STORE"
+ envLimit = "KHAOS_RANGE_LIMIT"
+
+ // What the control gateway authenticates WITH. It verifies a signature
+ // rather than remembering what it issued, because the cold-start
+ // experiment restarts it.
+ envControlSecret = "KHAOS_RANGE_CONTROL_SECRET"
+
+ // Deliberately unreachable, like every other dependency default: a
+ // plausible one would let the gateway start and measure nothing.
+ defaultLimitStore = "127.0.0.1:0"
+ defaultPolicyStore = "http://host.docker.internal:0"
+ defaultRulesStore = "http://host.docker.internal:0"
+
+ // Generous enough that khaos's own probe interval never exhausts it, so
+ // the budget the experiment measures is the flagged credential's zero and
+ // never an artefact of how fast khaos asks.
+ defaultLimit = 1000
+
// Where the service-to-service caller asks for an authorization decision,
// which is khaos itself: no correct dependency answers a definitive deny
// on request.
@@ -87,7 +114,7 @@ const (
// look like the fault's.
readHeaderTimeout = 120 * time.Second
- usage = "usage: khaos-range <idp|gateway|verifier|tls|rogue|resolver|db|secret|authz|healthcheck|alivecheck>"
+ usage = "usage: khaos-range <idp|gateway|verifier|tls|rogue|resolver|db|secret|authz|control|controld|healthcheck|alivecheck>"
selfBase = "http://127.0.0.1:8080"
selfHealthPath = "/health"
selfAlivePath = "/alive"
@@ -182,6 +209,17 @@ func roleHandler(role string) (http.Handler, error) {
envOr(envAuthzBase, defaultAuthzBase),
envOr(envAuthzMode, AuthzDistinguishes),
).routes(), nil
+ case roleControl:
+ return newControlGateway(
+ envOr(envLimitStore, defaultLimitStore),
+ envOr(envPolicyStore, defaultPolicyStore),
+ envOr(envRulesStore, defaultRulesStore),
+ envOr(envFailMode, FailClosed),
+ os.Getenv(envControlSecret),
+ int64Or(envLimit, defaultLimit),
+ ).routes(), nil
+ case roleControld:
+ return newControlBackend().routes(), nil
case roleSecret:
return newSecretGateway(
envOr(envSecretPath, defaultSecretPath),
@@ -251,6 +289,14 @@ func envOr(name, fallback string) string {
return fallback
}
+func int64Or(name string, fallback int64) int64 {
+ value, err := strconv.ParseInt(os.Getenv(name), 10, 64)
+ if err != nil || value < 0 {
+ return fallback
+ }
+ return value
+}
+
func durationOr(name string, fallback time.Duration) time.Duration {
value, err := time.ParseDuration(os.Getenv(name))
if err != nil || value <= 0 {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resp.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resp.go
new file mode 100644
index 00000000..aa9dccdd
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resp.go
@@ -0,0 +1,135 @@
+// ©AngelaMos | 2026
+// resp.go
+
+package main
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "net"
+ "strconv"
+ "strings"
+ "time"
+)
+
+const (
+ // INCR and EXPIRE are integer-reply in RESP2 and RESP3 alike, so this
+ // sends no HELLO. redis:7.4-alpine patches protected-mode off at build
+ // time and sets no password, so the first command on a socket runs.
+ respINCR = "INCR"
+ respEXPIRE = "EXPIRE"
+
+ respInteger = ':'
+ respError = '-'
+ respCRLF = "\r\n"
+
+ // A severed datastore has to reach the fault-tolerance branch inside one
+ // request, or the observable is a timeout rather than a decision.
+ respDialTimeout = 400 * time.Millisecond
+ respIOTimeout = 400 * time.Millisecond
+
+ errRESPReply = "the datastore answered %q, which is not an integer reply"
+)
+
+var errRESPRefused = errors.New("the datastore refused the command")
+
+// respCount is one fixed-window increment, dialled per call: a pooled client
+// would hide the severance behind a connection established before it.
+func respCount(address, key string, window time.Duration) (int64, error) {
+ conn, err := net.DialTimeout("tcp", address, respDialTimeout)
+ if err != nil {
+ return 0, err
+ }
+ defer func() { _ = conn.Close() }()
+
+ if err = conn.SetDeadline(time.Now().Add(respIOTimeout)); err != nil {
+ return 0, err
+ }
+
+ reader := bufio.NewReader(conn)
+
+ count, err := respCommand(conn, reader, respINCR, key)
+ if err != nil {
+ return 0, err
+ }
+
+ // INCR leaves an existing timeout untouched, so the window is set on the
+ // increment that created the key. A key per window means an EXPIRE that
+ // never lands leaks one key rather than disabling the limit.
+ if count == 1 {
+ seconds := strconv.FormatInt(int64(window.Seconds()), 10)
+ if _, err = respCommand(
+ conn,
+ reader,
+ respEXPIRE,
+ key,
+ seconds,
+ ); err != nil {
+ return 0, err
+ }
+ }
+
+ return count, nil
+}
+
+func respCommand(
+ conn net.Conn,
+ reader *bufio.Reader,
+ parts ...string,
+) (int64, error) {
+ if _, err := conn.Write([]byte(respEncode(parts))); err != nil {
+ return 0, err
+ }
+
+ return respInteger64(reader)
+}
+
+func respEncode(parts []string) string {
+ var out strings.Builder
+
+ out.WriteString("*" + strconv.Itoa(len(parts)) + respCRLF)
+
+ for _, part := range parts {
+ out.WriteString("$" + strconv.Itoa(len(part)) + respCRLF)
+ out.WriteString(part + respCRLF)
+ }
+
+ return out.String()
+}
+
+// respInteger64 reads the only two replies these commands produce. WRONGTYPE,
+// a non-integer value and an int64 overflow all arrive on the error line and
+// must never be read as a count.
+func respInteger64(reader *bufio.Reader) (int64, error) {
+ line, err := reader.ReadString('\n')
+ if err != nil {
+ return 0, err
+ }
+
+ line = strings.TrimRight(line, respCRLF)
+
+ // Load-bearing rather than defensive: without it line[0] on an empty
+ // reply is an index panic that takes this process down mid-run.
+ if line == "" {
+ return 0, fmt.Errorf(errRESPReply, line)
+ }
+
+ if line[0] == respError {
+ return 0, fmt.Errorf("%w: %s", errRESPRefused, line[1:])
+ }
+
+ if line[0] != respInteger {
+ return 0, fmt.Errorf(errRESPReply, line)
+ }
+
+ // Zero on failure, explicitly. ParseInt returns MaxInt64 alongside its
+ // error on overflow, and a count returned beside an error is one a caller
+ // can read before noticing it is garbage.
+ count, err := strconv.ParseInt(line[1:], 10, 64)
+ if err != nil {
+ return 0, err
+ }
+
+ return count, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resp_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resp_test.go
new file mode 100644
index 00000000..80df0873
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resp_test.go
@@ -0,0 +1,105 @@
+// ©AngelaMos | 2026
+// resp_test.go
+
+package main
+
+import (
+ "bufio"
+ "errors"
+ "strings"
+ "testing"
+)
+
+// An error reply must never be read as a count. A limiter that read -ERR as a
+// number would compare garbage against its budget and decide from it, which
+// is worse than failing: it would decide WRONGLY and report confidently.
+//
+// The empty row is the one whose guard is load-bearing rather than redundant:
+// without it `line[0]` on an empty reply is an index panic that takes the
+// gateway down mid-run.
+func TestOnlyAnIntegerReplyIsReadAsACount(t *testing.T) {
+ for _, row := range []struct {
+ name string
+ reply string
+ want int64
+ counted bool
+ }{
+ {"an integer", ":5\r\n", 5, true},
+ {"zero", ":0\r\n", 0, true},
+ {"an unknown command", "-ERR unknown command\r\n", 0, false},
+ {
+ "the wrong type",
+ "-WRONGTYPE Operation against a key holding the wrong kind of value\r\n",
+ 0,
+ false,
+ },
+ {
+ "a non-integer value",
+ "-ERR value is not an integer or out of range\r\n",
+ 0,
+ false,
+ },
+ {"an empty reply", "\r\n", 0, false},
+ {"a simple string", "+PONG\r\n", 0, false},
+ {"an int64 overflow", ":9223372036854775808\r\n", 0, false},
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ count, err := respInteger64(
+ bufio.NewReader(strings.NewReader(row.reply)),
+ )
+
+ if row.counted {
+ if err != nil {
+ t.Fatalf("an integer reply errored: %v", err)
+ }
+
+ if count != row.want {
+ t.Fatalf("count = %d, want %d", count, row.want)
+ }
+
+ return
+ }
+
+ if err == nil {
+ t.Fatalf(
+ "%q was read as the count %d rather than refused",
+ row.reply,
+ count,
+ )
+ }
+
+ if count != 0 {
+ t.Fatalf(
+ "%q errored AND returned %d; a caller reading the count "+
+ "before the error gets garbage",
+ row.reply,
+ count,
+ )
+ }
+ })
+ }
+}
+
+// The error line is classified, so a refusal is distinguishable from a
+// transport failure by something other than its text.
+func TestARefusalIsDistinguishableFromATransportFailure(t *testing.T) {
+ _, err := respInteger64(
+ bufio.NewReader(strings.NewReader("-ERR unknown command\r\n")),
+ )
+
+ if !errors.Is(err, errRESPRefused) {
+ t.Fatalf("an -ERR reply did not classify as a refusal: %v", err)
+ }
+}
+
+// The exact bytes, because the stub datastore's own parser discards the bulk
+// length lines: a wrong length is invisible to it and a protocol error to
+// real Redis.
+func TestTheEncodingIsAnArrayOfBulkStrings(t *testing.T) {
+ got := respEncode([]string{respINCR, "a-key"})
+ want := "*2\r\n$4\r\nINCR\r\n$5\r\na-key\r\n"
+
+ if got != want {
+ t.Fatalf("respEncode = %q, want %q", got, want)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
index f21ed690..3946b7a7 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
@@ -799,6 +799,7 @@ func rangeSupplies() []string {
experiment.ParamTarget,
experiment.ParamURL,
experiment.ParamToken,
+ experiment.ParamRefusedFrom,
experiment.ParamAuthority,
experiment.ParamRevocable,
experiment.ParamIssuer,
@@ -838,9 +839,29 @@ const (
tierSecret
tierEnv
tierAuthz
+
+ // Three tiers over ONE gateway, because that gateway carries three
+ // controls and each has its own measurement path. The mechanism cannot
+ // tell them apart: all three sever with the same proxy.
+ tierControlLimiter
+ tierControlPolicy
+ tierControlRules
)
-func tierOf(kind string) tier {
+func tierOf(definition experiment.Definition) tier {
+ switch definition.Control {
+ case experiment.SubjectLimiter:
+ return tierControlLimiter
+ case experiment.SubjectPolicy:
+ return tierControlPolicy
+ case experiment.SubjectRules:
+ return tierControlRules
+ }
+
+ return tierForKind(definition.FaultKind)
+}
+
+func tierForKind(kind string) tier {
switch {
case inject.NeedsProxy(kind):
return tierProxy
@@ -901,6 +922,24 @@ type tierAnswers struct {
readyURL string
secretPath string
variable string
+
+ // Where khaos fetches a fresh input the control must refuse, once per
+ // probe. The control family's negative control is signed, so it cannot
+ // carry an appended nonce and has to be re-issued instead.
+ refusedFrom string
+}
+
+// controlPath is the endpoint whose answer one control decides. Three tiers
+// share one gateway, so the path is the only thing that separates them.
+func controlPath(at tier) string {
+ switch at {
+ case tierControlLimiter:
+ return rangelab.LimitedPath
+ case tierControlPolicy:
+ return rangelab.GovernedPath
+ default:
+ return rangelab.FilteredPath
+ }
}
func answersFor(lab rangelab.Lab, at tier) tierAnswers {
@@ -973,6 +1012,14 @@ func answersFor(lab rangelab.Lab, at tier) tierAnswers {
answers.url = lab.AuthzProtectedURL()
answers.target = rangelab.AuthzGatewayName
+ // One gateway, three endpoints. The target is the gateway in all three:
+ // the fault lands on khaos's proxy in front of a dependency, so the
+ // gateway is the only container whose consent is needed.
+ case tierControlLimiter, tierControlPolicy, tierControlRules:
+ answers.url = lab.ControlGatewayURL() + controlPath(at)
+ answers.target = rangelab.ControlGatewayName
+ answers.refusedFrom = lab.RefusedInputURL()
+
default:
// The direct tier's URL and target are the values set above.
}
@@ -1019,6 +1066,7 @@ func rangeDefaults(
// sends an operator looking for a flag that does nothing.
for name, value := range map[string]string{
experiment.ParamURL: tiered.url,
+ experiment.ParamRefusedFrom: tiered.refusedFrom,
experiment.ParamEngine: tiered.engine,
experiment.ParamEngineTarget: tiered.engineTarget,
experiment.ParamSubjectRole: tiered.subjectRole,
@@ -1068,7 +1116,7 @@ func fillFromRange(
}
// Refuse before announcing, or the note names a URL resolved to port 0.
- at := tierOf(definition.FaultKind)
+ at := tierOf(definition)
if err := requireTier(lab, at); err != nil {
return nil, err
@@ -1160,6 +1208,22 @@ func requireTier(lab rangelab.Lab, at tier) error {
case tierAuthz:
return lab.RequireAuthzTier()
+ // The limiter is the one control whose dependency is a real engine, so it
+ // is the one that needs the opt-in stateful tier. The other two are
+ // refused only when the gateway or its address is missing.
+ case tierControlLimiter:
+ if err := lab.RequireStatefulTier(); err != nil {
+ return err
+ }
+
+ return lab.RequireControlTier(rangelab.ControlLimiter)
+
+ case tierControlPolicy:
+ return lab.RequireControlTier(rangelab.ControlPolicy)
+
+ case tierControlRules:
+ return lab.RequireControlTier(rangelab.ControlRules)
+
default:
// The direct tier needs only the gateway Discover already found.
}
@@ -1239,7 +1303,7 @@ func startProxy(
) (*proxy.Proxy, error) {
listen, upstream := spec.ProxyListen, spec.ProxyUpstream
- fault, err := proxy.Open(listen, upstream)
+ fault, err := proxy.Open(listen, upstream, spec.ProxyProbe)
if err != nil {
if errors.Is(err, proxy.ErrUnsafeBind) {
return nil, err
@@ -1923,7 +1987,8 @@ func portOf(issuer string) (int, error) {
}
func credentialParam(name string) bool {
- return name == experiment.ParamToken || name == experiment.ParamRevocable
+ return name == experiment.ParamToken ||
+ name == experiment.ParamRevocable
}
// askForCredentials mints whichever credentials this experiment declared, from
@@ -1966,6 +2031,27 @@ func askForCredentials(
return nil
}
+// proxyEndpointFor resolves which dependency this run's proxy sits in front
+// of. Every control family member severs with the same mechanism, so the
+// address comes from the tier rather than from the fault.
+func proxyEndpointFor(
+ ctx context.Context,
+ reader rangelab.PortReader,
+ lab rangelab.Lab,
+ at tier,
+) (string, string, error) {
+ switch at {
+ case tierControlLimiter:
+ return lab.ControlEndpoint(ctx, reader, rangelab.ControlLimiter)
+ case tierControlPolicy:
+ return lab.ControlEndpoint(ctx, reader, rangelab.ControlPolicy)
+ case tierControlRules:
+ return lab.ControlEndpoint(ctx, reader, rangelab.ControlRules)
+ default:
+ return lab.ProxyEndpoint(ctx, reader)
+ }
+}
+
// issuerFor is whichever fixture will be the one VERIFYING. For the secret
// family that is the fixture itself, because khaos never reads the secret and a
// token signed with any other key would be refused for a reason that has
@@ -1996,6 +2082,9 @@ func issuerFor(
return lab.IssueAuthzToken, nil
+ case tierControlLimiter, tierControlPolicy, tierControlRules:
+ return lab.IssueControlToken, nil
+
default:
// Every other tier's credential comes from the range's own identity
// provider through the direct gateway.
@@ -2019,7 +2108,7 @@ func askTheRange(
if want(experiment.ParamProxyListen) ||
want(experiment.ParamProxyUpstream) {
- listen, upstream, err := lab.ProxyEndpoint(ctx, reader)
+ listen, upstream, err := proxyEndpointFor(ctx, reader, lab, at)
if err != nil {
return rangeAnswers{}, err
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
index f06bb715..34a7ae6f 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
@@ -181,6 +181,9 @@ func allTiers() []tier {
tierSecret,
tierEnv,
tierAuthz,
+ tierControlLimiter,
+ tierControlPolicy,
+ tierControlRules,
}
}
@@ -204,6 +207,12 @@ func tierName(at tier) string {
return "at the variable the service verifies with"
case tierAuthz:
return "at the service whose authorization dependency khaos answers"
+ case tierControlLimiter:
+ return "at the rate limiter's datastore"
+ case tierControlPolicy:
+ return "at the policy engine's policy store"
+ case tierControlRules:
+ return "at the rule matcher's rules backend"
default:
return "direct"
}
@@ -239,6 +248,14 @@ func fullLab() rangelab.Lab {
// the caller itself publishes.
AuthzPort: 30014,
AuthzBase: "http://host.docker.internal:30013",
+
+ // The control gateway, and the three addresses it was told to find
+ // its three dependencies on. Distinct from each other because the
+ // whole point of the tier is that khaos severs exactly one.
+ ControlPort: 30015,
+ ControlStoreAddr: "host.docker.internal:30016",
+ ControlPolicyAddr: "http://host.docker.internal:30017",
+ ControlRulesAddr: "http://host.docker.internal:30018",
}
}
@@ -750,7 +767,7 @@ func TestTheRangeCanSatisfyEveryExperimentsRequiredParameters(t *testing.T) {
defaults := rangeDefaults(
lab,
testAnswers,
- tierOf(definition.FaultKind),
+ tierOf(definition),
)
supplied := map[string]string{}
@@ -808,6 +825,10 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
tierSecret: lab.SecretProtectedURL(),
tierEnv: lab.EnvProtectedURL(),
tierAuthz: lab.AuthzProtectedURL(),
+
+ tierControlLimiter: lab.ControlGatewayURL() + rangelab.LimitedPath,
+ tierControlPolicy: lab.ControlGatewayURL() + rangelab.GovernedPath,
+ tierControlRules: lab.ControlGatewayURL() + rangelab.FilteredPath,
}
seen := map[string]tier{}
@@ -826,7 +847,7 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
for _, definition := range experiment.Catalog() {
t.Run(definition.Name, func(t *testing.T) {
- at := tierOf(definition.FaultKind)
+ at := tierOf(definition)
exercised[at] = true
defaults := rangeDefaults(lab, testAnswers, at)
@@ -883,6 +904,13 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
case tierAuthz:
wantTarget = rangelab.AuthzGatewayName
+ // One gateway for all three, because the fault lands on khaos's
+ // proxy in front of a dependency rather than on the dependency.
+ // The three tiers differ in which of its controls is measured,
+ // never in whose consent is needed.
+ case tierControlLimiter, tierControlPolicy, tierControlRules:
+ wantTarget = rangelab.ControlGatewayName
+
default:
// The direct and proxy tiers fault the dependency itself.
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
index ab70925c..49941fea 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
@@ -52,7 +52,7 @@ const (
flagAuthzMode = "authz-mode"
flagDNSMode = "dns-mode"
- mintedSecrets = 4
+ mintedSecrets = 5
rangeShort = "bring up a target stack that is deliberately breakable"
rangeLong = `The range is a small stack khaos ships so you have something to break before you
@@ -253,6 +253,13 @@ func mintRangeSecrets(lab *rangelab.Lab) error {
// image is what lets it be mounted read-only.
lab.SecretValue = secrets[3]
+ // The control gateway's signing key. It authenticates by SIGNATURE rather
+ // than from a set it remembers, because the cold-start experiment restarts
+ // it: an in-memory set is empty afterwards, every credential khaos holds
+ // reads as unknown, and the run measures the restart instead of the
+ // policy. Measured on a live range before this key existed.
+ lab.ControlSecret = secrets[4]
+
return nil
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
index 66000b1d..7378ff5e 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
@@ -58,6 +58,11 @@ type Shape struct {
ProxyListen string
ProxyUpstream string
+ // What the confirmation probe says to the dependency. Empty keeps the
+ // HTTP request; a dependency that does not speak HTTP answers one with
+ // nothing, which the oracle correctly reads as unreachable.
+ ProxyProbe string
+
IdP *idp.Provider
IdPTrap *idp.Trap
@@ -119,6 +124,12 @@ type Shape struct {
ShiftDetail string
ShiftWhat string
+ // ColdStart restarts the target inside the fault window, so it
+ // INITIALISES with the dependency already unreachable. The baseline has
+ // to precede it: a target that boots broken refuses nothing, so khaos's
+ // own guard would void the run rather than measure one.
+ ColdStart bool
+
Unsupportable string
CacheRequired string
DeliveredWhat string
@@ -139,8 +150,14 @@ type Definition struct {
Hypothesis Hypothesis
Polarity Polarity
FaultKind string
- Params []Param
- Build func(*Params) (Shape, error)
+
+ // Control is which of a gateway's controls this experiment severs, for
+ // the family whose four members share one mechanism and one gateway and
+ // differ only in the dependency they take away.
+ Control ControlSubject
+
+ Params []Param
+ Build func(*Params) (Shape, error)
}
const (
@@ -293,6 +310,7 @@ func Build(
ProxyFault: shape.ProxyFault,
ProxyListen: shape.ProxyListen,
ProxyUpstream: shape.ProxyUpstream,
+ ProxyProbe: shape.ProxyProbe,
IdP: shape.IdP,
IdPTrap: shape.IdPTrap,
@@ -305,6 +323,7 @@ func Build(
Shift: shape.Shift,
ShiftDetail: shape.ShiftDetail,
ShiftWhat: shape.ShiftWhat,
+ ColdStart: shape.ColdStart,
AuthorityBase: shape.AuthorityBase,
AuthorityListen: shape.AuthorityListen,
AuthorityTLS: shape.AuthorityTLS,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/control.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/control.go
new file mode 100644
index 00000000..89e21034
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/control.go
@@ -0,0 +1,308 @@
+// ©AngelaMos | 2026
+// control.go
+
+package experiment
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ FamilyControl = "control"
+
+ NameRatelimitStoreDown = "control.ratelimit_store_down"
+ NamePolicyStoreDown = "control.policy_store_down"
+ NamePolicyStoreColdStart = "control.policy_store_cold_start"
+ NameWAFRulesBackendDown = "control.waf_rules_backend_down"
+
+ // An ENDPOINT rather than a value, and that is forced rather than chosen.
+ // This family's negative control has to be authentic and still refused by
+ // the control, so it is signed -- and a signed credential cannot carry an
+ // appended nonce without becoming a different fault. Fetching a fresh one
+ // per probe is the only way to vary it, and vary it khaos must: a control
+ // with a decision cache answers a repeated input from one cached refusal,
+ // so a fixed one makes the fail-closed direction unmeasurable.
+ //
+ // Its own parameter rather than --invalid-token for the same reason: the
+ // range can only answer it for this family, and a name the range claims
+ // to answer everywhere and answers in one place sends an operator looking
+ // for a flag that does nothing.
+ ParamRefusedFrom = "refused-from"
+)
+
+// ControlSubject is which of a gateway's controls an experiment severs. The
+// experiment declares it because the mechanism cannot be read off the fault:
+// all four sever with the same proxy and differ only in what they sever.
+type ControlSubject string
+
+const (
+ SubjectLimiter ControlSubject = "limiter"
+ SubjectPolicy ControlSubject = "policy"
+ SubjectRules ControlSubject = "rules"
+)
+
+const (
+ usageControlURL = "the endpoint the control you are testing guards, which is the only endpoint whose answer that control decides"
+
+ usageControlRefused = "an endpoint that issues an input this control should refuse: for a rate limiter a credential whose budget is already exhausted, for a policy engine a subject the policy denies, for a rule matcher a request a rule matches. khaos fetches a fresh one for every probe, because a control holding a decision cache answers a repeated input from one cached refusal. This is the negative control, and a run in which khaos never sees it refused is voided rather than concluded"
+
+ conditionRatelimit = "%s losing the datastore its rate limiter counts in"
+ conditionPolicy = "%s losing the store its authorization policy is loaded from"
+ conditionColdStart = "%s starting up with the store its authorization policy is loaded from already unreachable"
+ conditionRules = "%s losing the backend its rule set is loaded from"
+
+ summaryRatelimit = "take away the datastore a rate limiter counts in, which is the one control in the taxonomy whose best-known implementation disables itself by default when it cannot count"
+
+ summaryPolicy = "take away the store an authorization policy is loaded from while the target is running, which finds a service that drops the policy it already had"
+
+ summaryColdStart = "start the target with its policy store already unreachable, which finds a service whose empty policy set denies nothing -- a different bug from losing a policy it had loaded, and one that reports the same verdict"
+
+ summaryRules = "take away the backend a rule set is loaded from, which finds the operator's own fallback rather than a vendor default: of the vendors that document one, both deny"
+
+ responseRatelimit = "continuing to refuse requests over the budget, rather than serving them because it cannot count them"
+
+ responsePolicy = "continuing to refuse the subjects its policy denies, rather than serving them because it cannot re-read the policy"
+
+ responseColdStart = "refusing to serve requests it cannot evaluate, rather than treating an empty policy set as a policy that denies nothing"
+
+ responseRules = "continuing to refuse the requests its rules match, rather than passing traffic it cannot inspect"
+
+ defectRatelimit = "a rate limiter that disables itself when its datastore is unreachable"
+
+ defectPolicy = "an authorization policy that stops being enforced when its store is unreachable"
+
+ defectColdStart = "an empty policy set read as a policy that denies nothing"
+
+ defectRules = "a rule set that stops being matched when its backend is unreachable"
+
+ caveatControlHeld = "khaos severed the dependency and confirmed it, and this control kept refusing what it refused before; that is the correct outcome and it carries an availability cost khaos did not measure"
+
+ probeRefusedName = "input-this-control-refuses"
+
+ describeRefused = "an input this control refused while its dependency was reachable, fetched fresh per probe and presented again once it was not"
+
+ errRefusedStatus = "%s answered %d rather than issuing an input this control should refuse"
+
+ errRefusedEmpty = "%s issued an empty input, so this run has no negative control"
+
+ // Short: a negative control khaos cannot fetch inside one sample interval
+ // is a probe failure rather than target behaviour, and the run says so.
+ refusedTimeout = 3 * time.Second
+)
+
+var refusedClient = &http.Client{Timeout: refusedTimeout}
+
+// controlPolarity declares PERMITTED alone. DENIED is this control refusing
+// to act on information it does not have, which is correct with an
+// availability cost rather than a defect.
+//
+// The rationale states the counter-argument out loud for the limiter: a
+// limiter that disables itself when its datastore is gone is a defensible
+// trade when it is deliberate, Kong ships exactly that, and khaos still
+// reports it because the operator ran an experiment named after a control.
+var controlPolarity = Polarity{
+ Defects: []Direction{Permitted},
+ Rationale: "a control that stops refusing what it refused a moment ago has stopped being a control; refusing to serve while it cannot consult its dependency is the correct outcome with an availability cost, not a defect",
+}
+
+func controlParams() []Param {
+ return []Param{
+ {Name: ParamTarget, Usage: usageTarget, Required: true},
+ {Name: ParamURL, Usage: usageControlURL, Required: true},
+ {Name: ParamToken, Usage: usageToken, Required: true},
+ {
+ Name: ParamRefusedFrom,
+ Usage: usageControlRefused,
+ Required: true,
+ },
+ {Name: ParamProxyListen, Usage: usageProxyListen, Required: true},
+ {
+ Name: ParamProxyUpstream,
+ Usage: usageProxyUpstream,
+ Required: true,
+ },
+ }
+}
+
+// controlProbes drives the guarded endpoint with an input the control allows
+// and one it must refuse. Both are credentials the service AUTHENTICATES, so
+// a 401 is never what either stream measures.
+func controlProbes(params *Params) probe.Pair {
+ url := params.Get(ParamURL)
+
+ return probe.Pair{
+ Valid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeValidName,
+ URL: url,
+ Token: params.Get(ParamToken),
+ Credential: probe.Valid,
+ }),
+ Invalid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeRefusedName,
+ Describe: describeRefused,
+ URL: url,
+ Credential: probe.Invalid,
+ Mint: mintRefused(params.Get(ParamRefusedFrom)),
+ }),
+ }
+}
+
+// mintRefused fetches one refused input per probe. A signed credential cannot
+// carry an appended nonce without becoming a different fault, so this is how
+// the negative control varies.
+func mintRefused(from string) func(context.Context) (string, error) {
+ return func(ctx context.Context) (string, error) {
+ request, err := http.NewRequestWithContext(
+ ctx,
+ http.MethodPost,
+ from,
+ nil,
+ )
+ if err != nil {
+ return "", err
+ }
+
+ response, err := refusedClient.Do(request)
+ if err != nil {
+ return "", err
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ if response.StatusCode != http.StatusOK {
+ return "", fmt.Errorf(errRefusedStatus, from, response.StatusCode)
+ }
+
+ var body struct {
+ Token string `json:"token"`
+ }
+
+ if err := json.NewDecoder(response.Body).Decode(&body); err != nil {
+ return "", err
+ }
+
+ if body.Token == "" {
+ return "", fmt.Errorf(errRefusedEmpty, from)
+ }
+
+ return body.Token, nil
+ }
+}
+
+// buildControl severs one dependency with the proxy tier. closed is what this
+// control's refusal looks like beyond a Rejected: a limiter answers 429,
+// which says nothing about a credential and is still a refusal.
+func buildControl(
+ condition, defect, probePayload string,
+ closed []probe.Disposition,
+ cold bool,
+) func(*Params) (Shape, error) {
+ return func(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+ listen, upstream := proxyEndpoint(params)
+
+ shape := Shape{
+ Condition: fmt.Sprintf(condition, target),
+ Target: target,
+ Probes: controlProbes(params),
+ ProxyFault: &proxy.Fault{Kind: proxy.Reset},
+ ProxyListen: listen,
+ ProxyUpstream: upstream,
+ ProxyProbe: probePayload,
+ Defect: defect,
+ HeldCaveat: caveatControlHeld,
+ ClosedWhen: closed,
+ }
+
+ if cold {
+ shape.ColdStart = true
+ }
+
+ return shape, nil
+ }
+}
+
+func controlDefinition(
+ name string,
+ subject ControlSubject,
+ summary, condition, response, defect, probePayload string,
+ closed []probe.Disposition,
+ cold bool,
+) Definition {
+ return Definition{
+ Name: name,
+ Family: FamilyControl,
+ Summary: summary,
+ Hypothesis: Hypothesis{
+ Condition: fmt.Sprintf(condition, "the target"),
+ Response: response,
+ },
+ Polarity: controlPolarity,
+ FaultKind: inject.ProxyKind(proxy.Reset),
+ Control: subject,
+ Params: controlParams(),
+ Build: buildControl(condition, defect, probePayload, closed, cold),
+ }
+}
+
+func init() {
+ catalogAdd(controlDefinition(
+ NameRatelimitStoreDown,
+ SubjectLimiter,
+ summaryRatelimit,
+ conditionRatelimit,
+ responseRatelimit,
+ defectRatelimit,
+ // The only dependency in the catalogue that does not speak HTTP, and
+ // the oracle needs it to ANSWER on the direct path.
+ proxy.ProbeRESP,
+ // A limiter's refusal is a 429, on both streams: the over-budget
+ // request it must keep refusing, and the request it refuses because
+ // it can no longer count at all.
+ []probe.Disposition{probe.Throttled},
+ false,
+ ))
+
+ catalogAdd(controlDefinition(
+ NamePolicyStoreDown,
+ SubjectPolicy,
+ summaryPolicy,
+ conditionPolicy,
+ responsePolicy,
+ defectPolicy,
+ "",
+ nil,
+ false,
+ ))
+
+ catalogAdd(controlDefinition(
+ NamePolicyStoreColdStart,
+ SubjectPolicy,
+ summaryColdStart,
+ conditionColdStart,
+ responseColdStart,
+ defectColdStart,
+ "",
+ nil,
+ true,
+ ))
+
+ catalogAdd(controlDefinition(
+ NameWAFRulesBackendDown,
+ SubjectRules,
+ summaryRules,
+ conditionRules,
+ responseRules,
+ defectRules,
+ "",
+ nil,
+ false,
+ ))
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
index 8caed1aa..ffadf81b 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
@@ -15,17 +15,18 @@ import (
type Stage string
const (
- StageBaseline Stage = "baseline"
- StageInject Stage = "inject"
- StageConfirm Stage = "confirm"
- StageObserve Stage = "observe"
- StageAct Stage = "act"
- StageShift Stage = "shift"
- StageIdle Stage = "idle"
- StageResume Stage = "resume"
- StageRevert Stage = "revert"
- StageRecover Stage = "recover"
- StageVerdict Stage = "verdict"
+ StageBaseline Stage = "baseline"
+ StageInject Stage = "inject"
+ StageConfirm Stage = "confirm"
+ StageObserve Stage = "observe"
+ StageAct Stage = "act"
+ StageShift Stage = "shift"
+ StageColdStart Stage = "cold-start"
+ StageIdle Stage = "idle"
+ StageResume Stage = "resume"
+ StageRevert Stage = "revert"
+ StageRecover Stage = "recover"
+ StageVerdict Stage = "verdict"
)
const (
@@ -42,6 +43,12 @@ const (
detailShiftFailed = "the second phase could not be established, so khaos has one window where it needs two: %v"
detailIdle = "a credential leaked; pausing traffic for %s to see whether the cache is being re-armed"
detailResume = "resuming traffic with the fault still applied"
+ detailColdStart = "restarting the target so it initialises with the dependency already unreachable"
+
+ // How long the target gets to answer again after the restart. The
+ // settling samples are discarded: counting them would report khaos's own
+ // restart as the target having stopped answering.
+ settleFor = 30 * time.Second
// Breaks aliasing against the periodic target behaviour these experiments
// measure: cache TTLs, token refresh and crons.
@@ -62,8 +69,16 @@ type Injector interface {
Inject(ctx context.Context, kind, target string) error
Confirm(ctx context.Context) (string, error)
Revert(ctx context.Context) error
+
+ // Restart restarts the container the fault was applied to, for the one
+ // experiment whose fault is the target initialising without a dependency.
+ Restart(ctx context.Context) error
}
+var ErrNeverCameBack = errors.New(
+ "khaos restarted the target and it never answered again, so there is no window to measure; the fault is still applied and was reverted",
+)
+
var ErrNoProbePair = errors.New(
"an experiment needs both a valid and an invalid probe; the invalid one is the negative control that makes the verdict mean anything",
)
@@ -72,6 +87,11 @@ type Engine struct {
Injector Injector
Report Reporter
Now func() time.Time
+
+ // SettleFor bounds how long a cold-started target gets to answer again.
+ // Injectable for the same reason Now is: the default is longer than a
+ // test can wait for.
+ SettleFor time.Duration
}
// abandon reverts the fault and then concludes. The cause is joined with the
@@ -239,6 +259,7 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
observation.Baseline,
observation.ForgedBaseline,
observation.MintedByKhaos,
+ closedWhen(observation),
); unmet {
return e.conclude(result, observation, spec.Polarity, nil)
}
@@ -272,6 +293,10 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
return e.abandon(ctx, spec, result, observation, actErr)
}
+ if coldErr := e.coldStart(ctx, spec, timing); coldErr != nil {
+ return e.abandon(ctx, spec, result, observation, coldErr)
+ }
+
observation = e.measureFault(ctx, spec, timing, observation)
observation = e.shift(ctx, spec, timing, observation)
@@ -287,6 +312,67 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
return e.conclude(result, observation, spec.Polarity, revertErr)
}
+// coldStart restarts the target with the fault already applied, so it
+// initialises without a dependency it held a moment ago. The baseline is
+// already measured by this point, which is the whole ordering: a target
+// booted broken refuses nothing and voids its own run.
+//
+// It then waits for the target to ANSWER rather than for the daemon to call
+// it running, and the settling samples are discarded: counting them would
+// report khaos's own restart as the target having stopped answering.
+func (e Engine) coldStart(
+ ctx context.Context,
+ spec Experiment,
+ timing Timing,
+) error {
+ if !spec.ColdStart {
+ return nil
+ }
+
+ e.emit(StageColdStart, detailColdStart)
+
+ if err := e.Injector.Restart(ctx); err != nil {
+ return err
+ }
+
+ if e.settled(ctx, spec, timing) {
+ return nil
+ }
+
+ return ErrNeverCameBack
+}
+
+// settled polls the valid stream until the target answers at all. A refusal
+// counts: a correct target that cannot evaluate its policy refuses every
+// request, and that is an answer.
+func (e Engine) settled(
+ ctx context.Context,
+ spec Experiment,
+ timing Timing,
+) bool {
+ deadline := e.now().Add(e.settleBudget())
+
+ for e.now().Before(deadline) {
+ if ctx.Err() != nil {
+ return false
+ }
+
+ if spec.Probes.Valid.Run(ctx).Disposition.Answered() {
+ return true
+ }
+
+ timer := time.NewTimer(timing.SampleInterval)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return false
+ case <-timer.C:
+ }
+ }
+
+ return false
+}
+
// measureFault records whether the target ever MET the fault, not just whether
// khaos armed it: Go's HTTP transport resolves a name once per kept-alive
// connection, so a DNS fault can be armed, confirmed, and reach nothing.
@@ -527,6 +613,13 @@ func jitter(interval time.Duration) time.Duration {
return time.Duration(rand.Float64() * span)
}
+func (e Engine) settleBudget() time.Duration {
+ if e.SettleFor > 0 {
+ return e.SettleFor
+ }
+ return settleFor
+}
+
func (e Engine) now() time.Time {
if e.Now != nil {
return e.Now()
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine_test.go
index b459c6e8..9ed5f52b 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine_test.go
@@ -45,9 +45,15 @@ type scriptedInjector struct {
injectErr error
revertErr error
confirmErr error
+ restartErr error
unconfirmable bool
injects int
reverts int
+ restarts int
+
+ // restarted lets a probe see that the target went away, so a test can
+ // model a container that does not come back.
+ restarted *bool
}
func (i *scriptedInjector) Inject(context.Context, string, string) error {
@@ -75,6 +81,20 @@ func (i *scriptedInjector) Revert(context.Context) error {
return i.revertErr
}
+func (i *scriptedInjector) Restart(context.Context) error {
+ i.restarts++
+
+ if i.restartErr != nil {
+ return i.restartErr
+ }
+
+ if i.restarted != nil {
+ *i.restarted = true
+ }
+
+ return nil
+}
+
func quickTiming() Timing {
return Timing{
SampleInterval: time.Millisecond,
@@ -391,3 +411,189 @@ func TestTheHypothesisUsesTheCanonicalTemplate(t *testing.T) {
t.Fatalf("got %q, want %q", got, want)
}
}
+
+// goneAfterRestart answers normally until the target is restarted and never
+// again, which is what a container that does not come back looks like. The
+// baseline has to be measurable or the run is voided before the restart and
+// the test passes for the wrong reason.
+type goneAfterRestart struct {
+ credential probe.Credential
+ restarted *bool
+}
+
+func (p goneAfterRestart) Name() string { return "gone-after-restart" }
+
+func (p goneAfterRestart) Describe() string { return "a probe that stops answering" }
+
+func (p goneAfterRestart) Credential() probe.Credential { return p.credential }
+
+func (p goneAfterRestart) Run(context.Context) probe.Outcome {
+ disposition := probe.Accepted
+ if *p.restarted {
+ disposition = probe.Unreachable
+ }
+
+ return probe.Outcome{
+ At: time.Now().UTC(),
+ Credential: p.credential,
+ Disposition: disposition,
+ }
+}
+
+// coldSpec is a run whose fault is the target INITIALISING without a
+// dependency it had a moment ago.
+func coldSpec(valid, invalid probe.Probe) Experiment {
+ out := spec(valid, invalid)
+ out.ColdStart = true
+
+ return out
+}
+
+// The restart is the injection for this experiment, so a coldStart that
+// silently did nothing would leave the verdict, the exit code and the stage
+// line unchanged: the stage is emitted BEFORE the restart is attempted.
+func TestAColdStartRunActuallyRestartsTheTarget(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ injector := &scriptedInjector{faulted: &faulted}
+ engine := Engine{Injector: injector}
+
+ if _, err := engine.Run(context.Background(), coldSpec(
+ scriptedProbe{
+ probe.Valid,
+ probe.Accepted,
+ probe.Accepted,
+ &faulted,
+ &calls,
+ },
+ scriptedProbe{
+ probe.Invalid,
+ probe.Rejected,
+ probe.Rejected,
+ &faulted,
+ &calls,
+ },
+ )); err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ if injector.restarts != 1 {
+ t.Fatalf(
+ "a cold-start run restarted the target %d time(s), want exactly 1",
+ injector.restarts,
+ )
+ }
+}
+
+// And an experiment that does NOT declare it must not restart anything, or
+// every other family pays for this one.
+func TestAnOrdinaryRunNeverRestartsTheTarget(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ injector := &scriptedInjector{faulted: &faulted}
+ engine := Engine{Injector: injector}
+
+ if _, err := engine.Run(context.Background(), spec(
+ scriptedProbe{
+ probe.Valid,
+ probe.Accepted,
+ probe.Accepted,
+ &faulted,
+ &calls,
+ },
+ scriptedProbe{
+ probe.Invalid,
+ probe.Rejected,
+ probe.Rejected,
+ &faulted,
+ &calls,
+ },
+ )); err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ if injector.restarts != 0 {
+ t.Fatalf(
+ "an experiment that declares no cold start restarted the target %d time(s)",
+ injector.restarts,
+ )
+ }
+}
+
+// A restart khaos could not perform means the fault window would measure a
+// target in the wrong state, so the run is abandoned and the fault reverted
+// rather than reported.
+func TestAFailedRestartAbandonsTheRunAndRevertsTheFault(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ wanted := errors.New("the daemon refused the restart")
+
+ injector := &scriptedInjector{faulted: &faulted, restartErr: wanted}
+ engine := Engine{Injector: injector}
+
+ _, err := engine.Run(context.Background(), coldSpec(
+ scriptedProbe{
+ probe.Valid,
+ probe.Accepted,
+ probe.Accepted,
+ &faulted,
+ &calls,
+ },
+ scriptedProbe{
+ probe.Invalid,
+ probe.Rejected,
+ probe.Rejected,
+ &faulted,
+ &calls,
+ },
+ ))
+
+ if !errors.Is(err, wanted) {
+ t.Fatalf("the restart failure was not surfaced: %v", err)
+ }
+
+ if injector.reverts == 0 {
+ t.Fatal("the fault was left applied after the restart failed")
+ }
+
+ if faulted {
+ t.Fatal("the fault is still armed after the run abandoned")
+ }
+}
+
+// A target that never answers again produces a NAMED refusal rather than a
+// measured window: there is nothing to measure, and reporting HELD would
+// claim a survival khaos never saw.
+func TestATargetThatNeverComesBackIsNamedRatherThanMeasured(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ injector := &scriptedInjector{faulted: &faulted}
+ engine := Engine{Injector: injector, SettleFor: 20 * time.Millisecond}
+
+ restarted := false
+ injector.restarted = &restarted
+
+ run := coldSpec(
+ goneAfterRestart{probe.Valid, &restarted},
+ scriptedProbe{
+ probe.Invalid,
+ probe.Rejected,
+ probe.Rejected,
+ &faulted,
+ &calls,
+ },
+ )
+
+ _, err := engine.Run(context.Background(), run)
+
+ if !errors.Is(err, ErrNeverCameBack) {
+ t.Fatalf(
+ "a target that never answered after the restart did not produce ErrNeverCameBack: %v",
+ err,
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
index 21bb1c5b..4e70a26f 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
@@ -109,6 +109,9 @@ type Experiment struct {
ProxyListen string
ProxyUpstream string
+ // What the confirmation probe says to the dependency. Empty keeps HTTP.
+ ProxyProbe string
+
IdP *idp.Provider
IdPTrap *idp.Trap
@@ -152,6 +155,11 @@ type Experiment struct {
ClosedWhen []probe.Disposition
+ // ColdStart restarts the target inside the fault window so it initialises
+ // with the dependency already gone. The baseline precedes it deliberately:
+ // a target that boots broken refuses nothing and voids its own run.
+ ColdStart bool
+
// The liveness guard for an experiment that faults an ENDPOINT rather than a
// credential, where the probe streams cannot see whether the fault landed.
Consulted func() (int64, error)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go
index 90448a79..14588af1 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go
@@ -292,6 +292,7 @@ func TestTheTwoForgedGuardsDoNotFireOnEachOthersExpectation(t *testing.T) {
acceptedThroughout,
probe.ExpectAccepted,
true,
+ nil,
); unmet {
t.Error(
"ExpectAccepted voided a baseline where the forged stream WAS accepted",
@@ -302,6 +303,7 @@ func TestTheTwoForgedGuardsDoNotFireOnEachOthersExpectation(t *testing.T) {
rejectedThroughout,
probe.ExpectRejected,
true,
+ nil,
); unmet {
t.Error(
"ExpectRejected voided a baseline where the forged stream WAS rejected",
@@ -316,7 +318,7 @@ func TestAnAbsentForgedStreamIsNotGuarded(t *testing.T) {
bare := window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed)
for _, expect := range []probe.Expectation{"", probe.ExpectAccepted, probe.ExpectRejected} {
- if _, unmet := baselineUnmet(bare, expect, true); unmet {
+ if _, unmet := baselineUnmet(bare, expect, true, nil); unmet {
t.Errorf(
"a baseline with no forged stream was voided under expectation %q",
expect,
@@ -355,7 +357,7 @@ func TestEveryStreamThatLeakedIsStated(t *testing.T) {
t.Errorf("the forged stream's finding is missing: %s", joined)
}
- if !strings.Contains(joined, "had already rejected") {
+ if !strings.Contains(joined, "had already refused") {
t.Errorf("the monotonic finding is missing: %s", joined)
}
}
@@ -419,14 +421,14 @@ func TestTheIdentityTierSaysWhyItsOwnCredentialWasRefused(t *testing.T) {
MinimumToClaimClosed,
)
- minted, unmet := baselineUnmet(refused, "", true)
+ minted, unmet := baselineUnmet(refused, "", true, nil)
if !unmet {
t.Fatal(
"a baseline where khaos's own credential was never accepted was not voided",
)
}
- operator, unmet := baselineUnmet(refused, "", false)
+ operator, unmet := baselineUnmet(refused, "", false, nil)
if !unmet {
t.Fatal(
"a baseline where the operator's credential was never accepted was not voided",
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idleness_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idleness_test.go
index 45d00445..70b37efe 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idleness_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idleness_test.go
@@ -109,6 +109,10 @@ func (i *cachingInjector) Revert(context.Context) error {
return nil
}
+func (i *cachingInjector) Restart(context.Context) error {
+ return nil
+}
+
// The negative control here is deliberately NOT unique, and that is the opposite
// of what network.dependency_outage needs. There, the control is a credential no
// verifier could ever accept, so a FIXED one lets a cached refusal answer every
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go
index b5adbaed..359190c2 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go
@@ -34,6 +34,7 @@ var fastTiming = Timing{
type service struct {
dependencyUp atomic.Bool
failOpen bool
+ restarts atomic.Int64
}
func (s *service) handler() http.Handler {
@@ -79,6 +80,13 @@ func (i *serviceInjector) Revert(context.Context) error {
return nil
}
+// Restart is the real thing for this fake: the service drops what it loaded
+// from its dependency, which is what a cold start under fault looks like.
+func (i *serviceInjector) Restart(context.Context) error {
+ i.svc.restarts.Add(1)
+ return nil
+}
+
func runAgainst(t *testing.T, svc *service) Result {
t.Helper()
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go
index 0a780937..16746163 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go
@@ -5,6 +5,7 @@ package experiment
import (
"context"
+ "crypto/rand"
"errors"
"net/http"
"net/http/httptest"
@@ -404,8 +405,22 @@ func TestTheCatalogueAsksForAFreshNegativeControlCredential(t *testing.T) {
var mu sync.Mutex
seen := map[string]bool{}
+ // The same server also ISSUES, for the family whose negative
+ // control is signed and therefore cannot carry an appended
+ // nonce: it varies by being re-issued per probe, so a census
+ // with nowhere to issue from measures zero distinct credentials
+ // and cannot tell a fresh control from a fixed one.
server := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == http.MethodPost {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write(
+ []byte(`{"token":"` + rand.Text() + `"}`),
+ )
+
+ return
+ }
+
mu.Lock()
seen[r.Header.Get("Authorization")] = true
mu.Unlock()
@@ -417,6 +432,10 @@ func TestTheCatalogueAsksForAFreshNegativeControlCredential(t *testing.T) {
supplied := fill(definition.Params)
supplied[ParamURL] = server.URL
+ if _, declared := supplied[ParamRefusedFrom]; declared {
+ supplied[ParamRefusedFrom] = server.URL
+ }
+
spec, err := Build(definition.Name, supplied, fastTiming)
if err != nil {
t.Fatalf("build: %v", err)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/revocation_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/revocation_test.go
index 7b44f00f..eb16d89f 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/revocation_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/revocation_test.go
@@ -124,6 +124,10 @@ func (i *severingInjector) Revert(context.Context) error {
return nil
}
+func (i *severingInjector) Restart(context.Context) error {
+ return nil
+}
+
func registerRevocationExperiment(
t *testing.T,
url string,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/throttled_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/throttled_test.go
new file mode 100644
index 00000000..2b1e8823
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/throttled_test.go
@@ -0,0 +1,218 @@
+// ©AngelaMos | 2026
+// throttled_test.go
+
+package experiment
+
+import (
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+// A rate limiter never inspects the credential it is asked about, so its
+// negative control is the request over the budget and its refusal is a 429.
+// Three guards read the invalid stream, and each test below pins one of them
+// against both a declared and an undeclared experiment: without the
+// declaration the old behaviour must still hold, or this is a blanket
+// loosening rather than a declared one.
+
+var throttles = []probe.Disposition{probe.Throttled}
+
+// limiterBaseline is a limiter enforcing its budget.
+func limiterBaseline() probe.Window {
+ return probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Throttled, MinimumPaired)...),
+ }
+}
+
+func TestABaselineRefusedWithAThrottleCountsAsEnforcement(t *testing.T) {
+ reason, unmet := baselineUnmet(
+ limiterBaseline(),
+ "",
+ false,
+ closedWhen(Observation{ClosedWhen: throttles}),
+ )
+ if unmet {
+ t.Fatalf(
+ "a limiter that throttled every over-budget request was read as never enforcing: %s",
+ reason,
+ )
+ }
+}
+
+func TestAThrottledBaselineIsStillUnmetWhenNothingDeclaresIt(t *testing.T) {
+ if _, unmet := baselineUnmet(
+ limiterBaseline(),
+ "",
+ false,
+ closedWhen(Observation{}),
+ ); !unmet {
+ t.Fatal(
+ "an experiment that declared no refusal beyond Rejected accepted a throttle as enforcement, so the set is not being read at all",
+ )
+ }
+}
+
+// The declared set extends Rejected rather than replacing it.
+func TestADeclaredSetStillAcceptsAnOrdinaryRejection(t *testing.T) {
+ if _, unmet := baselineUnmet(
+ healthyBaseline(),
+ "",
+ false,
+ closedWhen(Observation{ClosedWhen: throttles}),
+ ); unmet {
+ t.Fatal(
+ "declaring Throttled stopped a plain 403 baseline from counting as enforcement",
+ )
+ }
+}
+
+// leakObservation is the limiter's defect: the over-budget request was
+// throttled while the datastore was reachable and served once it was severed.
+func leakObservation(closed []probe.Disposition) Observation {
+ return Observation{
+ ClosedWhen: closed,
+ Baseline: limiterBaseline(),
+ Fault: probe.Window{
+ Valid: stream(
+ probe.Valid,
+ repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Accepted, MinimumPaired)...),
+ },
+ }
+}
+
+func TestAnOverBudgetRequestServedDuringTheFaultIsCounted(t *testing.T) {
+ leaked := leakedMonotonic(leakObservation(throttles))
+ if leaked != MinimumPaired {
+ t.Fatalf(
+ "the limiter served %d over-budget requests during the fault and khaos counted %d",
+ MinimumPaired,
+ leaked,
+ )
+ }
+}
+
+func TestAnUndeclaredThrottleLeaksNothing(t *testing.T) {
+ if leaked := leakedMonotonic(leakObservation(nil)); leaked != 0 {
+ t.Fatalf(
+ "an experiment declaring nothing counted %d leaks from a throttled baseline, so the guard is no longer reading the declaration",
+ leaked,
+ )
+ }
+}
+
+// A limiter has recovered when the over-budget request is throttled again.
+func TestRecoveryReadsTheDeclaredRefusal(t *testing.T) {
+ recovered := recoveryOf(limiterBaseline(), throttles)
+ if recovered != Recovered {
+ t.Fatalf(
+ "a limiter throttling every over-budget request again was read as %q rather than recovered",
+ recovered,
+ )
+ }
+}
+
+// Through closedWhen rather than a bare nil: the bare nil skips the
+// defaulting the other two guards go through, and cannot see a change that
+// makes Throttled closed for every experiment.
+func TestRecoveryIsStillDegradedWhenNothingDeclaresTheThrottle(t *testing.T) {
+ if recovered := recoveryOf(
+ limiterBaseline(),
+ closedWhen(Observation{}),
+ ); recovered == Recovered {
+ t.Fatal(
+ "an experiment declaring nothing read a throttle as a recovered refusal",
+ )
+ }
+}
+
+func TestRecoveryStillReadsAPlainRejection(t *testing.T) {
+ if recovered := recoveryOf(
+ healthyBaseline(),
+ closedWhen(Observation{ClosedWhen: throttles}),
+ ); recovered != Recovered {
+ t.Fatalf(
+ "declaring Throttled stopped a plain 403 recovery from being read as recovered, got %q",
+ recovered,
+ )
+ }
+}
+
+// The three guards above are EPISTEMIC: they ask whether khaos observed this
+// control decide at all, and whether the steady state came back. A declared
+// refusal only answers that when the disposition is the target DECIDING.
+//
+// `Throttled` is a decision, so the limiter needs it. `Unavailable` is not --
+// probe.go says a 503 "is never Rejected" and Answered() is false for it --
+// and four shipped families declare it. Letting it through these three turns
+// a target that 500s its negative control into one khaos reports as enforcing.
+var unavailable = []probe.Disposition{probe.Unavailable}
+
+// deadNegativeControl is a target whose auth path throws instead of refusing:
+// the valid credential is served and the bogus one gets a 500. It was never
+// observed enforcing anything, so the run has to be voided.
+func deadNegativeControl() probe.Window {
+ return probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Unavailable, MinimumPaired)...),
+ }
+}
+
+func TestA503OnTheNegativeControlIsNotEnforcement(t *testing.T) {
+ reason, unmet := baselineUnmet(
+ deadNegativeControl(),
+ "",
+ false,
+ closedWhen(Observation{ClosedWhen: unavailable}),
+ )
+ if !unmet {
+ t.Fatal(
+ "a baseline whose negative control only ever 500'd was read as a control that was enforcing; khaos would then print that the invalid credential kept being rejected",
+ )
+ }
+
+ _ = reason
+}
+
+func TestA503InTheBaselineCannotSeedTheLeakLatch(t *testing.T) {
+ leaked := leakedMonotonic(Observation{
+ ClosedWhen: unavailable,
+ Baseline: deadNegativeControl(),
+ Fault: probe.Window{
+ Valid: stream(
+ probe.Valid,
+ repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Accepted, MinimumPaired)...),
+ },
+ })
+
+ if leaked != 0 {
+ t.Fatalf(
+ "khaos counted %d inputs as 'already refused and then accepted' when the baseline only ever 500'd them; it never saw a refusal to contradict",
+ leaked,
+ )
+ }
+}
+
+func TestA503InRecoveryIsNotASteadyState(t *testing.T) {
+ recovered := recoveryOf(
+ deadNegativeControl(),
+ closedWhen(Observation{ClosedWhen: unavailable}),
+ )
+
+ if recovered == Recovered {
+ t.Fatal(
+ "a recovery window still 500ing the negative control was read as the steady state returning",
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
index ab738752..7e12d50b 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
@@ -107,7 +107,7 @@ const (
reasonRedirects = "the invalid credential was answered with a redirect %d time(s) during the baseline and never with a refusal; point --url at an endpoint that answers with a status rather than a login redirect"
reasonNoService = "the valid credential was never accepted during the baseline, so the target was not serving before the fault"
reasonNoServiceMinted = "the target never accepted a credential from khaos's own identity provider during the baseline. It may not resolve keys from the issuer khaos is serving; and if it caches its key set, a PREVIOUS khaos run's keys may still be cached, because khaos mints a fresh key set every run. Restart the target or shorten its key cache lifetime"
- reasonPermitted = "a credential the target had already rejected was accepted %d time(s) during the fault"
+ reasonPermitted = "%s: an input the target had already refused was accepted %d time(s) during the fault"
reasonDenied = "the valid credential stopped being accepted %d time(s) during the fault"
reasonCrashed = "the target stopped answering %d of %d probes during the fault"
reasonHeld = "the valid credential kept being accepted and the invalid one kept being rejected"
@@ -344,6 +344,7 @@ func epistemic(observation Observation) (Verdict, bool) {
observation.Baseline,
observation.ForgedBaseline,
observation.MintedByKhaos,
+ closedWhen(observation),
); unmet {
verdict.Epistemic = BaselineNotMet
verdict.Reasons = append(verdict.Reasons, reason)
@@ -503,7 +504,10 @@ func finish(
)
}
- verdict.Recovery = recoveryOf(observation.Recovery)
+ verdict.Recovery = recoveryOf(
+ observation.Recovery,
+ closedWhen(observation),
+ )
switch verdict.Recovery {
case Degraded:
@@ -583,10 +587,22 @@ func leaked(observation Observation) []string {
)
}
+ // The DECLARED defect, like the two paths above. Without it, two
+ // experiments that read the same direction off the same observable print
+ // the same reason, and the verdict alone cannot tell an operator which
+ // bug they have: losing a loaded policy and never loading one are
+ // different fixes.
if monotonic := leakedMonotonic(
observation,
); monotonic >= MinimumToProveOpen {
- reasons = append(reasons, sprintf(reasonPermitted, monotonic))
+ reasons = append(
+ reasons,
+ sprintf(
+ reasonPermitted,
+ orUnnamed(observation.Defect),
+ monotonic,
+ ),
+ )
}
if served := servedWhileUnknown(
@@ -778,11 +794,19 @@ func orUnnamed(defect string) string {
return defect
}
-// leakedMonotonic counts credentials the target had already rejected and then
-// accepted during the fault.
+// leakedMonotonic counts inputs the target had already refused and then
+// accepted during the fault. Both halves read the declared refusal: for a
+// limiter neither one is a Rejected.
func leakedMonotonic(observation Observation) int {
- rejectedInBaseline := observation.Baseline.Invalid.Any(probe.Rejected)
- return observation.Fault.Invalid.AcceptedAfter(rejectedInBaseline)
+ closed := closedWhen(observation)
+
+ // The LATCH is epistemic -- it is the premise of the sentence this reason
+ // prints, "an input the target had already refused" -- while the count
+ // itself is directional.
+ return observation.Fault.Invalid.AcceptedAfter(
+ observation.Baseline.Invalid.AnyClosed(decided(closed)...),
+ closed...,
+ )
}
// revokedHonoured counts acceptances of a credential khaos invalidated at the
@@ -894,16 +918,23 @@ func resolveCause(
)
}
+// closed is this control's own refusal, declared by the experiment. It
+// extends Rejected and never replaces it.
+//
+// The invalid stream needs it as much as the valid one does: a rate limiter's
+// negative control is the request over the budget, not a credential it never
+// inspects, and that refusal is a 429.
func baselineUnmet(
baseline probe.Window,
expect probe.Expectation,
minted bool,
+ closed []probe.Disposition,
) (string, bool) {
if paired := baseline.Paired(); paired < MinimumPaired {
return sprintf(reasonThinBaseline, paired, MinimumPaired), true
}
- if !baseline.Invalid.Any(probe.Rejected) {
+ if !baseline.Invalid.AnyClosed(decided(closed)...) {
if redirects := baseline.Invalid.Count(
probe.Redirected,
); redirects > 0 {
@@ -1067,6 +1098,40 @@ func closedWhen(observation Observation) []probe.Disposition {
)
}
+// shutBy is closedWhen applied to one outcome, for the guards that walk a
+// stream position by position rather than asking it for a count.
+func shutBy(disposition probe.Disposition, closed []probe.Disposition) bool {
+ return slices.Contains(closed, disposition)
+}
+
+// decided narrows a declared set to the dispositions that are the target
+// DECIDING, and it is the difference between a directional question and an
+// epistemic one.
+//
+// ClosedWhen declares what failing closed looks like UNDER THE FAULT, which is
+// directional. Three guards ask something else: did khaos ever observe this
+// control decide about its negative control, and did the steady state return.
+// A 429 answers both -- a limiter that throttles has decided -- and a 503 does
+// not, which is why Answered() is false for it. Four families declare
+// Unavailable, and letting it through here read a target that 500s its
+// negative control as one that was enforcing, then printed that the invalid
+// credential kept being rejected.
+func decided(closed []probe.Disposition) []probe.Disposition {
+ out := make([]probe.Disposition, 0, len(closed))
+
+ for _, disposition := range closed {
+ if disposition.Answered() {
+ out = append(out, disposition)
+ }
+ }
+
+ if len(out) == 0 {
+ return []probe.Disposition{probe.Rejected}
+ }
+
+ return out
+}
+
// unanswered counts across BOTH streams, and a disposition the experiment
// DECLARED as failing closed counts as the target answering. Without that, a
// service that correctly refused every request because it had lost its secret
@@ -1099,7 +1164,7 @@ func unanswered(
// recoveryOf requires a run of consecutive healthy pairs, because a state
// transition declared on one lucky sample is how a flaky recovery reads as a
// clean one. An empty window is UNOBSERVED, never recovered.
-func recoveryOf(recovery probe.Window) Recovery {
+func recoveryOf(recovery probe.Window, closed []probe.Disposition) Recovery {
if recovery.Paired() == 0 {
return Unobserved
}
@@ -1107,7 +1172,10 @@ func recoveryOf(recovery probe.Window) Recovery {
run := 0
for i := 0; i < recovery.Valid.Len() && i < recovery.Invalid.Len(); i++ {
if recovery.Valid.Outcomes[i].Disposition == probe.Accepted &&
- recovery.Invalid.Outcomes[i].Disposition == probe.Rejected {
+ shutBy(
+ recovery.Invalid.Outcomes[i].Disposition,
+ decided(closed),
+ ) {
run++
if run >= RecoveryRunLength {
return Recovered
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/deadlock_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/deadlock_test.go
index 3dfa5e15..a26c3c75 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/deadlock_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/deadlock_test.go
@@ -27,7 +27,7 @@ func TestCloseDoesNotDeadlockAgainstAWedgedDependency(t *testing.T) {
}
}()
- p, err := Open("127.0.0.1:0", wedged.Addr().String())
+ p, err := Open("127.0.0.1:0", wedged.Addr().String(), "")
if err != nil {
t.Fatalf("open: %v", err)
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe.go
index fe29c6f2..a6149a76 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe.go
@@ -179,7 +179,7 @@ func (p *Proxy) measure(
started := time.Now()
- if _, err := io.WriteString(conn, probePayload); err != nil {
+ if _, err := io.WriteString(conn, p.probe); err != nil {
return reading{}, err
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe_test.go
index 7010e9ff..290ac78f 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe_test.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
+ "net"
"net/http"
"net/http/httptest"
"strings"
@@ -142,7 +143,11 @@ func TestASlowDependencyDoesNotConfirmAFaultThatIsNotHappening(t *testing.T) {
{Kind: Latency, Delay: 300 * time.Millisecond},
} {
t.Run(string(fault.Kind), func(t *testing.T) {
- p, err := Open(loopback, strings.TrimPrefix(slow.URL, "http://"))
+ p, err := Open(
+ loopback,
+ strings.TrimPrefix(slow.URL, "http://"),
+ "",
+ )
if err != nil {
t.Fatalf("open: %v", err)
}
@@ -173,7 +178,7 @@ func TestASlowDependencyDoesNotConfirmAFaultThatIsNotHappening(t *testing.T) {
// unattributable observation, and saying so voids the run instead of crediting
// khaos for an outage it did not cause.
func TestAnUnreachableDependencyCannotConfirmAnything(t *testing.T) {
- p, err := Open(loopback, "127.0.0.1:1")
+ p, err := Open(loopback, "127.0.0.1:1", "")
if err != nil {
t.Fatalf("open: %v", err)
}
@@ -243,7 +248,7 @@ func TestPartialDoesNotConfirmWhenNothingWasTruncated(t *testing.T) {
)
defer server.Close()
- p, err := Open(loopback, strings.TrimPrefix(server.URL, "http://"))
+ p, err := Open(loopback, strings.TrimPrefix(server.URL, "http://"), "")
if err != nil {
t.Fatalf("open: %v", err)
}
@@ -364,3 +369,76 @@ func TestMeasureReadsBothPathsOfTheDifferential(t *testing.T) {
)
}
}
+
+// The oracle needs the dependency to ANSWER on the direct path, and an HTTP
+// request is not something every dependency answers. Measured on
+// redis:7.4-alpine: the HTTP payload gets zero bytes and a close, so the
+// oracle reported the datastore unreachable and voided the limiter's run.
+func TestTheConfirmationProbeSpeaksTheDependencysProtocol(t *testing.T) {
+ // A dependency that answers RESP and nothing else, which is what the
+ // limiter's datastore is.
+ listener, err := net.Listen("tcp", loopback)
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+ defer func() { _ = listener.Close() }()
+
+ go func() {
+ for {
+ conn, err := listener.Accept()
+ if err != nil {
+ return
+ }
+
+ go func() {
+ defer func() { _ = conn.Close() }()
+
+ buffer := make([]byte, 64)
+
+ read, err := conn.Read(buffer)
+ if err != nil || !strings.Contains(
+ string(buffer[:read]),
+ "PING",
+ ) {
+ return
+ }
+
+ _, _ = io.WriteString(conn, "+PONG\r\n")
+ }()
+ }
+ }()
+
+ upstream := listener.Addr().String()
+
+ for _, row := range []struct {
+ name string
+ probe string
+ answer bool
+ }{
+ {"the default HTTP request", "", false},
+ {"RESP", ProbeRESP, true},
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ p, err := Open(loopback, upstream, row.probe)
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ defer func() { _ = p.Close() }()
+
+ direct, _ := p.measure(
+ context.Background(),
+ p.upstream,
+ observeGrace,
+ )
+
+ if answered := direct.total > 0; answered != row.answer {
+ t.Fatalf(
+ "%s measured %d bytes on the direct path, want answered=%v",
+ row.name,
+ direct.total,
+ row.answer,
+ )
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
index 34210db8..5e141470 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
@@ -32,6 +32,11 @@ const (
probePayload = "GET / HTTP/1.1\r\nHost: khaos.confirm\r\nConnection: close\r\n\r\n"
+ // RESP, for a dependency that speaks it. Measured on redis:7.4-alpine:
+ // the HTTP payload above gets zero bytes and a close, while this gets
+ // `+PONG\r\n`, and the oracle needs bytes on the direct path.
+ ProbeRESP = "*1\r\n$4\r\nPING\r\n"
+
observedHang = "the dependency answers khaos directly but never through its proxy"
observedReset = "the dependency answers khaos directly but resets through its proxy"
observedEOF = "the dependency answers khaos directly but closes without answering through its proxy"
@@ -59,6 +64,13 @@ type Proxy struct {
listener net.Listener
state atomic.Pointer[armed]
+ // What the confirmation probe SAYS to the dependency. The differential
+ // needs the dependency to answer on the direct path, and a dependency
+ // that does not speak HTTP answers an HTTP request with nothing at all:
+ // measured on Redis 7.4, which returns zero bytes and closes, so the
+ // oracle reported the dependency unreachable and voided the run.
+ probe string
+
// Both ends are held; closing the client cannot unblock the dependency.
mu sync.Mutex
held map[net.Conn]struct{}
@@ -76,8 +88,9 @@ type Proxy struct {
probes atomic.Int64
}
-// Open binds the proxy and refuses an address that would expose it.
-func Open(listenAddr, upstream string) (*Proxy, error) {
+// Open binds the proxy and refuses an address that would expose it. An empty
+// probe keeps the HTTP request every dependency before Redis spoke.
+func Open(listenAddr, upstream, probe string) (*Proxy, error) {
if err := bindguard.Check(listenAddr, SubjectProxy); err != nil {
return nil, err
}
@@ -87,9 +100,14 @@ func Open(listenAddr, upstream string) (*Proxy, error) {
return nil, err
}
+ if probe == "" {
+ probe = probePayload
+ }
+
p := &Proxy{
upstream: upstream,
listener: listener,
+ probe: probe,
held: map[net.Conn]struct{}{},
closed: make(chan struct{}),
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go
index c4f54f30..97a02008 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go
@@ -45,7 +45,7 @@ func through(t *testing.T) *Proxy {
server := upstreamServer(t)
- p, err := Open(loopback, strings.TrimPrefix(server.URL, "http://"))
+ p, err := Open(loopback, strings.TrimPrefix(server.URL, "http://"), "")
if err != nil {
t.Fatalf("open: %v", err)
}
@@ -501,7 +501,7 @@ func TestAFaultThatCannotExpressItselfIsRefused(t *testing.T) {
}
func TestAnUnreachableDependencyIsNotMistakenForAFault(t *testing.T) {
- p, err := Open(loopback, "127.0.0.1:1")
+ p, err := Open(loopback, "127.0.0.1:1", "")
if err != nil {
t.Fatalf("open: %v", err)
}
@@ -719,7 +719,7 @@ func TestAHalfClosingClientStillReceivesTheWholeAnswer(t *testing.T) {
// input it MUST refuse rather than a comment saying what callers should pass.
func TestOpenRefusesAnAddressThatWouldExposeTheProxy(t *testing.T) {
for _, addr := range []string{"0.0.0.0:0", ":0", "[::]:0", "0.0.0.0:45001"} {
- p, err := Open(addr, "127.0.0.1:1")
+ p, err := Open(addr, "127.0.0.1:1", "")
if err == nil {
bound := p.Addr()
_ = p.Close()
@@ -808,7 +808,7 @@ func TestOpenRefusesThisHostsRoutableAddress(t *testing.T) {
t.Skip("this host has no non-bridge routable IPv4 address to try")
}
- p, err := Open(net.JoinHostPort(routable, "45999"), "127.0.0.1:1")
+ p, err := Open(net.JoinHostPort(routable, "45999"), "127.0.0.1:1", "")
if err == nil {
bound := p.Addr()
_ = p.Close()
@@ -856,6 +856,7 @@ func TestOpenAcceptsABridgeAddress(t *testing.T) {
p, err := Open(
net.JoinHostPort(network.IP.String(), "0"),
"127.0.0.1:1",
+ "",
)
if err != nil {
t.Fatalf(
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
index 318d771f..ea6f7ebc 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
@@ -35,7 +35,11 @@ func armed(t *testing.T, fault proxy.Fault) Env {
)
t.Cleanup(server.Close)
- p, err := proxy.Open(loopback, strings.TrimPrefix(server.URL, "http://"))
+ p, err := proxy.Open(
+ loopback,
+ strings.TrimPrefix(server.URL, "http://"),
+ "",
+ )
if err != nil {
t.Fatalf("open: %v", err)
}
@@ -338,7 +342,7 @@ func TestInProcessKindsAreNotOfferedForDirectInjection(t *testing.T) {
}
func TestTheProxyBindRefusesAnExposedAddress(t *testing.T) {
- if _, err := proxy.Open("0.0.0.0:0", "127.0.0.1:1"); err == nil {
+ if _, err := proxy.Open("0.0.0.0:0", "127.0.0.1:1", ""); err == nil {
t.Fatal("the fault proxy bound every address on the host")
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/control_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/control_test.go
new file mode 100644
index 00000000..baaf31e6
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/control_test.go
@@ -0,0 +1,146 @@
+// ©AngelaMos | 2026
+// control_test.go
+
+package rangelab
+
+import (
+ "errors"
+ "strings"
+ "testing"
+)
+
+// controlPort is the only address parser in the project that accepts two
+// shapes: the limiter's datastore is a bare host:port and the other two
+// controls are URLs. Neither its success paths nor its three refusals had a
+// test, and a port it misreads is a proxy khaos binds where nothing dials.
+func TestControlPortReadsBothAddressShapes(t *testing.T) {
+ for _, row := range []struct {
+ name string
+ address string
+ want int
+ }{
+ {"a bare host and port", "host.docker.internal:34567", 34567},
+ {"a URL", "http://host.docker.internal:34567", 34567},
+ {"a loopback URL", "http://127.0.0.1:1", 1},
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ port, err := controlPort(row.address)
+ if err != nil {
+ t.Fatalf("controlPort(%q): %v", row.address, err)
+ }
+
+ if port != row.want {
+ t.Fatalf("port = %d, want %d", port, row.want)
+ }
+ })
+ }
+}
+
+// Each refusal names the address it could not read, because an operator
+// seeing "khaos cannot read that" needs to know which of three it meant.
+func TestControlPortRefusesWhatItCannotRead(t *testing.T) {
+ for _, row := range []struct{ name, address string }{
+ {"no port at all", "host.docker.internal"},
+ {"port zero", "host.docker.internal:0"},
+ {"a non-numeric port", "host.docker.internal:redis"},
+ {"a URL with no port", "http://host.docker.internal"},
+ {"an empty address", ""},
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ port, err := controlPort(row.address)
+ if err == nil {
+ t.Fatalf(
+ "controlPort(%q) returned %d rather than refusing",
+ row.address,
+ port,
+ )
+ }
+
+ if !errors.Is(err, ErrRangeIncomplete) {
+ t.Fatalf("the refusal is not a range-incomplete: %v", err)
+ }
+ })
+ }
+}
+
+// The two ways this tier can be missing need different fixes, so they are
+// different refusals: no gateway means bring the range up, and a gateway
+// carrying no dependency address means that container predates the variable.
+func TestRequireControlTierSeparatesTheTwoWaysItIsMissing(t *testing.T) {
+ absent := Lab{}
+
+ err := absent.RequireControlTier(ControlPolicy)
+ if err == nil {
+ t.Fatal("a range with no control gateway was not refused")
+ }
+
+ if !strings.Contains(err.Error(), ControlGatewayName) {
+ t.Fatalf("the refusal does not name the missing gateway: %v", err)
+ }
+
+ // The gateway is there and carries no address for THIS control, which is
+ // a different fix: the container has to be recreated.
+ partial := Lab{
+ ControlPort: 30015,
+ ControlStoreAddr: "host.docker.internal:30016",
+ }
+
+ err = partial.RequireControlTier(ControlPolicy)
+ if err == nil {
+ t.Fatal(
+ "a gateway carrying no policy-store address was not refused",
+ )
+ }
+
+ if !strings.Contains(err.Error(), EnvPolicyStore) {
+ t.Fatalf(
+ "the refusal does not name the variable that is missing: %v",
+ err,
+ )
+ }
+
+ // And the control that IS wired must not be refused, or one missing
+ // address would take the other two experiments down with it.
+ if err := partial.RequireControlTier(ControlLimiter); err != nil {
+ t.Fatalf(
+ "the limiter was refused although its address is present: %v",
+ err,
+ )
+ }
+}
+
+// Each control resolves to its OWN address and variable. Collapsing any two
+// would make khaos sever one dependency while reporting another.
+func TestEachControlResolvesItsOwnDependency(t *testing.T) {
+ lab := Lab{
+ ControlPort: 30015,
+ ControlStoreAddr: "host.docker.internal:30016",
+ ControlPolicyAddr: "http://host.docker.internal:30017",
+ ControlRulesAddr: "http://host.docker.internal:30018",
+ }
+
+ seen := map[string]Control{}
+
+ for _, at := range []Control{
+ ControlLimiter,
+ ControlPolicy,
+ ControlRules,
+ } {
+ address, variable := lab.controlDependency(at)
+
+ if address == "" || variable == "" {
+ t.Fatalf("%s resolved to %q / %q", at, address, variable)
+ }
+
+ if other, clash := seen[address]; clash {
+ t.Fatalf(
+ "%s and %s both resolve to %s, so one fault would sever both",
+ at,
+ other,
+ address,
+ )
+ }
+
+ seen[address] = at
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
index c2a09c0b..a1336762 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
@@ -65,6 +65,11 @@ const (
// whose dependency is not a container: khaos plays that itself.
AuthzGatewayName = "khaos-range-gateway-authz"
+ // The backend publishes nothing: khaos forwards to it by its address on
+ // the range network.
+ ControlGatewayName = "khaos-range-gateway-control"
+ ControldName = "khaos-range-controld"
+
LabelRange = "khaos.range"
FailModeOpen = "open"
FailModeShut = "closed"
@@ -116,7 +121,24 @@ const (
EnvAuthzPort = "KHAOS_RANGE_AUTHZ_PORT"
EnvAuthzDep = "KHAOS_RANGE_AUTHZ_DEP_PORT"
EnvAuthzBase = "KHAOS_RANGE_AUTHZ_BASE"
- EnvAuthzMode = "KHAOS_RANGE_AUTHZ_MODE"
+
+ // One published port and one dependency port per control, read back out
+ // of the gateway's own environment rather than remembered: the container
+ // is the only authority on where it will actually dial.
+ EnvControlPort = "KHAOS_RANGE_CONTROL_PORT"
+ EnvControlStorePort = "KHAOS_RANGE_CONTROL_STORE_PORT"
+ EnvControlPolicyPort = "KHAOS_RANGE_CONTROL_POLICY_PORT"
+ EnvControlRulesPort = "KHAOS_RANGE_CONTROL_RULES_PORT"
+
+ EnvLimitStore = "KHAOS_RANGE_LIMIT_STORE"
+ EnvPolicyStore = "KHAOS_RANGE_POLICY_STORE"
+ EnvRulesStore = "KHAOS_RANGE_RULES_STORE"
+
+ // The control gateway authenticates by SIGNATURE against this key rather
+ // than from a set it remembers, because the cold-start experiment
+ // restarts it and an in-memory set does not survive that.
+ EnvControlSecret = "KHAOS_RANGE_CONTROL_SECRET"
+ EnvAuthzMode = "KHAOS_RANGE_AUTHZ_MODE"
// AuthzDistinguishes reads the dependency's three answers as three.
// AuthzCollapses is research 07 section 6.2's shape: anything that is not
@@ -172,6 +194,7 @@ const (
// before anything has read a published port.
PostgresAddress = "10.244.244.30"
postgresPort = "5432"
+ redisPort = "6379"
adminRole = "postgres"
PoolUnbounded = "unbounded"
@@ -201,6 +224,14 @@ const (
protectedPath = "/protected"
tokenPath = "/token"
+ flaggedTokenPath = "/flagged-token"
+
+ // The endpoint each control guards, which is also the only endpoint whose
+ // answer that control decides.
+ LimitedPath = "/limited"
+ GovernedPath = "/governed"
+ FilteredPath = "/filtered"
+
errNoCompose = "cannot find the range compose file; run khaos from the project directory, or set %s to the directory holding khaos's own range/compose.yml"
errUnreadable = "%w: %s could not be read: %v"
errForeign = "%w: %s declares the compose project %s, and khaos only ever acts on %s"
@@ -250,7 +281,11 @@ const (
errNoAuthzBase = "%w: this experiment answers a service's own authorization call, and %s does not carry %s, so khaos cannot tell where that service looks for its dependency; bring the range down and up again to recreate it"
- errNoStatefulTier = "%w: this experiment acts on a real database engine, and the running range has none. khaos does not replace these dependencies with its own code, because the finding IS what three correct engines do, so the stack has to be brought up with them: `khaos range down && khaos range up --with-db`"
+ errNoControlGateway = "%w: this experiment severs one of a gateway's security controls from its dependency, and the running range has no %s; bring the range down and up again to create it"
+
+ errNoControlAddr = "%w: this experiment severs %s, and %s does not carry %s, so khaos cannot tell where that control looks for it; bring the range down and up again to recreate it"
+
+ errNoStatefulTier = "%w: this experiment acts on a real stateful dependency -- a database engine, or the datastore a rate limiter counts in -- and the running range has none. khaos does not replace these with its own code, because the finding IS what correct ones do, so the stack has to be brought up with them: `khaos range down && khaos range up --with-db`"
)
// Lab is a located range. The khaos-side ports are chosen at `range up` rather
@@ -289,6 +324,20 @@ type Lab struct {
AuthzDepPort int
AuthzBase string
+ // ControlPort is the gateway's published port; the three that follow are
+ // where it was told to find each control's dependency. The three Addr
+ // fields are read back out of its own environment.
+ ControlPort int
+ ControlStorePort int
+ ControlPolicyPort int
+ ControlRulesPort int
+
+ ControlStoreAddr string
+ ControlPolicyAddr string
+ ControlRulesAddr string
+
+ ControlSecret string
+
// Read back from the TLS gateway's own environment, never remembered.
Dependency string
AuthorityBase string
@@ -325,6 +374,11 @@ func Locate(ports Ports) (Lab, error) {
EnvPort: ports.Env,
AuthzPort: ports.Authz,
AuthzDepPort: ports.AuthzDep,
+
+ ControlPort: ports.Control,
+ ControlStorePort: ports.ControlStore,
+ ControlPolicyPort: ports.ControlPolicy,
+ ControlRulesPort: ports.ControlRules,
}, nil
}
@@ -359,6 +413,14 @@ type Ports struct {
// here because the caller's environment names it at creation.
Authz int
AuthzDep int
+
+ // The control gateway's published port, and the address each of its three
+ // controls dials its own dependency on. Three rather than one is what
+ // lets khaos sever the datastore while the other two keep answering.
+ Control int
+ ControlStore int
+ ControlPolicy int
+ ControlRules int
}
// FreePorts picks every host port the range needs in one go, all distinct.
@@ -400,6 +462,10 @@ func FreePorts(also ...string) (Ports, error) {
&chosen.Env,
&chosen.Authz,
&chosen.AuthzDep,
+ &chosen.Control,
+ &chosen.ControlStore,
+ &chosen.ControlPolicy,
+ &chosen.ControlRules,
} {
port, err := pick()
if err != nil {
@@ -575,6 +641,11 @@ func (l Lab) Up(
EnvAuthzPort+"="+strconv.Itoa(l.AuthzPort),
EnvAuthzDep+"="+strconv.Itoa(l.AuthzDepPort),
EnvAuthzMode+"="+authzMode,
+ EnvControlPort+"="+strconv.Itoa(l.ControlPort),
+ EnvControlStorePort+"="+strconv.Itoa(l.ControlStorePort),
+ EnvControlPolicyPort+"="+strconv.Itoa(l.ControlPolicyPort),
+ EnvControlRulesPort+"="+strconv.Itoa(l.ControlRulesPort),
+ EnvControlSecret+"="+l.ControlSecret,
EnvReresolve+"="+reresolve,
)
@@ -668,6 +739,11 @@ func (l Lab) Down(ctx context.Context, out io.Writer) error {
EnvEnvPort+"="+strconv.Itoa(teardownPort),
EnvAuthzPort+"="+strconv.Itoa(teardownPort),
EnvAuthzDep+"="+strconv.Itoa(teardownPort),
+ EnvControlPort+"="+strconv.Itoa(teardownPort),
+ EnvControlStorePort+"="+strconv.Itoa(teardownPort),
+ EnvControlPolicyPort+"="+strconv.Itoa(teardownPort),
+ EnvControlRulesPort+"="+strconv.Itoa(teardownPort),
+ EnvControlSecret+"="+teardownPassword,
)
// The profile is ALWAYS passed on the way down. Compose only removes the
@@ -915,10 +991,11 @@ func (l *Lab) readM9Fixtures(
running map[string]dockerx.Target,
) error {
for name, into := range map[string]*int{
- DBGatewayName: &l.DBGatewayPort,
- SecretGatewayName: &l.SecretPort,
- EnvGatewayName: &l.EnvPort,
- AuthzGatewayName: &l.AuthzPort,
+ DBGatewayName: &l.DBGatewayPort,
+ SecretGatewayName: &l.SecretPort,
+ EnvGatewayName: &l.EnvPort,
+ AuthzGatewayName: &l.AuthzPort,
+ ControlGatewayName: &l.ControlPort,
} {
gateway, ok := running[name]
if !ok {
@@ -954,6 +1031,33 @@ func (l *Lab) readM9Fixtures(
// RequireAuthzTier refuses by name for the experiments that need it.
l.AuthzBase = environ[EnvAuthzBase]
+ return l.readControlTier(ctx, reader, running)
+}
+
+// readControlTier reads the three dependency addresses out of the gateway that
+// dials them. A container of that name carrying none of them is the tier being
+// ABSENT rather than an error: this runs inside Discover, which every
+// experiment calls, so a hard error here would take unrelated runs down over
+// one stale container. RequireControlTier refuses by name instead.
+func (l *Lab) readControlTier(
+ ctx context.Context,
+ reader PortReader,
+ running map[string]dockerx.Target,
+) error {
+ gateway, ok := running[ControlGatewayName]
+ if !ok {
+ return nil
+ }
+
+ environ, err := reader.ContainerEnv(ctx, gateway.ID)
+ if err != nil {
+ return err
+ }
+
+ l.ControlStoreAddr = environ[EnvLimitStore]
+ l.ControlPolicyAddr = environ[EnvPolicyStore]
+ l.ControlRulesAddr = environ[EnvRulesStore]
+
return nil
}
@@ -1046,6 +1150,170 @@ func (l Lab) RequireEnvTier() error {
return fmt.Errorf(errNoEnvGateway, ErrRangeIncomplete, EnvGatewayName)
}
+// Control names one of the gateway's three controls, so a refusal and a
+// listen address can both be resolved from the experiment's own subject.
+type Control string
+
+const (
+ ControlLimiter Control = "the rate limiter from its datastore"
+ ControlPolicy Control = "the policy engine from its policy store"
+ ControlRules Control = "the rule matcher from its rules backend"
+)
+
+func (l Lab) ControlGatewayURL() string {
+ return "http://" + net.JoinHostPort(
+ loopback,
+ strconv.Itoa(l.ControlPort),
+ )
+}
+
+// IssueControlToken asks the gateway for a credential it will authenticate.
+func (l Lab) IssueControlToken(ctx context.Context) (string, error) {
+ return l.issueFrom(ctx, l.ControlGatewayURL()+tokenPath)
+}
+
+// RefusedInputURL is where khaos fetches a fresh input every control refuses,
+// once per probe rather than once per run: the credential is signed, so it
+// cannot carry an appended nonce, and a repeated one is answered from a
+// decision cache rather than by the control.
+func (l Lab) RefusedInputURL() string {
+ return l.ControlGatewayURL() + flaggedTokenPath
+}
+
+func (l Lab) controlDependency(at Control) (string, string) {
+ switch at {
+ case ControlLimiter:
+ return l.ControlStoreAddr, EnvLimitStore
+ case ControlPolicy:
+ return l.ControlPolicyAddr, EnvPolicyStore
+ default:
+ return l.ControlRulesAddr, EnvRulesStore
+ }
+}
+
+func (l Lab) RequireControlTier(at Control) error {
+ if l.ControlPort == 0 {
+ return fmt.Errorf(
+ errNoControlGateway,
+ ErrRangeIncomplete,
+ ControlGatewayName,
+ )
+ }
+
+ address, variable := l.controlDependency(at)
+ if address == "" {
+ return fmt.Errorf(
+ errNoControlAddr,
+ ErrRangeIncomplete,
+ at,
+ ControlGatewayName,
+ variable,
+ )
+ }
+
+ return nil
+}
+
+// ControlEndpoint says where khaos binds the proxy for one control and which
+// real dependency that proxy forwards to. The listen port comes from the
+// gateway's own environment and the upstream from the dependency's address on
+// the range network, which the host routes to.
+func (l Lab) ControlEndpoint(
+ ctx context.Context,
+ reader PortReader,
+ at Control,
+) (string, string, error) {
+ if err := l.RequireControlTier(at); err != nil {
+ return "", "", err
+ }
+
+ gatewayIP, err := reader.BridgeGateway(ctx)
+ if err != nil {
+ return "", "", err
+ }
+
+ address, _ := l.controlDependency(at)
+
+ port, err := controlPort(address)
+ if err != nil {
+ return "", "", err
+ }
+
+ upstream, err := l.controlUpstream(ctx, reader, at)
+ if err != nil {
+ return "", "", err
+ }
+
+ return net.JoinHostPort(gatewayIP, strconv.Itoa(port)), upstream, nil
+}
+
+// controlPort reads the port out of either shape the gateway holds: the
+// limiter's datastore is a bare host:port and the other two are URLs.
+func controlPort(address string) (int, error) {
+ if strings.Contains(address, "://") {
+ return portOfURL(address)
+ }
+
+ _, raw, err := net.SplitHostPort(address)
+ if err != nil {
+ return 0, fmt.Errorf(
+ errBadUpstream,
+ ErrRangeIncomplete,
+ ControlGatewayName,
+ address,
+ err,
+ )
+ }
+
+ port, err := strconv.Atoi(raw)
+ if err != nil || port <= 0 {
+ return 0, fmt.Errorf(
+ errBadUpstream,
+ ErrRangeIncomplete,
+ ControlGatewayName,
+ address,
+ err,
+ )
+ }
+
+ return port, nil
+}
+
+func (l Lab) controlUpstream(
+ ctx context.Context,
+ reader PortReader,
+ at Control,
+) (string, error) {
+ name, port := ControldName, strconv.Itoa(GatewayInternalPort)
+ if at == ControlLimiter {
+ name, port = RedisName, redisPort
+ }
+
+ targets, err := reader.Targets(ctx, dockerx.Filter{Project: ProjectName})
+ if err != nil {
+ return "", err
+ }
+
+ for _, target := range targets {
+ if target.Name != name || !target.Running {
+ continue
+ }
+
+ address, err := reader.NetworkAddress(ctx, target.ID, NetworkName)
+ if err != nil {
+ return "", err
+ }
+
+ return net.JoinHostPort(address, port), nil
+ }
+
+ return "", fmt.Errorf(
+ errNoControlGateway,
+ ErrRangeIncomplete,
+ name,
+ )
+}
+
func (l Lab) SecretGatewayURL() string {
return "http://" + net.JoinHostPort(
loopback,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector.go
index ae1e1c08..ca7bf2f9 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector.go
@@ -5,10 +5,15 @@ package run
import (
"context"
+ "errors"
"github.com/CarterPerez-dev/khaos/internal/dockerx"
)
+var ErrNothingApplied = errors.New(
+ "khaos was asked to restart the target before any fault had been applied, so there is no container it can be sure was meant",
+)
+
type Resolver interface {
Accept(
ctx context.Context,
@@ -84,3 +89,15 @@ func (i *Injector) Confirm(ctx context.Context) (string, error) {
func (i *Injector) Revert(ctx context.Context) error {
return i.Session.RevertAll(ctx, i.Progress)
}
+
+// Restart restarts the container this run's fault was applied to, which is the
+// container the engine resolved and consent-checked before injecting. A
+// restart asked for before anything was injected is a refusal rather than a
+// guess at which container was meant.
+func (i *Injector) Restart(ctx context.Context) error {
+ if len(i.applied) == 0 {
+ return ErrNothingApplied
+ }
+
+ return i.Session.Restart(ctx, i.applied[0].target)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector_test.go
new file mode 100644
index 00000000..d5a401c0
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector_test.go
@@ -0,0 +1,31 @@
+// ©AngelaMos | 2026
+// injector_test.go
+
+package run
+
+import (
+ "context"
+ "errors"
+ "testing"
+)
+
+// The cold-start experiment restarts the container the fault was applied to.
+// Asked before anything is applied, khaos has no container it can be SURE was
+// meant, and guessing would restart something the operator never consented
+// for. Mutating this guard to a nil return makes the whole phase a silent
+// no-op, which is indistinguishable from a target that restarted cleanly.
+func TestRestartingBeforeAnythingIsAppliedIsRefused(t *testing.T) {
+ injector := &Injector{}
+
+ err := injector.Restart(context.Background())
+
+ if err == nil {
+ t.Fatal(
+ "khaos was asked to restart the target before any fault was applied and did not refuse",
+ )
+ }
+
+ if !errors.Is(err, ErrNothingApplied) {
+ t.Fatalf("the refusal is not ErrNothingApplied: %v", err)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go
index 9bddcff8..d8e0beee 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go
@@ -34,6 +34,10 @@ var ErrRevertFailed = errors.New(
"one or more faults could not be reverted; run `khaos reclaim` once the cause is fixed",
)
+var ErrNoDaemon = errors.New(
+ "this run holds no Docker client, so khaos cannot restart the target",
+)
+
type Progress func(record journal.Record, err error)
type Registry func(kind string) (inject.Mutator, error)
@@ -308,6 +312,30 @@ func revertWith(
return log.Append(record)
}
+// Restart stops and starts the target in place, for the one experiment whose
+// fault is the target INITIALISING without a dependency it already had.
+//
+// In place rather than recreated, which keeps the published port: a recreate
+// moves it under khaos's own probe mid-run.
+//
+// It journals nothing, unlike every other mutation here, and that is correct
+// rather than an omission: a stop followed by a start leaves the container
+// where it was, so there is no state a cold-started revert would have to undo.
+func (s *Session) Restart(
+ ctx context.Context,
+ target dockerx.Target,
+) error {
+ if s.env.Docker == nil {
+ return ErrNoDaemon
+ }
+
+ if err := s.env.Docker.Stop(ctx, target.ID); err != nil {
+ return err
+ }
+
+ return s.env.Docker.Start(ctx, target.ID)
+}
+
func (s *Session) Confirm(
ctx context.Context,
kind string,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/justfile b/PROJECTS/intermediate/chaos-engineering-tool/justfile
index df13da39..eb426091 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/justfile
+++ b/PROJECTS/intermediate/chaos-engineering-tool/justfile
@@ -97,7 +97,7 @@ fix:
check: lint tidy-check test colors
-gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate credential-gate
+gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate credential-gate control-gate
colors: build
./scripts/colors.sh
@@ -129,5 +129,8 @@ dns-gate: build
credential-gate: build
./scripts/credential.sh
+control-gate: build
+ ./scripts/control.sh
+
clean:
rm -f {{binary}}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
index b53e0db0..adf7c2c5 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
@@ -600,6 +600,72 @@ services:
retries: 15
restart: "no"
+ # One gateway carrying three security controls, the way a real API gateway
+ # does. Each control dials its OWN address, so khaos's proxy severs exactly
+ # one and the other two keep answering.
+ #
+ # In the default range: only the limiter needs the stateful tier, so only
+ # control.ratelimit_store_down is refused without --with-db.
+ gateway-control:
+ container_name: khaos-range-gateway-control
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["control"]
+ environment:
+ KHAOS_RANGE_LIMIT_STORE: "host.docker.internal:${KHAOS_RANGE_CONTROL_STORE_PORT:?khaos chooses this port; run `khaos range up`}"
+ KHAOS_RANGE_POLICY_STORE: "http://host.docker.internal:${KHAOS_RANGE_CONTROL_POLICY_PORT:?khaos chooses this port; run `khaos range up`}"
+ KHAOS_RANGE_RULES_STORE: "http://host.docker.internal:${KHAOS_RANGE_CONTROL_RULES_PORT:?khaos chooses this port; run `khaos range up`}"
+ KHAOS_RANGE_FAIL_MODE: "${KHAOS_RANGE_FAIL_MODE:-closed}"
+ # Authentication is by SIGNATURE, not from a remembered set: the
+ # cold-start experiment restarts this container and a remembered set is
+ # empty afterwards.
+ KHAOS_RANGE_CONTROL_SECRET: "${KHAOS_RANGE_CONTROL_SECRET:?khaos mints this; run `khaos range up`}"
+ # host-gateway is the DEFAULT bridge's gateway, measured, not this
+ # network's gateway: a listener on this network's own gateway address is
+ # unreachable from a container on it.
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ ports:
+ - "127.0.0.1:${KHAOS_RANGE_CONTROL_PORT:?khaos chooses this port; run `khaos range up`}:8080"
+ networks:
+ range:
+ ipv4_address: 10.244.244.19
+ # PROCESS liveness: a healthcheck on a measured endpoint would make the
+ # fixture unstartable the moment khaos severed a dependency.
+ healthcheck:
+ test: ["CMD", "/usr/local/bin/khaos-range", "alivecheck"]
+ interval: 2s
+ timeout: 2s
+ retries: 15
+ restart: "no"
+
+ # The policy document and the rule set. It publishes nothing, like the
+ # resolver: khaos's proxy forwards to it by its address on this network.
+ controld:
+ container_name: khaos-range-controld
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["controld"]
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ networks:
+ range:
+ ipv4_address: 10.244.244.21
+ healthcheck:
+ test: ["CMD", "/usr/local/bin/khaos-range", "alivecheck"]
+ interval: 2s
+ timeout: 2s
+ retries: 15
+ restart: "no"
+
networks:
range:
name: khaos-range
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh
new file mode 100755
index 00000000..25b08686
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh
@@ -0,0 +1,402 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# control.sh
+
+set -euo pipefail
+
+# M9's `control.` exit gate. Five things have to be true of this family.
+#
+# 1. EVERY experiment is asserted in BOTH directions on a live range. A
+# fail-open target reports `deviated / PERMITTED` exit 2 and a fail-closed
+# one reports `passed / HELD` exit 0 on the identical fault. One direction
+# alone cannot tell a defect from a control that was never consulted.
+#
+# 2. The limiter is DATASTORE-backed, and these two rows are the proof.
+# Kong's rate-limiting plugin defaults to `policy = local`, an in-memory
+# per-node counter with no datastore to partition: against one of those,
+# severing the datastore changes nothing and both rows come back HELD. The
+# rows differing IS the assertion that the counter is on the severed path.
+#
+# 3. The three controls are severed INDEPENDENTLY, asserted two ways: the
+# three dependency addresses read out of ONE running gateway are pairwise
+# distinct, and the rule matcher still refuses what its rules match while
+# the policy store is reachable. Address distinctness is the structure;
+# the second is the property that matters.
+#
+# 4. The cold start RAN, asserted on the container's own start time rather
+# than on khaos's stage line -- the line is emitted BEFORE the restart is
+# attempted, so it stays green against a restart that never happened.
+#
+# 5. The profile split is real. Only the limiter needs the stateful tier, so
+# only it is refused without --with-db, and the other three must RUN on a
+# default range. Asserting one without the other would pass against a
+# family that quietly required the opt-in tier everywhere.
+
+BIN="${BIN:-$PWD/khaos}"
+
+KHAOS_STATE_DIR="$(mktemp -d)"
+export KHAOS_STATE_DIR
+
+# The fault window has to clear MinimumToClaimClosed, which is 30 paired
+# samples, or a HELD verdict is honestly marked inconclusive and exits 4 rather
+# than 0. At 300ms this gives ~66.
+TIMING=(--baseline-for 5s --fault-for 20s --recover-for 3s --sample-every 300ms)
+
+# A longer window for the cold start. NOT because the restart eats into it --
+# coldStart runs before measureFault, so the restart and the settle poll cost
+# the window nothing -- but because the target it measures has just booted,
+# and a wider window is what makes a thin sample count unambiguous.
+COLD_TIMING=(--baseline-for 5s --fault-for 60s --recover-for 3s --sample-every 300ms)
+
+failures=0
+report=""
+last_code=0
+ran=()
+
+cleanup() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ rm -rf "$KHAOS_STATE_DIR"
+}
+trap cleanup EXIT INT TERM HUP
+
+pass() { echo "ok $1"; }
+fail() { echo "FAIL $1"; failures=$((failures + 1)); }
+
+# The fail mode is fixed at container creation, deliberately, so a gate
+# spanning both directions has to recreate the stack. `with_db` is passed only
+# for the limiter: the other three controls are in the default range and
+# gating them on the stateful tier would cost two more engine startups.
+start_range() {
+ local fail_mode="$1" with_db="${2:-no}"
+ local -a args=(range up --fail-mode "$fail_mode" --no-color --quiet)
+
+ [[ "$with_db" == "yes" ]] && args+=(--with-db)
+
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ if ! "$BIN" "${args[@]}" >/dev/null 2>&1; then
+ fail "range up --fail-mode $fail_mode (with-db=$with_db)"
+ return 1
+ fi
+ return 0
+}
+
+# Sets the globals `report` and `last_code`. Never call it through a command
+# substitution: that runs it in a subshell and the report is lost while the
+# exit code still comes back, which looks like a passing gate with no evidence.
+run_experiment() {
+ local experiment="$1"
+ shift
+ ran+=("$experiment")
+ set +e
+ report="$("$BIN" run "$experiment" --no-color "$@" 2>&1)"
+ last_code=$?
+ set -e
+}
+
+assert_verdict() {
+ local label="$1" want="$2"
+ if grep -qE "^ verdict .*${want}" <<<"$report"; then
+ pass "${label} verdict is ${want}"
+ else
+ fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"
+ fi
+}
+
+assert_exit() {
+ local label="$1" want="$2"
+ if [[ "$last_code" == "$want" ]]; then
+ pass "${label} exit code ${want}"
+ else
+ fail "${label} exit code ${last_code}, want ${want}"
+ fi
+}
+
+assert_confirmed() {
+ local label="$1"
+ if grep -qE '^ plan confirm fault confirmed by ' <<<"$report"; then
+ pass "${label} confirmed by a measured observable"
+ else
+ fail "${label} reported no confirmation, so the run measured a fault it never proved landed"
+ fi
+}
+
+# `--` before the pattern, or grep reads a phrase that starts with a dash as
+# its own option: asserting that a refusal names `--with-db` died with
+# "unrecognized option" and reported the product as silent about it.
+assert_report_has() {
+ local label="$1" want="$2"
+ if grep -qF -- "$want" <<<"$report"; then
+ pass "${label}"
+ else
+ fail "${label}: the report never says ${want}"
+ fi
+}
+
+assert_report_lacks() {
+ local label="$1" want="$2"
+ if grep -qF -- "$want" <<<"$report"; then
+ fail "${label}: the report says ${want}"
+ else
+ pass "${label}"
+ fi
+}
+
+# Every grepping helper ends in `|| true`: with `set -e`, a grep inside a
+# command substitution that legitimately matches nothing kills the script, and
+# `grep -c` does it while printing a perfectly good 0.
+# started_at is the restart OBSERVABLE. The cold-start stage line is emitted
+# BEFORE khaos attempts the restart, so asserting the line proves only that
+# khaos intended to restart: gutting Injector.Restart to a nil return leaves
+# it green. The container's own start time cannot be faked by khaos.
+started_at() {
+ docker inspect -f '{{.State.StartedAt}}' \
+ khaos-range-gateway-control 2>/dev/null || true
+}
+
+# dependency_ports reads all three addresses out of ONE running gateway, which
+# is the only way this comparison means anything: each `range up` draws fresh
+# random ports, so comparing addresses captured across two range lifetimes
+# tests that FreePorts returns distinct numbers rather than that the three
+# controls dial distinct dependencies.
+dependency_ports() {
+ docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' \
+ khaos-range-gateway-control 2>/dev/null |
+ grep -E 'KHAOS_RANGE_(LIMIT|POLICY|RULES)_STORE=' |
+ sed -E 's/.*:([0-9]+)$/\1/' | sort || true
+}
+
+# A fail-open target serves what the control refused a moment ago. A
+# fail-closed one keeps refusing it and pays an availability cost khaos does
+# not measure, which is the correct outcome rather than a defect.
+assert_open() {
+ local label="$1"
+ assert_confirmed "$label"
+ assert_verdict "$label" "deviated"
+ assert_report_has "${label} names the direction" "PERMITTED"
+ assert_exit "$label" 2
+}
+
+assert_closed() {
+ local label="$1"
+ assert_confirmed "$label"
+ assert_verdict "$label" "passed"
+ assert_report_has "${label} names the direction" "HELD"
+ assert_exit "$label" 0
+}
+
+# The limiter's own fail-closed row, and it is NOT the one above. A limiter
+# that cannot count refuses EVERY request rather than only the over-budget
+# one, so the valid stream stops being served and khaos reads DENIED --
+# correct, with an availability cost, and recorded without being called a
+# defect because the polarity declares only PERMITTED. Measured on the live
+# range at 62 refusals of the valid credential across a 20s window.
+assert_denied() {
+ local label="$1"
+ assert_confirmed "$label"
+ assert_verdict "$label" "deviated"
+ assert_report_has "${label} names the direction" "DENIED"
+ assert_report_has \
+ "${label} is not called a defect" \
+ "without calling it a defect"
+ assert_exit "$label" 0
+}
+
+echo "== the three default-range controls, fail-open =="
+
+if start_range open; then
+ run_experiment control.policy_store_down "${TIMING[@]}"
+ assert_open "policy_store_down fail-open"
+
+ # The counterpart of the cold-start assertion below: these two differ in
+ # the DECLARED defect and in nothing else khaos prints.
+ assert_report_has \
+ "policy_store_down names the dropped policy" \
+ "stops being enforced when its store is unreachable"
+
+ run_experiment control.waf_rules_backend_down "${TIMING[@]}"
+ assert_open "waf_rules_backend_down fail-open"
+
+ before_restart="$(started_at)"
+
+ run_experiment control.policy_store_cold_start "${COLD_TIMING[@]}"
+ assert_open "policy_store_cold_start fail-open"
+
+ # The container's OWN start time, which khaos cannot fake. Without this
+ # the fail-open cold-start row is byte-identical in outcome to
+ # policy_store_down's, and a gutted Injector.Restart stays green.
+ after_restart="$(started_at)"
+ if [[ -n "$before_restart" && "$before_restart" != "$after_restart" ]]; then
+ pass "policy_store_cold_start restarted the container (start time moved)"
+ else
+ fail "the container's start time did not move, so no restart happened: ${before_restart} -> ${after_restart}"
+ fi
+
+ # The phase, not the verdict. This experiment reports the same direction on
+ # the same observable as policy_store_down, so without this assertion a
+ # deleted restart leaves the verdict, the exit code and the gate unchanged.
+ assert_report_has \
+ "policy_store_cold_start actually restarted the target" \
+ "restarting the target so it initialises"
+
+ # And the finding has to name the EMPTY policy set rather than a dropped
+ # one. Both experiments read the same direction off the same observable,
+ # so without this the two reports are identical and the verdict alone
+ # cannot tell an operator which bug they have.
+ assert_report_has \
+ "policy_store_cold_start names the empty policy set" \
+ "an empty policy set read as a policy that denies nothing"
+
+ assert_report_lacks \
+ "policy_store_cold_start does not claim a policy was dropped" \
+ "stops being enforced when its store is unreachable"
+
+ # The limiter needs the stateful tier and this range does not have it, so
+ # it is refused BY NAME rather than reported as a control that was not
+ # enforcing. That is the M4 finding arriving through another door.
+ run_experiment control.ratelimit_store_down "${TIMING[@]}"
+ if [[ "$last_code" == 0 ]]; then
+ fail "ratelimit_store_down ran without the stateful tier, so it measured a datastore that is not there"
+ else
+ pass "ratelimit_store_down is refused without the stateful tier"
+ fi
+ assert_report_has \
+ "the refusal names the tier rather than just saying no" \
+ "--with-db"
+fi
+
+echo "== the same three, fail-closed =="
+
+if start_range closed; then
+ run_experiment control.policy_store_down "${TIMING[@]}"
+ assert_closed "policy_store_down fail-closed"
+
+ # The caveat is what a HELD finding carries instead of a leak reason, and
+ # the availability cost is the half khaos must not claim to have measured.
+ assert_report_has \
+ "the held run states the availability cost it did not measure" \
+ "availability cost"
+
+ assert_report_lacks \
+ "no credential is claimed to have been accepted" \
+ "was accepted"
+
+ run_experiment control.waf_rules_backend_down "${TIMING[@]}"
+ assert_closed "waf_rules_backend_down fail-closed"
+
+ # DENIED rather than HELD, and for the same reason as the limiter: a
+ # target that INITIALISED with no policy at all refuses every request
+ # rather than only the one the policy denies, so the valid stream stops
+ # being served. Measured at 183 refusals across a 60s window. The
+ # polarity declares only PERMITTED, so khaos records the direction and
+ # explicitly does not call it a defect.
+ run_experiment control.policy_store_cold_start "${COLD_TIMING[@]}"
+ assert_denied "policy_store_cold_start fail-closed"
+fi
+
+echo "== the limiter, both directions, on the stateful tier =="
+
+if start_range open yes; then
+ run_experiment control.ratelimit_store_down "${TIMING[@]}"
+ assert_open "ratelimit_store_down fail-open"
+
+ # The defect in its own words. Kong's fault_tolerant = true is the one
+ # genuine fail-open default in the twenty-two-row taxonomy, and this is
+ # the sentence that says khaos measured it rather than inferred it.
+ assert_report_has \
+ "the run names the limiter's own defect" \
+ "disables itself when its datastore is unreachable"
+fi
+
+if start_range closed yes; then
+ run_experiment control.ratelimit_store_down "${TIMING[@]}"
+ assert_denied "ratelimit_store_down fail-closed"
+fi
+
+echo "== the three controls are severed independently =="
+
+# The structural half of the one-gateway design. Two of these dependencies
+# terminate at the SAME backend container, so the only thing that makes them
+# two faults is the gateway dialling a different address for each.
+#
+# Read from ONE running gateway. The previous version compared addresses
+# captured across two different `range up` lifetimes, so it was comparing
+# ports from two unrelated random draws: it would have passed with all three
+# controls wired to the same port, and could only ever fail on a coincidence.
+if start_range closed; then
+ ports="$(dependency_ports)"
+ distinct="$(echo "$ports" | sort -u | grep -c . || true)"
+ total="$(echo "$ports" | grep -c . || true)"
+
+ if [[ "$total" != 3 ]]; then
+ fail "the control gateway carries ${total} dependency addresses, want 3; this section cannot fail"
+ elif [[ "$distinct" == 3 ]]; then
+ pass "all three controls dial distinct dependency addresses"
+ else
+ fail "the three controls resolve to ${distinct} distinct addresses, so one fault severs more than one control"
+ fi
+
+ # And the behavioural half: with the policy store severed mid-run, the
+ # rule matcher must still refuse what its own rules match. Address
+ # distinctness is structure; this is the property that matters.
+ token_url="http://127.0.0.1:$(docker port khaos-range-gateway-control 8080 | head -1 | sed -E 's/.*://')"
+ flagged="$(curl -s -X POST "${token_url}/flagged-token" |
+ sed -E 's/.*"token":"([^"]+)".*/\1/' || true)"
+
+ if [[ -z "$flagged" ]]; then
+ fail "could not obtain a flagged credential, so the behavioural check cannot fail"
+ else
+ filtered_status="$(curl -s -o /dev/null -w '%{http_code}' \
+ -H "Authorization: Bearer ${flagged}" \
+ "${token_url}/filtered" || true)"
+
+ if [[ "$filtered_status" == 403 ]]; then
+ pass "the rule matcher refuses a matching request while the policy store is reachable"
+ else
+ fail "the rule matcher answered ${filtered_status} to a request its rules match, want 403"
+ fi
+ fi
+fi
+
+echo "== the census: every catalogued fault was actually run =="
+
+# Read from `run --help` rather than from `catalog`: catalog TRUNCATES an
+# experiment name at 30 characters, and control.policy_store_cold_start is
+# longer than that, so a census built on catalog would compare a truncated
+# name and fail for the wrong reason.
+catalogued="$(NO_COLOR=1 "$BIN" run --help 2>/dev/null |
+ grep -oE 'control\.[a-z_]+' | sort -u || true)"
+
+if [[ -z "$catalogued" ]]; then
+ fail "the census read no control. experiments at all, so it cannot fail"
+fi
+
+missing=0
+for name in $catalogued; do
+ found=0
+ for executed in "${ran[@]}"; do
+ [[ "$executed" == "$name" ]] && found=1
+ done
+ if [[ "$found" == 0 ]]; then
+ fail "the catalogue advertises ${name} and this gate never RAN it"
+ missing=$((missing + 1))
+ fi
+done
+
+if [[ "$missing" == 0 ]]; then
+ pass "every catalogued control. fault was actually run by this gate (${#ran[@]} runs)"
+fi
+
+# And the reverse: a name this gate ran that is NOT catalogued means the gate
+# is asserting against something that no longer exists.
+for executed in "${ran[@]}"; do
+ if ! grep -qx "$executed" <<<"$catalogued"; then
+ fail "${executed} was run by this gate and is not in the catalogue"
+ fi
+done
+
+if [[ "$failures" -gt 0 ]]; then
+ echo "control: ${failures} assertion(s) failed"
+ exit 1
+fi
+
+echo "control: all assertions passed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
index 235e52c8..627e5b81 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
@@ -288,7 +288,7 @@ if "$BIN" range up --no-color --quiet >/dev/null 2>&1; then
"khaos range up --with-db"
assert_report_has \
"the refusal says why khaos does not fake it" \
- "the finding IS what three correct engines do"
+ "the finding IS what correct ones do"
else
fail "range up (without the stateful tier)"
fi
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh
index 2e450e43..720e6eab 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh
@@ -177,9 +177,9 @@ assert_confirmed_by_an_observable() {
# happened: "a credential khaos revoked was still accepted" is not "a credential
# khaos minted with a deliberate defect was accepted".
# Per CLASS, not an OR over every phrase khaos can print. An OR let a forgery
-# experiment satisfy this with the generic monotonic reason, which says a
-# credential the target had already rejected got through and says nothing about
-# the trap the experiment is named for.
+# experiment satisfy this with the generic monotonic reason, which says an
+# input the target had already refused got through and says nothing about the
+# trap the experiment is named for.
assert_states_a_finding() {
local label="$1" want="$2"
@@ -367,7 +367,7 @@ for row in "${CATCHING[@]}"; do
assert_reports_the_exposure_bound "$label"
;;
identity.introspection_error | identity.malformed_response | identity.idp_latency)
- assert_states_a_finding "$label" "had already rejected"
+ assert_states_a_finding "$label" "had already refused"
assert_does_not_claim_a_revocation "$label"
;;
*)