Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(khaos): the authorization tier, and the one experiment that needs two fault windows
CarterPerez-dev2ca0a1c93e48
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.gonew file mode 100644index 00000000..aa6fd6df--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go@@ -0,0 +1,216 @@+// ©AngelaMos | 2026+// authzgateway.go++package main++import (+ "crypto/rand"+ "net/http"+ "strings"+ "sync"+ "time"+)++const (+ // AuthzDistinguishes is correct: an authorization dependency has three+ // outcomes - allow, deny and unknown - and only unknown is eligible for a+ // fallback policy.+ AuthzDistinguishes = "distinguishes"++ // AuthzCollapses is research 07 section 6.2's exact shape:+ //+ // if err != nil || resp.StatusCode != 200 { return Allow }+ //+ // The dependency said no, loudly and correctly, and the caller read it as+ // "no answer". It is the single most common fail-open in service-to-service+ // code because error handling is usually written as "did it work" rather+ // than "what did it say".+ AuthzCollapses = "collapses"++ // AuthzRetries reaches the right verdict the wrong way: generic retry+ // middleware that retries any non-2xx, so a definitive refusal is asked+ // again and again. It fails CLOSED, which is why it is a third mode rather+ // than a variant of the second, and it is the shape research 07 section 6.2+ // step 4 names as the retry storm.+ AuthzRetries = "retries"++ headerAuthzReason = "X-Khaos-Authz"++ reasonNotAuthenticated = "this service never issued that credential, which is its own check and not the dependency's"++ reasonAuthzAllowed = "the authorization dependency allowed it"++ reasonAuthzDenied = "the authorization dependency denied it"++ reasonAuthzUnknown = "the authorization dependency did not answer, and this mode refuses when it cannot tell"++ reasonAuthzFellOpen = "the authorization dependency did not answer, so this mode allowed it"++ reasonCollapsed = "the authorization dependency DENIED it and this mode read that as no answer, so it allowed it"++ // Three attempts, because the finding is the ask COUNT and one retry would+ // make it indistinguishable from a second probe arriving.+ authzAttempts = 3+ authzBackoff = 50 * time.Millisecond+)++// authzGateway is the CALLER, and the only range fixture whose dependency is+// not a container: khaos serves that itself, because no correct authorization+// service answers a definitive deny on request.+//+// AUTHENTICATION is this service's own job and it is correct in every mode.+// That separation is what makes the experiment measurable: a caller that allows+// on any non-200 from its authorization dependency has no authorization check+// at all, so khaos's baseline guard would correctly void a run against one that+// did nothing else. Research 07's row is about a service whose other checks+// work and whose authorization check collapses.+type authzGateway struct {+ dependency string+ mode string+ client *http.Client++ mu sync.Mutex+ issued map[string]bool+}++func newAuthzGateway(dependency, mode string) *authzGateway {+ return &authzGateway{+ dependency: dependency,+ mode: mode,+ client: &http.Client{Timeout: upstreamTimeout},+ issued: map[string]bool{},+ }+}++func (g *authzGateway) routes() *http.ServeMux {+ mux := http.NewServeMux()++ mux.HandleFunc(routeAlive, func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusOK)+ })+ mux.HandleFunc(routeHealth, func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusOK)+ })+ mux.HandleFunc(routeProtected, g.protected)+ mux.HandleFunc(issuePath, g.issue)++ return mux+}++// issue hands out a credential this service will AUTHENTICATE, which is what+// lets khaos present one it will refuse as well as one it will accept.+func (g *authzGateway) issue(w http.ResponseWriter, _ *http.Request) {+ credential := rand.Text()++ g.mu.Lock()+ g.issued[credential] = true+ g.mu.Unlock()++ w.Header().Set(contentType, mimeJSON)+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"token":"` + credential + `"}`))+}++func (g *authzGateway) protected(w http.ResponseWriter, r *http.Request) {+ credential, scheme := strings.CutPrefix(+ r.Header.Get(headerAuthorization),+ bearerPrefix,+ )++ if !scheme || credential == "" {+ w.Header().Set(headerAuthzReason, reasonNoCredential)+ w.WriteHeader(http.StatusUnauthorized)++ return+ }++ g.mu.Lock()+ known := g.issued[credential]+ g.mu.Unlock()++ if !known {+ w.Header().Set(headerAuthzReason, reasonNotAuthenticated)+ w.WriteHeader(http.StatusUnauthorized)++ return+ }++ allow, reason := g.decide(credential)++ w.Header().Set(headerAuthzReason, reason)++ if allow {+ w.WriteHeader(http.StatusOK)+ return+ }++ w.WriteHeader(http.StatusForbidden)+}++// decide is the whole fixture. One branch reads the dependency's three answers+// as three; the second reads "not 200" as "no answer"; the third reads them+// correctly and asks again every time it does not like one.+func (g *authzGateway) decide(credential string) (bool, string) {+ status, reached := g.ask(credential)++ switch g.mode {+ case AuthzCollapses:+ if reached && status == http.StatusForbidden {+ return true, reasonCollapsed+ }++ if !reached || status != http.StatusOK {+ return true, reasonAuthzFellOpen+ }++ return true, reasonAuthzAllowed++ case AuthzRetries:+ for attempt := 1; attempt < authzAttempts; attempt++ {+ if reached && status == http.StatusOK {+ break+ }++ time.Sleep(authzBackoff)++ status, reached = g.ask(credential)+ }+ }++ if !reached {+ return false, reasonAuthzUnknown+ }++ switch status {+ case http.StatusOK:+ return true, reasonAuthzAllowed++ case http.StatusForbidden, http.StatusUnauthorized:+ return false, reasonAuthzDenied++ default:+ // Unknown, and only unknown is eligible for a fallback policy. This+ // fixture's policy is to refuse.+ return false, reasonAuthzUnknown+ }+}++// ask forwards the credential to the authorization dependency the way an+// Envoy `ext_authz` filter does: the original request's Authorization header,+// and the RESPONSE STATUS is the decision.+func (g *authzGateway) ask(credential string) (int, bool) {+ request, err := http.NewRequest(http.MethodGet, g.dependency, nil)+ if err != nil {+ return 0, false+ }++ request.Header.Set(headerAuthorization, bearerPrefix+credential)++ response, err := g.client.Do(request)+ if err != nil {+ return 0, false+ }+ defer func() { _ = response.Body.Close() }()++ return response.StatusCode, true+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.gonew file mode 100644index 00000000..c0d4e7ea--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go@@ -0,0 +1,265 @@+// ©AngelaMos | 2026+// authzgateway_test.go++package main++import (+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+)++// stubAuthz is khaos's side of the experiment, reduced to what the fixture can+// observe: an answer it is told to give, and a count of how many times it was+// asked for one.+type stubAuthz struct {+ status atomic.Int64+ asked atomic.Int64+ server *httptest.Server+}++func newStubAuthz(t *testing.T) *stubAuthz {+ t.Helper()++ stub := &stubAuthz{}+ stub.status.Store(http.StatusOK)++ stub.server = httptest.NewServer(+ http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ stub.asked.Add(1)+ w.WriteHeader(int(stub.status.Load()))+ }),+ )+ t.Cleanup(stub.server.Close)++ return stub+}++func authzFixture(t *testing.T, mode string) (*authzGateway, *stubAuthz) {+ t.Helper()++ dependency := newStubAuthz(t)++ return newAuthzGateway(dependency.server.URL, mode), dependency+}++// issued asks the fixture for a credential it will AUTHENTICATE, which is the+// property that makes the whole experiment measurable: a caller that allows on+// any non-200 from its authorization dependency refuses nothing at all, and+// khaos's baseline guard would void the run rather than measure one.+func issued(t *testing.T, g *authzGateway) string {+ t.Helper()++ recorder := httptest.NewRecorder()+ request := httptest.NewRequest(http.MethodPost, issuePath, nil)++ g.routes().ServeHTTP(recorder, request)++ var body struct {+ Token string `json:"token"`+ }++ if err := json.NewDecoder(recorder.Body).Decode(&body); err != nil {+ t.Fatalf("decode the issued credential: %v", err)+ }++ if body.Token == "" {+ t.Fatal("the fixture issued no credential")+ }++ return body.Token+}++func askAuthz(t *testing.T, g *authzGateway, credential string) int {+ t.Helper()++ request := httptest.NewRequest(http.MethodGet, routeProtected, nil)+ if credential != "" {+ request.Header.Set(headerAuthorization, bearerPrefix+credential)+ }++ recorder := httptest.NewRecorder()+ g.routes().ServeHTTP(recorder, request)++ return recorder.Code+}++// Authentication is the fixture's OWN check and it is correct in every mode.+// Without it there is no baseline: the collapsing caller would refuse nothing,+// and an experiment against it could only ever report that the target was not+// enforcing anything to begin with.+func TestEveryModeAuthenticatesLocally(t *testing.T) {+ for _, mode := range []string{+ AuthzDistinguishes,+ AuthzCollapses,+ AuthzRetries,+ } {+ t.Run(mode, func(t *testing.T) {+ gateway, dependency := authzFixture(t, mode)++ if got := askAuthz(+ t,+ gateway,+ "a-credential-this-service-never-issued",+ ); got != http.StatusUnauthorized {+ t.Fatalf(+ "an unissued credential returned %d, want %d: this check is what gives the experiment a baseline",+ got,+ http.StatusUnauthorized,+ )+ }++ if got := askAuthz(t, gateway, ""); got != http.StatusUnauthorized {+ t.Fatalf("no credential returned %d, want %d",+ got, http.StatusUnauthorized)+ }++ if asked := dependency.asked.Load(); asked != 0 {+ t.Fatalf(+ "the fixture asked its authorization dependency %d times about a credential it had not authenticated; the two checks are separate and khaos counts the second one",+ asked,+ )+ }+ })+ }+}++// The three callers, which are three cells of research 07 section 6.2's+// matrix. The deny is the row that separates them.+func TestTheThreeCallersReadADenyDifferently(t *testing.T) {+ for _, row := range []struct {+ mode string+ wantDeny int+ wantOut int+ wantAsked int64+ }{+ {+ mode: AuthzDistinguishes,+ wantDeny: http.StatusForbidden,+ wantOut: http.StatusForbidden,+ wantAsked: 1,+ },+ {+ mode: AuthzCollapses,+ wantDeny: http.StatusOK,+ wantOut: http.StatusOK,+ wantAsked: 1,+ },+ {+ mode: AuthzRetries,+ wantDeny: http.StatusForbidden,+ wantOut: http.StatusForbidden,+ wantAsked: authzAttempts,+ },+ } {+ t.Run(row.mode, func(t *testing.T) {+ gateway, dependency := authzFixture(t, row.mode)+ credential := issued(t, gateway)++ if got := askAuthz(+ t,+ gateway,+ credential,+ ); got != http.StatusOK {+ t.Fatalf("an allowed credential returned %d, want 200", got)+ }++ dependency.status.Store(http.StatusForbidden)+ dependency.asked.Store(0)++ if got := askAuthz(t, gateway, credential); got != row.wantDeny {+ t.Fatalf(+ "under a definitive 403 from its dependency the caller answered %d, want %d",+ got,+ row.wantDeny,+ )+ }++ if asked := dependency.asked.Load(); asked != row.wantAsked {+ t.Fatalf(+ "the caller asked %d time(s) about a credential its dependency had refused, want %d",+ asked,+ row.wantAsked,+ )+ }++ dependency.status.Store(http.StatusServiceUnavailable)++ if got := askAuthz(t, gateway, credential); got != row.wantOut {+ t.Fatalf(+ "with its dependency unable to answer the caller answered %d, want %d",+ got,+ row.wantOut,+ )+ }+ })+ }+}++// The collapse has to be reached through the STATUS and not through a transport+// error, because that is the distinction the whole experiment is about: every+// network fault khaos already has produces the error, and only this tier can+// produce a definitive refusal.+func TestTheCollapseNamesWhichAnswerItIgnored(t *testing.T) {+ gateway, dependency := authzFixture(t, AuthzCollapses)+ credential := issued(t, gateway)++ dependency.status.Store(http.StatusForbidden)++ request := httptest.NewRequest(http.MethodGet, routeProtected, nil)+ request.Header.Set(headerAuthorization, bearerPrefix+credential)++ recorder := httptest.NewRecorder()+ gateway.routes().ServeHTTP(recorder, request)++ reason := recorder.Header().Get(headerAuthzReason)++ if !strings.Contains(reason, "DENIED") {+ t.Fatalf(+ "the collapsing caller allowed a denied credential and said %q, which does not say the dependency had refused it",+ reason,+ )+ }++ dependency.status.Store(http.StatusServiceUnavailable)++ outage := httptest.NewRecorder()+ gateway.routes().ServeHTTP(+ outage,+ request.Clone(request.Context()),+ )++ if outage.Header().Get(headerAuthzReason) == reason {+ t.Fatal(+ "the collapsing caller gives the same reason for a deny and for an outage; the fixture is the thing khaos measures, so the two have to be distinguishable in it even though its behaviour is not",+ )+ }+}++// A caller that cannot reach its dependency at all is the branch every network+// fault already reaches, and the correct modes must refuse there too.+func TestAnUnreachableDependencyIsRefusedByTheCorrectModes(t *testing.T) {+ for _, mode := range []string{AuthzDistinguishes, AuthzRetries} {+ t.Run(mode, func(t *testing.T) {+ gateway, dependency := authzFixture(t, mode)+ credential := issued(t, gateway)++ dependency.server.Close()++ if got := askAuthz(+ t,+ gateway,+ credential,+ ); got != http.StatusForbidden {+ t.Fatalf(+ "with its dependency unreachable the caller answered %d, want %d",+ got,+ http.StatusForbidden,+ )+ }+ })+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.goindex d1389473..5d2e2dbc 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go@@ -28,6 +28,7 @@ const (roleResolver = "resolver"roleDB = "db"roleSecret = "secret"+ roleAuthz = "authz"roleHealth = "healthcheck"roleAlive = "alivecheck"@@ -60,6 +61,13 @@ const (// the file faults target: whichever is set is where the secret comes from.envSecretVar = "KHAOS_RANGE_SECRET_VAR"+ envAuthzMode = "KHAOS_RANGE_AUTHZ_MODE"++ // Where the service-to-service caller asks for an authorization decision,+ // which is khaos itself: no correct dependency answers a definitive deny+ // on request.+ envAuthzBase = "KHAOS_RANGE_AUTHZ_BASE"+defaultSecretPath = "/run/khaos-secrets/hmac"envCacheTTL = "KHAOS_RANGE_CACHE_TTL"@@ -70,6 +78,7 @@ const (// start and measure nothing.defaultDependency = "https://host.docker.internal:0"defaultAuthorityBase = "http://host.docker.internal:0"+ defaultAuthzBase = "http://host.docker.internal:0"shutdownGrace = 2 * time.Second@@ -78,7 +87,7 @@ const (// look like the fault's.readHeaderTimeout = 120 * time.Second- usage = "usage: khaos-range <idp|gateway|verifier|tls|rogue|resolver|db|secret|healthcheck|alivecheck>"+ usage = "usage: khaos-range <idp|gateway|verifier|tls|rogue|resolver|db|secret|authz|healthcheck|alivecheck>"selfBase = "http://127.0.0.1:8080"selfHealthPath = "/health"selfAlivePath = "/alive"@@ -168,6 +177,11 @@ func roleHandler(role string) (http.Handler, error) {).routes(), nilcase roleRogue:return newRogueProvider().routes(), nil+ case roleAuthz:+ return newAuthzGateway(+ envOr(envAuthzBase, defaultAuthzBase),+ envOr(envAuthzMode, AuthzDistinguishes),+ ).routes(), nilcase roleSecret:return newSecretGateway(envOr(envSecretPath, defaultSecretPath),diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.gonew file mode 100644index 00000000..ebf429bd--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go@@ -0,0 +1,363 @@+// ©AngelaMos | 2026+// authz.go++package authz++import (+ "context"+ "crypto/rand"+ "errors"+ "fmt"+ "io"+ "net"+ "net/http"+ "strings"+ "sync"+ "sync/atomic"+ "time"++ "github.com/CarterPerez-dev/khaos/internal/bindguard"+)++const (+ // Any path, because this is the Envoy `ext_authz` HTTP contract rather than+ // an endpoint of khaos's own invention: the filter forwards the request's+ // headers to whatever path it was configured with and reads the RESPONSE+ // STATUS as the decision. 200 allows, 403 denies, and a 5xx is the only+ // answer the `failure_mode_allow` knob is permitted to act on.+ routeAny = "/"++ headerAuthorization = "Authorization"+ bearerPrefix = "Bearer "+ headerReason = "X-Khaos-Authz"++ reasonAllowed = "khaos's authorization dependency has not been told to deny this credential"+ reasonDenied = "khaos's authorization dependency denies this credential: a definitive verdict, with full reachability"+ reasonOutage = "khaos's authorization dependency cannot answer; this is the absence of a decision, not a decision"++ subjectAuthz = "an authorization service that allows every credential it was not told to deny"++ askTimeout = 2 * time.Second+ shutdownGrace = 2 * time.Second+ controlLen = 16++ errAlreadyListening = "%w: khaos's authorization dependency is already listening on %s"+ errAsk = "khaos could not ask its own authorization dependency: %w"+ errNotListening = "%w: khaos's authorization dependency is not listening, so there is nothing to read back"++ confirmDenied = "khaos's own authorization dependency answers %d for the credential under test and %d for a control credential it was never told to deny, read back through its own HTTP surface"++ confirmOutage = "khaos's own authorization dependency answers %d for every credential, including a control one it was never told to deny"+)++var ErrListening = errors.New(+ "khaos's authorization dependency is already listening",+)++var ErrNotListening = errors.New(+ "khaos's authorization dependency has no listener",+)++var ErrClosed = errors.New(+ "khaos's authorization dependency is closed",+)++// Dependency is khaos playing an authorization service, for the same reason it+// plays an identity provider and a certificate authority: no correct dependency+// will answer a definitive deny on command, and the defect this tier hunts only+// exists when one does.+//+// It ALLOWS by default. A stand-in authority that refused anything it was not+// told to deny would make the baseline khaos's posture rather than the target's,+// and the whole measurement is the difference between two windows.+type Dependency struct {+ mu sync.RWMutex+ trap Trap++ denied map[string]bool+ outage bool++ listener net.Listener+ server *http.Server++ asked atomic.Int64+ refused atomic.Int64++ wg sync.WaitGroup+ once sync.Once+ closed chan struct{}+}++func New() *Dependency {+ return &Dependency{+ denied: map[string]bool{},+ closed: make(chan struct{}),+ }+}++// Listen binds where the TARGET already looks, and refuses an address khaos+// must not serve this from. The refusal is not ceremony: until it is told+// otherwise this dependency ALLOWS every credential presented to it, so a+// wildcard bind would put an allow-everything authorization service on every+// interface the host has.+func (d *Dependency) Listen(addr string) error {+ if err := bindguard.Check(addr, subjectAuthz); err != nil {+ return err+ }++ // Guard and bind under ONE acquisition, or two callers both bind and the+ // second orphans the first listener and its goroutine.+ d.mu.Lock()+ defer d.mu.Unlock()++ if d.listener != nil {+ return fmt.Errorf(+ errAlreadyListening,+ ErrListening,+ d.listener.Addr().String(),+ )+ }++ listener, err := net.Listen("tcp", addr)+ if err != nil {+ return err+ }++ server := &http.Server{+ Handler: d.routes(),+ ReadHeaderTimeout: shutdownGrace,+ }++ d.listener = listener+ d.server = server++ d.wg.Add(1)++ go func() {+ defer d.wg.Done()+ _ = server.Serve(listener)+ }()++ return nil+}++func (d *Dependency) Addr() string {+ d.mu.RLock()+ defer d.mu.RUnlock()++ if d.listener == nil {+ return ""+ }++ return d.listener.Addr().String()+}++func (d *Dependency) Close() error {+ var err error++ d.once.Do(func() {+ close(d.closed)++ d.mu.RLock()+ server := d.server+ d.mu.RUnlock()++ if server != nil {+ err = server.Close()+ }++ d.wg.Wait()+ })++ return err+}++func (d *Dependency) routes() *http.ServeMux {+ mux := http.NewServeMux()+ mux.HandleFunc(routeAny, d.authorize)++ return mux+}++func (d *Dependency) authorize(w http.ResponseWriter, r *http.Request) {+ d.asked.Add(1)++ credential, _ := strings.CutPrefix(+ r.Header.Get(headerAuthorization),+ bearerPrefix,+ )++ status, reason := d.decide(credential)++ // Only the DEFINITIVE refusals are counted, and only the credential under+ // test is ever answered with one. That makes this count the number of+ // times the target asked again about a credential it had already been+ // told no about, which nothing else can measure from outside it.+ if status == http.StatusForbidden {+ d.refused.Add(1)+ }++ w.Header().Set(headerReason, reason)+ w.WriteHeader(status)+}++// decide is the whole dependency. Three outcomes, because an authorization+// dependency has three and only the third is eligible for a fallback policy.+func (d *Dependency) decide(credential string) (int, string) {+ d.mu.RLock()+ defer d.mu.RUnlock()++ if d.outage {+ return http.StatusServiceUnavailable, reasonOutage+ }++ if d.denied[credential] {+ return http.StatusForbidden, reasonDenied+ }++ return http.StatusOK, reasonAllowed+}++// Arm applies a trap. An inert arm prepares everything and tells the dependency+// nothing, so the confirmation that reads it back through its own surface has+// something it must reject.+func (d *Dependency) Arm(trap Trap) error {+ select {+ case <-d.closed:+ return ErrClosed+ default:+ }++ if err := trap.Validate(); err != nil {+ return err+ }++ d.mu.Lock()+ defer d.mu.Unlock()++ d.trap = trap++ if trap.Inert {+ return nil+ }++ switch trap.Kind {+ case Deny:+ d.denied[trap.Credential] = true+ case Outage:+ d.outage = true+ }++ return nil+}++// Disarm restores the allow-everything posture. Nothing is removed from the+// denied set by Arm, so this is the single place the fault is lifted.+func (d *Dependency) Disarm() {+ d.mu.Lock()+ defer d.mu.Unlock()++ d.denied = map[string]bool{}+ d.outage = false+}++// Asked is the liveness guard: the number of authorization decisions the target+// asked for. Zero during the baseline means the target does not consult an+// authorization dependency at all, and the fault is one it cannot experience.+//+// khaos's own two-sided confirmation reads this dependency through the same+// surface and is counted here too. Every guard built on this number reads a+// DIFFERENCE across a window, and the confirmations run between windows rather+// than inside one, so none of them sees khaos's own traffic.+func (d *Dependency) Asked() (int64, error) {+ return d.asked.Load(), nil+}++func (d *Dependency) Refused() int64 {+ return d.refused.Load()+}++// Observe is the two-sided confirmation, and it reads the dependency back+// through its OWN HTTP surface rather than the flag that was set: a dependency+// whose handler was wrong would otherwise confirm a fault it is not serving.+//+// Both sides are needed. The credential under test must be refused, and a+// control credential khaos never named must still be allowed, or a dependency+// that had started refusing everything would read as a successful deny.+func (d *Dependency) Observe(ctx context.Context) (string, bool, error) {+ d.mu.RLock()+ trap := d.trap+ d.mu.RUnlock()++ control, err := d.ask(ctx, rand.Text()[:controlLen])+ if err != nil {+ return "", false, err+ }++ if trap.Kind == Outage {+ return confirms(trap.Kind, control, control)+ }++ subject, err := d.ask(ctx, trap.Credential)+ if err != nil {+ return "", false, err+ }++ return confirms(trap.Kind, subject, control)+}++// confirms is the two-sided judgement, kept separate from the dialling so that+// every combination of answers can be put to it. The control side is not+// reachable through this type's own API -- nothing can tell the dependency to+// refuse everything -- and a regression that made it do so is precisely what+// the control side exists to catch: khaos would otherwise measure a run where+// its own dependency denied the valid credential, and report the target's+// correct refusal of it as a defect.+func confirms(kind Kind, subject, control int) (string, bool, error) {+ if kind == Outage {+ if control != http.StatusServiceUnavailable {+ return "", false, nil+ }++ return fmt.Sprintf(confirmOutage, control), true, nil+ }++ if subject != http.StatusForbidden || control != http.StatusOK {+ return "", false, nil+ }++ return fmt.Sprintf(confirmDenied, subject, control), true, nil+}++// ask dials khaos's own listener, which is the only way to read what the+// dependency ANSWERS rather than what khaos believes it armed.+func (d *Dependency) ask(ctx context.Context, credential string) (int, error) {+ addr := d.Addr()+ if addr == "" {+ return 0, fmt.Errorf(errNotListening, ErrNotListening)+ }++ request, err := http.NewRequestWithContext(+ ctx,+ http.MethodGet,+ "http://"+addr+routeAny,+ nil,+ )+ if err != nil {+ return 0, fmt.Errorf(errAsk, err)+ }++ request.Header.Set(headerAuthorization, bearerPrefix+credential)++ client := &http.Client{Timeout: askTimeout}++ response, err := client.Do(request)+ if err != nil {+ return 0, fmt.Errorf(errAsk, err)+ }+ defer func() { _ = response.Body.Close() }()++ _, _ = io.Copy(io.Discard, response.Body)++ return response.StatusCode, nil+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz_test.gonew file mode 100644index 00000000..2d649784--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz_test.go@@ -0,0 +1,440 @@+// ©AngelaMos | 2026+// authz_test.go++package authz++import (+ "context"+ "errors"+ "io"+ "net/http"+ "testing"++ "github.com/CarterPerez-dev/khaos/internal/bindguard"+)++const (+ subject = "the-credential-under-test"+ other = "a-credential-nobody-named"+ loop = "127.0.0.1:0"+)++func serving(t *testing.T) *Dependency {+ t.Helper()++ dependency := New()++ if err := dependency.Listen(loop); err != nil {+ t.Fatalf("listen: %v", err)+ }++ t.Cleanup(func() { _ = dependency.Close() })++ return dependency+}++// through reads the dependency the way the TARGET does, over HTTP, because+// every claim khaos makes about this tier is a claim about what the dependency+// answered rather than about what khaos set.+func through(t *testing.T, dependency *Dependency, credential string) int {+ t.Helper()++ request, err := http.NewRequestWithContext(+ context.Background(),+ http.MethodGet,+ "http://"+dependency.Addr()+"/",+ nil,+ )+ if err != nil {+ t.Fatalf("request: %v", err)+ }++ request.Header.Set(headerAuthorization, bearerPrefix+credential)++ response, err := http.DefaultClient.Do(request)+ if err != nil {+ t.Fatalf("ask: %v", err)+ }+ defer func() { _ = response.Body.Close() }()++ _, _ = io.Copy(io.Discard, response.Body)++ return response.StatusCode+}++// An authorization dependency has three outcomes and khaos's stand-in has to+// be able to give all three, because the whole tier exists for the one in the+// middle: a definitive refusal from a dependency that is perfectly reachable.+func TestTheDependencyAnswersAllowDenyAndUnknown(t *testing.T) {+ dependency := serving(t)++ if status := through(t, dependency, subject); status != http.StatusOK {+ t.Fatalf(+ "before any fault the dependency answered %d, want %d: a stand-in that refuses anything it was not told to refuse makes the baseline khaos's posture rather than the target's",+ status,+ http.StatusOK,+ )+ }++ if err := dependency.Arm(+ Trap{Kind: Deny, Credential: subject},+ ); err != nil {+ t.Fatalf("arm the deny: %v", err)+ }++ if status := through(+ t,+ dependency,+ subject,+ ); status != http.StatusForbidden {+ t.Fatalf("the denied credential was answered %d, want %d",+ status, http.StatusForbidden)+ }++ if err := dependency.Arm(Trap{Kind: Outage}); err != nil {+ t.Fatalf("arm the outage: %v", err)+ }++ if status := through(+ t,+ dependency,+ subject,+ ); status != http.StatusServiceUnavailable {+ t.Fatalf("under the outage the dependency answered %d, want %d",+ status, http.StatusServiceUnavailable)+ }+}++// The property the confirmation's control side exists for. A dependency that+// answered 403 to everything would confirm every deny khaos ever armed, and+// the target would be refusing for a reason khaos invented.+func TestADenyIsScopedToOneCredential(t *testing.T) {+ dependency := serving(t)++ if err := dependency.Arm(+ Trap{Kind: Deny, Credential: subject},+ ); err != nil {+ t.Fatalf("arm: %v", err)+ }++ if status := through(t, dependency, other); status != http.StatusOK {+ t.Fatalf(+ "denying one credential answered %d for another, want %d: the experiment reads a target refusing every OTHER credential as a defect, so khaos must not be the cause of it",+ status,+ http.StatusOK,+ )+ }+}++// THE negative control. An inert arm prepares everything and tells the+// dependency nothing, so a confirmation that reads khaos's own armed flag+// instead of the dependency's answer says yes here and is worthless.+func TestAnInertArmIsRefusedByTheConfirmation(t *testing.T) {+ dependency := serving(t)++ if err := dependency.Arm(+ Trap{Kind: Deny, Credential: subject, Inert: true},+ ); err != nil {+ t.Fatalf("arm: %v", err)+ }++ detail, confirmed, err := dependency.Observe(context.Background())+ if err != nil {+ t.Fatalf("observe: %v", err)+ }++ if confirmed {+ t.Fatalf(+ "the confirmation accepted an inert arm, with %q: it is reading khaos's own state rather than what the dependency answers",+ detail,+ )+ }++ if status := through(t, dependency, subject); status != http.StatusOK {+ t.Fatalf("an inert arm changed the answer to %d", status)+ }+}++// And the same control for the second phase: the trap is still Outage after a+// disarm, so a confirmation keyed on the trap rather than the answer passes.+func TestAnOutageConfirmationNeedsThe503(t *testing.T) {+ dependency := serving(t)++ if err := dependency.Arm(Trap{Kind: Outage}); err != nil {+ t.Fatalf("arm: %v", err)+ }++ if _, confirmed, err := dependency.Observe(+ context.Background(),+ ); err != nil || !confirmed {+ t.Fatalf("an armed outage did not confirm: %v, %v", confirmed, err)+ }++ dependency.Disarm()++ if _, confirmed, err := dependency.Observe(+ context.Background(),+ ); err != nil || confirmed {+ t.Fatalf(+ "the outage confirmed after a disarm (%v, %v), so it is reading the trap rather than the answer",+ confirmed,+ err,+ )+ }+}++func TestTheConfirmationReportsWhatItRead(t *testing.T) {+ dependency := serving(t)++ if err := dependency.Arm(+ Trap{Kind: Deny, Credential: subject},+ ); err != nil {+ t.Fatalf("arm: %v", err)+ }++ detail, confirmed, err := dependency.Observe(context.Background())+ if err != nil || !confirmed {+ t.Fatalf("an armed deny did not confirm: %v, %v", confirmed, err)+ }++ if detail == "" {+ t.Fatal(+ "the confirmation reported no observable; a constant describing the technique is not evidence",+ )+ }+}++// The control side of the two-sided confirmation, driven directly, because the+// input it exists to reject cannot be produced through the dependency's own+// API: a dependency that refused EVERYTHING would confirm the deny khaos armed+// while also refusing the valid credential, and this experiment reports that+// refusal as a defect of the target. The row that matters is the third.+func TestTheConfirmationNeedsBothSides(t *testing.T) {+ for _, row := range []struct {+ name string+ kind Kind+ subject int+ control int+ want bool+ }{+ {+ name: "the credential is refused and the control is allowed",+ kind: Deny,+ subject: http.StatusForbidden,+ control: http.StatusOK,+ want: true,+ },+ {+ name: "the credential was never refused",+ kind: Deny,+ subject: http.StatusOK,+ control: http.StatusOK,+ want: false,+ },+ {+ name: "the dependency refuses everything",+ kind: Deny,+ subject: http.StatusForbidden,+ control: http.StatusForbidden,+ want: false,+ },+ {+ name: "the outage answers everything",+ kind: Outage,+ subject: http.StatusServiceUnavailable,+ control: http.StatusServiceUnavailable,+ want: true,+ },+ {+ name: "the outage was never armed",+ kind: Outage,+ subject: http.StatusOK,+ control: http.StatusOK,+ want: false,+ },+ } {+ t.Run(row.name, func(t *testing.T) {+ detail, confirmed, err := confirms(+ row.kind,+ row.subject,+ row.control,+ )+ if err != nil {+ t.Fatalf("confirms: %v", err)+ }++ if confirmed != row.want {+ t.Fatalf(+ "subject %d and control %d confirmed %v, want %v",+ row.subject,+ row.control,+ confirmed,+ row.want,+ )+ }++ if confirmed && detail == "" {+ t.Fatal("confirmed with no observable to report")+ }+ })+ }+}++func TestDisarmRestoresTheAllowEverythingPosture(t *testing.T) {+ dependency := serving(t)++ if err := dependency.Arm(+ Trap{Kind: Deny, Credential: subject},+ ); err != nil {+ t.Fatalf("arm: %v", err)+ }++ dependency.Disarm()++ if status := through(t, dependency, subject); status != http.StatusOK {+ t.Fatalf(+ "after the revert the dependency still answered %d for the credential it had denied, want %d: the recovery window measures the target serving again",+ status,+ http.StatusOK,+ )+ }+}++// Asked is a liveness guard three separate verdict branches read, and Refused+// is the retry-storm instrument. Only a definitive refusal increments the+// second, or a caller asking once per request would read as one asking twice.+func TestTheCountersSeparateDecisionsFromRefusals(t *testing.T) {+ dependency := serving(t)++ through(t, dependency, subject)+ through(t, dependency, other)++ asked, err := dependency.Asked()+ if err != nil {+ t.Fatalf("asked: %v", err)+ }++ if asked != 2 {+ t.Fatalf("the dependency counted %d decisions, want 2", asked)+ }++ if refused := dependency.Refused(); refused != 0 {+ t.Fatalf(+ "the dependency counted %d refusals before any fault, want 0",+ refused,+ )+ }++ if err := dependency.Arm(+ Trap{Kind: Deny, Credential: subject},+ ); err != nil {+ t.Fatalf("arm: %v", err)+ }++ through(t, dependency, subject)+ through(t, dependency, subject)+ through(t, dependency, other)++ if refused := dependency.Refused(); refused != 2 {+ t.Fatalf(+ "the dependency counted %d refusals, want 2: a credential it allowed must not count, or khaos would report a retry storm on a caller that asked once",+ refused,+ )+ }+}++func TestATrapWithoutACredentialIsRefusedByName(t *testing.T) {+ dependency := serving(t)++ err := dependency.Arm(Trap{Kind: Deny})+ if err == nil {+ t.Fatal(+ "a deny with no credential was armed; it would deny nothing and the confirmation would have nothing to read",+ )+ }++ if status := through(t, dependency, ""); status != http.StatusOK {+ t.Fatalf("the refused arm still changed the answer to %d", status)+ }+}++func TestAnUnknownTrapIsRefusedByName(t *testing.T) {+ dependency := serving(t)++ if err := dependency.Arm(Trap{Kind: "not-a-real-fault"}); err == nil {+ t.Fatal("an invented authorization fault was armed")+ }+}++// Injectable is deliberately smaller than Kinds, and the gap is the whole+// design: every other tier can make a dependency unreachable, so an outage+// exists here only as the second phase of a deny.+func TestInjectableIsASubsetOfKindsAndExcludesTheOutage(t *testing.T) {+ for _, kind := range Injectable() {+ found := false+ for _, known := range Kinds() {+ if kind == known {+ found = true+ }+ }++ if !found {+ t.Fatalf(+ "%s is offered for injection and is not a known kind",+ kind,+ )+ }++ if kind == string(Outage) {+ t.Fatal(+ "the outage is offered as a fault of its own; it would duplicate every network fault khaos already has, and its value is the comparison with the deny",+ )+ }+ }+}++// This dependency ALLOWS every credential until it is told otherwise, so a+// wildcard bind would put an allow-everything authorization service on every+// interface the host has. The refusal is shared with the proxy tier because+// both listeners are things an operator would be horrified to find exposed.+func TestAnExposedBindIsRefused(t *testing.T) {+ for _, addr := range []string{+ "0.0.0.0:45999",+ ":45999",+ "[::]:45999",+ } {+ dependency := New()++ err := dependency.Listen(addr)+ if err == nil {+ bound := dependency.Addr()+ _ = dependency.Close()+ t.Fatalf(+ "the dependency bound %s, which is reachable from outside this host",+ bound,+ )+ }++ if !errors.Is(err, bindguard.ErrUnsafeBind) {+ t.Fatalf("%s: err = %v, want ErrUnsafeBind", addr, err)+ }+ }++ // And loopback MUST still be accepted, or the refusal is simply a broken+ // listener rather than a guard.+ dependency := New()+ if err := dependency.Listen(loop); err != nil {+ t.Fatalf("loopback was refused: %v", err)+ }+ _ = dependency.Close()+}++func TestEveryKindDescribesItself(t *testing.T) {+ for _, name := range Kinds() {+ trap := Trap{Kind: Kind(name), Credential: subject}++ if trap.Describe() == "" {+ t.Fatalf("%s describes itself as nothing", name)+ }+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/trap.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/trap.gonew file mode 100644index 00000000..d1f0b5c4--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/trap.go@@ -0,0 +1,96 @@+// ©AngelaMos | 2026+// trap.go++package authz++import (+ "errors"+ "fmt"+ "slices"+ "strings"+)++type Kind string++const (+ // Deny is a DEFINITIVE answer about one credential, and the whole point of+ // the tier: the dependency said no, loudly and correctly, with full+ // reachability and a 403. No network fault can produce it, because every+ // network fault only ever makes a dependency unreachable.+ Deny Kind = "deny"++ // Outage is the discriminator phase. It is armed second, on the same run,+ // and research 07 section 6.2 is why: a correct caller behaves DIFFERENTLY+ // under a deny than under an absence of an answer, and a caller that+ // behaves identically has collapsed the two into one branch.+ Outage Kind = "outage"+)++const (+ errUnknownTrap = "%w: %q (known authorization faults: %s)"++ errNoCredential = "%w: a deny needs the credential it is a verdict about"++ describeDeny = "have khaos's own authorization dependency answer a definitive 403 for one credential the target is honouring, while it keeps answering 200 for every other credential; khaos stands in for that dependency for the length of the run, so it allows everything it has not been told to deny"++ describeOutage = "then stop answering at all, with a 503, which is the only answer a fallback policy may act on"+)++var ErrUnknownTrap = errors.New(+ "no authorization fault is registered under that name",+)++var ErrNoCredential = errors.New(+ "this authorization fault has no credential to be a verdict about",+)++// Trap is one authorization fault. The credential is a value khaos was given+// rather than one it minted, because the dependency is a stand-in for the+// operator's own and the credential belongs to their target.+type Trap struct {+ Kind Kind `json:"kind"`+ Credential string `json:"-"`++ // Inert is the negative control. Arm prepares everything and never tells+ // the dependency, so a confirmation that reads the dependency back through+ // its own surface MUST say no.+ Inert bool `json:"inert,omitempty"`+}++func Kinds() []string {+ return []string{string(Deny), string(Outage)}+}++// Injectable is the subset khaos registers as a fault an operator can ask for.+// An outage is NOT one of them: every other tier already makes a dependency+// unreachable, and this one exists for the answer that no network fault can+// produce. It is reachable only as the second phase of a deny, where its whole+// value is the comparison with the first.+func Injectable() []string {+ return []string{string(Deny)}+}++func (t Trap) Validate() error {+ if !slices.Contains(Kinds(), string(t.Kind)) {+ return fmt.Errorf(+ errUnknownTrap,+ ErrUnknownTrap,+ t.Kind,+ strings.Join(Kinds(), ", "),+ )+ }++ if t.Kind == Deny && t.Credential == "" {+ return fmt.Errorf(errNoCredential, ErrNoCredential)+ }++ return nil+}++func (t Trap) Describe() string {+ if t.Kind == Outage {+ return describeOutage+ }++ return describeDeny+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.gonew file mode 100644index 00000000..1d3be2fc--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go@@ -0,0 +1,116 @@+// ©AngelaMos | 2026+// bindguard.go++package bindguard++import (+ "errors"+ "fmt"+ "net"+ "os"+ "path/filepath"+ "strconv"+)++const (+ lowestUnprivilegedPort = 1024+ ephemeralPort = 0++ // Exported because the tiers' own tests derive a bridge and a non-bridge+ // address from sysfs INDEPENDENTLY, rather than asking this package which+ // is which: a test that picked its input with the predicate under test+ // would agree with that predicate's mistake. These two are facts about+ // Linux, not logic, so sharing them costs nothing.+ SysClassNet = "/sys/class/net"+ BridgeMarker = "bridge"++ errWildcard = "%w: %q binds every address on this host, and %s must not be reachable off it; bind the Docker bridge gateway instead"++ errPrivileged = "%w: %q is a privileged port"++ errBadAddr = "%w: %q is not a host and port (%v)"++ errOffBox = "%w: %q is not on loopback or on a Docker bridge, so it is reachable from outside this host and a container cannot reach it anyway; bind the bridge gateway the daemon reports"+)++// ErrUnsafeBind is one sentinel for every listener khaos opens. Each tier+// wraps it with its own subject, so a refusal names what was about to be+// exposed rather than the rule that stopped it.+var ErrUnsafeBind = errors.New("khaos will not bind there")++// Check refuses an address khaos must not serve a deliberately broken+// dependency on. It is shared rather than restated because every one of these+// listeners is a thing an operator would be horrified to find on their+// network: a fault proxy in front of a dependency under test, and an+// authorization service that allows every credential it was not told to deny.+//+// subject is what the refusal names, in words, from the tier's own vocabulary.+//+// The test is the ROUTING property rather than the address: loopback, or a+// bridge, identified by its sysfs `bridge` directory rather than by its name,+// because a target in a container has to be able to reach it and nothing else+// should.+func Check(listenAddr, subject string) error {+ host, portText, err := net.SplitHostPort(listenAddr)+ if err != nil {+ return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)+ }++ ip := net.ParseIP(host)+ if ip == nil || ip.IsUnspecified() {+ return fmt.Errorf(errWildcard, ErrUnsafeBind, listenAddr, subject)+ }++ port, err := strconv.Atoi(portText)+ if err != nil {+ return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)+ }++ if port != ephemeralPort && port < lowestUnprivilegedPort {+ return fmt.Errorf(errPrivileged, ErrUnsafeBind, listenAddr)+ }++ if ip.IsLoopback() {+ return nil+ }++ onBridge, err := bridgeHosts(ip)+ if err != nil {+ return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)+ }++ if !onBridge {+ return fmt.Errorf(errOffBox, ErrUnsafeBind, listenAddr)+ }++ return nil+}++func bridgeHosts(ip net.IP) (bool, error) {+ interfaces, err := net.Interfaces()+ if err != nil {+ return false, err+ }++ for _, iface := range interfaces {+ addrs, err := iface.Addrs()+ if err != nil {+ continue+ }++ for _, addr := range addrs {+ network, ok := addr.(*net.IPNet)+ if !ok || !network.IP.Equal(ip) {+ continue+ }++ _, err := os.Stat(+ filepath.Join(SysClassNet, iface.Name, BridgeMarker),+ )++ return err == nil, nil+ }+ }++ return false, nil+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.gonew file mode 100644index 00000000..315b6f5f--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go@@ -0,0 +1,83 @@+// ©AngelaMos | 2026+// bindguard_test.go++package bindguard++import (+ "errors"+ "strings"+ "testing"+)++const subject = "a listener under test"++// The syntactic branches, which are the ones no tier's own test reaches: the+// proxy's cases cover the wildcards and this host's routable address, and the+// authorization tier's cover the wildcards again, so a privileged port was+// refused by code nothing exercised. Loopback is in the same table because a+// guard that refuses everything is not a guard.+func TestTheGuardRefusesWhatItMustAndAcceptsLoopback(t *testing.T) {+ for _, row := range []struct {+ name string+ addr string+ want bool+ }{+ {name: "an ipv4 wildcard", addr: "0.0.0.0:45999", want: false},+ {name: "a bare port", addr: ":45999", want: false},+ {name: "an ipv6 wildcard", addr: "[::]:45999", want: false},+ {name: "a privileged port", addr: "127.0.0.1:80", want: false},+ {name: "the last privileged port", addr: "127.0.0.1:1023", want: false},+ {name: "not a host and port", addr: "not-an-address", want: false},+ {name: "a hostname rather than an ip", addr: "localhost:45999", want: false},+ {name: "no port at all", addr: "127.0.0.1", want: false},+ {name: "loopback on a high port", addr: "127.0.0.1:45999", want: true},+ {name: "loopback on an ephemeral port", addr: "127.0.0.1:0", want: true},+ {+ name: "the first unprivileged port",+ addr: "127.0.0.1:1024",+ want: true,+ },+ {name: "ipv6 loopback", addr: "[::1]:45999", want: true},+ } {+ t.Run(row.name, func(t *testing.T) {+ err := Check(row.addr, subject)++ if row.want && err != nil {+ t.Fatalf(+ "%s was refused (%v); a guard that refuses an address khaos has to use stops the tier from running at all",+ row.addr,+ err,+ )+ }++ if row.want {+ return+ }++ if err == nil {+ t.Fatalf("%s was accepted", row.addr)+ }++ if !errors.Is(err, ErrUnsafeBind) {+ t.Fatalf("%s: err = %v, want ErrUnsafeBind", row.addr, err)+ }+ })+ }+}++// The refusal has to name what was about to be exposed, in the tier's own+// words, because one sentinel now serves several listeners and "khaos will not+// bind there" alone does not tell an operator which one.+func TestAWildcardRefusalNamesTheSubject(t *testing.T) {+ err := Check("0.0.0.0:45999", subject)+ if err == nil {+ t.Fatal("a wildcard was accepted")+ }++ if !strings.Contains(err.Error(), subject) {+ t.Fatalf(+ "the refusal does not say what it was about to expose: %v",+ err,+ )+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.goindex 3aceff55..e3d89238 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go@@ -16,6 +16,7 @@ import ("strings""time"+ "github.com/CarterPerez-dev/khaos/internal/bindguard""github.com/CarterPerez-dev/khaos/internal/dbx""github.com/CarterPerez-dev/khaos/internal/dnsrv""github.com/CarterPerez-dev/khaos/internal/dockerx"@@ -123,6 +124,10 @@ enforcing in the first place.`errNoPooledPath = "%w: this experiment rotates the credential a service's own connection pool authenticated with, and the running range has no %s; bring the range down and up again with `khaos range up --with-db`"+ errAuthzBind = "khaos could not bind the authorization dependency it plays on %s, which is where the target's own authorization check is already looking: %w. Something else took that port since the range came up; bring the range down and up again to pick a new one"++ msgAuthzUp = "khaos is answering authorization decisions on %s, allowing every credential until it is told otherwise"+errNoSubjectSecret = "%s does not carry %s in its environment, so khaos cannot read the credential its own connection pool authenticated with, and a rotation it cannot undo is not one khaos will perform"errTargetOffNetwork = "%s is not attached to %s, so it cannot be using the chaos resolver that serves it and khaos could not count its queries: %w"@@ -521,12 +526,17 @@ func runExperiment(dockerx.Filter{Project: project},guard,)++ // BEFORE the error check, because standUpTiers hands the closer back WITH+ // its error: a tier that failed is one position past tiers that already+ // bound a listener, and khaos's claim that nothing is left listening has to+ // hold by construction rather than because the process happens to exit.+ defer closeTiers()+if err != nil {return err}- defer closeTiers()-session := run.NewSession(env, work.Journal, ttl)engine := wire(@@ -766,6 +776,7 @@ func rangeSupplies() []string {experiment.ParamReadyURL,experiment.ParamSecretPath,experiment.ParamEnvVar,+ experiment.ParamAuthzListen,}}@@ -783,12 +794,15 @@ const (tierPooltierSecrettierEnv+ tierAuthz)func tierOf(kind string) tier {switch {case inject.NeedsProxy(kind):return tierProxy+ case inject.NeedsAuthz(kind):+ return tierAuthzcase inject.NeedsIdP(kind):return tierIdPcase inject.NeedsPKI(kind):@@ -823,6 +837,11 @@ type rangeAnswers struct {AuthorityBase stringCertHost string++ // Where khaos binds the authorization dependency it plays. Same shape as+ // AuthorityListen and for the same reason: the fixture's own call is+ // already addressed at a fixed port on the bridge gateway.+ AuthzListen string}// tierAnswers is what one tier's own gateway and fixtures answer. Split out of@@ -904,6 +923,13 @@ func answersFor(lab rangelab.Lab, at tier) tierAnswers {answers.target = rangelab.EnvGatewayNameanswers.variable = rangelab.SecretVariable+ // The CALLER is the target. khaos replaces the dependency it asks, so the+ // fixture is the only container whose consent is needed, and the+ // dependency is not a container at all.+ case tierAuthz:+ answers.url = lab.AuthzProtectedURL()+ answers.target = rangelab.AuthzGatewayName+default:// The direct tier's URL and target are the values set above.}@@ -932,6 +958,7 @@ func rangeDefaults(experiment.ParamAuthorityTLS: answers.AuthorityTLS,experiment.ParamAuthorityBase: answers.AuthorityBase,experiment.ParamCertHost: answers.CertHost,+ experiment.ParamAuthzListen: answers.AuthzListen,experiment.ParamDNSHost: rangelab.DNSDependencyName,experiment.ParamResolverNetwork: rangelab.NetworkName,@@ -1087,6 +1114,9 @@ func requireTier(lab rangelab.Lab, at tier) error {case tierEnv:return lab.RequireEnvTier()+ case tierAuthz:+ return lab.RequireAuthzTier()+default:// The direct tier needs only the gateway Discover already found.}@@ -1246,6 +1276,9 @@ func standUpTiers(func() (func(), error) {return standUpPKITier(out, spec, &env)},+ func() (func(), error) {+ return standUpAuthzTier(out, spec, &env)+ },func() (func(), error) {return standUpDNSTier(ctx, out, work.Docker, spec, target, &env)},@@ -1345,6 +1378,38 @@ func standUpPKITier(return func() { _ = spec.Authority.Close() }, nil}+// standUpAuthzTier binds khaos's authorization dependency BEFORE the baseline,+// which is not optional: the target's authorization call has to be answered+// through the whole run, or the baseline measures a dependency that is down and+// the fault window is the only one where it is up.+func standUpAuthzTier(+ out io.Writer,+ spec *experiment.Experiment,+ env *inject.Env,+) (func(), error) {+ if spec.AuthzTrap == nil {+ return nil, nil+ }++ if err := spec.Authz.Listen(spec.AuthzListen); err != nil {+ // The guard refuses BEFORE the bind is attempted, so the port-collision+ // advice below is wrong for the one refusal an operator will actually+ // hit: an address khaos will not serve this from.+ if errors.Is(err, bindguard.ErrUnsafeBind) {+ return nil, err+ }++ return nil, fmt.Errorf(errAuthzBind, spec.AuthzListen, err)+ }++ note(out, fmt.Sprintf(msgAuthzUp, spec.Authz.Addr()))++ env.Authz = spec.Authz+ env.AuthzTrap = *spec.AuthzTrap++ return func() { _ = spec.Authz.Close() }, nil+}+// standUpDNSTier ATTACHES rather than starts: `--dns` is fixed at container// creation, so a resolver started now would be one nothing is pointed at.func standUpDNSTier(@@ -1740,10 +1805,8 @@ func credentialParam(name string) bool {return name == experiment.ParamToken || name == experiment.ParamRevocable}-// askForCredentials asks whichever fixture will be the one VERIFYING. For the-// secret family that is the fixture itself, because khaos never reads the-// secret and a token signed with any other key would be refused for a reason-// that has nothing to do with the fault.+// askForCredentials mints whichever credentials this experiment declared, from+// the one fixture that will be asked to verify them.func askForCredentials(ctx context.Context,lab rangelab.Lab,@@ -1751,27 +1814,12 @@ func askForCredentials(at tier,answers *rangeAnswers,) error {- if want(experiment.ParamToken) {- issue := lab.IssueToken-- switch at {- case tierSecret:- if err := lab.RequireSecretTier(); err != nil {- return err- }- issue = lab.IssueSecretToken-- case tierEnv:- if err := lab.RequireEnvTier(); err != nil {- return err- }- issue = lab.IssueEnvToken-- default:- // Every other tier's credential comes from the range's own- // identity provider through the direct gateway.- }+ issue, err := issuerFor(lab, at)+ if err != nil {+ return err+ }+ if want(experiment.ParamToken) {token, err := issue(ctx)if err != nil {return err@@ -1782,8 +1830,11 @@ func askForCredentials(// A SECOND credential: revoking the one khaos keeps presenting as valid// would also take away the stream proving the target is serving at all.+ // It comes from the same issuer, because a fixture that authenticates+ // locally refuses anything it did not hand out and khaos would be+ // measuring that refusal instead of the fault.if want(experiment.ParamRevocable) {- revocable, err := lab.IssueToken(ctx)+ revocable, err := issue(ctx)if err != nil {return err}@@ -1794,6 +1845,43 @@ func askForCredentials(return nil}+// issuerFor is whichever fixture will be the one VERIFYING. For the secret+// family that is the fixture itself, because khaos never reads the secret and a+// token signed with any other key would be refused for a reason that has+// nothing to do with the fault.+func issuerFor(+ lab rangelab.Lab,+ at tier,+) (func(context.Context) (string, error), error) {+ switch at {+ case tierSecret:+ if err := lab.RequireSecretTier(); err != nil {+ return nil, err+ }++ return lab.IssueSecretToken, nil++ case tierEnv:+ if err := lab.RequireEnvTier(); err != nil {+ return nil, err+ }++ return lab.IssueEnvToken, nil++ case tierAuthz:+ if err := lab.RequireAuthzTier(); err != nil {+ return nil, err+ }++ return lab.IssueAuthzToken, nil++ default:+ // Every other tier's credential comes from the range's own identity+ // provider through the direct gateway.+ return lab.IssueToken, nil+ }+}+func askTheRange(ctx context.Context,reader rangelab.PortReader,@@ -1830,6 +1918,14 @@ func askTheRange(answers.Authority = authority}+ if want(experiment.ParamAuthzListen) {+ listen, err := lab.AuthzListenAddr(ctx, reader)+ if err != nil {+ return rangeAnswers{}, err+ }+ answers.AuthzListen = listen+ }+if want(experiment.ParamAuthorityListen) ||want(experiment.ParamAuthorityTLS) ||want(experiment.ParamAuthorityBase) ||diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.goindex c9a74630..39250fe4 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go@@ -37,6 +37,7 @@ var testAnswers = rangeAnswers{AuthorityTLS: "10.0.0.1:30007",AuthorityBase: "http://host.docker.internal:30006",CertHost: "dependency.khaos.invalid",+ AuthzListen: "10.0.0.1:30013",}// refuser fails the test if khaos goes looking for a range it was never meant to@@ -179,6 +180,7 @@ func allTiers() []tier {tierPool,tierSecret,tierEnv,+ tierAuthz,}}@@ -200,6 +202,8 @@ func tierName(at tier) string {return "at the file the service verifies with"case tierEnv:return "at the variable the service verifies with"+ case tierAuthz:+ return "at the service whose authorization dependency khaos answers"default:return "direct"}@@ -229,6 +233,12 @@ func fullLab() rangelab.Lab {DBGatewayPort: 30010,SecretPort: 30011,EnvPort: 30012,++ // The caller whose authorization dependency khaos plays. Its base URL+ // is what khaos reads the port to bind out of, so it is NOT the port+ // the caller itself publishes.+ AuthzPort: 30014,+ AuthzBase: "http://host.docker.internal:30013",}}@@ -797,6 +807,7 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(tierPool: lab.DBProtectedURL(),tierSecret: lab.SecretProtectedURL(),tierEnv: lab.EnvProtectedURL(),+ tierAuthz: lab.AuthzProtectedURL(),}seen := map[string]tier{}@@ -866,6 +877,12 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(case tierEnv:wantTarget = rangelab.EnvGatewayName+ // The CALLER, which is the only container in this tier: khaos+ // replaces the dependency it asks, and that dependency is a+ // listener in khaos's own process rather than a container at all.+ case tierAuthz:+ wantTarget = rangelab.AuthzGatewayName+default:// The direct and proxy tiers fault the dependency itself.}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.goindex 057a3e0e..577aa492 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go@@ -30,6 +30,10 @@ var ErrUnknownPoolMode = errors.New("that is not a pool configuration the range fixture understands",)+var ErrUnknownAuthzMode = errors.New(+ "that is not an authorization-error handling the range fixture understands",+)+var ErrUnknownFailMode = errors.New("the range has no such fail mode")const (@@ -41,6 +45,7 @@ const (flagWithDB = "with-db"flagPoolMode = "pool-mode"flagSecretMode = "secret-mode"+ flagAuthzMode = "authz-mode"mintedSecrets = 4@@ -69,6 +74,8 @@ running.`quietUsage = "suppress compose build and start output"secretModeUsage = "how the range's secret fixture handles the secret it reads. validated checks the length after every read, which is the correct handling and the fix for CVE-2019-20933; unchecked lets an empty value flow into the crypto, which is that CVE's own shape. An unrecognised value is refused rather than quietly leaving the fixture correct"+ authzModeUsage = "how the range's service-to-service caller reads its authorization dependency's answers. distinguishes reads allow, deny and unknown as three outcomes, which is correct; collapses reads anything that is not a 200 as an absence of an answer and allows it, which is research 07's F20 and the single most common fail-open in service-to-service code; retries reaches the right verdict the wrong way, asking again every time it does not like an answer, which hammers the dependency with a credential it has already been told is dead. An unrecognised value is refused rather than quietly leaving the caller correct"+poolModeUsage = "how the stateful tier's pooled fixture configures its connection pool. unbounded is the DEFAULT configuration of a Go service rather than a broken one, because database/sql closes a connection for age only when SetConnMaxLifetime is positive; bounded sets a lifetime, which is the knob that turns a rotation into a revocation; held checks one connection out and never returns it, which no lifetime bounds at all"withDBUsage = "also start the stateful tier: PostgreSQL 18, MySQL 8.4, MariaDB 11.4 and a Redis for the rate limiter's datastore. These four are the only range services khaos does not implement itself, because the credential family's finding IS what three correct engines do, and they cost real startup time, so they are opt-in"@@ -84,6 +91,7 @@ running.`msgTokenHint = "a credential is issued on demand, so `khaos run` against the range needs no flags"errUnknownPoolMode = "%w: %q (known pool modes: %s)"errUnknownSecretMode = "%w: %q (known secret modes: %s)"+ errUnknownAuthzMode = "%w: %q (known authorization modes: %s)"msgStatefulUp = "the stateful tier is up: PostgreSQL, MySQL, MariaDB and Redis, none of them published to a host port"msgPortHint = "host port %d was chosen at random; pass --port to pin it"@@ -94,6 +102,7 @@ running.`msgTierVerifier = "identity tier %s, trusting khaos's provider as %s"msgTierTLS = "certificate tier %s, dialling khaos's authority at %s as the %s client"msgTierDNS = "dns tier %s, resolving %s through khaos's chaos resolver at %s"+ msgTierAuthz = "authorization tier %s, asking khaos for a decision at %s")func newRangeCmd() *cobra.Command {@@ -150,6 +159,10 @@ func newRangeUpCmd() *cobra.Command {if err != nil {return err}+ authzMode, err := cmd.Flags().GetString(flagAuthzMode)+ if err != nil {+ return err+ }return runRangeUp(cmd.Context(),cmd.OutOrStdout(),@@ -159,8 +172,11 @@ func newRangeUpCmd() *cobra.Command {port,quiet,withDB,- poolMode,- secretMode,+ rangeModes{+ pool: poolMode,+ secret: secretMode,+ authz: authzMode,+ },)},}@@ -177,10 +193,24 @@ func newRangeUpCmd() *cobra.Command {rangelab.SecretUnchecked,secretModeUsage,)+ cmd.Flags().String(+ flagAuthzMode,+ rangelab.AuthzCollapses,+ authzModeUsage,+ )return cmd}+// rangeModes is the per-fixture configuration one `range up` carries. Grouped+// because four of them are now passed through two functions, and a fifth+// positional string is how two of them end up swapped.+type rangeModes struct {+ pool string+ secret string+ authz string+}+// mintRangeSecrets gives the range a fresh nonce and a fresh database password// per `range up`. One that outlived the range would let anything that had seen// it arm the resolver, and a credential written into a public compose file is a@@ -230,6 +260,17 @@ func knownPoolModes() []string {}}+// knownAuthzModes is validated against the whole set for the same reason as+// the other three: `collapses` is the DEFAULT here, so a typo would hand the+// experiment a correct caller and report the defect as absent.+func knownAuthzModes() []string {+ return []string{+ rangelab.AuthzDistinguishes,+ rangelab.AuthzCollapses,+ rangelab.AuthzRetries,+ }+}+func newRangeDownCmd() *cobra.Command {return &cobra.Command{Use: "down",@@ -295,20 +336,25 @@ func knownFailModes() []string {// range behaving CORRECTLY, which makes a typo the worst possible outcome: the// stack comes up, nothing is broken, and every experiment reports a clean// survival against a fixture that was supposed to carry the bug.-func validateModes(failMode, poolMode, secretMode string) error {+func validateModes(failMode string, modes rangeModes) error {if !slices.Contains(knownFailModes(), failMode) {return fmt.Errorf(errUnknownFailMode, ErrUnknownFailMode, failMode,strings.Join(knownFailModes(), listSep))}- if !slices.Contains(knownPoolModes(), poolMode) {- return fmt.Errorf(errUnknownPoolMode, ErrUnknownPoolMode, poolMode,+ if !slices.Contains(knownPoolModes(), modes.pool) {+ return fmt.Errorf(errUnknownPoolMode, ErrUnknownPoolMode, modes.pool,strings.Join(knownPoolModes(), listSep))}- if !slices.Contains(knownSecretModes(), secretMode) {+ if !slices.Contains(knownSecretModes(), modes.secret) {return fmt.Errorf(errUnknownSecretMode, ErrUnknownSecretMode,- secretMode, strings.Join(knownSecretModes(), listSep))+ modes.secret, strings.Join(knownSecretModes(), listSep))+ }++ if !slices.Contains(knownAuthzModes(), modes.authz) {+ return fmt.Errorf(errUnknownAuthzMode, ErrUnknownAuthzMode,+ modes.authz, strings.Join(knownAuthzModes(), listSep))}return nil@@ -320,9 +366,9 @@ func runRangeUp(failMode, cacheTTL, tlsMode string,port int,quiet, withDB bool,- poolMode, secretMode string,+ modes rangeModes,) error {- if err := validateModes(failMode, poolMode, secretMode); err != nil {+ if err := validateModes(failMode, modes); err != nil {return err}@@ -375,8 +421,9 @@ func runRangeUp(failMode,cacheTTL,tlsMode,- poolMode,- secretMode,+ modes.pool,+ modes.secret,+ modes.authz,withDB,)if err == nil {@@ -509,5 +556,16 @@ func describeEntryPoints()}+ if lab.AuthzPort != 0 {+ say(+ out,+ fmt.Sprintf(+ msgTierAuthz,+ lab.AuthzProtectedURL(),+ lab.AuthzBase,+ ),+ )+ }+fmt.Fprintln(out)}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/apikey.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/apikey.gonew file mode 100644index 00000000..13973963--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/apikey.go@@ -0,0 +1,202 @@+// ©AngelaMos | 2026+// apikey.go++package experiment++import (+ "context"+ "errors"+ "fmt"++ "github.com/CarterPerez-dev/khaos/internal/authz"+ "github.com/CarterPerez-dev/khaos/internal/inject"+ "github.com/CarterPerez-dev/khaos/internal/probe"+)++const (+ ParamAuthzListen = "authz-listen"++ NameAPIKeyRevoked = "credential.apikey_revoked"++ usageAuthzListen = "where khaos binds the authorization dependency it plays, which has to be the address the target's own authorization check already calls; khaos answers that call for the length of the run"++ usageAuthzToken = "a credential the target accepts, which khaos's authorization dependency keeps allowing; it is the stream that proves the target is still serving"++ usageAuthzRevocable = "a SECOND credential the target accepts, which khaos's authorization dependency answers a definitive 403 for once the baseline is established. It must NOT be one your live traffic is using: khaos denies it for the length of the run, so anything else presenting it is denied too, and the retry count khaos reports is the number of times THIS credential was asked about, which other clients presenting it would inflate"++ summaryAPIKeyRevoked = "have a service's authorization dependency answer a definitive deny, then answer nothing at all, and see whether the service can tell those apart"++ conditionAPIKeyRevoked = "%s's authorization dependency refusing one credential outright, and then becoming unable to answer at all"++ responseAPIKeyRevoked = "refusing the credential its dependency denied, continuing to serve every other credential, and refusing again once the dependency can no longer answer -- three outcomes, not two"++ // It names khaos's own posture as well as the fault, because this string is+ // what a LEAKING run prints and the HeldCaveat below is not: an operator+ // acting on a finding has to know khaos was answering their authorization+ // calls and allowing everything it was not told to deny.+ defectAPIKeyRevoked = "answering a definitive 403 for that credential at the authorization dependency khaos is playing, which stayed reachable throughout and kept allowing every other credential"++ consultsAuthz = "khaos's authorization dependency"++ deliversAuthz = "ask khaos's authorization dependency for a decision"++ hintAuthz = "A target that caches authorization decisions produces this shape: it asked once before the fault and answered every probe from that cached allow. What to check, in that order: the lifetime of the target's own decision cache, whether it caches a negative answer at all, and whether its authorization check runs per request or per session. Re-run with --fault-for longer than that lifetime"++ shiftAuthzWhat = "khaos's authorization dependency answered 503 to everything"++ shiftAuthzDetail = "making khaos's authorization dependency unable to answer at all, which is the only outcome a fallback policy may act on"++ caveatAuthzHeld = "khaos stood in for the authorization dependency for the length of this run and allowed every credential it was not told to deny, so this says nothing about the decisions that dependency normally makes; what was measured is how the target reads the three answers it can give"++ errShiftUnconfirmed = "%w: khaos armed the second phase and its own dependency did not answer 503 when read back, so the window would have measured the first fault twice"++ probeDeniedName = "denied-credential"++ describeAuthzValid = "%s with a credential khaos's authorization dependency keeps allowing"++ describeAuthzDenied = "%s with the credential khaos's authorization dependency denies"+)++var ErrShiftUnconfirmed = errors.New(+ "the second phase of this fault could not be confirmed",+)++// authzPolarity has TWO defects, and the second one is not symmetry for its own+// sake. khaos's dependency denies exactly one credential and keeps allowing+// every other, so a target that stops serving the valid one has turned a single+// revocation into an outage of everything, which is a real failure mode of a+// shared authorization client rather than an over-cautious success.+var authzPolarity = Polarity{+ Defects: []Direction{Permitted, Denied},+ Rationale: "an authorization dependency has three outcomes, allow, deny and unknown, and only unknown is eligible for a fallback policy: serving a credential it was told to refuse is a bypass, and refusing every other credential because one was denied is an outage the deny never called for",+}++const usageAuthzTarget = "the consenting container whose authorization dependency khaos answers; nothing of the target's is taken away and no dependency of its is paused, so this is the service being measured rather than the thing being broken"++var authzParams = []Param{+ {Name: ParamTarget, Usage: usageAuthzTarget, Required: true},+ {Name: ParamURL, Usage: usageURL, Required: true},+ {Name: ParamToken, Usage: usageAuthzToken, Required: true},+ {Name: ParamRevocable, Usage: usageAuthzRevocable, Required: true},+ {Name: ParamAuthzListen, Usage: usageAuthzListen, Required: true},+ {+ Name: ParamInvalidToken,+ Usage: usageInvalidToken,+ Default: DefaultInvalidToken,+ },+}++func buildAPIKeyRevoked(params *Params) (Shape, error) {+ target := params.Get(ParamTarget)+ url := params.Get(ParamURL)+ revocable := params.Get(ParamRevocable)++ dependency := authz.New()++ trap := authz.Trap{Kind: authz.Deny, Credential: revocable}+ if err := trap.Validate(); err != nil {+ return Shape{}, err+ }++ return Shape{+ Condition: fmt.Sprintf(conditionAPIKeyRevoked, target),+ Target: target,++ Probes: probe.Pair{+ Valid: probe.NewHTTP(probe.HTTPOptions{+ Name: probeValidName,+ Describe: fmt.Sprintf(describeAuthzValid, url),+ URL: url,+ Token: params.Get(ParamToken),+ Credential: probe.Valid,+ }),+ Invalid: probe.NewHTTP(probe.HTTPOptions{+ Name: probeInvalidName,+ URL: url,+ Token: params.Get(ParamInvalidToken),+ Credential: probe.Invalid,+ Unique: true,+ }),+ Revoked: probe.NewHTTP(probe.HTTPOptions{+ Name: probeDeniedName,+ Describe: fmt.Sprintf(describeAuthzDenied, url),+ URL: url,+ Token: revocable,+ Credential: probe.Revoked,+ }),+ },++ Authz: dependency,+ AuthzTrap: &trap,+ AuthzListen: params.Get(ParamAuthzListen),++ Defect: defectAPIKeyRevoked,+ HeldCaveat: caveatAuthzHeld,++ // A caller that answers 503 to a request whose authorization was+ // refused has not served it, and for this control that IS failing+ // closed. It is a worse answer than a 403 and not a defect.+ ClosedWhen: []probe.Disposition{probe.Unavailable},++ Consulted: dependency.Asked,+ ConsultedWhat: consultsAuthz,+ Refusals: refusalsOf(dependency),+ DeliveredWhat: deliversAuthz,+ DeliveredHint: hintAuthz,++ Shift: shiftToOutage(dependency),+ ShiftDetail: shiftAuthzDetail,+ ShiftWhat: shiftAuthzWhat,+ }, nil+}++// refusalsOf is the retry-storm instrument. Only the credential under test is+// ever answered with a definitive refusal, so this count IS the number of times+// the target asked about a credential it had already been refused for.+func refusalsOf(dependency *authz.Dependency) func() (int64, error) {+ return func() (int64, error) {+ return dependency.Refused(), nil+ }+}++// shiftToOutage is the second phase. It confirms before returning, because an+// unconfirmed shift would leave the deny in place and khaos would measure the+// FIRST fault twice and then compare a window with itself.+func shiftToOutage(+ dependency *authz.Dependency,+) func(context.Context) error {+ return func(ctx context.Context) error {+ if err := dependency.Arm(+ authz.Trap{Kind: authz.Outage},+ ); err != nil {+ return err+ }++ _, confirmed, err := dependency.Observe(ctx)+ if err != nil {+ return err+ }++ if !confirmed {+ return fmt.Errorf(errShiftUnconfirmed, ErrShiftUnconfirmed)+ }++ return nil+ }+}++func init() {+ catalogAdd(Definition{+ Name: NameAPIKeyRevoked,+ Family: FamilyCredential,+ Summary: summaryAPIKeyRevoked,+ Hypothesis: Hypothesis{+ Condition: "a service's authorization dependency refusing one credential outright, and then becoming unable to answer at all",+ Response: responseAPIKeyRevoked,+ },+ Polarity: authzPolarity,+ FaultKind: inject.AuthzKind(authz.Deny),+ Params: authzParams,+ Build: buildAPIKeyRevoked,+ })+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.goindex cdfd7d81..0cf583c1 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go@@ -11,6 +11,7 @@ import ("strings""time"+ "github.com/CarterPerez-dev/khaos/internal/authz""github.com/CarterPerez-dev/khaos/internal/dnsrv""github.com/CarterPerez-dev/khaos/internal/idp""github.com/CarterPerez-dev/khaos/internal/inject"@@ -63,6 +64,13 @@ type Shape struct {Authority *pki.AuthorityPKITrap *pki.Trap+ // The dependency khaos plays so one can answer a definitive deny. The+ // listen address is the one the target's own authorization check already+ // calls, so it is given rather than chosen.+ Authz *authz.Dependency+ AuthzTrap *authz.Trap+ AuthzListen string+DNSTrap *dnsrv.TrapResolverNetwork stringDNSHost string@@ -95,9 +103,21 @@ type Shape struct {Consulted func() (int64, error)ConsultedWhat string+ // Refusals counts the definitive denials the faulted dependency issued, for+ // the retry storm: a caller presenting a credential that has already been+ // refused once per request asks about it once per request.+ Refusals func() (int64, error)++ // Shift changes what the faulted dependency ANSWERS, between two fault+ // windows on the same run, so the two can be compared.+ Shift func(context.Context) error+ ShiftDetail string+ ShiftWhat string+Unsupportable stringCacheRequired stringDeliveredWhat string+ DeliveredHint stringIdleFor time.Duration@@ -273,6 +293,13 @@ func Build(Authority: shape.Authority,PKITrap: shape.PKITrap,+ Authz: shape.Authz,+ AuthzTrap: shape.AuthzTrap,+ AuthzListen: shape.AuthzListen,+ Refusals: shape.Refusals,+ Shift: shape.Shift,+ ShiftDetail: shape.ShiftDetail,+ ShiftWhat: shape.ShiftWhat,AuthorityBase: shape.AuthorityBase,AuthorityListen: shape.AuthorityListen,AuthorityTLS: shape.AuthorityTLS,@@ -284,6 +311,7 @@ func Build(Unsupportable: shape.Unsupportable,CacheRequired: shape.CacheRequired,DeliveredWhat: shape.DeliveredWhat,+ DeliveredHint: shape.DeliveredHint,DNSTrap: shape.DNSTrap,ResolverNetwork: shape.ResolverNetwork,DNSHost: shape.DNSHost,diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.goindex 3769daf0..b72f7d2f 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go@@ -41,6 +41,11 @@ const (caveatRedirectUnseen = "khaos served the redirect and measured that the target re-resolved, and it did NOT observe which host the target then connected to: the redirect address is an operator parameter and khaos has nothing listening at the far end of it. So this HELD means the target's control was not disturbed, and it does not establish that the target verified who it reached. If the address you redirected to answers at all, check its own logs for the target's requests before reading this as a pass"deliveredResolve = "re-resolve its dependency's name"++ // What to check when a DNS fault was armed, confirmed and never met. It+ // lives here rather than in the verdict's shared sentence because the only+ // other tier carrying that guard reaches it for an unrelated reason.+ hintResolve = "A kept-alive connection produces this shape -- Go's HTTP transport resolves a name once when it opens a connection and never again while that connection lives, measured at two lookups across forty-six requests -- and so would the target's own resolver cache, an /etc/hosts entry, or resolution over DoH, none of which khaos ruled out. What to check, in that order: the target's connection lifetime, its own DNS cache, /etc/hosts, and whether it resolves over plain DNS at all")// dnsPolarity omits two directions deliberately: DENIED is correct with an@@ -148,6 +153,7 @@ func buildDNS(spec dnsSpec) func(*Params) (Shape, error) {// one fault a target can simply not ask for again. The TTL pair is// exempt, because there not re-resolving IS the measurement.DeliveredWhat: deliveredFor(spec),+ DeliveredHint: hintFor(spec),}if spec.needsCache {@@ -207,6 +213,14 @@ func deliveredFor(spec dnsSpec) string {return deliveredResolve}+func hintFor(spec dnsSpec) string {+ if spec.needsCache {+ return ""+ }++ return hintResolve+}+// DNSDelayCeiling is exported so the CLI's refusal names the same number the// resolver enforces.func DNSDelayCeiling() time.Duration {diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.goindex 3467b207..24d4c539 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go@@ -20,6 +20,7 @@ const (StageConfirm Stage = "confirm"StageObserve Stage = "observe"StageAct Stage = "act"+ StageShift Stage = "shift"StageIdle Stage = "idle"StageResume Stage = "resume"StageRevert Stage = "revert"@@ -28,17 +29,19 @@ const ()const (- detailBaseline = "establishing steady state"- detailObserve = "measuring under fault"- detailRevert = "removing the fault"- detailRecover = "confirming the target came back"- detailInjecting = "%s on %s"- detailConfirmed = "fault confirmed by %s"- detailUnconfirm = "the fault was not confirmed, aborting before measuring"- detailVerdict = "%s / %s"- detailActFailed = "the mid-run action failed, so nothing measured after it means anything: %v"- detailIdle = "a credential leaked; pausing traffic for %s to see whether the cache is being re-armed"- detailResume = "resuming traffic with the fault still applied"+ detailBaseline = "establishing steady state"+ detailObserve = "measuring under fault"+ detailRevert = "removing the fault"+ detailRecover = "confirming the target came back"+ detailInjecting = "%s on %s"+ detailConfirmed = "fault confirmed by %s"+ detailUnconfirm = "the fault was not confirmed, aborting before measuring"+ detailVerdict = "%s / %s"+ detailActFailed = "the mid-run action failed, so nothing measured after it means anything: %v"+ detailShift = "%s, and measuring again"+ detailShiftFailed = "the second phase could not be established, so khaos has one window where it needs two: %v"+ detailIdle = "a credential leaked; pausing traffic for %s to see whether the cache is being re-armed"+ detailResume = "resuming traffic with the fault still applied"// Breaks aliasing against the periodic target behaviour these experiments// measure: cache TTLs, token refresh and crons.@@ -271,6 +274,8 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {observation = e.measureFault(ctx, spec, timing, observation)+ observation = e.shift(ctx, spec, timing, observation)+observation = e.discriminate(ctx, spec, observation, timing)observation, revertErr := e.restore(ctx, spec, timing, observation)@@ -294,6 +299,7 @@ func (e Engine) measureFault(e.emit(StageObserve, detailObserve)before, beforeErr := e.consulted(spec)+ refusedBefore, refusedBeforeErr := e.refusals(spec)observation.Fault = e.sample(ctx,@@ -303,9 +309,20 @@ func (e Engine) measureFault()after, afterErr := e.consulted(spec)+ refusedAfter, refusedAfterErr := e.refusals(spec)observation.Delivered = after - beforeobservation.DeliveredWhat = spec.DeliveredWhat+ observation.DeliveredHint = spec.DeliveredHint+ observation.Refusals = refusedAfter - refusedBefore++ // NOT folded into CounterUnreadable, which voids the run: this count feeds+ // one optional finding rather than a guard, and zero makes no claim.+ if refusedBeforeErr != nil ||+ refusedAfterErr != nil ||+ observation.Refusals < 0 {+ observation.Refusals = 0+ }if beforeErr != nil || afterErr != nil || observation.Delivered < 0 {observation.CounterUnreadable = errors.Join(@@ -319,6 +336,62 @@ func (e Engine) measureFault(return observation}+// shift is the SECOND fault phase. One experiment needs it and research 07+// section 6.2 is why: an authorization dependency's deny and its silence are+// different answers, a correct caller treats them differently, and a caller+// that behaves IDENTICALLY has collapsed three outcomes into two branches.+// One window cannot show that; two can.+//+// A failed shift is recorded rather than fatal. The first window is already+// measured and its finding stands on its own; what is lost is the discriminator,+// and the verdict says so instead of pretending it ran.+func (e Engine) shift(+ ctx context.Context,+ spec Experiment,+ timing Timing,+ observation Observation,+) Observation {+ if spec.Shift == nil || ctx.Err() != nil {+ return observation+ }++ observation.ShiftWhat = spec.ShiftWhat++ e.emit(StageShift, sprintf(detailShift, spec.ShiftDetail))++ if err := spec.Shift(ctx); err != nil {+ e.emit(StageShift, sprintf(detailShiftFailed, err))+ observation.ShiftFailed = true++ return observation+ }++ before, beforeErr := e.consulted(spec)++ observation.Shift = e.sample(+ ctx,+ spec.Probes,+ timing.FaultFor,+ timing.SampleInterval,+ )++ after, afterErr := e.consulted(spec)++ observation.Shifted = true+ observation.ShiftAsked = after - before++ if beforeErr != nil || afterErr != nil || observation.ShiftAsked < 0 {+ observation.CounterUnreadable = errors.Join(+ observation.CounterUnreadable,+ beforeErr,+ afterErr,+ )+ observation.ShiftAsked = 0+ }++ return observation+}+// discriminate pauses traffic past any plausible cache lifetime with the fault// still applied, then resumes, and only when the fault window actually leaked.func (e Engine) discriminate(@@ -390,6 +463,16 @@ func (e Engine) consulted(spec Experiment) (int64, error) {return spec.Consulted()}+// refusals counts the DEFINITIVE denials the faulted dependency issued, which+// is the same thing as how many times the target asked about a credential it+// had already been refused for: nothing else is answered that way.+func (e Engine) refusals(spec Experiment) (int64, error) {+ if spec.Refusals == nil {+ return 0, nil+ }+ return spec.Refusals()+}+func markInterrupted(observation Observation) Observation {observation.Interrupted = truereturn observationdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.goindex 7e3a097a..ff0e5d36 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go@@ -10,6 +10,7 @@ import ("strings""time"+ "github.com/CarterPerez-dev/khaos/internal/authz""github.com/CarterPerez-dev/khaos/internal/dnsrv""github.com/CarterPerez-dev/khaos/internal/idp""github.com/CarterPerez-dev/khaos/internal/inject"@@ -114,6 +115,12 @@ type Experiment struct {Authority *pki.AuthorityPKITrap *pki.Trap+ // A non-nil AuthzTrap is what tells the run to bind the authorization+ // dependency khaos plays.+ Authz *authz.Dependency+ AuthzTrap *authz.Trap+ AuthzListen string+// AuthorityBase is the URL the TARGET reaches the revocation endpoints by,// which is pinned rather than derived from wherever khaos binds.AuthorityBase string@@ -150,6 +157,17 @@ type Experiment struct {Consulted func() (int64, error)ConsultedWhat string+ // Refusals is the count of definitive denials the dependency issued, which+ // only the credential under test is ever answered with.+ Refusals func() (int64, error)++ // Shift is the second fault phase. The one fault class that needs it is an+ // authorization dependency, whose deny and whose silence are different+ // answers that a defective caller reads as the same one.+ Shift func(context.Context) error+ ShiftDetail string+ ShiftWhat string+// Nothing in a request count separates the two readings of a never-consulted// endpoint, so only an experiment declaring this treats silence as a finding.Unsupportable string@@ -157,6 +175,7 @@ type Experiment struct {CacheRequired stringDeliveredWhat string+ DeliveredHint string// IdleFor must exceed the target's own decision-cache lifetime, which khaos// cannot discover, so the experiment declares it.diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier_test.goindex 691c56c8..a9eb6436 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier_test.go@@ -469,11 +469,15 @@ func TestTheRolloverControlIsAJWTUnderAKeyThatIsNeverRotated(t *testing.T) {}}-// The rotation experiment is the only inversion in the family, and it is the-// only thing in the whole catalogue that makes the directional axis anything but-// decoration. If it ever stops treating DENIED as a defect, nothing else-// exercises that branch.-func TestTheRotationExperimentIsTheOnlyOneWhereRefusingIsTheDefect(+// TWO experiments treat refusing as a defect, and the set is pinned because it+// is the only thing in the whole catalogue that makes the directional axis+// anything but decoration: if both ever stopped, nothing would exercise that+// branch. Each one is here for the same reason -- the fault was scoped to one+// credential, so refusing the OTHER one is an outage the fault never called+// for. The rotation publishes both keys precisely so nothing breaks, and the+// authorization dependency denies exactly one credential and keeps allowing+// every other. A third entry has to earn that argument.+func TestRefusingIsTheDefectOnlyWhereTheFaultWasScopedToOneCredential(t *testing.T,) {inverted := []string{}@@ -484,11 +488,14 @@ func TestTheRotationExperimentIsTheOnlyOneWhereRefusingIsTheDefect(}}- if !slices.Equal(inverted, []string{NameKeyRotation}) {+ want := []string{NameAPIKeyRevoked, NameKeyRotation}+ slices.Sort(want)++ if !slices.Equal(inverted, want) {t.Fatalf(- "the experiments treating DENIED as a defect are %v, want exactly [%s]",+ "the experiments treating DENIED as a defect are %v, want exactly %v",inverted,- NameKeyRotation,+ want,)}}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.goindex a233f33e..0a780937 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go@@ -497,3 +497,58 @@ func TestAnExplicitlyEmptyRequiredValueIsStillRefused(t *testing.T) {)}}++// The delivery guard's ADVICE is per experiment and its sentence is not. Two+// tiers carry that guard and they reach it for unrelated reasons -- a name the+// target never resolved again, and a decision it never asked for again -- so a+// hint shared between them tells one of the two to check /etc/hosts about an+// authorization cache. The first version did exactly that.+func TestEveryDeliveryGuardCarriesItsOwnAdvice(t *testing.T) {+ seen := map[string]string{}++ for _, definition := range Catalog() {+ spec, err := Build(definition.Name, fill(definition.Params), fastTiming)+ if err != nil {+ t.Fatalf("%s: build: %v", definition.Name, err)+ }++ if spec.DeliveredWhat == "" {+ if spec.DeliveredHint != "" {+ t.Fatalf(+ "%s declares advice for a delivery guard it does not carry, so nothing will ever print it",+ definition.Name,+ )+ }++ continue+ }++ if spec.DeliveredHint == "" {+ t.Fatalf(+ "%s carries the delivery guard with no advice, so a run that armed a fault the target never met tells an operator nothing to check",+ definition.Name,+ )+ }++ seen[spec.DeliveredWhat] = spec.DeliveredHint+ }++ if len(seen) < 2 {+ t.Fatalf(+ "only %d experiment(s) carry the delivery guard, so this census cannot catch one tier being given another tier's advice",+ len(seen),+ )+ }++ hints := map[string]string{}+ for what, hint := range seen {+ if first, clash := hints[hint]; clash {+ t.Fatalf(+ "%q and %q print the same advice, and they fail the guard for unrelated reasons",+ first,+ what,+ )+ }+ hints[hint] = what+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.gonew file mode 100644index 00000000..605423be--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go@@ -0,0 +1,790 @@+// ©AngelaMos | 2026+// shift_test.go++package experiment++import (+ "context"+ "errors"+ "strings"+ "testing"+ "time"++ "github.com/CarterPerez-dev/khaos/internal/inject"+ "github.com/CarterPerez-dev/khaos/internal/probe"+)++const (+ whatShifted = "khaos's dependency answered 503 to everything"++ deniedCredentialRefused = "the credential the dependency denied"+)++// deniedWindow is the FIRST fault phase: the dependency answered a definitive+// 403 for one credential and kept allowing the other.+func deniedWindow(revoked probe.Disposition) probe.Window {+ return probe.Window{+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),+ Invalid: stream(+ probe.Invalid,+ repeat(probe.Rejected, MinimumPaired)...),+ Revoked: stream(probe.Revoked, repeat(revoked, MinimumPaired)...),+ }+}++// unknownWindow is the SECOND phase: the dependency answered nothing at all,+// so what the valid stream does here is a different branch of the caller.+func unknownWindow(valid probe.Disposition) probe.Window {+ return probe.Window{+ Valid: stream(probe.Valid, repeat(valid, MinimumPaired)...),+ Invalid: stream(+ probe.Invalid,+ repeat(probe.Rejected, MinimumPaired)...),+ Revoked: stream(probe.Revoked, repeat(valid, MinimumPaired)...),+ }+}++func shifted(revoked, validUnderOutage probe.Disposition) Observation {+ return Observation{+ Baseline: probe.Window{+ Valid: stream(+ probe.Valid,+ repeat(probe.Accepted, MinimumPaired)...),+ Invalid: stream(+ probe.Invalid,+ repeat(probe.Rejected, MinimumPaired)...),+ Revoked: stream(+ probe.Revoked,+ repeat(probe.Accepted, MinimumPaired)...),+ },+ Fault: deniedWindow(revoked),+ Shift: unknownWindow(validUnderOutage),+ Shifted: true,+ ShiftWhat: whatShifted,+ Recovery: healthyRecovery(),+ FaultConfirmed: true,+ Defect: deniedCredentialRefused,+ DeliveredWhat: "ask its authorization dependency",+ Delivered: int64(MinimumPaired * 2),++ // The SECOND window's counter, and leaving it out is not neutral: a+ // declared delivery guard plus a zero count is a window the target+ // never met, which the verdict voids rather than reads. Every row+ // below is about a target that did consult its dependency.+ ShiftAsked: int64(MinimumPaired * 2),+ }+}++// The matrix research 07 section 6.2 prescribes. Each phase measures an+// INDEPENDENT branch of the caller's error handling, so all four cells exist+// and only one of them is a caller with three outcomes. A tool that ran one+// window could not tell the second row from the fourth.+func TestTheTwoPhasesReadFourDifferentCallers(t *testing.T) {+ for _, row := range []struct {+ name string+ revoked probe.Disposition+ validUnderOut probe.Disposition+ wantDirection Direction+ wantDefect bool+ wantReason string+ }{+ {+ name: "three branches, which is the correct caller",+ revoked: probe.Rejected,+ validUnderOut: probe.Rejected,+ wantDirection: Held,+ wantDefect: false,+ wantReason: "acted on what the dependency SAID",+ },+ {+ name: "honours the deny and treats silence as permission",+ revoked: probe.Rejected,+ validUnderOut: probe.Accepted,+ wantDirection: Permitted,+ wantDefect: true,+ wantReason: "this target's fallback is to allow",+ },+ {+ name: "honours silence and ignores the deny",+ revoked: probe.Accepted,+ validUnderOut: probe.Rejected,+ wantDirection: Permitted,+ wantDefect: true,+ wantReason: "the collapse inverted",+ },+ {+ name: "two branches where it needs three",+ revoked: probe.Accepted,+ validUnderOut: probe.Accepted,+ wantDirection: Permitted,+ wantDefect: true,+ wantReason: "served in BOTH phases",+ },+ } {+ t.Run(row.name, func(t *testing.T) {+ verdict := Judge(+ shifted(row.revoked, row.validUnderOut),+ authzPolarity,+ )++ if verdict.Direction != row.wantDirection {+ t.Fatalf(+ "direction %s, want %s (reasons: %v)",+ verdict.Direction,+ row.wantDirection,+ verdict.Reasons,+ )+ }++ if verdict.Defect != row.wantDefect {+ t.Fatalf("defect %v, want %v", verdict.Defect, row.wantDefect)+ }++ if !hasReason(verdict, row.wantReason) {+ t.Fatalf(+ "no reason mentions %q, so khaos reached the right verdict without saying which caller it saw: %v",+ row.wantReason,+ verdict.Reasons,+ )+ }++ if !strings.Contains(+ strings.Join(verdict.Reasons, " "),+ whatShifted,+ ) {+ t.Fatalf(+ "the reasons never name what the second phase did: %v",+ verdict.Reasons,+ )+ }+ })+ }+}++// A refusal in the second phase is the CORRECT outcome, so it must never be+// read as a direction of its own: the valid credential is refused there by a+// caller that is behaving properly, and reading that window the way the first+// one is read would report the correct caller as having failed closed.+func TestTheSecondPhaseIsNeverReadAsADenial(t *testing.T) {+ verdict := Judge(shifted(probe.Rejected, probe.Rejected), authzPolarity)++ if verdict.Direction != Held {+ t.Fatalf(+ "a caller that refused in both phases was judged %s: the second window is a comparison, not a direction",+ verdict.Direction,+ )+ }++ if verdict.Epistemic != Passed {+ t.Fatalf("epistemic %s, want %s", verdict.Epistemic, Passed)+ }+}++// An acceptance at the START of the second window is what a decision cache+// produces: the caller was correctly serving that credential a moment earlier.+// Only the trailing run counts, or khaos would report a correct caller with a+// two-second cache as failing open.+func TestACacheLapsingIntoRefusalIsNotAFinding(t *testing.T) {+ observation := shifted(probe.Rejected, probe.Rejected)++ observation.Shift.Valid = stream(+ probe.Valid,+ probe.Accepted,+ probe.Accepted,+ probe.Rejected,+ probe.Rejected,+ probe.Rejected,+ )++ verdict := Judge(observation, authzPolarity)++ if verdict.Direction != Held {+ t.Fatalf(+ "a cache that lapsed into refusal was judged %s: the acceptances came before the last refusal, which is what a bounded cache looks like (%v)",+ verdict.Direction,+ verdict.Reasons,+ )+ }+}++// And the mirror: a refusal followed by acceptances is a cache that lapsed and+// a caller that then decided to allow anyway, which IS the finding.+func TestServingAfterTheLastRefusalIsAFinding(t *testing.T) {+ observation := shifted(probe.Rejected, probe.Rejected)++ observation.Shift.Valid = stream(+ probe.Valid,+ probe.Rejected,+ probe.Rejected,+ probe.Accepted,+ probe.Accepted,+ probe.Accepted,+ )++ verdict := Judge(observation, authzPolarity)++ if verdict.Direction != Permitted {+ t.Fatalf(+ "a caller that started serving again while the dependency could not answer was judged %s (%v)",+ verdict.Direction,+ verdict.Reasons,+ )+ }+}++// THE DEFECT AN AUDIT FOUND IN GREEN CODE. A caller that answers 503 when it+// cannot reach its authorization dependency is failing CLOSED -- this+// experiment declares exactly that with ClosedWhen -- and the trailing-run+// guard only ever reset on Rejected, so one acceptance from a one-second+// decision cache followed by 39 correct refusals was reported as+// deviated / PERMITTED with "this target's fallback is to allow". The shipped+// fixture answers 403, so neither the gate nor the matrix above could see it.+func TestA503RefusalEndsTheRunOfAcceptances(t *testing.T) {+ observation := shifted(probe.Rejected, probe.Rejected)+ observation.ClosedWhen = []probe.Disposition{probe.Unavailable}++ observation.Shift.Valid = stream(+ probe.Valid,+ append(+ []probe.Disposition{probe.Accepted},+ repeat(probe.Unavailable, MinimumPaired*2)...,+ )...,+ )++ verdict := Judge(observation, authzPolarity)++ if verdict.Direction == Permitted {+ t.Fatalf(+ "a caller that served one cached request and then refused %d with the disposition this experiment DECLARES as closed was judged PERMITTED: %v",+ MinimumPaired*2,+ verdict.Reasons,+ )+ }++ if hasReason(verdict, "this target's fallback is to allow") {+ t.Fatalf(+ "khaos told an operator their fail-closed caller allows by default: %v",+ verdict.Reasons,+ )+ }++ // And the mirror, on the FIRST window's revoked stream: the same hardcoded+ // Rejected made every cached acceptance count when the target then 503'd.+ window := shifted(probe.Rejected, probe.Rejected)+ window.ClosedWhen = []probe.Disposition{probe.Unavailable}+ window.Fault.Revoked = stream(+ probe.Revoked,+ append(+ []probe.Disposition{probe.Accepted},+ repeat(probe.Unavailable, MinimumPaired*2)...,+ )...,+ )++ if revokedHonoured(window) != 0 {+ t.Fatalf(+ "a bounded cache that lapsed into 503s counted %d unexplained acceptances; a correct bounded cache explains all of them",+ revokedHonoured(window),+ )+ }+}++// The exposure figure has to describe the SAME set as the count beside it. The+// count is the trailing run; a first-to-last span over the whole window+// includes the period the target spent refusing, and quoting the two in one+// sentence states an exposure khaos did not measure.+func TestTheExposureSpanDescribesTheRunItCounts(t *testing.T) {+ observation := shifted(probe.Rejected, probe.Rejected)++ observation.Shift.Valid = stream(+ probe.Valid,+ probe.Accepted,+ probe.Accepted,+ probe.Rejected,+ probe.Rejected,+ probe.Accepted,+ probe.Accepted,+ )++ // stream() spaces its outcomes one second apart, so the whole window spans+ // 5s and the trailing run of two acceptances spans 1s.+ if span := observation.Shift.Valid.AcceptedSpanSinceLastRefusal(+ probe.Rejected,+ ); span != time.Second {+ t.Fatalf(+ "the trailing run spans %s, want 1s: a first-to-last span over this window is 5s, 4s of which the target was refusing",+ span,+ )+ }++ if !hasReason(+ Judge(observation, authzPolarity),+ "1s is a lower bound on the exposure",+ ) {+ t.Fatalf(+ "the reason quotes a span that is not the run it counted: %v",+ Judge(observation, authzPolarity).Reasons,+ )+ }+}++// Window one aborts the run when the target never met the fault. Window two had+// no such guard, so a target that answered the whole second window from a+// cached allow -- without asking its dependency once -- was reported as a+// conclusive defect for serving "requests whose authorization it could not have+// checked". It checked nothing, because it asked nothing.+func TestASecondWindowTheTargetNeverMetIsNotAFinding(t *testing.T) {+ observation := shifted(probe.Rejected, probe.Accepted)+ observation.DeliveredWhat = "ask its authorization dependency"+ observation.ShiftAsked = 0++ verdict := Judge(observation, authzPolarity)++ if verdict.Direction == Permitted {+ t.Fatalf(+ "a second window the target never met was reported as a bypass: %v",+ verdict.Reasons,+ )+ }++ if !hasReason(verdict, "did not observe the target meeting it") {+ t.Fatalf(+ "khaos did not say the second window never reached the target: %v",+ verdict.Reasons,+ )+ }++ // With the counter moving, the same window IS the finding.+ delivered := shifted(probe.Rejected, probe.Accepted)+ delivered.DeliveredWhat = "ask its authorization dependency"+ delivered.ShiftAsked = int64(MinimumPaired)++ if Judge(delivered, authzPolarity).Direction != Permitted {+ t.Fatal(+ "with the dependency actually consulted, serving the valid credential while it could not answer is the finding and must still be reported",+ )+ }+}++// A stream that was only ever redirected or throttled decided nothing, and+// every four-cell sentence reads an absence of acceptances as an observed+// refusal. baselineUnmet already guards redirect-as-refusal; the fault+// window's revoked stream needed the same.+func TestAnUndecidedFirstWindowIsNotReadAsARefusal(t *testing.T) {+ observation := shifted(probe.Rejected, probe.Rejected)+ observation.Fault.Revoked = stream(+ probe.Revoked,+ repeat(probe.Redirected, MinimumPaired)...,+ )++ verdict := Judge(observation, authzPolarity)++ if hasReason(verdict, "That is the third branch") {+ t.Fatalf(+ "a stream that only ever redirected was credited with refusing the denied credential: %v",+ verdict.Reasons,+ )+ }++ if !hasReason(verdict, "no decision to compare") {+ t.Fatalf(+ "the undecided first window was not reported: %v",+ verdict.Reasons,+ )+ }+}++// Each phase is guarded by whether it was MEASURABLE. The absence of an+// acceptance is only evidence of a refusal when the target answered at all, and+// a caller that fell over in the second phase must not read as one that+// correctly stopped serving.+func TestAnUnmeasurablePhaseIsSaidRatherThanRead(t *testing.T) {+ silent := shifted(probe.Rejected, probe.Rejected)+ silent.Shift.Valid = stream(+ probe.Valid,+ repeat(probe.ProbeFailed, MinimumPaired)...)++ if !hasReason(+ Judge(silent, authzPolarity),+ "cannot tell a target that correctly stopped serving from one that fell over",+ ) {+ t.Fatalf(+ "a second phase the target never answered was read as a correct refusal: %v",+ Judge(silent, authzPolarity).Reasons,+ )+ }++ unanswered := shifted(probe.Rejected, probe.Rejected)+ unanswered.Fault.Revoked = stream(+ probe.Revoked,+ repeat(probe.ProbeFailed, MinimumPaired)...)++ if !hasReason(+ Judge(unanswered, authzPolarity),+ "produced no answer at all during the first window",+ ) {+ t.Fatalf(+ "the two phases were compared with nothing measured in the first: %v",+ Judge(unanswered, authzPolarity).Reasons,+ )+ }+}++// A failed shift leaves khaos with one window where it needs two. The first+// one's finding stands on its own, and the missing comparison is stated rather+// than passed over in silence.+func TestAFailedShiftIsStatedAndNotFatal(t *testing.T) {+ observation := shifted(probe.Rejected, probe.Rejected)+ observation.Shifted = false+ observation.Shift = probe.Window{}+ observation.ShiftFailed = true++ verdict := Judge(observation, authzPolarity)++ if verdict.Direction != Held {+ t.Fatalf(+ "a failed second phase changed the first phase's verdict to %s",+ verdict.Direction,+ )+ }++ if !hasReason(verdict, "could not establish the second phase") {+ t.Fatalf("the missing comparison was not reported: %v", verdict.Reasons)+ }+}++// An experiment with no second phase must read exactly as it did before the+// mechanism existed, or every other family's verdicts move.+func TestAnExperimentWithoutASecondPhaseSaysNothingAboutOne(t *testing.T) {+ observation := Observation{+ Baseline: healthyBaseline(),+ Fault: deniedWindow(probe.Rejected),+ Recovery: healthyRecovery(),+ FaultConfirmed: true,+ }++ for _, reason := range Judge(observation, authPolarity).Reasons {+ if strings.Contains(reason, "second phase") ||+ strings.Contains(reason, "dependency SAID") {+ t.Fatalf(+ "an experiment with no second phase was told about one: %q",+ reason,+ )+ }+ }+}++// The retry storm, which is a REASON rather than a direction: no credential got+// through, and what it tells an operator is that their caller will hammer a+// dependency with a key it has already been told is dead.+func TestTheRetryStormIsCountedAgainstTheRequests(t *testing.T) {+ quiet := shifted(probe.Rejected, probe.Rejected)+ quiet.Refusals = int64(MinimumPaired)++ if hasReason(+ Judge(quiet, authzPolarity),+ "asked again about a credential",+ ) {+ t.Fatalf(+ "a caller that asked exactly once per request was reported as retrying: %v",+ Judge(quiet, authzPolarity).Reasons,+ )+ }++ storm := shifted(probe.Rejected, probe.Rejected)+ storm.Refusals = int64(MinimumPaired * 3)++ if !hasReason(+ Judge(storm, authzPolarity),+ "asked again about a credential",+ ) {+ t.Fatalf(+ "three refusals per request was not reported as a retry: %v",+ Judge(storm, authzPolarity).Reasons,+ )+ }+}++// The census. A tier whose whole value is the comparison between two windows+// must not ship an experiment that measures one of them: the first window's+// finding is real on its own, so the omission would be invisible in every+// verdict. And a second phase khaos cannot NAME produces evidence an operator+// cannot act on, in both the plan line and the reason text.+func TestEveryAuthorizationExperimentMeasuresBothWindows(t *testing.T) {+ covered := 0++ for _, definition := range Catalog() {+ if !inject.NeedsAuthz(definition.FaultKind) {+ continue+ }++ t.Run(definition.Name, func(t *testing.T) {+ covered++++ spec, err := Build(+ definition.Name,+ fill(definition.Params),+ fastTiming,+ )+ if err != nil {+ t.Fatalf("build: %v", err)+ }++ if spec.Shift == nil {+ t.Fatal(+ "this experiment arms a definitive deny and never makes the dependency unable to answer, so it measures one branch of the caller and claims to measure the discriminator",+ )+ }++ if spec.ShiftDetail == "" {+ t.Fatal(+ "the second phase has no plan line, so a run shows an operator a stage with no reason for it",+ )+ }++ if spec.ShiftWhat == "" {+ t.Fatal(+ "the second phase is unnamed, so every reason quoting it reads as a placeholder",+ )+ }++ if spec.Refusals == nil {+ t.Fatal(+ "this experiment has no refusal counter, so the retry storm research 07 section 6.2 names is unmeasurable here",+ )+ }++ if spec.Consulted == nil {+ t.Fatal(+ "this experiment has no consultation counter, so a target that never asks its authorization dependency reads as one that survived the fault",+ )+ }+ })+ }++ if covered == 0 {+ t.Fatal(+ "no catalogued experiment needs the authorization dependency, so this census cannot fail for the right reason",+ )+ }+}++// phasedProbe answers differently in all THREE phases, which is what a shift+// test needs and what the two-phase scripted probe cannot express.+type phasedProbe struct {+ credential probe.Credential+ before probe.Disposition+ during probe.Disposition+ after probe.Disposition+ faulted *bool+ shifted *bool+}++func (p phasedProbe) Name() string { return "phased" }+func (p phasedProbe) Describe() string { return "a phased probe" }+func (p phasedProbe) Credential() probe.Credential { return p.credential }++func (p phasedProbe) Run(context.Context) probe.Outcome {+ disposition := p.before++ switch {+ case *p.shifted:+ disposition = p.after+ case *p.faulted:+ disposition = p.during+ }++ return probe.Outcome{+ At: time.Now().UTC(),+ Credential: p.credential,+ Disposition: disposition,+ }+}++// The verdict tests above hand-build their observations, which proves nothing+// about the code that FEEDS them: a control on a pure function says nothing+// about its caller. This is the other end -- the engine really arming the+// second phase, really sampling a second window, and really recording it where+// the verdict reads it.+func TestTheEngineMeasuresASecondWindowAfterTheShift(t *testing.T) {+ faulted := false+ wasShifted := false+ asked := int64(0)++ injector := &scriptedInjector{faulted: &faulted}+ engine := Engine{Injector: injector}++ var stages []Stage+ engine.Report = func(e Event) { stages = append(stages, e.Stage) }++ run := spec(+ phasedProbe{+ credential: probe.Valid,+ before: probe.Accepted,+ during: probe.Accepted,+ after: probe.Rejected,+ faulted: &faulted,+ shifted: &wasShifted,+ },+ phasedProbe{+ credential: probe.Invalid,+ before: probe.Rejected,+ during: probe.Rejected,+ after: probe.Rejected,+ faulted: &faulted,+ shifted: &wasShifted,+ },+ )++ run.Probes.Revoked = phasedProbe{+ credential: probe.Revoked,+ before: probe.Accepted,+ during: probe.Rejected,+ after: probe.Rejected,+ faulted: &faulted,+ shifted: &wasShifted,+ }++ run.Consulted = func() (int64, error) {+ asked+++ return asked, nil+ }+ run.ConsultedWhat = "khaos's authorization dependency"+ run.ShiftWhat = whatShifted+ run.ShiftDetail = "making the dependency unable to answer"+ run.Shift = func(context.Context) error {+ wasShifted = true+ return nil+ }++ result, err := engine.Run(context.Background(), run)+ if err != nil {+ t.Fatalf("run: %v", err)+ }++ if !slicesContains(stages, StageShift) {+ t.Fatalf("the engine never reported a shift stage: %v", stages)+ }++ if !result.Observation.Shifted {+ t.Fatal("the engine armed the second phase and did not record it")+ }++ if result.Observation.Shift.Valid.Len() == 0 {+ t.Fatal(+ "the engine recorded no second window, so every comparison the verdict makes would read a zero-valued one",+ )+ }++ if result.Observation.Fault.Valid.Count(probe.Accepted) == 0 {+ t.Fatal(+ "the first window recorded no acceptance of the valid credential",+ )+ }++ if result.Observation.Shift.Valid.Count(probe.Rejected) == 0 {+ t.Fatalf(+ "the second window recorded no refusal, so the two windows are the same measurement: %v",+ result.Observation.Shift.Valid.Distribution(),+ )+ }++ if result.Observation.ShiftWhat != whatShifted {+ t.Fatalf(+ "the observation does not name the second phase: %q",+ result.Observation.ShiftWhat,+ )+ }++ if result.Observation.ShiftAsked <= 0 {+ t.Fatal(+ "the dependency counter was not read across the second window, so a target that stopped asking would read as one that kept asking",+ )+ }++ if result.Verdict.Direction != Held {+ t.Fatalf(+ "a caller that refused in both phases was judged %s: %v",+ result.Verdict.Direction,+ result.Verdict.Reasons,+ )+ }+}++// A failed shift must not take the first window's finding with it, and must not+// skip the rollback.+func TestAFailedShiftStillRevertsAndStillConcludes(t *testing.T) {+ faulted := false+ wasShifted := false++ injector := &scriptedInjector{faulted: &faulted}+ engine := Engine{Injector: injector}++ run := spec(+ phasedProbe{+ credential: probe.Valid,+ before: probe.Accepted,+ during: probe.Accepted,+ after: probe.Accepted,+ faulted: &faulted,+ shifted: &wasShifted,+ },+ phasedProbe{+ credential: probe.Invalid,+ before: probe.Rejected,+ during: probe.Rejected,+ after: probe.Rejected,+ faulted: &faulted,+ shifted: &wasShifted,+ },+ )++ run.Shift = func(context.Context) error {+ return errors.New("the dependency would not answer 503")+ }++ result, err := engine.Run(context.Background(), run)+ if err != nil {+ t.Fatalf("run: %v", err)+ }++ if !result.Observation.ShiftFailed {+ t.Fatal("a failed shift was not recorded")+ }++ if result.Observation.Shifted {+ t.Fatal(+ "a failed shift recorded a second window, so the verdict would compare against one that was never measured",+ )+ }++ if injector.reverts != 1 {+ t.Fatalf(+ "reverts=%d after a failed shift, want 1: the fault is still applied",+ injector.reverts,+ )+ }++ if result.Verdict.Direction != Held {+ t.Fatalf("direction %s, want %s", result.Verdict.Direction, Held)+ }+}++func slicesContains(stages []Stage, want Stage) bool {+ for _, stage := range stages {+ if stage == want {+ return true+ }+ }++ return false+}++func hasReason(verdict Verdict, want string) bool {+ for _, reason := range verdict.Reasons {+ if strings.Contains(reason, want) {+ return true+ }+ }++ return false+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.goindex d87030e3..b1307862 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go@@ -135,12 +135,55 @@ const (reasonTTLExtension = "the credential survived %s of STEADY traffic and then stopped being accepted after a %s pause, and no fixed lifetime behaves that way: every failed check was re-arming whatever the target had cached, a decision or a key set, so the lifetime was bounding IDLENESS rather than staleness. The fix is to stop writing to the cache in the error branch"reasonLapsedOnPause = "the credential stopped being accepted after the %s pause, but the steady-traffic window before it was only %s, so an ordinary fixed expiry between those two durations produces this exact result and khaos will not call it re-arming. Re-run with --fault-for at least %s to separate them"reasonStalePastPause = "the credential was still accepted after traffic was paused for %s, so whatever the target cached did not lapse across that pause. khaos cannot tell a cache with no expiry from one whose lifetime simply exceeds %s, and it will not claim the first from an observation that only establishes the second; raise --idle-for to raise the bound"- reasonNeverDelivered = "the fault was armed and confirmed, and khaos did not observe the target meeting it: it did not %s once during the fault window, so nothing khaos changed there can be shown to have reached it. khaos does NOT establish why. A kept-alive connection produces this shape -- Go's HTTP transport resolves a name once when it opens a connection and never again while that connection lives, measured at two lookups across forty-six requests -- and so would the target's own resolver cache, an /etc/hosts entry, or resolution over DoH, none of which khaos ruled out. What to check, in that order: the target's connection lifetime, its own DNS cache, /etc/hosts, and whether it resolves over plain DNS at all"- reasonCounterUnreadable = "khaos could not read its own count of the target's use of %s, so it cannot tell a target that never went there from an instrument it failed to read: %v. Every guard built on that count makes a statement about the TARGET, and khaos will not make one from a number it does not have"- reasonNoCache = "the target holds no cached answer, so %s governs nothing it does: it asked %s %d times during a baseline of %d requests, while khaos was serving a ten-minute lifetime for that name. A target honouring it resolves once for the whole window. There is no cache lifetime to have been honoured or ignored, so its behaviour under this fault is not a defect and khaos makes no security claim here. Go's resolver has no answer cache and ignores TTL entirely, which khaos measured rather than assumed from the target's language"- reasonNeverConsulted = "the target never used %s during the baseline, so the fault this experiment applies there is one it cannot experience; a target that validates credentials locally never calls it, and khaos would otherwise report a clean survival of a fault that never reached it"- reasonNoResumedEvidence = "the resumed window produced %d usable samples in %s and never a refusal, so khaos saw nothing that could tell a cache being re-armed from one that outlasts the pause"- reasonStaleWindow = "the credential khaos invalidated was accepted over a span of at least %s, which is the lower bound on the exposure and not a measurement of the cache lifetime"+ // The HINT is declared per experiment and this sentence is not, because the+ // two experiments that carry this guard reach it for unrelated reasons: a+ // name the target never resolved again, and a decision it never asked for+ // again. The first version hardcoded the DNS paragraph, so an+ // authorization run that never reached the dependency would have been told+ // to check /etc/hosts.+ reasonNeverDelivered = "the fault was armed and confirmed, and khaos did not observe the target meeting it: it did not %s once during the fault window, so nothing khaos changed there can be shown to have reached it. khaos does NOT establish why. %s"+ reasonCounterUnreadable = "khaos could not read its own count of the target's use of %s, so it cannot tell a target that never went there from an instrument it failed to read: %v. Every guard built on that count makes a statement about the TARGET, and khaos will not make one from a number it does not have"+ reasonNoCache = "the target holds no cached answer, so %s governs nothing it does: it asked %s %d times during a baseline of %d requests, while khaos was serving a ten-minute lifetime for that name. A target honouring it resolves once for the whole window. There is no cache lifetime to have been honoured or ignored, so its behaviour under this fault is not a defect and khaos makes no security claim here. Go's resolver has no answer cache and ignores TTL entirely, which khaos measured rather than assumed from the target's language"+ reasonNeverConsulted = "the target never used %s during the baseline, so the fault this experiment applies there is one it cannot experience; a target that validates credentials locally never calls it, and khaos would otherwise report a clean survival of a fault that never reached it"+ reasonNoResumedEvidence = "the resumed window produced %d usable samples in %s and never a refusal, so khaos saw nothing that could tell a cache being re-armed from one that outlasts the pause"+ reasonStaleWindow = "the credential khaos invalidated was accepted over a span of at least %s, which is the lower bound on the exposure and not a measurement of the cache lifetime"++ reasonServedWhileUnknown = "the valid credential was still being accepted %d time(s) at the END of a window in which %s, so the target served requests whose authorization it could not have checked. It had been accepting that credential before the fault as well, so a decision cache produces the same shape -- which is why this counts only the acceptances after the last refusal, and why %s is a lower bound on the exposure rather than a cache lifetime"++ reasonRetriedRefused = "the dependency issued %d definitive refusals across %d requests presenting the refused credential, so the target asked again about a credential it had already been told no about; retrying an unchanged credential cannot succeed and is indistinguishable from a brute-force attempt. The count is every ask about THAT credential, so it holds only if khaos was the one presenting it -- which is why the flag that supplies it says not to use one your live traffic is using"++ // The four cells of research 07 section 6.2's matrix. Each phase measures+ // one branch of the caller's error handling, and the pair is the finding:+ // what a caller does with a DENY and what it does with SILENCE are+ // independent, and a correct caller has a third branch for the second.+ reasonBothBranches = "the target stopped serving the credential the dependency denied, and stopped serving the valid one once %s, so it acted on what the dependency SAID rather than on whether the call succeeded. That is the third branch, and khaos observed it rather than inferring it"++ // A stream that was only ever redirected or throttled answered without+ // deciding, and the sentences above read an absence of acceptances as an+ // observed refusal. baselineUnmet already carries a dedicated guard for+ // redirect-as-refusal; the fault window's revoked stream needs one too.+ reasonDenyUndecided = "the second phase ran and khaos will not compare the two: the credential under test was never accepted and never refused during the first window -- every answer was a redirect or a throttle -- so there is no decision to compare the second phase against"++ reasonOnlyDeny = "the target refused the credential the dependency denied, and then kept serving once %s: it honours a definitive deny and treats the absence of a decision as permission. Only the unknown outcome is eligible for a fallback policy, and this target's fallback is to allow"++ reasonOnlyUnknown = "the target served the credential the dependency DENIED and stopped serving once %s, which is the collapse inverted: it branches on whether the call worked and never on what came back, so a 403 and a 200 are the same answer to it"++ // Deliberately NOT "behaved identically". The two phases are counted on+ // different streams with different credentials, so no comparison of+ // behaviour was performed: what khaos measured is that it SERVED in both,+ // which is the finding and is all this may claim.+ reasonNeitherBranch = "the target served in BOTH phases -- the credential the dependency denied, and the valid one once %s -- so it has two branches where it needs three and neither of them is reading the dependency's answer. This is the finding research 07 names: error handling written as 'did it work' rather than 'what did it say'"++ reasonDenyUnmeasured = "the second phase ran, and khaos will not compare the two: the credential under test produced no answer at all during the first window, so there is nothing to compare the second against"++ reasonUnknownUnmeasured = "the second phase ran and the target answered none of its probes, so khaos cannot tell a target that correctly stopped serving from one that fell over, and it will not claim the first"++ // "could not establish", not "could not arm": the arm succeeds before the+ // read-back runs, so the failure khaos actually hit is a confirmation it+ // could not get, and the fault may well be in effect.+ reasonShiftUnarmed = "khaos could not establish the second phase, so this run measured what the target does with a definitive deny and nothing about what it does when the dependency cannot answer; those are different branches and a correct caller needs both"++ reasonShiftNotDelivered = "the second phase was armed and confirmed, and khaos did not observe the target meeting it: it did not %s once during that window, so it answered every probe from something it had already cached and nothing khaos changed there reached it. Window one treats that observation as a void run rather than a finding, and so does this one")type Verdict struct {@@ -207,6 +250,34 @@ type Observation struct {Delivered int64 `json:"delivered,omitempty"`DeliveredWhat string `json:"delivered_what,omitempty"`+ // DeliveredHint is what to check, in this fault's own terms. The guard is+ // shared and the causes are not: a name the target never resolved again and+ // a decision it never asked for again have nothing in common to advise on.+ DeliveredHint string `json:"delivered_hint,omitempty"`++ // Refusals is how many DEFINITIVE denials the dependency issued during the+ // first fault window. Only the credential under test is ever answered that+ // way, so a count above one per request presenting it is the target asking+ // again about a credential it has already been refused for.+ Refusals int64 `json:"refusals,omitempty"`++ // Shift is the SECOND fault window, measured after khaos changed what the+ // faulted dependency answers: a definitive deny in the first, no answer at+ // all in the second. It is never read as a direction of its own. A refusal+ // here is the CORRECT outcome, so only an acceptance is evidence, which is+ // the same asymmetric bar every other window is read with.+ Shift probe.Window `json:"shift,omitempty"`+ Shifted bool `json:"shifted,omitempty"`+ ShiftAsked int64 `json:"shift_asked,omitempty"`++ // ShiftFailed is khaos failing to arm the second phase. The first window+ // still stands; what is gone is the comparison, and the verdict says so.+ ShiftFailed bool `json:"shift_failed,omitempty"`++ // ShiftWhat names the second phase in the words of the fault, because the+ // reason text has to be true of the dependency rather than generic.+ ShiftWhat string `json:"shift_what,omitempty"`+// CacheRequired is the INVERSE of Unsupportable: that reads a count of zero// as the answer, this reads a count that scales with traffic. Declared per// experiment, because a target re-resolving per request is correct for every@@ -433,6 +504,14 @@ func finish(verdict.Reasons = append(verdict.Reasons, observation.HeldCaveat)}+ if reading := shiftReading(observation); reading != "" {+ verdict.Reasons = append(verdict.Reasons, reading)+ }++ if storm := retryStorm(observation); storm != "" {+ verdict.Reasons = append(verdict.Reasons, storm)+ }+// EVERY direction is put to the declared polarity, including HELD. For a// certificate fault a target that carries on unchanged has ACCEPTED the// broken certificate, so "nothing looks wrong" is the finding.@@ -489,9 +568,121 @@ func leaked(observation Observation) []string {reasons = append(reasons, sprintf(reasonPermitted, monotonic))}+ if served := servedWhileUnknown(+ observation,+ ); served >= MinimumToProveOpen {+ reasons = append(+ reasons,+ sprintf(+ reasonServedWhileUnknown,+ served,+ orUnnamed(observation.ShiftWhat),+ observation.Shift.Valid.AcceptedSpanSinceLastRefusal(+ closedWhen(observation)...,+ ).Round(time.Second),+ ),+ )+ }+return reasons}+// retryStorm reports the target asking again about a credential the dependency+// has already definitively refused. It is a REASON rather than a direction: no+// credential got through, and what it tells an operator is that their caller+// will hammer a dependency with a key it has been told is dead.+//+// The comparison is the dependency's own refusal count against the number of+// requests khaos made with that credential. One refusal per request is a caller+// asking once; more than that is a caller asking twice about the same no.+func retryStorm(observation Observation) string {+ requests := int64(observation.Fault.Revoked.Countable())++ if requests == 0 || observation.Refusals <= requests {+ return ""+ }++ return sprintf(reasonRetriedRefused, observation.Refusals, requests)+}++// shiftReading says which cell of the two-phase matrix this run landed in. The+// two phases measure INDEPENDENT branches of the caller's error handling -- what+// it does with a definitive deny, and what it does with no answer at all -- so+// four readings exist and only one of them is a caller with three outcomes.+//+// Each phase is guarded by whether it was measurable, because the absence of an+// acceptance is only evidence of a refusal when the target answered.+func shiftReading(observation Observation) string {+ if observation.ShiftFailed {+ return reasonShiftUnarmed+ }++ if !observation.Shifted {+ return ""+ }++ if observation.Fault.Revoked.Answered() == 0 {+ return reasonDenyUnmeasured+ }++ // A DECISION, not merely an answer: a stream that was only ever redirected+ // or throttled decided nothing, and every sentence below reads the absence+ // of an acceptance as an observed refusal.+ if !observation.Fault.Revoked.Any(probe.Accepted) &&+ !observation.Fault.Revoked.AnyClosed(closedWhen(observation)...) {+ return reasonDenyUndecided+ }++ if observation.Shift.Valid.Answered() == 0 {+ return reasonUnknownUnmeasured+ }++ if shiftNotDelivered(observation) {+ return sprintf(reasonShiftNotDelivered, observation.DeliveredWhat)+ }++ what := orUnnamed(observation.ShiftWhat)++ honouredDeny := revokedHonoured(observation) < MinimumToProveOpen+ honouredUnknown := servedWhileUnknown(observation) < MinimumToProveOpen++ switch {+ case honouredDeny && honouredUnknown:+ return sprintf(reasonBothBranches, what)+ case honouredDeny:+ return sprintf(reasonOnlyDeny, what)+ case honouredUnknown:+ return sprintf(reasonOnlyUnknown, what)+ default:+ return sprintf(reasonNeitherBranch, what)+ }+}++// servedWhileUnknown is the second fault window's only finding: a credential+// accepted while the dependency could answer nothing at all. It counts the+// TRAILING run of acceptances rather than all of them, because the target was+// correctly serving that credential a moment earlier and a decision cache would+// put acceptances at the start of this window too.+func servedWhileUnknown(observation Observation) int {+ if !observation.Shifted || shiftNotDelivered(observation) {+ return 0+ }++ return observation.Shift.Valid.AcceptedSinceLastRefusal(+ closedWhen(observation)...,+ )+}++// shiftNotDelivered is faultNeverDelivered for the SECOND window, and it has to+// exist for the same reason: a target that answered every probe from a cached+// allow without asking its dependency once never met the fault, so it did not+// serve "requests whose authorization it could not have checked" -- it checked+// nothing. Window one aborts the run on that observation; reporting it as a+// conclusive defect here would be the same claim khaos refuses to make there.+func shiftNotDelivered(observation Observation) bool {+ return observation.DeliveredWhat != "" && observation.ShiftAsked == 0+}+// Leaking reports whether ANY stream found a bypass. The engine needs it// mid-run, because the idleness discriminator costs a traffic pause and there is// nothing to diagnose on a target that never leaked. It must cover every stream:@@ -557,7 +748,9 @@ func leakedMonotonic(observation Observation) int {// which is a defensible design. Counting every acceptance in the window reports// that as a bypass.func revokedHonoured(observation Observation) int {- return observation.Fault.Revoked.AcceptedDespiteLapse()+ return observation.Fault.Revoked.AcceptedDespiteLapse(+ closedWhen(observation)...,+ )}// diagnose separates the two causes of a permitted credential. Both look@@ -580,6 +773,19 @@ func diagnose(observation Observation, reasons []string) (Cause, []string) {return Undiagnosed, reasons}+ // Same guard for a leak carried ONLY by the second fault window. The pause+ // says nothing about it: that window measured a different fault, and the+ // cause would be read off the invalid stream, which never leaked, and come+ // back as a cache being re-armed by failed checks. Unreachable today,+ // because the one experiment with a second phase declares no pause -- and+ // this is the file where that stops being true by accident.+ if servedWhileUnknown(observation) > 0 &&+ revokedHonoured(observation) == 0 &&+ forgedAccepted(observation) == 0 &&+ leakedMonotonic(observation) == 0 {+ return Undiagnosed, reasons+ }+// The resumed window is read on whichever stream carried the leak. A revoked// credential and a never-valid one are different streams, and diagnosing the// wrong one would report the pause's effect on something that never leaked.@@ -705,7 +911,11 @@ func faultNeverDelivered(observation Observation) (string, bool) {return "", false}- return sprintf(reasonNeverDelivered, observation.DeliveredWhat), true+ return sprintf(+ reasonNeverDelivered,+ observation.DeliveredWhat,+ orUnnamed(observation.DeliveredHint),+ ), true}// cacheNotObserved reports that the target holds no cached answer, for andiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/authzfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/authzfault.gonew file mode 100644index 00000000..e007e10e--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/authzfault.go@@ -0,0 +1,149 @@+// ©AngelaMos | 2026+// authzfault.go++package inject++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"++ "github.com/CarterPerez-dev/khaos/internal/authz"+ "github.com/CarterPerez-dev/khaos/internal/dockerx"+ "github.com/CarterPerez-dev/khaos/internal/journal"+)++const (+ KindAuthzPrefix = "authz-"++ errNoAuthz = "%w: %s needs khaos's own authorization dependency, and this run did not start one"++ describeAuthz = "%s. khaos serves that dependency itself, on the address the target already asks, because no correct authorization service will answer a deny on request"+)++var ErrNoAuthz = errors.New(+ "this mutation needs khaos's authorization dependency, which is not running",+)++func AuthzKind(kind authz.Kind) string {+ return KindAuthzPrefix + string(kind)+}++// authzMutator leaves the dependency REACHABLE and answering. That is the whole+// tier: a paused container and a severed path both produce an absent answer, so+// a caller's error branch is the only one they can ever reach, and the defect+// this hunts lives in the branch that reads a definitive refusal.+type authzMutator struct {+ kind authz.Kind+}++func init() {+ for _, name := range authz.Injectable() {+ kind := authz.Kind(name)+ registry[AuthzKind(kind)] = authzMutator{kind: kind}+ }+}++func (m authzMutator) Describe(env Env, _ dockerx.Target) string {+ trap := env.AuthzTrap+ if trap.Kind != m.kind {+ trap = authz.Trap{Kind: m.kind}+ }++ return fmt.Sprintf(describeAuthz, trap.Describe())+}++// Plan journals the SHAPE of the fault and never the credential. The credential+// belongs to the operator's target, a journal entry would leave it on disk, and+// the revert does not need it: khaos's dependency allows everything it has not+// been told to deny, so lifting the fault is a reset rather than an undo.+func (m authzMutator) Plan(+ _ context.Context,+ env Env,+ _ dockerx.Target,+) (json.RawMessage, error) {+ if env.Authz == nil {+ return nil, fmt.Errorf(errNoAuthz, ErrNoAuthz, AuthzKind(m.kind))+ }++ if env.AuthzTrap.Kind != m.kind {+ return nil, fmt.Errorf(+ errWrongTrapKind,+ ErrTrapMismatch,+ env.AuthzTrap.Kind,+ m.kind,+ )+ }++ if err := env.AuthzTrap.Validate(); err != nil {+ return nil, err+ }++ return journal.EncodePayload(env.AuthzTrap)+}++func (m authzMutator) Apply(+ _ context.Context,+ env Env,+ _ dockerx.Target,+ _ json.RawMessage,+) error {+ if env.Authz == nil {+ return fmt.Errorf(errNoAuthz, ErrNoAuthz, AuthzKind(m.kind))+ }++ return env.Authz.Arm(env.AuthzTrap)+}++// Confirm reads the dependency back through its own HTTP surface against a+// control credential, never the flag that was set: a dependency whose handler+// was wrong would otherwise confirm a fault it is not serving.+func (m authzMutator) Confirm(+ ctx context.Context,+ env Env,+ _ dockerx.Target,+) (string, bool, error) {+ if env.Authz == nil {+ return "", false, fmt.Errorf(+ errNoAuthz,+ ErrNoAuthz,+ AuthzKind(m.kind),+ )+ }++ return env.Authz.Observe(ctx)+}++// Revert is a cold start's no-op by construction. khaos's dependency lives in+// khaos's process, so a killed run takes the fault with it and there is nothing+// outstanding for a reclaim to undo.+func (m authzMutator) Revert(+ _ context.Context,+ env Env,+ _ journal.Record,+) error {+ if env.Authz == nil {+ return nil+ }++ env.Authz.Disarm()++ return nil+}++// NeedsAuthz tells the run to stand the dependency up before the baseline,+// derived from the registry rather than from the kind's spelling.+func NeedsAuthz(kind string) bool {+ mutator, ok := registry[kind]+ if !ok {+ return false+ }++ _, isAuthz := mutator.(authzMutator)++ return isAuthz+}++// Prefix declares this tier's kinds to the registry.+func (authzMutator) Prefix() string { return KindAuthzPrefix }diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/authzfault_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/authzfault_test.gonew file mode 100644index 00000000..40f507f6--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/authzfault_test.go@@ -0,0 +1,301 @@+// ©AngelaMos | 2026+// authzfault_test.go++package inject++import (+ "context"+ "errors"+ "strings"+ "testing"++ "github.com/CarterPerez-dev/khaos/internal/authz"+ "github.com/CarterPerez-dev/khaos/internal/dockerx"+ "github.com/CarterPerez-dev/khaos/internal/journal"+)++const authzCredential = "the-credential-the-dependency-will-deny"++func dependency(t *testing.T) *authz.Dependency {+ t.Helper()++ served := authz.New()++ if err := served.Listen("127.0.0.1:0"); err != nil {+ t.Fatalf("listen: %v", err)+ }++ t.Cleanup(func() { _ = served.Close() })++ return served+}++// The census. A fault the catalogue can advertise and the engine cannot apply+// is the shape that shipped thirty-six unusable `pki-*` kinds.+func TestEveryAuthorizationFaultIsRegistered(t *testing.T) {+ registered := 0+ for _, kind := range Kinds() {+ if strings.HasPrefix(kind, KindAuthzPrefix) {+ registered+++ }+ }++ if registered != len(authz.Injectable()) {+ t.Fatalf(+ "%d authorization faults are registered and %d are injectable",+ registered,+ len(authz.Injectable()),+ )+ }++ for _, name := range authz.Injectable() {+ kind := AuthzKind(authz.Kind(name))++ if _, err := Lookup(kind); err != nil {+ t.Errorf("the engine cannot resolve %s: %v", kind, err)+ }++ if !NeedsAuthz(kind) {+ t.Errorf("%s does not ask the run for a dependency", kind)+ }++ if IsDirect(kind) {+ t.Errorf(+ "`khaos inject` offers %s, which needs a dependency it never starts",+ kind,+ )+ }+ }+}++// The whole mutation, end to end: journal it, arm it, confirm it by read-back+// on the dependency's own surface, revert it, and prove the dependency allows+// again afterwards, because the recovery window measures exactly that.+func TestTheAuthorizationFaultAppliesConfirmsAndReverts(t *testing.T) {+ served := dependency(t)++ trap := authz.Trap{Kind: authz.Deny, Credential: authzCredential}+ env := Env{Authz: served, AuthzTrap: trap}++ mutator, err := Lookup(AuthzKind(authz.Deny))+ if err != nil {+ t.Fatalf("lookup: %v", err)+ }++ payload, err := mutator.Plan(context.Background(), env, dockerx.Target{})+ if err != nil {+ t.Fatalf("plan: %v", err)+ }++ if _, confirmed, err := mutator.Confirm(+ context.Background(),+ env,+ dockerx.Target{},+ ); err != nil || confirmed {+ t.Fatalf(+ "the fault confirmed BEFORE it was applied (%v, %v), so the confirmation is not reading the dependency",+ confirmed,+ err,+ )+ }++ if err := mutator.Apply(+ context.Background(),+ env,+ dockerx.Target{},+ payload,+ ); err != nil {+ t.Fatalf("apply: %v", err)+ }++ detail, confirmed, err := mutator.Confirm(+ context.Background(),+ env,+ dockerx.Target{},+ )+ if err != nil || !confirmed {+ t.Fatalf("the applied fault did not confirm: %v, %v", confirmed, err)+ }++ if detail == "" {+ t.Fatal("the confirmation reported no observable")+ }++ if err := mutator.Revert(+ context.Background(),+ env,+ journal.Record{Payload: payload},+ ); err != nil {+ t.Fatalf("revert: %v", err)+ }++ if _, confirmed, err := mutator.Confirm(+ context.Background(),+ env,+ dockerx.Target{},+ ); err != nil || confirmed {+ t.Fatalf(+ "the fault still confirms after the revert (%v, %v)",+ confirmed,+ err,+ )+ }+}++// The credential belongs to the OPERATOR's target. A journal entry would leave+// it on disk after the run, and the revert does not need it: khaos's dependency+// allows everything it was not told to deny, so lifting the fault is a reset.+func TestTheJournalledFaultCarriesNoCredential(t *testing.T) {+ served := dependency(t)++ trap := authz.Trap{Kind: authz.Deny, Credential: authzCredential}+ env := Env{Authz: served, AuthzTrap: trap}++ mutator, err := Lookup(AuthzKind(authz.Deny))+ if err != nil {+ t.Fatalf("lookup: %v", err)+ }++ payload, err := mutator.Plan(context.Background(), env, dockerx.Target{})+ if err != nil {+ t.Fatalf("plan: %v", err)+ }++ if strings.Contains(string(payload), authzCredential) {+ t.Fatalf(+ "the journalled fault carries the credential, which would leave it on disk after the run: %s",+ payload,+ )+ }++ record := journal.Record{Payload: payload}++ var back authz.Trap+ if err := record.DecodePayload(&back); err != nil {+ t.Fatalf("decode: %v", err)+ }++ if back.Kind != authz.Deny {+ t.Fatalf("the journalled kind is %q, want %q", back.Kind, authz.Deny)+ }+}++// A mutation whose run never started a dependency refuses by name rather than+// arming nothing and reporting success.+func TestApplyingAnAuthorizationFaultWithNoDependencyIsRefused(t *testing.T) {+ mutator, err := Lookup(AuthzKind(authz.Deny))+ if err != nil {+ t.Fatalf("lookup: %v", err)+ }++ if _, err := mutator.Plan(+ context.Background(),+ Env{},+ dockerx.Target{},+ ); !errors.Is(err, ErrNoAuthz) {+ t.Fatalf("plan without a dependency returned %v, want ErrNoAuthz", err)+ }++ if err := mutator.Apply(+ context.Background(),+ Env{},+ dockerx.Target{},+ nil,+ ); !errors.Is(err, ErrNoAuthz) {+ t.Fatalf("apply without a dependency returned %v, want ErrNoAuthz", err)+ }++ if _, _, err := mutator.Confirm(+ context.Background(),+ Env{},+ dockerx.Target{},+ ); !errors.Is(err, ErrNoAuthz) {+ t.Fatalf(+ "confirm without a dependency returned %v, want ErrNoAuthz",+ err,+ )+ }+}++// A cold start has nothing to disarm: the dependency lived in the process that+// died, so the fault went with it. Reporting a failure there would leave a+// reclaim retrying something that cannot exist.+func TestRevertingWithoutADependencyIsResolvedRatherThanFailed(t *testing.T) {+ mutator, err := Lookup(AuthzKind(authz.Deny))+ if err != nil {+ t.Fatalf("lookup: %v", err)+ }++ if err := mutator.Revert(+ context.Background(),+ Env{},+ journal.Record{},+ ); err != nil {+ t.Fatalf("revert after a cold start returned %v, want nil", err)+ }+}++func TestTheAuthorizationMutatorRefusesATrapTheRunDidNotArm(t *testing.T) {+ served := dependency(t)++ mutator, err := Lookup(AuthzKind(authz.Deny))+ if err != nil {+ t.Fatalf("lookup: %v", err)+ }++ env := Env{+ Authz: served,+ AuthzTrap: authz.Trap{Kind: authz.Outage},+ }++ if _, err := mutator.Plan(+ context.Background(),+ env,+ dockerx.Target{},+ ); !errors.Is(err, ErrTrapMismatch) {+ t.Fatalf("plan with a mismatched trap returned %v", err)+ }+}++// Keyed off the registry rather than the kind's spelling, which is the shape+// that gave the proxy and identity tiers their own bugs: a predicate reading+// the name agrees with itself about a kind that does not exist.+func TestNeedsAuthzIsDerivedFromTheRegistry(t *testing.T) {+ if NeedsAuthz(KindAuthzPrefix + "not-a-real-fault") {+ t.Error(+ "NeedsAuthz answered yes for a kind no mutation is registered under",+ )+ }++ for _, kind := range Kinds() {+ if strings.HasPrefix(kind, KindAuthzPrefix) != NeedsAuthz(kind) {+ t.Errorf("%s: NeedsAuthz reports %v", kind, NeedsAuthz(kind))+ }+ }+}++func TestTheAuthorizationFaultDescribesItself(t *testing.T) {+ for _, name := range authz.Injectable() {+ mutator, err := Lookup(AuthzKind(authz.Kind(name)))+ if err != nil {+ t.Fatalf("lookup %s: %v", name, err)+ }++ described := mutator.Describe(Env{}, dockerx.Target{Name: "caller"})++ if described == "" {+ t.Fatalf("%s describes itself as nothing", name)+ }++ // The operator has to be told that khaos is standing in for their+ // dependency, because for the length of the run it allows every+ // credential it was not told to deny.+ if !strings.Contains(described, "khaos serves that dependency itself") {+ t.Fatalf(+ "%s does not say khaos is playing the dependency: %q",+ name,+ described,+ )+ }+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.goindex 6ffa7423..0bbd264b 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go@@ -5,15 +5,13 @@ package proxyimport ("errors"- "fmt""io""net"- "os"- "path/filepath"- "strconv""sync""sync/atomic""time"++ "github.com/CarterPerez-dev/khaos/internal/bindguard")const (@@ -30,9 +28,7 @@ const (settleWindow = 250 * time.Millisecond- errWildcard = "%w: %q binds every address on this host, and a fault proxy in front of a dependency under test must not be reachable off it; bind the Docker bridge gateway instead"- errPrivileged = "%w: %q is a privileged port"- errBadAddr = "%w: %q is not a host and port (%v)"+ subjectProxy = "a fault proxy in front of a dependency under test"probePayload = "GET / HTTP/1.1\r\nHost: khaos.confirm\r\nConnection: close\r\n\r\n"@@ -43,19 +39,13 @@ const (observedPartial = "%d bytes arrive through khaos's proxy where the dependency answers %d directly"observedSlowClose = "the close arrives %s after the last byte, which it does not do directly"errWrongArmed = "%w: the proxy holds %q but %q was expected"-- lowestUnprivilegedPort = 1024- ephemeralPort = 0-- sysClassNet = "/sys/class/net"- bridgeMarker = "bridge"-- errOffBox = "%w: %q is not on loopback or on a Docker bridge, so it is reachable from outside this host and a container cannot reach it anyway; bind the bridge gateway the daemon reports")var ErrClosed = errors.New("the fault proxy is closed")-var ErrUnsafeBind = errors.New("khaos will not bind the fault proxy there")+// ErrUnsafeBind is the shared sentinel. It stays spelled here because every+// caller and test in this tier asks about it by this name.+var ErrUnsafeBind = bindguard.ErrUnsafeBindtype armed struct {fault Fault@@ -88,7 +78,7 @@ type Proxy struct {// Open binds the proxy and refuses an address that would expose it.func Open(listenAddr, upstream string) (*Proxy, error) {- if err := safeBind(listenAddr); err != nil {+ if err := bindguard.Check(listenAddr, subjectProxy); err != nil {return nil, err}@@ -112,73 +102,6 @@ func Open(listenAddr, upstream string) (*Proxy, error) {return p, nil}-// safeBind tests the routing property rather than the address: loopback, or a-// bridge, identified by its sysfs `bridge` directory rather than by its name.-func safeBind(listenAddr string) error {- host, portText, err := net.SplitHostPort(listenAddr)- if err != nil {- return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)- }-- ip := net.ParseIP(host)- if ip == nil || ip.IsUnspecified() {- return fmt.Errorf(errWildcard, ErrUnsafeBind, listenAddr)- }-- port, err := strconv.Atoi(portText)- if err != nil {- return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)- }-- if port != ephemeralPort && port < lowestUnprivilegedPort {- return fmt.Errorf(errPrivileged, ErrUnsafeBind, listenAddr)- }-- if ip.IsLoopback() {- return nil- }-- onBridge, err := bridgeHosts(ip)- if err != nil {- return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)- }-- if !onBridge {- return fmt.Errorf(errOffBox, ErrUnsafeBind, listenAddr)- }-- return nil-}--func bridgeHosts(ip net.IP) (bool, error) {- interfaces, err := net.Interfaces()- if err != nil {- return false, err- }-- for _, iface := range interfaces {- addrs, err := iface.Addrs()- if err != nil {- continue- }-- for _, addr := range addrs {- network, ok := addr.(*net.IPNet)- if !ok || !network.IP.Equal(ip) {- continue- }-- _, err := os.Stat(- filepath.Join(sysClassNet, iface.Name, bridgeMarker),- )-- return err == nil, nil- }- }-- return false, nil-}-func (p *Proxy) Addr() string { return p.listener.Addr().String() }func (p *Proxy) Upstream() string { return p.upstream }diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.goindex d0793b1e..c4f54f30 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go@@ -16,6 +16,8 @@ import ("syscall""testing""time"++ "github.com/CarterPerez-dev/khaos/internal/bindguard")const (@@ -779,7 +781,11 @@ func TestOpenRefusesThisHostsRoutableAddress(t *testing.T) {for _, iface := range interfaces {if _, err := os.Stat(- filepath.Join(sysClassNet, iface.Name, bridgeMarker),+ filepath.Join(+ bindguard.SysClassNet,+ iface.Name,+ bindguard.BridgeMarker,+ ),); err == nil {continue}@@ -827,7 +833,11 @@ func TestOpenAcceptsABridgeAddress(t *testing.T) {for _, iface := range interfaces {if _, err := os.Stat(- filepath.Join(sysClassNet, iface.Name, bridgeMarker),+ filepath.Join(+ bindguard.SysClassNet,+ iface.Name,+ bindguard.BridgeMarker,+ ),); err != nil {continue}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.goindex 7973a665..355ed0a4 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go@@ -11,6 +11,7 @@ import ("slices""strings"+ "github.com/CarterPerez-dev/khaos/internal/authz""github.com/CarterPerez-dev/khaos/internal/dbx""github.com/CarterPerez-dev/khaos/internal/dnsrv""github.com/CarterPerez-dev/khaos/internal/dockerx"@@ -54,6 +55,12 @@ type Env struct {Authority *pki.AuthorityPKITrap pki.Trap+ // The authority khaos plays so that a dependency can answer a definitive+ // DENY while staying perfectly reachable, which no network fault can+ // produce and no correct dependency will do on request.+ Authz *authz.Dependency+ AuthzTrap authz.Trap+// The one authority NOT in khaos's process, so khaos holds a CLIENT for a// container. ResolverNetwork travels so a cold-start revert finds it.Resolver *dnsrv.Clientdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.goindex 7b42ff2c..a360ba14 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go@@ -62,14 +62,20 @@ func (s Stream) Distribution() map[Disposition]int {// AcceptedAfter counts only acceptances that follow a refusal, because an// acceptance before the target ever refused anything is a warm-up. The caller// seeds rejected from an earlier phase.-func (s Stream) AcceptedAfter(rejected bool) int {+//+// closed is what THIS control's failing closed looks like, and an empty set+// means Rejected alone. Every order-sensitive count below takes it for the same+// reason ClosedAfter does: a 503 is not a statement about a credential, but for+// a control that declared it, becoming unavailable IS the refusal, and a latch+// that only watches Rejected never trips against a target that answers 503.+func (s Stream) AcceptedAfter(rejected bool, closed ...Disposition) int {count := 0for _, outcome := range s.Outcomes {- switch outcome.Disposition {- case Rejected:+ switch {+ case shut(outcome.Disposition, closed):rejected = true- case Accepted:+ case outcome.Disposition == Accepted:if rejected {count++}@@ -81,6 +87,28 @@ func (s Stream) AcceptedAfter(rejected bool) int {return count}+// shut reads the declared set, defaulting to Rejected so a caller that names+// nothing keeps the meaning it had before the set existed.+func shut(disposition Disposition, closed []Disposition) bool {+ if len(closed) == 0 {+ return disposition == Rejected+ }++ return slices.Contains(closed, disposition)+}++// AnyClosed reports whether the stream ever failed closed, by this control's+// own definition of closed.+func (s Stream) AnyClosed(closed ...Disposition) bool {+ for _, outcome := range s.Outcomes {+ if shut(outcome.Disposition, closed) {+ return true+ }+ }++ return false+}+func (s Stream) AcceptedAfterRejected() int {return s.AcceptedAfter(false)}@@ -109,14 +137,14 @@ func (s Stream) AcceptedSpan() time.Duration {// AcceptedSinceLastRefusal is the trailing run of acceptances, so a bounded// cache that lapses and stays lapsed scores zero.-func (s Stream) AcceptedSinceLastRefusal() int {+func (s Stream) AcceptedSinceLastRefusal(closed ...Disposition) int {count := 0for _, outcome := range s.Outcomes {- switch outcome.Disposition {- case Rejected:+ switch {+ case shut(outcome.Disposition, closed):count = 0- case Accepted:+ case outcome.Disposition == Accepted:count++default:// Neither extends nor breaks the run.@@ -126,15 +154,46 @@ func (s Stream) AcceptedSinceLastRefusal() int {return count}+// AcceptedSpanSinceLastRefusal is the span of THAT run and not of every+// acceptance in the window, because the two describe different sets the moment+// a window contains a refusal in the middle: the count is order-sensitive and a+// first-to-last span is not, so quoting them in one sentence states an exposure+// over a period the target spent refusing.+func (s Stream) AcceptedSpanSinceLastRefusal(+ closed ...Disposition,+) time.Duration {+ var first, last time.Time++ for _, outcome := range s.Outcomes {+ switch {+ case shut(outcome.Disposition, closed):+ first, last = time.Time{}, time.Time{}+ case outcome.Disposition == Accepted:+ if first.IsZero() {+ first = outcome.At+ }+ last = outcome.At+ default:+ // Neither extends nor breaks the run.+ }+ }++ if first.IsZero() {+ return 0+ }++ return last.Sub(first)+}+// AcceptedDespiteLapse counts acceptances a correct bounded cache cannot// explain: all of them when the stream never lapsed, and only the ones// following a refusal when it did.-func (s Stream) AcceptedDespiteLapse() int {- if !s.Any(Rejected) {+func (s Stream) AcceptedDespiteLapse(closed ...Disposition) int {+ if !s.AnyClosed(closed...) {return s.Count(Accepted)}- return s.AcceptedAfter(false)+ return s.AcceptedAfter(false, closed...)}// RejectedAfter is the mirror of AcceptedAfter, for the credential that shoulddiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.goindex 73c18e80..1458d90e 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go@@ -61,6 +61,10 @@ const (SecretGatewayName = "khaos-range-gateway-secret"EnvGatewayName = "khaos-range-gateway-env"+ // The CALLER in the service-to-service experiment. It is the only fixture+ // whose dependency is not a container: khaos plays that itself.+ AuthzGatewayName = "khaos-range-gateway-authz"+LabelRange = "khaos.range"FailModeOpen = "open"FailModeShut = "closed"@@ -106,6 +110,23 @@ const (EnvSecretValue = "KHAOS_RANGE_SECRET_VALUE"EnvEnvPort = "KHAOS_RANGE_ENV_PORT"+ // The caller's own port, and the port khaos's authorization dependency is+ // addressed on. The second is read back out of the container rather than+ // remembered, like every other address a fixture was created with.+ EnvAuthzPort = "KHAOS_RANGE_AUTHZ_PORT"+ EnvAuthzDep = "KHAOS_RANGE_AUTHZ_DEP_PORT"+ EnvAuthzBase = "KHAOS_RANGE_AUTHZ_BASE"+ EnvAuthzMode = "KHAOS_RANGE_AUTHZ_MODE"++ // AuthzDistinguishes reads the dependency's three answers as three.+ // AuthzCollapses is research 07 section 6.2's shape: anything that is not+ // a 200 is read as an absence of an answer, and allowed. AuthzRetries+ // reaches the right verdict the wrong way, asking again every time it does+ // not like an answer, which is that section's retry storm.+ AuthzDistinguishes = "distinguishes"+ AuthzCollapses = "collapses"+ AuthzRetries = "retries"+// SecretVariable is the variable the environment faults take away, which// khaos names rather than discovers: the fixture is configured to read its// secret from exactly this one.@@ -210,6 +231,10 @@ const (errNoSecretGateway = "%w: this experiment breaks a secret FILE, and the running range has no %s; bring the range down and up again to create it"+ errNoAuthzGateway = "%w: this experiment answers a service's own authorization call, and the running range has no %s; bring the range down and up again to create it"++ errNoAuthzBase = "%w: this experiment answers a service's own authorization call, and %s does not carry %s, so khaos cannot tell where that service looks for its dependency; bring the range down and up again to recreate it"+errNoStatefulTier = "%w: this experiment acts on a real database engine, and the running range has none. khaos does not replace these dependencies with its own code, because the finding IS what three correct engines do, so the stack has to be brought up with them: `khaos range down && khaos range up --with-db`")@@ -241,6 +266,14 @@ type Lab struct {EnvPort intSecretValue string+ // AuthzPort is the caller's published port, AuthzDepPort is where that+ // caller was told to find its authorization dependency, and AuthzBase is+ // the URL it was created with, read back out of its own environment so+ // khaos binds where it is already being called.+ AuthzPort int+ AuthzDepPort int+ AuthzBase string+// Read back from the TLS gateway's own environment, never remembered.Dependency stringAuthorityBase string@@ -275,6 +308,8 @@ func Locate(ports Ports) (Lab, error) {DBGatewayPort: ports.DBGateway,SecretPort: ports.Secret,EnvPort: ports.Env,+ AuthzPort: ports.Authz,+ AuthzDepPort: ports.AuthzDep,}, nil}@@ -302,6 +337,13 @@ type Ports struct {DBGateway intSecret intEnv int++ // Two for the service-to-service tier: one the caller publishes, and one+ // khaos's own authorization dependency is reached on from inside the+ // range. The second is not published by anything, and it is still chosen+ // here because the caller's environment names it at creation.+ Authz int+ AuthzDep int}// FreePorts picks every host port the range needs in one go, all distinct.@@ -338,6 +380,8 @@ func FreePorts() (Ports, error) {&chosen.DBGateway,&chosen.Secret,&chosen.Env,+ &chosen.Authz,+ &chosen.AuthzDep,} {port, err := pick()if err != nil {@@ -464,7 +508,7 @@ func (l Lab) ProxiedProtectedURL() string {func (l Lab) Up(ctx context.Context,out io.Writer,- failMode, cacheTTL, tlsMode, poolMode, secretMode string,+ failMode, cacheTTL, tlsMode, poolMode, secretMode, authzMode string,withDB bool,) error {environ := append(os.Environ(),@@ -489,6 +533,9 @@ func (l Lab) Up(EnvSecretMode+"="+secretMode,EnvSecretValue+"="+l.SecretValue,EnvEnvPort+"="+strconv.Itoa(l.EnvPort),+ EnvAuthzPort+"="+strconv.Itoa(l.AuthzPort),+ EnvAuthzDep+"="+strconv.Itoa(l.AuthzDepPort),+ EnvAuthzMode+"="+authzMode,)args := []string{"up", "--build", "--detach", "--wait"}@@ -579,6 +626,8 @@ func (l Lab) Down(ctx context.Context, out io.Writer) error {EnvSecretPort+"="+strconv.Itoa(teardownPort),EnvSecretValue+"="+teardownPassword,EnvEnvPort+"="+strconv.Itoa(teardownPort),+ EnvAuthzPort+"="+strconv.Itoa(teardownPort),+ EnvAuthzDep+"="+strconv.Itoa(teardownPort),)// The profile is ALWAYS passed on the way down. Compose only removes the@@ -817,7 +866,7 @@ func Discover(ctx context.Context, reader PortReader) (Lab, error) {return lab, nil}-// readM9Fixtures reads the three gateways M9 added. Each one's absence is not+// readM9Fixtures reads the four gateways M9 added. Each one's absence is not// fatal, for the same reason as every other gateway: the experiments that do// not need it must still work.func (l *Lab) readM9Fixtures(@@ -829,6 +878,7 @@ func (l *Lab) readM9Fixtures(DBGatewayName: &l.DBGatewayPort,SecretGatewayName: &l.SecretPort,EnvGatewayName: &l.EnvPort,+ AuthzGatewayName: &l.AuthzPort,} {gateway, ok := running[name]if !ok {@@ -843,9 +893,97 @@ func (l *Lab) readM9Fixtures(*into = port}+ caller, ok := running[AuthzGatewayName]+ if !ok {+ return nil+ }++ environ, err := reader.ContainerEnv(ctx, caller.ID)+ if err != nil {+ return err+ }++ // The address the CALLER was created with, which is the one khaos has to+ // bind. A container's environment is fixed at creation, so this is read+ // back rather than recomputed from whatever ports this khaos chose.+ //+ // A container of that name carrying no such address is the tier being+ // ABSENT rather than an error, and that distinction is load-bearing: this+ // runs inside Discover, which every experiment calls, so returning an+ // error here would take every unrelated run down over one stale container.+ // RequireAuthzTier refuses by name for the experiments that need it.+ l.AuthzBase = environ[EnvAuthzBase]+return nil}+func (l Lab) AuthzGatewayURL() string {+ return "http://" + net.JoinHostPort(loopback, strconv.Itoa(l.AuthzPort))+}++func (l Lab) AuthzProtectedURL() string {+ return l.AuthzGatewayURL() + protectedPath+}++// IssueAuthzToken asks the caller for a credential it will AUTHENTICATE. The+// fixture authenticates locally and correctly in all three of its modes, which is+// what gives the experiment a baseline: a service that allows on any non-200+// from its authorization dependency refuses nothing at all, and khaos's own+// guard would void the run rather than measure one.+func (l Lab) IssueAuthzToken(ctx context.Context) (string, error) {+ return l.issueFrom(ctx, l.AuthzGatewayURL()+tokenPath)+}++// RequireAuthzTier separates the two ways this tier can be missing, because+// they need different fixes: no caller at all means bring the range up, and a+// caller carrying no dependency address means that container predates the+// variable and has to be recreated.+func (l Lab) RequireAuthzTier() error {+ if l.AuthzPort == 0 {+ return fmt.Errorf(+ errNoAuthzGateway,+ ErrRangeIncomplete,+ AuthzGatewayName,+ )+ }++ if l.AuthzBase == "" {+ return fmt.Errorf(+ errNoAuthzBase,+ ErrRangeIncomplete,+ AuthzGatewayName,+ EnvAuthzBase,+ )+ }++ return nil+}++// AuthzListenAddr is where khaos binds the dependency the caller already asks.+// It is the DEFAULT bridge's gateway for the same measured reason as every+// other in-process tier: a listener on the range network's own gateway is+// unreachable from a container on it, and a container's loopback is its own.+func (l Lab) AuthzListenAddr(+ ctx context.Context,+ reader PortReader,+) (string, error) {+ if err := l.RequireAuthzTier(); err != nil {+ return "", err+ }++ gatewayIP, err := reader.BridgeGateway(ctx)+ if err != nil {+ return "", err+ }++ port, err := portOfURL(l.AuthzBase)+ if err != nil {+ return "", err+ }++ return net.JoinHostPort(gatewayIP, strconv.Itoa(port)), nil+}+func (l Lab) EnvGatewayURL() string {return "http://" + net.JoinHostPort(loopback, strconv.Itoa(l.EnvPort))}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.ymlindex 3c70929d..3b8e8855 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml@@ -560,6 +560,46 @@ services:retries: 15restart: "no"+ # The service-to-service caller, and the only fixture whose dependency is not+ # in this file at all: khaos answers its authorization calls from the host.+ # That is the whole tier. No correct authorization service will answer a+ # definitive 403 on request, and a definitive 403 with full reachability is+ # exactly what no network fault can produce.+ #+ # It authenticates every credential itself, locally and correctly, in all+ # three modes. Without that a collapsing caller refuses nothing at all and+ # khaos's own baseline guard would void the run instead of measuring it.+ gateway-authz:+ container_name: khaos-range-gateway-authz+ build:+ context: ..+ dockerfile: range/Dockerfile+ image: khaos-range:latest+ command: ["authz"]+ environment:+ KHAOS_RANGE_AUTHZ_BASE: "http://host.docker.internal:${KHAOS_RANGE_AUTHZ_DEP_PORT:?khaos chooses this port; run `khaos range up`}"+ KHAOS_RANGE_AUTHZ_MODE: "${KHAOS_RANGE_AUTHZ_MODE:-collapses}"+ # host-gateway is the DEFAULT bridge's gateway, measured, not this network's+ # gateway: a listener on this network's own gateway address is unreachable.+ extra_hosts:+ - "host.docker.internal:host-gateway"+ labels:+ khaos.target: "true"+ khaos.range: "true"+ ports:+ - "127.0.0.1:${KHAOS_RANGE_AUTHZ_PORT:?khaos chooses this port; run `khaos range up`}:8080"+ networks:+ range:+ ipv4_address: 10.244.244.18+ # PROCESS liveness, not dependency liveness: /health would make the fixture+ # unstartable until khaos is listening, and khaos only listens during a run.+ healthcheck:+ test: ["CMD", "/usr/local/bin/khaos-range", "alivecheck"]+ interval: 2s+ timeout: 2s+ retries: 15+ restart: "no"+networks:range:name: khaos-rangediff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.shindex 3453de9a..b1715aba 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh@@ -177,8 +177,13 @@ assert_report_lacks() {# reporting a finding, and this asserts the run was NOT voided that way.assert_control_held() {local label="$1"+ # The subject is a parameter because this one helper now serves two kinds of+ # target. For ten experiments it is an engine checking a password; for the+ # service-to-service one it is a caller authenticating a credential, and+ # that distinction is the whole reason the fixture is measurable at all.+ local subject="${2:-the engine checks passwords}"assert_report_lacks \- "${label} established that the engine checks passwords at all" \+ "${label} established that ${subject} at all" \"the invalid credential was never rejected"}@@ -824,7 +829,162 @@ if [[ "${#differential_env[@]}" == "${#ENGINES[@]}" ]]; thenfifi-echo "== census: every catalogued credential.db_ experiment was run here =="+echo "== service to service: a DENY and a SILENCE are different answers =="++# The one fault in this family whose authority is khaos itself again, and the+# only one that needs TWO fault windows. Research 07 section 6.2: an+# authorization dependency has three outcomes - allow, deny and unknown - and+# only `unknown` is eligible for a fallback policy. A caller written as+#+# if err != nil || resp.StatusCode != 200 { return Allow }+#+# has two branches where it needs three, and one window cannot show that: the+# acceptance under a deny is already a finding, but what PROVES the mechanism is+# that the same caller behaves identically when the dependency answers nothing+# at all. So khaos denies one credential, measures, then makes its own+# dependency unavailable, measures again, and reads the pair.+#+# Each phase measures an INDEPENDENT branch, so all four cells of that matrix+# exist and three of the modes below are three of them. `collapses` ignores the+# deny, `distinguishes` honours both, and `retries` honours both and asks again+# every time, which is the same section's retry storm.+#+# This tier needs no stateful tier: it is in the DEFAULT range, and that is+# asserted by bringing the range up without `--with-db`.+start_authz_range() {+ local mode="$1"++ "$BIN" range down --no-color >/dev/null 2>&1 || true+ if ! "$BIN" range up --authz-mode "$mode" --no-color --quiet \+ >/dev/null 2>&1; then+ fail "range up --authz-mode $mode"+ return 1+ fi+ return 0+}++declare -A WANT_AUTHZ=(+ [collapses]="deviated / PERMITTED"+ [distinguishes]="passed / HELD"+ [retries]="passed / HELD"+)++declare -A WANT_AUTHZ_EXIT=(+ [collapses]=2+ [distinguishes]=0+ [retries]=0+)++for mode in collapses distinguishes retries; do+ if ! start_authz_range "$mode"; then+ continue+ fi++ run_experiment credential.apikey_revoked++ label="apikey_revoked against a ${mode} caller"++ assert_verdict "$label" "${WANT_AUTHZ[$mode]}"+ assert_exit "$label" "${WANT_AUTHZ_EXIT[$mode]}"+ assert_confirmed "$label"+ assert_control_held "$label" "the caller authenticates credentials"++ # The confirmation is TWO-SIDED, and both halves are quoted: the credential+ # under test is refused AND a control credential khaos never named is still+ # allowed. Without the second half a dependency that had started refusing+ # everything would confirm every deny khaos armed, and this experiment+ # reports a refusal of the valid credential as a defect of the target.+ assert_report_has \+ "${label}: the confirmation reads both sides of the deny" \+ "403 for the credential under test and 200 for a control credential"++ # The SECOND WINDOW has to have run. Without this the gate would pass on+ # phase one alone: the collapsing caller's acceptance of a denied+ # credential is a finding by itself, so deleting the shift entirely would+ # leave the verdict and the exit code unchanged.+ assert_report_has \+ "${label}: khaos measured a second window" \+ "plan shift"+ assert_report_has \+ "${label}: the second window is named in the evidence" \+ "answered 503 to everything"+done++echo "== the two phases are read as four callers, not two =="++# The discriminator, stated as the gate's own comparison. The verdicts above+# already differ; what these assert is that khaos SAYS which caller it saw,+# because "deviated" with no mechanism is a finding an operator cannot act on.+if start_authz_range collapses; then+ run_experiment credential.apikey_revoked++ assert_report_has \+ "the collapse is named as serving in both phases" \+ "the target served in BOTH phases"+ assert_report_has \+ "and it is named as two branches where three are needed" \+ "two branches where it needs three"+ assert_report_lacks \+ "a collapsing caller is not credited with the third branch" \+ "That is the third branch"+fi++if start_authz_range distinguishes; then+ run_experiment credential.apikey_revoked++ assert_report_has \+ "the correct caller is credited with the third branch, observed" \+ "That is the third branch, and khaos observed it rather than inferring it"+ assert_report_lacks \+ "and is not reported as having collapsed anything" \+ "served in BOTH phases"++ # The negative control for the retry-storm instrument. A caller that asks+ # once per request must NOT be reported as retrying, or the instrument says+ # yes to everything and the row below proves nothing.+ assert_report_lacks \+ "a caller that asked once per request is not called a retry storm" \+ "asked again about a credential it had already been told no about"+fi++if start_authz_range retries; then+ run_experiment credential.apikey_revoked++ # Measured from OUTSIDE the target, by counting what khaos's own dependency+ # answered: only the credential under test is ever answered with a+ # definitive refusal, so that count is the number of times the caller asked+ # about a credential it had already been told no about.+ assert_report_has \+ "the retry storm is counted against the requests that caused it" \+ "asked again about a credential it had already been told no about"++ # And it still fails CLOSED, which is why this is a third mode rather than+ # a variant of the collapse: the right verdict reached the wrong way.+ assert_report_has \+ "a retrying caller is still credited with both branches" \+ "That is the third branch"+fi++echo "== the authorization fixture is refused by name when it is not running =="++# The refusal has to arrive with a range UP and this one fixture stopped, which+# is the state an operator reaches when a container has died. A missing fixture+# otherwise surfaces as a published port of zero, and the run blames the target+# for not enforcing anything.+if start_authz_range distinguishes; then+ docker stop khaos-range-gateway-authz >/dev/null 2>&1 || true++ run_unmeasured credential.apikey_revoked++ assert_exit "an authorization experiment with no caller fixture" 1+ assert_report_has \+ "the refusal names the missing fixture" \+ "the running range has no khaos-range-gateway-authz"++ docker start khaos-range-gateway-authz >/dev/null 2>&1 || true+fi++echo "== census: every catalogued credential. experiment was run here =="catalogued="$(NO_COLOR=1 "$BIN" run --help |grep -oE '^ credential\.[a-z_]+' | tr -d ' ' | sort -u || true)"