Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(khaos): M9's credential. family, on the first authority that behaves correctly
CarterPerez-dev3880c26a9f98
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/go.mod b/PROJECTS/intermediate/chaos-engineering-tool/go.modindex bc26108e..edcf9023 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/go.mod+++ b/PROJECTS/intermediate/chaos-engineering-tool/go.mod@@ -6,6 +6,8 @@ require (charm.land/lipgloss/v2 v2.0.6github.com/charmbracelet/colorprofile v0.4.3github.com/containerd/errdefs v1.0.0+ github.com/go-sql-driver/mysql v1.10.1+ github.com/lib/pq v1.12.3github.com/miekg/dns v1.1.73github.com/moby/moby/api v1.56.1github.com/moby/moby/client v0.6.1@@ -18,6 +20,7 @@ require ()require (+ filippo.io/edwards25519 v1.2.0 // indirectgithub.com/Microsoft/go-winio v0.6.2 // indirectgithub.com/carapace-sh/carapace v1.11.6 // indirectgithub.com/carapace-sh/carapace-shlex v1.1.1 // indirectdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/go.sum b/PROJECTS/intermediate/chaos-engineering-tool/go.sumindex a49d2a67..d628e028 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/go.sum+++ b/PROJECTS/intermediate/chaos-engineering-tool/go.sum@@ -1,5 +1,7 @@charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ=charm.land/lipgloss/v2 v2.0.6/go.mod h1:ipDDJNSGa1hlwDtSfW1s2/xR8Vdhbut4PXh2zEKZd0Q=+filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=+filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=github.com/carapace-sh/carapace v1.11.6 h1:fUZv+oAMgbiDEpNPNis4n35tzqE3h8yshOohLJ2Mz4Y=@@ -46,6 +48,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=+github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=+github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=@@ -60,6 +64,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=+github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=+github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss=github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.goindex e08674fc..eb71a28d 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go@@ -16,6 +16,7 @@ import ("strings""time"+ "github.com/CarterPerez-dev/khaos/internal/dbx""github.com/CarterPerez-dev/khaos/internal/dnsrv""github.com/CarterPerez-dev/khaos/internal/dockerx""github.com/CarterPerez-dev/khaos/internal/experiment"@@ -116,6 +117,12 @@ enforcing in the first place.`errNoResolverOn = "no khaos chaos resolver is serving %s. The DNS family needs one, and the target has to have been CREATED pointing at it: `--dns` is fixed at container creation and Docker offers no way to change it afterwards. Run `khaos resolver up --network %[1]s` and recreate the target against the address it prints, or `khaos range up` for a stack that is already wired: %w"errResolverSilent = "khaos's chaos resolver %s did not answer its control surface at %s: %w"+ credTierFormat = "credential tier open on %s at %s over %s, which reports %q, acting on the role %s"++ errNoAdminSecret = "%s does not carry %s in its environment, so khaos has no administrative credential for the %s and will not guess one. khaos reads it from the container rather than taking it on a flag, because a flag is world-readable in /proc and a journal entry would outlive the run. Point --%s at the variable that holds it, or run `khaos range up --with-db` for a stack that is already wired"++ errNoSubjectSecret = "%s does not carry %s in its environment, so khaos cannot read the credential its own connection pool authenticated with, and a rotation it cannot undo is not one khaos will perform"+errTargetOffNetwork = "%s is not attached to %s, so it cannot be using the chaos resolver that serves it and khaos could not count its queries: %w"errProxyBind = "khaos could not bind its fault proxy on %s, which is where the range's proxied gateway is already looking for it: %w. Something else took that port since the range came up; bring the range down and up again to pick a new one"msgProxyUp = "khaos's fault proxy is listening on %s, forwarding to %s"@@ -741,6 +748,7 @@ func rangeSupplies() []string {experiment.ParamDNSHost,experiment.ParamResolverNetwork,experiment.ParamDNSAddress,+ experiment.ParamEngine,}}@@ -754,6 +762,7 @@ const (tierIdPtierTLStierDNS+ tierCred)func tierOf(kind string) tier {@@ -766,6 +775,8 @@ func tierOf(kind string) tier {return tierTLScase inject.NeedsResolver(kind):return tierDNS+ case inject.NeedsCred(kind):+ return tierCreddefault:return tierDirect}@@ -795,6 +806,7 @@ func rangeDefaults() map[string]string {url := lab.ProtectedURL()target := rangelab.IDPName+ engine := ""switch at {case tierProxy:@@ -813,13 +825,22 @@ func rangeDefaults(url = lab.DNSProtectedURL()target = rangelab.DNSGatewayName+ // The target IS the engine here. Nothing of the operator's is mutated: the+ // fault acts on a role khaos mints on that engine for the run. PostgreSQL+ // is the zero-flag default because its own image carries the loopback+ // `trust` rule that research 07 section 6.1 measured a probe through, so it+ // is the engine where khaos's negative control earns its keep.+ case tierCred:+ url = ""+ target = rangelab.PostgresName+ engine = string(dbx.Postgres)+default:// The direct tier's URL and target are the values set above.}- return map[string]string{+ defaults := map[string]string{experiment.ParamTarget: target,- experiment.ParamURL: url,experiment.ParamToken: answers.Token,experiment.ParamRevocable: answers.Revocable,experiment.ParamAuthority: answers.Authority,@@ -840,6 +861,21 @@ func rangeDefaults(// every credential; an unroutable address would only be another outage.experiment.ParamDNSAddress: rangelab.RogueAddress,}++ // Two answers are TIER-CONDITIONAL and are left out rather than filled with+ // an empty string. The credential tier has no url because its probes are+ // database connections, and no other tier has an engine. An empty answer+ // would read as "the range claims to answer this and cannot", which is the+ // shape that sends an operator looking for a flag that does nothing.+ if url != "" {+ defaults[experiment.ParamURL] = url+ }++ if engine != "" {+ defaults[experiment.ParamEngine] = engine+ }++ return defaults}// fillFromRange fills every unsupplied parameter from a running range, all of@@ -892,6 +928,12 @@ func fillFromRange()}+ if at == tierCred {+ if err := lab.RequireStatefulTier(); err != nil {+ return nil, err+ }+ }+answers, err := askTheRange(ctx,reader,@@ -1048,6 +1090,9 @@ func startProvider(// standUpTiers starts whichever in-process tiers this experiment needs. Every// one must answer before the BASELINE is measured through it, or the difference// between the phases is khaos rather than the fault.+// standUpTiers opens whichever authorities this experiment declares. Each tier+// has the same shape — stand it up, hand back the closer — so a sixth tier adds+// one branch rather than another paragraph of lifecycle.func standUpTiers(ctx context.Context,out io.Writer,@@ -1065,69 +1110,253 @@ func standUpTiers(}}- if spec.ProxyFault != nil {- fault, err := startProxy(ctx, out, work.Docker, *spec)+ tiers := []func() (func(), error){+ func() (func(), error) {+ return standUpProxyTier(ctx, out, work.Docker, spec, &env)+ },+ func() (func(), error) {+ return standUpIdPTier(ctx, out, work.Docker, spec, &env)+ },+ func() (func(), error) {+ return standUpPKITier(out, spec, &env)+ },+ func() (func(), error) {+ return standUpDNSTier(ctx, out, work.Docker, spec, target, &env)+ },+ func() (func(), error) {+ return standUpCredTier(ctx, out, work.Docker, spec, target, &env)+ },+ }++ for _, stand := range tiers {+ closer, err := stand()if err != nil {return env, closeAll, err}+ if closer != nil {+ closers = append(closers, closer)+ }+ }- closers = append(closers, func() { _ = fault.Close() })+ return env, closeAll, nil+}- env.Proxy = fault- env.ProxyFault = *spec.ProxyFault+func standUpProxyTier(+ ctx context.Context,+ out io.Writer,+ docker *dockerx.Client,+ spec *experiment.Experiment,+ env *inject.Env,+) (func(), error) {+ if spec.ProxyFault == nil {+ return nil, nil}- if spec.IdPTrap != nil {- if err := startProvider(ctx, out, work.Docker, *spec); err != nil {- return env, closeAll, err- }+ fault, err := startProxy(ctx, out, docker, *spec)+ if err != nil {+ return nil, err+ }++ env.Proxy = fault+ env.ProxyFault = *spec.ProxyFault- closers = append(closers, func() { _ = spec.IdP.Close() })+ return func() { _ = fault.Close() }, nil+}- env.IdP = spec.IdP- env.IdPTrap = *spec.IdPTrap+func standUpIdPTier(+ ctx context.Context,+ out io.Writer,+ docker *dockerx.Client,+ spec *experiment.Experiment,+ env *inject.Env,+) (func(), error) {+ if spec.IdPTrap == nil {+ return nil, nil}- if spec.PKITrap != nil {- if err := startAuthority(out, *spec); err != nil {- return env, closeAll, err- }+ if err := startProvider(ctx, out, docker, *spec); err != nil {+ return nil, err+ }++ env.IdP = spec.IdP+ env.IdPTrap = *spec.IdPTrap++ return func() { _ = spec.IdP.Close() }, nil+}- closers = append(closers, func() { _ = spec.Authority.Close() })+func standUpPKITier(+ out io.Writer,+ spec *experiment.Experiment,+ env *inject.Env,+) (func(), error) {+ if spec.PKITrap == nil {+ return nil, nil+ }- env.Authority = spec.Authority- env.PKITrap = *spec.PKITrap+ if err := startAuthority(out, *spec); err != nil {+ return nil, err}- // ATTACHED, not started: `--dns` is fixed at container creation, so a- // resolver started now would be one nothing is pointed at.- if spec.DNSTrap != nil {- client, err := attachResolver(ctx, out, work.Docker, *spec)- if err != nil {- return env, closeAll, err- }+ env.Authority = spec.Authority+ env.PKITrap = *spec.PKITrap- if err := wireArrivalGuard(- ctx,- work.Docker,- spec,- client,- target,- ); err != nil {- return env, closeAll, err+ return func() { _ = spec.Authority.Close() }, nil+}++// standUpDNSTier ATTACHES rather than starts: `--dns` is fixed at container+// creation, so a resolver started now would be one nothing is pointed at.+func standUpDNSTier(+ ctx context.Context,+ out io.Writer,+ docker *dockerx.Client,+ spec *experiment.Experiment,+ target dockerx.Target,+ env *inject.Env,+) (func(), error) {+ if spec.DNSTrap == nil {+ return nil, nil+ }++ client, err := attachResolver(ctx, out, docker, *spec)+ if err != nil {+ return nil, err+ }++ if err := wireArrivalGuard(ctx, docker, spec, client, target); err != nil {+ return nil, err+ }++ env.Resolver = client+ env.ResolverNetwork = spec.ResolverNetwork+ env.DNSTrap = *spec.DNSTrap++ // This resolver outlives the khaos process; armed, it would stay armed.+ return func() {+ _, _ = client.Disarm(context.WithoutCancel(ctx))+ }, nil+}++// standUpCredTier opens the tier and hands back the closer that removes the+// role khaos minted, so a minted role cannot outlive the run even when the+// revert already restored the credential.+func standUpCredTier(+ ctx context.Context,+ out io.Writer,+ docker *dockerx.Client,+ spec *experiment.Experiment,+ target dockerx.Target,+ env *inject.Env,+) (func(), error) {+ if spec.Cred == nil {+ return nil, nil+ }++ plan, err := openCredTier(ctx, out, docker, *spec, target)+ if err != nil {+ return nil, err+ }++ env.Cred = spec.Cred.Tier+ env.CredTrap = spec.Cred.Tier.Trap()+ env.CredPlan = plan++ return func() {+ _ = spec.Cred.Tier.Close(context.WithoutCancel(ctx))+ }, nil+}++// openCredTier reads the administrative credential out of the TARGET's own+// environment and dials the engine on the container's own address. Neither the+// credential nor a published port is involved: the first would outlive the run+// in a journal or a command line, and the second would need the operator to+// publish a database to the host.+func openCredTier(+ ctx context.Context,+ out io.Writer,+ docker *dockerx.Client,+ spec experiment.Experiment,+ target dockerx.Target,+) (inject.CredPlan, error) {+ cred := spec.Cred++ environ, err := docker.ContainerEnv(ctx, target.ID)+ if err != nil {+ return inject.CredPlan{}, err+ }++ password, found := environ[cred.AdminVar]+ if !found || password == "" {+ return inject.CredPlan{}, fmt.Errorf(+ errNoAdminSecret,+ target.Name,+ cred.AdminVar,+ cred.Engine,+ experiment.ParamAdminVar,+ )+ }++ network, address, err := docker.OwnAddress(ctx, target.ID)+ if err != nil {+ return inject.CredPlan{}, err+ }++ config := dbx.Config{+ Engine: cred.Engine,+ Address: net.JoinHostPort(address, cred.Port),+ Database: cred.Database,+ Admin: dbx.Credential{Role: cred.Admin, Password: password},+ }++ subject := dbx.Credential{}++ plan := inject.CredPlan{+ Engine: cred.Engine,+ Address: config.Address,+ Database: cred.Database,+ Admin: inject.CredSource{+ ContainerID: target.ID,+ Role: cred.Admin,+ PasswordVar: cred.AdminVar,+ },+ }++ if cred.Subject != "" {+ value, ok := environ[cred.SubjectVar]+ if !ok || value == "" {+ return inject.CredPlan{}, fmt.Errorf(+ errNoSubjectSecret,+ target.Name,+ cred.SubjectVar,+ )}- // This resolver outlives the khaos process; armed, it would stay armed.- closers = append(closers, func() {- _, _ = client.Disarm(context.WithoutCancel(ctx))- })+ subject = dbx.Credential{Role: cred.Subject, Password: value}+ plan.Subject = &inject.CredSource{+ ContainerID: target.ID,+ Role: cred.Subject,+ PasswordVar: cred.SubjectVar,+ }+ }- env.Resolver = client- env.ResolverNetwork = spec.ResolverNetwork- env.DNSTrap = *spec.DNSTrap+ if err := cred.Tier.Open(ctx, config, subject); err != nil {+ return inject.CredPlan{}, err}- return env, closeAll, nil+ event(+ out,+ markPlan,+ theme.TextFaint,+ fmt.Sprintf(+ credTierFormat,+ cred.Engine,+ config.Address,+ network,+ cred.Tier.Version(),+ cred.Tier.Role(),+ ),+ )++ return plan, nil}// wireArrivalGuard counts the TARGET's own queries for the name under fault,diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.goindex 51771890..51b50ecb 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go@@ -131,17 +131,45 @@ func outage(t *testing.T) experiment.Definition {// decides whether khaos looks for a range at all. Keeping it in a second place// is only safe if the two cannot drift.func TestRangeSuppliesIsExactlyTheKeySetOfRangeDefaults(t *testing.T) {+ union := map[string]bool{}+for _, at := range allTiers() {t.Run(tierName(at), func(t *testing.T) {- assertKeySet(t, rangeDefaults(fullLab(), testAnswers, at))+ for name := range assertKeySet(+ t,+ rangeDefaults(fullLab(), testAnswers, at),+ ) {+ union[name] = true+ }})}++ // Per tier the answers are a SUBSET, because two of them are+ // tier-conditional: the credential tier has no url and no other tier has an+ // engine. Across every tier the union has to be exact, or khaos would+ // refuse a parameter it can fill or try to fill one it cannot.+ supplies := slices.Clone(rangeSupplies())+ slices.Sort(supplies)++ answered := make([]string, 0, len(union))+ for name := range union {+ answered = append(answered, name)+ }+ slices.Sort(answered)++ if !slices.Equal(supplies, answered) {+ t.Fatalf(+ "rangeSupplies() = %v but no tier answers %v",+ supplies,+ answered,+ )+ }}// allTiers is every tier the range serves. A test that iterated only the two it// knew about would go on passing while a whole gateway's answers drifted.func allTiers() []tier {- return []tier{tierDirect, tierProxy, tierIdP, tierTLS, tierDNS}+ return []tier{tierDirect, tierProxy, tierIdP, tierTLS, tierDNS, tierCred}}func tierName(at tier) string {@@ -154,6 +182,8 @@ func tierName(at tier) string {return "against khaos's own certificate authority"case tierDNS:return "through khaos's own chaos resolver"+ case tierCred:+ return "over the engine's own administrative protocol"default:return "direct"}@@ -179,13 +209,14 @@ func fullLab() rangelab.Lab {}}-func assertKeySet(t *testing.T, defaults map[string]string) {+// assertKeySet checks one tier and returns the names it answered, so the caller+// can require the union across tiers to be exact.+func assertKeySet(t *testing.T, defaults map[string]string) map[string]bool {t.Helper()- supplies := slices.Clone(rangeSupplies())- slices.Sort(supplies)+ supplies := rangeSupplies()+ answered := map[string]bool{}- keys := make([]string, 0, len(defaults))for name, value := range defaults {if value == "" {t.Fatalf(@@ -193,18 +224,18 @@ func assertKeySet(t *testing.T, defaults map[string]string) {name,)}- keys = append(keys, name)- }- slices.Sort(keys)- if !slices.Equal(supplies, keys) {- t.Fatalf(- "rangeSupplies() = %v but rangeDefaults answers %v; khaos would either refuse a "+- "parameter it can fill or try to fill one it cannot",- supplies,- keys,- )+ if !slices.Contains(supplies, name) {+ t.Fatalf(+ "rangeDefaults answers %q, which rangeSupplies() does not list, so khaos would try to fill a parameter it does not believe it can",+ name,+ )+ }++ answered[name] = true}++ return answered}// The M4 defect, generalised. Fusing the operator's target with the range's URL@@ -729,12 +760,17 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects() {lab := fullLab()+ // The credential tier has NO url, and that is a property rather than an+ // omission: its probe streams are database connections, so there is no+ // gateway in front of the dependency and nothing to address over HTTP. A+ // url filled in here would be a flag the operator can set to no effect.want := map[tier]string{tierDirect: lab.ProtectedURL(),tierProxy: lab.ProxiedProtectedURL(),tierIdP: lab.VerifierProtectedURL(),tierTLS: lab.TLSProtectedURL(),tierDNS: lab.DNSProtectedURL(),+ tierCred: "",}seen := map[string]tier{}@@ -783,6 +819,13 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(wantTarget = rangelab.TLSGatewayNamecase tierDNS:wantTarget = rangelab.DNSGatewayName++ // The one tier whose target IS the dependency, because khaos does+ // not replace this authority: the fault is performed on the engine+ // itself, on a role khaos mints there for the run.+ case tierCred:+ wantTarget = rangelab.PostgresName+default:// The direct and proxy tiers fault the dependency itself.}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.goindex 37aaf611..edea41df 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go@@ -30,6 +30,7 @@ const (flagTLSMode = "tls-mode"flagPort = "port"flagQuiet = "quiet"+ flagWithDB = "with-db"rangeShort = "bring up a target stack that is deliberately breakable"rangeLong = `The range is a small stack khaos ships so you have something to break before you@@ -54,6 +55,7 @@ running.`tlsModeUsage = "which reference TLS client the range's TLS gateway runs. `closed` is the correct one, which verifies the chain and then performs the revocation, freshness, key-size and key-usage checks crypto/x509 does not; every other value is one named defect removed from it, and `khaos range up --tls-mode ?` lists them. An unrecognised value is refused rather than quietly leaving the gateway correct, because a gateway that silently became correct would report every certificate experiment as fail-closed"portUsage = "host port for the gateway; the default is a random high port chosen at startup"quietUsage = "suppress compose build and start output"+ withDBUsage = "also start the stateful tier: PostgreSQL 18, MySQL 8.4, MariaDB 11.4 and a Redis for the rate limiter's datastore. These four are the only range services khaos does not implement itself, because the credential family's finding IS what three correct engines do, and they cost real startup time, so they are opt-in"msgStarting = "building and starting the range"msgPortRetry = "host port %d was taken between the check and the bind; picking another"@@ -64,6 +66,7 @@ running.`msgRangeStopped = "the range is not running; start it with `khaos range up`"msgFailMode = "fail mode %s"msgTokenHint = "a credential is issued on demand, so `khaos run` against the range needs no flags"+ msgStatefulUp = "the stateful tier is up: PostgreSQL, MySQL, MariaDB and Redis, none of them published to a host port"msgPortHint = "host port %d was chosen at random; pass --port to pin it"errUnknownFailMode = "%w: %q (known modes: %s)"@@ -116,6 +119,10 @@ func newRangeUpCmd() *cobra.Command {if err != nil {return err}+ withDB, err := cmd.Flags().GetBool(flagWithDB)+ if err != nil {+ return err+ }return runRangeUp(cmd.Context(),cmd.OutOrStdout(),@@ -124,6 +131,7 @@ func newRangeUpCmd() *cobra.Command {tlsMode,port,quiet,+ withDB,)},}@@ -133,10 +141,32 @@ func newRangeUpCmd() *cobra.Command {cmd.Flags().String(flagTLSMode, string(tlsclient.Correct), tlsModeUsage)cmd.Flags().Int(flagPort, 0, portUsage)cmd.Flags().Bool(flagQuiet, false, quietUsage)+ cmd.Flags().Bool(flagWithDB, false, withDBUsage)return cmd}+// mintRangeSecrets gives the range a fresh nonce and a fresh database password+// per `range up`. One that outlived the range would let anything that had seen+// it arm the resolver, and a credential written into a public compose file is a+// credential forever.+func mintRangeSecrets(lab *rangelab.Lab) error {+ nonce, err := newNonce()+ if err != nil {+ return err+ }++ password, err := newNonce()+ if err != nil {+ return err+ }++ lab.ResolverNonce = nonce+ lab.DBPassword = password++ return nil+}+func newRangeDownCmd() *cobra.Command {return &cobra.Command{Use: "down",@@ -202,7 +232,7 @@ func runRangeUp(out io.Writer,failMode, cacheTTL, tlsMode string,port int,- quiet bool,+ quiet, withDB bool,) error {if !slices.Contains(knownFailModes(), failMode) {return fmt.Errorf(errUnknownFailMode, ErrUnknownFailMode, failMode,@@ -248,15 +278,11 @@ func runRangeUp(return err}- // A fresh nonce per `range up`: one that outlived the range would let- // anything that had seen it arm the resolver.- nonce, err := newNonce()- if err != nil {+ if err := mintRangeSecrets(&lab); err != nil {return err}- lab.ResolverNonce = nonce- err = lab.Up(ctx, progress, failMode, cacheTTL, tlsMode)+ err = lab.Up(ctx, progress, failMode, cacheTTL, tlsMode, withDB)if err == nil {break}@@ -271,6 +297,10 @@ func runRangeUp(event(out, markOK, theme.StatusOK, msgRangeUp)note(out, fmt.Sprintf(msgFailMode, failMode))note(out, fmt.Sprintf(msgPortHint, lab.Port))+ if withDB {+ note(out, msgStatefulUp)+ }+note(out, msgTokenHint)fmt.Fprintln(out)diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/answer.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/answer.gonew file mode 100644index 00000000..cce6b4bb--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/answer.go@@ -0,0 +1,243 @@+// ©AngelaMos | 2026+// answer.go++package dbx++import (+ "context"+ "database/sql"+ "database/sql/driver"+ "errors"+ "fmt"+ "io"+ "net"++ "github.com/go-sql-driver/mysql"+ "github.com/lib/pq"+)++// Answer is what the engine said about one credential, not whether khaos's+// request succeeded. Terminated is never produced by classification: it needs+// a second observation proving the server is still up.+type Answer string++const (+ // Unknown is the zero value on purpose, so a half-built Result reads as+ // khaos not knowing rather than as the engine serving.+ Unknown Answer = "unknown"++ Served Answer = "served"+ Refused Answer = "refused"+ Locked Answer = "locked"++ // Expired and MustReset are the same operator action arriving at two+ // different points. MySQL 8.4 refuses an expired credential at connect;+ // MariaDB 11.4 authenticates it and then refuses to execute.+ Expired Answer = "expired"+ MustReset Answer = "must-reset"++ // Terminated is a session the engine closed under khaos while remaining+ // reachable, which is the one control that revokes a live credential.+ Terminated Answer = "terminated"++ // Unreachable is the engine, not the credential.+ Unreachable Answer = "unreachable"++ // Misconfigured is khaos's own fault and is excluded from every rate.+ Misconfigured Answer = "misconfigured"+)++const (+ codeInvalidPassword = "28P01"+ codeNotPermitted = "28000"+ codeAdminShutdown = "57P01"+ codeBadCatalog = "3D000"+ codeBadRole = "42704"++ numberAccessDenied = 1045+ numberLockedMySQL = 3118+ numberLockedMariaDB = 4151+ numberMustReset = 1820+ numberMustChange = 1862+ numberNoSuchDB = 1049+ numberDBAccess = 1044++ detailServed = "the engine accepted the credential and answered a query"++ // PostgreSQL's three-way collision, stated in the detail rather than+ // resolved. Measured: a wrong password, a VALID UNTIL in the past and a+ // role that does not exist all produce this one message, and the+ // distinguishing DETAIL line exists only in the server log.+ detailRefusedPostgres = "%s; PostgreSQL returns this same message for a wrong password, for a credential whose validity has passed and for a role that does not exist, so khaos cannot resolve which from the client side"++ detailRefused = "%s"+ detailTerminated = "the engine closed this session while remaining reachable to khaos's own administrative connection: %s"+ detailLocked = "the account is locked or not permitted to log in: %s"+ detailMustReset = "the credential authenticated and then refused to execute until it is reset: %s"++ // Which end of the connection an expiry lands on depends on a capability+ // the client advertises. khaos's driver does not advertise it, so MySQL+ // refuses at connect with 1862 where a CLI passing+ // --connect-expired-password is admitted and then fails at execute with+ // 1820, which is what MariaDB does to every client.+ detailExpired = "the engine refused the connection outright because the credential has expired: %s"++ detailUnreachable = "khaos could not reach the engine at all: %s"+ detailKhaosFault = "khaos's own connection details are wrong rather than the credential: %s"+)++// Result is one observation of one credential against one engine.+type Result struct {+ Answer Answer+ Detail string+ Err error+ Version string+}++// Decided reports whether the engine made a statement about the credential.+// An unreachable engine and khaos's own misconfiguration are not statements.+func (r Result) Decided() bool {+ switch r.Answer {+ case Served, Refused, Locked, Expired, MustReset, Terminated:+ return true+ default:+ return false+ }+}++func served() Result {+ return Result{Answer: Served, Detail: detailServed}+}++// classify reads the engine's own error code rather than its message text,+// except in the detail, which carries the message verbatim because a restated+// constant is not evidence.+func classify(engine Engine, err error) Result {+ if err == nil {+ return served()+ }++ if engine.MySQLFamily() {+ return classifyMySQL(err)+ }++ return classifyPostgres(err)+}++func classifyPostgres(err error) Result {+ var pgErr *pq.Error+ if errors.As(err, &pgErr) {+ switch string(pgErr.Code) {+ case codeInvalidPassword:+ return Result{+ Answer: Refused,+ Detail: fmt.Sprintf(detailRefusedPostgres, pgErr.Message),+ Err: err,+ }+ case codeNotPermitted:+ return Result{+ Answer: Locked,+ Detail: fmt.Sprintf(detailLocked, pgErr.Message),+ Err: err,+ }+ case codeAdminShutdown:+ return Result{+ Answer: Terminated,+ Detail: fmt.Sprintf(detailTerminated, pgErr.Message),+ Err: err,+ }+ case codeBadCatalog, codeBadRole:+ return Result{+ Answer: Misconfigured,+ Detail: fmt.Sprintf(detailKhaosFault, pgErr.Message),+ Err: err,+ }+ default:+ return Result{+ Answer: Refused,+ Detail: fmt.Sprintf(detailRefused, pgErr.Message),+ Err: err,+ }+ }+ }++ return transport(err)+}++func classifyMySQL(err error) Result {+ var myErr *mysql.MySQLError+ if errors.As(err, &myErr) {+ switch myErr.Number {+ case numberAccessDenied:+ return Result{+ Answer: Refused,+ Detail: fmt.Sprintf(detailRefused, myErr.Message),+ Err: err,+ }+ case numberLockedMySQL, numberLockedMariaDB:+ return Result{+ Answer: Locked,+ Detail: fmt.Sprintf(detailLocked, myErr.Message),+ Err: err,+ }+ case numberMustChange:+ return Result{+ Answer: Expired,+ Detail: fmt.Sprintf(detailExpired, myErr.Message),+ Err: err,+ }+ case numberMustReset:+ return Result{+ Answer: MustReset,+ Detail: fmt.Sprintf(detailMustReset, myErr.Message),+ Err: err,+ }+ case numberNoSuchDB, numberDBAccess:+ return Result{+ Answer: Misconfigured,+ Detail: fmt.Sprintf(detailKhaosFault, myErr.Message),+ Err: err,+ }+ default:+ return Result{+ Answer: Refused,+ Detail: fmt.Sprintf(detailRefused, myErr.Message),+ Err: err,+ }+ }+ }++ return transport(err)+}++// transport is everything that is not the engine answering: a dead socket, a+// closed session, a timeout. It reads as Unreachable until a liveness control+// says otherwise, because a session khaos lost tells it nothing on its own.+func transport(err error) Result {+ return Result{+ Answer: Unreachable,+ Detail: fmt.Sprintf(detailUnreachable, err),+ Err: err,+ }+}++// lostSession reports whether this error means the connection went away rather+// than the engine refusing something.+func lostSession(err error) bool {+ if err == nil {+ return false+ }++ if errors.Is(err, driver.ErrBadConn) ||+ errors.Is(err, mysql.ErrInvalidConn) ||+ errors.Is(err, sql.ErrConnDone) ||+ errors.Is(err, io.EOF) ||+ errors.Is(err, io.ErrUnexpectedEOF) ||+ errors.Is(err, context.DeadlineExceeded) {+ return true+ }++ var netErr net.Error++ return errors.As(err, &netErr)+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/dbx.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/dbx.gonew file mode 100644index 00000000..0d1b6467--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/dbx.go@@ -0,0 +1,466 @@+// ©AngelaMos | 2026+// dbx.go++package dbx++import (+ "context"+ "database/sql"+ "errors"+ "fmt"+ "net/url"+ "regexp"+ "slices"+ "strings"+ "time"++ "github.com/go-sql-driver/mysql"+)++type Engine string++const (+ Postgres Engine = "postgres"+ MySQL Engine = "mysql"+ MariaDB Engine = "mariadb"+)++const (+ driverPostgres = "postgres"+ driverMySQL = "mysql"++ // sslmode is disabled deliberately. The credential tier measures what an+ // engine does with an authenticated session when its credential changes,+ // which is independent of the transport, and no engine in the range serves+ // TLS by default. MySQL needs no secure channel either: the driver performs+ // the caching_sha2_password RSA public-key exchange, which is what makes a+ // credential error observable over plaintext at all.+ postgresOptions = "sslmode=disable&connect_timeout=5"++ dialTimeout = 5 * time.Second+ queryTimeout = 5 * time.Second++ probeQuery = "SELECT 1"+ versionQuery = "SELECT version()"++ createRolePostgres = "CREATE ROLE %s LOGIN PASSWORD %s"+ createUserMySQL = "CREATE USER %s@'%%' IDENTIFIED BY %s"+ dropRolePostgres = "DROP ROLE IF EXISTS %s"+ dropUserMySQL = "DROP USER IF EXISTS %s@'%%'"+ rotatePostgres = "ALTER ROLE %s PASSWORD %s"+ rotateMySQL = "ALTER USER %s@'%%' IDENTIFIED BY %s"+ lockPostgres = "ALTER ROLE %s NOLOGIN"+ unlockPostgres = "ALTER ROLE %s LOGIN"+ lockMySQL = "ALTER USER %s@'%%' ACCOUNT LOCK"+ unlockMySQL = "ALTER USER %s@'%%' ACCOUNT UNLOCK"+ expirePostgres = "ALTER ROLE %s VALID UNTIL '1970-01-01'"+ unexpirePostgres = "ALTER ROLE %s VALID UNTIL 'infinity'"+ expireMySQL = "ALTER USER %s@'%%' PASSWORD EXPIRE"++ terminatePostgres = `SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity+WHERE usename = $1 AND pid <> pg_backend_pid()`+ listSessionsMySQL = `SELECT id FROM information_schema.processlist+WHERE user = ? AND id <> CONNECTION_ID()`+ killMySQL = "KILL %d"++ errUnknownEngine = "%w: %q (known engines: %s)"+ errBadIdentifier = "%w: %q"+ errNoAddress = "%w: the credential tier needs an address for the engine it is faulting"+ errNoAdmin = "%w: the credential tier needs an administrative role, read from the target container's own environment"+ errTerminateUnknown = "%w: khaos cannot terminate sessions on %s"+)++var (+ ErrUnknownEngine = errors.New(+ "no database engine is registered under that name",+ )++ // ErrBadIdentifier guards a role name that is interpolated into DDL. No+ // engine accepts a parameter in place of a role name or a password, so the+ // statement is built by hand and the input is constrained instead.+ ErrBadIdentifier = errors.New(+ "that role name is not one khaos is willing to put into a statement",+ )++ ErrIncomplete = errors.New(+ "this credential-tier client is missing something it cannot work without",+ )++ safeIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)+)++func Engines() []string {+ return []string{string(MariaDB), string(MySQL), string(Postgres)}+}++func ParseEngine(name string) (Engine, error) {+ engine := Engine(strings.ToLower(strings.TrimSpace(name)))+ if !slices.Contains(Engines(), string(engine)) {+ return "", fmt.Errorf(+ errUnknownEngine,+ ErrUnknownEngine,+ name,+ strings.Join(Engines(), ", "),+ )+ }+ return engine, nil+}++// MySQLFamily reports whether this engine speaks the MySQL wire protocol. It+// does NOT imply a shared authentication path: MySQL 8.4 defaults to+// caching_sha2_password and MariaDB 11.4 to mysql_native_password.+func (e Engine) MySQLFamily() bool {+ return e == MySQL || e == MariaDB+}++func (e Engine) driver() string {+ if e.MySQLFamily() {+ return driverMySQL+ }+ return driverPostgres+}++func (e Engine) String() string {+ return string(e)+}++// Credential is a role and the password presented for it. khaos only ever+// rotates a credential it minted itself, except for the pooled-service+// experiment, where the role belongs to the target.+type Credential struct {+ Role string `json:"role"`++ // Never serialised, structurally. khaos journals where a credential is+ // READ FROM and never the credential, so the type it travels in refuses to+ // marshal it rather than relying on every call site to remember.+ Password string `json:"-"`+}++type Config struct {+ Engine Engine+ Address string+ Database string+ Admin Credential+}++type Client struct {+ config Config+ admin *sql.DB+}++// Open validates the configuration and opens the administrative pool. It does+// not dial: Ping is what proves the engine is reachable, and the caller wants+// that failure separately from a configuration error.+func Open(config Config) (*Client, error) {+ if _, err := ParseEngine(string(config.Engine)); err != nil {+ return nil, err+ }++ if config.Address == "" {+ return nil, fmt.Errorf(errNoAddress, ErrIncomplete)+ }++ if config.Admin.Role == "" || config.Admin.Password == "" {+ return nil, fmt.Errorf(errNoAdmin, ErrIncomplete)+ }++ pool, err := sql.Open(+ config.Engine.driver(),+ dsn(config.Engine, config.Address, config.Database, config.Admin),+ )+ if err != nil {+ return nil, err+ }++ pool.SetMaxOpenConns(2)+ pool.SetMaxIdleConns(1)++ return &Client{config: config, admin: pool}, nil+}++func (c *Client) Engine() Engine {+ return c.config.Engine+}++func (c *Client) Close() error {+ return c.admin.Close()+}++func (c *Client) Ping(ctx context.Context) error {+ ctx, cancel := context.WithTimeout(ctx, dialTimeout)+ defer cancel()++ return c.admin.PingContext(ctx)+}++// Version returns what the SERVER reports, never the image tag. On this host+// `postgres:18` resolved to two different patch releases hours apart, so a run+// record that names a tag names nothing.+func (c *Client) Version(ctx context.Context) (string, error) {+ ctx, cancel := context.WithTimeout(ctx, queryTimeout)+ defer cancel()++ var version string+ if err := c.admin.QueryRowContext(+ ctx,+ versionQuery,+ ).Scan(&version); err != nil {+ return "", err+ }++ return version, nil+}++func (c *Client) CreateRole(ctx context.Context, cred Credential) error {+ statement := createRolePostgres+ if c.config.Engine.MySQLFamily() {+ statement = createUserMySQL+ }++ return c.alter(ctx, statement, cred)+}++func (c *Client) Rotate(ctx context.Context, cred Credential) error {+ statement := rotatePostgres+ if c.config.Engine.MySQLFamily() {+ statement = rotateMySQL+ }++ return c.alter(ctx, statement, cred)+}++func (c *Client) DropRole(ctx context.Context, role string) error {+ statement := dropRolePostgres+ if c.config.Engine.MySQLFamily() {+ statement = dropUserMySQL+ }++ return c.exec(ctx, statement, role)+}++func (c *Client) Lock(ctx context.Context, role string) error {+ statement := lockPostgres+ if c.config.Engine.MySQLFamily() {+ statement = lockMySQL+ }++ return c.exec(ctx, statement, role)+}++func (c *Client) Unlock(ctx context.Context, role string) error {+ statement := unlockPostgres+ if c.config.Engine.MySQLFamily() {+ statement = unlockMySQL+ }++ return c.exec(ctx, statement, role)+}++// Expire puts the credential's validity in the past. On the MySQL family this+// does not refuse a new connection: it authenticates and then fails at execute+// with 1820, which a connect-only liveness probe reads as healthy.+func (c *Client) Expire(ctx context.Context, role string) error {+ statement := expirePostgres+ if c.config.Engine.MySQLFamily() {+ statement = expireMySQL+ }++ return c.exec(ctx, statement, role)+}++// Unexpire restores validity. On the MySQL family the only way back is to set+// the password again, so the caller supplies the credential rather than a name.+func (c *Client) Unexpire(ctx context.Context, cred Credential) error {+ if c.config.Engine.MySQLFamily() {+ return c.Rotate(ctx, cred)+ }++ return c.exec(ctx, unexpirePostgres, cred.Role)+}++// Terminate is the only control in any of the three engines that revokes a+// LIVE session. It returns how many it closed, and it never closes khaos's own+// administrative connection.+func (c *Client) Terminate(ctx context.Context, role string) (int, error) {+ if err := validate(role); err != nil {+ return 0, err+ }++ ctx, cancel := context.WithTimeout(ctx, queryTimeout)+ defer cancel()++ switch {+ case c.config.Engine == Postgres:+ var killed int+ if err := c.admin.QueryRowContext(+ ctx,+ terminatePostgres,+ role,+ ).Scan(&killed); err != nil {+ return 0, err+ }+ return killed, nil++ case c.config.Engine.MySQLFamily():+ return c.killMySQLSessions(ctx, role)++ default:+ return 0, fmt.Errorf(+ errTerminateUnknown,+ ErrUnknownEngine,+ c.config.Engine,+ )+ }+}++// killMySQLSessions reads the identifiers first and kills afterwards, because+// KILL closes the connection the result set is being read from when it lands on+// khaos's own.+func (c *Client) killMySQLSessions(+ ctx context.Context,+ role string,+) (int, error) {+ rows, err := c.admin.QueryContext(ctx, listSessionsMySQL, role)+ if err != nil {+ return 0, err+ }++ ids := []int64{}+ for rows.Next() {+ var id int64+ if err := rows.Scan(&id); err != nil {+ _ = rows.Close()+ return 0, err+ }+ ids = append(ids, id)+ }+ if err := rows.Err(); err != nil {+ _ = rows.Close()+ return 0, err+ }+ if err := rows.Close(); err != nil {+ return 0, err+ }++ killed := 0+ var failures []error++ for _, id := range ids {+ if _, err := c.admin.ExecContext(+ ctx,+ fmt.Sprintf(killMySQL, id),+ ); err != nil {+ failures = append(failures, err)+ continue+ }+ killed+++ }++ return killed, errors.Join(failures...)+}++func (c *Client) alter(+ ctx context.Context,+ statement string,+ cred Credential,+) error {+ if err := validate(cred.Role); err != nil {+ return err+ }++ ctx, cancel := context.WithTimeout(ctx, queryTimeout)+ defer cancel()++ _, err := c.admin.ExecContext(+ ctx,+ fmt.Sprintf(+ statement,+ quoteIdentifier(c.config.Engine, cred.Role),+ quoteLiteral(cred.Password),+ ),+ )++ return err+}++func (c *Client) exec(+ ctx context.Context,+ statement string,+ role string,+) error {+ if err := validate(role); err != nil {+ return err+ }++ ctx, cancel := context.WithTimeout(ctx, queryTimeout)+ defer cancel()++ _, err := c.admin.ExecContext(+ ctx,+ fmt.Sprintf(statement, quoteIdentifier(c.config.Engine, role)),+ )++ return err+}++// validate is the guard that makes string-built DDL safe. Every role khaos+// creates is generated, so the constraint costs nothing and refuses anything+// that could carry a statement.+func validate(role string) error {+ if !safeIdentifier.MatchString(role) {+ return fmt.Errorf(errBadIdentifier, ErrBadIdentifier, role)+ }++ return nil+}++func quoteIdentifier(engine Engine, name string) string {+ if engine.MySQLFamily() {+ return "`" + name + "`"+ }++ return `"` + name + `"`+}++// quoteLiteral is single-quote doubling, which both dialects accept, and it is+// applied to passwords rather than relying on the input being generated.+func quoteLiteral(value string) string {+ return "'" + strings.ReplaceAll(value, "'", "''") + "'"+}++// dsn names no default database on the MySQL family, deliberately. khaos's+// probe asks only whether the engine ACCEPTS the credential, and a role with no+// grants can connect and run SELECT 1 with no schema selected — while naming a+// database it has no privilege on earns error 1044, which is khaos's own+// misconfiguration and not a statement about the credential. The alternative+// would be granting a khaos-minted role access to the operator's data, which is+// a larger blast radius for no measurement. PostgreSQL requires a database on+// every connection, so it keeps one.+func dsn(+ engine Engine,+ address, database string,+ cred Credential,+) string {+ if engine.MySQLFamily() {+ config := mysql.NewConfig()+ config.User = cred.Role+ config.Passwd = cred.Password+ config.Net = "tcp"+ config.Addr = address+ config.Timeout = dialTimeout+ config.ReadTimeout = queryTimeout+ config.WriteTimeout = queryTimeout++ return config.FormatDSN()+ }++ target := url.URL{+ Scheme: "postgres",+ User: url.UserPassword(cred.Role, cred.Password),+ Host: address,+ Path: "/" + database,+ RawQuery: postgresOptions,+ }++ return target.String()+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/dbx_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/dbx_test.gonew file mode 100644index 00000000..f51b0f70--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/dbx_test.go@@ -0,0 +1,440 @@+// ©AngelaMos | 2026+// dbx_test.go++package dbx++import (+ "encoding/json"+ "errors"+ "strings"+ "testing"++ "github.com/go-sql-driver/mysql"+ "github.com/lib/pq"+)++const (+ testAddress = "10.244.244.9:5432"+ testDatabase = "appdb"+)++// TestValidateRefusesAnythingThatCouldCarryAStatement is the control for the+// guard that makes string-built DDL safe. No engine accepts a parameter in+// place of a role name or a password, so the statement is interpolated and the+// input is constrained; this is the input set that must be refused.+func TestValidateRefusesAnythingThatCouldCarryAStatement(t *testing.T) {+ refused := []string{+ "",+ "robert'); DROP TABLE students;--",+ "khaos cred",+ "khaos-cred",+ "KhaosCred",+ "1khaos",+ `khaos"cred`,+ "khaos`cred",+ "khaos;cred",+ "khaos\ncred",+ "khaos%cred",+ strings.Repeat("k", 64),+ }++ for _, role := range refused {+ if err := validate(role); !errors.Is(err, ErrBadIdentifier) {+ t.Errorf(+ "validate(%q) = %v, want ErrBadIdentifier",+ role,+ err,+ )+ }+ }++ accepted := []string{+ "appuser",+ "khaos_cred_9f2c",+ "k",+ strings.Repeat("k", 63),+ }++ for _, role := range accepted {+ if err := validate(role); err != nil {+ t.Errorf("validate(%q) = %v, want nil", role, err)+ }+ }+}++func TestQuoteLiteralDoublesSingleQuotes(t *testing.T) {+ cases := map[string]string{+ "plain": "'plain'",+ "it's": "'it''s'",+ "''": "''''''",+ "a'b'c": "'a''b''c'",+ "back\\slash": "'back\\slash'",+ }++ for in, want := range cases {+ if got := quoteLiteral(in); got != want {+ t.Errorf("quoteLiteral(%q) = %q, want %q", in, got, want)+ }+ }+}++// TestDSNCarriesTheCredentialAndNoTLS pins both halves of the transport+// decision: the credential reaches the driver intact even when it contains+// characters a URL would otherwise eat, and neither engine is asked for TLS.+func TestDSNCarriesTheCredentialAndNoTLS(t *testing.T) {+ awkward := Credential{Role: "appuser", Password: "p@ss:w/rd?&=#"}++ postgres := dsn(Postgres, testAddress, testDatabase, awkward)++ if !strings.Contains(postgres, "sslmode=disable") {+ t.Errorf("postgres dsn does not disable sslmode: %s", postgres)+ }+ if strings.Contains(postgres, awkward.Password) {+ t.Errorf("postgres dsn carries an unescaped password: %s", postgres)+ }++ parsed, err := pq.ParseURL(postgres)+ if err != nil {+ t.Fatalf("lib/pq cannot parse the dsn khaos built: %v", err)+ }+ if !strings.Contains(parsed, "password='p@ss:w/rd?&=#'") {+ t.Errorf("round-tripped postgres dsn lost the password: %s", parsed)+ }++ for _, engine := range []Engine{MySQL, MariaDB} {+ raw := dsn(engine, testAddress, testDatabase, awkward)++ config, err := mysql.ParseDSN(raw)+ if err != nil {+ t.Fatalf("%s: go-sql-driver cannot parse %q: %v", engine, raw, err)+ }+ if config.Passwd != awkward.Password {+ t.Errorf(+ "%s: dsn round-tripped the password as %q, want %q",+ engine,+ config.Passwd,+ awkward.Password,+ )+ }+ if config.TLS != nil {+ t.Errorf(+ "%s: dsn asks for TLS, which no range engine serves",+ engine,+ )+ }+ if !config.AllowNativePasswords {+ t.Errorf(+ "%s: native passwords are disabled, which MariaDB 11.4 needs",+ engine,+ )+ }+ if config.AllowCleartextPasswords {+ t.Errorf(+ "%s: cleartext passwords are enabled, and the RSA exchange is what khaos relies on",+ engine,+ )+ }+ }+}++func TestParseEngine(t *testing.T) {+ for _, name := range []string{"postgres", "MySQL", " mariadb "} {+ if _, err := ParseEngine(name); err != nil {+ t.Errorf("ParseEngine(%q) = %v, want nil", name, err)+ }+ }++ for _, name := range []string{"", "sqlite", "postgresql", "mysql8"} {+ if _, err := ParseEngine(name); !errors.Is(err, ErrUnknownEngine) {+ t.Errorf("ParseEngine(%q) = %v, want ErrUnknownEngine", name, err)+ }+ }+}++func TestOpenRefusesAnIncompleteClient(t *testing.T) {+ cases := []struct {+ name string+ config Config+ want error+ }{+ {+ name: "no engine",+ config: Config{Address: testAddress},+ want: ErrUnknownEngine,+ },+ {+ name: "no address",+ config: Config{Engine: Postgres},+ want: ErrIncomplete,+ },+ {+ name: "no administrative credential",+ config: Config{+ Engine: Postgres,+ Address: testAddress,+ },+ want: ErrIncomplete,+ },+ {+ name: "an administrative role with no password",+ config: Config{+ Engine: Postgres,+ Address: testAddress,+ Admin: Credential{Role: "postgres"},+ },+ want: ErrIncomplete,+ },+ }++ for _, test := range cases {+ t.Run(test.name, func(t *testing.T) {+ if _, err := Open(test.config); !errors.Is(err, test.want) {+ t.Errorf("Open() = %v, want %v", err, test.want)+ }+ })+ }+}++// TestClassifyReadsTheCodeAndQuotesTheMessage pins the per-engine mapping. The+// live differential in live_test.go is what proves the engines actually emit+// these codes; this is the mapping, including the one case khaos must refuse to+// resolve.+func TestClassifyReadsTheCodeAndQuotesTheMessage(t *testing.T) {+ cases := []struct {+ name string+ engine Engine+ err error+ want Answer+ }{+ {+ name: "postgres refuses a password",+ engine: Postgres,+ err: &pq.Error{+ Code: "28P01",+ Message: `password authentication failed for user "appuser"`,+ },+ want: Refused,+ },+ {+ name: "postgres refuses a login",+ engine: Postgres,+ err: &pq.Error{+ Code: "28000",+ Message: `role "appuser" is not permitted to log in`,+ },+ want: Locked,+ },+ {+ name: "postgres closed the session",+ engine: Postgres,+ err: &pq.Error{+ Code: "57P01",+ Message: "terminating connection due to administrator command",+ },+ want: Terminated,+ },+ {+ name: "postgres has no such database",+ engine: Postgres,+ err: &pq.Error{+ Code: "3D000",+ Message: `database "nope" does not exist`,+ },+ want: Misconfigured,+ },+ {+ name: "mysql denies access",+ engine: MySQL,+ err: &mysql.MySQLError{+ Number: 1045,+ Message: "Access denied for user 'appuser'@'10.0.0.2'",+ },+ want: Refused,+ },+ {+ name: "mysql reports a locked account",+ engine: MySQL,+ err: &mysql.MySQLError{+ Number: 3118,+ Message: "Access denied for user 'appuser'. Account is locked.",+ },+ want: Locked,+ },+ {+ name: "mariadb reports a locked account",+ engine: MariaDB,+ err: &mysql.MySQLError{+ Number: 4151,+ Message: "Access denied, this account is locked",+ },+ want: Locked,+ },+ {+ name: "the mysql family demands a reset",+ engine: MySQL,+ err: &mysql.MySQLError{+ Number: 1820,+ Message: "You must reset your password using ALTER USER",+ },+ want: MustReset,+ },+ {+ name: "a dead socket is not a statement about the credential",+ engine: MySQL,+ err: mysql.ErrInvalidConn,+ want: Unreachable,+ },+ {+ name: "nothing went wrong",+ engine: Postgres,+ err: nil,+ want: Served,+ },+ }++ for _, test := range cases {+ t.Run(test.name, func(t *testing.T) {+ result := classify(test.engine, test.err)++ if result.Answer != test.want {+ t.Fatalf(+ "classify(%s, %v) = %q, want %q",+ test.engine,+ test.err,+ result.Answer,+ test.want,+ )+ }++ if test.err == nil {+ return+ }++ var (+ pgErr *pq.Error+ myErr *mysql.MySQLError+ )++ switch {+ case errors.As(test.err, &pgErr):+ if !strings.Contains(result.Detail, pgErr.Message) {+ t.Errorf(+ "detail %q does not quote the engine's own message %q",+ result.Detail,+ pgErr.Message,+ )+ }+ case errors.As(test.err, &myErr):+ if !strings.Contains(result.Detail, myErr.Message) {+ t.Errorf(+ "detail %q does not quote the engine's own message %q",+ result.Detail,+ myErr.Message,+ )+ }+ }+ })+ }+}++// TestPostgresRefusalRefusesToNameACause is the control for the one collision+// khaos must not resolve. A wrong password, a credential whose validity has+// passed and a role that does not exist all produce 28P01, so a detail that+// named any single cause would be confidently wrong a third of the time.+func TestPostgresRefusalRefusesToNameACause(t *testing.T) {+ result := classify(Postgres, &pq.Error{+ Code: "28P01",+ Message: `password authentication failed for user "appuser"`,+ })++ for _, phrase := range []string{+ "validity has passed",+ "does not exist",+ "cannot resolve",+ } {+ if !strings.Contains(result.Detail, phrase) {+ t.Errorf(+ "a 28P01 detail must say %q, got %q",+ phrase,+ result.Detail,+ )+ }+ }+}++func TestDecidedExcludesWhatTheEngineNeverSaid(t *testing.T) {+ decided := []Answer{Served, Refused, Locked, MustReset, Terminated}+ silent := []Answer{Unknown, Unreachable, Misconfigured, ""}++ for _, answer := range decided {+ if !(Result{Answer: answer}).Decided() {+ t.Errorf("%q should count as the engine deciding", answer)+ }+ }++ for _, answer := range silent {+ if (Result{Answer: answer}).Decided() {+ t.Errorf("%q is not the engine deciding anything", answer)+ }+ }+}++// TestWrongMintsAFreshPasswordPerSample is the control for the negative+// control. A fixed invalid credential keeps a target's decision cache warm, so+// the catalogue census requires freshness and this is where it is observable:+// the engine cannot tell khaos whether the password it refused was new.+func TestWrongMintsAFreshPasswordPerSample(t *testing.T) {+ tier := NewTier(Trap{Kind: Rotate, Role: "khaos_cred_abcdef", Minted: true})++ const rounds = 8++ seen := map[string]bool{}++ for range rounds {+ cred := tier.wrong()++ if cred.Role != "khaos_cred_abcdef" {+ t.Fatalf(+ "the control changed the ROLE to %q; it must vary the password of the role under test, or it is measuring a different account",+ cred.Role,+ )+ }++ seen[cred.Password] = true+ }++ if len(seen) != rounds {+ t.Fatalf(+ "%d calls produced %d distinct passwords; a repeated control credential can be answered from a cache",+ rounds,+ len(seen),+ )+ }+}++// TestACredentialCannotBeSerialisedWithItsSecret is the structural control for+// the rule that khaos journals where a credential is read from and never the+// credential. Enforcing it in the type beats enforcing it at every call site.+func TestACredentialCannotBeSerialisedWithItsSecret(t *testing.T) {+ encoded, err := json.Marshal(Credential{+ Role: "appuser",+ Password: "hunter2-should-never-appear",+ })+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }++ if strings.Contains(string(encoded), "hunter2") {+ t.Fatalf(+ "a Credential serialised its password: %s",+ encoded,+ )+ }++ if !strings.Contains(string(encoded), "appuser") {+ t.Fatalf(+ "a Credential dropped the role as well, which the journal does need: %s",+ encoded,+ )+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/live_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/live_test.gonew file mode 100644index 00000000..cce80d5b--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/live_test.go@@ -0,0 +1,540 @@+// ©AngelaMos | 2026+// live_test.go++package dbx++import (+ "context"+ "crypto/rand"+ "os"+ "strings"+ "testing"+ "time"+)++// Every assertion in this file is a differential against a real engine. The+// mapping tests in dbx_test.go prove khaos reads a code correctly; only these+// prove the engines emit the codes khaos reads, and a self-consistent client+// agrees with its own mistake.+//+// Skipped unless the engine's address is in the environment, so `go test ./...`+// stays hermetic. scripts/credential.sh supplies them from the range.+//+// KHAOS_DBX_LIVE_POSTGRES=host:port,database,adminrole,adminpassword+// KHAOS_DBX_LIVE_MYSQL=...+// KHAOS_DBX_LIVE_MARIADB=...+const (+ liveEnvPrefix = "KHAOS_DBX_LIVE_"+ liveFields = 4+ liveTimeout = 30 * time.Second+)++func liveConfig(t *testing.T, engine Engine) Config {+ t.Helper()++ name := liveEnvPrefix + strings.ToUpper(string(engine))++ raw := os.Getenv(name)+ if raw == "" {+ t.Skipf(+ "%s is not set, so there is no live %s to measure against",+ name,+ engine,+ )+ }++ parts := strings.Split(raw, ",")+ if len(parts) != liveFields {+ t.Fatalf(+ "%s = %q, want host:port,database,adminrole,adminpassword",+ name,+ raw,+ )+ }++ return Config{+ Engine: engine,+ Address: parts[0],+ Database: parts[1],+ Admin: Credential{Role: parts[2], Password: parts[3]},+ }+}++// TestRotationIsNotRevocation is the milestone's exit gate as a differential.+// One held session, one rotation, and the engine's own answer at both ends.+func TestRotationIsNotRevocation(t *testing.T) {+ for _, engine := range []Engine{Postgres, MySQL, MariaDB} {+ t.Run(string(engine), func(t *testing.T) {+ config := liveConfig(t, engine)++ ctx, cancel := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer cancel()++ client, role := liveRole(t, ctx, config)++ first := Credential{Role: role, Password: "khaos" + rand.Text()}+ second := Credential{Role: role, Password: "khaos" + rand.Text()}++ if err := client.Rotate(ctx, first); err != nil {+ t.Fatalf("setting the first password: %v", err)+ }++ wantAnswer(t, "the first password", client.Try(ctx, first), Served)++ wrong := Credential{Role: role, Password: "khaos" + rand.Text()}+ wantAnswer(+ t,+ "a password that was never correct",+ client.Try(ctx, wrong),+ Refused,+ )++ session, err := client.Hold(ctx, first)+ if err != nil {+ t.Fatalf("holding a session on the first password: %v", err)+ }+ defer func() { _ = session.Close() }()++ wantAnswer(+ t,+ "the held session before the rotation",+ session.Query(ctx),+ Served,+ )++ if err := client.Rotate(ctx, second); err != nil {+ t.Fatalf("rotating: %v", err)+ }++ wantAnswer(+ t,+ "THE FINDING: the held session after the rotation",+ session.Query(ctx),+ Served,+ )++ wantAnswer(+ t,+ "a NEW connection on the rotated-away password",+ client.Try(ctx, first),+ Refused,+ )++ wantAnswer(+ t,+ "a NEW connection on the rotated-to password",+ client.Try(ctx, second),+ Served,+ )++ killed, err := client.Terminate(ctx, role)+ if err != nil {+ t.Fatalf("terminating: %v", err)+ }+ if killed < 1 {+ t.Fatalf(+ "terminate closed %d sessions, want at least 1",+ killed,+ )+ }++ wantAnswer(+ t,+ "THE CONTRAST: the held session after termination",+ session.Query(ctx),+ Terminated,+ )+ })+ }+}++// TestAccountLockDoesNotRevokeALiveSession and the expiry test below are the+// two other controls an operator reaches for believing they revoke. Neither+// does, on any engine.+func TestAccountLockDoesNotRevokeALiveSession(t *testing.T) {+ for _, engine := range []Engine{Postgres, MySQL, MariaDB} {+ t.Run(string(engine), func(t *testing.T) {+ config := liveConfig(t, engine)++ ctx, cancel := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer cancel()++ client, role := liveRole(t, ctx, config)++ cred := Credential{Role: role, Password: "khaos" + rand.Text()}+ if err := client.Rotate(ctx, cred); err != nil {+ t.Fatalf("setting the password: %v", err)+ }++ session, err := client.Hold(ctx, cred)+ if err != nil {+ t.Fatalf("holding a session: %v", err)+ }+ defer func() { _ = session.Close() }()++ if err := client.Lock(ctx, role); err != nil {+ t.Fatalf("locking: %v", err)+ }++ wantAnswer(+ t,+ "THE FINDING: the held session after the lock",+ session.Query(ctx),+ Served,+ )++ wantAnswer(+ t,+ "a NEW connection against a locked account",+ client.Try(ctx, cred),+ Locked,+ )++ // A locked account presented with a WRONG password reports the+ // wrong-password error on both MySQL-family engines, so lock state+ // is not observable without a valid credential. Measured, and it+ // bounds what khaos can claim.+ wrong := Credential{Role: role, Password: "khaos" + rand.Text()}+ locked := client.Try(ctx, wrong)+ if locked.Answer != Locked && locked.Answer != Refused {+ t.Errorf(+ "a locked account with a wrong password answered %q, want locked or refused",+ locked.Answer,+ )+ }++ if err := client.Unlock(ctx, role); err != nil {+ t.Fatalf("unlocking: %v", err)+ }++ wantAnswer(+ t,+ "a NEW connection after the unlock",+ client.Try(ctx, cred),+ Served,+ )+ })+ }+}++func TestExpiryDoesNotRevokeALiveSession(t *testing.T) {+ for _, engine := range []Engine{Postgres, MySQL, MariaDB} {+ t.Run(string(engine), func(t *testing.T) {+ config := liveConfig(t, engine)++ ctx, cancel := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer cancel()++ client, role := liveRole(t, ctx, config)++ cred := Credential{Role: role, Password: "khaos" + rand.Text()}+ if err := client.Rotate(ctx, cred); err != nil {+ t.Fatalf("setting the password: %v", err)+ }++ session, err := client.Hold(ctx, cred)+ if err != nil {+ t.Fatalf("holding a session: %v", err)+ }+ defer func() { _ = session.Close() }()++ if err := client.Expire(ctx, role); err != nil {+ t.Fatalf("expiring: %v", err)+ }++ wantAnswer(+ t,+ "THE FINDING: the held session after the credential expired",+ session.Query(ctx),+ Served,+ )++ // Three engines, three answers, measured. PostgreSQL refuses an+ // expired credential with the same message it uses for a wrong+ // password. MySQL 8.4 refuses it at connect with 1862, because+ // khaos's client does not advertise the expired-password+ // capability. MariaDB 11.4 admits it and then refuses to execute+ // with 1820, which is a state a connect-only liveness probe reads+ // as healthy.+ want := map[Engine]Answer{+ Postgres: Refused,+ MySQL: Expired,+ MariaDB: MustReset,+ }[config.Engine]++ wantAnswer(+ t,+ "a NEW connection on an expired credential",+ client.Try(ctx, cred),+ want,+ )++ if err := client.Unexpire(ctx, cred); err != nil {+ t.Fatalf("restoring validity: %v", err)+ }++ wantAnswer(+ t,+ "a NEW connection after validity was restored",+ client.Try(ctx, cred),+ Served,+ )+ })+ }+}++// TestRotatingToTheSamePasswordChangesNothing is the inert control for the+// tier's confirmation oracle, and it needs no test-only flag: a rotation that+// changes nothing leaves the old password working, so a confirmation that+// requires the old password to be refused MUST say no.+func TestRotatingToTheSamePasswordChangesNothing(t *testing.T) {+ for _, engine := range []Engine{Postgres, MySQL, MariaDB} {+ t.Run(string(engine), func(t *testing.T) {+ config := liveConfig(t, engine)++ ctx, cancel := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer cancel()++ client, role := liveRole(t, ctx, config)++ cred := Credential{Role: role, Password: "khaos" + rand.Text()}+ if err := client.Rotate(ctx, cred); err != nil {+ t.Fatalf("setting the password: %v", err)+ }++ if err := client.Rotate(ctx, cred); err != nil {+ t.Fatalf("rotating to the same password: %v", err)+ }++ wantAnswer(+ t,+ "the unchanged password after an inert rotation",+ client.Try(ctx, cred),+ Served,+ )+ })+ }+}++// TestVersionComesFromTheServer pins the rule that a tag is not a version.+func TestVersionComesFromTheServer(t *testing.T) {+ for _, engine := range []Engine{Postgres, MySQL, MariaDB} {+ t.Run(string(engine), func(t *testing.T) {+ config := liveConfig(t, engine)++ ctx, cancel := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer cancel()++ client, err := Open(config)+ if err != nil {+ t.Fatalf("opening: %v", err)+ }+ defer func() { _ = client.Close() }()++ version, err := client.Version(ctx)+ if err != nil {+ t.Fatalf("reading the version: %v", err)+ }++ if version == "" {+ t.Fatal("the engine reported an empty version")+ }++ t.Logf("%s reports %q", engine, version)+ })+ }+}++// liveRole opens a client and mints a role that only exists for this test, so+// nothing belonging to the operator is ever mutated.+func liveRole(+ t *testing.T,+ ctx context.Context,+ config Config,+) (*Client, string) {+ t.Helper()++ client, err := Open(config)+ if err != nil {+ t.Fatalf("opening a client: %v", err)+ }++ if err := client.Ping(ctx); err != nil {+ t.Fatalf("the engine at %s is not reachable: %v", config.Address, err)+ }++ role := "khaos_live_" + strings.ToLower(rand.Text()[:10])++ if err := client.CreateRole(+ ctx,+ Credential{Role: role, Password: "khaos" + rand.Text()},+ ); err != nil {+ t.Fatalf("creating %s: %v", role, err)+ }++ t.Cleanup(func() {+ cleanup, cancel := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer cancel()++ if _, err := client.Terminate(cleanup, role); err != nil {+ t.Errorf("terminating %s during cleanup: %v", role, err)+ }+ if err := client.DropRole(cleanup, role); err != nil {+ t.Errorf("dropping %s: %v", role, err)+ }+ if err := client.Close(); err != nil {+ t.Errorf("closing the client: %v", err)+ }+ })++ return client, role+}++func wantAnswer(t *testing.T, what string, got Result, want Answer) {+ t.Helper()++ if got.Answer != want {+ t.Fatalf(+ "%s answered %q, want %q (detail: %s)",+ what,+ got.Answer,+ want,+ got.Detail,+ )+ }+}++// TestAnInertArmIsRefusedByTheConfirmation is the oracle's negative control,+// and it is the one assertion that cannot be satisfied by a confirmation gutted+// to `return true`. Arm prepares everything and never tells the engine, so the+// old password still works and a read-back MUST say no.+//+// Every kind is driven, because each one reads a different thing back and a+// control on one arm alibis the others.+func TestAnInertArmIsRefusedByTheConfirmation(t *testing.T) {+ for _, kind := range []Kind{Rotate, RotateTerminate, Lock, Expire} {+ t.Run(string(kind), func(t *testing.T) {+ config := liveConfig(t, Postgres)++ ctx, cancel := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer cancel()++ tier := NewTier(Trap{+ Kind: kind,+ Role: MintedRole(),+ Minted: true,+ Inert: true,+ })++ if err := tier.Open(ctx, config, Credential{}); err != nil {+ t.Fatalf("opening the tier: %v", err)+ }++ t.Cleanup(func() {+ closing, stop := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer stop()++ if err := tier.Close(closing); err != nil {+ t.Errorf("closing the tier: %v", err)+ }+ })++ if err := tier.Arm(ctx); err != nil {+ t.Fatalf("arming inert: %v", err)+ }++ observable, confirmed, err := tier.Observe(ctx)+ if err != nil {+ t.Fatalf("observing: %v", err)+ }++ if confirmed {+ t.Fatalf(+ "an inert %s arm was CONFIRMED, so the oracle is not reading the engine back: %s",+ kind,+ observable,+ )+ }+ })+ }+}++// TestAnArmedFaultIsConfirmed is the other end of the same control: the same+// kinds, armed for real, must all confirm. Without this the inert control above+// is satisfied by an oracle that always says no.+func TestAnArmedFaultIsConfirmed(t *testing.T) {+ for _, kind := range []Kind{Rotate, RotateTerminate, Lock, Expire} {+ t.Run(string(kind), func(t *testing.T) {+ config := liveConfig(t, Postgres)++ ctx, cancel := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer cancel()++ tier := NewTier(Trap{+ Kind: kind,+ Role: MintedRole(),+ Minted: true,+ })++ if err := tier.Open(ctx, config, Credential{}); err != nil {+ t.Fatalf("opening the tier: %v", err)+ }++ t.Cleanup(func() {+ closing, stop := context.WithTimeout(+ context.Background(),+ liveTimeout,+ )+ defer stop()++ if err := tier.Close(closing); err != nil {+ t.Errorf("closing the tier: %v", err)+ }+ })++ if err := tier.Arm(ctx); err != nil {+ t.Fatalf("arming: %v", err)+ }++ observable, confirmed, err := tier.Observe(ctx)+ if err != nil {+ t.Fatalf("observing: %v", err)+ }++ if !confirmed {+ t.Fatalf("an armed %s arm was not confirmed", kind)+ }++ if observable == "" {+ t.Fatal("confirmed with no observable, which is not evidence")+ }+ })+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/session.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/session.gonew file mode 100644index 00000000..3432807d--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/session.go@@ -0,0 +1,190 @@+// ©AngelaMos | 2026+// session.go++package dbx++import (+ "context"+ "database/sql"+ "fmt"+ "time"+)++const (+ errHoldFailed = "khaos could not establish the session it needs to hold across the fault: %w"++ detailAlreadyLost = "this session is still closed, and khaos's administrative connection still reaches the engine: %s"+)++// Try opens a connection of its own and asks the engine one question. It never+// reuses a pooled connection, because the whole measurement is what happens at+// authentication time and a reused connection skips it.+func (c *Client) Try(ctx context.Context, cred Credential) Result {+ pool, err := sql.Open(+ c.config.Engine.driver(),+ dsn(c.config.Engine, c.config.Address, c.config.Database, cred),+ )+ if err != nil {+ return Result{+ Answer: Misconfigured,+ Detail: fmt.Sprintf(detailKhaosFault, err),+ Err: err,+ }+ }+ defer func() { _ = pool.Close() }()++ pool.SetMaxIdleConns(0)++ ctx, cancel := context.WithTimeout(ctx, dialTimeout+queryTimeout)+ defer cancel()++ var one int++ return classify(+ c.config.Engine,+ pool.QueryRowContext(ctx, probeQuery).Scan(&one),+ )+}++// Session is one connection, authenticated once and then held. It is pinned+// through sql.Conn rather than left in a pool on purpose: a checked-out+// connection is age-unbounded in database/sql, which is the shape a real+// service reproduces with an open transaction or a long poll.+type Session struct {+ client *Client+ pool *sql.DB+ conn *sql.Conn+ cred Credential++ openedAt time.Time+ lost bool+ lostWhy string+}++// Hold authenticates now and proves the session works now, so a later refusal+// is the fault rather than a credential that was never accepted.+func (c *Client) Hold(+ ctx context.Context,+ cred Credential,+) (*Session, error) {+ pool, err := sql.Open(+ c.config.Engine.driver(),+ dsn(c.config.Engine, c.config.Address, c.config.Database, cred),+ )+ if err != nil {+ return nil, fmt.Errorf(errHoldFailed, err)+ }++ pool.SetMaxOpenConns(1)+ pool.SetMaxIdleConns(1)++ dialCtx, cancel := context.WithTimeout(ctx, dialTimeout+queryTimeout)+ defer cancel()++ conn, err := pool.Conn(dialCtx)+ if err != nil {+ _ = pool.Close()+ return nil, fmt.Errorf(errHoldFailed, err)+ }++ var one int+ if err := conn.QueryRowContext(+ dialCtx,+ probeQuery,+ ).Scan(&one); err != nil {+ _ = conn.Close()+ _ = pool.Close()++ return nil, fmt.Errorf(errHoldFailed, err)+ }++ return &Session{+ client: c,+ pool: pool,+ conn: conn,+ cred: cred,+ openedAt: time.Now().UTC(),+ }, nil+}++func (s *Session) Credential() Credential {+ return s.cred+}++func (s *Session) OpenedAt() time.Time {+ return s.openedAt+}++// Query asks the held session one question. A connection-level failure is put+// to a liveness control before it is called a termination: khaos's own+// administrative connection must still reach the engine, or losing the session+// says nothing about the credential.+func (s *Session) Query(ctx context.Context) Result {+ if s.lost {+ return s.confirmStillLost(ctx)+ }++ queryCtx, cancel := context.WithTimeout(ctx, queryTimeout)+ defer cancel()++ var one int++ err := s.conn.QueryRowContext(queryCtx, probeQuery).Scan(&one)+ if err == nil {+ return served()+ }++ result := classify(s.client.config.Engine, err)++ if result.Answer == Terminated {+ s.markLost(result.Detail)+ return result+ }++ if result.Answer != Unreachable || !lostSession(err) {+ return result+ }++ if pingErr := s.client.Ping(ctx); pingErr != nil {+ return result+ }++ terminated := Result{+ Answer: Terminated,+ Detail: fmt.Sprintf(detailTerminated, err),+ Err: err,+ }+ s.markLost(terminated.Detail)++ return terminated+}++// confirmStillLost re-runs the liveness control on every later sample rather+// than latching on the first one, so the claim stays measured for as long as+// it is reported.+func (s *Session) confirmStillLost(ctx context.Context) Result {+ if err := s.client.Ping(ctx); err != nil {+ return transport(err)+ }++ return Result{+ Answer: Terminated,+ Detail: fmt.Sprintf(detailAlreadyLost, s.lostWhy),+ }+}++func (s *Session) markLost(why string) {+ s.lost = true+ s.lostWhy = why+}++func (s *Session) Close() error {+ connErr := s.conn.Close()+ poolErr := s.pool.Close()++ if connErr != nil {+ return connErr+ }++ return poolErr+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/tier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/tier.gonew file mode 100644index 00000000..0bf21ca4--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/tier.go@@ -0,0 +1,450 @@+// ©AngelaMos | 2026+// tier.go++package dbx++import (+ "context"+ "crypto/rand"+ "errors"+ "fmt"+ "strings"+ "sync"+)++const (+ rolePrefix = "khaos_cred_"+ roleNonceLen = 12+ passwordLabel = "khaos"++ errNotOpen = "%w: the credential tier was never opened, so khaos has no engine to ask"++ errAlreadyOpen = "%w: this credential tier is already open"++ errMintRole = "minting %s on the %s: %w"++ errNoOriginal = "%w: khaos has no original credential to restore for %s, so it will not guess one"++ confirmRotated = "a new connection on the rotated-away password is refused by the engine (%s) while the rotated-to password is accepted"++ confirmLocked = "a new connection presenting the correct password is refused because the account is locked (%s)"++ confirmExpired = "a new connection presenting the correct password is refused because the credential's validity has passed (%s)"++ detailNotOpen = "khaos's own credential tier is not open, so this probe measured nothing"+)++var ErrNotOpen = errors.New(+ "this credential tier has no open client",+)++var ErrAlreadyOpen = errors.New(+ "this credential tier is already open",+)++var ErrNoOriginal = errors.New(+ "khaos cannot restore a credential it never held",+)++// Tier is the per-run state of the credential tier: built by the experiment so+// the probe streams can close over it, and opened by the run, which is the only+// place that has a daemon to read an administrative credential out of.+//+// Every method is safe for the concurrent probe streams the engine fires.+type Tier struct {+ mu sync.Mutex++ trap Trap+ client *Client++ session *Session++ // current is the credential that should work right now, original is what+ // the revert restores, and previous is what the fault rotated away from.+ current Credential+ original Credential+ previous Credential++ version string+ armed bool+}++func NewTier(trap Trap) *Tier {+ return &Tier{trap: trap}+}++func (t *Tier) Trap() Trap {+ t.mu.Lock()+ defer t.mu.Unlock()++ return t.trap+}++func (t *Tier) Engine() Engine {+ t.mu.Lock()+ defer t.mu.Unlock()++ if t.client == nil {+ return ""+ }++ return t.client.Engine()+}++func (t *Tier) Version() string {+ t.mu.Lock()+ defer t.mu.Unlock()++ return t.version+}++func (t *Tier) Role() string {+ t.mu.Lock()+ defer t.mu.Unlock()++ return t.trap.Role+}++// Open dials the engine, records the version the SERVER reports, mints the role+// when the fault is one khaos owns, and holds a session that must work now so a+// later refusal is the fault rather than a credential that never worked.+//+// subject is the credential a pooled-service fault acts on, read out of the+// target's own environment by the caller; it is ignored for a minted role.+func (t *Tier) Open(+ ctx context.Context,+ config Config,+ subject Credential,+) error {+ t.mu.Lock()+ defer t.mu.Unlock()++ if t.client != nil {+ return fmt.Errorf(errAlreadyOpen, ErrAlreadyOpen)+ }++ client, err := Open(config)+ if err != nil {+ return err+ }++ if err := client.Ping(ctx); err != nil {+ _ = client.Close()+ return err+ }++ version, err := client.Version(ctx)+ if err != nil {+ _ = client.Close()+ return err+ }++ t.client = client+ t.version = version++ if t.trap.Minted {+ minted := Credential{+ Role: t.trap.Role,+ Password: newPassword(),+ }++ if err := client.CreateRole(ctx, minted); err != nil {+ _ = client.Close()+ t.client = nil++ return fmt.Errorf(errMintRole, minted.Role, config.Engine, err)+ }++ t.original = minted+ } else {+ if subject.Role == "" || subject.Password == "" {+ _ = client.Close()+ t.client = nil++ return fmt.Errorf(errNoOriginal, ErrNoOriginal, t.trap.Role)+ }++ t.original = subject+ }++ t.current = t.original++ session, err := client.Hold(ctx, t.current)+ if err != nil {+ return errors.Join(err, t.closeLocked(ctx))+ }++ t.session = session++ return nil+}++// MintedRole is the role name khaos will create for a fault it owns, generated+// rather than fixed so two runs cannot collide on one engine.+func MintedRole() string {+ return rolePrefix + strings.ToLower(rand.Text()[:roleNonceLen])+}++func newPassword() string {+ return passwordLabel + rand.Text()+}++// Arm performs the mutation. It prepares everything first and tells the engine+// only when the trap is not inert, so an inert arm leaves a state the+// confirmation has to reject.+func (t *Tier) Arm(ctx context.Context) error {+ t.mu.Lock()+ defer t.mu.Unlock()++ if t.client == nil {+ return fmt.Errorf(errNotOpen, ErrNotOpen)+ }++ replacement := Credential{+ Role: t.trap.Role,+ Password: newPassword(),+ }++ if t.trap.Inert {+ replacement = t.current+ }++ t.previous = t.current++ switch t.trap.Kind {+ case Rotate, PoolRotate, RotateTerminate:+ if err := t.client.Rotate(ctx, replacement); err != nil {+ return err+ }++ t.current = replacement++ if t.trap.Kind != RotateTerminate || t.trap.Inert {+ break+ }++ if _, err := t.client.Terminate(ctx, t.trap.Role); err != nil {+ return err+ }++ case Lock:+ if t.trap.Inert {+ break+ }+ if err := t.client.Lock(ctx, t.trap.Role); err != nil {+ return err+ }++ case Expire:+ if t.trap.Inert {+ break+ }+ if err := t.client.Expire(ctx, t.trap.Role); err != nil {+ return err+ }++ default:+ return t.trap.Validate()+ }++ t.armed = true++ return nil+}++// Observe is the two-sided confirmation. It never reads khaos's own armed+// state: ALTER ROLE succeeds when it changes nothing, which is the same class+// of answer as a pause the daemon acknowledges.+func (t *Tier) Observe(ctx context.Context) (string, bool, error) {+ t.mu.Lock()+ client := t.client+ trap := t.trap+ previous := t.previous+ current := t.current+ t.mu.Unlock()++ if client == nil {+ return "", false, fmt.Errorf(errNotOpen, ErrNotOpen)+ }++ if trap.RotatesPassword() {+ away := client.Try(ctx, previous)+ if away.Answer != Refused {+ return "", false, nil+ }++ if live := client.Try(ctx, current); live.Answer != Served {+ return "", false, nil+ }++ return fmt.Sprintf(confirmRotated, away.Detail), true, nil+ }++ answer := client.Try(ctx, current)++ switch trap.Kind {+ case Lock:+ if answer.Answer != Locked {+ return "", false, nil+ }++ return fmt.Sprintf(confirmLocked, answer.Detail), true, nil++ case Expire:+ // PostgreSQL refuses an expired credential with the string it also uses+ // for a wrong password, so the proof is that THIS password was served+ // during the baseline and is refused now.+ switch answer.Answer {+ case Expired, MustReset, Refused:+ return fmt.Sprintf(confirmExpired, answer.Detail), true, nil+ default:+ return "", false, nil+ }++ default:+ return "", false, nil+ }+}++// Disarm RESTORES and never removes, including for a role khaos minted. The+// recovery window measures the engine serving the credential again, so dropping+// the role here would make khaos's own cleanup read as a target that never came+// back — measured, and it reported a degraded recovery on a run where nothing+// about the engine had degraded. Removal belongs to Close, which runs after the+// recovery window, and to the cold-start revert, where there is no window to+// measure.+func (t *Tier) Disarm(ctx context.Context) error {+ t.mu.Lock()+ defer t.mu.Unlock()++ if t.client == nil {+ return fmt.Errorf(errNotOpen, ErrNotOpen)+ }++ var failures []error++ if t.original.Password == "" {+ return fmt.Errorf(errNoOriginal, ErrNoOriginal, t.trap.Role)+ }++ switch t.trap.Kind {+ case Lock:+ if err := t.client.Unlock(ctx, t.trap.Role); err != nil {+ failures = append(failures, err)+ }+ case Expire:+ if err := t.client.Unexpire(ctx, t.original); err != nil {+ failures = append(failures, err)+ }+ default:+ if err := t.client.Rotate(ctx, t.original); err != nil {+ failures = append(failures, err)+ }+ }++ if len(failures) == 0 {+ t.current = t.original+ t.armed = false+ }++ return errors.Join(failures...)+}++// Current is the valid probe stream: a fresh connection presenting whatever+// credential should work right now.+func (t *Tier) Current(ctx context.Context) Result {+ t.mu.Lock()+ client := t.client+ cred := t.current+ t.mu.Unlock()++ if client == nil {+ return notOpen()+ }++ return client.Try(ctx, cred)+}++// Wrong is the negative control, and it is the control research 07 section 6.1+// was missing when a loopback `trust` rule made every password look accepted.+// A fresh password per sample, so nothing can be answered from a cache.+func (t *Tier) Wrong(ctx context.Context) Result {+ t.mu.Lock()+ client := t.client+ t.mu.Unlock()++ if client == nil {+ return notOpen()+ }++ return client.Try(ctx, t.wrong())+}++// wrong is separate so the freshness property has a control. The engine cannot+// tell khaos whether the password it refused was a new one, so the only place+// this is observable is here.+func (t *Tier) wrong() Credential {+ t.mu.Lock()+ defer t.mu.Unlock()++ return Credential{Role: t.trap.Role, Password: newPassword()}+}++// Held is the revoked stream: the session that authenticated before the fault+// and was never asked again.+func (t *Tier) Held(ctx context.Context) Result {+ t.mu.Lock()+ session := t.session+ t.mu.Unlock()++ if session == nil {+ return notOpen()+ }++ return session.Query(ctx)+}++func notOpen() Result {+ return Result{Answer: Misconfigured, Detail: detailNotOpen}+}++func (t *Tier) Close(ctx context.Context) error {+ t.mu.Lock()+ defer t.mu.Unlock()++ return t.closeLocked(ctx)+}++func (t *Tier) closeLocked(ctx context.Context) error {+ var failures []error++ if t.session != nil {+ if err := t.session.Close(); err != nil {+ failures = append(failures, err)+ }+ t.session = nil+ }++ if t.client == nil {+ return errors.Join(failures...)+ }++ // A minted role must not outlive the run even when the revert already ran,+ // because the lease exists to make that true without khaos's cooperation.+ if t.trap.Minted {+ if _, err := t.client.Terminate(ctx, t.trap.Role); err != nil {+ failures = append(failures, err)+ }+ if err := t.client.DropRole(ctx, t.trap.Role); err != nil {+ failures = append(failures, err)+ }+ }++ if err := t.client.Close(); err != nil {+ failures = append(failures, err)+ }++ t.client = nil++ return errors.Join(failures...)+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/trap.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/trap.gonew file mode 100644index 00000000..19319b10--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dbx/trap.go@@ -0,0 +1,125 @@+// ©AngelaMos | 2026+// trap.go++package dbx++import (+ "errors"+ "fmt"+ "slices"+ "strings"+)++type Kind string++const (+ // Rotate is the headline: change the password and leave every live session+ // alone, which is what an operator believes revokes access.+ Rotate Kind = "rotate"++ // RotateTerminate is the contrast that makes the pair worth running. One+ // statement more, and the same fault inverts.+ RotateTerminate Kind = "rotate_terminate"++ Lock Kind = "lock"+ Expire Kind = "expire"++ // PoolRotate acts on a role the TARGET is using, so the finding is a+ // property of the target's connection pool rather than of the engine.+ PoolRotate Kind = "pool_rotate"+)++const (+ errUnknownTrap = "%w: %q (known credential faults: %s)"+ errNoRole = "%w: a credential fault needs the role it acts on"++ describeRotate = "rotate %s's password at the %s it authenticates against, and leave every established session alone"++ describeRotateTerminate = "rotate %s's password at the %s it authenticates against and then terminate its sessions, which is the only control of the three that reaches a live connection"++ describeLock = "lock %s's account at the %s it authenticates against, which is what an incident runbook reaches for when a credential leaks"++ describeExpire = "put %s's credential validity in the past at the %s it authenticates against"++ describePoolRotate = "rotate the password %s's own connection pool authenticated with, without telling %s"+)++var ErrUnknownTrap = errors.New(+ "no credential fault is registered under that name",+)++var ErrNoRole = errors.New(+ "this credential fault has no role to act on",+)++// Trap is one credential fault. Minted records whether khaos created the role,+// which decides whether the revert restores a password or drops the role+// outright.+type Trap struct {+ Kind Kind `json:"kind"`+ Role string `json:"role"`+ Minted bool `json:"minted"`++ // Inert is the negative control. Arm still prepares everything, including+ // the replacement credential, and simply never tells the engine, so a+ // confirmation that reads the engine back MUST say no.+ Inert bool `json:"inert,omitempty"`+}++func Kinds() []string {+ return []string{+ string(Expire),+ string(Lock),+ string(PoolRotate),+ string(Rotate),+ string(RotateTerminate),+ }+}++func (t Trap) Validate() error {+ if !slices.Contains(Kinds(), string(t.Kind)) {+ return fmt.Errorf(+ errUnknownTrap,+ ErrUnknownTrap,+ t.Kind,+ strings.Join(Kinds(), ", "),+ )+ }++ if t.Role == "" {+ return fmt.Errorf(errNoRole, ErrNoRole)+ }++ return validate(t.Role)+}++// Describe names the engine as well as the role, because the same fault means+// something different against an engine that re-authenticates and none of the+// three do.+func (t Trap) Describe(target string, engine Engine) string {+ switch t.Kind {+ case Rotate:+ return fmt.Sprintf(describeRotate, t.Role, engine)+ case RotateTerminate:+ return fmt.Sprintf(describeRotateTerminate, t.Role, engine)+ case Lock:+ return fmt.Sprintf(describeLock, t.Role, engine)+ case Expire:+ return fmt.Sprintf(describeExpire, t.Role, engine)+ case PoolRotate:+ return fmt.Sprintf(describePoolRotate, target, target)+ default:+ return string(t.Kind)+ }+}++// RotatesPassword reports whether this fault replaces the credential, which is+// what the two-sided confirmation reads back.+func (t Trap) RotatesPassword() bool {+ switch t.Kind {+ case Rotate, RotateTerminate, PoolRotate:+ return true+ default:+ return false+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.goindex 7578ab73..9a4ac000 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go@@ -7,6 +7,7 @@ import ("context""errors""fmt"+ "slices""strconv""strings"@@ -27,6 +28,7 @@ const (errEnv = "read the environment of %s: %w"errAddress = "read the address of %s on network %s: %w"errNoAddress = "%s has no address on network %s"+ errNoNetwork = "%s has no network address at all, so khaos has nothing to dial it on"errBridge = "read the gateway of the default bridge network: %w"errNoBridge = "the default bridge network reports no gateway address, so khaos cannot find an address a container can reach it on"@@ -198,6 +200,38 @@ func (c *Client) NetworkAddress(return settings.IPAddress.String(), nil}+// OwnAddress is where khaos dials a target DIRECTLY, on the container's own+// address rather than on a published host port. The host routes to a bridge+// network's addresses, which is what makes a stateful dependency reachable with+// nothing published, and the network with the lowest name is chosen so two runs+// against a multi-homed container agree.+func (c *Client) OwnAddress(+ ctx context.Context,+ containerID string,+) (string, string, error) {+ endpoints, err := c.Endpoints(ctx, containerID)+ if err != nil {+ return "", "", err+ }++ names := make([]string, 0, len(endpoints))+ for name := range endpoints {+ names = append(names, name)+ }+ slices.Sort(names)++ for _, name := range names {+ settings := endpoints[name]+ if settings == nil || !settings.IPAddress.IsValid() {+ continue+ }++ return name, settings.IPAddress.String(), nil+ }++ return "", "", fmt.Errorf(errNoNetwork, ShortID(containerID))+}+// BridgeGateway is the address a container reaches the host on: the DEFAULT// bridge's gateway, discovered rather than assumed, since it is not 172.17.0.1// on every host. A user-defined network's own gateway is that network's defaultdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.goindex 52b13e96..91b10610 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go@@ -67,6 +67,10 @@ type Shape struct {ResolverNetwork stringDNSHost string+ // Grouped rather than spread: this tier needs seven inputs and Shape+ // already carries one field per tier.+ Cred *CredSpec+// Separate fields because the target addresses khaos by a name khaos cannot// see from its own listener, so neither can be derived from the other.AuthorityBase string@@ -178,6 +182,12 @@ var catalog = map[string]Definition{},}+// catalogAdd is how a family registers from its own init, which keeps every+// family's definitions in the file that explains them.+func catalogAdd(definition Definition) {+ catalog[definition.Name] = definition+}+func Names() []string {names := make([]string, 0, len(catalog))for name := range catalog {@@ -266,6 +276,7 @@ func Build(DNSTrap: shape.DNSTrap,ResolverNetwork: shape.ResolverNetwork,DNSHost: shape.DNSHost,+ Cred: shape.Cred,IdleFor: shape.IdleFor,Act: shape.Act,ActDetail: shape.ActDetail,diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credential.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credential.gonew file mode 100644index 00000000..7818e24a--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credential.go@@ -0,0 +1,168 @@+// ©AngelaMos | 2026+// credential.go++package experiment++import (+ "fmt"++ "github.com/CarterPerez-dev/khaos/internal/dbx"+ "github.com/CarterPerez-dev/khaos/internal/inject"+)++const (+ FamilyCredential = "credential"++ NameDBRotate = "credential.db_rotate"+ NameDBRotateTerminate = "credential.db_rotate_terminate"+ NameDBAccountLock = "credential.db_account_lock"+ NameDBExpiry = "credential.db_credential_expiry"++ summaryDBRotate = "rotate a database password and find out whether that revoked anything, which on all three engines it does not"++ summaryDBRotateTerminate = "rotate a database password AND terminate the sessions, which is the same fault plus the one statement that makes it a revocation"++ summaryDBAccountLock = "lock a database account, which an incident runbook reaches for when a credential leaks, and see what it does to a connection that is already open"++ summaryDBExpiry = "put a database credential's validity in the past and see whether an established session notices"++ conditionDBRotate = "%s's password being rotated at the engine with no session terminated"++ conditionDBRotateTerminate = "%s's password being rotated at the engine and its sessions terminated"++ conditionDBAccountLock = "%s's account being locked at the engine"++ conditionDBExpiry = "%s's credential validity being put in the past at the engine"++ responseRevoked = "losing access on the credential that was taken away, rather than serving on it for the life of a connection that authenticated before the change"++ responseTerminated = "closing the sessions that authenticated with the credential that was replaced"++ defectDBRotate = "rotating its password at the engine and terminating nothing"++ defectDBRotateTerminate = "rotating its password at the engine and terminating its sessions"++ defectDBAccountLock = "locking its account at the engine"++ defectDBExpiry = "putting its credential validity in the past at the engine"++ caveatTerminated = "the session khaos held was closed by the engine, which is what makes termination the only one of these controls that reaches a connection that is already authenticated"++ caveatRotated = "khaos held one session across the rotation; a target with a connection pool holds as many as its pool is wide, for as long as its own configuration allows"+)++// credentialPolarity is PERMITTED for every experiment in this family, and the+// rationale is the whole teaching point: an operator rotates a credential in+// order to revoke it, and on these engines rotation is a change to the+// admission policy for FUTURE connections and nothing else.+var credentialPolarity = Polarity{+ Defects: []Direction{Permitted},+ Rationale: "a credential that keeps working after it was taken away at the authority is access that outlived its own revocation",+}++func buildCredential(+ trap dbx.Trap,+ condition, defect, caveat string,+) func(*Params) (Shape, error) {+ return func(params *Params) (Shape, error) {+ target := params.Get(ParamTarget)++ spec, err := credSpec(params)+ if err != nil {+ return Shape{}, err+ }++ armed := trap+ armed.Role = dbx.MintedRole()+ armed.Minted = true++ spec.Tier = dbx.NewTier(armed)++ return Shape{+ Condition: fmt.Sprintf(condition, target),+ Target: target,+ Probes: credProbes(spec, spec.Engine),+ Cred: spec,+ Defect: defect,+ HeldCaveat: caveat,+ }, nil+ }+}++func init() {+ catalogAdd(Definition{+ Name: NameDBRotate,+ Family: FamilyCredential,+ Summary: summaryDBRotate,+ Hypothesis: Hypothesis{+ Condition: "a database password being rotated with no session terminated",+ Response: responseRevoked,+ },+ Polarity: credentialPolarity,+ FaultKind: inject.CredKind(dbx.Rotate),+ Params: credParams,+ Build: buildCredential(+ dbx.Trap{Kind: dbx.Rotate},+ conditionDBRotate,+ defectDBRotate,+ caveatRotated,+ ),+ })++ catalogAdd(Definition{+ Name: NameDBRotateTerminate,+ Family: FamilyCredential,+ Summary: summaryDBRotateTerminate,+ Hypothesis: Hypothesis{+ Condition: "a database password being rotated and its sessions terminated",+ Response: responseTerminated,+ },+ Polarity: credentialPolarity,+ FaultKind: inject.CredKind(dbx.RotateTerminate),+ Params: credParams,+ Build: buildCredential(+ dbx.Trap{Kind: dbx.RotateTerminate},+ conditionDBRotateTerminate,+ defectDBRotateTerminate,+ caveatTerminated,+ ),+ })++ catalogAdd(Definition{+ Name: NameDBAccountLock,+ Family: FamilyCredential,+ Summary: summaryDBAccountLock,+ Hypothesis: Hypothesis{+ Condition: "a database account being locked at the engine",+ Response: responseRevoked,+ },+ Polarity: credentialPolarity,+ FaultKind: inject.CredKind(dbx.Lock),+ Params: credParams,+ Build: buildCredential(+ dbx.Trap{Kind: dbx.Lock},+ conditionDBAccountLock,+ defectDBAccountLock,+ caveatRotated,+ ),+ })++ catalogAdd(Definition{+ Name: NameDBExpiry,+ Family: FamilyCredential,+ Summary: summaryDBExpiry,+ Hypothesis: Hypothesis{+ Condition: "a database credential's validity being put in the past",+ Response: responseRevoked,+ },+ Polarity: credentialPolarity,+ FaultKind: inject.CredKind(dbx.Expire),+ Params: credParams,+ Build: buildCredential(+ dbx.Trap{Kind: dbx.Expire},+ conditionDBExpiry,+ defectDBExpiry,+ caveatRotated,+ ),+ })+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.gonew file mode 100644index 00000000..adb1e8b5--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go@@ -0,0 +1,198 @@+// ©AngelaMos | 2026+// credtier.go++package experiment++import (+ "context"+ "fmt"++ "github.com/CarterPerez-dev/khaos/internal/dbx"+ "github.com/CarterPerez-dev/khaos/internal/probe"+)++const (+ ParamEngine = "engine"+ ParamEnginePort = "engine-port"+ ParamDatabase = "database"+ ParamAdminRole = "admin-role"+ ParamAdminVar = "admin-password-var"++ usageEngine = "which database engine the target runs; the range fills this from the container's own image"++ usageEnginePort = "the port the engine listens on INSIDE its container, which khaos dials directly rather than through a published port"++ usageDatabase = "the database khaos's own administrative connection uses; PostgreSQL requires one and the MySQL family does not"++ usageAdminRole = "the role khaos performs the rotation as, which is never the role under test"++ usageAdminVar = "the environment variable on the target container that holds the administrative password; khaos reads it from the container rather than taking it on a flag, because a flag is world-readable in /proc and a journal entry would outlive the run"++ errNoEngine = "this experiment needs --%s (one of %s), because the fault is performed over the engine's own administrative protocol and the three do not share one"++ probeCurrentName = "current-credential"+ probeWrongName = "never-valid-credential"+ probeHeldName = "session-held-across-the-fault"++ describeCurrent = "a NEW connection to the %s presenting whatever credential should work right now"++ describeWrong = "a NEW connection to the %s presenting a password that was never correct, minted fresh per sample so nothing can answer it from a cache"++ describeHeld = "the session that authenticated BEFORE the fault, queried again and never re-authenticated"+)++// CredSpec is grouped rather than spread across Shape, which already carries+// one field per tier and would otherwise carry seven for this one.+type CredSpec struct {+ Tier *dbx.Tier++ Engine dbx.Engine+ Port string+ Database string++ Admin string+ AdminVar string++ // Subject is the role a pooled-service fault acts on, which belongs to the+ // target rather than to khaos. Empty for a role khaos mints.+ Subject string+ SubjectVar string+}++// credDefaults are per-engine and pure, so Build resolves them without a+// daemon. Each is what the official image ships.+type credDefaults struct {+ port string+ database string+ admin string+ adminVar string+}++var credByEngine = map[dbx.Engine]credDefaults{+ dbx.Postgres: {+ port: "5432",+ database: "postgres",+ admin: "postgres",+ adminVar: "POSTGRES_PASSWORD",+ },+ dbx.MySQL: {+ port: "3306",+ database: "",+ admin: "root",+ adminVar: "MYSQL_ROOT_PASSWORD",+ },+ dbx.MariaDB: {+ port: "3306",+ database: "",+ admin: "root",+ adminVar: "MARIADB_ROOT_PASSWORD",+ },+}++var credParams = []Param{+ {Name: ParamTarget, Usage: usageTargetEngine, Required: true},+ {Name: ParamEngine, Usage: usageEngine},+ {Name: ParamEnginePort, Usage: usageEnginePort},+ {Name: ParamDatabase, Usage: usageDatabase},+ {Name: ParamAdminRole, Usage: usageAdminRole},+ {Name: ParamAdminVar, Usage: usageAdminVar},+}++const usageTargetEngine = "the consenting container running the database whose credential you want to rotate"++// credSpec resolves the engine and then fills every unspecified input from what+// that engine's official image ships, so a run against the range needs no flags+// and an operator's own target needs only the engine.+func credSpec(params *Params) (*CredSpec, error) {+ raw := params.Get(ParamEngine)+ if raw == "" {+ return nil, fmt.Errorf(+ errNoEngine,+ ParamEngine,+ joinNames(dbx.Engines()),+ )+ }++ engine, err := dbx.ParseEngine(raw)+ if err != nil {+ return nil, err+ }++ defaults := credByEngine[engine]++ spec := &CredSpec{+ Engine: engine,+ Port: orElse(params.Get(ParamEnginePort), defaults.port),+ Database: orElse(params.Get(ParamDatabase), defaults.database),+ Admin: orElse(params.Get(ParamAdminRole), defaults.admin),+ AdminVar: orElse(params.Get(ParamAdminVar), defaults.adminVar),+ }++ return spec, nil+}++func orElse(value, fallback string) string {+ if value == "" {+ return fallback+ }+ return value+}++func joinNames(names []string) string {+ out := ""+ for i, name := range names {+ if i > 0 {+ out += nameSep+ }+ out += name+ }+ return out+}++// credProbes wires the three streams onto the tier. Every one of them is a+// fresh dial except the held session, which is the whole point: authentication+// happens once per connection on all three engines.+func credProbes(spec *CredSpec, engine dbx.Engine) probe.Pair {+ return probe.Pair{+ Valid: probe.NewCall(probe.CallOptions{+ Name: probeCurrentName,+ Describe: fmt.Sprintf(describeCurrent, engine),+ Credential: probe.Valid,+ Call: func(ctx context.Context) (probe.Disposition, string) {+ return dispositionOf(spec.Tier.Current(ctx))+ },+ }),+ Invalid: probe.NewCall(probe.CallOptions{+ Name: probeWrongName,+ Describe: fmt.Sprintf(describeWrong, engine),+ Credential: probe.Invalid,+ Call: func(ctx context.Context) (probe.Disposition, string) {+ return dispositionOf(spec.Tier.Wrong(ctx))+ },+ }),+ Revoked: probe.NewCall(probe.CallOptions{+ Name: probeHeldName,+ Describe: describeHeld,+ Credential: probe.Revoked,+ Call: func(ctx context.Context) (probe.Disposition, string) {+ return dispositionOf(spec.Tier.Held(ctx))+ },+ }),+ }+}++// dispositionOf maps what the engine said to what it means about the+// credential. Every refusal shape is a refusal; khaos's own problems are+// excluded from every rate rather than counted as target behaviour.+func dispositionOf(result dbx.Result) (probe.Disposition, string) {+ switch result.Answer {+ case dbx.Served:+ return probe.Accepted, result.Detail+ case dbx.Refused, dbx.Locked, dbx.Expired, dbx.MustReset, dbx.Terminated:+ return probe.Rejected, result.Detail+ case dbx.Unreachable:+ return probe.Unavailable, result.Detail+ default:+ return probe.ProbeFailed, result.Detail+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.goindex d20494b3..9f40d649 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go@@ -126,6 +126,10 @@ type Experiment struct {ResolverNetwork stringDNSHost string+ // Cred is the one authority khaos does not replace, so the run opens a+ // client for a stock engine instead of standing a fake one up.+ Cred *CredSpec+Defect stringHeldCaveat stringdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.goindex 80305cb3..4b1b8f85 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go@@ -14,6 +14,7 @@ import ("testing""time"+ "github.com/CarterPerez-dev/khaos/internal/dbx""github.com/CarterPerez-dev/khaos/internal/probe")@@ -26,10 +27,23 @@ const (// that declares a default takes it, because a typed parameter such as a duration// or a byte count cannot accept an arbitrary sentinel and a build that refuses the// value never reaches the assertion under test.+// legalSample is for a parameter whose domain is not "any string". Every other+// parameter takes a sentinel, which is what proves the builder reads the value+// it was given rather than a literal of its own. A constrained parameter has to+// be given something its builder will accept, or the refusal under test is the+// domain check rather than the thing the test is about.+var legalSample = map[string]string{+ ParamEngine: string(dbx.Postgres),+}+func fill(declared []Param) map[string]string {supplied := map[string]string{}for _, param := range declared {+ if sample, constrained := legalSample[param.Name]; constrained {+ supplied[param.Name] = sample+ continue+ }if param.Default != "" {supplied[param.Name] = param.Defaultcontinue@@ -331,6 +345,39 @@ func TestTheNegativeControlCredentialIsNotAcceptableToAnyVerifier(}}+// httpControlled reports whether this experiment's negative control is an HTTP+// probe, which is the only transport this test can observe. The freshness+// property itself is not optional for the others: it lives in the tier that+// owns the transport, and dbx's own suite proves the credential tier mints a+// new password per sample.+//+// The non-HTTP set is pinned by FAMILY rather than derived, because a derived+// exemption would silently swallow a sixth family that shipped a fixed control+// credential by accident. A new family lands here and has to be looked at.+func httpControlled(t *testing.T, definition Definition) bool {+ t.Helper()++ supplied := fill(definition.Params)++ spec, err := Build(definition.Name, supplied, fastTiming)+ if err != nil {+ t.Fatalf("%s: build: %v", definition.Name, err)+ }++ _, overHTTP := spec.Probes.Invalid.(*probe.HTTPProbe)++ if overHTTP == (definition.Family == FamilyCredential) {+ t.Fatalf(+ "%s is in family %q and its negative control is over HTTP = %v; the credential family is the only one whose control is not an HTTP probe, so one of the two needs updating",+ definition.Name,+ definition.Family,+ overHTTP,+ )+ }++ return overHTTP+}+func flagSpelling(name string) string {return "--" + name}@@ -341,6 +388,10 @@ func flagSpelling(name string) string {// reason both ends need a control.func TestTheCatalogueAsksForAFreshNegativeControlCredential(t *testing.T) {for _, definition := range Catalog() {+ if !httpControlled(t, definition) {+ continue+ }+t.Run(definition.Name, func(t *testing.T) {var mu sync.Mutexseen := map[string]bool{}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/credfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/credfault.gonew file mode 100644index 00000000..8b155c4e--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/credfault.go@@ -0,0 +1,268 @@+// ©AngelaMos | 2026+// credfault.go++package inject++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"++ "github.com/CarterPerez-dev/khaos/internal/dbx"+ "github.com/CarterPerez-dev/khaos/internal/dockerx"+ "github.com/CarterPerez-dev/khaos/internal/journal"+)++const (+ KindCredPrefix = "cred-"++ errNoCredTier = "%w: %s needs khaos's credential tier, and this run has none"++ errNoCredRecipe = "%w: this credential fault has nothing recorded about how to reach the engine again"++ errCredSecretGone = "%s is not in %s's environment any more, so khaos cannot re-read the credential it would need to undo this"++ describeCred = "%s, which khaos performs over the engine's own administrative protocol rather than through the daemon"+)++var ErrNoCredTier = errors.New(+ "this mutation needs khaos's credential tier, which is not open",+)++var ErrNoCredRecipe = errors.New(+ "this credential fault cannot be reverted from a cold start",+)++// CredSource is where a credential is READ FROM rather than the credential.+// The journal records this and never the secret: a flag would put it in+// /proc/<pid>/cmdline, which is world-readable on a normal host, and a journal+// entry would leave it on disk after the run.+type CredSource struct {+ ContainerID string `json:"container_id"`+ Role string `json:"role"`+ PasswordVar string `json:"password_var"`+}++// CredPlan is everything a cold-started khaos needs to undo a credential fault.+type CredPlan struct {+ Engine dbx.Engine `json:"engine"`+ Address string `json:"address"`+ Database string `json:"database"`+ Admin CredSource `json:"admin"`+ Subject *CredSource `json:"subject,omitempty"`+}++func CredKind(kind dbx.Kind) string {+ return KindCredPrefix + string(kind)+}++// credMutator's authority is a stock database that behaves CORRECTLY. Every+// other tier exists because no correct dependency emits the defect; here the+// finding is that three correct engines never re-evaluate authentication.+type credMutator struct {+ kind dbx.Kind+}++func init() {+ for _, name := range dbx.Kinds() {+ kind := dbx.Kind(name)+ registry[CredKind(kind)] = credMutator{kind: kind}+ }+}++func (m credMutator) Describe(env Env, target dockerx.Target) string {+ trap := env.CredTrap+ if trap.Kind != m.kind {+ trap = dbx.Trap{Kind: m.kind, Role: trap.Role}+ }++ return fmt.Sprintf(+ describeCred,+ trap.Describe(target.Name, env.CredPlan.Engine),+ )+}++type credPayload struct {+ Trap dbx.Trap `json:"trap"`+ Plan CredPlan `json:"plan"`+}++func (m credMutator) Plan(+ _ context.Context,+ env Env,+ _ dockerx.Target,+) (json.RawMessage, error) {+ if env.Cred == nil {+ return nil, fmt.Errorf(errNoCredTier, ErrNoCredTier, CredKind(m.kind))+ }++ trap := env.Cred.Trap()++ if trap.Kind != m.kind {+ return nil, fmt.Errorf(+ errWrongTrapKind,+ ErrTrapMismatch,+ trap.Kind,+ m.kind,+ )+ }++ if err := trap.Validate(); err != nil {+ return nil, err+ }++ return journal.EncodePayload(credPayload{Trap: trap, Plan: env.CredPlan})+}++func (m credMutator) Apply(+ ctx context.Context,+ env Env,+ _ dockerx.Target,+ _ json.RawMessage,+) error {+ if env.Cred == nil {+ return fmt.Errorf(errNoCredTier, ErrNoCredTier, CredKind(m.kind))+ }++ return env.Cred.Arm(ctx)+}++func (m credMutator) Confirm(+ ctx context.Context,+ env Env,+ _ dockerx.Target,+) (string, bool, error) {+ if env.Cred == nil {+ return "", false, fmt.Errorf(+ errNoCredTier,+ ErrNoCredTier,+ CredKind(m.kind),+ )+ }++ return env.Cred.Observe(ctx)+}++// Revert prefers the open tier, which knows the credential it replaced. From a+// cold start it rebuilds a client out of the journal's recipe and re-reads both+// credentials from the containers that hold them.+func (m credMutator) Revert(+ ctx context.Context,+ env Env,+ record journal.Record,+) error {+ if env.Cred != nil {+ return env.Cred.Disarm(ctx)+ }++ var payload credPayload+ if len(record.Payload) == 0 {+ return fmt.Errorf(errNoCredRecipe, ErrNoCredRecipe)+ }+ if err := record.DecodePayload(&payload); err != nil {+ return err+ }++ return revertFromRecipe(ctx, env, payload)+}++func revertFromRecipe(+ ctx context.Context,+ env Env,+ payload credPayload,+) error {+ if env.Docker == nil || payload.Plan.Address == "" {+ return fmt.Errorf(errNoCredRecipe, ErrNoCredRecipe)+ }++ admin, err := readSource(ctx, env.Docker, payload.Plan.Admin)+ if err != nil {+ return err+ }++ client, err := dbx.Open(dbx.Config{+ Engine: payload.Plan.Engine,+ Address: payload.Plan.Address,+ Database: payload.Plan.Database,+ Admin: admin,+ })+ if err != nil {+ return err+ }+ defer func() { _ = client.Close() }()++ var failures []error++ // A role khaos minted takes the fault with it, so the revert is removal+ // rather than restoration and there is nothing to guess.+ if payload.Trap.Minted {+ if _, err := client.Terminate(ctx, payload.Trap.Role); err != nil {+ failures = append(failures, err)+ }+ if err := client.DropRole(ctx, payload.Trap.Role); err != nil {+ failures = append(failures, err)+ }++ return errors.Join(failures...)+ }++ if payload.Plan.Subject == nil {+ return fmt.Errorf(errNoCredRecipe, ErrNoCredRecipe)+ }++ subject, err := readSource(ctx, env.Docker, *payload.Plan.Subject)+ if err != nil {+ return err+ }++ switch payload.Trap.Kind {+ case dbx.Lock:+ return client.Unlock(ctx, payload.Trap.Role)+ case dbx.Expire:+ return client.Unexpire(ctx, subject)+ default:+ return client.Rotate(ctx, subject)+ }+}++// readSource re-reads a credential from the container that owns it. The+// container is the only authority on it, and the value never travels through+// khaos's own state.+func readSource(+ ctx context.Context,+ docker *dockerx.Client,+ source CredSource,+) (dbx.Credential, error) {+ environ, err := docker.ContainerEnv(ctx, source.ContainerID)+ if err != nil {+ return dbx.Credential{}, err+ }++ password, found := environ[source.PasswordVar]+ if !found || password == "" {+ return dbx.Credential{}, fmt.Errorf(+ errCredSecretGone,+ source.PasswordVar,+ dockerx.ShortID(source.ContainerID),+ )+ }++ return dbx.Credential{Role: source.Role, Password: password}, nil+}++// NeedsCred tells the run to open a credential tier before the baseline,+// derived from the registry rather than from the kind's spelling.+func NeedsCred(kind string) bool {+ mutator, ok := registry[kind]+ if !ok {+ return false+ }++ _, isCred := mutator.(credMutator)++ return isCred+}++// Prefix declares this tier's kinds to the registry.+func (credMutator) Prefix() string { return KindCredPrefix }diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.goindex 9cd8fca9..b701cfde 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go@@ -11,6 +11,7 @@ import ("slices""strings"+ "github.com/CarterPerez-dev/khaos/internal/dbx""github.com/CarterPerez-dev/khaos/internal/dnsrv""github.com/CarterPerez-dev/khaos/internal/dockerx""github.com/CarterPerez-dev/khaos/internal/idp"@@ -58,6 +59,13 @@ type Env struct {Resolver *dnsrv.ClientResolverNetwork stringDNSTrap dnsrv.Trap++ // The one authority that behaves CORRECTLY, so khaos drives a stock engine+ // instead of replacing it. CredPlan is how a cold start reaches it again,+ // and it names where each credential is read from rather than carrying one.+ Cred *dbx.Tier+ CredTrap dbx.Trap+ CredPlan CredPlan}// Mutator is one injectable fault. Plan returns what the revert needs so thediff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/call.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/call.gonew file mode 100644index 00000000..0c73525f--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/call.go@@ -0,0 +1,65 @@+// ©AngelaMos | 2026+// call.go++package probe++import (+ "context"+ "time"+)++// CallOptions is a probe whose transport is not HTTP. The caller supplies the+// one function that asks a question and says what the answer was, which keeps+// this package free of every protocol khaos learns to speak.+type CallOptions struct {+ Name string+ Describe string+ Credential Credential+ Call func(context.Context) (Disposition, string)+}++type CallProbe struct {+ options CallOptions+}++func NewCall(options CallOptions) *CallProbe {+ return &CallProbe{options: options}+}++func (p *CallProbe) Name() string {+ return p.options.Name+}++func (p *CallProbe) Credential() Credential {+ return p.options.Credential+}++func (p *CallProbe) Describe() string {+ return p.options.Describe+}++func (p *CallProbe) Run(ctx context.Context) Outcome {+ started := time.Now().UTC()++ outcome := Outcome{+ At: started,+ Credential: p.options.Credential,+ Disposition: Unknown,+ }++ if p.options.Call == nil {+ outcome.Disposition = ProbeFailed+ outcome.Detail = detailNoCall+ return outcome+ }++ disposition, detail := p.options.Call(ctx)++ outcome.SetLatency(time.Since(started))+ outcome.Disposition = disposition+ outcome.Detail = detail++ return outcome+}++const detailNoCall = "this probe was built with nothing to call, which is khaos's own fault and not a target behaviour"diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.goindex ea05c8d2..5b4bc140 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go@@ -48,7 +48,15 @@ const (DNSControlName = "control.khaos.invalid."- IDPName = "khaos-range-idp"+ IDPName = "khaos-range-idp"++ // The stateful tier, opt-in behind `range up --with-db`. These are the only+ // range services khaos does not implement itself.+ PostgresName = "khaos-range-postgres"+ MySQLName = "khaos-range-mysql"+ MariaDBName = "khaos-range-mariadb"+ RedisName = "khaos-range-redis"+LabelRange = "khaos.range"FailModeOpen = "open"FailModeShut = "closed"@@ -78,6 +86,14 @@ const (EnvTLSMode = "KHAOS_RANGE_TLS_MODE"+ // Minted per `range up` rather than written into the compose file, so the+ // public repo carries no credential and khaos reads it back out of the+ // container that holds it.+ EnvDBPassword = "KHAOS_RANGE_DB_PASSWORD"++ profileFlag = "--profile"+ dbProfile = "db"+composeDir = "range"composeFile = "compose.yml"@@ -117,6 +133,10 @@ const (// no resolver, or `range down` fails and the range cannot be removed at all.teardownNonce = "teardown"+ // The compose file requires the database password even for a teardown that+ // creates no database, for the same reason as the nonce above.+ teardownPassword = "teardown"+EnvComposeDir = "KHAOS_RANGE_DIR"EnvUpstream = "KHAOS_RANGE_UPSTREAM"@@ -131,6 +151,8 @@ const (errNoTLSGateway = "%w: this experiment arms khaos's own certificate authority, and the running range has no %s; bring the range down and up again to create it"errNoDNSGateway = "%w: this experiment arms khaos's chaos resolver, and the running range has no %s; bring the range down and up again to create it"errNoRangeResolver = "%w: this experiment arms khaos's chaos resolver, and the running range has no %s; bring the range down and up again to create it"++ errNoStatefulTier = "%w: this experiment acts on a real database engine, and the running range has none. khaos does not replace these dependencies with its own code, because the finding IS what three correct engines do, so the stack has to be brought up with them: `khaos range down && khaos range up --with-db`")// Lab is a located range. The khaos-side ports are chosen at `range up` rather@@ -150,6 +172,12 @@ type Lab struct {DNSPort intResolverNonce string+ // Stateful is whether `range up --with-db` created the database tier. The+ // password is minted per up and read back from the containers, never+ // remembered from what khaos chose.+ Stateful bool+ DBPassword string+// Read back from the TLS gateway's own environment, never remembered.Dependency stringAuthorityBase string@@ -350,10 +378,14 @@ func (l Lab) ProxiedProtectedURL() string {return l.ProxiedGatewayURL() + protectedPath}+// Up brings the range up. withDB adds the stateful tier, which is opt-in+// because those four containers are the only ones khaos does not implement+// itself and they cost real startup time.func (l Lab) Up(ctx context.Context,out io.Writer,failMode, cacheTTL, tlsMode string,+ withDB bool,) error {environ := append(os.Environ(),EnvGatewayPort+"="+strconv.Itoa(l.Port),@@ -369,17 +401,15 @@ func (l Lab) Up(EnvFailMode+"="+failMode,EnvCacheTTL+"="+cacheTTL,EnvTLSMode+"="+tlsMode,+ EnvDBPassword+"="+l.DBPassword,)- if err := l.compose(- ctx,- out,- environ,- "up",- "--build",- "--detach",- "--wait",- ); err != nil {+ args := []string{"up", "--build", "--detach", "--wait"}+ if withDB {+ args = append([]string{profileFlag, dbProfile}, args...)+ }++ if err := l.compose(ctx, out, environ, args...); err != nil {return err}@@ -398,12 +428,18 @@ func (l Lab) Down(ctx context.Context, out io.Writer) error {EnvResolverNonce+"="+teardownNonce,EnvAuthorityPort+"="+strconv.Itoa(teardownPort),EnvDependencyPort+"="+strconv.Itoa(teardownPort),+ EnvDBPassword+"="+teardownPassword,)+ // The profile is ALWAYS passed on the way down. Compose only removes the+ // services in the active profile set, so a teardown without it leaves the+ // database tier running and `range down` would report a range that is down+ // while four containers of it are up.return l.compose(ctx,out,environ,+ profileFlag, dbProfile,"down","--volumes","--remove-orphans",@@ -606,32 +642,76 @@ func Discover(ctx context.Context, reader PortReader) (Lab, error) {return Lab{}, err}- tlsGateway, ok := running[TLSGatewayName]- if !ok {+ complete, err := lab.readTLSTier(ctx, reader, running)+ if err != nil {+ return Lab{}, err+ }+ if !complete {return lab, nil}- if lab.TLSPort, err = reader.PublishedPort(- ctx,- tlsGateway.ID,- GatewayInternalPort,- ); err != nil {+ if err = lab.readDNSTier(ctx, reader, running); err != nil {return Lab{}, err}- if err = lab.readAuthorityWiring(- ctx,- reader,- tlsGateway.ID,- ); err != nil {- return Lab{}, err+ lab.readStatefulTier(running)++ return lab, nil+}++// readTLSTier reports whether the TLS gateway was there to read, because its+// absence is not fatal: the experiments that do not need it must still work.+func (l *Lab) readTLSTier(+ ctx context.Context,+ reader PortReader,+ running map[string]dockerx.Target,+) (bool, error) {+ tlsGateway, ok := running[TLSGatewayName]+ if !ok {+ return false, nil}- if err = lab.readDNSTier(ctx, reader, running); err != nil {- return Lab{}, err+ port, err := reader.PublishedPort(ctx, tlsGateway.ID, GatewayInternalPort)+ if err != nil {+ return false, err}- return lab, nil+ l.TLSPort = port++ if err := l.readAuthorityWiring(ctx, reader, tlsGateway.ID); err != nil {+ return false, err+ }++ return true, nil+}++// readStatefulTier records whether `range up --with-db` created the database+// tier. Every engine is required rather than any one of them: the credential+// family's exit gate runs the same pair against all three, and a partial tier+// would report a missing engine as an experiment that cannot run.+func (l *Lab) readStatefulTier(running map[string]dockerx.Target) {+ for _, name := range []string{+ PostgresName,+ MySQLName,+ MariaDBName,+ RedisName,+ } {+ if _, up := running[name]; !up {+ return+ }+ }++ l.Stateful = true+}++// RequireStatefulTier refuses before the baseline rather than letting a run+// report a database that is not there as a target that was not enforcing.+func (l Lab) RequireStatefulTier() error {+ if l.Stateful {+ return nil+ }++ return fmt.Errorf(errNoStatefulTier, ErrRangeIncomplete)}func (l *Lab) readDNSTier(diff --git a/PROJECTS/intermediate/chaos-engineering-tool/justfile b/PROJECTS/intermediate/chaos-engineering-tool/justfileindex aa5785e7..6006b59d 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/justfile+++ b/PROJECTS/intermediate/chaos-engineering-tool/justfile@@ -47,7 +47,7 @@ tidy-check:check: lint tidy-check test colors-gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate+gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate credential-gatecolors: build./scripts/colors.sh@@ -76,5 +76,8 @@ tls-gate: builddns-gate: build./scripts/dns.sh+credential-gate: build+ ./scripts/credential.sh+clean:rm -f {{binary}}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.ymlindex 54e3f181..26de8498 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml@@ -14,11 +14,17 @@ services:labels:khaos.target: "true"khaos.range: "true"- # The address is PINNED, and only on this network, because khaos's chaos- # resolver has to be authoritative for a name that maps to it and the zone- # is passed as a literal at container creation. Docker refuses a static- # address on a network whose subnet it chose, which is why the range network- # declares its own below.+ # EVERY address in this range is pinned, and it is load-bearing rather than+ # tidy. Three of them have to be: the resolver's and the dependency's+ # because the gateway's `dns:` and the resolver's zone are literals fixed at+ # container creation, and the rogue's for the same reason. The rest are+ # pinned because a DYNAMIC address competes with those three. Docker+ # allocates sequentially from the bottom of the subnet, so with five+ # dynamic gateways the fourth one reaches .5 and the resolver then fails to+ # start with "Address already in use" - measured, when the stateful tier+ # added four more containers and postgres took the resolver's address.+ # Pinning removes the race by construction instead of relying on compose+ # creating the pinned services first.networks:range:ipv4_address: 10.244.244.10@@ -47,7 +53,9 @@ services:- "127.0.0.1:${KHAOS_RANGE_GATEWAY_PORT:?khaos chooses this port; run `khaos range up`}:8080"depends_on:- idp- networks: [range]+ networks:+ range:+ ipv4_address: 10.244.244.11restart: "no"# The same gateway, reaching the same identity provider, but addressing it@@ -79,7 +87,9 @@ services:- "127.0.0.1:${KHAOS_RANGE_PROXIED_PORT:?khaos chooses this port; run `khaos range up`}:8080"depends_on:- idp- networks: [range]+ networks:+ range:+ ipv4_address: 10.244.244.12restart: "no"# The third gateway, and the only one whose identity provider is khaos itself@@ -113,7 +123,9 @@ services:khaos.range: "true"ports:- "127.0.0.1:${KHAOS_RANGE_VERIFIER_PORT:?khaos chooses this port; run `khaos range up`}:8080"- networks: [range]+ networks:+ range:+ ipv4_address: 10.244.244.13restart: "no"# The fourth gateway, and the only one that reaches a dependency over TLS from@@ -165,7 +177,9 @@ services:- "127.0.0.1:${KHAOS_RANGE_TLS_PORT:?khaos chooses this port; run `khaos range up`}:8080"depends_on:- idp- networks: [range]+ networks:+ range:+ ipv4_address: 10.244.244.14restart: "no"@@ -277,7 +291,155 @@ services:depends_on:- idp- resolver- networks: [range]+ networks:+ range:+ ipv4_address: 10.244.244.15+ restart: "no"++ # THE STATEFUL TIER, opt-in behind `khaos range up --with-db`.+ #+ # These four are the only services in the range khaos does not implement+ # itself, and that is the point of the credential family: the finding is what+ # three CORRECT engines do with an authenticated session when its credential+ # changes, so a khaos-written fake would prove nothing. Every other tier+ # exists because no correct dependency will emit the defect.+ #+ # Nothing is published to the host. khaos dials the container's own address on+ # this network, which the host routes to, so a database never appears on a+ # host port. The administrative password is minted per `khaos range up` rather+ # than written here, and khaos reads it back out of the container's own+ # environment.+ #+ # The tags are the ones the research measured. The VERSION is read from the+ # server at run time and recorded in the run, because a tag is not a version:+ # `postgres:18` resolved to two different patch releases on this host hours+ # apart.+ postgres:+ container_name: khaos-range-postgres+ # The data directory is a TMPFS, which is not a performance choice. These+ # images apply the administrative password only when they INITIALISE an+ # empty data directory, and compose carries an anonymous volume over to the+ # recreated container, so a second `range up` produced a container whose+ # environment held the password khaos had just minted and whose database+ # still held the previous one - measured, as an authentication failure+ # against khaos's own range. A tmpfs makes "empty every time" structural,+ # leaves nothing on disk after `range down`, and removes the anonymous+ # volume the image would otherwise declare.+ tmpfs:+ - /var/lib/postgresql+ image: postgres:18+ profiles: ["db"]+ command: ["-c", "log_connections=all"]+ environment:+ POSTGRES_PASSWORD: "${KHAOS_RANGE_DB_PASSWORD:?khaos mints this; run `khaos range up --with-db`}"+ POSTGRES_DB: "appdb"+ POSTGRES_USER: "postgres"+ labels:+ khaos.target: "true"+ khaos.range: "true"+ healthcheck:+ test: ["CMD-SHELL", "pg_isready -U postgres -d appdb"]+ interval: 2s+ timeout: 3s+ retries: 30+ networks:+ range:+ ipv4_address: 10.244.244.30+ restart: "no"++ mysql:+ container_name: khaos-range-mysql+ # The data directory is a TMPFS, which is not a performance choice. These+ # images apply the administrative password only when they INITIALISE an+ # empty data directory, and compose carries an anonymous volume over to the+ # recreated container, so a second `range up` produced a container whose+ # environment held the password khaos had just minted and whose database+ # still held the previous one - measured, as an authentication failure+ # against khaos's own range. A tmpfs makes "empty every time" structural,+ # leaves nothing on disk after `range down`, and removes the anonymous+ # volume the image would otherwise declare.+ tmpfs:+ - /var/lib/mysql+ image: mysql:8.4+ profiles: ["db"]+ environment:+ MYSQL_ROOT_PASSWORD: "${KHAOS_RANGE_DB_PASSWORD:?khaos mints this; run `khaos range up --with-db`}"+ MYSQL_DATABASE: "appdb"+ labels:+ khaos.target: "true"+ khaos.range: "true"+ healthcheck:+ test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root -p\"$$MYSQL_ROOT_PASSWORD\" --silent"]+ interval: 2s+ timeout: 3s+ retries: 45+ networks:+ range:+ ipv4_address: 10.244.244.31+ restart: "no"++ mariadb:+ container_name: khaos-range-mariadb+ # The data directory is a TMPFS, which is not a performance choice. These+ # images apply the administrative password only when they INITIALISE an+ # empty data directory, and compose carries an anonymous volume over to the+ # recreated container, so a second `range up` produced a container whose+ # environment held the password khaos had just minted and whose database+ # still held the previous one - measured, as an authentication failure+ # against khaos's own range. A tmpfs makes "empty every time" structural,+ # leaves nothing on disk after `range down`, and removes the anonymous+ # volume the image would otherwise declare.+ tmpfs:+ - /var/lib/mysql+ image: mariadb:11.4+ profiles: ["db"]+ environment:+ MARIADB_ROOT_PASSWORD: "${KHAOS_RANGE_DB_PASSWORD:?khaos mints this; run `khaos range up --with-db`}"+ MARIADB_DATABASE: "appdb"+ labels:+ khaos.target: "true"+ khaos.range: "true"+ healthcheck:+ test: ["CMD-SHELL", "mariadb-admin ping -h 127.0.0.1 -u root -p\"$$MARIADB_ROOT_PASSWORD\" --silent"]+ interval: 2s+ timeout: 3s+ retries: 30+ networks:+ range:+ ipv4_address: 10.244.244.32+ restart: "no"++ # The rate limiter's datastore. Kong's `fault_tolerant = true` is the only+ # genuine fail-open default in the whole fail-open taxonomy, and it is a+ # limiter that disables itself when THIS is unreachable. The limiter fixture+ # speaks RESP directly rather than taking a client dependency: a fixed-window+ # counter needs INCR and EXPIRE and nothing else.+ redis:+ container_name: khaos-range-redis+ # The data directory is a TMPFS, which is not a performance choice. These+ # images apply the administrative password only when they INITIALISE an+ # empty data directory, and compose carries an anonymous volume over to the+ # recreated container, so a second `range up` produced a container whose+ # environment held the password khaos had just minted and whose database+ # still held the previous one - measured, as an authentication failure+ # against khaos's own range. A tmpfs makes "empty every time" structural,+ # leaves nothing on disk after `range down`, and removes the anonymous+ # volume the image would otherwise declare.+ tmpfs:+ - /data+ image: redis:7.4-alpine+ profiles: ["db"]+ labels:+ khaos.target: "true"+ khaos.range: "true"+ healthcheck:+ test: ["CMD", "redis-cli", "ping"]+ interval: 2s+ timeout: 3s+ retries: 15+ networks:+ range:+ ipv4_address: 10.244.244.33restart: "no"networks:diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.shnew file mode 100755index 00000000..07cbc9c9--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh@@ -0,0 +1,470 @@+#!/usr/bin/env bash+# ©AngelaMos | 2026+# credential.sh++set -euo pipefail++# M9's exit gate for the database half of the `credential.` family: rotation+# distinguishes rotation from revocation on all three engines.+#+# This is the first family whose authority behaves CORRECTLY. Every other tier+# exists because no correct dependency will emit the defect - no correct+# provider mints `alg: none`, no correct CA signs an expired leaf. Here the+# engines are right and the operator's model is wrong: authentication is+# evaluated once, at connection establishment, and never again. So khaos drives+# three stock engines rather than replacing them, and the gate's job is to prove+# the pair inverts.+#+# 1. THE PAIR. `db_rotate` returns `deviated / PERMITTED` exit 2 on every+# engine, because the session khaos held keeps answering. The same fault+# plus one statement, `db_rotate_terminate`, returns `passed / HELD` exit 0+# on the same engine. One statement is the entire difference between a+# change of admission policy and a revocation.+#+# 2. THE NEGATIVE CONTROL EARNS ITS KEEP HERE. Research 07 section 6.1 is a+# write-up of a probe that reported PostgreSQL accepting a rotated-away+# password: it connected over loopback inside the server container, where+# `postgres:18` ships `host all all 127.0.0.1 trust` AHEAD of scram, so no+# password was ever evaluated. khaos's invalid stream presents a password+# that was never correct and requires the engine to refuse it, which is the+# control that was missing. A run that cannot establish it is void.+#+# 3. EVERY OBSERVABLE IS THE ENGINE'S OWN MESSAGE. The confirmation quotes+# what the engine said, per engine, and the four faults produce DISTINCT+# strings. A shared constant describing the technique measures nothing,+# which `scripts/proxy.sh` learned at the cost of two implementations.+#+# 4. THE VERSION COMES FROM THE SERVER. On this host `postgres:18` resolved to+# 18.3 for one measurement and 18.6 hours later, and `mariadb:11.4` moved a+# patch release the same way. A run record naming a tag names nothing.+#+# 5. NOTHING IS LEFT BEHIND. khaos mints a role per run and the engine must+# hold none of them afterwards. The revert RESTORES rather than removes,+# because dropping the role would make khaos's own cleanup read as a target+# that never came back - measured, as a degraded recovery on a run where+# nothing about the engine had degraded.+#+# 6. THE CLASSIFIER IS A DIFFERENTIAL, not a table khaos agrees with itself+# about. `internal/dbx`'s live tests run against all three engines from+# here, including the confirmation oracle's inert control, because a+# self-consistent client agrees with its own mistake.++BIN="${BIN:-$PWD/khaos}"++KHAOS_STATE_DIR="$(mktemp -d)"+export KHAOS_STATE_DIR++# The HELD half of the pair has to clear MinimumToClaimClosed, which is 30+# paired samples: khaos refuses to call a HELD run conclusive below that and+# exits 4 instead of 0. `tls.sh` and `dns.sh` both failed their first run on+# exactly this, so the window is sized for it rather than discovered again.+TIMING=(--baseline-for 4s --fault-for 20s --recover-for 3s --sample-every 300ms)++ENGINES=(postgres mysql mariadb)++declare -A CONTAINER=(+ [postgres]=khaos-range-postgres+ [mysql]=khaos-range-mysql+ [mariadb]=khaos-range-mariadb+)++declare -A ADMIN_VAR=(+ [postgres]=POSTGRES_PASSWORD+ [mysql]=MYSQL_ROOT_PASSWORD+ [mariadb]=MARIADB_ROOT_PASSWORD+)++declare -A ADMIN_ROLE=(+ [postgres]=postgres+ [mysql]=root+ [mariadb]=root+)++declare -A ENGINE_PORT=(+ [postgres]=5432+ [mysql]=3306+ [mariadb]=3306+)++failures=0+report=""+last_code=0+ran=()++cleanup() {+ "$BIN" range down --no-color >/dev/null 2>&1 || true+ rm -rf "$KHAOS_STATE_DIR"+}+trap cleanup EXIT INT TERM HUP++pass() { echo "ok $1"; }+fail() { echo "FAIL $1"; failures=$((failures + 1)); }++# Never call through a command substitution: that runs it in a subshell and the+# report is lost while the exit code still comes back, which looks like a+# passing gate with no evidence.+run_experiment() {+ local experiment="$1"+ shift+ ran+=("$experiment")+ run_unmeasured "$experiment" "$@"+}++# run_unmeasured is for a run that is NOT a measurement of the experiment: a+# refusal, or a read of something khaos printed before the baseline. It does not+# enter the census, because a refused run that counted would alibi an experiment+# this gate never actually measured.+#+# The first version of this script achieved that by clearing the census array+# after such a run, which wiped the record of every earlier one and reported+# that the gate had run nothing at all.+run_unmeasured() {+ local experiment="$1"+ shift+ set +e+ report="$("$BIN" run "$experiment" --no-color "${TIMING[@]}" "$@" 2>&1)"+ last_code=$?+ set -e+}++assert_verdict() {+ local label="$1" want="$2"+ if grep -qE "^ verdict .*${want}" <<<"$report"; then+ pass "${label} verdict is ${want}"+ else+ fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"+ fi+}++assert_exit() {+ local label="$1" want="$2"+ if [[ "$last_code" == "$want" ]]; then+ pass "${label} exit code ${want}"+ else+ fail "${label} exit code ${last_code}, want ${want}"+ fi+}++assert_confirmed() {+ local label="$1"+ if grep -qE '^ plan confirm fault confirmed by ' <<<"$report"; then+ pass "${label} confirmed by a measured observable"+ else+ fail "${label} reported no confirmation, so the run measured a fault it never proved landed"+ fi+}++assert_report_has() {+ local label="$1" want="$2"+ if grep -qF "$want" <<<"$report"; then+ pass "${label}"+ else+ fail "${label}: the report never says ${want}"+ fi+}++assert_report_lacks() {+ local label="$1" want="$2"+ if grep -qF "$want" <<<"$report"; then+ fail "${label}: the report says ${want}"+ else+ pass "${label}"+ fi+}++# The baseline guard that research 07 section 6.1 paid for. If the engine ever+# accepted the never-valid password, khaos voids the run by name rather than+# reporting a finding, and this asserts the run was NOT voided that way.+assert_control_held() {+ local label="$1"+ assert_report_lacks \+ "${label} established that the engine checks passwords at all" \+ "the invalid credential was never rejected"+}++observable_of() {+ grep -E '^ plan confirm fault confirmed by ' <<<"$report" |+ sed -E 's/^.*fault confirmed by //'+}++admin_password() {+ docker inspect "${CONTAINER[$1]}" \+ --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null |+ grep "^${ADMIN_VAR[$1]}=" | cut -d= -f2-+}++engine_address() {+ docker inspect "${CONTAINER[$1]}" \+ --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \+ 2>/dev/null+}++# leftover_roles counts roles khaos minted and did not remove. Read over+# loopback INSIDE the container on purpose: that is the `trust` path research+# 07 section 6.1 warns about, which makes it useless as a probe and exactly+# right as the gate's own administrative read.+leftover_roles() {+ local engine="$1"+ case "$engine" in+ postgres)+ docker exec "${CONTAINER[$engine]}" psql -U postgres -tAc \+ "SELECT count(*) FROM pg_roles WHERE rolname LIKE 'khaos_cred_%'" \+ 2>/dev/null | tail -1 | tr -d '[:space:]'+ ;;+ *)+ docker exec "${CONTAINER[$engine]}" sh -c \+ "exec \"\$(command -v mariadb || command -v mysql)\" -uroot -p\"\$${ADMIN_VAR[$engine]}\" -N -B -e \"SELECT count(*) FROM mysql.user WHERE user LIKE 'khaos\\_cred\\_%'\"" \+ 2>/dev/null | tail -1 | tr -d '[:space:]'+ ;;+ esac+}++start_range() {+ "$BIN" range down --no-color >/dev/null 2>&1 || true+ if ! "$BIN" range up --with-db --no-color --quiet >/dev/null 2>&1; then+ fail "range up --with-db"+ return 1+ fi+ return 0+}++echo "== the stateful tier is refused by name when it is not running =="++# The refusal has to arrive with a range UP and the database tier DOWN, which is+# the state an operator reaches by running `khaos range up` and then asking for+# a credential experiment. With no range at all the run dies in fillFromRange+# with a different message, and a gate that accepted either would pass against+# the wrong one.+"$BIN" range down --no-color >/dev/null 2>&1 || true+if "$BIN" range up --no-color --quiet >/dev/null 2>&1; then+ run_unmeasured credential.db_rotate++ assert_exit "a credential experiment with no stateful tier" 1+ assert_report_has \+ "the refusal names the missing tier" \+ "the running range has none"+ assert_report_has \+ "the refusal names the command that fixes it" \+ "khaos range up --with-db"+ assert_report_has \+ "the refusal says why khaos does not fake it" \+ "the finding IS what three correct engines do"+else+ fail "range up (without the stateful tier)"+fi++echo "== THE PAIR: rotation is not revocation, and termination is =="++if start_range; then+ for engine in "${ENGINES[@]}"; do+ container="${CONTAINER[$engine]}"++ run_experiment credential.db_rotate \+ --target "$container" --engine "$engine"++ assert_verdict "${engine} db_rotate" "deviated / PERMITTED"+ assert_exit "${engine} db_rotate" 2+ assert_confirmed "${engine} db_rotate"+ assert_control_held "${engine} db_rotate"+ assert_report_has \+ "${engine} db_rotate names what khaos did to the credential" \+ "by rotating its password at the engine and terminating nothing"+ assert_report_has \+ "${engine} db_rotate reports the exposure as a lower bound" \+ "which is the lower bound on the exposure"++ rotate_observable="$(observable_of)"++ run_experiment credential.db_rotate_terminate \+ --target "$container" --engine "$engine"++ assert_verdict "${engine} db_rotate_terminate" "passed / HELD"+ assert_exit "${engine} db_rotate_terminate" 0+ assert_confirmed "${engine} db_rotate_terminate"+ assert_control_held "${engine} db_rotate_terminate"+ assert_report_has \+ "${engine} db_rotate_terminate says what HELD means here" \+ "closed by the engine, which is what makes termination"++ # The inversion, read off the FINDING rather than off the verdict+ # label: the rotation half reports the invalidated credential being+ # honoured, and the half that also terminated must report no such+ # thing. Two assertions on two reports, which is what makes the pair+ # the exit gate rather than two unrelated runs.+ assert_report_lacks \+ "${engine} nothing was honoured once the sessions were terminated" \+ "was still accepted"++ if [[ -n "$rotate_observable" ]]; then+ pass "${engine} the rotation produced an observable to compare"+ else+ fail "${engine} the rotation produced no observable to compare"+ fi+ done+fi++echo "== the two other controls an operator reaches for do not revoke either =="++for engine in "${ENGINES[@]}"; do+ container="${CONTAINER[$engine]}"++ run_experiment credential.db_account_lock \+ --target "$container" --engine "$engine"++ assert_verdict "${engine} db_account_lock" "deviated / PERMITTED"+ assert_exit "${engine} db_account_lock" 2+ assert_confirmed "${engine} db_account_lock"+ assert_control_held "${engine} db_account_lock"+ assert_report_has \+ "${engine} db_account_lock confirmation quotes the engine on the lock" \+ "because the account is locked"++ lock_observable="$(observable_of)"++ run_experiment credential.db_credential_expiry \+ --target "$container" --engine "$engine"++ assert_verdict "${engine} db_credential_expiry" "deviated / PERMITTED"+ assert_exit "${engine} db_credential_expiry" 2+ assert_confirmed "${engine} db_credential_expiry"+ assert_control_held "${engine} db_credential_expiry"+ assert_report_has \+ "${engine} db_credential_expiry confirmation names the validity" \+ "because the credential's validity has passed"++ expiry_observable="$(observable_of)"++ # Distinct observables, per engine. The lock and the expiry both refuse a+ # correct password at a new connection, so a confirmation that described the+ # technique rather than reading the engine back would be identical for both.+ if [[ "$lock_observable" != "$expiry_observable" ]]; then+ pass "${engine} the lock and the expiry produce DISTINCT observables"+ else+ fail "${engine} the lock and the expiry produced the same observable, so neither is evidence"+ fi+done++echo "== the engine version comes from the server, not from the image tag =="++for engine in "${ENGINES[@]}"; do+ run_unmeasured credential.db_rotate \+ --target "${CONTAINER[$engine]}" --engine "$engine"++ if grep -qE '^ plan credential tier open on .*which reports "' <<<"$report"; then+ reported="$(grep -oE 'which reports "[^"]+"' <<<"$report")"++ case "$engine" in+ postgres) expect='PostgreSQL 18\.[0-9]+' ;;+ mysql) expect='8\.4\.[0-9]+' ;;+ mariadb) expect='11\.4\.[0-9]+-MariaDB' ;;+ esac++ if grep -qE "$expect" <<<"$reported"; then+ pass "${engine} reports a patch level its image tag does not carry (${reported})"+ else+ fail "${engine} reported ${reported}, which does not carry a patch level and so could be the tag echoed back"+ fi+ else+ fail "${engine} never reported the version the server gave it"+ fi+done++echo "== khaos leaves no role of its own behind =="++for engine in "${ENGINES[@]}"; do+ left="$(leftover_roles "$engine")"++ if [[ "$left" == "0" ]]; then+ pass "${engine} holds no khaos-minted role after the runs"+ else+ fail "${engine} still holds ${left:-<unreadable>} khaos-minted role(s)"+ fi+done++echo "== the classifier is a differential against all three engines =="++# Including the confirmation oracle's inert control, which is the one assertion+# an oracle gutted to `return true` cannot satisfy.+differential_env=()+for engine in "${ENGINES[@]}"; do+ address="$(engine_address "$engine")"+ password="$(admin_password "$engine")"+ database=""+ if [[ "$engine" == "postgres" ]]; then+ database="postgres"+ fi++ if [[ -z "$address" || -z "$password" ]]; then+ fail "could not read ${engine}'s address or administrative credential from its own container"+ continue+ fi++ name="KHAOS_DBX_LIVE_$(echo "$engine" | tr '[:lower:]' '[:upper:]')"+ differential_env+=(+ "${name}=${address}:${ENGINE_PORT[$engine]},${database},${ADMIN_ROLE[$engine]},${password}"+ )+done++if [[ "${#differential_env[@]}" == "${#ENGINES[@]}" ]]; then+ set +e+ differential="$(env "${differential_env[@]}" \+ go test ./internal/dbx/ -count=1 2>&1)"+ differential_code=$?+ set -e++ if [[ "$differential_code" == 0 ]]; then+ pass "internal/dbx agrees with all three live engines"+ else+ fail "internal/dbx disagrees with a live engine: $(tail -5 <<<"$differential")"+ fi++ # A differential that SKIPPED every engine would also exit 0, which is the+ # inert-control shape this project has shipped before.+ skipped="$(env "${differential_env[@]}" \+ go test ./internal/dbx/ -count=1 -v -run TestRotationIsNotRevocation 2>&1 |+ grep -c 'SKIP' || true)"++ if [[ "$skipped" == "0" ]]; then+ pass "the differential ran against every engine rather than skipping them"+ else+ fail "the differential SKIPPED ${skipped} engine(s), so it proved nothing"+ fi+fi++echo "== census: every catalogued credential.db_ experiment was run here =="++catalogued="$(NO_COLOR=1 "$BIN" run --help |+ grep -oE '^ credential\.[a-z_]+' | tr -d ' ' | sort -u)"++if [[ -z "$catalogued" ]]; then+ fail "the census read no credential experiments at all, so it cannot fail for the right reason"+fi++missing=""+for name in $catalogued; do+ found=0+ for was in "${ran[@]}"; do+ if [[ "$was" == "$name" ]]; then+ found=1+ break+ fi+ done+ if [[ "$found" == 0 ]]; then+ missing="${missing} ${name}"+ fi+done++if [[ -z "$missing" ]]; then+ pass "every catalogued credential. fault was actually run by this gate"+else+ fail "this gate never ran:${missing}"+fi++echo+if [[ "$failures" == 0 ]]; then+ echo "credential gate: all assertions passed"+ exit 0+fi++echo "credential gate: ${failures} assertion(s) failed"+exit 1