Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇

feat(khaos): security chaos engineering tool, M0 through M6

CarterPerez-dev28d6c4bc946b
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/.dockerignore b/PROJECTS/intermediate/chaos-engineering-tool/.dockerignore
new file mode 100644
index 00000000..19816542
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/.dockerignore
@@ -0,0 +1,8 @@
+# ©AngelaMos | 2026
+# .dockerignore
+
+*
+!go.mod
+!go.sum
+!cmd/
+!internal/
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/.github/workflows/ci.yml b/PROJECTS/intermediate/chaos-engineering-tool/.github/workflows/ci.yml
new file mode 100644
index 00000000..49a60938
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/.github/workflows/ci.yml
@@ -0,0 +1,52 @@
+# ©AngelaMos | 2026
+# ci.yml
+
+name: ci
+
+on:
+ push:
+ branches: ["**"]
+ pull_request:
+
+permissions:
+ contents: read
+
+jobs:
+ check:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+
+ - name: gofmt
+ run: test -z "$(gofmt -l .)"
+
+ - name: vet
+ run: go vet ./...
+
+ - name: module tidiness
+ run: go mod tidy -diff
+
+ - name: test
+ run: go test -race ./...
+
+ - name: build
+ run: go build -o khaos ./cmd/khaos
+
+ - name: color degradation gate
+ run: ./scripts/colors.sh
+
+ - name: refusal gate against a real daemon
+ run: ./scripts/smoke.sh
+
+ - name: rollback survives kill -9
+ run: ./scripts/survive.sh
+
+ - name: the range produces the right verdict both ways
+ run: ./scripts/range.sh
+
+ - name: non-tty guard
+ run: timeout 15s ./khaos < /dev/null > /dev/null
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/.gitignore b/PROJECTS/intermediate/chaos-engineering-tool/.gitignore
new file mode 100644
index 00000000..507219ce
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/.gitignore
@@ -0,0 +1,12 @@
+# ©AngelaMos | 2026
+# .gitignore
+
+docs/
+dist/
+/khaos
+*.db
+*.sqlite
+*.prom
+
+.env
+.env.development
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/.golangci.yml b/PROJECTS/intermediate/chaos-engineering-tool/.golangci.yml
new file mode 100644
index 00000000..3514d8fb
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/.golangci.yml
@@ -0,0 +1,113 @@
+# AngelaMos | 2026
+# .golangci.yml
+
+version: "2"
+
+linters:
+ default: none
+ enable:
+ - errcheck
+ - govet
+ - gosec
+ - bodyclose
+ - nilerr
+ - errorlint
+ - exhaustive
+ - gocritic
+ - funlen
+ - gocognit
+ - dupl
+ - goconst
+ - ineffassign
+ - unused
+ - unconvert
+ - unparam
+ - testifylint
+ - fatcontext
+
+ exclusions:
+ rules:
+ - path: _test\.go
+ linters:
+ - funlen
+ - dupl
+ - goconst
+ - gosec
+ paths:
+ - vendor
+ - testdata
+ max-same-issues: 50
+
+ settings:
+ errcheck:
+ check-type-assertions: true
+ check-blank: true
+
+ funlen:
+ lines: 88
+ statements: 50
+
+ gocognit:
+ min-complexity: 20
+
+ govet:
+ enable-all: true
+ disable:
+ - fieldalignment
+
+ revive:
+ rules:
+ - name: blank-imports
+ - name: context-as-argument
+ - name: context-keys-type
+ - name: error-return
+ - name: error-strings
+ - name: error-naming
+ - name: exported
+ - name: increment-decrement
+ - name: var-declaration
+ - name: package-comments
+ disabled: true
+ - name: range
+ - name: receiver-naming
+ - name: time-naming
+ - name: unexported-return
+ - name: indent-error-flow
+ - name: errorf
+ - name: empty-block
+ - name: superfluous-else
+ - name: unreachable-code
+
+ staticcheck:
+ checks:
+ - all
+
+ gosec:
+ excludes:
+ - G104
+
+ sloglint:
+ no-mixed-args: true
+ kv-only: true
+ context: all
+
+formatters:
+ enable:
+ - gci
+ - gofumpt
+ - golines
+ settings:
+ golines:
+ max-len: 80
+ reformat-tags: true
+ goimports:
+ local-prefixes:
+ - github.com/carterperez-dev/khaos
+ gci:
+ sections:
+ - standard
+ - default
+ - prefix(github.com/carterperez-dev)
+ custom-order: true
+ gofumpt:
+ extra-rules: true
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/LICENSE b/PROJECTS/intermediate/chaos-engineering-tool/LICENSE
new file mode 100644
index 00000000..0ad25db4
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/LICENSE
@@ -0,0 +1,661 @@
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway.go
new file mode 100644
index 00000000..f267e854
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway.go
@@ -0,0 +1,245 @@
+// ©AngelaMos | 2026
+// gateway.go
+
+package main
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+)
+
+const (
+ routeProtected = "/protected"
+
+ headerAuthorization = "Authorization"
+ bearerPrefix = "Bearer "
+
+ // FailClosed refuses when it cannot reach the identity provider. This is
+ // the correct behaviour for an authentication check.
+ //
+ // FailOpen serves anyway. This is the defect khaos exists to find, and it
+ // is modelled on the Okta AD/LDAP delegated-authentication advisory, where
+ // a cached decision was consulted first and could be reached only while the
+ // agent was "down or cannot be reached".
+ FailClosed = "closed"
+ FailOpen = "open"
+
+ // FailStaleRearm is the bug research 07 section 3.3 says to name, build and
+ // detect, because it reads as a resilience feature in review. It serves a
+ // cached decision when the provider is unreachable AND re-arms that cache
+ // entry for a full lifetime on every failed check. The lifetime therefore
+ // stops bounding staleness and starts bounding IDLENESS: a cached allow
+ // becomes permanent exactly when the authority that could revoke it is
+ // unreachable, for as long as traffic keeps arriving.
+ //
+ // It is one line different from a defensible design. Serving a stale allow up
+ // to a fixed ceiling and never re-arming is the bounded-stale pattern, which
+ // is a trade plenty of teams make deliberately. The difference is whether the
+ // error branch writes to the cache.
+ FailStaleRearm = "stale-rearm"
+
+ introspectPath = "/introspect?token="
+ issuePath = "/token"
+ revokePath = "/revoke"
+ defaultCacheTTL = 2 * time.Second
+ upstreamTimeout = 500 * time.Millisecond
+
+ // Generous on purpose. The ceiling has to be wide enough that khaos's own
+ // probe interval cannot fall out of it by accident, or a run measures the
+ // fixture's timing rather than the bug.
+ staleCeilingFactor = 4
+)
+
+type gateway struct {
+ upstream string
+ failMode string
+ cacheTTL time.Duration
+
+ client *http.Client
+
+ mu sync.Mutex
+ cache map[string]cachedDecision
+}
+
+type cachedDecision struct {
+ active bool
+ expires time.Time
+}
+
+func newGateway(upstream, failMode string, cacheTTL time.Duration) *gateway {
+ if cacheTTL <= 0 {
+ cacheTTL = defaultCacheTTL
+ }
+
+ return &gateway{
+ upstream: upstream,
+ failMode: failMode,
+ cacheTTL: cacheTTL,
+ cache: map[string]cachedDecision{},
+ client: &http.Client{Timeout: upstreamTimeout},
+ }
+}
+
+func (g *gateway) routes() *http.ServeMux {
+ mux := http.NewServeMux()
+ mux.HandleFunc(routeHealth, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+ mux.HandleFunc(routeProtected, g.protected)
+ mux.HandleFunc(issuePath, g.forward(issuePath))
+ mux.HandleFunc(revokePath, g.forward(revokePath))
+ return mux
+}
+
+func (g *gateway) protected(w http.ResponseWriter, r *http.Request) {
+ token, scheme := strings.CutPrefix(r.Header.Get(headerAuthorization), bearerPrefix)
+
+ if !scheme || token == "" {
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ if decision, ok := g.cached(token); ok {
+ g.answer(w, decision)
+ return
+ }
+
+ active, reachable := g.introspect(token)
+
+ if reachable {
+ g.remember(token, active)
+ g.answer(w, active)
+ return
+ }
+
+ if g.failMode == FailOpen {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+
+ // The whole bug is the next three lines. A stale allow is served AND written
+ // back, so the next request finds it fresh and writes it back again.
+ if g.failMode == FailStaleRearm {
+ if decision, ok := g.stale(token); ok {
+ g.remember(token, decision)
+ g.answer(w, decision)
+ return
+ }
+ }
+
+ w.WriteHeader(http.StatusForbidden)
+}
+
+// forward passes credential administration through to the identity provider so
+// the range needs exactly one host port. It is deliberately unauthenticated:
+// this is a loopback-only fixture whose whole purpose is to be broken, and
+// khaos needs to mint itself a credential before it has one.
+func (g *gateway) forward(path string) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ target := g.upstream + path
+ if r.URL.RawQuery != "" {
+ target += "?" + r.URL.RawQuery
+ }
+
+ response, err := g.client.Post(target, "", nil)
+ if err != nil {
+ w.WriteHeader(http.StatusBadGateway)
+ return
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ w.Header().Set(contentType, mimeJSON)
+ w.WriteHeader(response.StatusCode)
+ _, _ = io.Copy(w, response.Body)
+ }
+}
+
+func (g *gateway) answer(w http.ResponseWriter, active bool) {
+ if active {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ w.WriteHeader(http.StatusForbidden)
+}
+
+func (g *gateway) introspect(token string) (bool, bool) {
+ response, err := g.client.Get(g.upstream + introspectPath + token)
+ if err != nil {
+ return false, false
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ // A refusal from the identity provider is an ANSWER, not an outage.
+ // Treating any non-200 as "dependency unavailable, proceed" is the single
+ // most common fail-open shape in service-to-service code, and a reference
+ // implementation that contains it makes the comparison dishonest.
+ if response.StatusCode == http.StatusUnauthorized || response.StatusCode == http.StatusForbidden {
+ return false, true
+ }
+
+ if response.StatusCode != http.StatusOK {
+ return false, false
+ }
+
+ var body introspection
+ if err := json.NewDecoder(response.Body).Decode(&body); err != nil {
+ return false, false
+ }
+
+ return body.Active, true
+}
+
+func (g *gateway) cached(token string) (bool, bool) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ decision, ok := g.cache[token]
+ if !ok || time.Now().After(decision.expires) {
+ return false, false
+ }
+
+ return decision.active, true
+}
+
+// stale returns a decision whose lifetime has passed but which is still inside a
+// bounded staleness ceiling.
+//
+// The ceiling is what makes this the TTL-EXTENSION bug rather than the other one.
+// Without it a stale entry is servable forever, which IS a cache with no expiry,
+// and khaos measures it as exactly that. With it, every failed check pushes the
+// entry back inside the window, so steady traffic keeps a cached allow alive
+// indefinitely and a pause longer than the window lets it lapse. One ceiling, and
+// the two bugs separate.
+func (g *gateway) stale(token string) (bool, bool) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ decision, ok := g.cache[token]
+ if !ok {
+ return false, false
+ }
+
+ if time.Now().After(decision.expires.Add(g.staleCeiling())) {
+ return false, false
+ }
+
+ return decision.active, true
+}
+
+// staleCeiling is how far past its lifetime a decision may still be served. It is
+// a multiple of the lifetime rather than a separate knob, so a range configured
+// with one number behaves predictably: steady traffic at any interval shorter
+// than this keeps a cached allow alive.
+func (g *gateway) staleCeiling() time.Duration {
+ return g.cacheTTL * staleCeilingFactor
+}
+
+func (g *gateway) remember(token string, active bool) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ g.cache[token] = cachedDecision{active: active, expires: time.Now().Add(g.cacheTTL)}
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway_test.go
new file mode 100644
index 00000000..e386dd84
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway_test.go
@@ -0,0 +1,332 @@
+// ©AngelaMos | 2026
+// gateway_test.go
+
+package main
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+const (
+ shortTTL = 100 * time.Millisecond
+ goodToken = "khaos-range-token-1"
+)
+
+type stubIDP struct {
+ server *httptest.Server
+ down atomic.Bool
+ status atomic.Int32
+ calls atomic.Int64
+ validOnly string
+}
+
+func newStubIDP(t *testing.T) *stubIDP {
+ t.Helper()
+
+ stub := &stubIDP{validOnly: goodToken}
+ stub.status.Store(http.StatusOK)
+
+ stub.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ stub.calls.Add(1)
+
+ if stub.down.Load() {
+ hijackAndDrop(w)
+ return
+ }
+
+ if status := int(stub.status.Load()); status != http.StatusOK {
+ w.WriteHeader(status)
+ return
+ }
+
+ token := r.URL.Query().Get(paramToken)
+ writeJSON(w, introspection{Active: token == stub.validOnly, User: "operator"})
+ }))
+
+ t.Cleanup(stub.server.Close)
+
+ return stub
+}
+
+func hijackAndDrop(w http.ResponseWriter) {
+ hijacker, ok := w.(http.Hijacker)
+ if !ok {
+ w.WriteHeader(http.StatusGatewayTimeout)
+ return
+ }
+
+ conn, _, err := hijacker.Hijack()
+ if err == nil {
+ _ = conn.Close()
+ }
+}
+
+func ask(t *testing.T, g *gateway, authorization string) int {
+ t.Helper()
+
+ request := httptest.NewRequest(http.MethodGet, routeProtected, nil)
+ if authorization != "" {
+ request.Header.Set(headerAuthorization, authorization)
+ }
+
+ recorder := httptest.NewRecorder()
+ g.routes().ServeHTTP(recorder, request)
+
+ return recorder.Code
+}
+
+func TestFailClosedServesFromCacheThenRefuses(t *testing.T) {
+ idp := newStubIDP(t)
+ g := newGateway(idp.server.URL, FailClosed, shortTTL)
+
+ if got := ask(t, g, bearerPrefix+goodToken); got != http.StatusOK {
+ t.Fatalf("baseline returned %d, want 200", got)
+ }
+
+ idp.down.Store(true)
+
+ if got := ask(t, g, bearerPrefix+goodToken); got != http.StatusOK {
+ t.Fatalf("immediately after the outage the gateway returned %d; it should still be "+
+ "serving from its decision cache", got)
+ }
+
+ time.Sleep(shortTTL * 2)
+
+ if got := ask(t, g, bearerPrefix+goodToken); got != http.StatusForbidden {
+ t.Fatalf("after the cache expired the gateway returned %d, want 403; a target that "+
+ "serves from cache and then refuses has STOPPED serving, and asking only whether "+
+ "it ever served reports that as a clean survival", got)
+ }
+}
+
+func TestFailOpenHonoursACredentialItHadRefused(t *testing.T) {
+ idp := newStubIDP(t)
+ g := newGateway(idp.server.URL, FailOpen, shortTTL)
+
+ if got := ask(t, g, bearerPrefix+"never-issued"); got != http.StatusForbidden {
+ t.Fatalf("baseline returned %d for an invalid credential, want 403", got)
+ }
+
+ idp.down.Store(true)
+ time.Sleep(shortTTL * 2)
+
+ if got := ask(t, g, bearerPrefix+"never-issued"); got != http.StatusOK {
+ t.Fatalf("returned %d; the fail-open mode exists to honour a credential it had already "+
+ "refused, and if it does not the range cannot produce the finding khaos is for", got)
+ }
+}
+
+func TestARefusalFromTheProviderIsAnAnswerNotAnOutage(t *testing.T) {
+ for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} {
+ t.Run(http.StatusText(status), func(t *testing.T) {
+ idp := newStubIDP(t)
+ idp.status.Store(int32(status))
+
+ g := newGateway(idp.server.URL, FailOpen, shortTTL)
+
+ if got := ask(t, g, bearerPrefix+goodToken); got != http.StatusForbidden {
+ t.Fatalf("the provider answered %d and the gateway returned %d; treating a "+
+ "refusal from a dependency as 'dependency unavailable, proceed' is the most "+
+ "common fail-open shape there is, and it must not be in the reference "+
+ "implementation unlabelled", status, got)
+ }
+ })
+ }
+}
+
+func TestTheBearerSchemeIsRequired(t *testing.T) {
+ idp := newStubIDP(t)
+ g := newGateway(idp.server.URL, FailClosed, shortTTL)
+
+ cases := map[string]int{
+ bearerPrefix + goodToken: http.StatusOK,
+ goodToken: http.StatusUnauthorized,
+ "Basic " + goodToken: http.StatusUnauthorized,
+ bearerPrefix: http.StatusUnauthorized,
+ "": http.StatusUnauthorized,
+ }
+
+ for authorization, want := range cases {
+ name := authorization
+ if name == "" {
+ name = "no header"
+ }
+
+ t.Run(name, func(t *testing.T) {
+ if got := ask(t, g, authorization); got != want {
+ t.Fatalf("%q returned %d, want %d", authorization, got, want)
+ }
+ })
+ }
+}
+
+func TestAFreshCredentialPerRequestDefeatsTheDecisionCache(t *testing.T) {
+ idp := newStubIDP(t)
+ g := newGateway(idp.server.URL, FailOpen, time.Hour)
+
+ const probes = 3
+
+ for range probes {
+ ask(t, g, bearerPrefix+"never-issued")
+ }
+
+ repeated := idp.calls.Load()
+
+ idp.calls.Store(0)
+ for i := range probes {
+ ask(t, g, bearerPrefix+"never-issued-"+strings.Repeat("x", i+1))
+ }
+
+ fresh := idp.calls.Load()
+
+ if repeated >= fresh {
+ t.Fatalf("a repeated credential produced %d provider calls and a fresh one produced %d; "+
+ "a negative control that reuses one token keeps the decision cache warm, so the "+
+ "cached refusal answers every probe and a control that stops enforcing is never "+
+ "exercised", repeated, fresh)
+ }
+}
+
+func TestTheProviderIssuesAndRevokes(t *testing.T) {
+ provider := newIdentityProvider()
+ server := httptest.NewServer(provider.routes())
+ defer server.Close()
+
+ response, err := http.Post(server.URL+routeToken, "", nil)
+ if err != nil {
+ t.Fatalf("issue: %v", err)
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ var issued map[string]string
+ if err := json.NewDecoder(response.Body).Decode(&issued); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+
+ token := issued[paramToken]
+ if token == "" {
+ t.Fatal("the provider issued an empty token")
+ }
+
+ if !active(t, server.URL, token) {
+ t.Fatal("a freshly issued token was not active")
+ }
+
+ if _, err := http.Post(server.URL+routeRevoke+"?"+paramToken+"="+token, "", nil); err != nil {
+ t.Fatalf("revoke: %v", err)
+ }
+
+ if active(t, server.URL, token) {
+ t.Fatal("a revoked token was still active")
+ }
+}
+
+func active(t *testing.T, base, token string) bool {
+ t.Helper()
+
+ response, err := http.Get(base + routeIntrospect + "?" + paramToken + "=" + token)
+ if err != nil {
+ t.Fatalf("introspect: %v", err)
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ var body introspection
+ if err := json.NewDecoder(response.Body).Decode(&body); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+
+ return body.Active
+}
+
+// The whole point of the stale-rearm mode: a cached allow that steady traffic
+// keeps alive indefinitely and that a pause lets lapse. Deterministic, because
+// the Docker version of this question cost two wrong predictions in a row.
+func TestStaleRearmSurvivesSteadyTrafficAndLapsesOnAPause(t *testing.T) {
+ idp := newStubIDP(t)
+ g := newGateway(idp.server.URL, FailStaleRearm, shortTTL)
+
+ // The provider is up, so this caches a genuine allow.
+ if status := ask(t, g, bearerPrefix+goodToken); status != http.StatusOK {
+ t.Fatalf("baseline status %d, want 200", status)
+ }
+
+ idp.down.Store(true)
+
+ // Steady traffic at an interval longer than the lifetime but well inside the
+ // ceiling. Every request lands on the stale path and re-arms the entry, so the
+ // allow has to survive all of them.
+ const beats = 6
+
+ for beat := range beats {
+ time.Sleep(shortTTL + shortTTL/2)
+
+ if status := ask(t, g, bearerPrefix+goodToken); status != http.StatusOK {
+ t.Fatalf("beat %d: status %d, want 200. Steady traffic must keep a re-armed cache "+
+ "alive, or this mode is not the TTL-extension bug", beat, status)
+ }
+ }
+
+ // Now stop. Nothing re-arms, so the entry falls out of the ceiling.
+ time.Sleep(shortTTL*staleCeilingFactor + shortTTL)
+
+ if status := ask(t, g, bearerPrefix+goodToken); status == http.StatusOK {
+ t.Fatalf("after a pause longer than the ceiling the gateway still served the cached " +
+ "allow, so this mode is a cache with no expiry rather than one being re-armed")
+ }
+}
+
+// The mirror, and the reason the ceiling exists. Without a bound this mode is
+// indistinguishable from a no-expiry cache, which is the OTHER bug khaos has to
+// tell it apart from.
+func TestWithoutACeilingStaleRearmWouldBeANoExpiryCache(t *testing.T) {
+ idp := newStubIDP(t)
+ g := newGateway(idp.server.URL, FailStaleRearm, shortTTL)
+
+ if ceiling := g.staleCeiling(); ceiling <= shortTTL {
+ t.Fatalf("the staleness ceiling is %s against a %s lifetime; a ceiling that is not "+
+ "wider than the lifetime cannot be re-armed into", ceiling, shortTTL)
+ }
+
+ if status := ask(t, g, bearerPrefix+goodToken); status != http.StatusOK {
+ t.Fatalf("baseline status %d, want 200", status)
+ }
+
+ idp.down.Store(true)
+
+ // One lifetime past expiry is inside the ceiling: served.
+ time.Sleep(shortTTL * 2)
+ if status := ask(t, g, bearerPrefix+goodToken); status != http.StatusOK {
+ t.Fatalf("status %d inside the ceiling, want 200", status)
+ }
+
+ // Far past it is not, and that is the bound a no-expiry cache lacks.
+ time.Sleep(shortTTL * (staleCeilingFactor + 2))
+ if status := ask(t, g, bearerPrefix+goodToken); status == http.StatusOK {
+ t.Fatal("a decision far outside the staleness ceiling was still served")
+ }
+}
+
+// Fail-closed must not grow a stale path by accident. It is the reference
+// implementation, and a 300-star teaching repo's correct side is what readers
+// copy.
+func TestFailClosedNeverServesAStaleDecision(t *testing.T) {
+ idp := newStubIDP(t)
+ g := newGateway(idp.server.URL, FailClosed, shortTTL)
+
+ if status := ask(t, g, bearerPrefix+goodToken); status != http.StatusOK {
+ t.Fatalf("baseline status %d, want 200", status)
+ }
+
+ idp.down.Store(true)
+ time.Sleep(shortTTL * 2)
+
+ if status := ask(t, g, bearerPrefix+goodToken); status == http.StatusOK {
+ t.Fatal("the fail-closed gateway served a stale decision")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/idp.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/idp.go
new file mode 100644
index 00000000..8ca0a530
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/idp.go
@@ -0,0 +1,94 @@
+// ©AngelaMos | 2026
+// idp.go
+
+package main
+
+import (
+ "encoding/json"
+ "net/http"
+ "strconv"
+ "sync"
+)
+
+const (
+ routeToken = "/token"
+ routeRevoke = "/revoke"
+ routeIntrospect = "/introspect"
+ routeHealth = "/health"
+
+ paramUser = "user"
+ paramToken = "token"
+
+ contentType = "Content-Type"
+ mimeJSON = "application/json"
+
+ tokenPrefix = "khaos-range-token-"
+)
+
+type introspection struct {
+ Active bool `json:"active"`
+ User string `json:"user,omitempty"`
+}
+
+type identityProvider struct {
+ mu sync.RWMutex
+ issued map[string]string
+ serial int
+}
+
+func newIdentityProvider() *identityProvider {
+ return &identityProvider{issued: map[string]string{}}
+}
+
+func (p *identityProvider) routes() *http.ServeMux {
+ mux := http.NewServeMux()
+ mux.HandleFunc(routeHealth, p.health)
+ mux.HandleFunc(routeToken, p.issue)
+ mux.HandleFunc(routeRevoke, p.revoke)
+ mux.HandleFunc(routeIntrospect, p.introspect)
+ return mux
+}
+
+func (p *identityProvider) health(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+}
+
+func (p *identityProvider) issue(w http.ResponseWriter, r *http.Request) {
+ user := r.URL.Query().Get(paramUser)
+ if user == "" {
+ user = "operator"
+ }
+
+ p.mu.Lock()
+ p.serial++
+ token := tokenPrefix + strconv.Itoa(p.serial)
+ p.issued[token] = user
+ p.mu.Unlock()
+
+ writeJSON(w, map[string]string{paramToken: token, paramUser: user})
+}
+
+func (p *identityProvider) revoke(w http.ResponseWriter, r *http.Request) {
+ token := r.URL.Query().Get(paramToken)
+
+ p.mu.Lock()
+ delete(p.issued, token)
+ p.mu.Unlock()
+
+ w.WriteHeader(http.StatusOK)
+}
+
+func (p *identityProvider) introspect(w http.ResponseWriter, r *http.Request) {
+ token := r.URL.Query().Get(paramToken)
+
+ p.mu.RLock()
+ user, active := p.issued[token]
+ p.mu.RUnlock()
+
+ writeJSON(w, introspection{Active: active, User: user})
+}
+
+func writeJSON(w http.ResponseWriter, body any) {
+ w.Header().Set(contentType, mimeJSON)
+ _ = json.NewEncoder(w).Encode(body)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
new file mode 100644
index 00000000..87a35577
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
@@ -0,0 +1,145 @@
+// ©AngelaMos | 2026
+// main.go
+
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/verifier"
+)
+
+const (
+ roleIDP = "idp"
+ roleGateway = "gateway"
+ roleVerifier = "verifier"
+ roleHealth = "healthcheck"
+
+ envListen = "KHAOS_RANGE_LISTEN"
+ envUpstream = "KHAOS_RANGE_UPSTREAM"
+ envFailMode = "KHAOS_RANGE_FAIL_MODE"
+ envCacheTTL = "KHAOS_RANGE_CACHE_TTL"
+
+ defaultListen = ":8080"
+ defaultUpstream = "http://idp:8080"
+
+ shutdownGrace = 2 * time.Second
+
+ // Generous on purpose. At three seconds this fixture closed khaos's upstream
+ // before a `--delay` beyond that could be observed, so khaos's one graduated
+ // fault had a usable window of roughly 500ms to 3s against its own range and
+ // aborted above it. That ceiling belonged to the fixture, not to the fault.
+ // This is a deliberately-breakable test stack, not a hardened service, so the
+ // header timeout exists only to stop a wedged connection living forever.
+ readHeaderTimeout = 120 * time.Second
+
+ usage = "usage: khaos-range <idp|gateway|verifier|healthcheck>"
+ selfURL = "http://127.0.0.1:8080/health"
+ healthWait = 2 * time.Second
+ errorFormat = "khaos-range: %v\n"
+ exitError = 1
+)
+
+func main() {
+ if err := run(); err != nil {
+ fmt.Fprintf(os.Stderr, errorFormat, err)
+ os.Exit(exitError)
+ }
+}
+
+func run() error {
+ if len(os.Args) < 2 {
+ return errors.New(usage)
+ }
+
+ if os.Args[1] == roleHealth {
+ return selfCheck()
+ }
+
+ handler, err := roleHandler(os.Args[1])
+ if err != nil {
+ return err
+ }
+
+ server := &http.Server{
+ Addr: envOr(envListen, defaultListen),
+ Handler: handler,
+ ReadHeaderTimeout: readHeaderTimeout,
+ }
+
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+
+ go func() {
+ <-ctx.Done()
+ shutdown, cancel := context.WithTimeout(context.Background(), shutdownGrace)
+ defer cancel()
+ _ = server.Shutdown(shutdown)
+ }()
+
+ if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+ return err
+ }
+
+ return nil
+}
+
+func roleHandler(role string) (http.Handler, error) {
+ switch role {
+ case roleIDP:
+ return newIdentityProvider().routes(), nil
+ case roleGateway:
+ return newGateway(
+ envOr(envUpstream, defaultUpstream),
+ envOr(envFailMode, FailClosed),
+ durationOr(envCacheTTL, defaultCacheTTL),
+ ).routes(), nil
+ case roleVerifier:
+ return newVerifierGateway(
+ envOr(envIssuer, defaultIssuer),
+ envOr(envFailMode, string(verifier.Correct)),
+ durationOr(envKeyTTL, verifier.DefaultKeyTTL),
+ durationOr(envCacheTTL, verifier.DefaultDecisionTTL),
+ ).routes(), nil
+ default:
+ return nil, errors.New(usage)
+ }
+}
+
+func selfCheck() error {
+ client := &http.Client{Timeout: healthWait}
+
+ response, err := client.Get(selfURL)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ if response.StatusCode != http.StatusOK {
+ return fmt.Errorf("health returned %d", response.StatusCode)
+ }
+
+ return nil
+}
+
+func envOr(name, fallback string) string {
+ if value := os.Getenv(name); value != "" {
+ return value
+ }
+ return fallback
+}
+
+func durationOr(name string, fallback time.Duration) time.Duration {
+ value, err := time.ParseDuration(os.Getenv(name))
+ if err != nil || value <= 0 {
+ return fallback
+ }
+ return value
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/verifier.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/verifier.go
new file mode 100644
index 00000000..edc351a4
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/verifier.go
@@ -0,0 +1,85 @@
+// ©AngelaMos | 2026
+// verifier.go
+
+package main
+
+import (
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/verifier"
+)
+
+const (
+ envIssuer = "KHAOS_RANGE_ISSUER"
+ envKeyTTL = "KHAOS_RANGE_KEY_TTL"
+
+ defaultIssuer = "http://host.docker.internal:0"
+
+ headerVerifierReason = "X-Khaos-Reason"
+)
+
+// verifierGateway is the third gateway the range runs, and the only one whose
+// identity provider is khaos itself. The other two introspect the range's own
+// provider container, which khaos cannot make lie; this one resolves keys and
+// introspects against khaos's in-process provider, which khaos can arm.
+//
+// It serves BOTH validation strategies, chosen by the shape of the credential,
+// because research 07 section 3.1 establishes that local validation and
+// introspection have genuinely different outage shapes and the `identity.`
+// family spans both. Real gateways dispatch the same way.
+type verifierGateway struct {
+ verifier *verifier.Verifier
+}
+
+func newVerifierGateway(issuer, failMode string, keyTTL, decisionTTL time.Duration) *verifierGateway {
+ issuer = strings.TrimSuffix(issuer, "/")
+
+ return &verifierGateway{
+ verifier: verifier.New(verifier.Config{
+ Issuer: issuer,
+ Audience: idp.DefaultAudience,
+ JWKSURI: issuer + idp.RouteJWKS,
+ IntrospectURI: issuer + idp.RouteIntrospect,
+ Mode: verifier.Resolve(failMode),
+ KeyTTL: keyTTL,
+ DecisionTTL: decisionTTL,
+ Client: &http.Client{Timeout: upstreamTimeout},
+ }),
+ }
+}
+
+func (g *verifierGateway) routes() *http.ServeMux {
+ mux := http.NewServeMux()
+ mux.HandleFunc(routeHealth, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+ mux.HandleFunc(routeProtected, g.protected)
+ return mux
+}
+
+func (g *verifierGateway) protected(w http.ResponseWriter, r *http.Request) {
+ token, scheme := strings.CutPrefix(r.Header.Get(headerAuthorization), bearerPrefix)
+
+ if !scheme || token == "" {
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ decision := g.verifier.Decide(token)
+
+ // The reason is a convenience for a human reading the fixture's responses,
+ // and khaos must never consume it. khaos judges a target from the outside and
+ // a real target volunteers nothing, so an experiment that read this header
+ // would pass against the range and be inert against everything else.
+ w.Header().Set(headerVerifierReason, decision.Reason)
+
+ if decision.Allow {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+
+ w.WriteHeader(http.StatusForbidden)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go
new file mode 100644
index 00000000..da548ebc
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go
@@ -0,0 +1,142 @@
+// ©AngelaMos | 2026
+// main.go
+
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "os/signal"
+ "strconv"
+ "strings"
+ "syscall"
+
+ "golang.org/x/term"
+
+ "github.com/CarterPerez-dev/khaos/internal/cli"
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/run"
+ "github.com/CarterPerez-dev/khaos/internal/shell"
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+ "github.com/CarterPerez-dev/khaos/internal/version"
+)
+
+const (
+ exitOK = 0
+ exitError = 1
+ exitDefect = 2
+
+ // Design section 8 assigns 4 to an inconclusive run. An interrupted run is a
+ // kind of inconclusive one, and Chaos Toolkit's lattice keeps `interrupted`
+ // on the same epistemic axis, so they share it until M10 builds the full
+ // taxonomy out. What matters here is that neither is 0.
+ exitInconclusive = 4
+ exitInterrupted = 4
+
+ exitRevertFailed = 6
+
+ noColorFlag = "--" + cli.FlagNoColor
+ noColorFlagTrue = noColorFlag + "="
+ errorFormat = "%s: %v\n"
+ fixFormat = " fix: %s\n"
+)
+
+var errInterrupted = errors.New("interrupted before the run could reach a verdict")
+
+func main() {
+ os.Exit(execute())
+}
+
+func execute() int {
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+
+ args := os.Args[1:]
+ out := theme.NewWriter(os.Stdout, os.Environ(), noColorRequested(args))
+
+ if interactive(args, os.Stdin, os.Stdout) {
+ if err := shell.Run(ctx, out); err != nil {
+ return fail(err)
+ }
+ return exitOK
+ }
+
+ root := cli.NewRoot()
+ root.SetOut(out)
+ root.SetErr(os.Stderr)
+ root.SetArgs(args)
+
+ if err := root.ExecuteContext(ctx); err != nil {
+ return fail(err)
+ }
+
+ return exitOK
+}
+
+func interactive(args []string, stdin, stdout *os.File) bool {
+ if len(args) > 0 {
+ return false
+ }
+ return term.IsTerminal(int(stdin.Fd())) && term.IsTerminal(int(stdout.Fd()))
+}
+
+func noColorRequested(args []string) bool {
+ for _, arg := range args {
+ if arg == noColorFlag {
+ return true
+ }
+ if value, ok := strings.CutPrefix(arg, noColorFlagTrue); ok {
+ parsed, err := strconv.ParseBool(value)
+ return err == nil && parsed
+ }
+ }
+ return false
+}
+
+// fail maps an error to an exit code, and the ORDER is the contract.
+//
+// Anything about the state khaos left the target in outranks anything about the
+// verdict, and both outrank interruption. Cancellation used to be checked first,
+// so an interrupted run that ALSO failed to revert its fault returned
+// "interrupted" and silently dropped the rollback failure — the one outcome an
+// operator must never miss, hidden by the one they already know about, because
+// `errors.Join` carries both and the first branch won.
+func fail(err error) int {
+ if errors.Is(err, run.ErrRevertFailed) {
+ fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
+ printFix(err)
+ return exitRevertFailed
+ }
+
+ if errors.Is(err, cli.ErrNotRecovered) {
+ fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
+ return exitRevertFailed
+ }
+
+ if errors.Is(err, context.Canceled) {
+ fmt.Fprintf(os.Stderr, errorFormat, version.Name, errInterrupted)
+ return exitInterrupted
+ }
+
+ fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
+ printFix(err)
+
+ if errors.Is(err, cli.ErrDefectFound) {
+ return exitDefect
+ }
+
+ if errors.Is(err, cli.ErrInconclusive) {
+ return exitInconclusive
+ }
+
+ return exitError
+}
+
+func printFix(err error) {
+ var refusal *dockerx.Refusal
+ if errors.As(err, &refusal) && refusal.Fix != "" {
+ fmt.Fprintf(os.Stderr, fixFormat, refusal.Fix)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go
new file mode 100644
index 00000000..0aa6cf52
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go
@@ -0,0 +1,145 @@
+// ©AngelaMos | 2026
+// main_test.go
+
+package main
+
+import (
+ "context"
+ "errors"
+
+ "github.com/CarterPerez-dev/khaos/internal/cli"
+ "github.com/CarterPerez-dev/khaos/internal/run"
+ "os"
+ "testing"
+)
+
+func notATTY(t *testing.T) *os.File {
+ t.Helper()
+
+ f, err := os.Open(os.DevNull)
+ if err != nil {
+ t.Fatalf("open %s: %v", os.DevNull, err)
+ }
+ t.Cleanup(func() { _ = f.Close() })
+
+ return f
+}
+
+func pipeEnd(t *testing.T) *os.File {
+ t.Helper()
+
+ reader, writer, err := os.Pipe()
+ if err != nil {
+ t.Fatalf("pipe: %v", err)
+ }
+ t.Cleanup(func() {
+ _ = reader.Close()
+ _ = writer.Close()
+ })
+
+ return reader
+}
+
+func TestInteractiveRefusesEveryNonTerminalCombination(t *testing.T) {
+ cases := []struct {
+ name string
+ args []string
+ stdin func(*testing.T) *os.File
+ stdout func(*testing.T) *os.File
+ }{
+ {"piped stdin, devnull stdout", nil, pipeEnd, notATTY},
+ {"devnull stdin, devnull stdout", nil, notATTY, notATTY},
+ {"piped stdin only", nil, pipeEnd, notATTY},
+ {"args present", []string{"version"}, notATTY, notATTY},
+ {"args present with a pipe", []string{"targets"}, pipeEnd, notATTY},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if interactive(tc.args, tc.stdin(t), tc.stdout(t)) {
+ t.Fatal("interactive() said yes without a terminal on both ends; " +
+ "the console spins forever on a non-TTY stdin, writing megabytes per second")
+ }
+ })
+ }
+}
+
+func TestInteractiveChecksStdinNotOnlyStdout(t *testing.T) {
+ stdin := pipeEnd(t)
+ stdout := notATTY(t)
+
+ if interactive(nil, stdin, stdout) {
+ t.Fatal("a piped stdin was accepted as interactive")
+ }
+
+ if interactive([]string{"version"}, stdin, stdout) {
+ t.Fatal("arguments did not force the one-shot path")
+ }
+}
+
+func TestNoColorFlagAcceptsEverySpellingPflagDoes(t *testing.T) {
+ cases := []struct {
+ arg string
+ want bool
+ }{
+ {"--no-color", true},
+ {"--no-color=true", true},
+ {"--no-color=TRUE", true},
+ {"--no-color=True", true},
+ {"--no-color=t", true},
+ {"--no-color=T", true},
+ {"--no-color=1", true},
+ {"--no-color=false", false},
+ {"--no-color=F", false},
+ {"--no-color=0", false},
+ {"--no-colour", false},
+ {"--color", false},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.arg, func(t *testing.T) {
+ if got := noColorRequested([]string{"banner", tc.arg}); got != tc.want {
+ t.Fatalf("noColorRequested(%q) = %v, want %v; cobra and this pre-parser must agree "+
+ "or the flag is accepted and silently ignored", tc.arg, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestNoColorIsAbsentByDefault(t *testing.T) {
+ if noColorRequested([]string{"targets", "--project", "shop"}) {
+ t.Fatal("no-color was inferred from unrelated arguments")
+ }
+}
+
+// Exit-code priority. Anything about the state khaos left the target in has to
+// outrank anything about the verdict, and both outrank interruption.
+//
+// Cancellation used to be checked first, so a run that was interrupted AND
+// failed to revert its fault reported "interrupted" and dropped the rollback
+// failure entirely. errors.Join carries both; the first branch simply won.
+func TestRollbackFailureOutranksEveryOtherExitCode(t *testing.T) {
+ cases := []struct {
+ name string
+ err error
+ want int
+ }{
+ {"rollback alone", run.ErrRevertFailed, exitRevertFailed},
+ {"rollback joined with cancellation", errors.Join(context.Canceled, run.ErrRevertFailed), exitRevertFailed},
+ {"rollback joined with a defect", errors.Join(cli.ErrDefectFound, run.ErrRevertFailed), exitRevertFailed},
+ {"rollback joined with inconclusive", errors.Join(cli.ErrInconclusive, run.ErrRevertFailed), exitRevertFailed},
+ {"a target that never came back", cli.ErrNotRecovered, exitRevertFailed},
+ {"cancellation alone", context.Canceled, exitInterrupted},
+ {"defect alone", cli.ErrDefectFound, exitDefect},
+ {"inconclusive alone", cli.ErrInconclusive, exitInconclusive},
+ {"anything else", errors.New("something broke"), exitError},
+ }
+
+ for _, each := range cases {
+ t.Run(each.name, func(t *testing.T) {
+ if got := fail(each.err); got != each.want {
+ t.Fatalf("fail(%v) = %d, want %d", each.err, got, each.want)
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/go.mod b/PROJECTS/intermediate/chaos-engineering-tool/go.mod
new file mode 100644
index 00000000..5d1e83cd
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/go.mod
@@ -0,0 +1,55 @@
+module github.com/CarterPerez-dev/khaos
+
+go 1.26.0
+
+require (
+ charm.land/lipgloss/v2 v2.0.6
+ github.com/charmbracelet/colorprofile v0.4.3
+ github.com/containerd/errdefs v1.0.0
+ github.com/moby/moby/api v1.56.1
+ github.com/moby/moby/client v0.6.1
+ github.com/reeflective/console v0.5.0
+ github.com/spf13/cobra v1.10.2
+ github.com/spf13/pflag v1.0.10
+ golang.org/x/sys v0.48.0
+ golang.org/x/term v0.46.0
+)
+
+require (
+ github.com/Microsoft/go-winio v0.6.2 // indirect
+ github.com/carapace-sh/carapace v1.11.6 // indirect
+ github.com/carapace-sh/carapace-shlex v1.1.1 // indirect
+ github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect
+ github.com/charmbracelet/x/ansi v0.11.8 // indirect
+ github.com/charmbracelet/x/term v0.2.2 // indirect
+ github.com/charmbracelet/x/termios v0.1.1 // indirect
+ github.com/charmbracelet/x/windows v0.2.2 // indirect
+ github.com/clipperhouse/displaywidth v0.11.0 // indirect
+ github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
+ github.com/containerd/errdefs/pkg v0.3.0 // indirect
+ github.com/distribution/reference v0.6.0 // indirect
+ github.com/docker/go-connections v0.8.1 // indirect
+ github.com/docker/go-units v0.5.0 // indirect
+ github.com/felixge/httpsnoop v1.0.4 // indirect
+ github.com/go-logr/logr v1.4.2 // indirect
+ github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+ github.com/lucasb-eyer/go-colorful v1.4.1 // indirect
+ github.com/mattn/go-runewidth v0.0.24 // indirect
+ github.com/moby/docker-image-spec v1.3.1 // indirect
+ github.com/muesli/cancelreader v0.2.2 // indirect
+ github.com/opencontainers/go-digest v1.0.0 // indirect
+ github.com/opencontainers/image-spec v1.1.1 // indirect
+ github.com/reeflective/readline v1.3.0 // indirect
+ github.com/rivo/uniseg v0.4.7 // indirect
+ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+ go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+ go.opentelemetry.io/otel v1.35.0 // indirect
+ go.opentelemetry.io/otel/metric v1.35.0 // indirect
+ go.opentelemetry.io/otel/trace v1.35.0 // indirect
+ golang.org/x/sync v0.22.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+ mvdan.cc/sh/v3 v3.13.1 // indirect
+)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/go.sum b/PROJECTS/intermediate/chaos-engineering-tool/go.sum
new file mode 100644
index 00000000..4eccf5f2
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/go.sum
@@ -0,0 +1,132 @@
+charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ=
+charm.land/lipgloss/v2 v2.0.6/go.mod h1:ipDDJNSGa1hlwDtSfW1s2/xR8Vdhbut4PXh2zEKZd0Q=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/carapace-sh/carapace v1.11.6 h1:fUZv+oAMgbiDEpNPNis4n35tzqE3h8yshOohLJ2Mz4Y=
+github.com/carapace-sh/carapace v1.11.6/go.mod h1:5MUSHyLN9GGb5/NY/j9VI68/TcZV4ApRCAHGg4WeU0s=
+github.com/carapace-sh/carapace-shlex v1.1.1 h1:ccmNeetAYZOk4IcV36youFDsXusT9uCNW2Njkw+QS+Q=
+github.com/carapace-sh/carapace-shlex v1.1.1/go.mod h1:lJ4ZsdxytE0wHJ8Ta9S7Qq0XpjgjU0mdfCqiI2FHx7M=
+github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
+github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
+github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 h1:rdnVWKgJpTVXKuKuJyxDJ+NFJdUaUqGvyGy61OcvlbA=
+github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886/go.mod h1:nAw0d9PhFp1qdzi2xhQU5YOu5sVpDIHWlaW2Uz/bCro=
+github.com/charmbracelet/x/ansi v0.11.8 h1:JMFwp0CgDC2+jcOB162HH5k7I3FVbgFSMMYg7dSPBQQ=
+github.com/charmbracelet/x/ansi v0.11.8/go.mod h1:ZNN+3mXny/516oTQPLMPIBeSINvNJJQ8uQXDgbeJxY0=
+github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
+github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
+github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY=
+github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo=
+github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM=
+github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k=
+github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8=
+github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
+github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
+github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M=
+github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 h1:AgcIVYPa6XJnU3phs104wLj8l5GEththEw6+F79YsIY=
+github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss=
+github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
+github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c=
+github.com/moby/moby/api v1.56.1/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk=
+github.com/moby/moby/client v0.6.1 h1:gvfKmdcWw+9MzXYP0vAUKQtBTGu1L8475K2nsmL7O98=
+github.com/moby/moby/client v0.6.1/go.mod h1:XHgTFqz9NCgS/VuoxXMmEDVmVgXQ4vFEc15wsCIwb24=
+github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
+github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/reeflective/console v0.5.0 h1:CNNMKxa35nOjQaBp5saQV+tg7l0jmmL3MDAkt5w7b14=
+github.com/reeflective/console v0.5.0/go.mod h1:kdGZ6hxATYj2aXIGdAuLhL2ONWpkz1x8QQi2dSZsmB8=
+github.com/reeflective/readline v1.3.0 h1:uh9c2SEmyoy7A/auequfXZjvK0NP5HVEAJFcL9Uf7qE=
+github.com/reeflective/readline v1.3.0/go.mod h1:bOpqx2/VqGlIoobyWR1Vgt/p5FiMfIHj4OicPuw6RfU=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
+go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
+go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
+go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
+go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
+go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
+go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
+go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
+go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
+go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
+go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/exp v0.0.0-20260529124908-c761662dc8c9 h1:4d4PbuBNwaxMXkXI8yiIYjydtMU+04RHeuSxJdgKftM=
+golang.org/x/exp v0.0.0-20260529124908-c761662dc8c9/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
+gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
+mvdan.cc/sh/v3 v3.13.1 h1:DP3TfgZhDkT7lerUdnp6PTGKyxxzz6T+cOlY/xEvfWk=
+mvdan.cc/sh/v3 v3.13.1/go.mod h1:lXJ8SexMvEVcHCoDvAGLZgFJ9Wsm2sulmoNEXGhYZD0=
+pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
+pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/banner.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/banner.go
new file mode 100644
index 00000000..9a43a5c5
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/banner.go
@@ -0,0 +1,51 @@
+// ©AngelaMos | 2026
+// banner.go
+
+package banner
+
+import (
+ _ "embed"
+ "io"
+ "strings"
+
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+ "github.com/CarterPerez-dev/khaos/internal/version"
+)
+
+//go:embed khaos.ascii
+var art string
+
+const (
+ indent = " "
+ metaGap = " "
+ nl = "\n"
+)
+
+func Lines() []string {
+ return strings.Split(strings.TrimRight(art, nl), nl)
+}
+
+func Render() string {
+ var b strings.Builder
+
+ b.WriteString(theme.Block(Lines()))
+ b.WriteString(nl)
+ b.WriteString(nl)
+
+ b.WriteString(indent)
+ b.WriteString(theme.Fg(theme.TextDim, version.Tagline))
+ b.WriteString(metaGap)
+ b.WriteString(theme.Fg(theme.TextBright, version.Version()))
+ b.WriteString(nl)
+
+ b.WriteString(indent)
+ b.WriteString(theme.Fg(theme.TextDim, version.Author))
+ b.WriteString(nl)
+
+ return b.String()
+}
+
+func Write(w io.Writer) error {
+ _, err := io.WriteString(w, Render())
+ return err
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/banner_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/banner_test.go
new file mode 100644
index 00000000..307e62c5
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/banner_test.go
@@ -0,0 +1,89 @@
+// ©AngelaMos | 2026
+// banner_test.go
+
+package banner
+
+import (
+ "bytes"
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+ "github.com/CarterPerez-dev/khaos/internal/version"
+)
+
+const (
+ wantRows = 9
+ wantColumns = 43
+)
+
+func TestArtDimensionsAreLocked(t *testing.T) {
+ lines := Lines()
+
+ if len(lines) != wantRows {
+ t.Fatalf("wordmark has %d rows, want %d", len(lines), wantRows)
+ }
+
+ widest := 0
+ for _, line := range lines {
+ if w := len([]rune(line)); w > widest {
+ widest = w
+ }
+ }
+
+ if widest != wantColumns {
+ t.Fatalf("wordmark is %d columns wide, want %d", widest, wantColumns)
+ }
+}
+
+func TestArtCarriesNoFileHeader(t *testing.T) {
+ for i, line := range Lines() {
+ if strings.Contains(line, "AngelaMos") || strings.HasPrefix(line, "#") {
+ t.Fatalf("line %d looks like a file header and would print inside the banner: %q", i, line)
+ }
+ }
+}
+
+func TestRenderIncludesVersionAndAuthor(t *testing.T) {
+ out := Render()
+
+ for _, want := range []string{version.Tagline, version.Author, version.Version()} {
+ if !strings.Contains(out, want) {
+ t.Fatalf("banner is missing %q", want)
+ }
+ }
+}
+
+func TestRenderThroughNoColorWriterIsPlainText(t *testing.T) {
+ var buf bytes.Buffer
+ w := theme.NewWriter(&buf, []string{"CLICOLOR_FORCE=1", "COLORTERM=truecolor"}, true)
+
+ if err := Write(w); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ if strings.Contains(buf.String(), "\x1b") {
+ t.Fatal("banner emitted escape sequences through a no-color writer")
+ }
+
+ if !strings.Contains(buf.String(), "█") {
+ t.Fatal("the block glyphs did not survive stripping")
+ }
+}
+
+func TestRenderIsDeterministic(t *testing.T) {
+ if Render() != Render() {
+ t.Fatal("banner output changed between two calls")
+ }
+}
+
+func TestWritePropagatesWriterErrors(t *testing.T) {
+ if err := Write(failingWriter{}); err == nil {
+ t.Fatal("Write swallowed a writer error")
+ }
+}
+
+type failingWriter struct{}
+
+func (failingWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/khaos.ascii b/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/khaos.ascii
new file mode 100644
index 00000000..df1d7e61
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/banner/khaos.ascii
@@ -0,0 +1,9 @@
+ ██ ▄█▀ ██░ ██ ▄▄▄ ▒█████ ██████
+ ██▄█▒ ▓██░ ██▒▒████▄ ▒██▒ ██▒▒██ ▒
+▓███▄░ ▒██▀▀██░▒██ ▀█▄ ▒██░ ██▒░ ▓██▄
+▓██ █▄ ░▓█ ░██ ░██▄▄▄▄██ ▒██ ██░ ▒ ██▒
+▒██▒ █▄░▓█▒░██▓ ▓█ ▓██▒░ ████▓▒░▒██████▒▒
+▒ ▒▒ ▓▒ ▒ ░░▒░▒ ▒▒ ▓▒█░░ ▒░▒░▒░ ▒ ▒▓▒ ▒ ░
+░ ░▒ ▒░ ▒ ░▒░ ░ ▒ ▒▒ ░ ░ ▒ ▒░ ░ ░▒ ░ ░
+░ ░░ ░ ░ ░░ ░ ░ ▒ ░ ░ ░ ▒ ░ ░ ░
+░ ░ ░ ░ ░ ░ ░ ░ ░ ░
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/doctor.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/doctor.go
new file mode 100644
index 00000000..8a1f3bac
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/doctor.go
@@ -0,0 +1,101 @@
+// ©AngelaMos | 2026
+// doctor.go
+
+package cli
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/render"
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+)
+
+const (
+ doctorShort = "diagnose the environment and name the fix for anything broken"
+ doctorLong = `doctor resolves the Docker endpoint the same way khaos itself does, checks that
+the daemon answers, and counts the containers that have consented to fault
+injection. Every failed check names the cause and the command that fixes it.
+
+A check that could not run is reported as skipped rather than hidden, so a
+passing report never means a check was quietly not performed.`
+
+ markPass = "ok "
+ markFail = "fail"
+ markSkipped = "skip"
+ checkWidth = 20
+ rowFormat = "%s%s %s %s\n"
+ fixFormat = "%s%s\n"
+)
+
+var errNotReady = errors.New("environment is not ready")
+
+func newDoctorCmd() *cobra.Command {
+ return &cobra.Command{
+ Use: "doctor",
+ Short: doctorShort,
+ Long: doctorLong,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ return runDoctor(cmd.Context(), cmd.OutOrStdout())
+ },
+ }
+}
+
+func runDoctor(ctx context.Context, out io.Writer) error {
+ checks := dockerx.Diagnose(ctx)
+ healthy := true
+
+ if _, err := fmt.Fprintln(out); err != nil {
+ return err
+ }
+
+ for _, check := range checks {
+ if check.Status == dockerx.StatusFail {
+ healthy = false
+ }
+
+ if _, err := fmt.Fprintf(out, rowFormat,
+ render.Indent,
+ statusMark(check.Status),
+ theme.Fg(theme.TextNormal, render.Pad(check.Name, checkWidth)),
+ theme.Fg(theme.TextDim, check.Detail)); err != nil {
+ return err
+ }
+
+ if check.Fix == "" {
+ continue
+ }
+
+ if _, err := fmt.Fprintf(out, fixFormat,
+ render.FixIndent,
+ theme.Fg(theme.TextBright, render.FixPrefix+check.Fix)); err != nil {
+ return err
+ }
+ }
+
+ if _, err := fmt.Fprintln(out); err != nil {
+ return err
+ }
+
+ if !healthy {
+ return errNotReady
+ }
+ return nil
+}
+
+func statusMark(status dockerx.Status) string {
+ switch status {
+ case dockerx.StatusPass:
+ return theme.Fg(theme.StateHeld, markPass)
+ case dockerx.StatusSkipped:
+ return theme.Fg(theme.StateUndetermined, markSkipped)
+ default:
+ return theme.Fg(theme.StatePermitted, markFail)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
new file mode 100644
index 00000000..e2ad3115
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
@@ -0,0 +1,1003 @@
+// ©AngelaMos | 2026
+// experiment.go
+
+package cli
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "maps"
+ "net"
+ "net/url"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+ "github.com/CarterPerez-dev/khaos/internal/lease"
+ "github.com/CarterPerez-dev/khaos/internal/rangelab"
+ "github.com/CarterPerez-dev/khaos/internal/render"
+ "github.com/CarterPerez-dev/khaos/internal/run"
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+ "github.com/CarterPerez-dev/khaos/internal/workspace"
+)
+
+const (
+ flagBaselineFor = "baseline-for"
+ flagFaultFor = "fault-for"
+ flagRecoverFor = "recover-for"
+ flagSampleEvery = "sample-every"
+
+ catalogShort = "list the experiments khaos ships"
+ catalogLong = `Each experiment pairs a fault with two probes: one carrying a credential the
+target should accept, and one carrying a credential it should refuse. The
+second is the negative control, and it is what makes the verdict mean
+anything.
+
+Each experiment also declares its polarity, which is the direction that counts
+as a defect for the control it is testing. Failing open is catastrophic for an
+authentication check and a defensible trade for a rate-limit counter, so khaos
+never assumes.`
+
+ planShort = "show exactly what an experiment would do, without doing any of it"
+ planLong = `plan resolves the target, checks it against every refusal predicate, and prints
+the hypothesis, the fault, the probes and the timing. It touches nothing.`
+
+ runShort = "run an experiment and report whether the target failed closed or failed open"
+ runLong = `run establishes a steady state, injects the fault, measures the target with both
+probes, reverts, and confirms recovery.
+
+It reports two independent verdicts. The epistemic one says whether the run can
+be believed at all. The directional one says which way the target broke. A
+single verdict cannot tell a target that failed closed from one that was never
+enforcing in the first place.`
+
+ baselineUsage = "how long to measure the steady state before injecting"
+ faultUsage = "how long to hold the fault and measure under it"
+ recoverUsage = "how long to measure recovery after reverting"
+ sampleUsage = "how often each probe fires"
+
+ headerExperiment = "EXPERIMENT"
+ headerFamily = "FAMILY"
+ headerFault = "FAULT"
+ // Wide enough for the longest NAME in the catalogue, because the experiment
+ // name is the thing an operator types. `identity.claim_foreign_issuer` was
+ // truncated to `identity.claim_foreign_is...` at 30, which is a listing that
+ // cannot be copied from.
+ colExperiment = 32
+ colFamily = 12
+ colFault = 30
+
+ labelHypothesis = "hypothesis"
+ labelFault = "fault"
+ labelTarget = "target"
+ labelValid = "valid probe"
+ labelInvalid = "invalid probe"
+ labelPolarity = "defect is"
+ labelTiming = "timing"
+ labelVerdict = "verdict"
+ labelCause = "cause"
+ labelEvidence = "evidence"
+ labelWarning = "warning"
+ labelRun = "run"
+ labelWhy = "why"
+ labelWidth = 14
+
+ fieldLayout = "%s%s%s\n"
+ timingFormat = "baseline %s, fault %s, recover %s, sampling every %s"
+ verdictFormat = "%s / %s"
+ evidenceFormat = "baseline %d paired samples, fault %d, recovery %d"
+ stageFormat = "%-9s %s"
+ planOnly = "plan only, nothing was injected"
+ msgUsingRange = "using the running range: %s"
+ errHalfSpecified = "you gave %s but not %s; khaos will not fill one half from its range and the other from you, because that pauses a container you named and measures a service you did not"
+ errNoExperiment = "name the experiment to %s (known experiments: %s)"
+ errNoRangeFor = "%w by passing %s"
+ errInconclusive = "%w (%s)"
+ verdictThinEvidence = "the sample cannot rule out a bypass"
+ verdictRecoveryUnobserved = "recovery was never observed"
+ errNoProxiedPath = "%w: this experiment injects through khaos's fault proxy, and the running range has no %s; bring the range down and up again to create it"
+ errNoVerifierPath = "%w: this experiment arms khaos's own identity provider, and the running range has no %s; bring the range down and up again to create it"
+ errIdPBind = "khaos could not bind its identity provider on %s, which is where the range's verifier gateway is already looking for it: %w. Something else took that port since the range came up; bring the range down and up again to pick a new one"
+ errIssuerPort = "%w: khaos's identity provider must bind the port its issuer names, and %q does not name a usable one (%v); pass --issuer as a URL with an explicit port, such as http://host.docker.internal:45000"
+ msgIdPUp = "khaos's identity provider is listening on %s, issuing as %s"
+ errProxyBind = "khaos could not bind its fault proxy on %s, which is where the range's proxied gateway is already looking for it: %w. Something else took that port since the range came up; bring the range down and up again to pick a new one"
+ msgProxyUp = "khaos's fault proxy is listening on %s, forwarding to %s"
+
+ groupExperiments = "experiments"
+ titleExperiments = "Experiments:"
+
+ paramDefault = "%s (defaults to %q)"
+ flagPrefix = "--"
+ pairFormat = "%s %s"
+ listSep = ", "
+)
+
+var ErrDefectFound = errors.New("the target broke in the direction this experiment declares a defect")
+
+var ErrNotRecovered = errors.New("the target did not return to its steady state after khaos reverted the fault, so it may still be degraded")
+
+var ErrUnusableIssuer = errors.New("that issuer does not tell khaos where to listen")
+
+var ErrInconclusive = errors.New("this run did not reach a conclusion, so nothing can be concluded from it either way")
+
+func newCatalogCmd() *cobra.Command {
+ return &cobra.Command{
+ Use: "catalog",
+ Short: catalogShort,
+ Long: catalogLong,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ return runCatalog(cmd.OutOrStdout())
+ },
+ }
+}
+
+// Every experiment is its own subcommand, so the flags an operator sees are
+// exactly the parameters the selected experiment declares. Registering the union
+// of every experiment's parameters on one `run` would show a beginner twenty
+// flags that do nothing for the experiment they picked, and deciding the set
+// from a positional argument is not available in the console surface at all:
+// the console regenerates the command tree without being told the line.
+func newPlanCmd() *cobra.Command {
+ cmd := experimentParent("plan <experiment>", planShort, planLong)
+
+ for _, definition := range experiment.Catalog() {
+ cmd.AddCommand(newPlanExperimentCmd(definition))
+ }
+
+ return cmd
+}
+
+func newRunCmd() *cobra.Command {
+ cmd := experimentParent("run <experiment>", runShort, runLong)
+
+ for _, definition := range experiment.Catalog() {
+ cmd.AddCommand(newRunExperimentCmd(definition))
+ }
+
+ return cmd
+}
+
+// experimentParent resolves a name and nothing else. Unknown flags are tolerated
+// on purpose: without that, `khaos run typo --url x` reports the unknown FLAG,
+// burying the only thing wrong with the line.
+func experimentParent(use, short, long string) *cobra.Command {
+ verb := strings.Fields(use)[0]
+
+ cmd := &cobra.Command{
+ Use: use,
+ Short: short,
+ Long: long,
+ Args: cobra.ArbitraryArgs,
+ FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true},
+ RunE: func(_ *cobra.Command, args []string) error {
+ return unknownExperiment(verb, args)
+ },
+ }
+
+ cmd.AddGroup(&cobra.Group{ID: groupExperiments, Title: titleExperiments})
+
+ return cmd
+}
+
+func unknownExperiment(verb string, args []string) error {
+ if len(args) > 0 {
+ if _, err := experiment.Lookup(args[0]); err != nil {
+ return err
+ }
+ }
+ return fmt.Errorf(errNoExperiment, verb, strings.Join(experiment.Names(), listSep))
+}
+
+func newPlanExperimentCmd(definition experiment.Definition) *cobra.Command {
+ cmd := &cobra.Command{
+ Use: definition.Name,
+ Short: definition.Summary,
+ Long: planLong,
+ GroupID: groupExperiments,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ given, err := experimentInputs(cmd, definition)
+ if err != nil {
+ return err
+ }
+ return runPlan(cmd.Context(), cmd.OutOrStdout(), definition, given)
+ },
+ }
+
+ experimentFlags(cmd, definition)
+
+ return cmd
+}
+
+func newRunExperimentCmd(definition experiment.Definition) *cobra.Command {
+ cmd := &cobra.Command{
+ Use: definition.Name,
+ Short: definition.Summary,
+ Long: runLong,
+ GroupID: groupExperiments,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ given, err := experimentInputs(cmd, definition)
+ if err != nil {
+ return err
+ }
+ ttl, err := cmd.Flags().GetDuration(flagTTL)
+ if err != nil {
+ return err
+ }
+ return runExperiment(cmd.Context(), cmd.OutOrStdout(), definition, ttl, given)
+ },
+ }
+
+ experimentFlags(cmd, definition)
+ cmd.Flags().Duration(flagTTL, lease.DefaultTTL, ttlUsage)
+
+ return cmd
+}
+
+func experimentFlags(cmd *cobra.Command, definition experiment.Definition) {
+ cmd.Flags().String(flagProject, "", projectUsage)
+
+ // Every parameter registers with an empty default even when it declares one,
+ // because an empty value is how khaos tells "the operator did not set this"
+ // from "the operator set this". Defaults are applied once, by Bind.
+ for _, param := range definition.Params {
+ cmd.Flags().String(param.Name, "", paramUsage(param))
+ }
+
+ cmd.Flags().Duration(flagBaselineFor, experiment.DefaultBaselineFor, baselineUsage)
+ cmd.Flags().Duration(flagFaultFor, experiment.DefaultFaultFor, faultUsage)
+ cmd.Flags().Duration(flagRecoverFor, experiment.DefaultRecoveryFor, recoverUsage)
+ cmd.Flags().Duration(flagSampleEvery, experiment.DefaultSampleInterval, sampleUsage)
+}
+
+func paramUsage(param experiment.Param) string {
+ if param.Default == "" {
+ return param.Usage
+ }
+ return fmt.Sprintf(paramDefault, param.Usage, param.Default)
+}
+
+type inputs struct {
+ Project string
+ Supplied map[string]string
+ Timing experiment.Timing
+}
+
+func experimentInputs(cmd *cobra.Command, definition experiment.Definition) (inputs, error) {
+ project, err := cmd.Flags().GetString(flagProject)
+ if err != nil {
+ return inputs{}, err
+ }
+
+ // Changed, not a non-empty value. Sniffing for "" cannot tell an operator who
+ // typed `--invalid-token ""` from one who typed nothing, and those must differ:
+ // an empty credential means no Authorization header at all, which is the
+ // canonical unauthenticated negative control. Treating it as unset silently
+ // replaced it with the default and made that control inexpressible.
+ supplied := map[string]string{}
+ for _, param := range definition.Params {
+ if !cmd.Flags().Changed(param.Name) {
+ continue
+ }
+
+ value, err := cmd.Flags().GetString(param.Name)
+ if err != nil {
+ return inputs{}, err
+ }
+
+ supplied[param.Name] = value
+ }
+
+ timing, err := experimentTiming(cmd)
+ if err != nil {
+ return inputs{}, err
+ }
+
+ return inputs{Project: project, Supplied: supplied, Timing: timing}, nil
+}
+
+func experimentTiming(cmd *cobra.Command) (experiment.Timing, error) {
+ durations := map[string]time.Duration{}
+
+ for _, name := range []string{flagBaselineFor, flagFaultFor, flagRecoverFor, flagSampleEvery} {
+ value, err := cmd.Flags().GetDuration(name)
+ if err != nil {
+ return experiment.Timing{}, err
+ }
+ durations[name] = value
+ }
+
+ return experiment.Timing{
+ BaselineFor: durations[flagBaselineFor],
+ FaultFor: durations[flagFaultFor],
+ RecoveryFor: durations[flagRecoverFor],
+ SampleInterval: durations[flagSampleEvery],
+ }, nil
+}
+
+func runCatalog(out io.Writer) error {
+ fmt.Fprintln(out)
+ fmt.Fprintf(out, rowLayout, render.Indent,
+ theme.Fg(theme.TextFaint, render.Cell(headerExperiment, colExperiment)),
+ theme.Fg(theme.TextFaint, render.Cell(headerFamily, colFamily)),
+ theme.Fg(theme.TextFaint, render.Cell(headerFault, colFault)),
+ theme.Fg(theme.TextFaint, "DEFECT IS"))
+
+ for _, definition := range experiment.Catalog() {
+ fmt.Fprintf(out, rowLayout, render.Indent,
+ theme.Fg(theme.TextBright, render.Cell(definition.Name, colExperiment)),
+ theme.Fg(theme.TextNormal, render.Cell(definition.Family, colFamily)),
+ theme.Fg(theme.TextDim, render.Cell(definition.FaultKind, colFault)),
+ theme.Fg(directionColor(definition.Polarity.Defects[0]), definition.Polarity.Declared()))
+
+ fmt.Fprintf(out, noteLayout, render.FixIndent, theme.Fg(theme.TextDim, definition.Summary))
+ }
+
+ fmt.Fprintln(out)
+ return nil
+}
+
+func runPlan(ctx context.Context, out io.Writer, definition experiment.Definition, given inputs) error {
+ work, err := workspace.OpenShared(ctx)
+ if err != nil {
+ return err
+ }
+ defer work.Close()
+
+ supplied, err := fillFromRange(ctx, out, work.Docker, definition, given.Supplied)
+ if err != nil {
+ return err
+ }
+
+ spec, err := experiment.Build(definition.Name, supplied, given.Timing)
+ if err != nil {
+ return err
+ }
+
+ project := given.Project
+ guard := dockerx.Guard{Project: project, SelfID: dockerx.SelfContainerID()}
+
+ target, err := work.Docker.Accept(ctx, spec.Fault.Target, dockerx.Filter{Project: project}, guard)
+ if err != nil {
+ return err
+ }
+
+ describe(out, spec, target.Name)
+ note(out, planOnly)
+ fmt.Fprintln(out)
+
+ return nil
+}
+
+func describe(out io.Writer, spec experiment.Experiment, resolved string) {
+ fmt.Fprintln(out)
+ field(out, labelHypothesis, spec.Hypothesis.String())
+ field(out, labelFault, spec.Fault.Kind)
+ field(out, labelTarget, resolved)
+ field(out, labelValid, spec.Probes.Valid.Describe())
+ field(out, labelInvalid, spec.Probes.Invalid.Describe())
+ field(out, labelPolarity, spec.Polarity.Declared()+" ("+spec.Polarity.Rationale+")")
+
+ timing := spec.Timing
+ field(out, labelTiming, fmt.Sprintf(timingFormat,
+ orDefault(timing.BaselineFor, experiment.DefaultBaselineFor),
+ orDefault(timing.FaultFor, experiment.DefaultFaultFor),
+ orDefault(timing.RecoveryFor, experiment.DefaultRecoveryFor),
+ orDefault(timing.SampleInterval, experiment.DefaultSampleInterval)))
+ fmt.Fprintln(out)
+}
+
+func runExperiment(
+ ctx context.Context,
+ out io.Writer,
+ definition experiment.Definition,
+ ttl time.Duration,
+ given inputs,
+) error {
+ work, err := workspace.OpenExclusive(ctx)
+ if err != nil {
+ return err
+ }
+ defer work.Close()
+
+ supplied, err := fillFromRange(ctx, out, work.Docker, definition, given.Supplied)
+ if err != nil {
+ return err
+ }
+
+ spec, err := experiment.Build(definition.Name, supplied, given.Timing)
+ if err != nil {
+ return err
+ }
+
+ project := given.Project
+ guard := dockerx.Guard{Project: project, SelfID: dockerx.SelfContainerID()}
+
+ // The target is resolved and checked against every refusal predicate BEFORE
+ // anything is printed, exactly as plan does it. Without this, a target khaos
+ // would refuse by name reached the engine instead, the baseline found nothing
+ // serving, and the run reported `baseline-not-met` and exit 0 against a target
+ // khaos never even accepted. That is the same "true statement about a run that
+ // never had a chance, blaming the target" failure the parameter model exists
+ // to kill, arriving on the output path instead of the input path.
+ accepted, err := work.Docker.Accept(ctx, spec.Fault.Target, dockerx.Filter{Project: project}, guard)
+ if err != nil {
+ return err
+ }
+
+ // The proxy has to be up before the BASELINE, not before the fault. The
+ // baseline is measured through it, so a proxy that arrives at injection time
+ // would make the two phases travel different paths and the difference between
+ // them would be the proxy rather than the fault.
+ env := work.Env()
+
+ if spec.ProxyFault != nil {
+ fault, err := startProxy(ctx, out, work.Docker, spec)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = fault.Close() }()
+
+ env.Proxy = fault
+ env.ProxyFault = *spec.ProxyFault
+ }
+
+ // The provider has to be listening before the BASELINE for the same reason
+ // the proxy does: the baseline is measured through it, so a provider that
+ // arrived at injection time would make the two phases see different
+ // topologies and the difference between them would be khaos rather than the
+ // fault. The experiment already BUILT it, because its probes mint out of it
+ // and minting needs no listener; all that is left is to bind it where the
+ // target is already looking.
+ if spec.IdPTrap != nil {
+ if err := startProvider(ctx, out, work.Docker, spec); err != nil {
+ return err
+ }
+ defer func() { _ = spec.IdP.Close() }()
+
+ env.IdP = spec.IdP
+ env.IdPTrap = *spec.IdPTrap
+ }
+
+ session := run.NewSession(env, work.Journal, ttl)
+
+ injector := &run.Injector{
+ Session: session,
+ Resolver: work.Docker,
+ Filter: dockerx.Filter{Project: project},
+ Guard: guard,
+ Progress: func(record journal.Record, failure error) {
+ if failure != nil {
+ event(out, markError, theme.StatusError,
+ fmt.Sprintf(faultFormat, record.Kind, record.TargetName, failure))
+ return
+ }
+ event(out, markRevert, theme.StatusOK, subject(record.Kind, record.TargetName))
+ },
+ }
+
+ engine := experiment.Engine{
+ Injector: injector,
+ Report: func(e experiment.Event) {
+ event(out, markPlan, theme.TextFaint, fmt.Sprintf(stageFormat, e.Stage, e.Detail))
+ },
+ }
+
+ describe(out, spec, accepted.Name)
+
+ // The lease must be renewed for as long as the run holds a fault. Without
+ // this, `khaos reap` reverts the fault partway through and the engine
+ // scores a target that was never under fault as a clean survival.
+ keepCtx, stopKeeping := context.WithCancel(ctx)
+ renewals := make(chan error, 1)
+
+ go func() {
+ renewals <- session.Lease().Keep(keepCtx, func(time.Time) error { return session.Renew() })
+ }()
+
+ result, runErr := engine.Run(ctx, spec)
+
+ stopKeeping()
+ if keepErr := <-renewals; !errors.Is(keepErr, context.Canceled) {
+ result.Verdict.Epistemic = experiment.Aborted
+ result.Verdict.Conclusive = false
+ result.Verdict.Reasons = append(result.Verdict.Reasons, keepErr.Error())
+ runErr = errors.Join(runErr, keepErr)
+ }
+
+ result.RunID = session.RunID()
+ reportVerdict(out, result)
+
+ if runErr != nil {
+ return runErr
+ }
+
+ if result.Observation.RollbackFailed {
+ return run.ErrRevertFailed
+ }
+
+ // A target khaos could not see come back is a target khaos may have left
+ // broken, and that must not exit 0. It ranks above the defect check for the
+ // same reason a failed rollback does: the operator's first job is the state
+ // of their system, not the verdict about it.
+ if result.Verdict.Recovery == experiment.Degraded {
+ return ErrNotRecovered
+ }
+
+ if result.Verdict.Defect {
+ return ErrDefectFound
+ }
+
+ // A run that measured nothing must not exit 0. Every inconclusive epistemic
+ // outcome used to, so `khaos run network.dependency_partial --bytes 999999`
+ // against a gateway explicitly configured to fail open was a GREEN build: the
+ // fault was too large to confirm, the engine aborted before measuring, the
+ // verdict text said so honestly, and the exit code said success. Design
+ // section 8 assigns 4 to inconclusive, and that is what this produces.
+ if !conclusive(result.Verdict.Epistemic) {
+ return fmt.Errorf(errInconclusive, ErrInconclusive, result.Verdict.Epistemic)
+ }
+
+ // khaos's OWN confidence field, which the exit code used to ignore entirely.
+ // A HELD verdict carrying `Conclusive=false` says in its own reasons that the
+ // sample cannot rule out a bypass, and then exited 0 anyway — the same
+ // dishonesty the inconclusive exit code was introduced to end, arriving one
+ // field over.
+ if !result.Verdict.Conclusive {
+ return fmt.Errorf(errInconclusive, ErrInconclusive, verdictThinEvidence)
+ }
+
+ // Recovery that was never observed is not recovery.
+ if result.Verdict.Recovery == experiment.Unobserved {
+ return fmt.Errorf(errInconclusive, ErrInconclusive, verdictRecoveryUnobserved)
+ }
+
+ return nil
+}
+
+// conclusive reports whether the epistemic axis reached a belief about the target
+// at all. Passed and Deviated are answers; everything else says the run could not
+// produce one.
+func conclusive(epistemic experiment.Epistemic) bool {
+ return epistemic == experiment.Passed || epistemic == experiment.Deviated
+}
+
+func reportVerdict(out io.Writer, result experiment.Result) {
+ fmt.Fprintln(out)
+
+ field(out, labelVerdict, theme.Fg(directionColor(result.Verdict.Direction),
+ fmt.Sprintf(verdictFormat, result.Verdict.Epistemic, result.Verdict.Direction)))
+
+ // The cause is only printed when something established one. An UNDIAGNOSED
+ // line on every run would read as a field khaos failed to fill rather than a
+ // diagnosis it never attempted.
+ if result.Verdict.Cause != experiment.Undiagnosed {
+ field(out, labelCause, theme.Fg(theme.StatePermitted, string(result.Verdict.Cause)))
+ }
+
+ if result.RunID != "" {
+ field(out, labelRun, result.RunID)
+ }
+
+ for _, warning := range result.Verdict.Warnings {
+ field(out, labelWarning, theme.Fg(theme.StatusWarn, string(warning)))
+ }
+
+ field(out, labelEvidence, fmt.Sprintf(evidenceFormat,
+ result.Observation.Baseline.Len(),
+ result.Observation.Fault.Len(),
+ result.Observation.Recovery.Len()))
+
+ for _, reason := range result.Verdict.Reasons {
+ field(out, labelWhy, reason)
+ }
+
+ fmt.Fprintln(out)
+}
+
+func field(out io.Writer, label, value string) {
+ fmt.Fprintf(out, fieldLayout, render.Indent,
+ theme.Fg(theme.TextFaint, render.Pad(label, labelWidth)), value)
+}
+
+func directionColor(direction experiment.Direction) theme.Color {
+ switch direction {
+ case experiment.Held:
+ return theme.StateHeld
+ case experiment.Denied:
+ return theme.StateDenied
+ case experiment.Permitted, experiment.Crashed:
+ return theme.StatePermitted
+ default:
+ return theme.StateUndetermined
+ }
+}
+
+func orDefault(value, fallback time.Duration) time.Duration {
+ if value <= 0 {
+ return fallback
+ }
+ return value
+}
+
+// rangeSupplies names the parameters a running range answers for itself. It is
+// the key set of rangeDefaults, pinned by a test, and it has to be knowable
+// without touching anything because it decides whether khaos looks for a range
+// at all.
+func rangeSupplies() []string {
+ return []string{
+ experiment.ParamTarget,
+ experiment.ParamURL,
+ experiment.ParamToken,
+ experiment.ParamAuthority,
+ experiment.ParamRevocable,
+ experiment.ParamIssuer,
+ experiment.ParamProxyListen,
+ experiment.ParamProxyUpstream,
+ }
+}
+
+// tier is which of the range's three gateways an experiment belongs to, and it
+// is derived from the registry rather than from the experiment's name. Each
+// gateway exists so one injection tier is measured on an uncontaminated path:
+// pointing a Docker-native experiment at a gateway that talks through a proxy
+// measures the pair, and pointing a forgery experiment at a gateway whose issuer
+// khaos cannot arm measures nothing at all.
+type tier int
+
+const (
+ tierDirect tier = iota
+ tierProxy
+ tierIdP
+)
+
+func tierOf(kind string) tier {
+ switch {
+ case inject.NeedsProxy(kind):
+ return tierProxy
+ case inject.NeedsIdP(kind):
+ return tierIdP
+ default:
+ return tierDirect
+ }
+}
+
+// rangeDefaults answers per EXPERIMENT, not just per range. A proxy-tier
+// experiment must probe the gateway whose own upstream runs through khaos's
+// proxy; pointing it at the direct gateway would arm a fault on a path that
+// gateway never uses and measure a service with no causal connection to it.
+//
+// The credential is always issued by the direct gateway. Both gateways share one
+// identity provider, so the credential is valid at either, and issuing it on the
+// direct path means token issuance never depends on the proxy already being up.
+// rangeAnswers is everything khaos has to go and get before it can answer for
+// the range. Keeping rangeDefaults pure over this is what lets a test pin the
+// VALUES rather than only the key set.
+type rangeAnswers struct {
+ Token string
+ Revocable string
+ Authority string
+
+ // Where the range's proxied gateway expects khaos's proxy, and the
+ // dependency behind it. Both come from the daemon rather than from what
+ // `range up` chose, for the reason recorded on proxyPortOf.
+ ProxyListen string
+ ProxyUpstream string
+}
+
+func rangeDefaults(lab rangelab.Lab, answers rangeAnswers, at tier) map[string]string {
+ url := lab.ProtectedURL()
+ target := rangelab.IDPName
+
+ switch at {
+ case tierProxy:
+ url = lab.ProxiedProtectedURL()
+ case tierIdP:
+ // The target is the VERIFIER GATEWAY, not the provider container. That is
+ // the one place this tier inverts the others: there the fault is applied
+ // to the dependency and the gateway is probed, while here the fault is
+ // applied to khaos's own in-process provider and the gateway is both the
+ // subject and the thing khaos needs consent for. Naming the provider
+ // container would ask consent for a container this experiment never
+ // touches and leave the one it does measure unchecked.
+ url = lab.VerifierProtectedURL()
+ target = rangelab.VerifierGatewayName
+ }
+
+ return map[string]string{
+ experiment.ParamTarget: target,
+ experiment.ParamURL: url,
+ experiment.ParamToken: answers.Token,
+ experiment.ParamRevocable: answers.Revocable,
+ experiment.ParamAuthority: answers.Authority,
+ experiment.ParamIssuer: lab.Issuer,
+
+ experiment.ParamProxyListen: answers.ProxyListen,
+ experiment.ParamProxyUpstream: answers.ProxyUpstream,
+ }
+}
+
+// fillFromRange supplies everything the operator did not, from a running range.
+// It is what makes the zero-flag commitment in the UX law real: khaos already
+// knows the gateway's randomly chosen port and can ask it for a credential, so
+// it does not make the operator repeat either one.
+//
+// Three states, not two, and the partition is derived from what the range can
+// answer rather than from a hand-written pair of field checks. Filling part of
+// an invocation fuses the operator's target with the range's URL, which pauses a
+// container they named and measures a service they did not.
+func fillFromRange(
+ ctx context.Context,
+ out io.Writer,
+ reader rangelab.PortReader,
+ definition experiment.Definition,
+ supplied map[string]string,
+) (map[string]string, error) {
+ given, missing := partitionFillable(definition, supplied)
+
+ if len(missing) == 0 {
+ return supplied, nil
+ }
+
+ if len(given) > 0 {
+ return nil, fmt.Errorf(errHalfSpecified, flagList(given), flagList(missing))
+ }
+
+ lab, err := rangelab.Discover(ctx, reader)
+ if err != nil {
+ if errors.Is(err, rangelab.ErrNoRange) {
+ return nil, fmt.Errorf(errNoRangeFor, err, flagList(missing))
+ }
+ return nil, err
+ }
+
+ // Refuse BEFORE announcing anything. Without this the note printed a URL
+ // khaos had already resolved to port 0, immediately above the refusal saying
+ // the part that would have supplied the port is missing.
+ at := tierOf(definition.FaultKind)
+
+ if at == tierProxy && lab.ProxiedPort == 0 {
+ return nil, fmt.Errorf(errNoProxiedPath, rangelab.ErrRangeIncomplete, rangelab.ProxiedGatewayName)
+ }
+
+ if at == tierIdP && (lab.VerifierPort == 0 || lab.Issuer == "") {
+ return nil, fmt.Errorf(errNoVerifierPath, rangelab.ErrRangeIncomplete, rangelab.VerifierGatewayName)
+ }
+
+ answers, err := askTheRange(ctx, reader, lab, missing)
+ if err != nil {
+ return nil, err
+ }
+
+ defaults := rangeDefaults(lab, answers, at)
+ filled := maps.Clone(supplied)
+ resolved := make([]string, 0, len(missing))
+
+ // Credentials are deliberately omitted from the note. Everything else khaos
+ // chose on the operator's behalf is named, because this line is the only thing
+ // telling them which stack they are about to break.
+ for _, name := range missing {
+ filled[name] = defaults[name]
+ if !credentialParam(name) {
+ resolved = append(resolved, fmt.Sprintf(pairFormat, name, defaults[name]))
+ }
+ }
+
+ note(out, fmt.Sprintf(msgUsingRange, strings.Join(resolved, listSep)))
+
+ return filled, nil
+}
+
+// partitionFillable splits the parameters this experiment declares AND the range
+// can answer into the ones the operator gave and the ones they did not.
+func partitionFillable(definition experiment.Definition, supplied map[string]string) ([]string, []string) {
+ answerable := rangeSupplies()
+ given := []string{}
+ missing := []string{}
+
+ for _, param := range definition.Params {
+ if !slices.Contains(answerable, param.Name) {
+ continue
+ }
+
+ // Presence, not emptiness: an operator who typed `--token ""` DID supply
+ // it, and refusing with "you gave --target, --url but not --token"
+ // contradicts the line they just typed. An empty value that is not allowed
+ // is Bind's refusal to make, by name.
+ if _, present := supplied[param.Name]; present {
+ given = append(given, param.Name)
+ continue
+ }
+
+ // A parameter the declaration already answers is not missing. Demanding a
+ // value the experiment supplies itself sends the operator looking for
+ // something khaos already has.
+ if param.Default != "" {
+ continue
+ }
+
+ missing = append(missing, param.Name)
+ }
+
+ return given, missing
+}
+
+func flagList(names []string) string {
+ out := make([]string, 0, len(names))
+ for _, name := range names {
+ out = append(out, flagPrefix+name)
+ }
+ return strings.Join(out, listSep)
+}
+
+// startProxy stands up khaos's in-process fault proxy where the target is already
+// looking for it. The address is not khaos's choice: the proxied gateway's
+// environment is fixed at container creation, so khaos binds the port that target
+// is ALREADY addressing rather than one it would have picked.
+//
+// The listen address is the Docker bridge gateway, discovered rather than assumed.
+// Measured: a container reaches a host listener there, a listener on the target
+// network's own gateway is unreachable, and binding that one address leaves the
+// proxy dead on loopback and dead on the host's LAN address.
+func startProxy(
+ _ context.Context,
+ out io.Writer,
+ _ rangelab.PortReader,
+ spec experiment.Experiment,
+) (*proxy.Proxy, error) {
+ // From the SPEC, not from a range. These are declared parameters now, so a
+ // fully specified invocation against somebody's own stack no longer fails
+ // looking for a fixture it was never going to use.
+ listen, upstream := spec.ProxyListen, spec.ProxyUpstream
+
+ fault, err := proxy.Open(listen, upstream)
+ if err != nil {
+ // A refusal to bind THERE is not a port collision, and telling the
+ // operator to bring the range down and up again prescribes a fix that
+ // cannot work. Only a genuine collision gets the collision advice.
+ if errors.Is(err, proxy.ErrUnsafeBind) {
+ return nil, err
+ }
+ return nil, fmt.Errorf(errProxyBind, listen, err)
+ }
+
+ note(out, fmt.Sprintf(msgProxyUp, fault.Addr(), fault.Upstream()))
+
+ return fault, nil
+}
+
+// startProvider binds khaos's own identity provider where the target is already
+// addressing it. The address is not khaos's choice: the verifier gateway's
+// environment is fixed at container creation, so khaos binds the port that
+// container will dial rather than one it would have picked.
+//
+// The listen address is the Docker bridge gateway, discovered rather than
+// assumed, which is the same measured fact the fault proxy rests on. A wildcard
+// bind would put a deliberately-wrong identity provider on the operator's
+// network, which is the one thing this tier must never do.
+func startProvider(
+ ctx context.Context,
+ out io.Writer,
+ reader rangelab.PortReader,
+ spec experiment.Experiment,
+) error {
+ // The ISSUER already says where to bind, so this tier needs no range at all.
+ //
+ // It used to call rangelab.Discover unconditionally, which meant a fully
+ // specified invocation against somebody's own stack failed looking for a
+ // range it was never going to use — while khaos's own refusal text invited
+ // exactly that: "or point khaos at your own stack instead". Seventeen of the
+ // eighteen identity experiments were unusable outside the bundled fixture.
+ //
+ // khaos cannot BIND the issuer's host, because the target addresses it as
+ // `host.docker.internal` and that name does not resolve here. It binds the
+ // Docker bridge gateway on the issuer's PORT, which is the same address the
+ // container reaches and the same discovery the fault proxy already does.
+ issuer := spec.IdP.Issuer()
+
+ port, err := portOf(issuer)
+ if err != nil {
+ return err
+ }
+
+ gateway, err := reader.BridgeGateway(ctx)
+ if err != nil {
+ return err
+ }
+
+ listen := net.JoinHostPort(gateway, strconv.Itoa(port))
+
+ if err := spec.IdP.Listen(listen); err != nil {
+ return fmt.Errorf(errIdPBind, listen, err)
+ }
+
+ note(out, fmt.Sprintf(msgIdPUp, listen, issuer))
+
+ return nil
+}
+
+// portOf reads the port an issuer names. It is the one thing khaos needs from
+// the issuer beyond minting under it, and a missing or unusable port is a
+// refusal by name rather than a bind to something arbitrary.
+func portOf(issuer string) (int, error) {
+ parsed, err := url.Parse(issuer)
+ if err != nil {
+ return 0, fmt.Errorf(errIssuerPort, ErrUnusableIssuer, issuer, err)
+ }
+
+ port, err := strconv.Atoi(parsed.Port())
+ if err != nil || port <= 0 {
+ return 0, fmt.Errorf(errIssuerPort, ErrUnusableIssuer, issuer, err)
+ }
+
+ return port, nil
+}
+
+func credentialParam(name string) bool {
+ return name == experiment.ParamToken || name == experiment.ParamRevocable
+}
+
+// askTheRange goes and gets only what this invocation is actually missing. A
+// second credential costs a round trip and the authority's address costs a
+// container inspection, and an experiment that needs neither should pay for
+// neither.
+func askTheRange(
+ ctx context.Context,
+ reader rangelab.PortReader,
+ lab rangelab.Lab,
+ missing []string,
+) (rangeAnswers, error) {
+ answers := rangeAnswers{}
+ want := func(name string) bool { return slices.Contains(missing, name) }
+
+ if want(experiment.ParamToken) {
+ token, err := lab.IssueToken(ctx)
+ if err != nil {
+ return rangeAnswers{}, err
+ }
+ answers.Token = token
+ }
+
+ // A SECOND credential, issued separately. The one khaos revokes has to be
+ // distinct from the one it keeps presenting as valid, or revoking it would
+ // take away the stream that proves the target is still serving at all and
+ // every verdict would read as a crash.
+ if want(experiment.ParamRevocable) {
+ revocable, err := lab.IssueToken(ctx)
+ if err != nil {
+ return rangeAnswers{}, err
+ }
+ answers.Revocable = revocable
+ }
+
+ if want(experiment.ParamProxyListen) || want(experiment.ParamProxyUpstream) {
+ listen, upstream, err := lab.ProxyEndpoint(ctx, reader)
+ if err != nil {
+ return rangeAnswers{}, err
+ }
+ answers.ProxyListen = listen
+ answers.ProxyUpstream = upstream
+ }
+
+ if want(experiment.ParamAuthority) {
+ authority, err := lab.AuthorityURL(ctx, reader)
+ if err != nil {
+ return rangeAnswers{}, err
+ }
+ answers.Authority = authority
+ }
+
+ return answers, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
new file mode 100644
index 00000000..9f03c870
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
@@ -0,0 +1,616 @@
+// ©AngelaMos | 2026
+// experiment_test.go
+
+package cli
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/spf13/cobra"
+ "github.com/spf13/pflag"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/rangelab"
+)
+
+const bogusExperiment = "network.no_such_experiment"
+
+var testAnswers = rangeAnswers{
+ Token: "a-token",
+ Revocable: "a-revocable-token",
+ Authority: "http://127.0.0.1:30009",
+ ProxyListen: "10.0.0.1:30003",
+ ProxyUpstream: "172.20.0.5:8080",
+}
+
+// refuser fails the test if khaos goes looking for a range it was never meant to
+// need. A reader that merely returns nothing cannot tell "khaos did not look"
+// from "khaos looked and found none", and those are different behaviours.
+type refuser struct{ t *testing.T }
+
+func (r refuser) Targets(context.Context, dockerx.Filter) ([]dockerx.Target, error) {
+ r.t.Fatal("khaos went looking for a range although the operator supplied every parameter " +
+ "a range could have answered; an operator pointed at their own stack must not need one running")
+ return nil, nil
+}
+
+func (r refuser) PublishedPort(context.Context, string, int) (int, error) {
+ r.t.Fatal("khaos read a published port although it needed nothing from a range")
+ return 0, nil
+}
+
+func (r refuser) ContainerEnv(context.Context, string) (map[string]string, error) {
+ r.t.Fatal("khaos read a container environment although it needed nothing from a range")
+ return nil, nil
+}
+
+func (r refuser) NetworkAddress(context.Context, string, string) (string, error) {
+ r.t.Fatal("khaos read a network address although it needed nothing from a range")
+ return "", nil
+}
+
+func (r refuser) BridgeGateway(context.Context) (string, error) {
+ r.t.Fatal("khaos read the bridge gateway although it needed nothing from a range")
+ return "", nil
+}
+
+// absent is a daemon with no range on it at all.
+type absent struct{}
+
+func (absent) Targets(context.Context, dockerx.Filter) ([]dockerx.Target, error) {
+ return nil, nil
+}
+
+func (absent) PublishedPort(context.Context, string, int) (int, error) {
+ return 0, nil
+}
+
+func (absent) ContainerEnv(context.Context, string) (map[string]string, error) {
+ return nil, nil
+}
+
+func (absent) NetworkAddress(context.Context, string, string) (string, error) {
+ return "", nil
+}
+
+func (absent) BridgeGateway(context.Context) (string, error) {
+ return "", nil
+}
+
+func outage(t *testing.T) experiment.Definition {
+ t.Helper()
+
+ definition, err := experiment.Lookup(experiment.NameDependencyOutage)
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ return definition
+}
+
+// The key set has to be knowable without starting anything, because it is what
+// decides whether khaos looks for a range at all. Keeping it in a second place
+// is only safe if the two cannot drift.
+func TestRangeSuppliesIsExactlyTheKeySetOfRangeDefaults(t *testing.T) {
+ for _, at := range allTiers() {
+ t.Run(tierName(at), func(t *testing.T) {
+ assertKeySet(t, rangeDefaults(fullLab(), testAnswers, at))
+ })
+ }
+}
+
+// allTiers is every tier the range serves. A test that iterated only the two it
+// knew about would go on passing while a whole gateway's answers drifted.
+func allTiers() []tier {
+ return []tier{tierDirect, tierProxy, tierIdP}
+}
+
+func tierName(at tier) string {
+ switch at {
+ case tierProxy:
+ return "through the proxy"
+ case tierIdP:
+ return "against khaos's own provider"
+ default:
+ return "direct"
+ }
+}
+
+// fullLab is a range with every gateway present. Each field has to be distinct
+// and non-zero or the assertions below cannot tell the gateways apart.
+func fullLab() rangelab.Lab {
+ return rangelab.Lab{
+ Port: 30001,
+ ProxiedPort: 30002,
+ ProxyPort: 30003,
+ VerifierPort: 30004,
+ IdPPort: 30005,
+ Issuer: "http://host.docker.internal:30005",
+ }
+}
+
+func assertKeySet(t *testing.T, defaults map[string]string) {
+ t.Helper()
+
+ supplies := slices.Clone(rangeSupplies())
+ slices.Sort(supplies)
+
+ keys := make([]string, 0, len(defaults))
+ for name, value := range defaults {
+ if value == "" {
+ t.Fatalf("the range claims to answer %q but answers it with an empty string", name)
+ }
+ keys = append(keys, name)
+ }
+ slices.Sort(keys)
+
+ if !slices.Equal(supplies, keys) {
+ t.Fatalf("rangeSupplies() = %v but rangeDefaults answers %v; khaos would either refuse a "+
+ "parameter it can fill or try to fill one it cannot", supplies, keys)
+ }
+}
+
+// The M4 defect, generalised. Fusing the operator's target with the range's URL
+// pauses a container they named and measures a service they did not.
+// Per EXPERIMENT, not per range. The refusal must name the parameters this
+// experiment declares and the range could have answered, and nothing else:
+// demanding a flag the experiment does not take would send the operator looking
+// for something that does not exist.
+func TestAPartlySuppliedInvocationIsRefusedRatherThanFused(t *testing.T) {
+ definition := outage(t)
+
+ answerable := answerableFor(definition)
+
+ if len(answerable) < 2 {
+ t.Fatalf("%s declares %d range-answerable parameters; with fewer than two there is no "+
+ "mixing to refuse and this test cannot fail", definition.Name, len(answerable))
+ }
+
+ for _, given := range answerable {
+ t.Run(given, func(t *testing.T) {
+ // absent, not refuser: gutting the mixing check must fail this test on
+ // the MESSAGE, not on a fake that trips the moment khaos looks for a
+ // range. A refuser here would alibi the assertions below.
+ _, err := fillFromRange(context.Background(), io.Discard, absent{},
+ definition, map[string]string{given: "something"})
+
+ if err == nil {
+ t.Fatalf("supplying only --%s was silently completed from the range", given)
+ }
+
+ if !strings.Contains(err.Error(), "will not fill one half") {
+ t.Fatalf("the refusal did not name the cause: %v", err)
+ }
+
+ if !strings.Contains(err.Error(), flagPrefix+given) {
+ t.Fatalf("the refusal did not name what was given: %v", err)
+ }
+
+ for _, other := range answerable {
+ if other == given {
+ continue
+ }
+ if !strings.Contains(err.Error(), flagPrefix+other) {
+ t.Fatalf("the refusal did not name the missing --%s, so the operator "+
+ "cannot tell what to add: %v", other, err)
+ }
+ }
+ })
+ }
+}
+
+// Supplying everything a range could answer means the operator is pointing khaos
+// at their own stack. khaos must not require a range to be running for that, and
+// must not overwrite a single value they gave.
+func TestAFullySuppliedInvocationNeverTouchesTheRange(t *testing.T) {
+ definition := outage(t)
+
+ supplied := map[string]string{}
+ for _, name := range answerableFor(definition) {
+ supplied[name] = "operator-" + name
+ }
+
+ filled, err := fillFromRange(context.Background(), io.Discard, refuser{t: t}, definition, supplied)
+ if err != nil {
+ t.Fatalf("fill: %v", err)
+ }
+
+ for name, want := range supplied {
+ if filled[name] != want {
+ t.Fatalf("khaos replaced the operator's --%s %q with %q", name, want, filled[name])
+ }
+ }
+}
+
+func TestAnInvocationWithNoFlagsAtAllLooksForARange(t *testing.T) {
+ definition := outage(t)
+
+ _, err := fillFromRange(context.Background(), io.Discard, absent{}, definition, map[string]string{})
+
+ if !errors.Is(err, rangelab.ErrNoRange) {
+ t.Fatalf("err = %v, want ErrNoRange; with nothing supplied khaos must go looking for a "+
+ "range, and say so when there is none", err)
+ }
+
+ // The fix has to be the whole fix. Naming a subset sends the operator straight
+ // into the half-specified refusal instead.
+ for _, name := range answerableFor(definition) {
+ if !strings.Contains(err.Error(), flagPrefix+name) {
+ t.Fatalf("the refusal does not name --%s, so following it lands the operator in the "+
+ "half-specified refusal: %v", name, err)
+ }
+ }
+}
+
+// A parameter an experiment does not declare must not be settable on it, and one
+// it does declare must be. Registering the union of every experiment's parameters
+// on one command is what this forbids.
+func TestEveryExperimentRegistersExactlyItsDeclaredParameters(t *testing.T) {
+ for _, verb := range []string{"run", "plan"} {
+ t.Run(verb, func(t *testing.T) {
+ parent := find(t, NewRoot(), verb)
+
+ subcommands := parent.Commands()
+ if len(subcommands) != len(experiment.Catalog()) {
+ t.Fatalf("%s has %d subcommands for %d experiments",
+ verb, len(subcommands), len(experiment.Catalog()))
+ }
+
+ for _, cmd := range subcommands {
+ definition, err := experiment.Lookup(cmd.Name())
+ if err != nil {
+ t.Fatalf("%s %s is not an experiment: %v", verb, cmd.Name(), err)
+ }
+
+ registered := stringFlags(cmd)
+ want := append([]string{flagProject}, names(definition.Params)...)
+ slices.Sort(registered)
+ slices.Sort(want)
+
+ if !slices.Equal(registered, want) {
+ t.Fatalf("%s %s registers %v, want %v", verb, cmd.Name(), registered, want)
+ }
+
+ for _, param := range definition.Params {
+ flag := cmd.Flags().Lookup(param.Name)
+ if !strings.Contains(flag.Usage, param.Usage) {
+ t.Fatalf("--%s on %s does not carry the usage the experiment declared",
+ param.Name, cmd.Name())
+ }
+ if flag.DefValue != "" {
+ t.Fatalf("--%s on %s registers a non-empty default %q, so khaos can no "+
+ "longer tell an operator who set it from one who did not",
+ param.Name, cmd.Name(), flag.DefValue)
+ }
+ }
+ }
+ })
+ }
+}
+
+func TestAFlagTheExperimentNeverDeclaredIsRejected(t *testing.T) {
+ root := NewRoot()
+ silence(root)
+ root.SetArgs([]string{"run", experiment.NameDependencyOutage, "--not-a-parameter-of-anything", "x"})
+
+ if err := root.Execute(); err == nil {
+ t.Fatal("an undeclared flag was accepted, so an operator cannot tell a typo from a parameter")
+ }
+}
+
+func TestAnUnknownExperimentNamesTheOnesThatExist(t *testing.T) {
+ for _, verb := range []string{"run", "plan"} {
+ t.Run(verb, func(t *testing.T) {
+ root := NewRoot()
+ out := &strings.Builder{}
+ root.SetOut(out)
+ root.SetErr(out)
+ root.SetArgs([]string{verb, bogusExperiment})
+
+ err := root.Execute()
+ if err == nil {
+ t.Fatalf("%s accepted %q as an experiment", verb, bogusExperiment)
+ }
+
+ if !errors.Is(err, experiment.ErrUnknownExperiment) {
+ t.Fatalf("err = %v, want ErrUnknownExperiment", err)
+ }
+
+ if !strings.Contains(err.Error(), experiment.NameDependencyOutage) {
+ t.Fatalf("the refusal did not name an experiment that does exist: %v", err)
+ }
+ })
+ }
+}
+
+// A typo'd experiment carrying flags must still report the experiment, not the
+// flag. The flag is only unknown because the experiment is.
+func TestAnUnknownExperimentIsReportedAheadOfItsFlags(t *testing.T) {
+ root := NewRoot()
+ silence(root)
+ root.SetArgs([]string{"run", bogusExperiment, "--" + experiment.ParamURL, "http://127.0.0.1:1"})
+
+ err := root.Execute()
+ if !errors.Is(err, experiment.ErrUnknownExperiment) {
+ t.Fatalf("err = %v, want ErrUnknownExperiment; reporting the unknown flag instead buries "+
+ "the only thing actually wrong with the line", err)
+ }
+}
+
+func TestNamingNoExperimentAtAllListsThem(t *testing.T) {
+ root := NewRoot()
+ silence(root)
+ root.SetArgs([]string{"run"})
+
+ err := root.Execute()
+ if err == nil {
+ t.Fatal("`khaos run` with no experiment exited cleanly without running anything")
+ }
+
+ if !strings.Contains(err.Error(), experiment.NameDependencyOutage) {
+ t.Fatalf("the refusal did not list the experiments: %v", err)
+ }
+}
+
+func find(t *testing.T, root *cobra.Command, name string) *cobra.Command {
+ t.Helper()
+
+ for _, cmd := range root.Commands() {
+ if cmd.Name() == name {
+ return cmd
+ }
+ }
+
+ t.Fatalf("no %q command is registered", name)
+ return nil
+}
+
+func stringFlags(cmd *cobra.Command) []string {
+ out := []string{}
+ cmd.Flags().VisitAll(func(flag *pflag.Flag) {
+ if flag.Value.Type() == "string" {
+ out = append(out, flag.Name)
+ }
+ })
+ return out
+}
+
+func names(params []experiment.Param) []string {
+ out := make([]string, 0, len(params))
+ for _, param := range params {
+ out = append(out, param.Name)
+ }
+ return out
+}
+
+// present is a daemon with a running range on it. Without it nothing executed
+// fillFromRange's SUCCESS path at all: refuser fatals the moment khaos looks for
+// a range and absent returns nothing, so Discover never succeeded in any test and
+// the values the range answers with were unpinned. Deleting the fillFromRange call
+// from runPlan entirely, and answering --url with the gateway root instead of
+// /protected, both left the Go suite green.
+type present struct {
+ gateway *httptest.Server
+ port int
+}
+
+func (p present) Targets(context.Context, dockerx.Filter) ([]dockerx.Target, error) {
+ return []dockerx.Target{{
+ ID: "range-gateway",
+ Name: rangelab.GatewayName,
+ Project: rangelab.ProjectName,
+ Running: true,
+ Inspected: true,
+ }}, nil
+}
+
+func (p present) PublishedPort(context.Context, string, int) (int, error) {
+ return p.port, nil
+}
+
+func (p present) ContainerEnv(context.Context, string) (map[string]string, error) {
+ return map[string]string{}, nil
+}
+
+func (p present) NetworkAddress(context.Context, string, string) (string, error) {
+ return "127.0.0.1", nil
+}
+
+func (p present) BridgeGateway(context.Context) (string, error) {
+ return "127.0.0.1", nil
+}
+
+func rangeOnLoopback(t *testing.T) present {
+ t.Helper()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/token" {
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = io.WriteString(w, `{"token":"`+rangeIssuedToken+`"}`)
+ }))
+ t.Cleanup(server.Close)
+
+ port, err := strconv.Atoi(strings.TrimPrefix(server.URL, "http://127.0.0.1:"))
+ if err != nil {
+ t.Fatalf("the test server is not on loopback: %v", err)
+ }
+
+ return present{gateway: server, port: port}
+}
+
+const rangeIssuedToken = "a-credential-the-range-issued"
+
+func TestARunningRangeAnswersEveryParameterItClaimsTo(t *testing.T) {
+ reader := rangeOnLoopback(t)
+ definition := outage(t)
+
+ filled, err := fillFromRange(context.Background(), io.Discard, reader, definition, map[string]string{})
+ if err != nil {
+ t.Fatalf("fill: %v", err)
+ }
+
+ for _, name := range answerableFor(definition) {
+ if filled[name] == "" {
+ t.Fatalf("the range claims to answer --%s and left it empty", name)
+ }
+ }
+
+ // The VALUES matter, not just the keys. Answering --url with the gateway root
+ // instead of the protected endpoint probes a path that needs no credential, so
+ // the baseline never observes enforcement and every verdict is baseline-not-met.
+ if filled[experiment.ParamURL] != reader.gateway.URL+"/protected" {
+ t.Fatalf("--url answered as %q, want the protected endpoint; probing the gateway root "+
+ "measures a path that needs no credential", filled[experiment.ParamURL])
+ }
+
+ // The target is the DEPENDENCY, not the gateway. Pausing the gateway pauses the
+ // thing being measured rather than the thing being taken away.
+ if filled[experiment.ParamTarget] != rangelab.IDPName {
+ t.Fatalf("--target answered as %q, want the identity provider %q",
+ filled[experiment.ParamTarget], rangelab.IDPName)
+ }
+
+ if filled[experiment.ParamToken] != rangeIssuedToken {
+ t.Fatalf("--token was not the credential the range issued, got %q",
+ filled[experiment.ParamToken])
+ }
+}
+
+// The note is the only thing telling an operator which stack khaos just chose for
+// them, and it must never print the credential.
+func TestTheRangeNoteNamesWhatWasChosenAndNeverTheCredential(t *testing.T) {
+ out := &strings.Builder{}
+
+ if _, err := fillFromRange(context.Background(), out, rangeOnLoopback(t), outage(t),
+ map[string]string{}); err != nil {
+ t.Fatalf("fill: %v", err)
+ }
+
+ printed := out.String()
+
+ if !strings.Contains(printed, "using the running range") {
+ t.Fatalf("khaos filled an invocation from the range without saying so: %q", printed)
+ }
+
+ if !strings.Contains(printed, rangelab.IDPName) {
+ t.Fatalf("the note does not name the target it chose: %q", printed)
+ }
+
+ if strings.Contains(printed, rangeIssuedToken) {
+ t.Fatalf("the note printed the credential: %q", printed)
+ }
+}
+
+// Required bits on the shipped catalogue were asserted by nothing. Every CLI path
+// reaches Build through fillFromRange, so if the range cannot answer an
+// experiment's required parameters the zero-flag commitment is broken for it.
+func TestTheRangeCanSatisfyEveryExperimentsRequiredParameters(t *testing.T) {
+ lab := fullLab()
+
+ for _, definition := range experiment.Catalog() {
+ t.Run(definition.Name, func(t *testing.T) {
+ defaults := rangeDefaults(lab, testAnswers, tierOf(definition.FaultKind))
+
+ supplied := map[string]string{}
+ for _, param := range definition.Params {
+ if value, answerable := defaults[param.Name]; answerable {
+ supplied[param.Name] = value
+ }
+ }
+
+ if _, err := experiment.Build(definition.Name, supplied, experiment.Timing{}); err != nil {
+ t.Fatalf("the range cannot satisfy %s, so it cannot run with zero flags: %v",
+ definition.Name, err)
+ }
+ })
+ }
+}
+
+// The range carries THREE gateways and they are not interchangeable. Each one
+// exists so one injection tier is measured on a path nothing else contaminates:
+// a proxy-tier experiment arms a fault on the path only the proxied gateway
+// uses, and an identity-tier one arms a provider only the verifier gateway
+// resolves keys from. Pointing either at the wrong gateway measures a service
+// with no causal connection to the fault and reports a clean survival over an
+// armed fault.
+func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(t *testing.T) {
+ lab := fullLab()
+
+ want := map[tier]string{
+ tierDirect: lab.ProtectedURL(),
+ tierProxy: lab.ProxiedProtectedURL(),
+ tierIdP: lab.VerifierProtectedURL(),
+ }
+
+ seen := map[string]tier{}
+ for at, url := range want {
+ if other, clash := seen[url]; clash {
+ t.Fatalf("%s and %s resolve to the same URL, so this test cannot tell them apart",
+ tierName(at), tierName(other))
+ }
+ seen[url] = at
+ }
+
+ exercised := map[tier]bool{}
+
+ for _, definition := range experiment.Catalog() {
+ t.Run(definition.Name, func(t *testing.T) {
+ at := tierOf(definition.FaultKind)
+ exercised[at] = true
+
+ defaults := rangeDefaults(lab, testAnswers, at)
+
+ if answered := defaults[experiment.ParamURL]; answered != want[at] {
+ t.Fatalf("%s injects %s and is probed at %q, want %q",
+ definition.Name, definition.FaultKind, answered, want[at])
+ }
+
+ // The identity tier's target is the gateway rather than the
+ // dependency, because that is the container khaos needs consent for
+ // and the one it actually measures.
+ wantTarget := rangelab.IDPName
+ if at == tierIdP {
+ wantTarget = rangelab.VerifierGatewayName
+ }
+
+ if answered := defaults[experiment.ParamTarget]; answered != wantTarget {
+ t.Fatalf("%s asks consent for %q, want %q",
+ definition.Name, answered, wantTarget)
+ }
+ })
+ }
+
+ for _, at := range allTiers() {
+ if !exercised[at] {
+ t.Errorf("the catalogue exercises no experiment %s, so this test cannot fail for that tier",
+ tierName(at))
+ }
+ }
+}
+
+// answerableFor is the intersection khaos actually acts on: the parameters this
+// experiment declares AND the range can answer. Iterating the global list
+// instead made these tests demand flags the experiment under test does not take.
+func answerableFor(definition experiment.Definition) []string {
+ out := []string{}
+
+ for _, param := range definition.Params {
+ if slices.Contains(rangeSupplies(), param.Name) {
+ out = append(out, param.Name)
+ }
+ }
+
+ return out
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/inject.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/inject.go
new file mode 100644
index 00000000..cc9f8945
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/inject.go
@@ -0,0 +1,274 @@
+// ©AngelaMos | 2026
+// inject.go
+
+package cli
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+ "github.com/CarterPerez-dev/khaos/internal/lease"
+ "github.com/CarterPerez-dev/khaos/internal/run"
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+ "github.com/CarterPerez-dev/khaos/internal/workspace"
+)
+
+const (
+ flagTTL = "ttl"
+ flagHold = "hold"
+ flagTimeout = "revert-timeout"
+
+ injectShort = "apply a single fault to one target and hold it until you stop"
+
+ ttlUsage = "how long the fault may outlive khaos before `khaos reap` or the next khaos invocation reclaims it; it does NOT limit how long the fault is held while khaos is alive"
+ holdUsage = "hold the fault for this long and then revert, instead of holding until you interrupt"
+ timeoutUsage = "give up waiting for the daemon during a revert after this long"
+
+ markInject = "->"
+ markRevert = "<-"
+ markOK = "ok"
+ markError = "!!"
+ markPlan = "plan"
+
+ defaultRevertTimeout = 60 * time.Second
+ holdDisabled = time.Duration(0)
+
+ msgInjected = "fault applied, holding. press ctrl-c to revert"
+ msgReverting = "reverting"
+ msgReverted = "reverted, target restored"
+ msgHoldOver = "hold elapsed"
+ msgInterrupt = "interrupted, reverting now (do not kill khaos, the revert is running)"
+ msgClamped = "--ttl was raised to %s, the shortest lease khaos will issue"
+ msgLeaseInfo = "run %s, lease %s, expires %s"
+
+ errNotDirect = "%s acts on the path between a target and its dependency rather than on a container, and it exists only for as long as the run that owns khaos's proxy, so `khaos inject` cannot apply it on its own; run it as an experiment with `khaos run`, or pick one of: %s"
+)
+
+func newInjectCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "inject <kind> <target>",
+ Short: injectShort,
+ Long: injectLong(),
+ Example: " khaos inject " + inject.KindPause + " shop-api-1\n" +
+ " khaos inject " + inject.KindDisconnect + " shop-api-1 --hold 30s",
+ Args: cobra.MatchAll(cobra.ExactArgs(2), onlyKnownKind),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ ttl, err := cmd.Flags().GetDuration(flagTTL)
+ if err != nil {
+ return err
+ }
+ hold, err := cmd.Flags().GetDuration(flagHold)
+ if err != nil {
+ return err
+ }
+ timeout, err := cmd.Flags().GetDuration(flagTimeout)
+ if err != nil {
+ return err
+ }
+ project, err := cmd.Flags().GetString(flagProject)
+ if err != nil {
+ return err
+ }
+ return runInject(cmd.Context(), cmd.OutOrStdout(),
+ args[0], args[1], project, ttl, hold, timeout)
+ },
+ ValidArgsFunction: completeKinds,
+ }
+
+ cmd.Flags().Duration(flagTTL, lease.DefaultTTL, ttlUsage)
+ cmd.Flags().Duration(flagHold, holdDisabled, holdUsage)
+ cmd.Flags().Duration(flagTimeout, defaultRevertTimeout, timeoutUsage)
+ cmd.Flags().String(flagProject, "", projectUsage)
+
+ return cmd
+}
+
+func injectLong() string {
+ return `inject applies one raw fault to one consenting target, holds it, and reverts it
+when you stop the command.
+
+Available kinds: ` + strings.Join(inject.DirectKinds(), ", ") + `
+
+The fault carries a lease. Everything the revert will need is written to the
+journal BEFORE the fault is applied, so a khaos killed outright still leaves a
+record that fully describes how to put the target back. The next khaos
+invocation reverts it, and ` + "`khaos reap`" + ` reverts it with no further
+invocation at all.`
+}
+
+func onlyKnownKind(_ *cobra.Command, args []string) error {
+ if len(args) == 0 {
+ return nil
+ }
+
+ if _, err := inject.Lookup(args[0]); err != nil {
+ return err
+ }
+
+ // A registered kind that `inject` cannot apply alone has to be refused by
+ // name rather than attempted. It would otherwise fail deep inside the
+ // mutation with a message about a proxy the operator never asked for.
+ if !inject.IsDirect(args[0]) {
+ return fmt.Errorf(errNotDirect, args[0], strings.Join(inject.DirectKinds(), ", "))
+ }
+
+ return nil
+}
+
+func completeKinds(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {
+ if len(args) > 0 {
+ return nil, cobra.ShellCompDirectiveNoFileComp
+ }
+ return inject.DirectKinds(), cobra.ShellCompDirectiveNoFileComp
+}
+
+func runInject(
+ ctx context.Context,
+ out io.Writer,
+ kind, name, project string,
+ ttl, hold, timeout time.Duration,
+) error {
+ work, err := workspace.OpenExclusive(ctx)
+ if err != nil {
+ return err
+ }
+ defer work.Close()
+
+ mutator, err := inject.Lookup(kind)
+ if err != nil {
+ return err
+ }
+
+ filter := dockerx.Filter{Project: project}
+ guard := dockerx.Guard{Project: project, SelfID: dockerx.SelfContainerID()}
+
+ planned, err := work.Docker.Accept(ctx, name, filter, guard)
+ if err != nil {
+ return err
+ }
+
+ fmt.Fprintln(out)
+ event(out, markPlan, theme.TextFaint, mutator.Describe(work.Env(), planned))
+
+ guard.ExpectedID = planned.ID
+ confirmed, err := work.Docker.Accept(ctx, name, filter, guard)
+ if err != nil {
+ return err
+ }
+
+ session := run.NewSession(work.Env(), work.Journal, ttl)
+
+ if session.Lease().TTL() != ttl {
+ note(out, fmt.Sprintf(msgClamped, session.Lease().TTL()))
+ }
+
+ if err := session.Inject(ctx, kind, confirmed); err != nil {
+ return errors.Join(err, revert(withoutCancel(ctx, timeout), out, session))
+ }
+
+ event(out, markInject, theme.StatusWarn, subject(kind, confirmed.Name))
+ note(out, fmt.Sprintf(msgLeaseInfo,
+ session.RunID(), session.Lease().ID(), session.Lease().ExpiresAt().Format(time.RFC3339)))
+ say(out, msgInjected)
+
+ waitErr := hold_(ctx, session, hold)
+
+ switch {
+ case errors.Is(waitErr, context.Canceled):
+ fmt.Fprintln(out)
+ say(out, msgInterrupt)
+ case waitErr != nil:
+ fmt.Fprintln(out)
+ event(out, markError, theme.StatusError, waitErr.Error())
+ default:
+ say(out, msgHoldOver)
+ }
+
+ revertErr := revert(withoutCancel(ctx, timeout), out, session)
+
+ if errors.Is(waitErr, context.Canceled) {
+ return errors.Join(waitErr, revertErr)
+ }
+
+ return errors.Join(skipCancellation(waitErr), revertErr)
+}
+
+func hold_(ctx context.Context, session *run.Session, hold time.Duration) error {
+ keepCtx, stop := context.WithCancel(ctx)
+ renewals := make(chan error, 1)
+
+ go func() {
+ renewals <- session.Lease().Keep(keepCtx, func(time.Time) error { return session.Renew() })
+ }()
+
+ defer func() {
+ stop()
+ <-renewals
+ }()
+
+ var timer <-chan time.Time
+ if hold > holdDisabled {
+ t := time.NewTimer(hold)
+ defer t.Stop()
+ timer = t.C
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case err := <-renewals:
+ renewals <- err
+ return err
+ case <-timer:
+ return nil
+ }
+}
+
+func withoutCancel(ctx context.Context, timeout time.Duration) context.Context {
+ detached := context.WithoutCancel(ctx)
+ if timeout <= 0 {
+ return detached
+ }
+ bounded, cancel := context.WithTimeout(detached, timeout)
+ context.AfterFunc(bounded, cancel)
+ return bounded
+}
+
+func skipCancellation(err error) error {
+ if errors.Is(err, context.Canceled) {
+ return nil
+ }
+ return err
+}
+
+func revert(ctx context.Context, out io.Writer, session *run.Session) error {
+ say(out, msgReverting)
+
+ err := session.RevertAll(ctx, func(record journal.Record, failure error) {
+ if failure != nil {
+ event(out, markError, theme.StatusError,
+ fmt.Sprintf(faultFormat, record.Kind, record.TargetName, failure))
+ return
+ }
+ event(out, markRevert, theme.StatusOK, subject(record.Kind, record.TargetName))
+ })
+
+ if err != nil {
+ fmt.Fprintln(out)
+ return err
+ }
+
+ event(out, markOK, theme.StatusOK, msgReverted)
+ fmt.Fprintln(out)
+
+ return nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/print.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/print.go
new file mode 100644
index 00000000..70e201ed
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/print.go
@@ -0,0 +1,45 @@
+// ©AngelaMos | 2026
+// print.go
+
+package cli
+
+import (
+ "fmt"
+ "io"
+
+ "github.com/CarterPerez-dev/khaos/internal/render"
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+)
+
+const (
+ lineLayout = "%s%s\n"
+ eventLayout = "%s%s %s\n"
+
+ subjectFormat = "%s on %s"
+ faultFormat = "%s on %s: %v"
+ leaseFormat = "%s on %s (run %s, lease %s)"
+)
+
+func say(out io.Writer, text string) {
+ fmt.Fprintf(out, lineLayout, render.Indent, theme.Fg(theme.TextNormal, text))
+}
+
+func note(out io.Writer, text string) {
+ fmt.Fprintf(out, lineLayout, render.Indent, theme.Fg(theme.TextDim, text))
+}
+
+func event(out io.Writer, mark string, markColor theme.Color, text string) {
+ fmt.Fprintf(out, eventLayout, render.Indent,
+ theme.Fg(markColor, mark), theme.Fg(theme.TextNormal, text))
+}
+
+func fixLine(out io.Writer, fix string) {
+ if fix == "" {
+ return
+ }
+ fmt.Fprintf(out, lineLayout, render.FixIndent, theme.Fg(theme.TextBright, render.FixPrefix+fix))
+}
+
+func subject(kind, target string) string {
+ return fmt.Sprintf(subjectFormat, kind, target)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
new file mode 100644
index 00000000..67d4bca3
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
@@ -0,0 +1,321 @@
+// ©AngelaMos | 2026
+// rangecmd.go
+
+package cli
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "slices"
+ "strings"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/rangelab"
+ "github.com/CarterPerez-dev/khaos/internal/render"
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+ "github.com/CarterPerez-dev/khaos/internal/verifier"
+ "github.com/CarterPerez-dev/khaos/internal/workspace"
+)
+
+var ErrUnknownFailMode = errors.New("the range has no such fail mode")
+
+const (
+ flagFailMode = "fail-mode"
+ flagCacheTTL = "cache-ttl"
+ flagPort = "port"
+ flagQuiet = "quiet"
+
+ rangeShort = "bring up a target stack that is deliberately breakable"
+ rangeLong = `The range is a small stack khaos ships so you have something to break before you
+point it at anything you care about.
+
+It is a gateway in front of an identity provider. Pause the provider and the
+gateway has to decide what to do without it. In ` + "`--fail-mode closed`" + ` it refuses,
+which is correct. In ` + "`--fail-mode open`" + ` it serves anyway, which is the defect
+khaos exists to find, modelled on the Okta delegated-authentication advisory
+where a cached decision was reachable only while the agent was unreachable.
+
+Both containers carry the consent label, and the gateway's host port is a
+random high one chosen at startup, so it never collides with anything already
+running.`
+
+ upShort = "build and start the range, then wait until it answers"
+ downShort = "stop the range and remove everything it created"
+ statusShort = "show whether the range is running and where to reach it"
+
+ failModeUsage = "which bug the range carries. The introspecting gateways understand closed, open and stale-rearm; khaos's verifier gateway understands closed plus one mode per verifier defect, and `khaos range up --fail-mode ?` lists them all. An unrecognised value is refused rather than quietly leaving the range correct"
+ cacheTTLUsage = "how long the gateway caches an authorization decision"
+ portUsage = "host port for the gateway; the default is a random high port chosen at startup"
+ quietUsage = "suppress compose build and start output"
+
+ msgStarting = "building and starting the range"
+ msgPortRetry = "host port %d was taken between the check and the bind; picking another"
+ errPortBound = "could not start the range on host port %d: %w"
+ portAttempts = 3
+ msgRangeUp = "range is up"
+ msgRangeDown = "range is down"
+ msgRangeStopped = "the range is not running; start it with `khaos range up`"
+ msgFailMode = "fail mode %s"
+ msgTokenHint = "a credential is issued on demand, so `khaos run` against the range needs no flags"
+ msgPortHint = "host port %d was chosen at random; pass --port to pin it"
+ errUnknownFailMode = "%w: %q (known modes: %s)"
+
+ msgTierDirect = "docker-native tier %s"
+ msgTierProxied = "proxy tier %s, with khaos's fault proxy expected on host port %d"
+ msgTierVerifier = "identity tier %s, trusting khaos's provider as %s"
+)
+
+func newRangeCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "range",
+ Short: rangeShort,
+ Long: rangeLong,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ return cmd.Help()
+ },
+ }
+
+ cmd.AddCommand(newRangeUpCmd(), newRangeDownCmd(), newRangeStatusCmd())
+
+ return cmd
+}
+
+func newRangeUpCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "up",
+ Short: upShort,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ failMode, err := cmd.Flags().GetString(flagFailMode)
+ if err != nil {
+ return err
+ }
+ cacheTTL, err := cmd.Flags().GetString(flagCacheTTL)
+ if err != nil {
+ return err
+ }
+ port, err := cmd.Flags().GetInt(flagPort)
+ if err != nil {
+ return err
+ }
+ quiet, err := cmd.Flags().GetBool(flagQuiet)
+ if err != nil {
+ return err
+ }
+ return runRangeUp(cmd.Context(), cmd.OutOrStdout(), failMode, cacheTTL, port, quiet)
+ },
+ }
+
+ cmd.Flags().String(flagFailMode, rangelab.FailModeShut, failModeUsage)
+ cmd.Flags().String(flagCacheTTL, "2s", cacheTTLUsage)
+ cmd.Flags().Int(flagPort, 0, portUsage)
+ cmd.Flags().Bool(flagQuiet, false, quietUsage)
+
+ return cmd
+}
+
+func newRangeDownCmd() *cobra.Command {
+ return &cobra.Command{
+ Use: "down",
+ Short: downShort,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ lab, err := rangelab.Locate(rangelab.Ports{})
+ if err != nil {
+ return err
+ }
+
+ work, err := workspace.OpenExclusive(cmd.Context())
+ if err != nil {
+ return err
+ }
+ defer work.Close()
+
+ if err := lab.Down(cmd.Context(), io.Discard); err != nil {
+ return err
+ }
+
+ out := cmd.OutOrStdout()
+ fmt.Fprintln(out)
+ say(out, msgRangeDown)
+ fmt.Fprintln(out)
+
+ return nil
+ },
+ }
+}
+
+func newRangeStatusCmd() *cobra.Command {
+ return &cobra.Command{
+ Use: "status",
+ Short: statusShort,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ return runRangeStatus(cmd.Context(), cmd.OutOrStdout())
+ },
+ }
+}
+
+// knownFailModes is every value the range answers to, across both kinds of
+// gateway. The introspecting gateways understand three and khaos's verifier
+// gateway understands ten, and an unrecognised value leaves EVERY gateway
+// behaving correctly.
+//
+// That makes a typo the worst possible outcome: the range comes up, nothing is
+// broken, and all eighteen identity experiments report a clean survival against
+// a stack that was supposed to be carrying the bug. Refusing by name is the only
+// safe behaviour, and the refusal lists what is available because nobody is
+// going to remember ten mode names.
+func knownFailModes() []string {
+ modes := []string{rangelab.FailModeShut, rangelab.FailModeOpen, rangelab.FailModeStaleRearm}
+
+ for _, mode := range verifier.Modes() {
+ if !slices.Contains(modes, mode) {
+ modes = append(modes, mode)
+ }
+ }
+
+ return modes
+}
+
+func runRangeUp(ctx context.Context, out io.Writer, failMode, cacheTTL string, port int, quiet bool) error {
+ if !slices.Contains(knownFailModes(), failMode) {
+ return fmt.Errorf(errUnknownFailMode, ErrUnknownFailMode, failMode,
+ strings.Join(knownFailModes(), listSep))
+ }
+
+ work, err := workspace.OpenExclusive(ctx)
+ if err != nil {
+ return err
+ }
+ defer work.Close()
+
+ progress := io.Writer(os.Stderr)
+ if quiet {
+ progress = io.Discard
+ }
+
+ fmt.Fprintln(out)
+ say(out, msgStarting)
+
+ pinned := port != 0
+ var lab rangelab.Lab
+
+ for attempt := range portAttempts {
+ // Only the direct gateway's port is pinnable. The other two are khaos's
+ // own plumbing: the second gateway's published port and the port that
+ // gateway expects khaos's fault proxy on. Letting an operator pin those
+ // would be a footgun with no use case.
+ ports, err := rangelab.FreePorts()
+ if err != nil {
+ return err
+ }
+
+ if pinned {
+ ports.Gateway = port
+ }
+ port = ports.Gateway
+
+ lab, err = rangelab.Locate(ports)
+ if err != nil {
+ return err
+ }
+
+ err = lab.Up(ctx, progress, failMode, cacheTTL)
+ if err == nil {
+ break
+ }
+
+ if pinned || !rangelab.PortTaken(err) || attempt == portAttempts-1 {
+ return fmt.Errorf(errPortBound, port, err)
+ }
+
+ note(out, fmt.Sprintf(msgPortRetry, port))
+ }
+
+ event(out, markOK, theme.StatusOK, msgRangeUp)
+ note(out, fmt.Sprintf(msgFailMode, failMode))
+ note(out, fmt.Sprintf(msgPortHint, lab.Port))
+ note(out, msgTokenHint)
+ fmt.Fprintln(out)
+
+ // Printed from the containers rather than from what khaos just chose, which
+ // is the same discipline the proxy port is read back under: the container is
+ // the only authority on where it will actually dial and what issuer it will
+ // actually trust. Printing khaos's intent instead would hide exactly the
+ // mismatch this is most useful for.
+ describeEntryPoints(ctx, out, work.Docker)
+
+ return nil
+}
+
+func runRangeStatus(ctx context.Context, out io.Writer) error {
+ work, err := workspace.OpenShared(ctx)
+ if err != nil {
+ return err
+ }
+ defer work.Close()
+
+ targets, err := work.Docker.Targets(ctx, dockerx.Filter{Project: rangelab.ProjectName})
+ if err != nil {
+ return err
+ }
+
+ fmt.Fprintln(out)
+
+ if len(targets) == 0 {
+ note(out, msgRangeStopped)
+ fmt.Fprintln(out)
+ return nil
+ }
+
+ for _, target := range targets {
+ status := theme.Fg(theme.StatusOK, statusReady)
+ if refusal := dockerx.Refuse(target, dockerx.Guard{}); refusal != nil {
+ status = theme.Fg(theme.StatusWarn, refusal.Predicate)
+ }
+
+ fmt.Fprintf(out, rowLayout, render.Indent,
+ theme.Fg(theme.TextBright, render.Cell(target.Name, colName)),
+ theme.Fg(theme.TextNormal, render.Cell(target.Project, colProject)),
+ theme.Fg(theme.TextDim, render.Cell(target.Image, colImage)),
+ status)
+ }
+
+ fmt.Fprintln(out)
+
+ // The entry points, which the container listing cannot give. It shows three
+ // gateways with truncated names and says nothing about which one is reached
+ // how, so an operator could not find the second or third tier from khaos's
+ // own output at all. Each tier is probed at a different URL and the whole
+ // point of running the same experiment against two of them is being able to
+ // address both.
+ describeEntryPoints(ctx, out, work.Docker)
+
+ return nil
+}
+
+func describeEntryPoints(ctx context.Context, out io.Writer, reader rangelab.PortReader) {
+ lab, err := rangelab.Discover(ctx, reader)
+ if err != nil {
+ return
+ }
+
+ say(out, fmt.Sprintf(msgTierDirect, lab.ProtectedURL()))
+
+ if lab.ProxiedPort != 0 {
+ say(out, fmt.Sprintf(msgTierProxied, lab.ProxiedProtectedURL(), lab.ProxyPort))
+ }
+
+ if lab.VerifierPort != 0 {
+ say(out, fmt.Sprintf(msgTierVerifier, lab.VerifierProtectedURL(), lab.Issuer))
+ }
+
+ fmt.Fprintln(out)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go
new file mode 100644
index 00000000..1acd1fef
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go
@@ -0,0 +1,180 @@
+// ©AngelaMos | 2026
+// reclaim.go
+
+package cli
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/khaos/internal/run"
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+ "github.com/CarterPerez-dev/khaos/internal/workspace"
+)
+
+const (
+ flagExpiredOnly = "expired-only"
+ flagInterval = "interval"
+
+ reclaimShort = "revert any fault left behind by a khaos that did not finish"
+ reclaimLong = `reclaim replays the write-ahead journal and reverts every fault that was never
+reverted, whether khaos crashed, was killed outright, or lost its daemon.
+
+Every revert is idempotent, so running this when nothing is outstanding is
+safe and does nothing.
+
+Without --expired-only this takes the exclusive lock, so it will refuse to run
+while another khaos is holding a live fault rather than pull that fault out
+from under it.`
+
+ reapShort = "watch the journal and revert faults as their leases expire"
+ reapLong = `reap is the dead man's switch. It reverts a fault once its lease expires, so a
+fault outlives a dead khaos by at most the remaining lease rather than forever.
+
+It runs until you stop it, and it only ever touches faults whose lease has
+already expired, so it is safe to leave running beside an active khaos. You do
+not need it for an ordinary ` + "`khaos inject`" + `, which reverts its own fault when
+you stop it; reap is what covers the case where khaos never gets the chance.
+
+A fault it cannot revert is reported and retried on the next tick rather than
+stopping the watch, because one stuck record must never disable the switch for
+every other fault on the machine.`
+
+ expiredOnlyUsage = "only revert faults whose lease has already expired, and do not take the exclusive lock"
+ intervalUsage = "how often to check the journal for expired leases"
+
+ defaultReapInterval = 5 * time.Second
+
+ msgNothing = "nothing outstanding"
+ msgReclaimed = "reclaimed %d fault(s)"
+ msgWatching = "watching %s every %s, until you stop it"
+ msgStopped = "stopped watching, %d fault(s) reclaimed"
+ msgDamaged = "the journal has a damaged line; khaos acted on every record it could read, but cannot promise that is all of them"
+)
+
+func newReclaimCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "reclaim",
+ Short: reclaimShort,
+ Long: reclaimLong,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ expiredOnly, err := cmd.Flags().GetBool(flagExpiredOnly)
+ if err != nil {
+ return err
+ }
+ return runReclaim(cmd.Context(), cmd.OutOrStdout(), expiredOnly)
+ },
+ }
+
+ cmd.Flags().Bool(flagExpiredOnly, false, expiredOnlyUsage)
+
+ return cmd
+}
+
+func newReapCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "reap",
+ Short: reapShort,
+ Long: reapLong,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ interval, err := cmd.Flags().GetDuration(flagInterval)
+ if err != nil {
+ return err
+ }
+ return runReap(cmd.Context(), cmd.OutOrStdout(), interval)
+ },
+ }
+
+ cmd.Flags().Duration(flagInterval, defaultReapInterval, intervalUsage)
+
+ return cmd
+}
+
+func runReclaim(ctx context.Context, out io.Writer, expiredOnly bool) error {
+ work, err := openFor(ctx, expiredOnly)
+ if err != nil {
+ return err
+ }
+ defer work.Close()
+
+ fmt.Fprintln(out)
+ sweep := run.Reclaim(ctx, work.Env(), work.Journal, time.Now().UTC(), expiredOnly)
+ report(out, sweep)
+
+ if len(sweep.Results) == 0 {
+ note(out, msgNothing)
+ }
+ fmt.Fprintln(out)
+
+ if sweep.Failed() {
+ return run.ErrRevertFailed
+ }
+
+ return sweep.Damage
+}
+
+func runReap(ctx context.Context, out io.Writer, interval time.Duration) error {
+ work, err := workspace.OpenShared(ctx)
+ if err != nil {
+ return err
+ }
+ defer work.Close()
+
+ fmt.Fprintln(out)
+ note(out, fmt.Sprintf(msgWatching, work.Journal.Path(), interval))
+
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+
+ reclaimed := 0
+
+ for {
+ select {
+ case <-ctx.Done():
+ fmt.Fprintln(out)
+ note(out, fmt.Sprintf(msgStopped, reclaimed))
+ fmt.Fprintln(out)
+ return nil
+ case <-ticker.C:
+ sweep := run.Reclaim(ctx, work.Env(), work.Journal, time.Now().UTC(), true)
+ report(out, sweep)
+ reclaimed += len(sweep.Results)
+ }
+ }
+}
+
+func openFor(ctx context.Context, expiredOnly bool) (*workspace.Workspace, error) {
+ if expiredOnly {
+ return workspace.OpenShared(ctx)
+ }
+ return workspace.OpenExclusive(ctx)
+}
+
+func report(out io.Writer, sweep run.Sweep) {
+ if sweep.Damage != nil {
+ event(out, markError, theme.StatusError, msgDamaged)
+ }
+
+ failed := false
+ for _, result := range sweep.Results {
+ if result.Err != nil {
+ failed = true
+ event(out, markError, theme.StatusError,
+ fmt.Sprintf(faultFormat, result.Record.Kind, result.Record.TargetName, result.Err))
+ continue
+ }
+ event(out, markRevert, theme.StatusOK,
+ fmt.Sprintf(leaseFormat, result.Record.Kind, result.Record.TargetName,
+ result.Record.RunID, result.Record.LeaseID))
+ }
+
+ if len(sweep.Results) > 0 && !failed {
+ note(out, fmt.Sprintf(msgReclaimed, len(sweep.Results)))
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go
new file mode 100644
index 00000000..87a636d5
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go
@@ -0,0 +1,83 @@
+// ©AngelaMos | 2026
+// root.go
+
+package cli
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/khaos/internal/banner"
+ "github.com/CarterPerez-dev/khaos/internal/version"
+)
+
+const (
+ FlagNoColor = "no-color"
+
+ rootShort = "inject security failures and measure whether controls fail closed"
+ rootLong = `khaos injects controlled security failures into a Docker-scoped target and
+measures whether the target failed closed or failed open.
+
+Every other chaos tool asks whether the system stayed up. khaos asks which way
+it broke, by probing with a legitimate credential and an invalid one at the
+same time and comparing how the two move.
+
+Run khaos with no arguments in a terminal to open the interactive console.`
+
+ versionShort = "print the khaos version"
+ bannerShort = "print the khaos banner"
+ noColorUsage = "disable all color output"
+
+ versionFormat = "%s %s\n"
+)
+
+func NewRoot() *cobra.Command {
+ root := &cobra.Command{
+ Use: version.Name,
+ Short: rootShort,
+ Long: rootLong,
+ SilenceUsage: true,
+ SilenceErrors: true,
+ }
+
+ root.PersistentFlags().Bool(FlagNoColor, false, noColorUsage)
+ root.AddCommand(
+ newVersionCmd(),
+ newBannerCmd(),
+ newDoctorCmd(),
+ newTargetsCmd(),
+ newInjectCmd(),
+ newRangeCmd(),
+ newCatalogCmd(),
+ newPlanCmd(),
+ newRunCmd(),
+ newReclaimCmd(),
+ newReapCmd(),
+ )
+
+ return root
+}
+
+func newVersionCmd() *cobra.Command {
+ return &cobra.Command{
+ Use: "version",
+ Short: versionShort,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ _, err := fmt.Fprintf(cmd.OutOrStdout(), versionFormat, version.Name, version.Version())
+ return err
+ },
+ }
+}
+
+func newBannerCmd() *cobra.Command {
+ return &cobra.Command{
+ Use: "banner",
+ Short: bannerShort,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ return banner.Write(cmd.OutOrStdout())
+ },
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root_test.go
new file mode 100644
index 00000000..31a41371
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root_test.go
@@ -0,0 +1,99 @@
+// ©AngelaMos | 2026
+// root_test.go
+
+package cli
+
+import (
+ "io"
+ "testing"
+
+ "github.com/spf13/cobra"
+)
+
+func silence(cmd *cobra.Command) {
+ cmd.SetOut(io.Discard)
+ cmd.SetErr(io.Discard)
+}
+
+func TestFreshTreeDoesNotInheritFlagState(t *testing.T) {
+ first := NewRoot()
+ silence(first)
+ first.SetArgs([]string{"version", "--" + FlagNoColor})
+
+ if err := first.Execute(); err != nil {
+ t.Fatalf("first execute: %v", err)
+ }
+
+ if set, err := first.PersistentFlags().GetBool(FlagNoColor); err != nil || !set {
+ t.Fatalf("first tree did not record the flag (set=%v err=%v)", set, err)
+ }
+
+ second := NewRoot()
+ silence(second)
+ second.SetArgs([]string{"version"})
+
+ if err := second.Execute(); err != nil {
+ t.Fatalf("second execute: %v", err)
+ }
+
+ set, err := second.PersistentFlags().GetBool(FlagNoColor)
+ if err != nil {
+ t.Fatalf("second tree flag lookup: %v", err)
+ }
+
+ if set {
+ t.Fatal("a freshly built tree inherited --no-color from a previous invocation")
+ }
+}
+
+func TestSharedTreeLeaksFlagState(t *testing.T) {
+ shared := NewRoot()
+ silence(shared)
+
+ shared.SetArgs([]string{"version", "--" + FlagNoColor})
+ if err := shared.Execute(); err != nil {
+ t.Fatalf("first execute: %v", err)
+ }
+
+ shared.SetArgs([]string{"version"})
+ if err := shared.Execute(); err != nil {
+ t.Fatalf("second execute: %v", err)
+ }
+
+ set, err := shared.PersistentFlags().GetBool(FlagNoColor)
+ if err != nil {
+ t.Fatalf("flag lookup: %v", err)
+ }
+
+ if !set {
+ t.Fatal("cobra no longer leaks flag state across invocations on a shared tree, " +
+ "which means TestFreshTreeDoesNotInheritFlagState proves nothing and the " +
+ "factory may no longer be load bearing")
+ }
+}
+
+func TestEveryCommandRejectsStrayArguments(t *testing.T) {
+ registered := NewRoot().Commands()
+ if len(registered) < 4 {
+ t.Fatalf("only %d commands registered; this test is named 'every' and must cover them all", len(registered))
+ }
+
+ for _, cmd := range registered {
+ name := cmd.Name()
+ if cmd.Args == nil {
+ t.Fatalf("%s declares no Args validator, so it silently accepts stray positional arguments", name)
+ }
+ }
+
+ for _, name := range []string{"version", "banner", "doctor", "targets"} {
+ t.Run(name, func(t *testing.T) {
+ root := NewRoot()
+ silence(root)
+ root.SetArgs([]string{name, "unexpected"})
+
+ if err := root.Execute(); err == nil {
+ t.Fatalf("%s accepted a stray positional argument", name)
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/targets.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/targets.go
new file mode 100644
index 00000000..3920912f
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/targets.go
@@ -0,0 +1,153 @@
+// ©AngelaMos | 2026
+// targets.go
+
+package cli
+
+import (
+ "context"
+ "fmt"
+ "io"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/render"
+ "github.com/CarterPerez-dev/khaos/internal/theme"
+)
+
+const (
+ flagProject = "project"
+ flagAll = "all"
+
+ targetsShort = "discover containers that have consented to fault injection"
+ targetsLong = `targets lists every container carrying the ` + dockerx.LabelConsent + `=` + dockerx.ConsentValue + ` label,
+and says for each one whether khaos would accept or refuse it as a target.
+
+You never write a selector. You pick a name from this list.`
+
+ projectUsage = "only consider containers in this compose project"
+ allUsage = "also show containers khaos would refuse, and why"
+
+ headerName = "TARGET"
+ headerProject = "PROJECT"
+ headerImage = "IMAGE"
+ headerStatus = "STATUS"
+
+ statusReady = "ready"
+ colName = 26
+ colProject = 18
+ colImage = 30
+
+ emptyMessage = "no consenting containers found"
+ refusedHint = "re-run with --all to see containers khaos refuses and why"
+ refusedFormat = "%d refused. %s"
+ rowLayout = "%s%s%s%s%s\n"
+ noteLayout = "%s%s\n"
+ endpointNote = "daemon: %s"
+ dashMarker = "-"
+)
+
+func newTargetsCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "targets",
+ Short: targetsShort,
+ Long: targetsLong,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ project, err := cmd.Flags().GetString(flagProject)
+ if err != nil {
+ return err
+ }
+ all, err := cmd.Flags().GetBool(flagAll)
+ if err != nil {
+ return err
+ }
+ return runTargets(cmd.Context(), cmd.OutOrStdout(), project, all)
+ },
+ }
+
+ cmd.Flags().String(flagProject, "", projectUsage)
+ cmd.Flags().Bool(flagAll, false, allUsage)
+
+ return cmd
+}
+
+func runTargets(ctx context.Context, out io.Writer, project string, all bool) error {
+ client, err := dockerx.Open(ctx)
+ if err != nil {
+ return fmt.Errorf("%w (run `khaos doctor` for the fix)", err)
+ }
+ defer func() { _ = client.Close() }()
+
+ filter := dockerx.Filter{Project: project}
+ if all {
+ filter = dockerx.Filter{}
+ }
+
+ guard := dockerx.Guard{Project: project, SelfID: dockerx.SelfContainerID()}
+
+ targets, err := client.Targets(ctx, filter)
+ if err != nil {
+ return err
+ }
+
+ fmt.Fprintln(out)
+ fmt.Fprintf(out, noteLayout, render.Indent,
+ theme.Fg(theme.TextFaint, fmt.Sprintf(endpointNote, client.Endpoint().Host)))
+
+ if len(targets) == 0 {
+ fmt.Fprintf(out, noteLayout, render.Indent, theme.Fg(theme.TextNormal, emptyMessage))
+ fmt.Fprintf(out, noteLayout, render.Indent, theme.Fg(theme.TextDim, dockerx.FixNoTargets))
+ fmt.Fprintln(out)
+ return nil
+ }
+
+ fmt.Fprintf(out, rowLayout, render.Indent,
+ theme.Fg(theme.TextFaint, render.Cell(headerName, colName)),
+ theme.Fg(theme.TextFaint, render.Cell(headerProject, colProject)),
+ theme.Fg(theme.TextFaint, render.Cell(headerImage, colImage)),
+ theme.Fg(theme.TextFaint, headerStatus))
+
+ refused := 0
+ for _, target := range targets {
+ refusal := dockerx.Refuse(target, guard)
+ if refusal != nil {
+ refused++
+ if !all {
+ continue
+ }
+ }
+
+ status := theme.Fg(theme.StateHeld, statusReady)
+ if refusal != nil {
+ status = theme.Fg(theme.StateDenied, refusal.Predicate)
+ }
+
+ fmt.Fprintf(out, rowLayout, render.Indent,
+ theme.Fg(theme.TextBright, render.Cell(target.Name, colName)),
+ theme.Fg(theme.TextNormal, render.Cell(orDash(target.Project), colProject)),
+ theme.Fg(theme.TextDim, render.Cell(target.Image, colImage)),
+ status)
+
+ if all && refusal != nil {
+ fmt.Fprintf(out, noteLayout, render.FixIndent, theme.Fg(theme.TextDim, refusal.Reason))
+ fmt.Fprintf(out, noteLayout, render.FixIndent, theme.Fg(theme.TextBright, render.FixPrefix+refusal.Fix))
+ }
+ }
+
+ if refused > 0 && !all {
+ fmt.Fprintln(out)
+ fmt.Fprintf(out, noteLayout, render.Indent,
+ theme.Fg(theme.TextDim, fmt.Sprintf(refusedFormat, refused, refusedHint)))
+ }
+ fmt.Fprintln(out)
+
+ return nil
+}
+
+func orDash(s string) string {
+ if s == "" {
+ return dashMarker
+ }
+ return s
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go
new file mode 100644
index 00000000..d0096c5d
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go
@@ -0,0 +1,212 @@
+// ©AngelaMos | 2026
+// actions.go
+
+package dockerx
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+
+ "github.com/containerd/errdefs"
+ "github.com/moby/moby/api/types/network"
+ "github.com/moby/moby/client"
+)
+
+const (
+ errPause = "pause %s: %w"
+ errUnpause = "unpause %s: %w"
+ errDisconnect = "disconnect %s from %s: %w"
+ errConnect = "reconnect %s to %s: %w"
+ errEndpoints = "read network endpoints of %s: %w"
+ errState = "read state of %s: %w"
+ errPorts = "read published ports of %s: %w"
+ errNoPort = "nothing is published for container port %d on %s"
+ errEnv = "read the environment of %s: %w"
+ errAddress = "read the address of %s on network %s: %w"
+ errNoAddress = "%s has no address on network %s"
+ errBridge = "read the gateway of the default bridge network: %w"
+ errNoBridge = "the default bridge network reports no gateway address, so khaos cannot find an address a container can reach it on"
+
+ bridgeNetwork = "bridge"
+ envPairParts = 2
+ envPairSep = "="
+)
+
+var ErrContainerGone = errors.New("the container no longer exists")
+
+var ErrNetworkGone = errors.New("the network no longer exists, so the container could not be reattached to it")
+
+func (c *Client) Pause(ctx context.Context, containerID string) error {
+ if _, err := c.api.ContainerPause(ctx, containerID, client.ContainerPauseOptions{}); err != nil {
+ return fmt.Errorf(errPause, ShortID(containerID), c.classify(err))
+ }
+ return nil
+}
+
+func (c *Client) Unpause(ctx context.Context, containerID string) error {
+ if _, err := c.api.ContainerUnpause(ctx, containerID, client.ContainerUnpauseOptions{}); err != nil {
+ return fmt.Errorf(errUnpause, ShortID(containerID), c.classify(err))
+ }
+ return nil
+}
+
+func (c *Client) Disconnect(ctx context.Context, networkID, containerID string) error {
+ _, err := c.api.NetworkDisconnect(ctx, networkID, client.NetworkDisconnectOptions{Container: containerID})
+ if err != nil {
+ return fmt.Errorf(errDisconnect, ShortID(containerID), networkID, c.classify(err))
+ }
+ return nil
+}
+
+func (c *Client) Connect(ctx context.Context, networkID, containerID string, settings *network.EndpointSettings) error {
+ _, err := c.api.NetworkConnect(ctx, networkID, client.NetworkConnectOptions{
+ Container: containerID,
+ EndpointConfig: settings,
+ })
+ if err != nil {
+ return fmt.Errorf(errConnect, ShortID(containerID), networkID, c.classifyNetwork(ctx, containerID, err))
+ }
+ return nil
+}
+
+func (c *Client) Endpoints(ctx context.Context, containerID string) (map[string]*network.EndpointSettings, error) {
+ result, err := c.api.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{})
+ if err != nil {
+ return nil, fmt.Errorf(errEndpoints, ShortID(containerID), c.classify(err))
+ }
+
+ if result.Container.NetworkSettings == nil {
+ return nil, nil
+ }
+
+ return result.Container.NetworkSettings.Networks, nil
+}
+
+func (c *Client) Paused(ctx context.Context, containerID string) (bool, error) {
+ result, err := c.api.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{})
+ if err != nil {
+ return false, fmt.Errorf(errState, ShortID(containerID), c.classify(err))
+ }
+ return result.Container.State != nil && result.Container.State.Paused, nil
+}
+
+// ContainerEnv reads a container's environment. khaos uses it to learn where a
+// target is ALREADY addressing its dependency, rather than assuming the address
+// khaos would have chosen: environment is fixed at container creation, so the
+// container is the only authority on what it will actually dial.
+func (c *Client) ContainerEnv(ctx context.Context, containerID string) (map[string]string, error) {
+ result, err := c.api.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{})
+ if err != nil {
+ return nil, fmt.Errorf(errEnv, ShortID(containerID), c.classify(err))
+ }
+
+ environ := map[string]string{}
+
+ if result.Container.Config == nil {
+ return environ, nil
+ }
+
+ for _, pair := range result.Container.Config.Env {
+ name, value, found := strings.Cut(pair, envPairSep)
+ if !found {
+ continue
+ }
+ environ[name] = value
+ }
+
+ return environ, nil
+}
+
+func (c *Client) NetworkAddress(ctx context.Context, containerID, networkName string) (string, error) {
+ endpoints, err := c.Endpoints(ctx, containerID)
+ if err != nil {
+ return "", fmt.Errorf(errAddress, ShortID(containerID), networkName, err)
+ }
+
+ settings, attached := endpoints[networkName]
+ if !attached || settings == nil || !settings.IPAddress.IsValid() {
+ return "", fmt.Errorf(errNoAddress, ShortID(containerID), networkName)
+ }
+
+ return settings.IPAddress.String(), nil
+}
+
+// BridgeGateway is the address a container reaches the host on. It is the DEFAULT
+// bridge's gateway and that is measured, not assumed: `host-gateway` resolves
+// there, a listener on a user-defined network's own gateway is unreachable from
+// containers on it, and the value is NOT the commonly cited 172.17.0.1 on every
+// host. Discovering it is the only correct way to find it.
+func (c *Client) BridgeGateway(ctx context.Context) (string, error) {
+ result, err := c.api.NetworkInspect(ctx, bridgeNetwork, client.NetworkInspectOptions{})
+ if err != nil {
+ return "", fmt.Errorf(errBridge, err)
+ }
+
+ for _, config := range result.Network.IPAM.Config {
+ if config.Gateway.IsValid() {
+ return config.Gateway.String(), nil
+ }
+ }
+
+ return "", errors.New(errNoBridge)
+}
+
+// classify turns a daemon NotFound into ErrContainerGone, which callers treat as
+// "the fault went with the target" and record as successfully reverted.
+//
+// It must therefore only be used where the MISSING THING IS THE CONTAINER. A
+// NetworkConnect returns NotFound when the NETWORK is gone, and routing that
+// through here told the session a detached container had been restored. See
+// classifyNetwork.
+func (c *Client) classify(err error) error {
+ if errdefs.IsNotFound(err) {
+ return fmt.Errorf("%w: %v", ErrContainerGone, err)
+ }
+ return err
+}
+
+// classifyNetwork is the same question for an operation that names two
+// resources. A NotFound here can mean either one is missing, and only the
+// container's absence makes the fault moot — a missing network leaves the
+// container detached and khaos still owing a restore.
+//
+// It resolves the ambiguity by ASKING: if the container still exists, the
+// network was the missing resource and the failure stays a failure.
+func (c *Client) classifyNetwork(ctx context.Context, containerID string, err error) error {
+ if !errdefs.IsNotFound(err) {
+ return err
+ }
+
+ if _, alive := c.api.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}); alive == nil {
+ return fmt.Errorf("%w: %v", ErrNetworkGone, err)
+ }
+
+ return fmt.Errorf("%w: %v", ErrContainerGone, err)
+}
+
+func (c *Client) PublishedPort(ctx context.Context, containerID string, internal int) (int, error) {
+ result, err := c.api.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{})
+ if err != nil {
+ return 0, fmt.Errorf(errPorts, ShortID(containerID), c.classify(err))
+ }
+
+ if result.Container.NetworkSettings == nil {
+ return 0, fmt.Errorf(errNoPort, internal, ShortID(containerID))
+ }
+
+ for port, bindings := range result.Container.NetworkSettings.Ports {
+ if int(port.Num()) != internal {
+ continue
+ }
+ for _, binding := range bindings {
+ if published, err := strconv.Atoi(binding.HostPort); err == nil && published > 0 {
+ return published, nil
+ }
+ }
+ }
+
+ return 0, fmt.Errorf(errNoPort, internal, ShortID(containerID))
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/client.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/client.go
new file mode 100644
index 00000000..8a80ed39
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/client.go
@@ -0,0 +1,138 @@
+// ©AngelaMos | 2026
+// client.go
+
+package dockerx
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "net"
+ "os"
+ "time"
+
+ "github.com/moby/moby/client"
+)
+
+var (
+ ErrNoEndpoint = errors.New("no Docker endpoint found")
+ ErrSocketMissing = errors.New("the Docker socket does not exist")
+ ErrSocketDenied = errors.New("permission denied on the Docker socket")
+ ErrNotASocket = errors.New("the Docker endpoint path is not a socket")
+ ErrNothingListening = errors.New("the Docker socket exists but nothing is listening on it")
+ ErrDaemonSilent = errors.New("the Docker endpoint is reachable but the daemon did not answer")
+)
+
+const (
+ dialTimeout = 3 * time.Second
+ unixNetwork = "unix"
+ errClientFmt = "docker client: %w"
+ errWrapPath = "%w: %s"
+ errWrapDetail = "%w: %v"
+)
+
+type Client struct {
+ api *client.Client
+ endpoint Endpoint
+ version string
+ daemon string
+}
+
+func Connect(ctx context.Context, endpoint Endpoint) (*Client, error) {
+ if endpoint.Host == "" {
+ return nil, ErrNoEndpoint
+ }
+
+ if path, ok := endpoint.SocketPath(); ok {
+ if err := probeSocket(path); err != nil {
+ return nil, err
+ }
+ }
+
+ api, err := client.New(client.WithHost(endpoint.Host))
+ if err != nil {
+ return nil, fmt.Errorf(errClientFmt, err)
+ }
+
+ ping, err := api.Ping(ctx, client.PingOptions{})
+ if err != nil {
+ _ = api.Close()
+ return nil, fmt.Errorf(errWrapDetail, ErrDaemonSilent, err)
+ }
+
+ c := &Client{api: api, endpoint: endpoint, version: ping.APIVersion}
+
+ // The daemon's own ID, read once. It is what the journal records so khaos
+ // can tell its own faults from another machine's, and it has to come from
+ // the daemon rather than from the endpoint string: the same daemon is
+ // reachable as a socket path, a TCP address and an SSH context, and all
+ // three are the same place.
+ if info, err := api.Info(ctx, client.InfoOptions{}); err == nil {
+ c.daemon = info.Info.ID
+ }
+
+ return c, nil
+}
+
+// Daemon is a stable identity for the daemon this client is attached to. It is
+// empty when the daemon would not say, which callers must treat as unknown
+// rather than as a match.
+func (c *Client) Daemon() string {
+ return c.daemon
+}
+
+func Open(ctx context.Context) (*Client, error) {
+ endpoint, err := ResolveEndpoint(os.Getenv, homeDir())
+ if err != nil {
+ return nil, err
+ }
+ return Connect(ctx, endpoint)
+}
+
+func (c *Client) Close() error {
+ return c.api.Close()
+}
+
+func (c *Client) Endpoint() Endpoint {
+ return c.endpoint
+}
+
+func (c *Client) APIVersion() string {
+ return c.version
+}
+
+func probeSocket(path string) error {
+ info, err := os.Stat(path)
+ if errors.Is(err, fs.ErrNotExist) {
+ return fmt.Errorf(errWrapPath, ErrSocketMissing, path)
+ }
+ if errors.Is(err, fs.ErrPermission) {
+ return fmt.Errorf(errWrapPath, ErrSocketDenied, path)
+ }
+ if err != nil {
+ return fmt.Errorf("stat %s: %w", path, err)
+ }
+
+ if info.Mode()&fs.ModeSocket == 0 {
+ return fmt.Errorf(errWrapPath, ErrNotASocket, path)
+ }
+
+ conn, err := net.DialTimeout(unixNetwork, path, dialTimeout)
+ if errors.Is(err, fs.ErrPermission) {
+ return fmt.Errorf(errWrapPath, ErrSocketDenied, path)
+ }
+ if err != nil {
+ return fmt.Errorf(errWrapPath, ErrNothingListening, path)
+ }
+
+ return conn.Close()
+}
+
+func homeDir() string {
+ home, err := os.UserHomeDir()
+ if err != nil {
+ return ""
+ }
+ return home
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover.go
new file mode 100644
index 00000000..b12f7c03
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover.go
@@ -0,0 +1,219 @@
+// ©AngelaMos | 2026
+// discover.go
+
+package dockerx
+
+import (
+ "cmp"
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "strings"
+
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/client"
+)
+
+const (
+ filterLabel = "label"
+ namePrefix = "/"
+ minIDPrefix = 4
+ errListFmt = "list containers: %w"
+ errInspect = "inspect %s: %w"
+ errNoMatch = "no consenting target named %q (run `khaos targets` to see what is available)"
+ errAmbiguous = "%q matches %d targets, name one exactly: %s"
+ errShortName = "a target name must be at least %d characters"
+)
+
+var (
+ ErrNoHostConfig = errors.New("the daemon returned a container with no host configuration, so khaos cannot tell whether it is safe to touch")
+ ErrEmptyName = errors.New("a target name is required")
+)
+
+func (c *Client) Targets(ctx context.Context, filter Filter) ([]Target, error) {
+ consenting, err := c.list(ctx, make(client.Filters).Add(filterLabel, LabelConsent))
+ if err != nil {
+ return nil, err
+ }
+
+ peers, err := c.namespacePeers(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ targets := make([]Target, 0, len(consenting))
+ for _, summary := range consenting {
+ target, err := c.inspect(ctx, summary.ID)
+ if err != nil {
+ return nil, err
+ }
+ if filter.Project != "" && target.Project != filter.Project {
+ continue
+ }
+ target.NamespacePeers = peers[target.ID]
+ targets = append(targets, target)
+ }
+
+ slices.SortFunc(targets, func(a, b Target) int {
+ return cmp.Or(cmp.Compare(a.Project, b.Project), cmp.Compare(a.Name, b.Name))
+ })
+
+ return targets, nil
+}
+
+func (c *Client) Lookup(ctx context.Context, name string, filter Filter) (Target, error) {
+ if strings.TrimSpace(name) == "" {
+ return Target{}, ErrEmptyName
+ }
+
+ if len(name) < minIDPrefix {
+ return Target{}, fmt.Errorf(errShortName, minIDPrefix)
+ }
+
+ targets, err := c.Targets(ctx, filter)
+ if err != nil {
+ return Target{}, err
+ }
+
+ var matches []Target
+ for _, t := range targets {
+ if t.Name == name || t.Service == name || strings.HasPrefix(t.ID, name) {
+ matches = append(matches, t)
+ }
+ }
+
+ switch len(matches) {
+ case 0:
+ return Target{}, fmt.Errorf(errNoMatch, name)
+ case 1:
+ return matches[0], nil
+ default:
+ return Target{}, fmt.Errorf(errAmbiguous, name, len(matches), joinNames(matches))
+ }
+}
+
+func (c *Client) Accept(ctx context.Context, name string, filter Filter, guard Guard) (Target, error) {
+ target, err := c.Lookup(ctx, name, filter)
+ if err != nil {
+ return Target{}, err
+ }
+
+ if refusal := Refuse(target, guard); refusal != nil {
+ return Target{}, refusal
+ }
+
+ return target, nil
+}
+
+func (c *Client) list(ctx context.Context, filters client.Filters) ([]container.Summary, error) {
+ listed, err := c.api.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: filters})
+ if err != nil {
+ return nil, fmt.Errorf(errListFmt, err)
+ }
+ return listed.Items, nil
+}
+
+func (c *Client) namespacePeers(ctx context.Context) (map[string][]string, error) {
+ all, err := c.list(ctx, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ byName := make(map[string]string, len(all))
+ for _, summary := range all {
+ for _, name := range summary.Names {
+ byName[strings.TrimPrefix(name, namePrefix)] = summary.ID
+ }
+ }
+
+ peers := make(map[string][]string)
+ for _, summary := range all {
+ reference, ok := strings.CutPrefix(summary.HostConfig.NetworkMode, NetworkModeContainerPfx)
+ if !ok {
+ continue
+ }
+
+ host := reference
+ if resolved, found := byName[reference]; found {
+ host = resolved
+ }
+
+ peers[host] = append(peers[host], displayName(summary))
+ }
+
+ return peers, nil
+}
+
+func (c *Client) inspect(ctx context.Context, id string) (Target, error) {
+ result, err := c.api.ContainerInspect(ctx, id, client.ContainerInspectOptions{})
+ if err != nil {
+ return Target{}, fmt.Errorf(errInspect, ShortID(id), err)
+ }
+
+ target, err := targetFrom(result.Container)
+ if err != nil {
+ return Target{}, fmt.Errorf(errInspect, ShortID(id), err)
+ }
+
+ return target, nil
+}
+
+func targetFrom(raw container.InspectResponse) (Target, error) {
+ if raw.HostConfig == nil {
+ return Target{}, ErrNoHostConfig
+ }
+
+ target := Target{
+ ID: raw.ID,
+ Name: strings.TrimPrefix(raw.Name, namePrefix),
+ Running: raw.State != nil && raw.State.Running,
+ Labels: map[string]string{},
+ NetworkMode: string(raw.HostConfig.NetworkMode),
+ PidMode: string(raw.HostConfig.PidMode),
+ IpcMode: string(raw.HostConfig.IpcMode),
+ UsernsMode: string(raw.HostConfig.UsernsMode),
+ Privileged: raw.HostConfig.Privileged,
+ SecurityOpt: raw.HostConfig.SecurityOpt,
+ Inspected: true,
+ }
+
+ target.Capabilities = append(target.Capabilities, raw.HostConfig.CapAdd...)
+
+ for _, device := range raw.HostConfig.Devices {
+ target.Devices = append(target.Devices, device.PathOnHost)
+ }
+
+ if raw.Config != nil {
+ target.Image = raw.Config.Image
+ for key, value := range raw.Config.Labels {
+ target.Labels[key] = value
+ }
+ }
+
+ target.Project = target.Labels[LabelComposeProject]
+ target.Service = target.Labels[LabelComposeService]
+
+ for _, mount := range raw.Mounts {
+ if mount.Source != "" {
+ target.BindSources = append(target.BindSources, mount.Source)
+ }
+ }
+
+ return target, nil
+}
+
+func displayName(summary container.Summary) string {
+ if len(summary.Names) > 0 {
+ return strings.TrimPrefix(summary.Names[0], namePrefix)
+ }
+ return ShortID(summary.ID)
+}
+
+func joinNames(targets []Target) string {
+ names := make([]string, len(targets))
+ for i, t := range targets {
+ names[i] = t.Name
+ }
+ return strings.Join(names, ", ")
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover_test.go
new file mode 100644
index 00000000..5b358646
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/discover_test.go
@@ -0,0 +1,149 @@
+// ©AngelaMos | 2026
+// discover_test.go
+
+package dockerx
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/moby/moby/api/types/container"
+)
+
+func inspectResponse() container.InspectResponse {
+ return container.InspectResponse{
+ ID: acceptableID,
+ Name: "/shop-api-1",
+ State: &container.State{Running: true},
+ HostConfig: &container.HostConfig{
+ NetworkMode: "shop_default",
+ },
+ Config: &container.Config{
+ Image: "shop/api:latest",
+ Labels: map[string]string{LabelConsent: ConsentValue, LabelComposeProject: project},
+ },
+ Mounts: []container.MountPoint{{Source: "/srv/shop/config"}},
+ }
+}
+
+func TestTargetFromProducesAnAcceptableTarget(t *testing.T) {
+ target, err := targetFrom(inspectResponse())
+ if err != nil {
+ t.Fatalf("targetFrom: %v", err)
+ }
+
+ if refusal := Refuse(target, Guard{Project: project}); refusal != nil {
+ t.Fatalf("a healthy inspect response produced a target refused by %s (%s)",
+ refusal.Predicate, refusal.Reason)
+ }
+}
+
+func TestTargetFromRefusesAnInspectResponseWithNoHostConfig(t *testing.T) {
+ raw := inspectResponse()
+ raw.HostConfig = nil
+
+ if _, err := targetFrom(raw); !errors.Is(err, ErrNoHostConfig) {
+ t.Fatalf("targetFrom returned %v for a nil HostConfig; it must refuse rather than zero-fill, "+
+ "because zero-filled host config silently disarms every container-property predicate", err)
+ }
+}
+
+func TestTargetFromCarriesEveryFieldThePredicatesRead(t *testing.T) {
+ cases := []struct {
+ name string
+ mutate func(*container.InspectResponse)
+ predicate string
+ }{
+ {"host network", func(r *container.InspectResponse) {
+ r.HostConfig.NetworkMode = NetworkModeHost
+ }, PredicateHostNetwork},
+ {"shared namespace", func(r *container.InspectResponse) {
+ r.HostConfig.NetworkMode = container.NetworkMode(NetworkModeContainerPfx + peerID)
+ }, PredicateSharedNamespace},
+ {"host pid namespace", func(r *container.InspectResponse) {
+ r.HostConfig.PidMode = ModeHost
+ }, PredicateHostNamespace},
+ {"host ipc namespace", func(r *container.InspectResponse) {
+ r.HostConfig.IpcMode = ModeHost
+ }, PredicateHostNamespace},
+ {"host user namespace", func(r *container.InspectResponse) {
+ r.HostConfig.UsernsMode = ModeHost
+ }, PredicateHostNamespace},
+ {"privileged", func(r *container.InspectResponse) {
+ r.HostConfig.Privileged = true
+ }, PredicatePrivileged},
+ {"dangerous capability", func(r *container.InspectResponse) {
+ r.HostConfig.CapAdd = []string{"CAP_SYS_MODULE"}
+ }, PredicateDangerousCap},
+ {"device passthrough", func(r *container.InspectResponse) {
+ r.HostConfig.Devices = []container.DeviceMapping{{PathOnHost: "/dev/sda"}}
+ }, PredicateDeviceAccess},
+ {"unconfined profile", func(r *container.InspectResponse) {
+ r.HostConfig.SecurityOpt = []string{"apparmor=unconfined"}
+ }, PredicateUnconfined},
+ {"socket mount", func(r *container.InspectResponse) {
+ r.Mounts = append(r.Mounts, container.MountPoint{Source: DockerSocketPath})
+ }, PredicateSocketReachable},
+ {"socket parent directory", func(r *container.InspectResponse) {
+ r.Mounts = append(r.Mounts, container.MountPoint{Source: "/var/run"})
+ }, PredicateSocketReachable},
+ {"not running", func(r *container.InspectResponse) {
+ r.State.Running = false
+ }, PredicateNotRunning},
+ {"consent removed", func(r *container.InspectResponse) {
+ delete(r.Config.Labels, LabelConsent)
+ }, PredicateConsentMissing},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ raw := inspectResponse()
+ tc.mutate(&raw)
+
+ target, err := targetFrom(raw)
+ if err != nil {
+ t.Fatalf("targetFrom: %v", err)
+ }
+
+ refusal := Refuse(target, Guard{Project: project})
+ if refusal == nil {
+ t.Fatalf("a container that is %s was built into an accepted target; "+
+ "the field the %s predicate reads is not being carried across from the daemon response",
+ tc.name, tc.predicate)
+ }
+
+ if refusal.Predicate != tc.predicate {
+ t.Fatalf("a container that is %s was refused by %s, want %s",
+ tc.name, refusal.Predicate, tc.predicate)
+ }
+ })
+ }
+}
+
+func TestTargetFromSurvivesAbsentOptionalSections(t *testing.T) {
+ raw := inspectResponse()
+ raw.Config = nil
+ raw.State = nil
+ raw.Mounts = nil
+
+ target, err := targetFrom(raw)
+ if err != nil {
+ t.Fatalf("targetFrom: %v", err)
+ }
+
+ refusal := Refuse(target, Guard{})
+ if refusal == nil {
+ t.Fatal("an inspect response with no Config and no State produced an accepted target")
+ }
+}
+
+func TestTargetFromMarksTheTargetInspected(t *testing.T) {
+ target, err := targetFrom(inspectResponse())
+ if err != nil {
+ t.Fatalf("targetFrom: %v", err)
+ }
+
+ if !target.Inspected {
+ t.Fatal("targetFrom did not mark the target as inspected, so it will be refused as incomplete")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/doctor.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/doctor.go
new file mode 100644
index 00000000..2ec56bc8
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/doctor.go
@@ -0,0 +1,223 @@
+// ©AngelaMos | 2026
+// doctor.go
+
+package dockerx
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "os/user"
+)
+
+type Status int
+
+const (
+ StatusPass Status = iota
+ StatusFail
+ StatusSkipped
+)
+
+type Check struct {
+ Name string
+ Status Status
+ Detail string
+ Fix string
+}
+
+func (c Check) OK() bool {
+ return c.Status == StatusPass
+}
+
+const (
+ checkEndpoint = "docker endpoint"
+ checkSocket = "socket"
+ checkDaemon = "daemon"
+ checkTargets = "consenting targets"
+
+ fixNoEndpoint = "install Docker, or set DOCKER_HOST to a reachable daemon"
+ fixDaemonLocal = "start the Docker daemon: systemctl --user start docker, or sudo systemctl start docker"
+ fixDaemonRemote = "check that %s is reachable and that the daemon there is listening"
+ fixContext = "run `docker context ls` to see what exists, then `docker context use <name>`, or unset DOCKER_CONTEXT"
+ FixNoTargets = "add `labels: [\"" + LabelConsent + "=" + ConsentValue + "\"]` to a service and recreate it, or run `khaos range up`"
+ fixRetry = "resolve the daemon problem above, then run this again"
+
+ detailNoEndpoint = "no DOCKER_HOST, no docker context, no rootless socket, no " + DockerSocketPath
+ detailSkipDaemon = "not checked, because the daemon did not answer"
+ detailSocketOK = "%s is present and a daemon is listening"
+ detailDaemonOK = "responding, API version %s"
+ detailEndpointFmt = "%s (via %s%s)"
+ detailTargets = "%d container(s) carry %s=%s"
+ detailNone = "none"
+
+ permissionFixFormat = "add yourself to the docker group and start a new login session: sudo usermod -aG docker %s. " +
+ "Rootless Docker avoids this entirely and khaos prefers it, since access to %s is equivalent to root on this machine"
+ fallbackUser = "your user"
+
+ checkCapacity = 4
+)
+
+func Diagnose(ctx context.Context) []Check {
+ return DiagnoseWith(ctx, os.Getenv, homeDir())
+}
+
+func DiagnoseWith(ctx context.Context, environ func(string) string, home string) []Check {
+ checks := make([]Check, 0, checkCapacity)
+
+ endpoint, err := ResolveEndpoint(environ, home)
+ if err != nil {
+ checks = append(checks, Check{
+ Name: checkEndpoint,
+ Status: StatusFail,
+ Detail: endpointFailureDetail(err),
+ Fix: endpointFix(err),
+ })
+ return append(checks, skipped(checkSocket), skipped(checkDaemon), skipped(checkTargets))
+ }
+
+ checks = append(checks, Check{
+ Name: checkEndpoint,
+ Status: StatusPass,
+ Detail: fmt.Sprintf(detailEndpointFmt, endpoint.Host, endpoint.Source, contextSuffix(endpoint)),
+ })
+
+ path, isSocket := endpoint.SocketPath()
+ if !isSocket {
+ checks = append(checks, skipped(checkSocket))
+ }
+
+ cli, connectErr := Connect(ctx, endpoint)
+
+ if isSocket {
+ checks = append(checks, socketCheck(path, connectErr))
+ }
+
+ if connectErr != nil {
+ checks = append(checks, Check{
+ Name: checkDaemon,
+ Status: StatusFail,
+ Detail: connectErr.Error(),
+ Fix: daemonFix(connectErr, endpoint),
+ })
+ return append(checks, skipped(checkTargets))
+ }
+ defer func() { _ = cli.Close() }()
+
+ checks = append(checks, Check{
+ Name: checkDaemon,
+ Status: StatusPass,
+ Detail: fmt.Sprintf(detailDaemonOK, cli.APIVersion()),
+ })
+
+ targets, err := cli.Targets(ctx, Filter{})
+ if err != nil {
+ return append(checks, Check{
+ Name: checkTargets,
+ Status: StatusFail,
+ Detail: err.Error(),
+ Fix: fixRetry,
+ })
+ }
+
+ return append(checks, targetsCheck(targets))
+}
+
+func skipped(name string) Check {
+ return Check{Name: name, Status: StatusSkipped, Detail: detailSkipDaemon}
+}
+
+func socketCheck(path string, connectErr error) Check {
+ if connectErr != nil && isSocketProblem(connectErr) {
+ return Check{
+ Name: checkSocket,
+ Status: StatusFail,
+ Detail: connectErr.Error(),
+ Fix: socketFix(connectErr, path),
+ }
+ }
+ return Check{Name: checkSocket, Status: StatusPass, Detail: fmt.Sprintf(detailSocketOK, path)}
+}
+
+func targetsCheck(targets []Target) Check {
+ consenting := 0
+ for _, t := range targets {
+ if t.Consented() {
+ consenting++
+ }
+ }
+
+ if consenting == 0 {
+ return Check{Name: checkTargets, Status: StatusFail, Detail: detailNone, Fix: FixNoTargets}
+ }
+
+ return Check{
+ Name: checkTargets,
+ Status: StatusPass,
+ Detail: fmt.Sprintf(detailTargets, consenting, LabelConsent, ConsentValue),
+ }
+}
+
+func isSocketProblem(err error) bool {
+ return errors.Is(err, ErrSocketMissing) ||
+ errors.Is(err, ErrSocketDenied) ||
+ errors.Is(err, ErrNotASocket) ||
+ errors.Is(err, ErrNothingListening)
+}
+
+func endpointFailureDetail(err error) string {
+ if errors.Is(err, ErrNoEndpoint) {
+ return detailNoEndpoint
+ }
+ return err.Error()
+}
+
+func endpointFix(err error) string {
+ if errors.Is(err, ErrContextUnresolvable) {
+ return fixContext
+ }
+ return fixNoEndpoint
+}
+
+func socketFix(err error, path string) string {
+ if isPermissionDenied(err) {
+ return permissionFix(path)
+ }
+ return fixDaemonLocal
+}
+
+func daemonFix(err error, endpoint Endpoint) string {
+ if isPermissionDenied(err) {
+ path, _ := endpoint.SocketPath()
+ if path == "" {
+ path = DockerSocketPath
+ }
+ return permissionFix(path)
+ }
+
+ if _, isSocket := endpoint.SocketPath(); !isSocket {
+ return fmt.Sprintf(fixDaemonRemote, endpoint.Host)
+ }
+
+ return fixDaemonLocal
+}
+
+func isPermissionDenied(err error) bool {
+ return errors.Is(err, ErrSocketDenied) || errors.Is(err, fs.ErrPermission)
+}
+
+func permissionFix(path string) string {
+ name := fallbackUser
+ if u, err := user.Current(); err == nil {
+ name = u.Username
+ }
+ return fmt.Sprintf(permissionFixFormat, name, path)
+}
+
+func contextSuffix(e Endpoint) string {
+ if e.Name == "" {
+ return ""
+ }
+ return fmt.Sprintf(" %q", e.Name)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint.go
new file mode 100644
index 00000000..afdf1096
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint.go
@@ -0,0 +1,167 @@
+// ©AngelaMos | 2026
+// endpoint.go
+
+package dockerx
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+const (
+ EnvDockerHost = "DOCKER_HOST"
+ EnvDockerContext = "DOCKER_CONTEXT"
+ EnvXDGRuntime = "XDG_RUNTIME_DIR"
+
+ SourceEnvHost = "DOCKER_HOST"
+ SourceEnvContext = "DOCKER_CONTEXT"
+ SourceConfigContext = "docker context"
+ SourceRootless = "rootless socket"
+ SourceDefault = "default socket"
+ SourceNone = "none"
+
+ unixScheme = "unix://"
+ defaultContext = "default"
+ dockerDir = ".docker"
+ configFile = "config.json"
+ contextsDir = "contexts"
+ contextMetaDir = "meta"
+ contextMeta = "meta.json"
+ dockerEndpoint = "docker"
+
+ errUnresolvableContext = "%w: %s names the context %q, but its metadata could not be read"
+)
+
+var ErrContextUnresolvable = errors.New("the selected Docker context could not be resolved")
+
+type Endpoint struct {
+ Host string
+ Source string
+ Name string
+}
+
+type dockerConfig struct {
+ CurrentContext string `json:"currentContext"`
+}
+
+type contextMetadata struct {
+ Name string `json:"Name"`
+ Endpoints map[string]struct {
+ Host string `json:"Host"`
+ } `json:"Endpoints"`
+}
+
+func ResolveEndpoint(environ func(string) string, home string) (Endpoint, error) {
+ if host := environ(EnvDockerHost); host != "" {
+ return Endpoint{Host: host, Source: SourceEnvHost}, nil
+ }
+
+ // An explicit DOCKER_CONTEXT=default SELECTS the default endpoint. It does
+ // not mean "unset", and treating it as unset is a divergence from the CLI
+ // that `khaos range` shells out to.
+ //
+ // Measured: with a saved current context of `ag-server` and DOCKER_CONTEXT
+ // set to `default`, the Docker CLI uses the LOCAL daemon while khaos used
+ // ag-server. khaos would then inject on one machine while `docker compose`
+ // acted on another, which is the worst shape this resolver can take.
+ //
+ // (The audit that raised this also claimed DOCKER_CONTEXT outranks
+ // DOCKER_HOST. Measured on docker 29.1.3, it does not: DOCKER_HOST wins, and
+ // the precedence above is correct.)
+ if name := environ(EnvDockerContext); name == defaultContext {
+ return defaultEndpoint(environ)
+ }
+
+ if name := environ(EnvDockerContext); name != "" {
+ host, ok := contextHost(home, name)
+ if !ok {
+ return Endpoint{}, fmt.Errorf(errUnresolvableContext, ErrContextUnresolvable, EnvDockerContext, name)
+ }
+ return Endpoint{Host: host, Source: SourceEnvContext, Name: name}, nil
+ }
+
+ if name, ok := currentContext(home); ok && name != defaultContext {
+ host, found := contextHost(home, name)
+ if !found {
+ return Endpoint{}, fmt.Errorf(errUnresolvableContext, ErrContextUnresolvable, SourceConfigContext, name)
+ }
+ return Endpoint{Host: host, Source: SourceConfigContext, Name: name}, nil
+ }
+
+ return defaultEndpoint(environ)
+}
+
+// defaultEndpoint is the `default` context: the rootless socket when one is
+// present, otherwise the system socket.
+func defaultEndpoint(environ func(string) string) (Endpoint, error) {
+ if runtimeDir := environ(EnvXDGRuntime); runtimeDir != "" {
+ path := filepath.Join(runtimeDir, socketBaseName)
+ if exists(path) {
+ return Endpoint{Host: unixScheme + path, Source: SourceRootless}, nil
+ }
+ }
+
+ if exists(DockerSocketPath) {
+ return Endpoint{Host: unixScheme + DockerSocketPath, Source: SourceDefault}, nil
+ }
+
+ return Endpoint{Source: SourceNone}, ErrNoEndpoint
+}
+
+func xdgRuntimeDir() string {
+ return os.Getenv(EnvXDGRuntime)
+}
+
+func (e Endpoint) SocketPath() (string, bool) {
+ if !strings.HasPrefix(e.Host, unixScheme) {
+ return "", false
+ }
+ return strings.TrimPrefix(e.Host, unixScheme), true
+}
+
+func currentContext(home string) (string, bool) {
+ raw, err := os.ReadFile(filepath.Join(home, dockerDir, configFile))
+ if err != nil {
+ return "", false
+ }
+
+ var cfg dockerConfig
+ if err := json.Unmarshal(raw, &cfg); err != nil {
+ return "", false
+ }
+
+ return cfg.CurrentContext, cfg.CurrentContext != ""
+}
+
+func contextHost(home, name string) (string, bool) {
+ digest := sha256.Sum256([]byte(name))
+ path := filepath.Join(home, dockerDir, contextsDir, contextMetaDir, hex.EncodeToString(digest[:]), contextMeta)
+
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ return "", false
+ }
+
+ var meta contextMetadata
+ if err := json.Unmarshal(raw, &meta); err != nil {
+ return "", false
+ }
+
+ endpoint, ok := meta.Endpoints[dockerEndpoint]
+ if !ok || endpoint.Host == "" {
+ return "", false
+ }
+
+ return endpoint.Host, true
+}
+
+func exists(path string) bool {
+ _, err := os.Stat(path)
+ return err == nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint_test.go
new file mode 100644
index 00000000..41fb7a05
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint_test.go
@@ -0,0 +1,278 @@
+// ©AngelaMos | 2026
+// endpoint_test.go
+
+package dockerx
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+const (
+ envHostValue = "tcp://10.0.0.5:2375"
+ ctxName = "orangepi"
+ ctxHostValue = "ssh://[email protected]"
+ otherCtxName = "staging"
+ otherCtxHost = "tcp://10.0.0.9:2375"
+ rootlessSockDir = "runtime"
+)
+
+func fakeEnv(pairs map[string]string) func(string) string {
+ return func(key string) string { return pairs[key] }
+}
+
+func writeContext(t *testing.T, home, name, host string) {
+ t.Helper()
+
+ digest := sha256.Sum256([]byte(name))
+ dir := filepath.Join(home, dockerDir, contextsDir, contextMetaDir, hex.EncodeToString(digest[:]))
+
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatalf("mkdir context: %v", err)
+ }
+
+ body := `{"Name":"` + name + `","Endpoints":{"docker":{"Host":"` + host + `"}}}`
+ if err := os.WriteFile(filepath.Join(dir, contextMeta), []byte(body), 0o644); err != nil {
+ t.Fatalf("write context meta: %v", err)
+ }
+}
+
+func writeCurrentContext(t *testing.T, home, name string) {
+ t.Helper()
+
+ dir := filepath.Join(home, dockerDir)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatalf("mkdir docker config: %v", err)
+ }
+
+ body := `{"currentContext":"` + name + `"}`
+ if err := os.WriteFile(filepath.Join(dir, configFile), []byte(body), 0o644); err != nil {
+ t.Fatalf("write docker config: %v", err)
+ }
+}
+
+func makeRootlessSocket(t *testing.T, home string) string {
+ t.Helper()
+
+ dir := filepath.Join(home, rootlessSockDir)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatalf("mkdir runtime: %v", err)
+ }
+
+ path := filepath.Join(dir, socketBaseName)
+ if err := os.WriteFile(path, nil, 0o600); err != nil {
+ t.Fatalf("write socket placeholder: %v", err)
+ }
+
+ return dir
+}
+
+func TestEnvHostWinsOverEverything(t *testing.T) {
+ home := t.TempDir()
+ writeContext(t, home, ctxName, ctxHostValue)
+ writeCurrentContext(t, home, ctxName)
+
+ env := fakeEnv(map[string]string{
+ EnvDockerHost: envHostValue,
+ EnvDockerContext: ctxName,
+ })
+
+ got, err := ResolveEndpoint(env, home)
+ if err != nil {
+ t.Fatalf("resolve: %v", err)
+ }
+
+ if got.Host != envHostValue || got.Source != SourceEnvHost {
+ t.Fatalf("got %+v, want %s via %s", got, envHostValue, SourceEnvHost)
+ }
+}
+
+func TestEnvContextWinsOverConfigContext(t *testing.T) {
+ home := t.TempDir()
+ writeContext(t, home, ctxName, ctxHostValue)
+ writeContext(t, home, otherCtxName, otherCtxHost)
+ writeCurrentContext(t, home, ctxName)
+
+ got, err := ResolveEndpoint(fakeEnv(map[string]string{EnvDockerContext: otherCtxName}), home)
+ if err != nil {
+ t.Fatalf("resolve: %v", err)
+ }
+
+ if got.Host != otherCtxHost || got.Source != SourceEnvContext || got.Name != otherCtxName {
+ t.Fatalf("got %+v, want %s via %s", got, otherCtxHost, SourceEnvContext)
+ }
+}
+
+func TestConfigContextIsResolvedByDigestPath(t *testing.T) {
+ home := t.TempDir()
+ writeContext(t, home, ctxName, ctxHostValue)
+ writeCurrentContext(t, home, ctxName)
+
+ got, err := ResolveEndpoint(fakeEnv(nil), home)
+ if err != nil {
+ t.Fatalf("resolve: %v", err)
+ }
+
+ if got.Host != ctxHostValue || got.Source != SourceConfigContext || got.Name != ctxName {
+ t.Fatalf("got %+v, want %s via %s", got, ctxHostValue, SourceConfigContext)
+ }
+}
+
+func TestDefaultContextFallsThroughToSockets(t *testing.T) {
+ home := t.TempDir()
+ writeCurrentContext(t, home, defaultContext)
+ runtimeDir := makeRootlessSocket(t, home)
+
+ got, err := ResolveEndpoint(fakeEnv(map[string]string{EnvXDGRuntime: runtimeDir}), home)
+ if err != nil {
+ t.Fatalf("resolve: %v", err)
+ }
+
+ if got.Source != SourceRootless {
+ t.Fatalf("got %+v, want the rootless socket", got)
+ }
+
+ want := filepath.Join(runtimeDir, socketBaseName)
+ if path, ok := got.SocketPath(); !ok || path != want {
+ t.Fatalf("SocketPath() = %q, %v; want %q", path, ok, want)
+ }
+}
+
+func TestAnUnresolvableNamedContextIsRefusedNotIgnored(t *testing.T) {
+ cases := map[string]func(t *testing.T, home string) func(string) string{
+ "named by DOCKER_CONTEXT": func(_ *testing.T, _ string) func(string) string {
+ return fakeEnv(map[string]string{EnvDockerContext: "a-context-with-no-metadata"})
+ },
+ "named by docker config": func(t *testing.T, home string) func(string) string {
+ writeCurrentContext(t, home, "a-context-with-no-metadata")
+ return fakeEnv(nil)
+ },
+ }
+
+ for name, setup := range cases {
+ t.Run(name, func(t *testing.T) {
+ home := t.TempDir()
+ env := setup(t, home)
+ makeRootlessSocket(t, home)
+
+ _, err := ResolveEndpoint(env, home)
+ if !errors.Is(err, ErrContextUnresolvable) {
+ t.Fatalf("got %v; an explicitly named context that cannot be resolved must refuse, "+
+ "never silently fall through to whatever daemon happens to be local", err)
+ }
+ })
+ }
+}
+
+func TestNoEndpointAnywhereIsReportedNotGuessed(t *testing.T) {
+ home := t.TempDir()
+
+ got, err := ResolveEndpoint(fakeEnv(nil), home)
+
+ if exists(DockerSocketPath) {
+ if err != nil || got.Source != SourceDefault {
+ t.Fatalf("got %+v (%v), want the default socket", got, err)
+ }
+ return
+ }
+
+ if !errors.Is(err, ErrNoEndpoint) {
+ t.Fatalf("got %+v (%v), want an explicit no-endpoint error", got, err)
+ }
+}
+
+func TestSocketPathRejectsNonUnixHosts(t *testing.T) {
+ e := Endpoint{Host: envHostValue, Source: SourceEnvHost}
+ if path, ok := e.SocketPath(); ok {
+ t.Fatalf("a tcp endpoint reported a socket path %q", path)
+ }
+}
+
+// An explicit DOCKER_CONTEXT=default selects the default endpoint. It is not
+// the same as leaving the variable unset, and khaos treated it as such.
+//
+// Measured against docker 29.1.3: with a saved current context of `ag-server`
+// and DOCKER_CONTEXT=default, the CLI used the LOCAL daemon. khaos used
+// ag-server. `khaos range` shells out to that CLI, so the two halves of one
+// command were acting on different machines.
+func TestAnExplicitDefaultContextSelectsTheLocalDaemon(t *testing.T) {
+ home := t.TempDir()
+ writeContext(t, home, "ag-server", "ssh://ag-server")
+ writeCurrentContext(t, home, "ag-server")
+
+ socket := t.TempDir()
+ runtime := func(name string) string {
+ if name == EnvXDGRuntime {
+ return socket
+ }
+ return ""
+ }
+
+ if err := os.WriteFile(filepath.Join(socket, socketBaseName), nil, 0o600); err != nil {
+ t.Fatalf("socket: %v", err)
+ }
+
+ // Control: with nothing set, the saved context wins.
+ saved, err := ResolveEndpoint(runtime, home)
+ if err != nil {
+ t.Fatalf("resolve: %v", err)
+ }
+
+ if saved.Name != "ag-server" {
+ t.Fatalf("with no env set khaos resolved %q, want the saved context ag-server", saved.Name)
+ }
+
+ withDefault := func(name string) string {
+ if name == EnvDockerContext {
+ return "default"
+ }
+ return runtime(name)
+ }
+
+ resolved, err := ResolveEndpoint(withDefault, home)
+ if err != nil {
+ t.Fatalf("resolve with DOCKER_CONTEXT=default: %v", err)
+ }
+
+ if resolved.Name == "ag-server" {
+ t.Fatal("DOCKER_CONTEXT=default resolved to the saved remote context; khaos would inject " +
+ "on another machine while `docker compose` acted locally")
+ }
+
+ if !strings.HasPrefix(resolved.Host, unixScheme) {
+ t.Fatalf("DOCKER_CONTEXT=default resolved to %q, want a local socket", resolved.Host)
+ }
+}
+
+// DOCKER_HOST still outranks DOCKER_CONTEXT, which is what the Docker CLI does.
+// Measured: DOCKER_HOST=tcp://127.0.0.1:1 with DOCKER_CONTEXT=ag-server made the
+// CLI try 127.0.0.1:1.
+func TestDockerHostStillOutranksDockerContext(t *testing.T) {
+ home := t.TempDir()
+ writeContext(t, home, "ag-server", "ssh://ag-server")
+
+ environ := func(name string) string {
+ switch name {
+ case EnvDockerHost:
+ return "tcp://127.0.0.1:1"
+ case EnvDockerContext:
+ return "ag-server"
+ }
+ return ""
+ }
+
+ resolved, err := ResolveEndpoint(environ, home)
+ if err != nil {
+ t.Fatalf("resolve: %v", err)
+ }
+
+ if resolved.Host != "tcp://127.0.0.1:1" {
+ t.Fatalf("resolved %q; the Docker CLI gives DOCKER_HOST precedence and khaos must match",
+ resolved.Host)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/refuse.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/refuse.go
new file mode 100644
index 00000000..1e19e9f1
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/refuse.go
@@ -0,0 +1,312 @@
+// ©AngelaMos | 2026
+// refuse.go
+
+package dockerx
+
+import (
+ "fmt"
+ "path/filepath"
+ "strings"
+)
+
+const (
+ refusalFormat = "refused %s: %s"
+ unconfinedValue = "unconfined"
+ noProject = "no compose project"
+ pathSeparator = "/"
+ rootPath = "/"
+)
+
+type Refusal struct {
+ Predicate string
+ Target string
+ Reason string
+ Fix string
+}
+
+func (r *Refusal) Error() string {
+ return fmt.Sprintf(refusalFormat, r.Target, r.Reason)
+}
+
+type Predicate struct {
+ Name string
+ Check func(Target, Guard) *Refusal
+}
+
+const (
+ PredicateNotInspected = "incomplete-inspection"
+ PredicateConsentMissing = "consent-label-missing"
+ PredicateConsentNotTrue = "consent-label-not-true"
+ PredicateIdentityChanged = "identity-changed"
+ PredicateSelfTarget = "self-target"
+ PredicateHostNetwork = "host-network-mode"
+ PredicateSharedNamespace = "shared-network-namespace"
+ PredicateNamespacePeer = "namespace-joined-by-peer"
+ PredicateHostNamespace = "host-namespace"
+ PredicatePrivileged = "privileged-container"
+ PredicateDangerousCap = "dangerous-capability"
+ PredicateDeviceAccess = "host-device-access"
+ PredicateUnconfined = "unconfined-security-profile"
+ PredicateSocketReachable = "docker-socket-reachable"
+ PredicateScopeMismatch = "outside-declared-scope"
+ PredicateNotRunning = "container-not-running"
+)
+
+var predicates = []Predicate{
+ {PredicateNotInspected, checkNotInspected},
+ {PredicateConsentMissing, checkConsentMissing},
+ {PredicateConsentNotTrue, checkConsentNotTrue},
+ {PredicateIdentityChanged, checkIdentityChanged},
+ {PredicateSelfTarget, checkSelfTarget},
+ {PredicateHostNetwork, checkHostNetwork},
+ {PredicateSharedNamespace, checkSharedNamespace},
+ {PredicateNamespacePeer, checkNamespacePeer},
+ {PredicateHostNamespace, checkHostNamespace},
+ {PredicatePrivileged, checkPrivileged},
+ {PredicateDangerousCap, checkDangerousCapability},
+ {PredicateDeviceAccess, checkDeviceAccess},
+ {PredicateUnconfined, checkUnconfined},
+ {PredicateSocketReachable, checkSocketReachable},
+ {PredicateScopeMismatch, checkScopeMismatch},
+ {PredicateNotRunning, checkNotRunning},
+}
+
+func Refuse(t Target, g Guard) *Refusal {
+ for _, p := range predicates {
+ if refusal := p.Check(t, g); refusal != nil {
+ return refusal
+ }
+ }
+ return nil
+}
+
+func refuse(predicate string, t Target, reason, fix string) *Refusal {
+ return &Refusal{Predicate: predicate, Target: t.Name, Reason: reason, Fix: fix}
+}
+
+func checkNotInspected(t Target, _ Guard) *Refusal {
+ if t.Inspected {
+ return nil
+ }
+ return refuse(PredicateNotInspected, t,
+ "this target was not built from a full container inspection, so the fields the safety checks read are unset rather than known-safe",
+ "resolve the target through khaos rather than constructing it from a cached or partial record")
+}
+
+func checkConsentMissing(t Target, _ Guard) *Refusal {
+ if _, ok := t.Labels[LabelConsent]; ok {
+ return nil
+ }
+ return refuse(PredicateConsentMissing, t,
+ "the container carries no "+LabelConsent+" label, so its owner never consented to fault injection",
+ "add `labels: [\""+LabelConsent+"="+ConsentValue+"\"]` to this service in your compose file and recreate it")
+}
+
+func checkConsentNotTrue(t Target, _ Guard) *Refusal {
+ value, ok := t.Labels[LabelConsent]
+ if !ok || value == ConsentValue {
+ return nil
+ }
+ return refuse(PredicateConsentNotTrue, t,
+ fmt.Sprintf("the %s label reads %q, and consent must be exactly %q", LabelConsent, value, ConsentValue),
+ "set "+LabelConsent+"="+ConsentValue+" exactly, or remove the label entirely")
+}
+
+func checkIdentityChanged(t Target, g Guard) *Refusal {
+ if g.ExpectedID == "" || t.ID == g.ExpectedID {
+ return nil
+ }
+ return refuse(PredicateIdentityChanged, t,
+ "the container was replaced since the plan was made: expected "+ShortID(g.ExpectedID)+", found "+t.Short(),
+ "re-run the plan so khaos resolves the container that exists now")
+}
+
+func checkSelfTarget(t Target, g Guard) *Refusal {
+ if g.SelfID == "" || !sameContainer(t.ID, g.SelfID) {
+ return nil
+ }
+ return refuse(PredicateSelfTarget, t,
+ "this is the khaos container itself, and a tool that injects faults into itself cannot roll them back",
+ "target a different container")
+}
+
+func checkHostNetwork(t Target, _ Guard) *Refusal {
+ if !strings.EqualFold(t.NetworkMode, NetworkModeHost) {
+ return nil
+ }
+ return refuse(PredicateHostNetwork, t,
+ "the container runs in host network mode, so its network is the host network and any fault would hit the machine",
+ "move the service onto a user-defined bridge network")
+}
+
+func checkSharedNamespace(t Target, _ Guard) *Refusal {
+ peer, ok := strings.CutPrefix(t.NetworkMode, NetworkModeContainerPfx)
+ if !ok {
+ return nil
+ }
+ return refuse(PredicateSharedNamespace, t,
+ "the container shares the network namespace of "+ShortID(peer)+", which has not consented, so a fault would reach it too",
+ "target "+ShortID(peer)+" directly once it carries the consent label")
+}
+
+func checkNamespacePeer(t Target, _ Guard) *Refusal {
+ if len(t.NamespacePeers) == 0 {
+ return nil
+ }
+ return refuse(PredicateNamespacePeer, t,
+ "the network namespace of this container is shared by "+strings.Join(t.NamespacePeers, ", ")+
+ ", which has not consented, so a fault here would reach it too",
+ "stop the sharing container, or give it the consent label so you are injecting into both deliberately")
+}
+
+func checkHostNamespace(t Target, _ Guard) *Refusal {
+ for _, ns := range []struct{ kind, mode string }{
+ {"process", t.PidMode},
+ {"IPC", t.IpcMode},
+ {"user", t.UsernsMode},
+ } {
+ if strings.EqualFold(ns.mode, ModeHost) {
+ return refuse(PredicateHostNamespace, t,
+ "the container shares the host "+ns.kind+" namespace, so it is not meaningfully isolated from the machine",
+ "remove the host "+ns.kind+" namespace setting from this service, or target a different one")
+ }
+ }
+ return nil
+}
+
+func checkPrivileged(t Target, _ Guard) *Refusal {
+ if !t.Privileged {
+ return nil
+ }
+ return refuse(PredicatePrivileged, t,
+ "the container is privileged, so it is not meaningfully isolated from the host",
+ "drop `privileged: true` from this service, or target a different one")
+}
+
+func checkDangerousCapability(t Target, _ Guard) *Refusal {
+ for _, capability := range t.Capabilities {
+ name := strings.TrimPrefix(strings.ToUpper(strings.TrimSpace(capability)), capPrefix)
+ if why, dangerous := dangerousCapabilities[name]; dangerous {
+ return refuse(PredicateDangerousCap, t,
+ "the container holds "+capability+", which grants "+why,
+ "remove "+capability+" from cap_add, or target a different service")
+ }
+ }
+ return nil
+}
+
+func checkDeviceAccess(t Target, _ Guard) *Refusal {
+ if len(t.Devices) == 0 {
+ return nil
+ }
+ return refuse(PredicateDeviceAccess, t,
+ "the container is given direct access to host device(s): "+strings.Join(t.Devices, ", "),
+ "remove the device passthrough, or target a different service")
+}
+
+func checkUnconfined(t Target, _ Guard) *Refusal {
+ for _, opt := range t.SecurityOpt {
+ if strings.Contains(strings.ToLower(opt), unconfinedValue) {
+ return refuse(PredicateUnconfined, t,
+ "the container runs with "+opt+", so its kernel confinement is disabled",
+ "remove that security_opt entry, or target a different service")
+ }
+ }
+ return nil
+}
+
+func checkSocketReachable(t Target, _ Guard) *Refusal {
+ for _, source := range t.BindSources {
+ if reason, reachable := socketReachableFrom(source); reachable {
+ return refuse(PredicateSocketReachable, t,
+ "the container bind-mounts "+source+", which "+reason+
+ ", and access to the Docker daemon is equivalent to root on this machine",
+ "remove that bind mount, or target a service that cannot reach the daemon")
+ }
+ }
+ return nil
+}
+
+func checkScopeMismatch(t Target, g Guard) *Refusal {
+ if g.Project == "" || t.Project == g.Project {
+ return nil
+ }
+ found := t.Project
+ if found == "" {
+ found = noProject
+ }
+ return refuse(PredicateScopeMismatch, t,
+ "the container belongs to "+found+" but khaos was scoped to "+g.Project,
+ "re-run with --project "+found+", or pick a target inside "+g.Project)
+}
+
+func checkNotRunning(t Target, _ Guard) *Refusal {
+ if t.Running {
+ return nil
+ }
+ return refuse(PredicateNotRunning, t,
+ "the container is not running, so there is no steady state to establish and no behavior to measure",
+ "start the container, then re-run")
+}
+
+func socketPaths() []string {
+ paths := []string{DockerSocketPath, runSocketPath}
+ if runtimeDir := xdgRuntimeDir(); runtimeDir != "" {
+ paths = append(paths, filepath.Join(runtimeDir, socketBaseName))
+ }
+ return paths
+}
+
+func socketReachableFrom(source string) (string, bool) {
+ resolved := resolvePath(source)
+
+ if overlaps(resolved, daemonStatePath) {
+ return "exposes the Docker daemon state directory", true
+ }
+
+ for _, socket := range socketPaths() {
+ if resolved == socket || resolved == resolvePath(socket) {
+ return "is the Docker socket", true
+ }
+ if isAncestor(resolved, socket) || isAncestor(resolved, resolvePath(socket)) {
+ return "contains the Docker socket at " + socket, true
+ }
+ }
+
+ if filepath.Base(resolved) == socketBaseName {
+ return "is named like a Docker socket", true
+ }
+
+ return "", false
+}
+
+func resolvePath(path string) string {
+ if path == "" {
+ return path
+ }
+ if resolved, err := filepath.EvalSymlinks(path); err == nil {
+ return filepath.Clean(resolved)
+ }
+ return filepath.Clean(path)
+}
+
+func isAncestor(dir, path string) bool {
+ if dir == rootPath {
+ return true
+ }
+ return strings.HasPrefix(path, dir+pathSeparator)
+}
+
+func overlaps(a, b string) bool {
+ return a == b || isAncestor(a, b) || isAncestor(b, a)
+}
+
+func sameContainer(id, other string) bool {
+ if id == "" || other == "" {
+ return false
+ }
+ if len(id) < len(other) {
+ id, other = other, id
+ }
+ return strings.HasPrefix(id, other)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/refuse_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/refuse_test.go
new file mode 100644
index 00000000..53d7cab1
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/refuse_test.go
@@ -0,0 +1,299 @@
+// ©AngelaMos | 2026
+// refuse_test.go
+
+package dockerx
+
+import (
+ "strings"
+ "testing"
+)
+
+const (
+ acceptableID = "9f1c2b3a4d5e6f708192a3b4c5d6e7f88192a3b4c5d6e7f89f1c2b3a4d5e6f70"
+ khaosOwnID = "0011223344556677889900aabbccddee0011223344556677889900aabbccddee"
+ peerID = "ffeeddccbbaa00998877665544332211ffeeddccbbaa009988776655443322ff"
+ project = "shop"
+)
+
+func acceptable() (Target, Guard) {
+ return Target{
+ ID: acceptableID,
+ Name: "shop-api-1",
+ Image: "shop/api:latest",
+ Project: project,
+ Service: "api",
+ Labels: map[string]string{LabelConsent: ConsentValue, LabelComposeProject: project},
+ Running: true,
+ NetworkMode: "shop_default",
+ BindSources: []string{"/srv/shop/config"},
+ Inspected: true,
+ }, Guard{
+ Project: project,
+ SelfID: khaosOwnID,
+ ExpectedID: acceptableID,
+ }
+}
+
+type control struct {
+ predicate string
+ mutate func(*Target, *Guard)
+}
+
+func controls() []control {
+ return []control{
+ {PredicateNotInspected, func(t *Target, _ *Guard) { t.Inspected = false }},
+ {PredicateConsentMissing, func(t *Target, _ *Guard) { delete(t.Labels, LabelConsent) }},
+ {PredicateConsentNotTrue, func(t *Target, _ *Guard) { t.Labels[LabelConsent] = "yes" }},
+ {PredicateIdentityChanged, func(_ *Target, g *Guard) { g.ExpectedID = peerID }},
+ {PredicateSelfTarget, func(t *Target, g *Guard) { g.SelfID = t.ID }},
+ {PredicateHostNetwork, func(t *Target, _ *Guard) { t.NetworkMode = NetworkModeHost }},
+ {PredicateSharedNamespace, func(t *Target, _ *Guard) {
+ t.NetworkMode = NetworkModeContainerPfx + peerID
+ }},
+ {PredicateNamespacePeer, func(t *Target, _ *Guard) {
+ t.NamespacePeers = []string{"sidecar-1"}
+ }},
+ {PredicateHostNamespace, func(t *Target, _ *Guard) { t.PidMode = ModeHost }},
+ {PredicatePrivileged, func(t *Target, _ *Guard) { t.Privileged = true }},
+ {PredicateDangerousCap, func(t *Target, _ *Guard) { t.Capabilities = []string{"NET_ADMIN"} }},
+ {PredicateDeviceAccess, func(t *Target, _ *Guard) { t.Devices = []string{"/dev/sda"} }},
+ {PredicateUnconfined, func(t *Target, _ *Guard) {
+ t.SecurityOpt = []string{"seccomp=unconfined"}
+ }},
+ {PredicateSocketReachable, func(t *Target, _ *Guard) {
+ t.BindSources = append(t.BindSources, DockerSocketPath)
+ }},
+ {PredicateScopeMismatch, func(t *Target, _ *Guard) { t.Project = "someone-elses-stack" }},
+ {PredicateNotRunning, func(t *Target, _ *Guard) { t.Running = false }},
+ }
+}
+
+func TestAcceptableTargetIsAccepted(t *testing.T) {
+ target, guard := acceptable()
+
+ if refusal := Refuse(target, guard); refusal != nil {
+ t.Fatalf("the control target was refused by %s (%s), so every refusal test below is vacuous",
+ refusal.Predicate, refusal.Reason)
+ }
+}
+
+func TestEachControlFiresExactlyOnePredicate(t *testing.T) {
+ for _, c := range controls() {
+ t.Run(c.predicate, func(t *testing.T) {
+ target, guard := acceptable()
+ c.mutate(&target, &guard)
+
+ var fired []string
+ for _, p := range predicates {
+ if p.Check(target, guard) != nil {
+ fired = append(fired, p.Name)
+ }
+ }
+
+ switch len(fired) {
+ case 0:
+ t.Fatalf("%s accepted a target that breaks exactly its own invariant", c.predicate)
+ case 1:
+ if fired[0] != c.predicate {
+ t.Fatalf("control for %s fired %s instead", c.predicate, fired[0])
+ }
+ default:
+ t.Fatalf("control for %s fired %d predicates (%s); they alibi each other, so gutting any one stays green",
+ c.predicate, len(fired), strings.Join(fired, ", "))
+ }
+ })
+ }
+}
+
+func TestEveryPredicateHasAControl(t *testing.T) {
+ covered := make(map[string]bool, len(controls()))
+ for _, c := range controls() {
+ covered[c.predicate] = true
+ }
+
+ for _, p := range predicates {
+ if !covered[p.Name] {
+ t.Fatalf("predicate %s has no negative control, so nothing proves it ever refuses anything", p.Name)
+ }
+ }
+
+ if len(covered) != len(predicates) {
+ t.Fatalf("%d controls for %d predicates", len(covered), len(predicates))
+ }
+}
+
+func TestEveryRefusalNamesAReasonAndAFix(t *testing.T) {
+ for _, c := range controls() {
+ t.Run(c.predicate, func(t *testing.T) {
+ target, guard := acceptable()
+ c.mutate(&target, &guard)
+
+ refusal := Refuse(target, guard)
+ if refusal == nil {
+ t.Fatalf("%s did not refuse its control", c.predicate)
+ }
+
+ if strings.TrimSpace(refusal.Reason) == "" {
+ t.Fatalf("%s refused without giving a reason", c.predicate)
+ }
+
+ if strings.TrimSpace(refusal.Fix) == "" {
+ t.Fatalf("%s refused without telling the operator how to fix it", c.predicate)
+ }
+ })
+ }
+}
+
+func TestConsentMustBeExactlyTrue(t *testing.T) {
+ cases := []struct {
+ value string
+ consented bool
+ }{
+ {"true", true},
+ {"TRUE", false},
+ {"True", false},
+ {"1", false},
+ {"yes", false},
+ {"false", false},
+ {" true", false},
+ {"true ", false},
+ {"", false},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.value, func(t *testing.T) {
+ target, guard := acceptable()
+ target.Labels[LabelConsent] = tc.value
+
+ refused := Refuse(target, guard) != nil
+ if refused == tc.consented {
+ t.Fatalf("consent value %q: refused=%v, want refused=%v", tc.value, refused, !tc.consented)
+ }
+ })
+ }
+}
+
+func TestDangerousCapabilitiesAreRefusedInEverySpelling(t *testing.T) {
+ spellings := []string{
+ "NET_ADMIN", "net_admin", "CAP_NET_ADMIN", "cap_sys_admin", "ALL",
+ "SYS_MODULE", "CAP_SYS_PTRACE", "DAC_READ_SEARCH", "SYS_RAWIO", "BPF", "NET_RAW",
+ }
+
+ for _, value := range spellings {
+ t.Run(value, func(t *testing.T) {
+ target, guard := acceptable()
+ target.Capabilities = []string{value}
+
+ refusal := Refuse(target, guard)
+ if refusal == nil || refusal.Predicate != PredicateDangerousCap {
+ t.Fatalf("capability %q was not refused as dangerous", value)
+ }
+ })
+ }
+}
+
+func TestHarmlessCapabilitiesAreAllowed(t *testing.T) {
+ target, guard := acceptable()
+ target.Capabilities = []string{"CHOWN", "SETUID", "KILL"}
+
+ if refusal := Refuse(target, guard); refusal != nil {
+ t.Fatalf("harmless capabilities were refused by %s", refusal.Predicate)
+ }
+}
+
+func TestSocketIsRefusedThroughEveryReachablePath(t *testing.T) {
+ reachable := []string{
+ DockerSocketPath,
+ "/run/docker.sock",
+ "/var/run",
+ "/run",
+ "/var",
+ "/",
+ "/var/lib/docker",
+ "/var/lib/docker/containers",
+ "/home/operator/dsock/docker.sock",
+ }
+
+ for _, source := range reachable {
+ t.Run(source, func(t *testing.T) {
+ target, guard := acceptable()
+ target.BindSources = []string{source}
+
+ refusal := Refuse(target, guard)
+ if refusal == nil || refusal.Predicate != PredicateSocketReachable {
+ t.Fatalf("a bind mount of %q reaches the Docker daemon and was not refused", source)
+ }
+ })
+ }
+}
+
+func TestOrdinaryBindMountsAreAllowed(t *testing.T) {
+ harmless := []string{"/srv/app/config", "/etc/myapp", "/var/log/myapp", "/opt/data"}
+
+ for _, source := range harmless {
+ t.Run(source, func(t *testing.T) {
+ target, guard := acceptable()
+ target.BindSources = []string{source}
+
+ if refusal := Refuse(target, guard); refusal != nil {
+ t.Fatalf("harmless mount %q was refused by %s", source, refusal.Predicate)
+ }
+ })
+ }
+}
+
+func TestHostNamespaceIsRefusedForEveryKind(t *testing.T) {
+ kinds := map[string]func(*Target){
+ "pid": func(t *Target) { t.PidMode = ModeHost },
+ "ipc": func(t *Target) { t.IpcMode = ModeHost },
+ "userns": func(t *Target) { t.UsernsMode = ModeHost },
+ }
+
+ for name, apply := range kinds {
+ t.Run(name, func(t *testing.T) {
+ target, guard := acceptable()
+ apply(&target)
+
+ refusal := Refuse(target, guard)
+ if refusal == nil || refusal.Predicate != PredicateHostNamespace {
+ t.Fatalf("host %s namespace was not refused", name)
+ }
+ })
+ }
+}
+
+func TestSelfTargetMatchesShortAndFullIDs(t *testing.T) {
+ target, guard := acceptable()
+ guard.SelfID = target.ID[:ShortIDLen]
+
+ refusal := Refuse(target, guard)
+ if refusal == nil || refusal.Predicate != PredicateSelfTarget {
+ t.Fatal("a short self ID did not match the full container ID")
+ }
+}
+
+func TestEmptyGuardStillEnforcesTargetProperties(t *testing.T) {
+ target, _ := acceptable()
+ target.Project = "anything"
+
+ if refusal := Refuse(target, Guard{}); refusal != nil {
+ t.Fatalf("an empty guard refused a consenting target via %s", refusal.Predicate)
+ }
+
+ target.Privileged = true
+ if refusal := Refuse(target, Guard{}); refusal == nil {
+ t.Fatal("an empty guard stopped enforcing container properties")
+ }
+}
+
+func TestZeroValueTargetIsRefused(t *testing.T) {
+ refusal := Refuse(Target{}, Guard{})
+ if refusal == nil {
+ t.Fatal("a zero-value Target was accepted; a partially built target must never pass")
+ }
+
+ if refusal.Predicate != PredicateNotInspected {
+ t.Fatalf("a zero-value Target was refused by %s rather than %s, so the completeness check is not first",
+ refusal.Predicate, PredicateNotInspected)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/self.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/self.go
new file mode 100644
index 00000000..a12d9e7b
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/self.go
@@ -0,0 +1,33 @@
+// ©AngelaMos | 2026
+// self.go
+
+package dockerx
+
+import (
+ "os"
+ "regexp"
+)
+
+const (
+ procMountInfo = "/proc/self/mountinfo"
+ procCgroup = "/proc/self/cgroup"
+)
+
+var containerIDPattern = regexp.MustCompile(`[0-9a-f]{64}`)
+
+func SelfContainerID() string {
+ for _, path := range []string{procMountInfo, procCgroup} {
+ if id := scanForContainerID(path); id != "" {
+ return id
+ }
+ }
+ return ""
+}
+
+func scanForContainerID(path string) string {
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ return ""
+ }
+ return containerIDPattern.FindString(string(raw))
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/target.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/target.go
new file mode 100644
index 00000000..f9e99f19
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/target.go
@@ -0,0 +1,87 @@
+// ©AngelaMos | 2026
+// target.go
+
+package dockerx
+
+const (
+ LabelConsent = "khaos.target"
+ ConsentValue = "true"
+
+ LabelComposeProject = "com.docker.compose.project"
+ LabelComposeService = "com.docker.compose.service"
+)
+
+const (
+ NetworkModeHost = "host"
+ NetworkModeContainerPfx = "container:"
+ ModeHost = "host"
+)
+
+const capPrefix = "CAP_"
+
+var dangerousCapabilities = map[string]string{
+ "ALL": "every capability the kernel offers",
+ "SYS_ADMIN": "mount, namespace and cgroup control, which is close to root on the host",
+ "SYS_MODULE": "loading kernel modules, which is root on the host",
+ "SYS_RAWIO": "raw I/O port and memory access",
+ "SYS_PTRACE": "tracing and injecting into other processes",
+ "DAC_READ_SEARCH": "reading any file on the host regardless of permissions",
+ "BPF": "loading BPF programs into the kernel",
+ "NET_ADMIN": "reconfiguring networking beyond its own namespace",
+ "NET_RAW": "raw sockets, which also lets it observe the probe traffic khaos measures",
+}
+
+const (
+ DockerSocketPath = "/var/run/docker.sock"
+ runSocketPath = "/run/docker.sock"
+ daemonStatePath = "/var/lib/docker"
+ socketBaseName = "docker.sock"
+)
+
+const ShortIDLen = 12
+
+type Target struct {
+ ID string
+ Name string
+ Image string
+ Project string
+ Service string
+ Labels map[string]string
+ Running bool
+ NetworkMode string
+ PidMode string
+ IpcMode string
+ UsernsMode string
+ Privileged bool
+ Capabilities []string
+ Devices []string
+ SecurityOpt []string
+ BindSources []string
+ NamespacePeers []string
+ Inspected bool
+}
+
+func (t Target) Consented() bool {
+ return t.Labels[LabelConsent] == ConsentValue
+}
+
+func (t Target) Short() string {
+ return ShortID(t.ID)
+}
+
+func ShortID(id string) string {
+ if len(id) > ShortIDLen {
+ return id[:ShortIDLen]
+ }
+ return id
+}
+
+type Filter struct {
+ Project string
+}
+
+type Guard struct {
+ Project string
+ SelfID string
+ ExpectedID string
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
new file mode 100644
index 00000000..66b7adfe
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
@@ -0,0 +1,240 @@
+// ©AngelaMos | 2026
+// catalog.go
+
+package experiment
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ FamilyNetwork = "network"
+
+ NameDependencyOutage = "network.dependency_outage"
+
+ // A credential no correct verifier can accept: it is not a signed token in
+ // any format, so a target that honours it is not verifying anything.
+ DefaultInvalidToken = "khaos.invalid.credential.not.a.signed.token"
+
+ probeValidName = "valid-credential"
+ probeInvalidName = "invalid-credential"
+ probeRevokedName = "revoked-credential"
+
+ nameSep = ", "
+ errUnknown = "%w: %q (known experiments: %s)"
+ errStray = "%w: %s reads %s, which it does not declare; declare them in its Params or stop reading them"
+)
+
+var ErrUnknownExperiment = errors.New("no experiment is registered under that name")
+
+var ErrStrayParam = errors.New("an experiment read a parameter it never declared")
+
+// Shape is the part of an experiment that depends on the operator's parameters.
+// Identity, polarity, fault kind and timing all come from the declaration, so a
+// builder cannot disagree with the catalogue about what it is or forget to carry
+// something universal through.
+type Shape struct {
+ Condition string
+ Target string
+ Probes probe.Pair
+
+ // ProxyFault is set only by an experiment that injects through khaos's own
+ // fault proxy. The experiment owns the magnitude because the operator declares
+ // it as a parameter, and the CLI must not have to infer a fault from a kind.
+ ProxyFault *proxy.Fault
+
+ // ProxyListen and ProxyUpstream are where the run binds the proxy and what
+ // it forwards to. They come from declared parameters, so an operator can
+ // point a proxy experiment at their own stack.
+ ProxyListen string
+ ProxyUpstream string
+
+ // IdP and IdPTrap are set only by an experiment that arms khaos's own
+ // identity provider, and the provider is built by the builder because its
+ // probes mint out of it.
+ IdP *idp.Provider
+ IdPTrap *idp.Trap
+
+ // Defect names what khaos did to the credential, for the verdict's wording.
+ Defect string
+
+ // HeldCaveat qualifies a HELD verdict that covers less than it appears to.
+ HeldCaveat string
+
+ // Consulted reads how many times the target has reached the endpoint this
+ // experiment faults, and ConsultedWhat names it. Together they are the
+ // liveness guard for a fault whose landing the probe streams cannot see.
+ Consulted func() int64
+ ConsultedWhat string
+
+ // IdleFor asks the engine for the idleness discriminator.
+ IdleFor time.Duration
+
+ // Act is the change khaos makes at an authority once the target can no longer
+ // reach it, and ActDetail is what the operator is told about it.
+ Act func(context.Context) error
+ ActDetail string
+}
+
+type Definition struct {
+ Name string
+ Family string
+ Summary string
+ Hypothesis Hypothesis
+ Polarity Polarity
+ FaultKind string
+ Params []Param
+ Build func(*Params) (Shape, error)
+}
+
+const (
+ outageSummary = "take a dependency away and see whether the control in front of it fails closed or fails open"
+ outageResponse = "refusing credentials it would have refused anyway, rather than letting them through"
+ outageCondition = "%s becoming unreachable"
+ outageGeneric = "a dependency of the target becoming unreachable"
+
+ usageTarget = "the consenting container whose outage you want to survive"
+ usageURL = "the endpoint khaos probes to see how the target behaves"
+ usageToken = "a credential the target should accept"
+ usageInvalidToken = "a credential the target should refuse; this is the negative control"
+)
+
+var outagePolarity = Polarity{
+ Defects: []Direction{Permitted},
+ Rationale: "a control that stops rejecting invalid credentials when its dependency is down has stopped being a control",
+}
+
+var outageParams = []Param{
+ {Name: ParamTarget, Usage: usageTarget, Required: true},
+ {Name: ParamURL, Usage: usageURL, Required: true},
+ {Name: ParamToken, Usage: usageToken, Required: true},
+ {Name: ParamInvalidToken, Usage: usageInvalidToken, Default: DefaultInvalidToken},
+}
+
+func probePair(params *Params) probe.Pair {
+ url := params.Get(ParamURL)
+
+ return probe.Pair{
+ Valid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeValidName,
+ URL: url,
+ Token: params.Get(ParamToken),
+ Credential: probe.Valid,
+ }),
+ Invalid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeInvalidName,
+ URL: url,
+ Token: params.Get(ParamInvalidToken),
+ Credential: probe.Invalid,
+ Unique: true,
+ }),
+ }
+}
+
+func buildDependencyOutage(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+
+ return Shape{
+ Condition: fmt.Sprintf(outageCondition, target),
+ Target: target,
+ Probes: probePair(params),
+ }, nil
+}
+
+var catalog = map[string]Definition{
+ NameDependencyOutage: {
+ Name: NameDependencyOutage,
+ Family: FamilyNetwork,
+ Summary: outageSummary,
+ Hypothesis: Hypothesis{Condition: outageGeneric, Response: outageResponse},
+ Polarity: outagePolarity,
+ FaultKind: inject.KindPause,
+ Params: outageParams,
+ Build: buildDependencyOutage,
+ },
+}
+
+func Names() []string {
+ names := make([]string, 0, len(catalog))
+ for name := range catalog {
+ names = append(names, name)
+ }
+ slices.Sort(names)
+ return names
+}
+
+func Catalog() []Definition {
+ out := make([]Definition, 0, len(catalog))
+ for _, name := range Names() {
+ out = append(out, catalog[name])
+ }
+ return out
+}
+
+func Lookup(name string) (Definition, error) {
+ definition, ok := catalog[name]
+ if !ok {
+ return Definition{}, fmt.Errorf(errUnknown, ErrUnknownExperiment, name, strings.Join(Names(), nameSep))
+ }
+ return definition, nil
+}
+
+func Build(name string, supplied map[string]string, timing Timing) (Experiment, error) {
+ definition, err := Lookup(name)
+ if err != nil {
+ return Experiment{}, err
+ }
+
+ params, err := Bind(definition.Params, supplied)
+ if err != nil {
+ return Experiment{}, err
+ }
+
+ shape, err := definition.Build(params)
+ if err != nil {
+ return Experiment{}, err
+ }
+
+ if stray := params.strayReads(); len(stray) > 0 {
+ return Experiment{}, fmt.Errorf(errStray, ErrStrayParam, name, strings.Join(stray, nameSep))
+ }
+
+ spec := Experiment{
+ Name: definition.Name,
+ Family: definition.Family,
+ Summary: definition.Summary,
+ Hypothesis: definition.Hypothesis,
+ Polarity: definition.Polarity,
+ Fault: Fault{Kind: definition.FaultKind, Target: shape.Target},
+ Probes: shape.Probes,
+ ProxyFault: shape.ProxyFault,
+ ProxyListen: shape.ProxyListen,
+ ProxyUpstream: shape.ProxyUpstream,
+ IdP: shape.IdP,
+ IdPTrap: shape.IdPTrap,
+ Defect: shape.Defect,
+ HeldCaveat: shape.HeldCaveat,
+ Consulted: shape.Consulted,
+ ConsultedWhat: shape.ConsultedWhat,
+ IdleFor: shape.IdleFor,
+ Act: shape.Act,
+ ActDetail: shape.ActDetail,
+ Timing: timing,
+ }
+
+ if shape.Condition != "" {
+ spec.Hypothesis.Condition = shape.Condition
+ }
+
+ return spec, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
new file mode 100644
index 00000000..75e9d3e1
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
@@ -0,0 +1,304 @@
+// ©AngelaMos | 2026
+// engine.go
+
+package experiment
+
+import (
+ "context"
+ "errors"
+ "math/rand/v2"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+type Stage string
+
+const (
+ StageBaseline Stage = "baseline"
+ StageInject Stage = "inject"
+ StageConfirm Stage = "confirm"
+ StageObserve Stage = "observe"
+ StageAct Stage = "act"
+ StageIdle Stage = "idle"
+ StageResume Stage = "resume"
+ StageRevert Stage = "revert"
+ StageRecover Stage = "recover"
+ StageVerdict Stage = "verdict"
+)
+
+const (
+ detailBaseline = "establishing steady state"
+ detailObserve = "measuring under fault"
+ detailRevert = "removing the fault"
+ detailRecover = "confirming the target came back"
+ detailInjecting = "%s on %s"
+ detailConfirmed = "fault confirmed by %s"
+ detailUnconfirm = "the fault was not confirmed, aborting before measuring"
+ detailVerdict = "%s / %s"
+ detailActFailed = "the mid-run action failed, so nothing measured after it means anything: %v"
+ detailIdle = "a credential leaked; pausing traffic for %s to see whether the cache is being re-armed"
+ detailResume = "resuming traffic with the fault still applied"
+
+ // Jitter breaks aliasing against periodic target behaviour: cache TTLs,
+ // token refresh and crons are exactly what these experiments measure, and
+ // a probe locked to a fixed period can sit in phase with them.
+ jitterFraction = 0.2
+)
+
+type Event struct {
+ Stage Stage
+ At time.Time
+ Detail string
+}
+
+type Reporter func(Event)
+
+// Injector is whatever can apply, confirm and undo the fault. The engine never
+// talks to Docker, so the whole lifecycle is testable without a daemon.
+type Injector interface {
+ Inject(ctx context.Context, kind, target string) error
+ Confirm(ctx context.Context) (string, error)
+ Revert(ctx context.Context) error
+}
+
+var ErrNoProbePair = errors.New("an experiment needs both a valid and an invalid probe; the invalid one is the negative control that makes the verdict mean anything")
+
+type Engine struct {
+ Injector Injector
+ Report Reporter
+ Now func() time.Time
+}
+
+func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
+ if !spec.Probes.Complete() {
+ return Result{}, ErrNoProbePair
+ }
+
+ timing := spec.Timing.withDefaults()
+ result := Result{
+ Experiment: spec.Name,
+ Hypothesis: spec.Hypothesis.String(),
+ StartedAt: e.now(),
+ }
+
+ observation := Observation{
+ ForgedBaseline: spec.Probes.ForgedBaseline,
+ MintedByKhaos: spec.IdP != nil,
+ FaultFor: timing.FaultFor,
+ Defect: spec.Defect,
+ HeldCaveat: spec.HeldCaveat,
+ }
+
+ e.emit(StageBaseline, detailBaseline)
+
+ // Measured across the BASELINE, where the target is healthy and whatever it
+ // normally does it is doing. A delta of zero there means the target never
+ // used the dependency this experiment is about to break.
+ consultedBefore := e.consulted(spec)
+
+ observation.Baseline = e.sample(ctx, spec.Probes, timing.BaselineFor, timing.SampleInterval)
+
+ observation.ConsultedWhat = spec.ConsultedWhat
+ observation.Consulted = e.consulted(spec) - consultedBefore
+
+ if ctx.Err() != nil {
+ return e.conclude(result, markInterrupted(observation), spec.Polarity, nil)
+ }
+
+ if _, unmet := baselineUnmet(observation.Baseline, observation.ForgedBaseline, observation.MintedByKhaos); unmet {
+ return e.conclude(result, observation, spec.Polarity, nil)
+ }
+
+ e.emit(StageInject, sprintf(detailInjecting, spec.Fault.Kind, spec.Fault.Target))
+
+ if err := e.Injector.Inject(ctx, spec.Fault.Kind, spec.Fault.Target); err != nil {
+ revertErr := e.Injector.Revert(context.WithoutCancel(ctx))
+ observation.RollbackFailed = revertErr != nil
+ result.Observation = observation
+ result.Verdict = Verdict{Epistemic: Errored, Direction: Undetermined, Reasons: []string{err.Error()}}
+ result.Duration = e.now().Sub(result.StartedAt)
+ // Joined, not discarded. The caller maps the rollback failure to its own
+ // exit code, and keeping only a boolean here meant the CLI returned this
+ // error first and never reached the boolean.
+ return result, errors.Join(err, revertErr)
+ }
+
+ method, confirmErr := e.Injector.Confirm(ctx)
+ observation.FaultConfirmed = confirmErr == nil && method != ""
+ observation.ConfirmMethod = method
+
+ if !observation.FaultConfirmed {
+ e.emit(StageConfirm, detailUnconfirm)
+ revertErr := e.Injector.Revert(context.WithoutCancel(ctx))
+ observation.RollbackFailed = revertErr != nil
+ return e.conclude(result, observation, spec.Polarity, errors.Join(confirmErr, revertErr))
+ }
+
+ e.emit(StageConfirm, sprintf(detailConfirmed, method))
+
+ // The mid-run action happens AFTER the fault is confirmed and before anything
+ // is measured under it, and that ordering is forced rather than convenient.
+ // Revoking a credential while the authority is still reachable makes the
+ // target cache a DENY, which a correct stale-serving control then serves
+ // correctly; the bug only exists when the cache holds an ALLOW at the moment
+ // the authority goes away. So the revocation has to be real and unreachable,
+ // which means it happens here or not at all.
+ if spec.Act != nil {
+ e.emit(StageAct, spec.ActDetail)
+
+ if err := spec.Act(ctx); err != nil {
+ e.emit(StageAct, sprintf(detailActFailed, err))
+ observation.ActFailed = true
+ revertErr := e.Injector.Revert(context.WithoutCancel(ctx))
+ observation.RollbackFailed = revertErr != nil
+ return e.conclude(result, observation, spec.Polarity, errors.Join(err, revertErr))
+ }
+
+ observation.Acted = true
+ }
+
+ e.emit(StageObserve, detailObserve)
+ observation.Fault = e.sample(ctx, spec.Probes, timing.FaultFor, timing.SampleInterval)
+
+ observation = e.discriminate(ctx, spec, observation, timing)
+
+ e.emit(StageRevert, detailRevert)
+ revertErr := e.Injector.Revert(context.WithoutCancel(ctx))
+ observation.RollbackFailed = revertErr != nil
+
+ e.emit(StageRecover, detailRecover)
+ observation.Recovery = e.sample(context.WithoutCancel(ctx), spec.Probes, timing.RecoveryFor, timing.SampleInterval)
+
+ if ctx.Err() != nil {
+ observation = markInterrupted(observation)
+ }
+
+ return e.conclude(result, observation, spec.Polarity, revertErr)
+}
+
+// discriminate is the idleness phase: stop sending traffic for longer than any
+// plausible cache lifetime, with the fault still applied, then resume and measure
+// again.
+//
+// It runs ONLY when the fault window actually leaked. The two causes it separates
+// are both causes of a PERMITTED credential, so on a target that held there is
+// nothing to diagnose and the pause would spend a minute of the operator's run
+// learning what khaos already knows. It also runs only when the experiment asks
+// for it, because the pause is meaningless without a cache lifetime to outlast
+// and khaos cannot invent one.
+func (e Engine) discriminate(
+ ctx context.Context,
+ spec Experiment,
+ observation Observation,
+ timing Timing,
+) Observation {
+ if spec.IdleFor <= 0 || ctx.Err() != nil {
+ return observation
+ }
+
+ if !Leaking(observation) {
+ return observation
+ }
+
+ e.emit(StageIdle, sprintf(detailIdle, spec.IdleFor))
+
+ timer := time.NewTimer(spec.IdleFor)
+ defer timer.Stop()
+
+ select {
+ case <-ctx.Done():
+ return observation
+ case <-timer.C:
+ }
+
+ e.emit(StageResume, detailResume)
+
+ observation.IdleFor = spec.IdleFor
+ observation.Resumed = e.sample(ctx, spec.Probes, timing.FaultFor, timing.SampleInterval)
+
+ return observation
+}
+
+func (e Engine) conclude(result Result, observation Observation, polarity Polarity, err error) (Result, error) {
+ result.Observation = observation
+ result.Verdict = Judge(observation, polarity)
+ result.Duration = e.now().Sub(result.StartedAt)
+
+ e.emit(StageVerdict, sprintf(detailVerdict, result.Verdict.Epistemic, result.Verdict.Direction))
+
+ return result, err
+}
+
+// consulted reads the experiment's endpoint counter, or zero when it declares
+// none.
+func (e Engine) consulted(spec Experiment) int64 {
+ if spec.Consulted == nil {
+ return 0
+ }
+ return spec.Consulted()
+}
+
+func markInterrupted(observation Observation) Observation {
+ observation.Interrupted = true
+ return observation
+}
+
+// sample fires both probes on a true period: the time the probes themselves
+// take is subtracted from the wait, so a slow target reduces latency headroom
+// rather than silently collapsing the sample count below the floor a verdict
+// needs.
+func (e Engine) sample(ctx context.Context, pair probe.Pair, duration, interval time.Duration) probe.Window {
+ window := probe.Window{}
+ deadline := e.now().Add(duration)
+
+ for {
+ started := e.now()
+
+ for _, p := range pair.All() {
+ window.Record(p.Run(ctx))
+ }
+
+ if !e.now().Before(deadline) || ctx.Err() != nil {
+ return window
+ }
+
+ wait := interval - e.now().Sub(started) + jitter(interval)
+ if remaining := deadline.Sub(e.now()); wait > remaining {
+ wait = remaining
+ }
+ if wait <= 0 {
+ continue
+ }
+
+ timer := time.NewTimer(wait)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return window
+ case <-timer.C:
+ }
+ }
+}
+
+func jitter(interval time.Duration) time.Duration {
+ span := float64(interval) * jitterFraction
+ if span <= 0 {
+ return 0
+ }
+ return time.Duration(rand.Float64() * span)
+}
+
+func (e Engine) now() time.Time {
+ if e.Now != nil {
+ return e.Now()
+ }
+ return time.Now().UTC()
+}
+
+func (e Engine) emit(stage Stage, detail string) {
+ if e.Report == nil {
+ return
+ }
+ e.Report(Event{Stage: stage, At: e.now(), Detail: detail})
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine_test.go
new file mode 100644
index 00000000..0de31c58
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine_test.go
@@ -0,0 +1,268 @@
+// ©AngelaMos | 2026
+// engine_test.go
+
+package experiment
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+type scriptedProbe struct {
+ credential probe.Credential
+ before probe.Disposition
+ during probe.Disposition
+ faulted *bool
+ calls *int
+}
+
+func (p scriptedProbe) Name() string { return "scripted" }
+func (p scriptedProbe) Describe() string { return "a scripted probe" }
+func (p scriptedProbe) Credential() probe.Credential { return p.credential }
+
+func (p scriptedProbe) Run(context.Context) probe.Outcome {
+ *p.calls++
+
+ disposition := p.before
+ if *p.faulted {
+ disposition = p.during
+ }
+
+ return probe.Outcome{
+ At: time.Now().UTC(),
+ Credential: p.credential,
+ Disposition: disposition,
+ }
+}
+
+type scriptedInjector struct {
+ faulted *bool
+ injectErr error
+ revertErr error
+ confirmErr error
+ unconfirmable bool
+ injects int
+ reverts int
+}
+
+func (i *scriptedInjector) Inject(context.Context, string, string) error {
+ i.injects++
+ if i.injectErr != nil {
+ return i.injectErr
+ }
+ *i.faulted = true
+ return nil
+}
+
+func (i *scriptedInjector) Confirm(context.Context) (string, error) {
+ if i.confirmErr != nil {
+ return "", i.confirmErr
+ }
+ if i.unconfirmable {
+ return "", nil
+ }
+ return "a scripted confirmation", nil
+}
+
+func (i *scriptedInjector) Revert(context.Context) error {
+ i.reverts++
+ *i.faulted = false
+ return i.revertErr
+}
+
+func quickTiming() Timing {
+ return Timing{
+ SampleInterval: time.Millisecond,
+ BaselineFor: 10 * time.Millisecond,
+ FaultFor: 10 * time.Millisecond,
+ RecoveryFor: 5 * time.Millisecond,
+ }
+}
+
+func spec(valid, invalid probe.Probe) Experiment {
+ return Experiment{
+ Name: "test.experiment",
+ Hypothesis: Hypothesis{Condition: "a dependency outage", Response: "a refusal"},
+ Fault: Fault{Kind: "container-pause", Target: "dep"},
+ Polarity: Polarity{Defects: []Direction{Permitted}},
+ Probes: probe.Pair{Valid: valid, Invalid: invalid},
+ Timing: quickTiming(),
+ }
+}
+
+func TestTheEngineRunsEveryStageInOrder(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ injector := &scriptedInjector{faulted: &faulted}
+ engine := Engine{Injector: injector}
+
+ var stages []Stage
+ engine.Report = func(e Event) { stages = append(stages, e.Stage) }
+
+ result, err := engine.Run(context.Background(), spec(
+ scriptedProbe{probe.Valid, probe.Accepted, probe.Accepted, &faulted, &calls},
+ scriptedProbe{probe.Invalid, probe.Rejected, probe.Rejected, &faulted, &calls},
+ ))
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ want := []Stage{StageBaseline, StageInject, StageConfirm, StageObserve, StageRevert, StageRecover, StageVerdict}
+ if len(stages) != len(want) {
+ t.Fatalf("stages = %v, want %v", stages, want)
+ }
+ for i := range want {
+ if stages[i] != want[i] {
+ t.Fatalf("stage %d = %s, want %s", i, stages[i], want[i])
+ }
+ }
+
+ if injector.injects != 1 || injector.reverts != 1 {
+ t.Fatalf("injects=%d reverts=%d, want 1 and 1", injector.injects, injector.reverts)
+ }
+
+ if result.Verdict.Direction != Held {
+ t.Fatalf("direction = %s, want %s", result.Verdict.Direction, Held)
+ }
+}
+
+func TestTheEngineRefusesAnExperimentWithoutANegativeControl(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ engine := Engine{Injector: &scriptedInjector{faulted: &faulted}}
+
+ incomplete := spec(scriptedProbe{probe.Valid, probe.Accepted, probe.Accepted, &faulted, &calls}, nil)
+
+ if _, err := engine.Run(context.Background(), incomplete); !errors.Is(err, ErrNoProbePair) {
+ t.Fatalf("got %v; an experiment without an invalid-credential probe has no negative "+
+ "control, so its verdict would mean nothing", err)
+ }
+}
+
+func TestAFailingInjectionRevertsAndNeverProducesADirection(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ injector := &scriptedInjector{faulted: &faulted, injectErr: errors.New("daemon said no")}
+ engine := Engine{Injector: injector}
+
+ result, err := engine.Run(context.Background(), spec(
+ scriptedProbe{probe.Valid, probe.Accepted, probe.Accepted, &faulted, &calls},
+ scriptedProbe{probe.Invalid, probe.Rejected, probe.Rejected, &faulted, &calls},
+ ))
+
+ if err == nil {
+ t.Fatal("a failing injection was reported as success")
+ }
+
+ if injector.reverts != 1 {
+ t.Fatalf("reverts = %d; a failed injection may have partially landed and must always "+
+ "be followed by a revert", injector.reverts)
+ }
+
+ if result.Verdict.Epistemic != Errored || result.Verdict.Direction != Undetermined {
+ t.Fatalf("verdict = %s/%s, want %s/%s",
+ result.Verdict.Epistemic, result.Verdict.Direction, Errored, Undetermined)
+ }
+}
+
+func TestTheEngineNeverInjectsIntoATargetThatWasNotEnforcing(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ injector := &scriptedInjector{faulted: &faulted}
+ engine := Engine{Injector: injector}
+
+ result, err := engine.Run(context.Background(), spec(
+ scriptedProbe{probe.Valid, probe.Accepted, probe.Accepted, &faulted, &calls},
+ scriptedProbe{probe.Invalid, probe.Accepted, probe.Accepted, &faulted, &calls},
+ ))
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ if injector.injects != 0 {
+ t.Fatal("khaos injected a fault into a target that was not enforcing before the run; " +
+ "there is nothing to learn and the operator paid the blast radius for it")
+ }
+
+ if result.Verdict.Epistemic != BaselineNotMet {
+ t.Fatalf("epistemic = %s, want %s", result.Verdict.Epistemic, BaselineNotMet)
+ }
+}
+
+func TestTheEngineDetectsAControlThatGaveWayUnderFault(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ engine := Engine{Injector: &scriptedInjector{faulted: &faulted}}
+
+ result, err := engine.Run(context.Background(), spec(
+ scriptedProbe{probe.Valid, probe.Accepted, probe.Accepted, &faulted, &calls},
+ scriptedProbe{probe.Invalid, probe.Rejected, probe.Accepted, &faulted, &calls},
+ ))
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ if result.Verdict.Direction != Permitted {
+ t.Fatalf("direction = %s, want %s; the invalid credential was rejected before the fault "+
+ "and accepted during it", result.Verdict.Direction, Permitted)
+ }
+
+ if !result.Verdict.Defect {
+ t.Fatal("failing open was not flagged as a defect for a control that declares it one")
+ }
+
+ if result.Verdict.Epistemic != Deviated {
+ t.Fatalf("epistemic = %s, want %s", result.Verdict.Epistemic, Deviated)
+ }
+}
+
+func TestAnInterruptedRunIsNotConclusive(t *testing.T) {
+ faulted := false
+ calls := 0
+
+ engine := Engine{Injector: &scriptedInjector{faulted: &faulted}}
+
+ ctx, cancel := context.WithCancel(context.Background())
+
+ slow := spec(
+ scriptedProbe{probe.Valid, probe.Accepted, probe.Accepted, &faulted, &calls},
+ scriptedProbe{probe.Invalid, probe.Rejected, probe.Rejected, &faulted, &calls},
+ )
+ slow.Timing.FaultFor = time.Second
+
+ go func() {
+ time.Sleep(20 * time.Millisecond)
+ cancel()
+ }()
+
+ result, err := engine.Run(ctx, slow)
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ if result.Verdict.Epistemic != Interrupted {
+ t.Fatalf("epistemic = %s, want %s", result.Verdict.Epistemic, Interrupted)
+ }
+
+ if result.Verdict.Conclusive {
+ t.Fatal("an interrupted run was marked conclusive")
+ }
+}
+
+func TestTheHypothesisUsesTheCanonicalTemplate(t *testing.T) {
+ h := Hypothesis{Condition: "the identity provider being unreachable", Response: "a refusal"}
+
+ want := "In the event of the identity provider being unreachable, we are confident that our system will respond with a refusal."
+ if got := h.String(); got != want {
+ t.Fatalf("got %q, want %q", got, want)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
new file mode 100644
index 00000000..cb903426
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
@@ -0,0 +1,188 @@
+// ©AngelaMos | 2026
+// experiment.go
+
+package experiment
+
+import (
+ "context"
+ "fmt"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ hypothesisTemplate = "In the event of %s, we are confident that our system will respond with %s."
+
+ DefaultSampleInterval = 500 * time.Millisecond
+ DefaultBaselineFor = 5 * time.Second
+ // Long enough to clear MinimumToClaimClosed at the default sample interval,
+ // with room for the jitter. At ten seconds a healthy target produced about
+ // twenty paired samples against a threshold of thirty, so every default run
+ // carried `Conclusive=false` and a thin-evidence warning — and exited 0
+ // anyway. Raising the window is the honest lever: it buys confidence with
+ // time rather than by probing the target harder.
+ DefaultFaultFor = 20 * time.Second
+ DefaultRecoveryFor = 5 * time.Second
+)
+
+// Polarity declares which way breaking is a defect for the control under test.
+// Failing open is not universally a defect: an authentication check that stops
+// enforcing is catastrophic, while a rate-limit counter that stops counting
+// when its datastore is unreachable is a trade plenty of teams make on purpose.
+// Defects is a SET, not one direction, and the single-direction version was a
+// live bug. `identity.key_rotation` declares DENIED its defect, so PERMITTED
+// fell to the else branch and khaos printed "this experiment declares PERMITTED
+// to be acceptable for this control" — which nobody declared, and which is not
+// true of any control khaos ships. The run then exited 0, because the exit code
+// keys off Defect alone, so a target that failed wide open passed a CI build.
+//
+// The axis still means what the design contract says: failing open is not
+// universally a defect, and a rate limiter that stops counting when its
+// datastore is unreachable is a trade teams make on purpose. That is why this is
+// a declared set rather than a hardcoded rule.
+type Polarity struct {
+ Defects []Direction
+ Rationale string
+}
+
+func (p Polarity) IsDefect(direction Direction) bool {
+ return slices.Contains(p.Defects, direction)
+}
+
+// Declared is the directions this control treats as a defect, for display.
+func (p Polarity) Declared() string {
+ out := make([]string, 0, len(p.Defects))
+ for _, direction := range p.Defects {
+ out = append(out, string(direction))
+ }
+ return strings.Join(out, " or ")
+}
+
+func (p Polarity) Explain(direction Direction) string {
+ if p.IsDefect(direction) {
+ return sprintf(reasonDefect, direction)
+ }
+ return sprintf(reasonNotDefect, direction)
+}
+
+type Hypothesis struct {
+ Condition string
+ Response string
+}
+
+func (h Hypothesis) String() string {
+ return fmt.Sprintf(hypothesisTemplate, h.Condition, h.Response)
+}
+
+type Fault struct {
+ Kind string
+ Target string
+}
+
+type Timing struct {
+ SampleInterval time.Duration
+ BaselineFor time.Duration
+ FaultFor time.Duration
+ RecoveryFor time.Duration
+}
+
+func (t Timing) withDefaults() Timing {
+ if t.SampleInterval <= 0 {
+ t.SampleInterval = DefaultSampleInterval
+ }
+ if t.BaselineFor <= 0 {
+ t.BaselineFor = DefaultBaselineFor
+ }
+ if t.FaultFor <= 0 {
+ t.FaultFor = DefaultFaultFor
+ }
+ if t.RecoveryFor <= 0 {
+ t.RecoveryFor = DefaultRecoveryFor
+ }
+ return t
+}
+
+type Experiment struct {
+ Name string
+ Family string
+ Summary string
+ Hypothesis Hypothesis
+ Fault Fault
+ Polarity Polarity
+ Probes probe.Pair
+ Timing Timing
+
+ // ProxyFault is non-nil only for an experiment that injects through khaos's
+ // own fault proxy, and it is what tells the run to stand one up at all.
+ ProxyFault *proxy.Fault
+
+ // ProxyListen is the host address the run binds the proxy on, and
+ // ProxyUpstream is the dependency it forwards to.
+ ProxyListen string
+ ProxyUpstream string
+
+ // IdP and IdPTrap are non-nil only for an experiment that arms khaos's own
+ // identity provider. The provider is BUILT here rather than by the run,
+ // because the probes have to mint credentials out of it and minting needs no
+ // listener: the run only has to bind it where the target is already looking.
+ IdP *idp.Provider
+ IdPTrap *idp.Trap
+
+ // Defect names what khaos did to the credential the finding is about, in
+ // words, so the verdict can state the actual observation rather than a
+ // constant that is only true of one experiment.
+ Defect string
+
+ // HeldCaveat qualifies a HELD verdict where the absence of a finding covers
+ // less than it appears to.
+ HeldCaveat string
+
+ // Consulted and ConsultedWhat are the liveness guard for the experiments
+ // that fault an ENDPOINT rather than a credential.
+ //
+ // Three of them break khaos's introspection endpoint, and their only
+ // enforcement evidence was that the invalid control got refused. A target
+ // doing purely local JWT validation refuses a non-JWT bearer token itself
+ // and never calls introspection at all, so the baseline passed, the trap
+ // confirmed at khaos's own provider, and the fault window looked exactly
+ // like the baseline: `passed / HELD`, conclusive, over a fault the target
+ // never experienced.
+ //
+ // khaos already had the instrument and was not using it. Research 07's
+ // requirement R5 names a fetch counter as a thing no third-party provider
+ // will give you, and khaos built one and then read it nowhere.
+ Consulted func() int64
+ ConsultedWhat string
+
+ // IdleFor turns on the idleness discriminator. It must be longer than the
+ // target's own decision-cache lifetime, which khaos cannot discover, so the
+ // experiment declares it as a parameter and the verdict names the pause it
+ // actually used.
+ IdleFor time.Duration
+
+ // Act runs once, after the fault is confirmed and before anything is measured
+ // under it. It exists for the one thing no amount of fault injection can
+ // express: making a change at an authority the target can no longer reach.
+ // ActDetail is what the operator is told khaos is about to do.
+ Act func(context.Context) error
+ ActDetail string
+}
+
+type Result struct {
+ Experiment string `json:"experiment"`
+ RunID string `json:"run_id"`
+ Hypothesis string `json:"hypothesis"`
+ StartedAt time.Time `json:"started_at"`
+ Duration time.Duration `json:"duration"`
+ Observation Observation `json:"observation"`
+ Verdict Verdict `json:"verdict"`
+}
+
+func sprintf(format string, args ...any) string {
+ return fmt.Sprintf(format, args...)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery.go
new file mode 100644
index 00000000..f9dbd208
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery.go
@@ -0,0 +1,142 @@
+// ©AngelaMos | 2026
+// forgery.go
+
+package experiment
+
+import (
+ "strings"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const caveatOneKeyForm = "this run keyed the forged MAC on the %s serialization only, and the same bug is unreachable under the wrong one; a target that loaded its key differently is not ruled out until each of %s has been tried"
+
+// The forgery experiments. Each one makes khaos's own identity provider mint a
+// credential that is wrong in EXACTLY ONE named way, leaves everything else
+// correct, and asks whether the target notices.
+//
+// Every one of them is anchored to a published CVE or a normative rule, and
+// every one has a different fix. That is why they are ten entries in the
+// catalogue rather than one experiment with a `--trap` flag: an operator reading
+// `khaos catalog` is reading a list of bugs to go and fix, and "accepts an
+// unsigned token" and "ignores the expiry claim" are not two settings of one
+// bug.
+//
+// The forged stream's baseline expectation is ACCEPTED for all ten, and that is
+// not a formality. khaos's provider mints correctly until the trap is armed, so
+// a target that refuses even those credentials is not pointed at khaos's
+// provider at all, and without the guard every one of these experiments would
+// report a clean survival against a target it never actually tested.
+const (
+ NameAlgNone = "identity.alg_none"
+ NameAlgConfusion = "identity.alg_confusion"
+ NameEmbeddedJWK = "identity.embedded_jwk"
+ NameClaimExpired = "identity.claim_expired"
+ NameClaimNotYetValid = "identity.claim_not_yet_valid"
+ NameClaimWrongAudience = "identity.claim_wrong_audience"
+ NameClaimForeignIssuer = "identity.claim_foreign_issuer"
+ NameSignatureTruncated = "identity.signature_truncated"
+ NameSignatureZero = "identity.signature_zero"
+ NameSignatureWeakKey = "identity.signature_weak_key"
+)
+
+const (
+ summaryAlgNone = "mint an unsigned token that declares `alg: none` and find out whether the target believes it, which is the cheapest high-signal probe in the whole tool because it needs no cooperation from anything"
+
+ summaryAlgConfusion = "sign a token with HMAC keyed on the issuer's PUBLIC key and find out whether the target verifies it, which works because the attacker has the secret by definition"
+
+ summaryEmbeddedJWK = "mint a token carrying its own signing key in the header and find out whether the target resolves keys from the token instead of from the key set it trusts"
+
+ summaryClaimExpired = "mint a correctly signed token that has already expired, which asks whether the target still has the only revocation mechanism a stateless verifier gets"
+
+ summaryClaimNotYetValid = "mint a correctly signed token that is not valid yet, which asks whether staged issuance means anything to the target"
+
+ summaryClaimWrongAudience = "mint a correctly signed token addressed to a different service, which asks whether a token minted for anything in the estate works everywhere in it"
+
+ summaryClaimForeignIssuer = "mint a correctly signed token from a second issuer whose keys the provider also publishes, which asks whether the key set is being mistaken for the trust boundary"
+
+ summarySignatureTruncated = "cut a valid signature down to one byte and find out whether the target checks its length before believing it"
+
+ summarySignatureZero = "mint an ES256 token whose signature is entirely zero bytes, which is the shape of the psychic-signatures bug and costs one request to rule out"
+
+ summarySignatureWeakKey = "sign with a modulus smaller than any correct verifier should accept, and find out whether the target checks key size or only that the arithmetic worked"
+)
+
+const (
+ conditionAlgNone = "%s being offered an unsigned token by its own identity provider"
+ conditionAlgConfusion = "%s being offered a token signed with its issuer's public key as an HMAC secret"
+ conditionEmbeddedJWK = "%s being offered a token that brings its own signing key"
+ conditionClaimExpired = "%s being offered an expired token with a correct signature"
+ conditionClaimNotYetValid = "%s being offered a not-yet-valid token with a correct signature"
+ conditionClaimWrongAudience = "%s being offered a correctly signed token meant for another service"
+ conditionClaimForeignIssuer = "%s being offered a correctly signed token from another issuer"
+ conditionSignatureTruncated = "%s being offered a token whose signature is one byte long"
+ conditionSignatureZero = "%s being offered a token whose signature is all zeroes"
+ conditionSignatureWeakKey = "%s being offered a token signed under an undersized key"
+)
+
+// The defect phrase goes into the verdict, so it has to read as a sentence
+// fragment naming what was wrong with the credential the target accepted.
+const (
+ defectAlgNone = "no signature at all and a header saying so"
+ defectAlgConfusion = "an HMAC keyed on its issuer's own public key"
+ defectEmbeddedJWK = "a signing key the issuer never published, carried in the token's own header"
+ defectClaimExpired = "an expiry already in the past"
+ defectClaimNotYetValid = "a validity window that has not started"
+ defectClaimWrongAudience = "an audience belonging to another service"
+ defectClaimForeignIssuer = "an issuer the target was not configured to trust"
+ defectSignatureTruncated = "a signature cut to a single byte"
+ defectSignatureZero = "a signature of nothing but zero bytes"
+ defectSignatureWeakKey = "a modulus below the size any correct verifier accepts"
+)
+
+// forged declares a forgery experiment. The wiring is identical across all ten
+// because the shape is identical: the provider mints correctly until it is
+// armed, so the forged stream mints per sample and has to have been accepted
+// during the baseline.
+func forged(name, summary, condition, defect string, trap idp.Kind, extra ...Param) idpSpec {
+ spec := idpSpec{
+ name: name,
+ summary: summary,
+ condition: condition,
+ defect: defect,
+ trap: trap,
+ wiring: streams{forged: probe.ExpectAccepted},
+ extra: extra,
+ }
+
+ switch trap {
+ case idp.ClaimExpired, idp.ClaimNotYetValid:
+ spec.magnitude = claimShiftMagnitude
+ case idp.AlgConfusion:
+ spec.magnitude = keyFormMagnitude
+ // Research 07's T2 is explicit that this bug is UNREACHABLE under the
+ // wrong serialization, and khaos tries one per run. An unqualified
+ // "no bypass was observed" would be khaos claiming it ruled out
+ // algorithm confusion when it ruled out one of four ways to express it.
+ spec.heldCaveat = func(params *Params) string {
+ return sprintf(caveatOneKeyForm, params.Get(ParamKeyForm), strings.Join(idp.KeyForms(), nameSep))
+ }
+ }
+
+ return spec
+}
+
+func init() {
+ for _, spec := range []idpSpec{
+ forged(NameAlgNone, summaryAlgNone, conditionAlgNone, defectAlgNone, idp.AlgNone),
+ forged(NameAlgConfusion, summaryAlgConfusion, conditionAlgConfusion, defectAlgConfusion, idp.AlgConfusion, keyFormParam),
+ forged(NameEmbeddedJWK, summaryEmbeddedJWK, conditionEmbeddedJWK, defectEmbeddedJWK, idp.EmbeddedJWK),
+ forged(NameClaimExpired, summaryClaimExpired, conditionClaimExpired, defectClaimExpired, idp.ClaimExpired, claimShiftParam),
+ forged(NameClaimNotYetValid, summaryClaimNotYetValid, conditionClaimNotYetValid, defectClaimNotYetValid, idp.ClaimNotYetValid, claimShiftParam),
+ forged(NameClaimWrongAudience, summaryClaimWrongAudience, conditionClaimWrongAudience, defectClaimWrongAudience, idp.ClaimWrongAudience),
+ forged(NameClaimForeignIssuer, summaryClaimForeignIssuer, conditionClaimForeignIssuer, defectClaimForeignIssuer, idp.ClaimForeignIssuer),
+ forged(NameSignatureTruncated, summarySignatureTruncated, conditionSignatureTruncated, defectSignatureTruncated, idp.SignatureTruncated),
+ forged(NameSignatureZero, summarySignatureZero, conditionSignatureZero, defectSignatureZero, idp.SignatureZero),
+ forged(NameSignatureWeakKey, summarySignatureWeakKey, conditionSignatureWeakKey, defectSignatureWeakKey, idp.SignatureWeakKey),
+ } {
+ definition := idpDefinition(spec)
+ catalog[definition.Name] = definition
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go
new file mode 100644
index 00000000..18792614
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go
@@ -0,0 +1,387 @@
+// ©AngelaMos | 2026
+// forgery_verdict_test.go
+
+package experiment
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const forgedDefectPhrase = "no signature at all and a header saying so"
+
+// window builds an observation window from a disposition per stream, repeated
+// enough times to clear every sample floor. Streams given no disposition are
+// left absent, which is how an experiment that declares no forged or revoked
+// stream looks.
+func window(valid, invalid, forged, revoked probe.Disposition, n int) probe.Window {
+ out := probe.Window{}
+
+ for range n {
+ at := time.Now().UTC()
+
+ out.Record(probe.Outcome{At: at, Credential: probe.Valid, Disposition: valid})
+ out.Record(probe.Outcome{At: at, Credential: probe.Invalid, Disposition: invalid})
+
+ if forged != "" {
+ out.Record(probe.Outcome{At: at, Credential: probe.Forged, Disposition: forged})
+ }
+
+ if revoked != "" {
+ out.Record(probe.Outcome{At: at, Credential: probe.Revoked, Disposition: revoked})
+ }
+ }
+
+ return out
+}
+
+func forgeryObservation(baseline, fault probe.Window, expect probe.Expectation) Observation {
+ return Observation{
+ Baseline: baseline,
+ Fault: fault,
+ Recovery: window(probe.Accepted, probe.Rejected, probe.Rejected, "", MinimumPaired),
+ FaultConfirmed: true,
+ ConfirmMethod: "khaos read its own forged token back off the provider",
+ ForgedBaseline: expect,
+ Defect: forgedDefectPhrase,
+ }
+}
+
+// One acceptance of a credential khaos PROVED defective is a proof, so the
+// finding does not wait for the sample floor. The engine refuses to measure a
+// fault it could not confirm, so by the time this is judged khaos has already
+// read the malformed token back off its own provider.
+func TestOneAcceptedForgedCredentialIsAConclusiveFinding(t *testing.T) {
+ observation := forgeryObservation(
+ window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumPaired),
+ window(probe.Accepted, probe.Rejected, probe.Accepted, "", 1),
+ probe.ExpectAccepted,
+ )
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Epistemic != Deviated || verdict.Direction != Permitted {
+ t.Fatalf("verdict = %s / %s, want %s / %s (%s)",
+ verdict.Epistemic, verdict.Direction, Deviated, Permitted,
+ strings.Join(verdict.Reasons, "; "))
+ }
+
+ if !verdict.Conclusive {
+ t.Fatal("a proven bypass was reported as inconclusive")
+ }
+
+ if !verdict.Defect {
+ t.Fatal("the identity tier declares PERMITTED a defect and this verdict did not")
+ }
+
+ joined := strings.Join(verdict.Reasons, "; ")
+
+ if !strings.Contains(joined, forgedDefectPhrase) {
+ t.Fatalf("the verdict does not name what was wrong with the credential: %s", joined)
+ }
+
+ // The evidence claim lives in the per-trap CONFIRM line, not in this reason.
+ // The shared reason used to assert that khaos had read the credential back
+ // off the provider, which is true of the ten forgery traps and false of
+ // `identity.jwks_unreachable`, whose confirmed observable is the key set
+ // answering non-200 and which never inspects the credential at all.
+ if observation.ConfirmMethod == "" {
+ t.Fatal("the run records no confirmation method, so nothing says how khaos established " +
+ "the credential was defective before presenting it")
+ }
+}
+
+// The forged stream being refused throughout is the target behaving correctly,
+// and it must not read as a finding.
+func TestAForgedCredentialRefusedThroughoutIsAPass(t *testing.T) {
+ observation := forgeryObservation(
+ window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumToClaimClosed),
+ window(probe.Accepted, probe.Rejected, probe.Rejected, "", MinimumToClaimClosed),
+ probe.ExpectAccepted,
+ )
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Epistemic != Passed || verdict.Direction != Held {
+ t.Fatalf("verdict = %s / %s, want %s / %s (%s)",
+ verdict.Epistemic, verdict.Direction, Passed, Held,
+ strings.Join(verdict.Reasons, "; "))
+ }
+
+ if verdict.Defect {
+ t.Fatal("a target that refused every forged credential was reported as defective")
+ }
+}
+
+// The liveness guard that matters most in the whole family. A target not pointed
+// at khaos's identity provider refuses every credential khaos mints, for reasons
+// that have nothing to do with any trap. Without this guard every one of the ten
+// forgery experiments reports a clean survival against a target it never tested.
+func TestATargetThatNeverTrustedKhaosProviderVoidsTheRun(t *testing.T) {
+ observation := forgeryObservation(
+ // Note the valid stream IS accepted: the target is serving, enforcing,
+ // and simply refusing khaos's issuer. That is exactly the case a naive
+ // baseline check passes.
+ window(probe.Accepted, probe.Rejected, probe.Rejected, "", MinimumToClaimClosed),
+ window(probe.Accepted, probe.Rejected, probe.Rejected, "", MinimumToClaimClosed),
+ probe.ExpectAccepted,
+ )
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Epistemic != BaselineNotMet {
+ t.Fatalf("epistemic = %s, want %s (%s)", verdict.Epistemic, BaselineNotMet,
+ strings.Join(verdict.Reasons, "; "))
+ }
+
+ if verdict.Conclusive {
+ t.Fatal("a run that never established the target trusts khaos's provider was conclusive")
+ }
+
+ if !strings.Contains(strings.Join(verdict.Reasons, "; "), "resolves keys from the issuer") {
+ t.Fatalf("the refusal does not tell the operator what to check: %s",
+ strings.Join(verdict.Reasons, "; "))
+ }
+}
+
+// The mirror guard, for the one experiment whose forged credential is malformed
+// from the start. A target that never refused it was not checking signatures, so
+// its acceptance during the fault proves nothing changed.
+func TestATargetThatNeverCheckedTheForgedCredentialVoidsTheRun(t *testing.T) {
+ observation := forgeryObservation(
+ window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumToClaimClosed),
+ window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumToClaimClosed),
+ probe.ExpectRejected,
+ )
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Epistemic != BaselineNotMet {
+ t.Fatalf("epistemic = %s, want %s (%s)", verdict.Epistemic, BaselineNotMet,
+ strings.Join(verdict.Reasons, "; "))
+ }
+
+ if !strings.Contains(strings.Join(verdict.Reasons, "; "), "not checking the property") {
+ t.Fatalf("the refusal does not say what was never established: %s",
+ strings.Join(verdict.Reasons, "; "))
+ }
+}
+
+// The two guards point in OPPOSITE directions, so each one has to be inert for
+// the other's expectation. A guard that fired on both would void every run in
+// the family.
+func TestTheTwoForgedGuardsDoNotFireOnEachOthersExpectation(t *testing.T) {
+ acceptedThroughout := window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumToClaimClosed)
+ rejectedThroughout := window(probe.Accepted, probe.Rejected, probe.Rejected, "", MinimumToClaimClosed)
+
+ if _, unmet := baselineUnmet(acceptedThroughout, probe.ExpectAccepted, true); unmet {
+ t.Error("ExpectAccepted voided a baseline where the forged stream WAS accepted")
+ }
+
+ if _, unmet := baselineUnmet(rejectedThroughout, probe.ExpectRejected, true); unmet {
+ t.Error("ExpectRejected voided a baseline where the forged stream WAS rejected")
+ }
+}
+
+// An experiment with no forged stream must be unaffected by either guard, or
+// adding the stream to the probe model would have broken the eight experiments
+// that do not use it.
+func TestAnAbsentForgedStreamIsNotGuarded(t *testing.T) {
+ bare := window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed)
+
+ for _, expect := range []probe.Expectation{"", probe.ExpectAccepted, probe.ExpectRejected} {
+ if _, unmet := baselineUnmet(bare, expect, true); unmet {
+ t.Errorf("a baseline with no forged stream was voided under expectation %q", expect)
+ }
+ }
+}
+
+// A forged acceptance and a monotonic leak in the same run are two different
+// observations, and both get stated. Returning on the first one dropped the
+// other from a run where both were true.
+func TestEveryStreamThatLeakedIsStated(t *testing.T) {
+ observation := forgeryObservation(
+ window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumPaired),
+ window(probe.Accepted, probe.Accepted, probe.Accepted, "", MinimumPaired),
+ probe.ExpectAccepted,
+ )
+
+ verdict := Judge(observation, idpPolarity)
+
+ joined := strings.Join(verdict.Reasons, "; ")
+
+ if !strings.Contains(joined, "khaos minted with") {
+ t.Errorf("the forged stream's finding is missing: %s", joined)
+ }
+
+ if !strings.Contains(joined, "had already rejected") {
+ t.Errorf("the monotonic finding is missing: %s", joined)
+ }
+}
+
+// The idleness discriminator is for CACHING causes. A forged credential getting
+// through is not a caching bug, so a forgery experiment must not spend a minute
+// of the operator's run pausing traffic to diagnose one.
+func TestAForgedFindingAloneDoesNotAskForTheIdlenessDiscriminator(t *testing.T) {
+ observation := forgeryObservation(
+ window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumPaired),
+ window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumPaired),
+ probe.ExpectAccepted,
+ )
+
+ if !Leaking(observation) {
+ t.Fatal("a proven forged bypass did not register as a leak, so no cause would ever be diagnosed")
+ }
+
+ verdict := Judge(observation, idpPolarity)
+
+ // Nothing resumed, because no forgery experiment sets IdleFor, so the cause
+ // stays honestly UNDIAGNOSED rather than being guessed.
+ if verdict.Cause != Undiagnosed {
+ t.Fatalf("cause = %s, want %s on a run with no idleness phase", verdict.Cause, Undiagnosed)
+ }
+}
+
+// A target whose key cache never expires is serving perfectly well and simply
+// does not know khaos's keys, which khaos regenerates every run. The generic
+// refusal says the target was not serving, which sends an operator to look at a
+// service that is fine.
+//
+// Measured against the range's forever-cache gateway: a second consecutive run
+// could not establish a baseline, and nothing in the output said why.
+func TestTheIdentityTierSaysWhyItsOwnCredentialWasRefused(t *testing.T) {
+ refused := window(probe.Rejected, probe.Rejected, "", "", MinimumToClaimClosed)
+
+ minted, unmet := baselineUnmet(refused, "", true)
+ if !unmet {
+ t.Fatal("a baseline where khaos's own credential was never accepted was not voided")
+ }
+
+ operator, unmet := baselineUnmet(refused, "", false)
+ if !unmet {
+ t.Fatal("a baseline where the operator's credential was never accepted was not voided")
+ }
+
+ if minted == operator {
+ t.Fatal("the identity tier and the operator-supplied tier give the same refusal, so the " +
+ "one case where the generic wording is misleading reads identically")
+ }
+
+ for _, want := range []string{"key set", "fresh key set every run", "Restart the target"} {
+ if !strings.Contains(minted, want) {
+ t.Errorf("the identity tier's refusal does not mention %q: %s", want, minted)
+ }
+ }
+
+ if strings.Contains(minted, "was not serving") {
+ t.Errorf("the identity tier's refusal still claims the target was not serving: %s", minted)
+ }
+}
+
+// Recovery is three-valued, and "not observed" is the state that used to lie.
+func TestRecoveryDistinguishesUnobservedFromRecovered(t *testing.T) {
+ healthy := window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed)
+
+ cases := []struct {
+ name string
+ recovery probe.Window
+ want Recovery
+ }{
+ {"healthy run of pairs", healthy, Recovered},
+ {"no evidence at all", probe.Window{}, Unobserved},
+ {"measured and still broken", window(probe.Rejected, probe.Rejected, "", "", MinimumPaired), Degraded},
+ }
+
+ for _, each := range cases {
+ t.Run(each.name, func(t *testing.T) {
+ observation := Observation{
+ Baseline: healthy,
+ Fault: healthy,
+ Recovery: each.recovery,
+ FaultConfirmed: true,
+ }
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Recovery != each.want {
+ t.Fatalf("recovery = %q, want %q (%s)", verdict.Recovery, each.want,
+ strings.Join(verdict.Reasons, "; "))
+ }
+ })
+ }
+}
+
+// An unobserved recovery must say so rather than pass silently.
+func TestAnUnobservedRecoveryIsWarnedAboutRatherThanAssumed(t *testing.T) {
+ healthy := window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed)
+
+ verdict := Judge(Observation{
+ Baseline: healthy,
+ Fault: healthy,
+ FaultConfirmed: true,
+ }, idpPolarity)
+
+ joined := strings.Join(verdict.Reasons, "; ")
+
+ if !strings.Contains(joined, "not the same as it having come back") {
+ t.Fatalf("khaos assumed a recovery it never measured: %s", joined)
+ }
+
+ found := false
+ for _, warning := range verdict.Warnings {
+ if warning == WarningRecoveryUnobserved {
+ found = true
+ }
+ }
+
+ if !found {
+ t.Fatalf("no recovery-unobserved warning: %v", verdict.Warnings)
+ }
+}
+
+// A required stream with no usable decisions cannot support a clean result.
+// Thirty valid acceptances and thirty invalid refusals used to clear the floor
+// while every forged probe failed, and the run reported that the target refused
+// a credential it was never successfully shown.
+func TestARequiredForgedStreamWithNoDecisionsCannotReportHeld(t *testing.T) {
+ baseline := window(probe.Accepted, probe.Rejected, probe.Accepted, "", MinimumToClaimClosed)
+
+ fault := probe.Window{}
+ for range MinimumToClaimClosed {
+ fault.Record(probe.Outcome{Credential: probe.Valid, Disposition: probe.Accepted})
+ fault.Record(probe.Outcome{Credential: probe.Invalid, Disposition: probe.Rejected})
+ fault.Record(probe.Outcome{Credential: probe.Forged, Disposition: probe.ProbeFailed})
+ }
+
+ observation := Observation{
+ Baseline: baseline,
+ Fault: fault,
+ Recovery: window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed),
+ FaultConfirmed: true,
+ ForgedBaseline: probe.ExpectAccepted,
+ }
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Epistemic == Passed {
+ t.Fatalf("a run where every forged probe failed reported a clean survival (%s)",
+ strings.Join(verdict.Reasons, "; "))
+ }
+
+ if verdict.Epistemic != ProbeError {
+ t.Fatalf("epistemic = %s, want %s", verdict.Epistemic, ProbeError)
+ }
+
+ // The control: the same run with real decisions on that stream judges
+ // normally, or this guard would have deleted the clean-result path.
+ decided := window(probe.Accepted, probe.Rejected, probe.Rejected, "", MinimumToClaimClosed)
+ observation.Fault = decided
+
+ if verdict := Judge(observation, idpPolarity); verdict.Epistemic != Passed {
+ t.Fatalf("a run WITH forged decisions was judged %s, want %s (%s)",
+ verdict.Epistemic, Passed, strings.Join(verdict.Reasons, "; "))
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/identity.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/identity.go
new file mode 100644
index 00000000..a53d31d7
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/identity.go
@@ -0,0 +1,166 @@
+// ©AngelaMos | 2026
+// identity.go
+
+package experiment
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/url"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ FamilyIdentity = "identity"
+
+ NameIDPOutage = "identity.idp_outage"
+
+ ParamIdleFor = "idle-for"
+ ParamAuthority = "authority"
+ ParamRevocable = "revocable-token"
+
+ usageIdleFor = "how long to pause traffic, with the fault still applied, to see whether the target's decision cache is being re-armed; it has to outlast the target's own cache lifetime"
+ usageAuthority = "khaos's OWN address for the identity provider, which it keeps while the target loses its path to it; this is what makes the revocation real and unreachable at the same time"
+ usageRevocable = "a credential the target is honouring and that khaos revokes once the target can no longer reach the authority"
+
+ // Six seconds outlasts the range's own two second decision cache by a wide
+ // margin. Against anything else the operator sets it, because khaos cannot
+ // discover a cache lifetime it is probing from outside.
+ defaultIdleFor = "6s"
+
+ revokePath = "/revoke"
+ revokeParam = "token"
+ revokeMethod = http.MethodPost
+
+ revokeTimeout = 5 * time.Second
+
+ errRevoke = "khaos could not revoke the credential at %s: %w"
+ errRevokeStatus = "the authority at %s answered %d when khaos asked it to revoke a credential, so the revocation did not happen"
+
+ actDetail = "revoking a credential at the authority, which the target can no longer reach"
+
+ // The phrase the verdict uses to say what khaos did. It is per experiment
+ // because the three experiments with a revoked stream invalidate their
+ // credential three different ways, and only this one does it at an authority
+ // the target cannot see.
+ defectIDPOutage = "revoking it at an authority the target could no longer reach"
+
+ idpOutageSummary = "sever the target from its identity provider, revoke a credential there, and find out WHY the target still honours it: a decision cache being re-armed by every failed check, or one that never expires at all"
+
+ idpOutageCondition = "%s becoming unreachable to the target while khaos revokes a credential there"
+
+ idpOutageResponse = "refusing the credential khaos revoked, and letting any cached decision lapse on schedule rather than on idleness"
+)
+
+// The research calls this the highest-value single experiment in its archive, and
+// the reason is the extra phases rather than the fault. Severing a target from its
+// identity provider is something several tools can do. Making a change at that
+// provider which the target cannot see, and then telling a cache that is being
+// RE-ARMED from one that has NO EXPIRY, from outside, with no access to the
+// target's source, is what no other tool does. The two have different fixes.
+var idpOutagePolarity = Polarity{
+ Defects: []Direction{Permitted},
+ Rationale: "a control that keeps honouring a credential while the authority that could revoke it is unreachable has stopped being a control, and the longer it does so the wider the exposure",
+}
+
+// The fault is a PROXY fault and not a pause, and that is forced. A paused
+// provider is unreachable by khaos too, so the revocation could not be made at
+// all. khaos severs the target's path with its own in-process proxy and keeps its
+// own direct path to the authority.
+func buildIDPOutage(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+ probeURL := params.Get(ParamURL)
+ authority := params.Get(ParamAuthority)
+ revocable := params.Get(ParamRevocable)
+
+ idle, err := params.Duration(ParamIdleFor)
+ if err != nil {
+ return Shape{}, err
+ }
+
+ pair := probePair(params)
+
+ // The third stream. It carries a credential the target is honouring, and its
+ // acceptance AFTER khaos revokes it is the finding. It is deliberately not
+ // unique per request: the whole question is whether the target's cached
+ // decision for THIS credential outlives the authority, and a fresh identity
+ // every time would never reach that cache.
+ pair.Revoked = probe.NewHTTP(probe.HTTPOptions{
+ Name: probeRevokedName,
+ URL: probeURL,
+ Token: revocable,
+ Credential: probe.Revoked,
+ })
+
+ listen, upstream := proxyEndpoint(params)
+
+ return Shape{
+ Condition: fmt.Sprintf(idpOutageCondition, target),
+ Target: target,
+ Probes: pair,
+ ProxyFault: &proxy.Fault{Kind: proxy.Hang},
+ ProxyListen: listen,
+ ProxyUpstream: upstream,
+ IdleFor: idle,
+ Act: revokeAt(authority, revocable),
+ ActDetail: actDetail,
+ Defect: defectIDPOutage,
+ }, nil
+}
+
+// revokeAt asks the authority to revoke one credential. A non-2xx answer is a
+// FAILED revocation, not a quiet success: the engine voids the run on it, because
+// a credential nobody revoked is one the target is correct to keep honouring.
+func revokeAt(authority, credential string) func(context.Context) error {
+ return func(ctx context.Context) error {
+ endpoint, err := url.Parse(authority + revokePath)
+ if err != nil {
+ return fmt.Errorf(errRevoke, authority, err)
+ }
+
+ query := endpoint.Query()
+ query.Set(revokeParam, credential)
+ endpoint.RawQuery = query.Encode()
+
+ request, err := http.NewRequestWithContext(ctx, revokeMethod, endpoint.String(), nil)
+ if err != nil {
+ return fmt.Errorf(errRevoke, authority, err)
+ }
+
+ client := &http.Client{Timeout: revokeTimeout}
+
+ response, err := client.Do(request)
+ if err != nil {
+ return fmt.Errorf(errRevoke, authority, err)
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
+ return fmt.Errorf(errRevokeStatus, authority, response.StatusCode)
+ }
+
+ return nil
+ }
+}
+
+func init() {
+ catalog[NameIDPOutage] = Definition{
+ Name: NameIDPOutage,
+ Family: FamilyIdentity,
+ Summary: idpOutageSummary,
+ Hypothesis: Hypothesis{Condition: outageGeneric, Response: idpOutageResponse},
+ Polarity: idpOutagePolarity,
+ FaultKind: inject.ProxyKind(proxy.Hang),
+ Params: append(proxyParams(),
+ Param{Name: ParamAuthority, Usage: usageAuthority, Required: true},
+ Param{Name: ParamRevocable, Usage: usageRevocable, Required: true},
+ Param{Name: ParamIdleFor, Usage: usageIdleFor, Default: defaultIdleFor},
+ ),
+ Build: buildIDPOutage,
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idleness_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idleness_test.go
new file mode 100644
index 00000000..a05401b3
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idleness_test.go
@@ -0,0 +1,359 @@
+// ©AngelaMos | 2026
+// idleness_test.go
+
+package experiment
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ cacheTTL = 60 * time.Millisecond
+ leakedToken = "a-credential-the-target-cached-an-allow-for"
+ idleName = "test.idleness"
+)
+
+// cachingService is a control in front of a dependency, with a decision cache
+// holding an allow for a credential it should not honour.
+//
+// reArm is the single line research 07 section 3.3 pins the TTL-extension bug to:
+// writing to the cache inside the ERROR branch, so every failed check extends the
+// cached decision by another full lifetime and the lifetime stops bounding
+// staleness and starts bounding IDLENESS.
+//
+// noExpiry is the other bug: a cache with no lifetime at all. Under steady
+// traffic the two are indistinguishable, which is the whole reason the
+// discriminator exists.
+type cachingService struct {
+ up atomic.Bool
+ reArm bool
+ noExpiry bool
+
+ mu sync.Mutex
+ cache map[string]time.Time
+}
+
+func newCachingService(reArm, noExpiry bool) *cachingService {
+ s := &cachingService{reArm: reArm, noExpiry: noExpiry, cache: map[string]time.Time{}}
+ s.up.Store(true)
+ s.cache[leakedToken] = time.Now().Add(cacheTTL)
+ return s
+}
+
+func (s *cachingService) handler() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ token := strings.TrimPrefix(r.Header.Get("Authorization"), bearer)
+
+ if s.up.Load() {
+ // While the dependency answers, the control enforces correctly: only
+ // the genuinely valid credential is accepted. This is what makes the
+ // baseline a steady state and the leaked credential a rejection.
+ if token != goodToken {
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ expires, known := s.cache[token]
+ if known && (s.noExpiry || time.Now().Before(expires)) {
+ if s.reArm {
+ s.cache[token] = time.Now().Add(cacheTTL)
+ }
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+
+ if token == goodToken {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+
+ w.WriteHeader(http.StatusForbidden)
+ })
+}
+
+type cachingInjector struct{ svc *cachingService }
+
+func (i *cachingInjector) Inject(context.Context, string, string) error {
+ i.svc.up.Store(false)
+ return nil
+}
+
+func (i *cachingInjector) Confirm(context.Context) (string, error) {
+ if i.svc.up.Load() {
+ return "", nil
+ }
+ return "the dependency reads as down", nil
+}
+
+func (i *cachingInjector) Revert(context.Context) error {
+ i.svc.up.Store(true)
+ return nil
+}
+
+// The negative control here is deliberately NOT unique, and that is the opposite
+// of what network.dependency_outage needs. There, the control is a credential no
+// verifier could ever accept, so a FIXED one lets a cached refusal answer every
+// probe and the control is never exercised. Here the control is a credential the
+// target has cached an ALLOW for, so it has to be the same credential every time
+// or the cached allow is never reached at all.
+func registerIdlenessExperiment(t *testing.T) {
+ t.Helper()
+
+ catalog[idleName] = Definition{
+ Name: idleName,
+ Family: FamilyNetwork,
+ Summary: "a cached allow outliving the authority that could revoke it",
+ Hypothesis: Hypothesis{Condition: outageGeneric, Response: outageResponse},
+ Polarity: outagePolarity,
+ FaultKind: "test-kind",
+ Params: []Param{
+ {Name: ParamTarget, Usage: usageTarget, Required: true},
+ {Name: ParamURL, Usage: usageURL, Required: true},
+ {Name: ParamToken, Usage: usageToken, Required: true},
+ },
+ Build: func(params *Params) (Shape, error) {
+ url := params.Get(ParamURL)
+
+ return Shape{
+ Target: params.Get(ParamTarget),
+ Probes: probe.Pair{
+ Valid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeValidName,
+ URL: url,
+ Token: params.Get(ParamToken),
+ Credential: probe.Valid,
+ }),
+ Invalid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeInvalidName,
+ URL: url,
+ Token: leakedToken,
+ Credential: probe.Invalid,
+ }),
+ },
+ }, nil
+ },
+ }
+
+ t.Cleanup(func() { delete(catalog, idleName) })
+}
+
+func runIdleness(t *testing.T, svc *cachingService, idle time.Duration) Result {
+ t.Helper()
+
+ registerIdlenessExperiment(t)
+
+ server := httptest.NewServer(svc.handler())
+ t.Cleanup(server.Close)
+
+ spec, err := Build(idleName, map[string]string{
+ ParamTarget: "the-identity-provider",
+ ParamURL: server.URL,
+ ParamToken: goodToken,
+ }, Timing{
+ SampleInterval: time.Millisecond,
+ BaselineFor: 20 * time.Millisecond,
+ // At least as long as the pause. The TTL-extension diagnosis is only
+ // sound when steady traffic kept the credential alive for at least the
+ // duration of the pause that then killed it: a fixed lifetime sitting
+ // between a short fault window and a longer pause produces the same
+ // observation. These tests used a 20ms window against a 600ms pause and
+ // asserted a causal claim khaos could not make from it.
+ FaultFor: maxDuration(20*time.Millisecond, idle),
+ RecoveryFor: 10 * time.Millisecond,
+ })
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ spec.IdleFor = idle
+
+ engine := Engine{Injector: &cachingInjector{svc: svc}}
+
+ result, err := engine.Run(context.Background(), spec)
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ return result
+}
+
+// THE milestone claim. Two targets that are indistinguishable under steady
+// traffic are separated by pausing traffic and resuming it, from outside, with no
+// access to either one's source.
+func TestTheIdlenessDiscriminatorSeparatesTwoCausesOfOneLeak(t *testing.T) {
+ const pause = 10 * cacheTTL
+
+ cases := []struct {
+ name string
+ svc *cachingService
+ cause Cause
+ }{
+ {"a cache re-armed on every failed check", newCachingService(true, false), TTLExtension},
+ {"a cache that outlasts the pause", newCachingService(false, true), StalePastPause},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ result := runIdleness(t, tc.svc, pause)
+
+ if result.Verdict.Direction != Permitted {
+ t.Fatalf("direction = %s, want %s (%s)", result.Verdict.Direction, Permitted,
+ strings.Join(result.Verdict.Reasons, "; "))
+ }
+
+ if result.Verdict.Cause != tc.cause {
+ t.Fatalf("cause = %s, want %s. These two targets are identical under steady "+
+ "traffic and need different fixes, so a verdict that cannot tell them apart "+
+ "sends the operator after the wrong bug (%s)",
+ result.Verdict.Cause, tc.cause, strings.Join(result.Verdict.Reasons, "; "))
+ }
+
+ if result.Observation.IdleFor != pause {
+ t.Fatalf("the run recorded a pause of %s, want %s; a finding whose pause is not "+
+ "reported cannot be checked", result.Observation.IdleFor, pause)
+ }
+ })
+ }
+}
+
+// The discriminator costs a pause and a second measurement window. On a target
+// that never leaked there is nothing to diagnose, so spending them would be the
+// tool deciding to learn nothing slowly.
+func TestTheDiscriminatorDoesNotRunOnATargetThatHeld(t *testing.T) {
+ registerIdlenessExperiment(t)
+
+ // A target whose cache holds nothing never leaks.
+ svc := &cachingService{cache: map[string]time.Time{}}
+ svc.up.Store(true)
+
+ server := httptest.NewServer(svc.handler())
+ defer server.Close()
+
+ spec, err := Build(idleName, map[string]string{
+ ParamTarget: "dep",
+ ParamURL: server.URL,
+ ParamToken: goodToken,
+ }, Timing{
+ SampleInterval: time.Millisecond,
+ BaselineFor: 20 * time.Millisecond,
+ FaultFor: 20 * time.Millisecond,
+ RecoveryFor: 10 * time.Millisecond,
+ })
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ spec.IdleFor = 10 * time.Second
+
+ started := time.Now()
+ result, err := Engine{Injector: &cachingInjector{svc: svc}}.Run(context.Background(), spec)
+ took := time.Since(started)
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ if took >= spec.IdleFor {
+ t.Fatalf("the run took %s against a %s pause, so it paused traffic on a target with "+
+ "nothing to diagnose", took, spec.IdleFor)
+ }
+
+ if result.Verdict.Cause != Undiagnosed {
+ t.Fatalf("cause = %s, want %s on a run where the discriminator never ran",
+ result.Verdict.Cause, Undiagnosed)
+ }
+
+ if result.Observation.Resumed.Paired() != 0 {
+ t.Fatal("a resumed window was measured although the discriminator did not run")
+ }
+}
+
+// An experiment that does not ask for the discriminator must never get it, and
+// must report its absence rather than a cause it did not establish.
+func TestWithoutAPauseTheCauseIsUndiagnosedRatherThanGuessed(t *testing.T) {
+ result := runIdleness(t, newCachingService(true, false), 0)
+
+ if result.Verdict.Direction != Permitted {
+ t.Fatalf("direction = %s, want %s", result.Verdict.Direction, Permitted)
+ }
+
+ if result.Verdict.Cause != Undiagnosed {
+ t.Fatalf("cause = %s, want %s. Nothing ran that could tell the causes apart, and naming "+
+ "one anyway is a finding khaos did not measure", result.Verdict.Cause, Undiagnosed)
+ }
+}
+
+// maxDuration keeps the steady-traffic window at least as long as the pause, so
+// the discriminator's causal claim is falsifiable rather than asserted.
+func maxDuration(a, b time.Duration) time.Duration {
+ if a > b {
+ return a
+ }
+ return b
+}
+
+// The soundness condition itself. A fixed, never-re-armed cache whose lifetime
+// sits between the steady-traffic window and the pause produces exactly the
+// observation TTL-EXTENSION is built on, so khaos must refuse the causal claim
+// when its own window was too short to exclude that.
+//
+// The gate shipped with `--fault-for 8s --idle-for 14s`, which is that case.
+func TestTTLExtensionIsRefusedWhenTheSteadyWindowIsShorterThanThePause(t *testing.T) {
+ lapsed := probe.Window{}
+ for range MinimumPaired {
+ lapsed.Record(probe.Outcome{Credential: probe.Revoked, Disposition: probe.Rejected})
+ }
+
+ leaked := window(probe.Accepted, probe.Rejected, "", probe.Accepted, MinimumPaired)
+
+ base := Observation{
+ Baseline: window(probe.Accepted, probe.Rejected, "", probe.Accepted, MinimumPaired),
+ Fault: leaked,
+ Recovery: window(probe.Accepted, probe.Rejected, "", "", MinimumPaired),
+ Resumed: lapsed,
+ FaultConfirmed: true,
+ IdleFor: 14 * time.Second,
+ Defect: "revoking it at the authority",
+ }
+
+ unsound := base
+ unsound.FaultFor = 8 * time.Second
+
+ if verdict := Judge(unsound, idpPolarity); verdict.Cause != LapsedOnPause {
+ t.Fatalf("an 8s steady window against a 14s pause produced %s; a fixed lifetime between "+
+ "those two durations explains it just as well (%s)",
+ verdict.Cause, strings.Join(verdict.Reasons, "; "))
+ }
+
+ // The control. The same observation with a steady window at least as long as
+ // the pause DOES support the causal claim, or this guard would simply have
+ // deleted the diagnosis.
+ sound := base
+ sound.FaultFor = 14 * time.Second
+
+ verdict := Judge(sound, idpPolarity)
+
+ if verdict.Cause != TTLExtension {
+ t.Fatalf("a 14s steady window against a 14s pause produced %s, want %s (%s)",
+ verdict.Cause, TTLExtension, strings.Join(verdict.Reasons, "; "))
+ }
+
+ if !strings.Contains(strings.Join(verdict.Reasons, "; "), "no fixed lifetime behaves that way") {
+ t.Fatalf("the TTL-extension reason does not state why a fixed lifetime is excluded: %s",
+ strings.Join(verdict.Reasons, "; "))
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier.go
new file mode 100644
index 00000000..f36f4b15
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier.go
@@ -0,0 +1,434 @@
+// ©AngelaMos | 2026
+// idptier.go
+
+package experiment
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ ParamIssuer = "issuer"
+ ParamKeyForm = "key-form"
+
+ usageIssuer = "the issuer khaos's own identity provider must mint under, which has to be the exact string the target was configured to trust"
+
+ usageKeyForm = "which serialization of the issuer's public key to use as the HMAC secret, because the attack only works with the bytes the target actually loaded: spki-pem, spki-der, pkcs1-pem or modulus"
+
+ probeForgedName = "forged-credential"
+
+ // The identity tier's own response. It is not the proxy tier's wording,
+ // because the dependency here is reachable and answering: the question is
+ // whether the target believes what it says.
+ idpResponse = "refusing a credential its own identity provider should never have produced"
+
+ // Generous against the range, for the same reason the proxy tier's delay
+ // default is: the hold has to exceed the target's own upstream timeout or the
+ // trap is not expressible at all. A caller whose timeout is longer than the
+ // hold sees a slow SUCCESS, which is correct behaviour and not a finding.
+ defaultIdPDelay = "2s"
+
+ // How far a corrupted claim is moved. Far enough that no clock skew
+ // tolerance a sane verifier carries can absorb it: the reference verifier
+ // allows 60 seconds and real ones allow more.
+ defaultClaimShift = "1h"
+
+ // The floor, and it is not decoration. Measured: a 30 second shift against a
+ // verifier allowing the textbook 60 seconds of skew reported `PERMITTED`,
+ // defect, exit 2 against a target doing exactly what RFC 7519 section 4.1.4
+ // permits. Five minutes clears the generous configurations as well as the
+ // typical ones.
+ claimShiftFloor = 5 * time.Minute
+
+ // The same hazard for a hold. A hold a target's timeout absorbs produces a
+ // slow SUCCESS, which is correct behaviour, so khaos can only report HELD
+ // and teaches the operator nothing. One second clears the sub-second
+ // timeouts that are common and is far above the jitter the differential
+ // confirmation has to see through.
+ idpDelayFloor = time.Second
+
+ becauseClockSkew = "a claim moved less than that sits inside the clock-skew allowance RFC 7519 permits a verifier to carry, so a correct target may accept it and khaos would report that as a bypass"
+
+ becauseTimeout = "a hold shorter than that is absorbed by most callers' timeouts, so the target sees a slow success rather than a failure and the experiment can only ever report that it held"
+)
+
+// idpPolarity is the identity tier's default. Honouring a credential the
+// provider marked wrong is the defect; refusing one is the control working.
+var idpPolarity = Polarity{
+ Defects: []Direction{Permitted},
+ Rationale: "a verifier that accepts a credential its own issuer declared defective has stopped being a verifier, and every credential it will ever be shown reaches it the same way",
+}
+
+// rotationPolarity is the ONE inversion in the family, and it is why khaos
+// carries a directional axis at all. An overlapping key rotation is the
+// availability-preserving operation every provider is supposed to perform, so
+// refusing a correctly signed token under the new key is the failure and
+// accepting it is correct. Reporting that as a bypass would be backwards.
+// BOTH directions, and the second is the point. An overlapping rotation is
+// supposed to break nothing, so refusing a correctly signed new-key token is the
+// failure this experiment is named for. But a target that started accepting
+// credentials it had been refusing is a defect under every experiment khaos
+// ships, and declaring only DENIED made khaos print "this experiment declares
+// PERMITTED to be acceptable for this control" over a target that failed wide
+// open, and exit 0.
+var rotationPolarity = Polarity{
+ Defects: []Direction{Denied, Permitted},
+ Rationale: "an overlapping rotation publishes both keys precisely so nothing breaks, so a verifier that refuses a correctly signed token under the new key has turned a routine key rotation into an outage",
+}
+
+// idpParams is the identity tier's parameter set. It needs no --token: the valid
+// credential is minted by khaos's own provider, so there is nothing for the
+// operator to supply and nothing for the range to issue.
+func idpParams(extra ...Param) []Param {
+ return append([]Param{
+ {Name: ParamTarget, Usage: usageTarget, Required: true},
+ {Name: ParamURL, Usage: usageURL, Required: true},
+ {Name: ParamIssuer, Usage: usageIssuer, Required: true},
+ {Name: ParamInvalidToken, Usage: usageInvalidToken, Default: DefaultInvalidToken},
+ }, extra...)
+}
+
+var keyFormParam = Param{Name: ParamKeyForm, Usage: usageKeyForm, Default: string(idp.SPKIPEM)}
+
+var idpDelayParam = Param{Name: ParamDelay, Usage: usageIdPDelay, Default: defaultIdPDelay}
+
+var claimShiftParam = Param{Name: ParamDelay, Usage: usageClaimShift, Default: defaultClaimShift}
+
+const (
+ // The two endpoints an identity experiment can fault, as they read in the
+ // refusal an operator sees.
+ ConsultsJWKS = "khaos's key set endpoint"
+ ConsultsIntrospection = "khaos's introspection endpoint"
+
+ usageIdPDelay = "how long khaos's identity provider holds each introspection call; it has to outlast the target's own timeout to reach the branch this experiment tests"
+ usageClaimShift = "how far into the past or future to move the claim khaos corrupts"
+)
+
+// streams is how one experiment wires its probe streams. Each field is a choice
+// the experiment makes, and they are not interchangeable: the research found the
+// SAME hazard pointing in OPPOSITE directions for two experiments that look
+// alike, so none of this is defaulted.
+type streams struct {
+ // validTrapped mints the valid stream under the armed trap instead of
+ // honestly. Only the rotation experiment wants it, because there the question
+ // is whether the target accepts a correctly signed token under a NEW key.
+ validTrapped bool
+
+ // validStable mints the valid stream under a key that is always published and
+ // never rotated, so proving the target is serving does not disturb what the
+ // experiment is measuring. See Provider.StableToken.
+ validStable bool
+
+ // validOpaque gives the valid stream an OPAQUE credential, so it travels the
+ // introspection path and never touches key resolution at all.
+ //
+ // The rollover experiment requires it, and the requirement is forced rather
+ // than chosen. Research 07 section 2.3: the eviction of a retired key is a
+ // side effect of the refetch that a token under the NEW key triggers. A valid
+ // stream minting JWTs supplies exactly that traffic, so khaos was causing the
+ // recovery it then reported.
+ //
+ // Measured with the range verifier's key cache set to an hour, so its timer
+ // could not fire inside a six-second fault window: the run still returned
+ // `passed / HELD`. Nothing but khaos's own valid stream could have evicted
+ // the retired key. There is no JWT that avoids this — the retired key is
+ // withdrawn and any new-key token IS the contaminating traffic — so the
+ // stream has to leave the key path entirely.
+ validOpaque bool
+
+ // forged declares a forged stream and what it must do during the baseline.
+ // Empty means no forged stream.
+ forged probe.Expectation
+
+ // forgedFixed mints the forged credential ONCE, before the fault, instead of
+ // per sample. The traps whose credential has to predate the fault need it:
+ // a token minted after a rotation is signed by the new key and proves
+ // nothing about the old one.
+ forgedFixed func(*idp.Provider) (string, error)
+
+ // revoked declares a revoked stream, which carries a credential that was
+ // legitimately accepted and that the fault takes away.
+ revoked func(*idp.Provider) (string, error)
+}
+
+// idpPair builds the probe streams for one identity-tier experiment.
+//
+// The valid stream MINTS PER SAMPLE rather than carrying a token captured at
+// build time. That is not a convenience: the provider's state changes mid-run,
+// and an experiment whose valid stream froze a token would be unable to tell a
+// target that stopped trusting the provider from one that never picked up a
+// rotation.
+func idpPair(params *Params, provider *idp.Provider, wiring streams) (probe.Pair, error) {
+ url := params.Get(ParamURL)
+
+ mintValid := provider.HonestToken
+ if wiring.validTrapped {
+ mintValid = provider.TrappedToken
+ }
+ if wiring.validStable {
+ mintValid = provider.StableToken
+ }
+
+ valid := probe.HTTPOptions{
+ Name: probeValidName,
+ URL: url,
+ Credential: probe.Valid,
+ Mint: mintEvery(mintValid),
+ }
+
+ if wiring.validOpaque {
+ // Fixed and NOT unique per request. An opaque credential is a grant the
+ // provider holds, so a fresh one per sample would be a credential the
+ // target has never been told about. It is also deliberately NOT the
+ // revocable one, or the mutator would revoke the stream that proves the
+ // target is still serving.
+ valid = probe.HTTPOptions{
+ Name: probeValidName,
+ URL: url,
+ Credential: probe.Valid,
+ Token: provider.Opaque(),
+ }
+ }
+
+ pair := probe.Pair{
+ Valid: probe.NewHTTP(valid),
+ Invalid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeInvalidName,
+ URL: url,
+ Token: params.Get(ParamInvalidToken),
+ Credential: probe.Invalid,
+ Unique: true,
+ }),
+ }
+
+ if wiring.forged != "" {
+ forged, err := forgedProbe(url, provider, wiring)
+ if err != nil {
+ return probe.Pair{}, err
+ }
+ pair.Forged = forged
+ pair.ForgedBaseline = wiring.forged
+ }
+
+ if wiring.revoked != nil {
+ credential, err := wiring.revoked(provider)
+ if err != nil {
+ return probe.Pair{}, err
+ }
+
+ // Deliberately NOT unique per request. The whole question is whether the
+ // target's cached decision for THIS credential outlives its revocation,
+ // and a fresh identity every time would never reach that cache. The
+ // invalid control above is the opposite: it must be unique, or a cached
+ // refusal answers every probe. Same hazard, opposite directions.
+ pair.Revoked = probe.NewHTTP(probe.HTTPOptions{
+ Name: probeRevokedName,
+ URL: url,
+ Token: credential,
+ Credential: probe.Revoked,
+ })
+ }
+
+ return pair, nil
+}
+
+func forgedProbe(url string, provider *idp.Provider, wiring streams) (probe.Probe, error) {
+ options := probe.HTTPOptions{
+ Name: probeForgedName,
+ URL: url,
+ Credential: probe.Forged,
+ }
+
+ if wiring.forgedFixed != nil {
+ credential, err := wiring.forgedFixed(provider)
+ if err != nil {
+ return nil, err
+ }
+ options.Token = credential
+ return probe.NewHTTP(options), nil
+ }
+
+ options.Mint = mintEvery(provider.TrappedToken)
+
+ return probe.NewHTTP(options), nil
+}
+
+// mintEvery adapts a provider mint to the probe's per-request signature. The
+// context is unused because minting is local arithmetic with no I/O, and giving
+// it one would suggest a cancellation point that does not exist.
+func mintEvery(mint func() (string, error)) func(context.Context) (string, error) {
+ return func(context.Context) (string, error) {
+ return mint()
+ }
+}
+
+// idpSpec is everything one identity-tier experiment declares beyond its name.
+type idpSpec struct {
+ name string
+ summary string
+ condition string
+ defect string
+
+ // heldCaveat qualifies a HELD verdict. It is per experiment because only
+ // some of them measure less than an unqualified "no bypass was observed"
+ // implies.
+ heldCaveat func(*Params) string
+ trap idp.Kind
+ polarity Polarity
+ wiring streams
+ extra []Param
+
+ // consults names the provider endpoint the target has to have been using for
+ // this experiment's fault to reach it. Empty means the fault lands on a
+ // credential khaos presents, which the probe streams see directly.
+ consults string
+
+ // magnitude reads the operator's parameter into the trap. Only the traps
+ // that carry one set it.
+ magnitude func(*Params, *idp.Trap) error
+}
+
+// idpDefinition turns one declaration into a catalogue entry. Every identity
+// experiment goes through here so none of them can forget to build a provider,
+// pin the issuer, or carry its polarity through.
+func idpDefinition(spec idpSpec) Definition {
+ polarity := spec.polarity
+ if len(polarity.Defects) == 0 {
+ polarity = idpPolarity
+ }
+
+ return Definition{
+ Name: spec.name,
+ Family: FamilyIdentity,
+ Summary: spec.summary,
+ Hypothesis: Hypothesis{Condition: outageGeneric, Response: idpResponse},
+ Polarity: polarity,
+ FaultKind: inject.IdPKind(spec.trap),
+ Params: idpParams(spec.extra...),
+ Build: buildIdP(spec),
+ }
+}
+
+func buildIdP(spec idpSpec) func(*Params) (Shape, error) {
+ return func(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+ issuer := params.Get(ParamIssuer)
+
+ provider, err := idp.New(idp.Config{})
+ if err != nil {
+ return Shape{}, err
+ }
+
+ // The issuer is pinned to what the TARGET was configured to trust, not to
+ // where khaos happens to be listening. khaos binds the bridge gateway's
+ // numeric address while the target addresses `host.docker.internal`, so
+ // minting under khaos's own listener address would get every credential
+ // refused on `iss` for a reason that has nothing to do with the trap, and
+ // a refusal for the wrong reason reads exactly like a target that caught
+ // it.
+ provider.SetIssuer(issuer)
+
+ trap := idp.Trap{Kind: spec.trap}
+
+ if spec.magnitude != nil {
+ if err := spec.magnitude(params, &trap); err != nil {
+ return Shape{}, err
+ }
+ }
+
+ pair, err := idpPair(params, provider, spec.wiring)
+ if err != nil {
+ return Shape{}, err
+ }
+
+ shape := Shape{
+ Condition: fmt.Sprintf(spec.condition, target),
+ Target: target,
+ Probes: pair,
+ IdP: provider,
+ IdPTrap: &trap,
+ Defect: spec.defect,
+ }
+
+ if spec.heldCaveat != nil {
+ shape.HeldCaveat = spec.heldCaveat(params)
+ }
+
+ // The liveness guard for a fault applied to an endpoint. Both halves are
+ // set together, because a declared endpoint with no counter is a guard
+ // that can never fire and a counter with no name is a refusal that
+ // cannot say what to check.
+ if spec.consults != "" {
+ shape.ConsultedWhat = spec.consults
+ shape.Consulted = counterFor(provider, spec.consults)
+ }
+
+ // Derived from the DECLARATION rather than from a second field. A
+ // separate flag saying "read --idle-for" sat next to the parameter that
+ // declares it and carried a duration of its own that nothing used, so it
+ // read like the default while the real default lived in the Param. The
+ // two could disagree and one of them would have been silently dead.
+ if spec.wantsIdleness() {
+ idle, err := params.Duration(ParamIdleFor)
+ if err != nil {
+ return Shape{}, err
+ }
+ shape.IdleFor = idle
+ }
+
+ return shape, nil
+ }
+}
+
+// wantsIdleness reports whether this experiment declared the idleness pause,
+// which is the only thing that makes reading the parameter legal: Params refuses
+// a read of a name the experiment never declared.
+func (s idpSpec) wantsIdleness() bool {
+ for _, param := range s.extra {
+ if param.Name == ParamIdleFor {
+ return true
+ }
+ }
+ return false
+}
+
+// claimShiftMagnitude and holdMagnitude both read --delay and differ only in
+// their floor, because the two faults are absorbed by different tolerances.
+// counterFor maps the declared endpoint onto the provider's own measurement of
+// it. The two endpoint names are constants rather than free text so a typo is a
+// compile error and not a silently absent guard.
+func counterFor(provider *idp.Provider, endpoint string) func() int64 {
+ if endpoint == ConsultsJWKS {
+ return provider.JWKSFetches
+ }
+ return provider.Introspections
+}
+
+func claimShiftMagnitude(params *Params, trap *idp.Trap) error {
+ shift, err := params.AtLeast(ParamDelay, claimShiftFloor, becauseClockSkew)
+ if err != nil {
+ return err
+ }
+ trap.Delay = shift
+ return nil
+}
+
+func holdMagnitude(params *Params, trap *idp.Trap) error {
+ hold, err := params.AtLeast(ParamDelay, idpDelayFloor, becauseTimeout)
+ if err != nil {
+ return err
+ }
+ trap.Delay = hold
+ return nil
+}
+
+func keyFormMagnitude(params *Params, trap *idp.Trap) error {
+ trap.KeyForm = idp.KeyForm(params.Get(ParamKeyForm))
+ return trap.Validate()
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier_test.go
new file mode 100644
index 00000000..0da0b536
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier_test.go
@@ -0,0 +1,617 @@
+// ©AngelaMos | 2026
+// idptier_test.go
+
+package experiment
+
+import (
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ testTargetName = "a-verifier"
+ testIssuerURL = "http://host.docker.internal:45678"
+)
+
+func identityInputs() map[string]string {
+ return map[string]string{
+ ParamTarget: testTargetName,
+ ParamURL: "http://127.0.0.1:1/protected",
+ ParamIssuer: testIssuerURL,
+ }
+}
+
+// identityTier is every catalogued experiment whose fault is a trap on khaos's
+// own identity provider.
+func identityTier() []Definition {
+ out := []Definition{}
+ for _, definition := range Catalog() {
+ if inject.NeedsIdP(definition.FaultKind) {
+ out = append(out, definition)
+ }
+ }
+ return out
+}
+
+// The census, both ways. A trap with no experiment is a capability khaos built
+// and never exposed; an experiment naming a trap that is not registered cannot
+// be injected at all. Either way the catalogue and the provider have drifted.
+func TestEveryIdentityTrapHasExactlyOneExperiment(t *testing.T) {
+ byTrap := map[string][]string{}
+
+ for _, definition := range identityTier() {
+ trap := strings.TrimPrefix(definition.FaultKind, inject.KindIdPPrefix)
+ byTrap[trap] = append(byTrap[trap], definition.Name)
+ }
+
+ for _, trap := range idp.Kinds() {
+ named := byTrap[trap]
+
+ switch len(named) {
+ case 0:
+ t.Errorf("trap %s has no experiment, so khaos can produce it and no operator can ask for it", trap)
+ case 1:
+ default:
+ t.Errorf("trap %s is claimed by %v; one trap is one experiment or `khaos catalog` describes the same fault twice",
+ trap, named)
+ }
+
+ delete(byTrap, trap)
+ }
+
+ for trap, named := range byTrap {
+ t.Errorf("%v name the trap %s, which khaos's provider does not register", named, trap)
+ }
+}
+
+// Every identity experiment has to build from what the range answers and nothing
+// else, because every CLI path reaches Build through fillFromRange.
+func TestEveryIdentityExperimentBuildsFromTheRangeAnswers(t *testing.T) {
+ for _, definition := range identityTier() {
+ t.Run(definition.Name, func(t *testing.T) {
+ spec, err := Build(definition.Name, identityInputs(), Timing{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ if spec.IdP == nil {
+ t.Fatal("no provider was built, so the probes have nothing to mint from")
+ }
+
+ if spec.IdPTrap == nil {
+ t.Fatal("no trap was declared, so the run would stand up a provider and arm nothing")
+ }
+
+ // Every trap has to be armable as declared, with ONE stated
+ // exception. The revocation's credential is issued by a probe, and no
+ // probe has fired when the experiment is built, so that trap is
+ // deliberately incomplete until the mutator resolves it at inject
+ // time. The exception is asserted in both directions here, so a
+ // change that made Revoke carry a build-time credential, or that made
+ // any other trap incomplete, fails this test rather than passing it.
+ err = spec.IdPTrap.Validate()
+
+ if spec.IdPTrap.Kind == idp.Revoke {
+ if err == nil {
+ t.Fatal("the revocation trap validated at build time, so it is carrying a credential " +
+ "from before the probe that issues one has fired")
+ }
+
+ resolved := *spec.IdPTrap
+ resolved.Credential = spec.IdP.Revocable()
+
+ if err := resolved.Validate(); err != nil {
+ t.Fatalf("the revocation trap is not armable even once a credential is resolved: %v", err)
+ }
+
+ return
+ }
+
+ if err != nil {
+ t.Fatalf("the declared trap is not armable: %v", err)
+ }
+
+ if spec.Fault.Kind != inject.IdPKind(spec.IdPTrap.Kind) {
+ t.Fatalf("the catalogued fault is %q and the trap is %q; the mutator would refuse its own experiment",
+ spec.Fault.Kind, spec.IdPTrap.Kind)
+ }
+
+ if spec.Fault.Target != testTargetName {
+ t.Fatalf("the fault names %q, want the target %q", spec.Fault.Target, testTargetName)
+ }
+ })
+ }
+}
+
+// The issuer has to reach the provider. khaos binds the bridge gateway's numeric
+// address while the target addresses `host.docker.internal`, so a provider left
+// to name itself would mint tokens the target refuses on `iss` for a reason that
+// has nothing to do with the trap, and a refusal for the wrong reason reads
+// exactly like a target that caught it.
+func TestEveryIdentityExperimentMintsUnderTheIssuerItWasGiven(t *testing.T) {
+ for _, definition := range identityTier() {
+ t.Run(definition.Name, func(t *testing.T) {
+ spec, err := Build(definition.Name, identityInputs(), Timing{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ if got := spec.IdP.Issuer(); got != testIssuerURL {
+ t.Fatalf("the provider issues as %q, want %q", got, testIssuerURL)
+ }
+
+ raw, err := spec.IdP.HonestToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ token, err := idp.Parse(raw)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ if token.Claims.Issuer != testIssuerURL {
+ t.Fatalf("the minted token says iss %q, want %q", token.Claims.Issuer, testIssuerURL)
+ }
+ })
+ }
+}
+
+// The stream wiring is the experiment's whole design and it is not
+// interchangeable. These are the expectations, stated once, so a change to any
+// one experiment's wiring has to be a change here too.
+func TestEachIdentityExperimentWiresTheStreamsItsFindingNeeds(t *testing.T) {
+ type expectation struct {
+ forged probe.Expectation
+ revoked bool
+ idleness bool
+ }
+
+ want := map[string]expectation{
+ NameAlgNone: {forged: probe.ExpectAccepted},
+ NameAlgConfusion: {forged: probe.ExpectAccepted},
+ NameEmbeddedJWK: {forged: probe.ExpectAccepted},
+ NameClaimExpired: {forged: probe.ExpectAccepted},
+ NameClaimNotYetValid: {forged: probe.ExpectAccepted},
+ NameClaimWrongAudience: {forged: probe.ExpectAccepted},
+ NameClaimForeignIssuer: {forged: probe.ExpectAccepted},
+ NameSignatureTruncated: {forged: probe.ExpectAccepted},
+ NameSignatureZero: {forged: probe.ExpectAccepted},
+ NameSignatureWeakKey: {forged: probe.ExpectAccepted},
+
+ // The one forged stream that must be REJECTED during the baseline. Its
+ // credential is malformed from the moment it is minted, so a target that
+ // never refused it was not checking signatures and its acceptance during
+ // the fault proves nothing changed.
+ NameJWKSUnreachable: {forged: probe.ExpectRejected},
+
+ // Revoked, not forged: the credential was legitimate when issued and the
+ // fault is what took it away. Both bring the idleness discriminator,
+ // because both leave a cache that either re-arms or never expires.
+ NameKidRollover: {revoked: true, idleness: true},
+ NameRevokedToken: {revoked: true, idleness: true},
+
+ // Neither. The valid stream carries the new key's token and a target that
+ // refuses it registers as DENIED, which is the whole experiment.
+ NameKeyRotation: {},
+
+ // Neither. The generic invalid control already carries a credential no
+ // provider would call active, so a target that accepts it while
+ // introspection is useless has invented a value RFC 7662 never gave it.
+ NameIntrospectionError: {},
+ NameIntrospectionMalformed: {},
+ NameIntrospectionSlow: {},
+ }
+
+ tier := identityTier()
+
+ if len(want) != len(tier) {
+ t.Fatalf("this test states expectations for %d experiments and the identity tier has %d",
+ len(want), len(tier))
+ }
+
+ for _, definition := range tier {
+ t.Run(definition.Name, func(t *testing.T) {
+ expected, stated := want[definition.Name]
+ if !stated {
+ t.Fatalf("%s has no stated wiring, so nothing pins its design", definition.Name)
+ }
+
+ spec, err := Build(definition.Name, identityInputs(), Timing{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ if (spec.Probes.Forged != nil) != (expected.forged != "") {
+ t.Errorf("forged stream present = %v, want %v", spec.Probes.Forged != nil, expected.forged != "")
+ }
+
+ if spec.Probes.ForgedBaseline != expected.forged {
+ t.Errorf("forged baseline expectation = %q, want %q", spec.Probes.ForgedBaseline, expected.forged)
+ }
+
+ if (spec.Probes.Revoked != nil) != expected.revoked {
+ t.Errorf("revoked stream present = %v, want %v", spec.Probes.Revoked != nil, expected.revoked)
+ }
+
+ if (spec.IdleFor > 0) != expected.idleness {
+ t.Errorf("idleness discriminator = %v, want %v", spec.IdleFor > 0, expected.idleness)
+ }
+
+ // Every stream that can carry a finding has to name what khaos did to
+ // it, or the verdict falls back to a phrase that describes every
+ // experiment identically.
+ if spec.Probes.Forged != nil || spec.Probes.Revoked != nil {
+ if spec.Defect == "" {
+ t.Error("this experiment can produce a finding and does not say what khaos did to the credential")
+ }
+ }
+ })
+ }
+}
+
+// A forged stream whose credential is captured once would carry the baseline's
+// HONEST token into the fault window, so the trap would never be presented and
+// every forgery experiment would report a clean survival.
+func TestTheForgeryStreamsMintPerSampleAndTheFixedOnesDoNot(t *testing.T) {
+ mintsPerSample := map[string]bool{
+ NameJWKSUnreachable: false,
+ }
+
+ for _, definition := range identityTier() {
+ spec, err := Build(definition.Name, identityInputs(), Timing{})
+ if err != nil {
+ t.Fatalf("build %s: %v", definition.Name, err)
+ }
+
+ if spec.Probes.Forged == nil {
+ continue
+ }
+
+ want, stated := mintsPerSample[definition.Name]
+ if !stated {
+ want = true
+ }
+
+ // Describe is the only thing a probe exposes about how it gets its
+ // credential, and it says so explicitly for exactly this reason.
+ perSample := strings.Contains(spec.Probes.Forged.Describe(), "minted per request")
+
+ if perSample != want {
+ t.Errorf("%s mints per sample = %v, want %v (%s)",
+ definition.Name, perSample, want, spec.Probes.Forged.Describe())
+ }
+ }
+}
+
+// The valid stream proves the target is serving WITHOUT disturbing what the
+// experiment measures, and that means a different credential per family.
+//
+// - A JWT minted per sample, for the forgery and JWKS experiments. Fresh every
+// time, so a target that stopped trusting khaos's provider is distinguishable
+// from one that never picked up a rotation.
+// - A JWT under a key that is never rotated, for `kid_rollover`. A control
+// under the CURRENT signing key becomes new-key traffic after the rotation,
+// and that traffic is what evicts the retired key — khaos causing the
+// recovery it reports.
+// - A fixed OPAQUE credential, for the four introspection experiments. Those
+// fault the introspection endpoint, so the control has to travel it; a JWT
+// control is validated locally and never reaches the fault at all.
+//
+// Asserted in every direction, because each choice is a trade and getting any of
+// them backwards reintroduces a blind spot this family has already had.
+func TestEachFamilysValidControlTravelsThePathItNeedsTo(t *testing.T) {
+ opaqueControl := map[string]bool{
+ NameIntrospectionError: true,
+ NameIntrospectionMalformed: true,
+ NameIntrospectionSlow: true,
+ NameRevokedToken: true,
+ }
+
+ seenOpaque, seenMinted := false, false
+
+ for _, definition := range identityTier() {
+ spec, err := Build(definition.Name, identityInputs(), Timing{})
+ if err != nil {
+ t.Fatalf("build %s: %v", definition.Name, err)
+ }
+
+ describe := spec.Probes.Valid.Describe()
+ perSample := strings.Contains(describe, "minted per request")
+
+ if opaqueControl[definition.Name] {
+ if perSample {
+ t.Errorf("%s faults the introspection endpoint and its valid control is minted "+
+ "per sample; a JWT control is validated locally and never reaches the fault: %s",
+ definition.Name, describe)
+ }
+ seenOpaque = true
+ continue
+ }
+
+ if !perSample {
+ t.Errorf("%s freezes its valid credential: %s", definition.Name, describe)
+ }
+ seenMinted = true
+ }
+
+ if !seenOpaque || !seenMinted {
+ t.Fatal("every experiment landed in one bucket, so this test cannot tell the two apart")
+ }
+}
+
+// `kid_rollover`'s control is a JWT, so it works against a JWT-only verifier,
+// and it is signed by a key that is never rotated, so resolving it never forces
+// the refetch that evicts the retired key.
+//
+// An opaque control also avoids the observer effect and was the first fix, but
+// it demands the target support introspection — trading one blind spot for
+// another.
+func TestTheRolloverControlIsAJWTUnderAKeyThatIsNeverRotated(t *testing.T) {
+ spec, err := Build(NameKidRollover, identityInputs(), Timing{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ raw, err := spec.IdP.StableToken()
+ if err != nil {
+ t.Fatalf("mint the control: %v", err)
+ }
+
+ control, err := idp.Parse(raw)
+ if err != nil {
+ t.Fatalf("the control is not a JWT, so a JWT-only verifier cannot baseline: %v", err)
+ }
+
+ if control.Header.Kid == spec.IdP.SigningKid() {
+ t.Fatal("the control is signed by the CURRENT signing key, so after the rotation it becomes " +
+ "the new-key traffic that evicts the retired key")
+ }
+
+ if !spec.IdP.PublishedSet().Has(control.Header.Kid) {
+ t.Fatal("the control's key is not published, so it cannot be resolved without a refetch")
+ }
+
+ // And it survives the rotation still published and still not the signing key,
+ // which is what stops it triggering a refetch mid-experiment.
+ if err := spec.IdP.Arm(idp.Trap{Kind: idp.KidRollover}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ if !spec.IdP.PublishedSet().Has(control.Header.Kid) {
+ t.Fatal("the rotation withdrew the control's key")
+ }
+
+ if control.Header.Kid == spec.IdP.SigningKid() {
+ t.Fatal("the rotation made the control's key the signing key")
+ }
+}
+
+// The rotation experiment is the only inversion in the family, and it is the
+// only thing in the whole catalogue that makes the directional axis anything but
+// decoration. If it ever stops treating DENIED as a defect, nothing else
+// exercises that branch.
+func TestTheRotationExperimentIsTheOnlyOneWhereRefusingIsTheDefect(t *testing.T) {
+ inverted := []string{}
+
+ for _, definition := range Catalog() {
+ if definition.Polarity.IsDefect(Denied) {
+ inverted = append(inverted, definition.Name)
+ }
+ }
+
+ if !slices.Equal(inverted, []string{NameKeyRotation}) {
+ t.Fatalf("the experiments treating DENIED as a defect are %v, want exactly [%s]",
+ inverted, NameKeyRotation)
+ }
+}
+
+// And PERMITTED is a defect for EVERY experiment khaos currently ships. The
+// rotation experiment declared only DENIED, so a target that started accepting
+// credentials it had been refusing drew "this experiment declares PERMITTED to
+// be acceptable for this control" and exit 0 — a CI pass over a target that
+// failed wide open.
+//
+// This is asserted over the whole catalogue rather than for that one experiment,
+// because the next inverted-polarity experiment will hit the same trap.
+func TestEveryExperimentTreatsPermittedAsADefect(t *testing.T) {
+ for _, definition := range Catalog() {
+ if !definition.Polarity.IsDefect(Permitted) {
+ t.Errorf("%s does not treat PERMITTED as a defect, so a target that failed open "+
+ "would be reported as acceptable and exit 0", definition.Name)
+ }
+
+ if len(definition.Polarity.Defects) == 0 {
+ t.Errorf("%s declares no defect direction at all", definition.Name)
+ }
+
+ if definition.Polarity.Rationale == "" {
+ t.Errorf("%s declares a polarity with no rationale", definition.Name)
+ }
+ }
+}
+
+// An unknown key form has to be refused at build time with the flag named,
+// rather than reaching the provider and producing a MAC over nothing.
+func TestAnUnknownKeyFormIsRefusedByName(t *testing.T) {
+ inputs := identityInputs()
+ inputs[ParamKeyForm] = "pem-ish"
+
+ _, err := Build(NameAlgConfusion, inputs, Timing{})
+ if err == nil {
+ t.Fatal("an unknown key form was accepted")
+ }
+
+ if !strings.Contains(err.Error(), "pem-ish") {
+ t.Fatalf("the refusal does not name what was typed: %v", err)
+ }
+}
+
+// Every key form the provider advertises has to be buildable, or `--help` lists
+// a value that does not work.
+func TestEveryAdvertisedKeyFormBuilds(t *testing.T) {
+ for _, form := range idp.KeyForms() {
+ inputs := identityInputs()
+ inputs[ParamKeyForm] = form
+
+ if _, err := Build(NameAlgConfusion, inputs, Timing{}); err != nil {
+ t.Errorf("key form %q is advertised and does not build: %v", form, err)
+ }
+ }
+}
+
+// The liveness guard for a fault applied to an ENDPOINT. Declared per
+// experiment, asserted in both directions, so neither dropping it from one of
+// the four nor adding it to an experiment that faults a credential passes.
+func TestOnlyTheEndpointFaultsDeclareAConsultedGuard(t *testing.T) {
+ want := map[string]string{
+ NameJWKSUnreachable: ConsultsJWKS,
+ NameIntrospectionError: ConsultsIntrospection,
+ NameIntrospectionMalformed: ConsultsIntrospection,
+ NameIntrospectionSlow: ConsultsIntrospection,
+ }
+
+ for _, definition := range identityTier() {
+ spec, err := Build(definition.Name, identityInputs(), Timing{})
+ if err != nil {
+ t.Fatalf("build %s: %v", definition.Name, err)
+ }
+
+ expected := want[definition.Name]
+
+ if spec.ConsultedWhat != expected {
+ t.Errorf("%s declares the guard %q, want %q", definition.Name, spec.ConsultedWhat, expected)
+ }
+
+ if (spec.Consulted != nil) != (expected != "") {
+ t.Errorf("%s has a counter = %v and a declared endpoint = %v; a declared endpoint with "+
+ "no counter is a guard that can never fire",
+ definition.Name, spec.Consulted != nil, expected != "")
+ }
+ }
+}
+
+// And the guard has to actually void the run. A target that validates
+// credentials locally never calls introspection, so its baseline looks healthy
+// and its fault window looks identical — `passed / HELD`, conclusive, over a
+// fault that never reached it.
+func TestATargetThatNeverUsedTheFaultedEndpointVoidsTheRun(t *testing.T) {
+ healthy := window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed)
+
+ observation := Observation{
+ Baseline: healthy,
+ Fault: healthy,
+ Recovery: healthy,
+ FaultConfirmed: true,
+ ConfirmMethod: "khaos read a 503 off its own introspection endpoint",
+ ConsultedWhat: ConsultsIntrospection,
+ Consulted: 0,
+ }
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Epistemic != BaselineNotMet {
+ t.Fatalf("epistemic = %s, want %s (%s)", verdict.Epistemic, BaselineNotMet,
+ strings.Join(verdict.Reasons, "; "))
+ }
+
+ if !strings.Contains(strings.Join(verdict.Reasons, "; "), ConsultsIntrospection) {
+ t.Fatalf("the refusal does not name the endpoint: %s", strings.Join(verdict.Reasons, "; "))
+ }
+
+ // The control: the same run where the target DID use the endpoint is judged
+ // normally. Without this, a guard that always fired would pass the assertion
+ // above.
+ observation.Consulted = 1
+
+ if verdict := Judge(observation, idpPolarity); verdict.Epistemic != Passed {
+ t.Fatalf("a target that used the endpoint was judged %s, want %s (%s)",
+ verdict.Epistemic, Passed, strings.Join(verdict.Reasons, "; "))
+ }
+}
+
+// An experiment that declares no endpoint must be unaffected, or adding the
+// guard would have voided the fourteen experiments that fault a credential.
+func TestAnExperimentWithNoDeclaredEndpointIsNotGuarded(t *testing.T) {
+ healthy := window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed)
+
+ observation := Observation{
+ Baseline: healthy,
+ Fault: healthy,
+ Recovery: healthy,
+ FaultConfirmed: true,
+ Consulted: 0,
+ }
+
+ if verdict := Judge(observation, idpPolarity); verdict.Epistemic != Passed {
+ t.Fatalf("epistemic = %s, want %s", verdict.Epistemic, Passed)
+ }
+}
+
+// The two experiments whose absence of a finding covers less than it reads.
+// Asserted in both directions: the other sixteen must carry no caveat, or a
+// caveat on everything would say nothing.
+func TestOnlyTheExperimentsThatMeasureLessCarryAHeldCaveat(t *testing.T) {
+ caveated := map[string]string{
+ NameAlgConfusion: "serialization",
+ NameIntrospectionSlow: "timeout is longer",
+ }
+
+ for _, definition := range identityTier() {
+ spec, err := Build(definition.Name, identityInputs(), Timing{})
+ if err != nil {
+ t.Fatalf("build %s: %v", definition.Name, err)
+ }
+
+ fragment, expected := caveated[definition.Name]
+
+ if (spec.HeldCaveat != "") != expected {
+ t.Errorf("%s carries a HELD caveat = %v, want %v (%q)",
+ definition.Name, spec.HeldCaveat != "", expected, spec.HeldCaveat)
+ continue
+ }
+
+ if expected && !strings.Contains(spec.HeldCaveat, fragment) {
+ t.Errorf("%s's caveat does not mention %q: %s", definition.Name, fragment, spec.HeldCaveat)
+ }
+ }
+}
+
+// And the caveat has to reach the verdict, which only happens on HELD.
+func TestTheHeldCaveatIsStatedOnlyWhenTheTargetHeld(t *testing.T) {
+ const caveat = "this run tried one of four serializations"
+
+ held := Observation{
+ Baseline: window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed),
+ Fault: window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed),
+ Recovery: window(probe.Accepted, probe.Rejected, "", "", MinimumToClaimClosed),
+ FaultConfirmed: true,
+ HeldCaveat: caveat,
+ }
+
+ verdict := Judge(held, idpPolarity)
+
+ if verdict.Direction != Held {
+ t.Fatalf("direction = %s, want %s", verdict.Direction, Held)
+ }
+
+ if !strings.Contains(strings.Join(verdict.Reasons, "; "), caveat) {
+ t.Fatalf("a HELD verdict did not state its caveat, so khaos claims it ruled out more than "+
+ "it tried: %s", strings.Join(verdict.Reasons, "; "))
+ }
+
+ // A run that found something does not need the caveat: it found the thing
+ // the caveat warns might have been missed.
+ leaked := held
+ leaked.Fault = window(probe.Accepted, probe.Accepted, "", "", MinimumPaired)
+
+ if found := Judge(leaked, idpPolarity); strings.Contains(strings.Join(found.Reasons, "; "), caveat) {
+ t.Fatalf("a run that found a bypass still warned that it might have missed one: %s",
+ strings.Join(found.Reasons, "; "))
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go
new file mode 100644
index 00000000..67a99e95
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go
@@ -0,0 +1,323 @@
+// ©AngelaMos | 2026
+// integration_test.go
+
+package experiment
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+const (
+ goodToken = "a-credential-the-service-accepts"
+ bearer = "Bearer "
+ gatewayURL = "/protected"
+)
+
+var fastTiming = Timing{
+ SampleInterval: time.Millisecond,
+ BaselineFor: 20 * time.Millisecond,
+ FaultFor: 20 * time.Millisecond,
+ RecoveryFor: 10 * time.Millisecond,
+}
+
+// service models a control in front of a dependency. When the dependency is
+// up it verifies credentials. When the dependency goes down it does whatever
+// its policy says, which is the thing khaos exists to measure.
+type service struct {
+ dependencyUp atomic.Bool
+ failOpen bool
+}
+
+func (s *service) handler() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ token := strings.TrimPrefix(r.Header.Get("Authorization"), bearer)
+
+ if !s.dependencyUp.Load() {
+ if s.failOpen {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ w.WriteHeader(http.StatusForbidden)
+ return
+ }
+
+ if token != goodToken {
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ w.WriteHeader(http.StatusOK)
+ })
+}
+
+type serviceInjector struct {
+ svc *service
+}
+
+func (i *serviceInjector) Inject(context.Context, string, string) error {
+ i.svc.dependencyUp.Store(false)
+ return nil
+}
+
+func (i *serviceInjector) Confirm(context.Context) (string, error) {
+ if i.svc.dependencyUp.Load() {
+ return "", nil
+ }
+ return "the dependency reads as down", nil
+}
+
+func (i *serviceInjector) Revert(context.Context) error {
+ i.svc.dependencyUp.Store(true)
+ return nil
+}
+
+func runAgainst(t *testing.T, svc *service) Result {
+ t.Helper()
+
+ svc.dependencyUp.Store(true)
+
+ server := httptest.NewServer(svc.handler())
+ t.Cleanup(server.Close)
+
+ spec, err := Build(NameDependencyOutage, map[string]string{
+ ParamTarget: "the-dependency",
+ ParamURL: server.URL + gatewayURL,
+ ParamToken: goodToken,
+ }, fastTiming)
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ engine := Engine{Injector: &serviceInjector{svc: svc}}
+
+ result, err := engine.Run(context.Background(), spec)
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ return result
+}
+
+func TestAControlThatFailsClosedIsReportedAsDenied(t *testing.T) {
+ result := runAgainst(t, &service{failOpen: false})
+
+ if result.Verdict.Direction != Denied {
+ t.Fatalf("direction = %s, want %s (%s)", result.Verdict.Direction, Denied,
+ strings.Join(result.Verdict.Reasons, "; "))
+ }
+
+ if result.Verdict.Defect {
+ t.Fatal("failing closed was flagged as a defect for a control whose declared defect is failing open")
+ }
+
+ if result.Verdict.Epistemic != Deviated {
+ t.Fatalf("epistemic = %s, want %s; refusing legitimate traffic is a deviation even though "+
+ "it is the safe direction", result.Verdict.Epistemic, Deviated)
+ }
+}
+
+func TestAControlThatFailsOpenIsCaughtAndFlaggedADefect(t *testing.T) {
+ result := runAgainst(t, &service{failOpen: true})
+
+ if result.Verdict.Direction != Permitted {
+ t.Fatalf("direction = %s, want %s (%s)", result.Verdict.Direction, Permitted,
+ strings.Join(result.Verdict.Reasons, "; "))
+ }
+
+ if !result.Verdict.Defect {
+ t.Fatal("a control that started honouring a credential it had been rejecting was not " +
+ "flagged as a defect")
+ }
+
+ if !result.Verdict.Conclusive {
+ t.Fatal("the finding was not marked conclusive")
+ }
+}
+
+func TestTheProbeReadsTheBodyAndClassifiesByStatus(t *testing.T) {
+ cases := []struct {
+ status int
+ want string
+ }{
+ {http.StatusOK, "accepted"},
+ {http.StatusNoContent, "accepted"},
+ {http.StatusUnauthorized, "rejected"},
+ {http.StatusForbidden, "rejected"},
+ {http.StatusTeapot, "rejected"},
+ {http.StatusBadRequest, "rejected"},
+ {http.StatusNotFound, "rejected"},
+ {http.StatusMethodNotAllowed, "rejected"},
+ {http.StatusTooManyRequests, "throttled"},
+ {http.StatusMovedPermanently, "redirected"},
+ {http.StatusFound, "redirected"},
+ {http.StatusSeeOther, "redirected"},
+ {http.StatusTemporaryRedirect, "redirected"},
+ {http.StatusInternalServerError, "unavailable"},
+ {http.StatusBadGateway, "unavailable"},
+ }
+
+ for _, tc := range cases {
+ t.Run(http.StatusText(tc.status), func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(tc.status)
+ }))
+ defer server.Close()
+
+ spec, err := Build(NameDependencyOutage, map[string]string{
+ ParamTarget: "x",
+ ParamURL: server.URL,
+ ParamToken: goodToken,
+ }, fastTiming)
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ outcome := spec.Probes.Valid.Run(context.Background())
+ if outcome.Disposition.String() != tc.want {
+ t.Fatalf("status %d classified as %s, want %s", tc.status, outcome.Disposition, tc.want)
+ }
+
+ if outcome.Status != tc.status {
+ t.Fatalf("recorded status %d, want %d", outcome.Status, tc.status)
+ }
+ })
+ }
+}
+
+func TestAnUnreachableEndpointIsNotMistakenForARejection(t *testing.T) {
+ spec, err := Build(NameDependencyOutage, map[string]string{
+ ParamTarget: "x",
+ ParamURL: "http://127.0.0.1:1/never-listening",
+ ParamToken: goodToken,
+ }, fastTiming)
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ outcome := spec.Probes.Invalid.Run(context.Background())
+
+ if outcome.Disposition != probe.Unavailable {
+ t.Fatalf("a connection failure was classified as %s; reading it as a rejection would "+
+ "make an outage look like a control doing its job", outcome.Disposition)
+ }
+}
+
+func TestARedirectingGatewayIsNamedNotBlamed(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.TrimPrefix(r.Header.Get("Authorization"), bearer) != goodToken {
+ http.Redirect(w, r, "/login", http.StatusFound)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+
+ spec, err := Build(NameDependencyOutage, map[string]string{
+ ParamTarget: "dep",
+ ParamURL: server.URL + gatewayURL,
+ ParamToken: goodToken,
+ }, fastTiming)
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ svc := &service{}
+ svc.dependencyUp.Store(true)
+
+ engine := Engine{Injector: &serviceInjector{svc: svc}}
+
+ result, err := engine.Run(context.Background(), spec)
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ if result.Verdict.Epistemic != BaselineNotMet {
+ t.Fatalf("epistemic = %s, want %s", result.Verdict.Epistemic, BaselineNotMet)
+ }
+
+ joined := strings.Join(result.Verdict.Reasons, " ")
+ if !strings.Contains(joined, "redirect") {
+ t.Fatalf("khaos refused to run against a redirecting gateway without naming the redirect "+
+ "as the cause: %q", joined)
+ }
+
+ if !strings.Contains(joined, "--url") {
+ t.Fatalf("the refusal names a cause but no fix: %q", joined)
+ }
+}
+
+func TestABaselineBlipDoesNotFabricateABypass(t *testing.T) {
+ baseline := healthyBaseline()
+ baseline.Invalid = stream(probe.Invalid,
+ probe.Rejected, probe.Accepted, probe.Rejected, probe.Rejected, probe.Rejected)
+
+ clean := probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, MinimumPaired)...),
+ }
+
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: baseline,
+ Fault: clean,
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Direction == Permitted {
+ t.Fatal("one flaky acceptance during the BASELINE was reported as a credential being " +
+ "honoured during the fault; the fault window was spotless")
+ }
+}
+
+func TestAProvenBypassIsReportedEvenOnThinEvidence(t *testing.T) {
+ thinButDamning := probe.Window{
+ Valid: stream(probe.Valid, probe.Accepted, probe.Accepted),
+ Invalid: stream(probe.Invalid, probe.Accepted, probe.Accepted),
+ }
+
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: healthyBaseline(),
+ Fault: thinButDamning,
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Direction != Permitted {
+ t.Fatalf("direction = %s; a credential the target had already refused being honoured even "+
+ "once is PROOF of a bypass, and burying it as thin evidence loses the finding "+
+ "khaos exists to produce", verdict.Direction)
+ }
+
+ if !verdict.Defect {
+ t.Fatal("the proven bypass was not flagged as a defect")
+ }
+}
+
+func TestACleanRunOnThinEvidenceIsNotConclusive(t *testing.T) {
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: healthyBaseline(),
+ Fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, MinimumPaired)...),
+ },
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Direction != Held {
+ t.Fatalf("direction = %s, want %s", verdict.Direction, Held)
+ }
+
+ if verdict.Conclusive {
+ t.Fatalf("a clean result on %d samples was called conclusive; the absence of a bypass "+
+ "needs a sample-size argument that %d does not supply", MinimumPaired, MinimumPaired)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/introspection.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/introspection.go
new file mode 100644
index 00000000..a9cd49b2
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/introspection.go
@@ -0,0 +1,129 @@
+// ©AngelaMos | 2026
+// introspection.go
+
+package experiment
+
+import (
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+)
+
+// The introspection experiments. All four leave khaos's provider REACHABLE and
+// ANSWERING, which is what separates them from `identity.idp_outage` and from
+// every `network.` experiment.
+//
+// That distinction is the point, not a detail. A caller's `err != nil` branch is
+// never taken here, so only its status handling, its decoder's error branch or
+// its timeout decides what happens. Those are different lines of code from the
+// outage path, reached by different bugs, and taxonomy row F20 is explicit that
+// collapsing a denial into an unknown is its own fail-open: returning 401 in one
+// phase and 503 in another and getting identical caller behaviour IS the
+// finding.
+//
+// None of the four needs a forged stream. The generic invalid control already
+// carries a credential no provider would call active, so a target that accepts
+// it while introspection is useless has invented a value RFC 7662 never gave it.
+const (
+ NameIntrospectionError = "identity.introspection_error"
+ NameIntrospectionMalformed = "identity.malformed_response"
+ NameIntrospectionSlow = "identity.idp_latency"
+ NameRevokedToken = "identity.revoked_token"
+)
+
+const (
+ summaryIntrospectionError = "make the identity provider answer introspection with a 503 while staying reachable, which asks whether the target treats a refusal it cannot parse as permission"
+
+ summaryIntrospectionMalformed = "answer introspection with a 200 and a body that is not JSON, which asks what the target's decode-error branch decided to do"
+
+ summaryIntrospectionSlow = "hold introspection past the target's own timeout while answering everything else promptly, which asks whether a slow authority and an absent one are the same thing to it"
+
+ summaryRevokedToken = "revoke a credential at a provider the target can still reach, and measure how long it keeps honouring it; this is the exposure window every revocation actually has"
+)
+
+const (
+ conditionIntrospectionError = "%s receiving a 503 from an identity provider it can still reach"
+ conditionIntrospectionMalformed = "%s receiving an unparseable introspection response"
+ conditionIntrospectionSlow = "%s waiting longer than its own timeout for an introspection answer"
+ conditionRevokedToken = "%s holding a credential its provider has revoked and can be asked about"
+)
+
+// The provider is reachable and will answer `active: false` on request. That
+// contrast with the outage experiment is the finding's whole point, so the
+// phrase has to carry it.
+const caveatHoldAbsorbed = "khaos held introspection for %s; a target whose own timeout is longer than that saw a slow success rather than a failure, so this run does not rule out a fail-open on a longer hold"
+
+const defectRevokedToken = "revoking it at a provider the target can still reach and ask"
+
+// The idleness discriminator applies here too, and this is where the contrast
+// with `identity.idp_outage` becomes a teaching point rather than a nuance. There
+// the authority is unreachable, so a stale decision is at least explicable. Here
+// the authority is answering `active: false` and the target is simply not
+// asking, and the pause tells a cache being re-armed from one with no expiry at
+// all.
+const defaultRevokeIdleFor = "6s"
+
+func init() {
+ for _, spec := range []idpSpec{
+ {
+ name: NameIntrospectionError,
+ summary: summaryIntrospectionError,
+ condition: conditionIntrospectionError,
+ trap: idp.IntrospectionError,
+ consults: ConsultsIntrospection,
+ wiring: streams{validOpaque: true},
+ },
+ {
+ name: NameIntrospectionMalformed,
+ summary: summaryIntrospectionMalformed,
+ condition: conditionIntrospectionMalformed,
+ trap: idp.IntrospectionMalformed,
+ consults: ConsultsIntrospection,
+ wiring: streams{validOpaque: true},
+ },
+ {
+ name: NameIntrospectionSlow,
+ summary: summaryIntrospectionSlow,
+ condition: conditionIntrospectionSlow,
+ trap: idp.IntrospectionSlow,
+ consults: ConsultsIntrospection,
+ wiring: streams{validOpaque: true},
+ magnitude: holdMagnitude,
+ extra: []Param{idpDelayParam},
+ // A hold the target's timeout absorbs is a slow SUCCESS, which is
+ // correct behaviour. khaos cannot discover a timeout it is probing
+ // from outside, so a HELD here has to say which hold it used.
+ heldCaveat: func(params *Params) string {
+ return sprintf(caveatHoldAbsorbed, params.Get(ParamDelay))
+ },
+ },
+ {
+ name: NameRevokedToken,
+ summary: summaryRevokedToken,
+ condition: conditionRevokedToken,
+ defect: defectRevokedToken,
+ trap: idp.Revoke,
+ // The revoked stream carries an OPAQUE credential, because a locally
+ // validated JWT cannot be revoked at all: research 07's asymmetry is
+ // that local validation does not fail open during a provider outage,
+ // it fails INVISIBLE, and revocation is a no-op until `exp`. Asking
+ // this question of a JWT would measure nothing.
+ // The valid control is opaque too. This experiment revokes an opaque
+ // credential and asks whether the target re-checks it, so the target
+ // must be one that introspects at all — and a JWT control would be
+ // validated locally, prove the target is serving, and prove nothing
+ // about the path under test.
+ wiring: streams{revoked: revocableCredential, validOpaque: true},
+ extra: []Param{{Name: ParamIdleFor, Usage: usageIdleFor, Default: defaultRevokeIdleFor}},
+ },
+ } {
+ definition := idpDefinition(spec)
+ catalog[definition.Name] = definition
+ }
+}
+
+// revocableCredential issues the opaque grant this run will revoke and records
+// it on the provider as such. The mutator reads it back at inject time, because
+// the trap cannot carry a credential that does not exist when the experiment is
+// built.
+func revocableCredential(provider *idp.Provider) (string, error) {
+ return provider.Revocable(), nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/keys.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/keys.go
new file mode 100644
index 00000000..52d950c6
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/keys.go
@@ -0,0 +1,141 @@
+// ©AngelaMos | 2026
+// keys.go
+
+package experiment
+
+import (
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+// The key experiments. These three change what khaos's provider SERVES rather
+// than what it mints, and between them they cover the two halves of key rotation
+// plus the trap research 07 calls the highest-value one in its archive.
+//
+// They need almost no new machinery, and that is a result rather than a
+// coincidence: `kid_rollover` is a credential that was legitimately accepted and
+// then taken away, which is exactly what the revoked stream already means, and
+// `key_rotation` is a correctly signed credential the target may refuse, which
+// is exactly what the directional axis already measures.
+const (
+ NameJWKSUnreachable = "identity.jwks_unreachable"
+ NameKidRollover = "identity.kid_rollover"
+ NameKeyRotation = "identity.key_rotation"
+)
+
+const (
+ summaryJWKSUnreachable = "take away the key set while leaving everything else up, then present a token whose signature cannot verify; a 2xx means the target stops checking signatures exactly when it cannot check them"
+
+ summaryKidRollover = "rotate the signing key and withdraw the retired one, then replay a token signed by it; a 2xx means rotation is not a revocation mechanism for this target and a leaked key stays usable"
+
+ summaryKeyRotation = "rotate the signing key with BOTH keys published, which is the rotation every provider is supposed to be able to perform, and find out whether the target picks the new one up or turns a routine rotation into an outage"
+)
+
+const (
+ conditionJWKSUnreachable = "%s being unable to fetch its issuer's key set while the issuer is otherwise up"
+ conditionKidRollover = "%s losing the key that signed a token it had been honouring"
+ conditionKeyRotation = "%s being offered a token under its issuer's newly rotated key"
+)
+
+const (
+ defectJWKSUnreachable = "a signature that does not verify under any published key"
+ // A withdrawal, not a revocation call, and the provider stays reachable
+ // throughout. Saying "while the authority was unreachable" here would be
+ // false.
+ defectKidRollover = "withdrawing the key that signed it from a key set the target can still fetch"
+)
+
+// The idleness discriminator applies to the rollover, and it is the same
+// question in a different place: a target still honouring a withdrawn key either
+// re-arms its key cache on every failed refetch, or never expires it at all.
+// Those are two different fixes.
+const defaultKeyIdleFor = "6s"
+
+func init() {
+ for _, spec := range []idpSpec{
+ {
+ name: NameJWKSUnreachable,
+ summary: summaryJWKSUnreachable,
+ condition: conditionJWKSUnreachable,
+ defect: defectJWKSUnreachable,
+ trap: idp.JWKSUnreachable,
+ consults: ConsultsJWKS,
+ wiring: streams{
+ // REJECTED, the opposite of every forgery experiment, and the
+ // reason the expectation is declared rather than derived. This
+ // credential is malformed from the moment it is minted, so a
+ // target that did not refuse it while the key set was reachable
+ // was never checking signatures and its acceptance during the
+ // fault proves nothing changed.
+ forged: probe.ExpectRejected,
+ forgedFixed: corruptedToken,
+ },
+ },
+ {
+ name: NameKidRollover,
+ summary: summaryKidRollover,
+ condition: conditionKidRollover,
+ defect: defectKidRollover,
+ trap: idp.KidRollover,
+ // The revoked stream, not the forged one. This credential was
+ // LEGITIMATE when it was issued and the rotation is what took it
+ // away, which is the revoked stream's exact meaning, and it brings
+ // the idleness discriminator with it for free.
+ //
+ // The valid stream is a JWT under a key that is NEVER rotated, and
+ // this experiment is not correct without that. A valid stream under
+ // the current signing key supplies the new-key traffic that evicts
+ // the retired one, so khaos caused the recovery it then reported:
+ // measured with the verifier's key cache set to an hour, the run
+ // still returned `passed / HELD` against a target with an unbounded
+ // stale window.
+ //
+ // An opaque credential also avoids that and was the first fix, but it
+ // demands the target support introspection — a JWT-only verifier
+ // cannot establish a baseline at all, which trades one blind spot for
+ // another.
+ wiring: streams{revoked: honestToken, validStable: true},
+ extra: []Param{{Name: ParamIdleFor, Usage: usageIdleFor, Default: defaultKeyIdleFor}},
+ },
+ {
+ name: NameKeyRotation,
+ summary: summaryKeyRotation,
+ condition: conditionKeyRotation,
+ trap: idp.KeyRotation,
+ polarity: rotationPolarity,
+ // No forged stream and no revoked one. The VALID stream mints under
+ // the armed trap, so it carries the new key's token during the fault,
+ // and a target that refuses it registers on the directional axis as
+ // DENIED. There is nothing forged anywhere in this experiment.
+ wiring: streams{validTrapped: true},
+ },
+ } {
+ definition := idpDefinition(spec)
+ catalog[definition.Name] = definition
+ }
+}
+
+// honestToken mints the credential the rollover will orphan. It is minted at
+// BUILD time, which is before the fault by construction, so it is signed by the
+// key the rotation is about to withdraw. A credential minted after the rotation
+// would be signed by the new key and would prove nothing.
+func honestToken(provider *idp.Provider) (string, error) {
+ return provider.HonestToken()
+}
+
+// corruptedToken mints a correct credential and then breaks one bit of its
+// signature. Everything else about it is right: the header, the kid, the claims
+// and the signature's length, so the only thing a verifier can refuse it for is
+// the signature itself.
+//
+// That precision is the whole experiment. A garbage string would be refused by
+// anything, including a verifier that skipped signature verification entirely,
+// so it could never distinguish the two.
+func corruptedToken(provider *idp.Provider) (string, error) {
+ honest, err := provider.HonestToken()
+ if err != nil {
+ return "", err
+ }
+
+ return idp.CorruptSignature(honest)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/network.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/network.go
new file mode 100644
index 00000000..b7060f5e
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/network.go
@@ -0,0 +1,187 @@
+// ©AngelaMos | 2026
+// network.go
+
+package experiment
+
+import (
+ "fmt"
+
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+)
+
+const (
+ NameDependencyHang = "network.dependency_hang"
+ NameDependencyReset = "network.dependency_reset"
+ NameDependencyEOF = "network.dependency_eof"
+ NameDependencyLatency = "network.dependency_latency"
+ NameDependencyPartial = "network.dependency_partial"
+ NameDependencySlowClose = "network.dependency_slow_close"
+
+ ParamDelay = "delay"
+ ParamBytes = "bytes"
+
+ // Where khaos binds its fault proxy, and what that proxy forwards to.
+ //
+ // They are declared PARAMETERS rather than something startProxy discovers,
+ // because discovery made every proxy experiment require the bundled range.
+ // khaos's own refusal text invited the opposite — "or point khaos at your
+ // own stack instead" — and then failed when an operator did.
+ //
+ // The range answers both, so a run against it still takes no flags.
+ ParamProxyListen = "proxy-listen"
+ ParamProxyUpstream = "proxy-upstream"
+
+ usageProxyListen = "the host address khaos binds its fault proxy on, which must be the address your target already dials for this dependency; it has to be a Docker bridge address, because a container cannot reach loopback and khaos will not bind a wildcard"
+ usageProxyUpstream = "the real dependency khaos's proxy forwards to, as host:port"
+
+ usageDelay = "how long khaos's proxy holds each exchange with the dependency"
+ usageBytes = "how many bytes of the dependency's answer to deliver before closing"
+
+ // Two seconds is past the range gateway's own upstream timeout, so the
+ // default reaches the branch the experiment exists to test rather than
+ // producing a slow success.
+ defaultDelay = "2s"
+
+ // Enough for a status line and the start of a body, so the target receives a
+ // well-formed beginning and a truncated end. That is the shape that finds a
+ // parser treating a short read as an empty-but-successful answer.
+ defaultBytes = "16"
+
+ hangCondition = "%s accepting connections and never answering them"
+ resetCondition = "%s resetting the connections its caller opens"
+ eofCondition = "%s closing the connections its caller opens without answering"
+ latencyCondition = "every exchange with %s taking %s longer"
+ partialCondition = "%s answering with only the first %d bytes and then closing"
+ slowCloseCondition = "%s answering in full but holding the connection open %s longer"
+
+ hangSummary = "accept the target's connections to its dependency and never answer, which is the failure shape that finds a missing timeout"
+ resetSummary = "reset the target's connections to its dependency, which is the failure shape that finds a swallowed error in the deny path"
+ eofSummary = "close the target's connections to its dependency cleanly, which is what a stopped dependency looks like to its peer"
+ latencySummary = "slow every exchange with the dependency, which finds a timeout that is too generous rather than missing"
+ partialSummary = "truncate the dependency's answer, which finds a parser treating a short read as an empty success"
+ slowCloseSummary = "answer in full but never close, which stalls a caller that waits for the close rather than for a length"
+
+ proxyResponse = "refusing credentials it would have refused anyway, rather than letting them through"
+)
+
+var proxyPolarity = Polarity{
+ Defects: []Direction{Permitted},
+ Rationale: "a control that stops rejecting invalid credentials when its dependency misbehaves has stopped being a control",
+}
+
+func proxyParams(extra ...Param) []Param {
+ return append([]Param{
+ {Name: ParamTarget, Usage: usageTarget, Required: true},
+ {Name: ParamURL, Usage: usageURL, Required: true},
+ {Name: ParamToken, Usage: usageToken, Required: true},
+ {Name: ParamProxyListen, Usage: usageProxyListen, Required: true},
+ {Name: ParamProxyUpstream, Usage: usageProxyUpstream, Required: true},
+ {Name: ParamInvalidToken, Usage: usageInvalidToken, Default: DefaultInvalidToken},
+ }, extra...)
+}
+
+// proxyEndpoint reads the two addresses every proxy-tier experiment needs.
+func proxyEndpoint(params *Params) (string, string) {
+ return params.Get(ParamProxyListen), params.Get(ParamProxyUpstream)
+}
+
+var delayParam = Param{Name: ParamDelay, Usage: usageDelay, Default: defaultDelay}
+
+var bytesParam = Param{Name: ParamBytes, Usage: usageBytes, Default: defaultBytes}
+
+// plainProxyShape covers the three faults that carry no magnitude. The condition
+// is resolved against the real target so the hypothesis names what was broken.
+func plainProxyShape(kind proxy.Kind, condition string) func(*Params) (Shape, error) {
+ return func(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+
+ listen, upstream := proxyEndpoint(params)
+
+ return Shape{
+ Condition: fmt.Sprintf(condition, target),
+ Target: target,
+ Probes: probePair(params),
+ ProxyFault: &proxy.Fault{Kind: kind},
+ ProxyListen: listen,
+ ProxyUpstream: upstream,
+ }, nil
+ }
+}
+
+func delayedProxyShape(kind proxy.Kind, condition string) func(*Params) (Shape, error) {
+ return func(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+
+ delay, err := params.Duration(ParamDelay)
+ if err != nil {
+ return Shape{}, err
+ }
+
+ listen, upstream := proxyEndpoint(params)
+
+ return Shape{
+ Condition: fmt.Sprintf(condition, target, delay),
+ Target: target,
+ Probes: probePair(params),
+ ProxyFault: &proxy.Fault{Kind: kind, Delay: delay},
+ ProxyListen: listen,
+ ProxyUpstream: upstream,
+ }, nil
+ }
+}
+
+func buildDependencyPartial(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+
+ budget, err := params.Count(ParamBytes)
+ if err != nil {
+ return Shape{}, err
+ }
+
+ listen, upstream := proxyEndpoint(params)
+
+ return Shape{
+ Condition: fmt.Sprintf(partialCondition, target, budget),
+ Target: target,
+ Probes: probePair(params),
+ ProxyFault: &proxy.Fault{Kind: proxy.Partial, Bytes: budget},
+ ProxyListen: listen,
+ ProxyUpstream: upstream,
+ }, nil
+}
+
+func proxyDefinition(name, summary, condition string, kind proxy.Kind, extra ...Param) Definition {
+ build := plainProxyShape(kind, condition)
+
+ switch kind {
+ case proxy.Latency, proxy.SlowClose:
+ build = delayedProxyShape(kind, condition)
+ case proxy.Partial:
+ build = buildDependencyPartial
+ }
+
+ return Definition{
+ Name: name,
+ Family: FamilyNetwork,
+ Summary: summary,
+ Hypothesis: Hypothesis{Condition: outageGeneric, Response: proxyResponse},
+ Polarity: proxyPolarity,
+ FaultKind: inject.ProxyKind(kind),
+ Params: proxyParams(extra...),
+ Build: build,
+ }
+}
+
+func init() {
+ for _, definition := range []Definition{
+ proxyDefinition(NameDependencyHang, hangSummary, hangCondition, proxy.Hang),
+ proxyDefinition(NameDependencyReset, resetSummary, resetCondition, proxy.Reset),
+ proxyDefinition(NameDependencyEOF, eofSummary, eofCondition, proxy.EOF),
+ proxyDefinition(NameDependencyLatency, latencySummary, latencyCondition, proxy.Latency, delayParam),
+ proxyDefinition(NameDependencySlowClose, slowCloseSummary, slowCloseCondition, proxy.SlowClose, delayParam),
+ proxyDefinition(NameDependencyPartial, partialSummary, partialCondition, proxy.Partial, bytesParam),
+ } {
+ catalog[definition.Name] = definition
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params.go
new file mode 100644
index 00000000..f2d20e95
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params.go
@@ -0,0 +1,187 @@
+// ©AngelaMos | 2026
+// params.go
+
+package experiment
+
+import (
+ "errors"
+ "fmt"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+)
+
+const (
+ ParamTarget = "target"
+ ParamURL = "url"
+ ParamToken = "token"
+ ParamInvalidToken = "invalid-token"
+
+ errUndeclared = "%w: --%s (this experiment accepts: %s)"
+ errNoParams = "%w: --%s (this experiment takes no parameters)"
+ errMissing = "this experiment needs --%s, %s"
+ errNotDuration = "--%s was given as %q, which is not a duration; write it like 2s, 500ms or 1m30s"
+ errNotCount = "--%s was given as %q, which is not a whole number"
+ errNotPositive = "--%s was given as %q, and a fault with no magnitude is indistinguishable from no fault at all"
+ errBelowFloor = "--%s was given as %s, and khaos needs at least %s: %s"
+)
+
+var ErrUndeclaredParam = errors.New("that is not a parameter of this experiment")
+
+// Param declares one input an experiment needs. The declaration is the single
+// place the CLI registers flags from, the range fills defaults into, and Bind
+// validates against, so an experiment can neither be handed an input it never
+// asked for nor be given one it silently ignores.
+type Param struct {
+ Name string
+ Usage string
+ Required bool
+ Default string
+}
+
+// Params is the bound, validated parameter set for one build. Get records every
+// read so Build can reject a builder reaching for a name nobody declared, which
+// is otherwise an empty string with no symptom.
+type Params struct {
+ values map[string]string
+ read map[string]bool
+ stray map[string]bool
+}
+
+func Bind(declared []Param, supplied map[string]string) (*Params, error) {
+ names := make([]string, 0, len(declared))
+ for _, param := range declared {
+ names = append(names, param.Name)
+ }
+
+ for _, name := range sorted(supplied) {
+ if slices.Contains(names, name) {
+ continue
+ }
+ if len(names) == 0 {
+ return nil, fmt.Errorf(errNoParams, ErrUndeclaredParam, name)
+ }
+ return nil, fmt.Errorf(errUndeclared, ErrUndeclaredParam, name, strings.Join(names, nameSep))
+ }
+
+ bound := &Params{
+ values: make(map[string]string, len(declared)),
+ read: map[string]bool{},
+ stray: map[string]bool{},
+ }
+
+ // Presence decides whether the default applies, not emptiness. An operator who
+ // explicitly asked for an empty value has asked for something meaningful: an
+ // empty credential is sent as no Authorization header, which is the
+ // unauthenticated negative control. Overwriting it with the default makes that
+ // control impossible to express.
+ for _, param := range declared {
+ value, given := supplied[param.Name]
+ if !given {
+ value = param.Default
+ }
+ if value == "" && param.Required {
+ return nil, fmt.Errorf(errMissing, param.Name, param.Usage)
+ }
+ bound.values[param.Name] = value
+ }
+
+ return bound, nil
+}
+
+func (p *Params) Get(name string) string {
+ value, declared := p.values[name]
+ if !declared {
+ p.stray[name] = true
+ return ""
+ }
+
+ p.read[name] = true
+
+ return value
+}
+
+func (p *Params) strayReads() []string {
+ return sorted(p.stray)
+}
+
+func (p *Params) unread() []string {
+ out := []string{}
+ for name := range p.values {
+ if !p.read[name] {
+ out = append(out, name)
+ }
+ }
+ slices.Sort(out)
+ return out
+}
+
+func sorted[V any](m map[string]V) []string {
+ out := make([]string, 0, len(m))
+ for key := range m {
+ out = append(out, key)
+ }
+ slices.Sort(out)
+ return out
+}
+
+// Duration and Count parse a declared parameter and refuse a value the fault
+// cannot use. The refusal names the flag and what was typed, because a fault
+// whose magnitude failed to parse would otherwise fall back to zero, and a zero
+// delay or a zero byte budget is indistinguishable from no fault at all.
+func (p *Params) Duration(name string) (time.Duration, error) {
+ raw := p.Get(name)
+
+ value, err := time.ParseDuration(raw)
+ if err != nil {
+ return 0, fmt.Errorf(errNotDuration, name, raw)
+ }
+
+ if value <= 0 {
+ return 0, fmt.Errorf(errNotPositive, name, raw)
+ }
+
+ return value, nil
+}
+
+// AtLeast is Duration with a floor, for the magnitudes where a value the target
+// is ALLOWED to tolerate is not a smaller fault but a different and false
+// finding.
+//
+// Measured: `identity.claim_expired --delay 30s` against a textbook verifier
+// that allows 60 seconds of clock skew, which RFC 7519 section 4.1.4 explicitly
+// permits, produced `deviated / PERMITTED`, defect, exit 2. khaos confirmed the
+// trap honestly — the token really was expired — and then reported a correct
+// target as broken, because an expiry inside the skew allowance is one a correct
+// verifier is supposed to accept.
+//
+// The refusal names the floor AND why it exists, because an operator's first
+// instinct on seeing it will be that khaos is being fussy about a magnitude.
+func (p *Params) AtLeast(name string, floor time.Duration, because string) (time.Duration, error) {
+ value, err := p.Duration(name)
+ if err != nil {
+ return 0, err
+ }
+
+ if value < floor {
+ return 0, fmt.Errorf(errBelowFloor, name, value, floor, because)
+ }
+
+ return value, nil
+}
+
+func (p *Params) Count(name string) (int, error) {
+ raw := p.Get(name)
+
+ value, err := strconv.Atoi(raw)
+ if err != nil {
+ return 0, fmt.Errorf(errNotCount, name, raw)
+ }
+
+ if value <= 0 {
+ return 0, fmt.Errorf(errNotPositive, name, raw)
+ }
+
+ return value, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go
new file mode 100644
index 00000000..c816bf63
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/params_test.go
@@ -0,0 +1,344 @@
+// ©AngelaMos | 2026
+// params_test.go
+
+package experiment
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ sentinelPrefix = "sentinel-"
+ strayName = "a-name-nobody-declared"
+)
+
+// fill supplies a distinguishable value for every declared parameter. A parameter
+// that declares a default takes it, because a typed parameter such as a duration
+// or a byte count cannot accept an arbitrary sentinel and a build that refuses the
+// value never reaches the assertion under test.
+func fill(declared []Param) map[string]string {
+ supplied := map[string]string{}
+
+ for _, param := range declared {
+ if param.Default != "" {
+ supplied[param.Name] = param.Default
+ continue
+ }
+ supplied[param.Name] = sentinelPrefix + param.Name
+ }
+
+ return supplied
+}
+
+func TestBindRejectsAParameterTheExperimentDoesNotDeclare(t *testing.T) {
+ declared := []Param{{Name: ParamURL, Usage: usageURL, Required: true}}
+
+ _, err := Bind(declared, map[string]string{ParamURL: "u", strayName: "v"})
+ if !errors.Is(err, ErrUndeclaredParam) {
+ t.Fatalf("err = %v, want ErrUndeclaredParam; an input nothing will read must not be silently accepted", err)
+ }
+
+ if !strings.Contains(err.Error(), ParamURL) {
+ t.Fatalf("the refusal did not name what this experiment does accept: %v", err)
+ }
+}
+
+func TestBindRejectsAMissingRequiredParameterAndNamesTheFix(t *testing.T) {
+ declared := []Param{{Name: ParamTarget, Usage: usageTarget, Required: true}}
+
+ _, err := Bind(declared, nil)
+ if err == nil {
+ t.Fatal("a missing required parameter was accepted, so the experiment would run against nothing")
+ }
+
+ for _, want := range []string{flagSpelling(ParamTarget), usageTarget} {
+ if !strings.Contains(err.Error(), want) {
+ t.Fatalf("the refusal did not name %q, so it does not reach the operator as a fix: %v", want, err)
+ }
+ }
+}
+
+func TestBindAppliesADeclaredDefaultAndAnOperatorValueOverridesIt(t *testing.T) {
+ declared := []Param{{Name: ParamInvalidToken, Usage: usageInvalidToken, Default: DefaultInvalidToken}}
+
+ bound, err := Bind(declared, nil)
+ if err != nil {
+ t.Fatalf("bind: %v", err)
+ }
+
+ if got := bound.Get(ParamInvalidToken); got != DefaultInvalidToken {
+ t.Fatalf("default not applied: got %q, want %q", got, DefaultInvalidToken)
+ }
+
+ bound, err = Bind(declared, map[string]string{ParamInvalidToken: "mine"})
+ if err != nil {
+ t.Fatalf("bind: %v", err)
+ }
+
+ if got := bound.Get(ParamInvalidToken); got != "mine" {
+ t.Fatalf("the declared default overrode the operator: got %q", got)
+ }
+}
+
+// A parameter with a default is never missing, so declaring it Required as well
+// would make the Required check untestable for it. This pins the distinction.
+func TestADefaultSatisfiesARequiredParameter(t *testing.T) {
+ declared := []Param{{Name: ParamToken, Usage: usageToken, Required: true, Default: "d"}}
+
+ if _, err := Bind(declared, nil); err != nil {
+ t.Fatalf("a required parameter with a default was reported missing: %v", err)
+ }
+}
+
+// The negative control for the stray-read rejection: a definition whose builder
+// reaches for a name it never declared. Without this check that read returns an
+// empty string with no symptom anywhere.
+func TestBuildRejectsABuilderThatReadsAnUndeclaredParameter(t *testing.T) {
+ const name = "test.stray_read"
+
+ catalog[name] = Definition{
+ Name: name,
+ Family: FamilyNetwork,
+ FaultKind: "test-kind",
+ Params: []Param{{Name: ParamTarget, Usage: usageTarget, Required: true}},
+ Build: func(params *Params) (Shape, error) {
+ return Shape{Target: params.Get(ParamTarget) + params.Get(strayName)}, nil
+ },
+ }
+ t.Cleanup(func() { delete(catalog, name) })
+
+ _, err := Build(name, map[string]string{ParamTarget: "t"}, Timing{})
+ if !errors.Is(err, ErrStrayParam) {
+ t.Fatalf("err = %v, want ErrStrayParam", err)
+ }
+
+ if !strings.Contains(err.Error(), strayName) {
+ t.Fatalf("the refusal did not name the stray parameter: %v", err)
+ }
+}
+
+// A declared parameter its builder never reads is a flag the operator can set to
+// no effect. This is the check that catches one the moment it is added.
+func TestEveryDeclaredParameterIsReadByItsBuilder(t *testing.T) {
+ definitions := Catalog()
+ if len(definitions) == 0 {
+ t.Fatal("the catalogue is empty, so this test proves nothing")
+ }
+
+ for _, definition := range definitions {
+ t.Run(definition.Name, func(t *testing.T) {
+ if len(definition.Params) == 0 {
+ t.Fatalf("%s declares no parameters at all", definition.Name)
+ }
+
+ bound, err := Bind(definition.Params, fill(definition.Params))
+ if err != nil {
+ t.Fatalf("bind: %v", err)
+ }
+
+ if _, err := definition.Build(bound); err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ if unread := bound.unread(); len(unread) > 0 {
+ t.Fatalf("%s declares %v but never reads them, so those flags do nothing",
+ definition.Name, unread)
+ }
+
+ if stray := bound.strayReads(); len(stray) > 0 {
+ t.Fatalf("%s reads %v without declaring them", definition.Name, stray)
+ }
+ })
+ }
+}
+
+// catalog and run must not be able to disagree about what an experiment is. The
+// builder returns a Shape precisely so it cannot restate any of this and get it
+// wrong, and this is the assertion that proves Build carries all of it across.
+func TestBuildTakesEveryIdentityFieldFromTheDeclaration(t *testing.T) {
+ timing := Timing{
+ SampleInterval: time.Millisecond,
+ BaselineFor: 2 * time.Millisecond,
+ FaultFor: 3 * time.Millisecond,
+ RecoveryFor: 4 * time.Millisecond,
+ }
+
+ for _, definition := range Catalog() {
+ t.Run(definition.Name, func(t *testing.T) {
+ spec, err := Build(definition.Name, fill(definition.Params), timing)
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ if spec.Name != definition.Name {
+ t.Fatalf("name = %q, want %q", spec.Name, definition.Name)
+ }
+ if spec.Family != definition.Family {
+ t.Fatalf("family = %q, want %q", spec.Family, definition.Family)
+ }
+ if spec.Summary != definition.Summary {
+ t.Fatalf("summary = %q, want %q", spec.Summary, definition.Summary)
+ }
+ if !slices.Equal(spec.Polarity.Defects, definition.Polarity.Defects) ||
+ spec.Polarity.Rationale != definition.Polarity.Rationale {
+ t.Fatalf("polarity = %+v, want %+v", spec.Polarity, definition.Polarity)
+ }
+ if spec.Fault.Target != sentinelPrefix+ParamTarget {
+ t.Fatalf("fault target = %q, want %q; this is the one value that decides WHICH "+
+ "container gets faulted and nothing else asserted it",
+ spec.Fault.Target, sentinelPrefix+ParamTarget)
+ }
+ if spec.Fault.Kind != definition.FaultKind {
+ t.Fatalf("fault kind = %q, want %q; `khaos catalog` would be advertising a "+
+ "different fault than `khaos run` applies", spec.Fault.Kind, definition.FaultKind)
+ }
+ if spec.Hypothesis.Response != definition.Hypothesis.Response {
+ t.Fatalf("response = %q, want %q", spec.Hypothesis.Response, definition.Hypothesis.Response)
+ }
+ if spec.Timing != timing {
+ t.Fatalf("timing = %+v, want %+v; a builder that forgets to carry timing "+
+ "silently ignores every timing flag", spec.Timing, timing)
+ }
+ if !spec.Probes.Complete() {
+ t.Fatal("the built experiment has no probe pair")
+ }
+ })
+ }
+}
+
+// The condition is the one hypothesis field a build may resolve, because it names
+// the actual target. The generic form belongs to the catalogue.
+func TestBuildResolvesTheHypothesisConditionAgainstTheTarget(t *testing.T) {
+ const target = "a-very-specific-container"
+
+ spec, err := Build(NameDependencyOutage, map[string]string{
+ ParamTarget: target,
+ ParamURL: "http://127.0.0.1:1",
+ ParamToken: "t",
+ }, Timing{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ if !strings.Contains(spec.Hypothesis.Condition, target) {
+ t.Fatalf("the hypothesis still reads %q and never names the target", spec.Hypothesis.Condition)
+ }
+
+ generic := catalog[NameDependencyOutage].Hypothesis.Condition
+ if spec.Hypothesis.Condition == generic {
+ t.Fatal("the built hypothesis is the catalogue's generic form, so a run reports a " +
+ "hypothesis that never mentions what it actually broke")
+ }
+}
+
+func TestTheNegativeControlCredentialIsNotAcceptableToAnyVerifier(t *testing.T) {
+ spec, err := Build(NameDependencyOutage, map[string]string{
+ ParamTarget: "t",
+ ParamURL: "http://127.0.0.1:1",
+ ParamToken: "t",
+ }, Timing{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ if spec.Probes.Invalid.Credential() != probe.Invalid {
+ t.Fatal("the second probe does not carry an invalid credential, so the run has no negative control")
+ }
+}
+
+func flagSpelling(name string) string {
+ return "--" + name
+}
+
+// Control at the other end. The probe package proves a Unique probe varies its
+// credential; this proves the CATALOGUE asks for one. Deleting `Unique: true`
+// from the builder left every probe-level assertion green, which is the whole
+// reason both ends need a control.
+func TestTheCatalogueAsksForAFreshNegativeControlCredential(t *testing.T) {
+ for _, definition := range Catalog() {
+ t.Run(definition.Name, func(t *testing.T) {
+ var mu sync.Mutex
+ seen := map[string]bool{}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ seen[r.Header.Get("Authorization")] = true
+ mu.Unlock()
+ w.WriteHeader(http.StatusUnauthorized)
+ }))
+ defer server.Close()
+
+ supplied := fill(definition.Params)
+ supplied[ParamURL] = server.URL
+
+ spec, err := Build(definition.Name, supplied, fastTiming)
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ const rounds = 4
+ for range rounds {
+ spec.Probes.Invalid.Run(context.Background())
+ }
+
+ mu.Lock()
+ distinct := len(seen)
+ mu.Unlock()
+
+ if distinct != rounds {
+ t.Fatalf("%s sent %d distinct invalid credentials across %d probes; a "+
+ "credential-keyed decision cache answers every repeat from one cached "+
+ "refusal, and a control that stops enforcing during the fault is never "+
+ "exercised. Measured against the range: a gateway configured to fail open "+
+ "with a cache TTL longer than the fault window reported passed / HELD, "+
+ "conclusive, exit 0", definition.Name, distinct, rounds)
+ }
+ })
+ }
+}
+
+// An explicitly empty value is a real request, not an absent one. An empty
+// credential is sent as no Authorization header at all, which is the canonical
+// unauthenticated negative control, so replacing it with the default makes that
+// control inexpressible for a tool whose whole thesis is the negative control.
+func TestAnExplicitlyEmptyValueIsNotReplacedByTheDefault(t *testing.T) {
+ declared := []Param{{Name: ParamInvalidToken, Usage: usageInvalidToken, Default: DefaultInvalidToken}}
+
+ bound, err := Bind(declared, map[string]string{ParamInvalidToken: ""})
+ if err != nil {
+ t.Fatalf("bind: %v", err)
+ }
+
+ if got := bound.Get(ParamInvalidToken); got != "" {
+ t.Fatalf("an explicit empty value became %q, so the unauthenticated negative control "+
+ "cannot be asked for", got)
+ }
+
+ absent, err := Bind(declared, map[string]string{})
+ if err != nil {
+ t.Fatalf("bind: %v", err)
+ }
+
+ if got := absent.Get(ParamInvalidToken); got != DefaultInvalidToken {
+ t.Fatalf("an absent value did not take the declared default, got %q", got)
+ }
+}
+
+func TestAnExplicitlyEmptyRequiredValueIsStillRefused(t *testing.T) {
+ declared := []Param{{Name: ParamToken, Usage: usageToken, Required: true}}
+
+ if _, err := Bind(declared, map[string]string{ParamToken: ""}); err == nil {
+ t.Fatal("an empty required credential was accepted; the valid probe would carry no " +
+ "credential and the baseline would blame the target for not serving")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/revocation_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/revocation_test.go
new file mode 100644
index 00000000..bdf90ac7
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/revocation_test.go
@@ -0,0 +1,324 @@
+// ©AngelaMos | 2026
+// revocation_test.go
+
+package experiment
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ revokedName = "test.revocation"
+ stillValid = "a-credential-that-stays-valid"
+ toBeRevoked = "a-credential-khaos-revokes-mid-run"
+ corruptToken = "a-token-with-a-corrupted-signature"
+)
+
+// authority is the identity provider. khaos can reach it throughout; the TARGET
+// loses its path to it when the fault lands, which is what makes a revocation
+// real and unreachable at the same time. A paused provider cannot express this,
+// because then khaos cannot reach it either.
+type authority struct {
+ mu sync.Mutex
+ revoked map[string]bool
+}
+
+func (a *authority) revoke(token string) {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ if a.revoked == nil {
+ a.revoked = map[string]bool{}
+ }
+ a.revoked[token] = true
+}
+
+func (a *authority) active(token string) bool {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return token != corruptToken && !a.revoked[token]
+}
+
+// cachingGateway serves a cached decision when it cannot reach the authority, and
+// reArm decides whether serving one also re-arms it.
+type cachingGateway struct {
+ authority *authority
+ reachable atomic.Bool
+ reArm bool
+ noExpiry bool
+ ttl time.Duration
+
+ mu sync.Mutex
+ cache map[string]time.Time
+}
+
+func newCachingGateway(a *authority, reArm, noExpiry bool, ttl time.Duration) *cachingGateway {
+ g := &cachingGateway{authority: a, reArm: reArm, noExpiry: noExpiry, ttl: ttl,
+ cache: map[string]time.Time{}}
+ g.reachable.Store(true)
+ return g
+}
+
+func (g *cachingGateway) handler() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ token := strings.TrimPrefix(r.Header.Get("Authorization"), bearer)
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ expires, known := g.cache[token]
+ if known && (g.noExpiry || time.Now().Before(expires)) {
+ if !g.reachable.Load() && g.reArm {
+ g.cache[token] = time.Now().Add(g.ttl)
+ }
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+
+ if !g.reachable.Load() {
+ w.WriteHeader(http.StatusForbidden)
+ return
+ }
+
+ if !g.authority.active(token) {
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ g.cache[token] = time.Now().Add(g.ttl)
+ w.WriteHeader(http.StatusOK)
+ })
+}
+
+type severingInjector struct{ gw *cachingGateway }
+
+func (i *severingInjector) Inject(context.Context, string, string) error {
+ i.gw.reachable.Store(false)
+ return nil
+}
+
+func (i *severingInjector) Confirm(context.Context) (string, error) {
+ if i.gw.reachable.Load() {
+ return "", nil
+ }
+ return "the path to the authority reads as severed", nil
+}
+
+func (i *severingInjector) Revert(context.Context) error {
+ i.gw.reachable.Store(true)
+ return nil
+}
+
+func registerRevocationExperiment(t *testing.T, url string, act func(context.Context) error) {
+ t.Helper()
+
+ catalog[revokedName] = Definition{
+ Name: revokedName,
+ Family: FamilyNetwork,
+ Summary: "a credential revoked at an authority the target can no longer reach",
+ Hypothesis: Hypothesis{Condition: outageGeneric, Response: outageResponse},
+ Polarity: outagePolarity,
+ FaultKind: "test-kind",
+ Params: []Param{{Name: ParamTarget, Usage: usageTarget, Required: true}},
+ Build: func(params *Params) (Shape, error) {
+ return Shape{
+ Target: params.Get(ParamTarget),
+ Probes: probe.Pair{
+ Valid: probe.NewHTTP(probe.HTTPOptions{
+ Name: "valid", URL: url, Token: stillValid, Credential: probe.Valid,
+ }),
+ Invalid: probe.NewHTTP(probe.HTTPOptions{
+ Name: "control", URL: url, Token: corruptToken, Credential: probe.Invalid,
+ }),
+ Revoked: probe.NewHTTP(probe.HTTPOptions{
+ Name: "revoked", URL: url, Token: toBeRevoked, Credential: probe.Revoked,
+ }),
+ },
+ Act: act,
+ ActDetail: "revoking a credential at the authority the target can no longer reach",
+ Defect: testDefectPhrase,
+ }, nil
+ },
+ }
+
+ t.Cleanup(func() { delete(catalog, revokedName) })
+}
+
+func runRevocation(t *testing.T, gw *cachingGateway, act func(context.Context) error, idle time.Duration) Result {
+ t.Helper()
+
+ server := httptest.NewServer(gw.handler())
+ t.Cleanup(server.Close)
+
+ registerRevocationExperiment(t, server.URL, act)
+
+ spec, err := Build(revokedName, map[string]string{ParamTarget: "the-authority"}, Timing{
+ SampleInterval: time.Millisecond,
+ BaselineFor: 20 * time.Millisecond,
+ // At least as long as the pause. The TTL-extension diagnosis is only
+ // sound when steady traffic kept the credential alive for at least the
+ // duration of the pause that then killed it: a fixed lifetime sitting
+ // between a short fault window and a longer pause produces the same
+ // observation. These tests used a 20ms window against a 600ms pause and
+ // asserted a causal claim khaos could not make from it.
+ FaultFor: maxDuration(20*time.Millisecond, idle),
+ RecoveryFor: 10 * time.Millisecond,
+ })
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ spec.IdleFor = idle
+
+ result, err := Engine{Injector: &severingInjector{gw: gw}}.Run(context.Background(), spec)
+ if err != nil && !errors.Is(err, errActRefused) {
+ t.Fatalf("run: %v", err)
+ }
+
+ return result
+}
+
+var errActRefused = errors.New("the authority refused khaos's change")
+
+// The phrase this experiment supplies for the verdict. It is asserted verbatim
+// below rather than loosely matched, because the whole point of the field is
+// that each experiment says what IT did: one shared constant previously claimed
+// every revocation happened "while that authority was unreachable", which was
+// false for two of the three experiments that have a revoked stream.
+const testDefectPhrase = "revoking it at an authority the target could no longer reach"
+
+// The finding this whole shape exists to produce, with its own claim. "A
+// credential khaos invalidated at the authority was still accepted" is not the
+// same statement as "a credential the target had already rejected was
+// accepted", and the verdict must make the one that actually happened.
+//
+// It must also say HOW khaos invalidated it, in this experiment's own words. A
+// verdict that falls back to the generic phrase is one that would describe a key
+// withdrawal and an unreachable-authority revocation identically.
+func TestARevokedCredentialStillHonouredIsItsOwnFinding(t *testing.T) {
+ auth := &authority{}
+ gw := newCachingGateway(auth, true, false, 60*time.Millisecond)
+
+ result := runRevocation(t, gw, func(context.Context) error {
+ auth.revoke(toBeRevoked)
+ return nil
+ }, 0)
+
+ if result.Verdict.Direction != Permitted {
+ t.Fatalf("direction = %s, want %s (%s)", result.Verdict.Direction, Permitted,
+ strings.Join(result.Verdict.Reasons, "; "))
+ }
+
+ joined := strings.Join(result.Verdict.Reasons, "; ")
+
+ if !strings.Contains(joined, "invalidated at the authority") {
+ t.Fatalf("the verdict does not state what khaos actually observed: %s", joined)
+ }
+
+ if !strings.Contains(joined, testDefectPhrase) {
+ t.Fatalf("the verdict does not say HOW khaos invalidated the credential, so it would read "+
+ "identically for every experiment that has a revoked stream: %s", joined)
+ }
+
+ if strings.Contains(joined, "had already rejected") {
+ t.Fatalf("the verdict claims the target honoured something it had already rejected, "+
+ "which is not what happened: %s", joined)
+ }
+
+ if !result.Observation.Acted {
+ t.Fatal("the run did not record that khaos made its change at the authority")
+ }
+}
+
+// A failed action means the credential khaos went on to measure was never
+// revoked, so every acceptance of it was correct and the run proves nothing.
+func TestAFailedMidRunActionVoidsTheRunRatherThanClearingTheTarget(t *testing.T) {
+ auth := &authority{}
+ gw := newCachingGateway(auth, true, false, 60*time.Millisecond)
+
+ result := runRevocation(t, gw, func(context.Context) error {
+ return errActRefused
+ }, 0)
+
+ if result.Verdict.Epistemic != Aborted {
+ t.Fatalf("epistemic = %s, want %s. khaos measured a credential nobody revoked, so a "+
+ "clean verdict would be khaos clearing a target it never tested (%s)",
+ result.Verdict.Epistemic, Aborted, strings.Join(result.Verdict.Reasons, "; "))
+ }
+
+ if !result.Observation.ActFailed {
+ t.Fatal("the run did not record that the action failed")
+ }
+
+ if result.Verdict.Direction == Permitted {
+ t.Fatal("a run whose action failed reported a bypass")
+ }
+}
+
+// Revoking a credential the target was never honouring takes nothing away, so
+// every later rejection is the target being correct.
+func TestTheBaselineRefusesWhenThereIsNothingToRevoke(t *testing.T) {
+ auth := &authority{}
+ auth.revoke(toBeRevoked)
+
+ gw := newCachingGateway(auth, true, false, 60*time.Millisecond)
+
+ result := runRevocation(t, gw, func(context.Context) error { return nil }, 0)
+
+ if result.Verdict.Epistemic != BaselineNotMet {
+ t.Fatalf("epistemic = %s, want %s (%s)", result.Verdict.Epistemic, BaselineNotMet,
+ strings.Join(result.Verdict.Reasons, "; "))
+ }
+
+ if !strings.Contains(strings.Join(result.Verdict.Reasons, "; "), "revoking it takes nothing away") {
+ t.Fatalf("the refusal does not name the cause: %s", strings.Join(result.Verdict.Reasons, "; "))
+ }
+}
+
+// The discriminator has to read the stream that actually leaked. Reading the
+// never-valid control instead would report the pause's effect on something that
+// was rejected throughout.
+func TestTheDiscriminatorReadsTheStreamThatLeaked(t *testing.T) {
+ const ttl = 50 * time.Millisecond
+
+ cases := []struct {
+ name string
+ reArm bool
+ none bool
+ cause Cause
+ }{
+ {"re-armed on every failed check", true, false, TTLExtension},
+ {"stale past the pause", false, true, StalePastPause},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ auth := &authority{}
+ gw := newCachingGateway(auth, tc.reArm, tc.none, ttl)
+
+ result := runRevocation(t, gw, func(context.Context) error {
+ auth.revoke(toBeRevoked)
+ return nil
+ }, 10*ttl)
+
+ if result.Verdict.Cause != tc.cause {
+ t.Fatalf("cause = %s, want %s (%s)", result.Verdict.Cause, tc.cause,
+ strings.Join(result.Verdict.Reasons, "; "))
+ }
+
+ if result.Observation.Resumed.Revoked.Countable() == 0 {
+ t.Fatal("the resumed window carries no revoked samples, so the diagnosis read " +
+ "a stream the fault never touched")
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/sampling_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/sampling_test.go
new file mode 100644
index 00000000..b0832c2a
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/sampling_test.go
@@ -0,0 +1,130 @@
+// ©AngelaMos | 2026
+// sampling_test.go
+
+package experiment
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+type slowProbe struct {
+ credential probe.Credential
+ delay time.Duration
+}
+
+func (p slowProbe) Name() string { return "slow" }
+func (p slowProbe) Describe() string { return "a probe that takes its time" }
+func (p slowProbe) Credential() probe.Credential { return p.credential }
+
+func (p slowProbe) Run(ctx context.Context) probe.Outcome {
+ select {
+ case <-ctx.Done():
+ case <-time.After(p.delay):
+ }
+
+ return probe.Outcome{
+ At: time.Now().UTC(),
+ Credential: p.credential,
+ Disposition: probe.Accepted,
+ }
+}
+
+func pairOf(delay time.Duration) probe.Pair {
+ return probe.Pair{
+ Valid: slowProbe{probe.Valid, delay},
+ Invalid: slowProbe{probe.Invalid, delay},
+ }
+}
+
+// A slow target must cost latency headroom, not sample count. Before the
+// interval became a true period, a target that hung under the fault produced
+// one paired sample where twenty-one were nominal, so the only verdict khaos
+// could reach was probe-error.
+func TestASlowProbeDoesNotCollapseTheSampleCount(t *testing.T) {
+ cases := []struct {
+ name string
+ duration time.Duration
+ interval time.Duration
+ delay time.Duration
+ atLeast int
+ }{
+ {"instant probes", 200 * time.Millisecond, 20 * time.Millisecond, 0, 6},
+ {"probe costs most of the period", 200 * time.Millisecond, 20 * time.Millisecond, 15 * time.Millisecond, 5},
+ {"probe costs more than the period", 200 * time.Millisecond, 10 * time.Millisecond, 25 * time.Millisecond, 3},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ engine := Engine{}
+
+ window := engine.sample(context.Background(), pairOf(tc.delay), tc.duration, tc.interval)
+
+ if got := window.Paired(); got < tc.atLeast {
+ t.Fatalf("collected %d paired samples in %v at a %v period with a %v probe, "+
+ "want at least %d; a slow target must not silently drop the run below the "+
+ "evidence floor", got, tc.duration, tc.interval, tc.delay, tc.atLeast)
+ }
+ })
+ }
+}
+
+func TestSamplingAlwaysProducesAtLeastOnePair(t *testing.T) {
+ engine := Engine{}
+
+ window := engine.sample(context.Background(), pairOf(0), 0, time.Second)
+
+ if window.Paired() < 1 {
+ t.Fatal("a zero-length phase produced no samples at all, so every verdict built on it " +
+ "would be an artefact of the timing rather than the target")
+ }
+}
+
+func TestSamplingHonoursTheRequestedDuration(t *testing.T) {
+ engine := Engine{}
+
+ started := time.Now()
+ engine.sample(context.Background(), pairOf(0), 150*time.Millisecond, 10*time.Millisecond)
+ elapsed := time.Since(started)
+
+ if elapsed < 150*time.Millisecond {
+ t.Fatalf("sampling returned after %v for a 150ms phase; the phase was cut short", elapsed)
+ }
+
+ if elapsed > time.Second {
+ t.Fatalf("sampling took %v for a 150ms phase", elapsed)
+ }
+}
+
+func TestSamplingStopsPromptlyOnCancellation(t *testing.T) {
+ engine := Engine{}
+
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ time.Sleep(30 * time.Millisecond)
+ cancel()
+ }()
+
+ started := time.Now()
+ engine.sample(ctx, pairOf(0), 10*time.Second, 10*time.Millisecond)
+ elapsed := time.Since(started)
+
+ if elapsed > time.Second {
+ t.Fatalf("sampling ran for %v after cancellation; a cancelled run must stop measuring", elapsed)
+ }
+}
+
+func TestAnIntervalLongerThanThePhaseDoesNotOverrun(t *testing.T) {
+ engine := Engine{}
+
+ started := time.Now()
+ engine.sample(context.Background(), pairOf(0), 20*time.Millisecond, time.Second)
+ elapsed := time.Since(started)
+
+ if elapsed > 500*time.Millisecond {
+ t.Fatalf("a 20ms phase with a 1s period took %v; the phase deadline must win", elapsed)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
new file mode 100644
index 00000000..e94c9fa9
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
@@ -0,0 +1,710 @@
+// ©AngelaMos | 2026
+// verdict.go
+
+package experiment
+
+import (
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+type Epistemic string
+
+const (
+ Passed Epistemic = "passed"
+ Deviated Epistemic = "deviated"
+ BaselineNotMet Epistemic = "baseline-not-met"
+ ProbeError Epistemic = "probe-error"
+ Aborted Epistemic = "aborted"
+ Interrupted Epistemic = "interrupted"
+ Errored Epistemic = "error"
+)
+
+type Direction string
+
+const (
+ Held Direction = "HELD"
+ Denied Direction = "DENIED"
+ Permitted Direction = "PERMITTED"
+ Crashed Direction = "CRASHED"
+ Undetermined Direction = "UNDETERMINED"
+)
+
+// Cause is a THIRD axis, and it is deliberately not a value on the directional
+// one. Direction answers which way the target broke; cause answers why it is
+// still broken, and the two are orthogonal. `PERMITTED` has two causes that look
+// identical from outside under steady traffic and need completely different
+// fixes, so collapsing them into the direction would lose the distinction khaos
+// exists to draw here.
+type Cause string
+
+const (
+ // Undiagnosed is the honest default. Nothing ran that could tell the causes
+ // apart, so khaos says so rather than guessing one.
+ Undiagnosed Cause = "UNDIAGNOSED"
+
+ // TTLExtension is the decision cache being re-armed on every failed check.
+ // Research 07 section 3.3 pins it to one line inside an error branch:
+ // `cache.Set(tok, stale, ttl)`. The TTL stops bounding staleness and starts
+ // bounding IDLENESS, so a cached allow becomes permanent exactly when the
+ // authority that could revoke it is unreachable. It survives any amount of
+ // steady traffic and dies the moment traffic pauses.
+ TTLExtension Cause = "TTL-EXTENSION"
+
+ // LapsedOnPause is enforcement returning after the pause, with khaos unable
+ // to say WHY. It is the honest middle between the two diagnoses.
+ //
+ // TTL-EXTENSION claims the cache was being re-armed by traffic. That only
+ // follows if steady traffic kept the credential alive for at least as long
+ // as the pause then killed it: otherwise an ordinary fixed expiry explains
+ // the same observation, and khaos would prescribe removing a cache write
+ // that does not exist.
+ LapsedOnPause Cause = "LAPSED-ON-PAUSE"
+
+ // StalePastPause is a cached state the traffic pause did not let lapse.
+ //
+ // It is NOT called NO-EXPIRY, and the rename is a correctness fix rather than
+ // a wording preference. Research 07's open question 5 is explicit: khaos
+ // cannot tell a cache with no expiry from one whose lifetime simply exceeds
+ // the experiment, "not without running longer than the TTL", and the
+ // experiment "should report an observed lower bound on the stale window and
+ // label it as a bound, never as 'caches forever'".
+ //
+ // The old label asserted the unbounded claim in two words while the reason
+ // beside it hedged, so a five-minute JWKS cache — the default in most
+ // libraries, and bounded, and defensible — was reported as having no expiry
+ // at all on the strength of a sixteen-second observation.
+ StalePastPause Cause = "STALE-PAST-PAUSE"
+)
+
+// Recovery is what khaos saw after it reverted the fault. It is three-valued
+// because "not observed" and "observed healthy" are different facts, and
+// collapsing them let an empty recovery window read as a clean comeback.
+type Recovery string
+
+const (
+ // Recovered is a healthy run of consecutive pairs after the revert.
+ Recovered Recovery = "recovered"
+
+ // Degraded is measured and NOT healthy: the target did not come back.
+ Degraded Recovery = "degraded"
+
+ // Unobserved is no usable evidence either way. It is not a pass.
+ Unobserved Recovery = "unobserved"
+)
+
+type Warning string
+
+const (
+ WarningRollbackFailed Warning = "rollback-failed"
+ WarningThinEvidence Warning = "thin-evidence"
+ WarningPartialOutage Warning = "partial-outage"
+ WarningRecoveryUnobserved Warning = "recovery-unobserved"
+)
+
+const (
+ // A single accepted credential that the target had already rejected is
+ // PROOF of a bypass, so PERMITTED needs n=1. The absence of one is not
+ // proof of its absence, so HELD needs a sample-size argument. The
+ // asymmetry is deliberate.
+ MinimumToProveOpen = 1
+ MinimumToClaimClosed = 30
+ MinimumPaired = 5
+ RecoveryRunLength = 2
+ crashFraction = 0.5
+)
+
+const (
+ reasonThinBaseline = "the baseline collected %d paired samples, fewer than the %d khaos needs to describe a steady state"
+ reasonThinForged = "the fault window produced %d usable decisions on the credential this experiment is actually about, fewer than the %d khaos needs; the target was never successfully shown the thing under test, so its not refusing one proves nothing"
+ reasonThinFault = "the fault window collected %d paired samples, fewer than the %d khaos needs to conclude anything from an absence of findings"
+ reasonNoEnforcement = "the invalid credential was never rejected during the baseline, so the target was not enforcing anything before the fault and nothing can be concluded from it failing to enforce during one"
+ reasonRedirects = "the invalid credential was answered with a redirect %d time(s) during the baseline and never with a refusal; point --url at an endpoint that answers with a status rather than a login redirect"
+ reasonNoService = "the valid credential was never accepted during the baseline, so the target was not serving before the fault"
+ reasonNoServiceMinted = "the target never accepted a credential from khaos's own identity provider during the baseline. It may not resolve keys from the issuer khaos is serving; and if it caches its key set, a PREVIOUS khaos run's keys may still be cached, because khaos mints a fresh key set every run. Restart the target or shorten its key cache lifetime"
+ reasonPermitted = "a credential the target had already rejected was accepted %d time(s) during the fault"
+ reasonDenied = "the valid credential stopped being accepted %d time(s) during the fault"
+ reasonCrashed = "the target stopped answering %d of %d probes during the fault"
+ reasonHeld = "the valid credential kept being accepted and the invalid one kept being rejected"
+ reasonWeakHeld = "no bypass was observed, but %d paired samples cannot rule one out; %d would"
+ reasonNotRecovered = "the target did not return to its steady state after the fault was reverted"
+ reasonRecoveryUnobserved = "khaos collected no usable recovery evidence, so it cannot say whether the target came back; this is not the same as it having come back"
+ reasonPartialOutage = "the target was unreachable for %d of %d probes during the fault, so the measurement is mixed"
+ reasonDefect = "this experiment declares %s to be the defect for this control"
+ reasonNotDefect = "this experiment declares %s to be acceptable for this control, so the deviation is recorded without calling it a defect"
+ reasonInterrupted = "the run was interrupted before it finished, so nothing it measured is conclusive"
+ reasonRollback = "the fault could not be fully reverted, so the target may still be degraded"
+ reasonFaultUnconfirmed = "khaos could not confirm the fault actually took effect, so nothing measured during the run can be attributed to it"
+ reasonNothingToRevoke = "the credential this experiment intends to revoke was never accepted during the baseline, so the target was not honouring it and revoking it takes nothing away"
+ reasonActFailed = "khaos could not make its change at the authority, so the credential it went on to measure was never revoked and every acceptance of it was correct"
+ reasonRevokedHonoured = "a credential khaos invalidated at the authority, by %s, was still accepted %d time(s)"
+ reasonForgedAccepted = "a credential khaos minted with %s was accepted %d time(s)"
+ reasonProviderNotTrusted = "the target never accepted a correct credential from khaos's own identity provider during the baseline, so it was not trusting that provider before the fault and its refusal of a forged one during it means nothing; check that the target resolves keys from the issuer khaos is serving"
+ reasonForgedNotChecked = "the target never refused the defective credential during the baseline, so it was not checking the property this experiment is about to break and nothing can be concluded from it accepting one during the fault"
+ reasonTTLExtension = "the credential survived %s of STEADY traffic and then stopped being accepted after a %s pause, and no fixed lifetime behaves that way: every failed check was re-arming whatever the target had cached, a decision or a key set, so the lifetime was bounding IDLENESS rather than staleness. The fix is to stop writing to the cache in the error branch"
+ reasonLapsedOnPause = "the credential stopped being accepted after the %s pause, but the steady-traffic window before it was only %s, so an ordinary fixed expiry between those two durations produces this exact result and khaos will not call it re-arming. Re-run with --fault-for at least %s to separate them"
+ reasonStalePastPause = "the credential was still accepted after traffic was paused for %s, so whatever the target cached did not lapse across that pause. khaos cannot tell a cache with no expiry from one whose lifetime simply exceeds %s, and it will not claim the first from an observation that only establishes the second; raise --idle-for to raise the bound"
+ reasonNeverConsulted = "the target never used %s during the baseline, so the fault this experiment applies there is one it cannot experience; a target that validates credentials locally never calls it, and khaos would otherwise report a clean survival of a fault that never reached it"
+ reasonNoResumedEvidence = "the resumed window produced %d usable samples in %s and never a refusal, so khaos saw nothing that could tell a cache being re-armed from one that outlasts the pause"
+ reasonStaleWindow = "the credential khaos invalidated was accepted over a span of at least %s, which is the lower bound on the exposure and not a measurement of the cache lifetime"
+)
+
+type Verdict struct {
+ Epistemic Epistemic `json:"epistemic"`
+ Recovery Recovery `json:"recovery"`
+ Direction Direction `json:"direction"`
+ Cause Cause `json:"cause"`
+ Conclusive bool `json:"conclusive"`
+ Defect bool `json:"defect"`
+ Warnings []Warning `json:"warnings,omitempty"`
+ Reasons []string `json:"reasons"`
+}
+
+type Observation struct {
+ Baseline probe.Window `json:"baseline"`
+ Fault probe.Window `json:"fault"`
+ Recovery probe.Window `json:"recovery"`
+ FaultConfirmed bool `json:"fault_confirmed"`
+ ConfirmMethod string `json:"fault_confirm_method,omitempty"`
+ RollbackFailed bool `json:"rollback_failed"`
+ Interrupted bool `json:"interrupted"`
+
+ // Resumed is the window measured AFTER traffic has been paused for IdleFor,
+ // with the fault still applied. It is the idleness discriminator: the two
+ // causes of a permitted credential are indistinguishable under steady traffic
+ // and separate completely across a pause.
+ Resumed probe.Window `json:"resumed"`
+ IdleFor time.Duration `json:"idle_for,omitempty"`
+
+ // FaultFor is how long steady traffic ran before the pause. It is what makes
+ // the TTL-extension diagnosis falsifiable rather than asserted.
+ FaultFor time.Duration `json:"fault_for,omitempty"`
+
+ // Acted records that khaos made its mid-run change at the authority. A run
+ // whose action failed measured a credential nobody revoked, so the finding it
+ // exists to produce is unreachable and the run is void rather than clean.
+ Acted bool `json:"acted,omitempty"`
+ ActFailed bool `json:"act_failed,omitempty"`
+
+ // ForgedBaseline is what the forged stream was DECLARED to do before the
+ // fault, carried into the record so a reader can audit the guard that was
+ // applied rather than having to know which experiment produced the run.
+ ForgedBaseline probe.Expectation `json:"forged_baseline,omitempty"`
+
+ // Consulted is how many times the target reached the faulted endpoint during
+ // the baseline, and ConsultedWhat names it. Zero means the fault khaos is
+ // about to apply is one the target cannot experience.
+ Consulted int64 `json:"consulted,omitempty"`
+ ConsultedWhat string `json:"consulted_what,omitempty"`
+
+ // MintedByKhaos records that the valid credential came from khaos's own
+ // identity provider rather than from the operator. It changes only the
+ // wording of one refusal, and that wording is the difference between telling
+ // an operator their target is down and telling them what to check.
+ MintedByKhaos bool `json:"minted_by_khaos,omitempty"`
+
+ // HeldCaveat qualifies a HELD verdict for the experiments whose absence of a
+ // finding covers less than it appears to. An unqualified "no bypass was
+ // observed" from a probe that tried one of four key serializations, or a
+ // hold the target's timeout may simply have absorbed, is khaos claiming more
+ // than it measured.
+ HeldCaveat string `json:"held_caveat,omitempty"`
+
+ // Defect names what khaos DID to the credential the finding is about, in
+ // words, so the verdict can state the actual observation.
+ //
+ // It is per experiment rather than a constant because the three experiments
+ // with a revoked stream invalidate their credential in three different ways,
+ // and one constant said all of them happened "while that authority was
+ // unreachable". That is true only of the outage experiment: a key rollover
+ // and a plain revocation both leave the provider fully reachable and
+ // answering, so the constant had khaos asserting something false about its
+ // own run on two of the three.
+ Defect string `json:"defect,omitempty"`
+}
+
+// Judge resolves two independent axes. The epistemic one answers whether the
+// run can be believed; the directional one answers which way the target broke.
+// A single-axis verdict cannot tell a target that failed closed from one that
+// was never enforcing to begin with.
+func Judge(observation Observation, polarity Polarity) Verdict {
+ // Every verdict carries an explicit cause. An empty one would read as a field
+ // khaos forgot rather than a diagnosis it never attempted.
+ verdict := Verdict{Direction: Undetermined, Cause: Undiagnosed}
+
+ if observation.RollbackFailed {
+ verdict.Warnings = append(verdict.Warnings, WarningRollbackFailed)
+ verdict.Reasons = append(verdict.Reasons, reasonRollback)
+ }
+
+ if observation.Interrupted {
+ verdict.Epistemic = Interrupted
+ verdict.Reasons = append(verdict.Reasons, reasonInterrupted)
+ return verdict
+ }
+
+ // A baseline that was never a steady state is the more fundamental finding
+ // and is knowable whether or not the fault went on to land.
+ if reason, unmet := baselineUnmet(observation.Baseline, observation.ForgedBaseline, observation.MintedByKhaos); unmet {
+ verdict.Epistemic = BaselineNotMet
+ verdict.Reasons = append(verdict.Reasons, reason)
+ return verdict
+ }
+
+ if reason, never := endpointNeverConsulted(observation); never {
+ verdict.Epistemic = BaselineNotMet
+ verdict.Reasons = append(verdict.Reasons, reason)
+ return verdict
+ }
+
+ if !observation.FaultConfirmed {
+ verdict.Epistemic = Aborted
+ verdict.Reasons = append(verdict.Reasons, reasonFaultUnconfirmed)
+ return verdict
+ }
+
+ // A failed mid-run action is NOT a clean run. The credential khaos measured
+ // was never revoked, so every acceptance of it is correct behaviour and the
+ // finding this experiment exists to produce was unreachable.
+ if observation.ActFailed {
+ verdict.Epistemic = Aborted
+ verdict.Reasons = append(verdict.Reasons, reasonActFailed)
+ return verdict
+ }
+
+ // A proven bypass is reported even on thin evidence, because one leaked
+ // request is a proof. Everything else waits for the sample floor.
+ //
+ // The three streams make three DIFFERENT claims and every one that fired is
+ // stated. "A credential khaos revoked at the authority was still accepted"
+ // is not "a credential the target had already rejected was accepted", and
+ // neither is "a credential khaos minted with a deliberate defect was
+ // accepted". Reporting one claim when a different stream carried the finding
+ // would be khaos describing an observation it never made, and returning on
+ // the first one would drop the others from a run where several are true.
+ if leaks := leaked(observation); len(leaks) > 0 {
+ verdict.Epistemic = Deviated
+ verdict.Direction = Permitted
+ verdict.Conclusive = true
+ verdict.Reasons = append(verdict.Reasons, leaks...)
+ verdict.Cause, verdict.Reasons = diagnose(observation, verdict.Reasons)
+ return finish(verdict, observation, polarity)
+ }
+
+ if paired := observation.Fault.Paired(); paired < MinimumPaired {
+ verdict.Epistemic = ProbeError
+ verdict.Warnings = append(verdict.Warnings, WarningThinEvidence)
+ verdict.Reasons = append(verdict.Reasons, sprintf(reasonThinFault, paired, MinimumPaired))
+ return verdict
+ }
+
+ // The stream that carries THIS experiment's hypothesis has to have produced
+ // usable decisions too. Paired() counts only the valid and invalid streams,
+ // so a run where every forged probe failed or timed out still cleared the
+ // floor above and fell through to HELD — reporting that the target refused a
+ // credential it was never successfully shown.
+ //
+ // This sits AFTER the leak checks on purpose: one observed acceptance of a
+ // credential khaos proved defective is a finding on its own, and must not be
+ // discarded for thin evidence.
+ if reason, thin := requiredStreamThin(observation); thin {
+ verdict.Epistemic = ProbeError
+ verdict.Warnings = append(verdict.Warnings, WarningThinEvidence)
+ verdict.Reasons = append(verdict.Reasons, reason)
+ return verdict
+ }
+
+ dir, reasons := direction(observation)
+ verdict.Direction = dir
+ verdict.Reasons = append(verdict.Reasons, reasons...)
+ verdict.Conclusive = true
+
+ if verdict.Direction == Held {
+ verdict.Epistemic = Passed
+ if paired := observation.Fault.Paired(); paired < MinimumToClaimClosed {
+ verdict.Conclusive = false
+ verdict.Warnings = append(verdict.Warnings, WarningThinEvidence)
+ verdict.Reasons = append(verdict.Reasons, sprintf(reasonWeakHeld, paired, MinimumToClaimClosed))
+ }
+ } else {
+ verdict.Epistemic = Deviated
+ }
+
+ return finish(verdict, observation, polarity)
+}
+
+func finish(verdict Verdict, observation Observation, polarity Polarity) Verdict {
+ if partial, total := unanswered(observation.Fault); partial > 0 && partial < total {
+ verdict.Warnings = append(verdict.Warnings, WarningPartialOutage)
+ verdict.Reasons = append(verdict.Reasons, sprintf(reasonPartialOutage, partial, total))
+ }
+
+ verdict.Recovery = recoveryOf(observation.Recovery)
+
+ switch verdict.Recovery {
+ case Degraded:
+ verdict.Epistemic = Deviated
+ verdict.Reasons = append(verdict.Reasons, reasonNotRecovered)
+ case Unobserved:
+ verdict.Warnings = append(verdict.Warnings, WarningRecoveryUnobserved)
+ verdict.Reasons = append(verdict.Reasons, reasonRecoveryUnobserved)
+ }
+
+ if verdict.Direction == Held && observation.HeldCaveat != "" {
+ verdict.Reasons = append(verdict.Reasons, observation.HeldCaveat)
+ }
+
+ if verdict.Direction != Held {
+ verdict.Defect = polarity.IsDefect(verdict.Direction)
+ verdict.Reasons = append(verdict.Reasons, polarity.Explain(verdict.Direction))
+ }
+
+ return verdict
+}
+
+// leaked returns one reason per stream that proved a bypass, in order of how
+// specific the claim is. An empty result means nothing leaked.
+func leaked(observation Observation) []string {
+ reasons := []string{}
+
+ if honoured := revokedHonoured(observation); honoured >= MinimumToProveOpen {
+ reasons = append(reasons, sprintf(reasonRevokedHonoured, orUnnamed(observation.Defect), honoured))
+
+ // The exposure as a NUMBER, so the finding is checkable rather than a
+ // count whose duration the reader has to infer from the timing flags.
+ // Research 07 section 2.2 calls this `stale_accept_window`, "the real
+ // exposure window of a leaked key", and it is explicitly a lower bound:
+ // khaos stopped looking when the run ended.
+ if span := staleWindow(observation); span > 0 {
+ reasons = append(reasons, sprintf(reasonStaleWindow, span.Round(time.Second)))
+ }
+ }
+
+ if accepted := forgedAccepted(observation); accepted >= MinimumToProveOpen {
+ reasons = append(reasons, sprintf(reasonForgedAccepted, orUnnamed(observation.Defect), accepted))
+ }
+
+ if monotonic := leakedMonotonic(observation); monotonic >= MinimumToProveOpen {
+ reasons = append(reasons, sprintf(reasonPermitted, monotonic))
+ }
+
+ return reasons
+}
+
+// Leaking reports whether ANY mechanism found a bypass. It is exported because
+// the ENGINE needs the answer mid-run: the idleness discriminator costs a traffic
+// pause and a second measurement window, and there is nothing to diagnose on a
+// target that never leaked.
+//
+// It has to cover every stream. Keyed on the monotonicity count alone, a
+// revocation experiment's finding sat entirely in the revoked stream and the
+// discriminator never ran, so the cause came back UNDIAGNOSED on a run that had
+// just proved a bypass.
+func Leaking(observation Observation) bool {
+ return len(leaked(observation)) > 0
+}
+
+// forgedAccepted counts acceptances of a credential khaos minted with one
+// deliberate defect.
+//
+// There is no monotonicity test here and that is deliberate, but the reason it
+// is sound differs by experiment and the shared sentence used to overstate it.
+// For the ten forgery traps the oracle reads the minted token back and inspects
+// the wrongness, so the credential is PROVED malformed before it is presented.
+// For `identity.jwks_unreachable` the confirmed observable is the key set
+// answering non-200 and nothing inspects the credential at all; its
+// defectiveness comes from CorruptSignature plus the ExpectRejected baseline
+// guard, which is adequate evidence but not the evidence the sentence named.
+// The reason now states only what is true of both, and the per-trap confirm line
+// carries the specific evidence.
+// staleWindow is the span from the first acceptance of the invalidated
+// credential to the last, across the fault window and the resumed one.
+func staleWindow(observation Observation) time.Duration {
+ fault := observation.Fault.Revoked.AcceptedSpan()
+ resumed := observation.Resumed.Revoked.AcceptedSpan()
+
+ // The idle pause sits between the two windows, so a run that leaked on both
+ // sides of it has an exposure spanning the pause as well.
+ if resumed > 0 {
+ return fault + observation.IdleFor + resumed
+ }
+
+ return fault
+}
+
+// requiredStreamThin reports whether a stream this experiment NEEDS could not be
+// measured. A declared forged stream is required: the experiment's whole
+// hypothesis is about that credential, so an absence of findings on it is only
+// meaningful if the target actually decided about it.
+func requiredStreamThin(observation Observation) (string, bool) {
+ if observation.ForgedBaseline == "" {
+ return "", false
+ }
+
+ if decided := observation.Fault.Forged.Countable(); decided < MinimumPaired {
+ return sprintf(reasonThinForged, decided, MinimumPaired), true
+ }
+
+ return "", false
+}
+
+func forgedAccepted(observation Observation) int {
+ return observation.Fault.Forged.Count(probe.Accepted)
+}
+
+func orUnnamed(defect string) string {
+ if defect == "" {
+ return "a deliberate defect"
+ }
+ return defect
+}
+
+// leakedMonotonic counts credentials the target had already rejected and then
+// accepted during the fault.
+func leakedMonotonic(observation Observation) int {
+ rejectedInBaseline := observation.Baseline.Invalid.Any(probe.Rejected)
+ return observation.Fault.Invalid.AcceptedAfter(rejectedInBaseline)
+}
+
+// revokedHonoured counts acceptances of a credential khaos invalidated at the
+// authority that were never followed by a refusal.
+//
+// Order-sensitive, for the same reason the DENIED test is. A target with a
+// correct BOUNDED cache serves the invalidated credential for exactly its cache
+// lifetime and then refuses it forever, which is the defensible bounded-stale
+// pattern research 07 section 3.3 lists as a PASS. Counting every acceptance in
+// the window reported that as a bypass.
+//
+// The outage experiment only escaped it by accident: its proxy confirmation
+// spends two seconds measuring both paths, which happens to be the range's cache
+// lifetime, so the stale allow had lapsed before the window opened. The identity
+// tier's confirmation is two local HTTP calls, so the same experiment shape
+// would have reported a false finding against a correct target.
+func revokedHonoured(observation Observation) int {
+ return observation.Fault.Revoked.AcceptedDespiteLapse()
+}
+
+// diagnose separates the two causes of a permitted credential. Both look
+// identical under steady traffic; they separate completely across a pause.
+//
+// The test is order-sensitive for the same reason the monotonicity rule is. The
+// fault window already established that the target accepts a credential it had
+// rejected. The only question left is whether it STILL accepts one once traffic
+// has stopped for longer than any plausible cache lifetime.
+func diagnose(observation Observation, reasons []string) (Cause, []string) {
+ // Deliberately NOT gated on Resumed.Paired(). That counts only the valid and
+ // invalid streams, so a resumed window full of decisions about the REVOKED
+ // credential — the stream that actually leaked — read as no evidence at all
+ // and skipped the diagnosis entirely. resolveCause below decides from the
+ // stream that carried the finding, and says so when that stream is empty.
+
+ // A forged credential getting through is not a caching bug, so there is no
+ // cause to diagnose. No experiment pairs a forged stream with the idleness
+ // pause today, and this guard is what stops the next one that does from
+ // being diagnosed off the INVALID stream's resumed window, which is a stream
+ // that never leaked.
+ if forgedAccepted(observation) > 0 &&
+ revokedHonoured(observation) == 0 &&
+ leakedMonotonic(observation) == 0 {
+ return Undiagnosed, reasons
+ }
+
+ // The resumed window is read on whichever stream carried the leak. A revoked
+ // credential and a never-valid one are different streams, and diagnosing the
+ // wrong one would report the pause's effect on something that never leaked.
+ //
+ // It is read with the same order-sensitive test the fault window uses, and
+ // for the same reason. The question is whether the credential is STILL being
+ // honoured once the pause has had its chance, so a window that starts
+ // accepted and settles into refusal is a cache that lapsed, however long it
+ // took. "Was it ever accepted after the pause" calls that one NO-EXPIRY,
+ // which is the opposite diagnosis and the opposite fix.
+ if revokedHonoured(observation) > 0 {
+ return resolveCause(observation.Resumed.Revoked, observation.FaultFor, observation.IdleFor, reasons)
+ }
+
+ return resolveCause(observation.Resumed.Invalid, observation.FaultFor, observation.IdleFor, reasons)
+}
+
+// resolveCause reads one resumed stream and refuses to diagnose from a window
+// that contains no evidence.
+//
+// Both halves are needed, and the second one was missing. `Resumed.Paired()`
+// ignores the revoked stream entirely, and AcceptedSinceLastRefusal returns zero
+// for a window that is ALL timeouts, because a timeout is neither an acceptance
+// nor a refusal. So a run whose resumed probes could not reach the target at all
+// fell through to TTL-EXTENSION and asserted that "pausing traffic restored
+// enforcement" — of a target that had stopped answering.
+//
+// That is reachable: `identity.idp_outage` severs the upstream with a proxy
+// hang, so a resumed probe against a wedged path times out rather than being
+// refused. TTL-EXTENSION also prescribes a fix, "stop writing to the cache in
+// the error branch", so the wrong diagnosis sends somebody to the wrong line.
+func resolveCause(resumed probe.Stream, steady, idle time.Duration, reasons []string) (Cause, []string) {
+ if resumed.AcceptedDespiteLapse() > 0 {
+ return StalePastPause, append(reasons, sprintf(reasonStalePastPause, idle, idle))
+ }
+
+ // Enforcement returning has to be OBSERVED, not inferred from an absence.
+ if resumed.Count(probe.Rejected) == 0 {
+ return Undiagnosed, append(reasons, sprintf(reasonNoResumedEvidence,
+ resumed.Countable(), idle))
+ }
+
+ // The soundness condition for the causal claim. A fixed, never-re-armed cache
+ // whose lifetime sits between the fault window and the pause produces exactly
+ // this observation, so without a steady-traffic window at least as long as
+ // the pause khaos cannot separate re-arming from ordinary expiry.
+ if steady < idle {
+ return LapsedOnPause, append(reasons, sprintf(reasonLapsedOnPause, idle, steady, idle))
+ }
+
+ return TTLExtension, append(reasons, sprintf(reasonTTLExtension, steady, idle))
+}
+
+func baselineUnmet(baseline probe.Window, expect probe.Expectation, minted bool) (string, bool) {
+ if paired := baseline.Paired(); paired < MinimumPaired {
+ return sprintf(reasonThinBaseline, paired, MinimumPaired), true
+ }
+
+ if !baseline.Invalid.Any(probe.Rejected) {
+ if redirects := baseline.Invalid.Count(probe.Redirected); redirects > 0 {
+ return sprintf(reasonRedirects, redirects), true
+ }
+ return reasonNoEnforcement, true
+ }
+
+ if !baseline.Valid.Any(probe.Accepted) {
+ // The identity tier gets its own wording, because the generic sentence is
+ // misleading there in the most likely failure case. A target whose key
+ // cache never expires is serving perfectly well and simply does not know
+ // khaos's keys, which khaos regenerates every run: measured against the
+ // range's forever-cache gateway, a second consecutive run could not
+ // establish a baseline and the generic reason said the target was not
+ // serving.
+ if minted {
+ return reasonNoServiceMinted, true
+ }
+ return reasonNoService, true
+ }
+
+ // An experiment that revokes a credential mid-run needs the target to have
+ // been HONOURING that credential first. If it was never accepted there is
+ // nothing for the revocation to take away, and every later rejection is the
+ // target being correct rather than khaos finding anything.
+ if baseline.Revoked.Countable() > 0 && !baseline.Revoked.Any(probe.Accepted) {
+ return reasonNothingToRevoke, true
+ }
+
+ return forgedBaselineUnmet(baseline, expect)
+}
+
+// endpointNeverConsulted is the liveness guard for a fault applied to an
+// endpoint rather than to a credential. Without it, three experiments report a
+// clean survival of a fault the target never experienced.
+func endpointNeverConsulted(observation Observation) (string, bool) {
+ if observation.ConsultedWhat == "" || observation.Consulted > 0 {
+ return "", false
+ }
+
+ return sprintf(reasonNeverConsulted, observation.ConsultedWhat), true
+}
+
+// forgedBaselineUnmet applies the guard the experiment DECLARED for its forged
+// stream. The two expectations point in opposite directions and both are
+// liveness guards rather than findings.
+//
+// ExpectAccepted catches the single most likely misconfiguration in the whole
+// family: a target that is not pointed at khaos's identity provider at all. It
+// refuses every credential khaos mints, for reasons that have nothing to do with
+// any trap, and without this guard a run against it reports a clean HELD for
+// every one of the forgery experiments.
+//
+// ExpectRejected catches the mirror: a target that was not checking the thing
+// the fault is about to break, so its later acceptance proves nothing changed.
+func forgedBaselineUnmet(baseline probe.Window, expect probe.Expectation) (string, bool) {
+ if baseline.Forged.Countable() == 0 {
+ return "", false
+ }
+
+ switch expect {
+ case probe.ExpectAccepted:
+ if !baseline.Forged.Any(probe.Accepted) {
+ return reasonProviderNotTrusted, true
+ }
+ case probe.ExpectRejected:
+ if !baseline.Forged.Any(probe.Rejected) {
+ return reasonForgedNotChecked, true
+ }
+ }
+
+ return "", false
+}
+
+func direction(observation Observation) (Direction, []string) {
+ down, total := unanswered(observation.Fault)
+
+ if total > 0 && float64(down) >= float64(total)*crashFraction {
+ return Crashed, []string{sprintf(reasonCrashed, down, total)}
+ }
+
+ // The baseline guarantees the valid credential was accepted, so the latch
+ // starts armed: any later refusal is the target having stopped serving.
+ if denied := observation.Fault.Valid.RejectedAfter(true); denied > 0 {
+ return Denied, []string{sprintf(reasonDenied, denied)}
+ }
+
+ return Held, []string{reasonHeld}
+}
+
+// unanswered counts across BOTH streams. A gateway that short-circuits the
+// invalid credential at the edge while hanging on the valid one is not a
+// crashed target, and judging the valid stream alone reports it as one.
+func unanswered(window probe.Window) (int, int) {
+ down := 0
+ total := 0
+
+ for _, stream := range []probe.Stream{window.Valid, window.Invalid} {
+ for _, outcome := range stream.Outcomes {
+ if !outcome.Disposition.Countable() {
+ continue
+ }
+ total++
+ if !outcome.Disposition.Answered() {
+ down++
+ }
+ }
+ }
+
+ return down, total
+}
+
+// recoveryOf requires a run of consecutive healthy pairs, because declaring a
+// state transition on a single lucky sample is how a flaky recovery reads as a
+// clean one.
+//
+// An empty window is UNOBSERVED rather than recovered. Returning true there
+// meant a run that collected no recovery evidence at all — an interrupted run,
+// a target that stopped answering, a recovery window too short to pair —
+// reported that the target came back, and exited 0.
+func recoveryOf(recovery probe.Window) Recovery {
+ if recovery.Paired() == 0 {
+ return Unobserved
+ }
+
+ run := 0
+ for i := 0; i < recovery.Valid.Len() && i < recovery.Invalid.Len(); i++ {
+ if recovery.Valid.Outcomes[i].Disposition == probe.Accepted &&
+ recovery.Invalid.Outcomes[i].Disposition == probe.Rejected {
+ run++
+ if run >= RecoveryRunLength {
+ return Recovered
+ }
+ continue
+ }
+ run = 0
+ }
+
+ return Degraded
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go
new file mode 100644
index 00000000..1b153aab
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go
@@ -0,0 +1,341 @@
+// ©AngelaMos | 2026
+// verdict_test.go
+
+package experiment
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+func stream(credential probe.Credential, dispositions ...probe.Disposition) probe.Stream {
+ s := probe.Stream{Credential: credential}
+ base := time.Now().UTC()
+
+ for i, d := range dispositions {
+ s.Outcomes = append(s.Outcomes, probe.Outcome{
+ At: base.Add(time.Duration(i) * time.Second),
+ Credential: credential,
+ Disposition: d,
+ })
+ }
+
+ return s
+}
+
+func repeat(d probe.Disposition, n int) []probe.Disposition {
+ out := make([]probe.Disposition, n)
+ for i := range out {
+ out[i] = d
+ }
+ return out
+}
+
+func healthyBaseline() probe.Window {
+ return probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, MinimumPaired)...),
+ }
+}
+
+func healthyRecovery() probe.Window {
+ return probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, 2)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, 2)...),
+ }
+}
+
+var authPolarity = Polarity{Defects: []Direction{Permitted}, Rationale: "an authentication check that stops enforcing is catastrophic"}
+
+func TestTheVerdictTruthTable(t *testing.T) {
+ cases := []struct {
+ name string
+ fault probe.Window
+ wantEpistemic Epistemic
+ wantDirection Direction
+ wantDefect bool
+ }{
+ {
+ name: "survived with enforcement intact",
+ fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, MinimumPaired)...),
+ },
+ wantEpistemic: Passed,
+ wantDirection: Held,
+ },
+ {
+ name: "failed closed, legitimate traffic refused",
+ fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Rejected, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, MinimumPaired)...),
+ },
+ wantEpistemic: Deviated,
+ wantDirection: Denied,
+ },
+ {
+ name: "failed open, a rejected credential was later accepted",
+ fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Accepted, MinimumPaired)...),
+ },
+ wantEpistemic: Deviated,
+ wantDirection: Permitted,
+ wantDefect: true,
+ },
+ {
+ name: "stopped answering entirely",
+ fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Unavailable, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Unavailable, MinimumPaired)...),
+ },
+ wantEpistemic: Deviated,
+ wantDirection: Crashed,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: healthyBaseline(),
+ Fault: tc.fault,
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Epistemic != tc.wantEpistemic {
+ t.Fatalf("epistemic = %s, want %s (%s)", verdict.Epistemic, tc.wantEpistemic,
+ strings.Join(verdict.Reasons, "; "))
+ }
+
+ if verdict.Direction != tc.wantDirection {
+ t.Fatalf("direction = %s, want %s (%s)", verdict.Direction, tc.wantDirection,
+ strings.Join(verdict.Reasons, "; "))
+ }
+
+ if verdict.Defect != tc.wantDefect {
+ t.Fatalf("defect = %v, want %v", verdict.Defect, tc.wantDefect)
+ }
+ })
+ }
+}
+
+func TestATargetThatWasNeverEnforcingIsNotAPass(t *testing.T) {
+ baseline := healthyBaseline()
+ baseline.Invalid = stream(probe.Invalid, repeat(probe.Accepted, MinimumPaired)...)
+
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: baseline,
+ Fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Accepted, MinimumPaired)...),
+ },
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Epistemic != BaselineNotMet {
+ t.Fatalf("epistemic = %s, want %s; a target that accepted an invalid credential BEFORE "+
+ "the fault was never enforcing, and a two-valued verdict would report this as a clean pass "+
+ "while teaching the operator nothing", verdict.Epistemic, BaselineNotMet)
+ }
+
+ if verdict.Conclusive {
+ t.Fatal("a run against a target that was not enforcing was marked conclusive")
+ }
+
+ if verdict.Direction != Undetermined {
+ t.Fatalf("direction = %s, want %s", verdict.Direction, Undetermined)
+ }
+}
+
+func TestAServiceStillServingAValidCredentialIsNotPermitted(t *testing.T) {
+ // A service that keeps honouring a credential that was valid when the
+ // outage began is behaving CORRECTLY. Only a credential it had already
+ // rejected, later accepted, is a control that stopped enforcing.
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: healthyBaseline(),
+ Fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, MinimumPaired)...),
+ },
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Direction == Permitted {
+ t.Fatal("a service that kept serving a still-valid credential was reported as failing open")
+ }
+
+ if verdict.Direction != Held {
+ t.Fatalf("direction = %s, want %s", verdict.Direction, Held)
+ }
+}
+
+func TestAnEarlyAcceptanceFollowedByEnforcementIsNotPermitted(t *testing.T) {
+ // A target that accepted once while warming up and then enforced
+ // consistently has not stopped enforcing. Counting acceptances instead of
+ // ordering them calls this a failure to enforce.
+ baseline := healthyBaseline()
+ baseline.Invalid = stream(probe.Invalid,
+ probe.Accepted, probe.Rejected, probe.Rejected, probe.Rejected, probe.Rejected)
+
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: baseline,
+ Fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, MinimumPaired)...),
+ },
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Direction == Permitted {
+ t.Fatal("an acceptance that happened BEFORE the target ever enforced was counted as the " +
+ "control giving way; the test is 'a credential it had already refused was later " +
+ "honoured', not 'something was accepted at some point'")
+ }
+
+ if verdict.Direction != Held {
+ t.Fatalf("direction = %s, want %s", verdict.Direction, Held)
+ }
+}
+
+func TestPermittedRequiresAPriorRejection(t *testing.T) {
+ baseline := healthyBaseline()
+ baseline.Invalid = stream(probe.Invalid, repeat(probe.Unavailable, MinimumPaired)...)
+
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: baseline,
+ Fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Accepted, MinimumPaired)...),
+ },
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Epistemic != BaselineNotMet {
+ t.Fatalf("epistemic = %s; with no rejection ever observed there is no enforcement to lose, "+
+ "so there is nothing to conclude", verdict.Epistemic)
+ }
+}
+
+func TestAThinSampleIsNotConclusive(t *testing.T) {
+ thin := probe.Window{
+ Valid: stream(probe.Valid, probe.Accepted),
+ Invalid: stream(probe.Invalid, probe.Rejected),
+ }
+
+ for _, tc := range []struct {
+ name string
+ observation Observation
+ }{
+ {"thin fault window after a healthy baseline", Observation{FaultConfirmed: true, Baseline: healthyBaseline(), Fault: thin}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ verdict := Judge(tc.observation, authPolarity)
+
+ if verdict.Conclusive {
+ t.Fatal("a run with too few samples was marked conclusive")
+ }
+
+ if verdict.Epistemic != ProbeError {
+ t.Fatalf("epistemic = %s, want %s", verdict.Epistemic, ProbeError)
+ }
+ })
+ }
+}
+
+func TestAFaultThatNeverLandedCannotProduceAVerdict(t *testing.T) {
+ verdict := Judge(Observation{
+ FaultConfirmed: false,
+ Baseline: healthyBaseline(),
+ Fault: healthyBaseline(),
+ Recovery: healthyRecovery(),
+ }, authPolarity)
+
+ if verdict.Epistemic != Aborted {
+ t.Fatalf("epistemic = %s, want %s; nothing measured can be attributed to a fault "+
+ "that was never confirmed applied", verdict.Epistemic, Aborted)
+ }
+
+ if verdict.Conclusive {
+ t.Fatal("a run whose fault never landed was marked conclusive")
+ }
+}
+
+func TestPolarityDecidesWhetherADeviationIsADefect(t *testing.T) {
+ failedOpen := Observation{
+ FaultConfirmed: true,
+ Baseline: healthyBaseline(),
+ Fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Accepted, MinimumPaired)...),
+ },
+ Recovery: healthyRecovery(),
+ }
+
+ strict := Judge(failedOpen, Polarity{Defects: []Direction{Permitted}})
+ if !strict.Defect {
+ t.Fatal("failing open was not a defect for a control that declares it one")
+ }
+
+ lenient := Judge(failedOpen, Polarity{Defects: []Direction{Denied}})
+ if lenient.Defect {
+ t.Fatal("failing open was called a defect for a control that deliberately tolerates it; " +
+ "a rate-limit counter that stops counting when its datastore is unreachable is a " +
+ "trade, not a bug")
+ }
+
+ if lenient.Direction != Permitted {
+ t.Fatalf("direction = %s; polarity changes whether a deviation is a defect, never which "+
+ "direction it went", lenient.Direction)
+ }
+}
+
+func TestAFailureToRecoverIsRecorded(t *testing.T) {
+ verdict := Judge(Observation{
+ FaultConfirmed: true,
+ Baseline: healthyBaseline(),
+ Fault: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Rejected, MinimumPaired)...),
+ },
+ Recovery: probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Unavailable, 2)...),
+ Invalid: stream(probe.Invalid, repeat(probe.Unavailable, 2)...),
+ },
+ }, authPolarity)
+
+ if verdict.Epistemic != Deviated {
+ t.Fatalf("epistemic = %s; a target that never came back after the revert did not pass",
+ verdict.Epistemic)
+ }
+}
+
+func TestEveryVerdictCarriesAReason(t *testing.T) {
+ observations := map[string]Observation{
+ "not applied": {FaultConfirmed: false},
+ "thin": {FaultConfirmed: true, Baseline: healthyBaseline()},
+ "no baseline": {FaultConfirmed: true, Baseline: probe.Window{Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired)...), Invalid: stream(probe.Invalid, repeat(probe.Accepted, MinimumPaired)...)}, Fault: healthyBaseline()},
+ "held": {FaultConfirmed: true, Baseline: healthyBaseline(), Fault: healthyBaseline(), Recovery: healthyRecovery()},
+ }
+
+ for name, observation := range observations {
+ t.Run(name, func(t *testing.T) {
+ verdict := Judge(observation, authPolarity)
+ if len(verdict.Reasons) == 0 {
+ t.Fatal("a verdict was produced with no stated reason")
+ }
+ for _, reason := range verdict.Reasons {
+ if strings.TrimSpace(reason) == "" {
+ t.Fatal("a verdict reason was empty")
+ }
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/helpers_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/helpers_test.go
new file mode 100644
index 00000000..c429096d
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/helpers_test.go
@@ -0,0 +1,50 @@
+// ©AngelaMos | 2026
+// helpers_test.go
+
+package idp
+
+import (
+ "context"
+ "net/http"
+ "testing"
+ "time"
+)
+
+const helperTimeout = 10 * time.Second
+
+func listening(t *testing.T) *Provider {
+ t.Helper()
+
+ p, err := New(Config{})
+ if err != nil {
+ t.Fatalf("build a provider: %v", err)
+ }
+
+ if err := p.Listen("127.0.0.1:0"); err != nil {
+ t.Fatalf("bind a provider: %v", err)
+ }
+
+ t.Cleanup(func() { _ = p.Close() })
+
+ return p
+}
+
+func httpClient() *http.Client {
+ return &http.Client{Timeout: helperTimeout}
+}
+
+func (p *Provider) active(t *testing.T, credential string) bool {
+ t.Helper()
+
+ addr, err := p.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ body, err := fetchIntrospection(context.Background(), httpClient(), p.introspectURL("http://"+addr, credential))
+ if err != nil {
+ t.Fatalf("introspect: %v", err)
+ }
+
+ return body.Active
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/jwk.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/jwk.go
new file mode 100644
index 00000000..5a7f9ddc
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/jwk.go
@@ -0,0 +1,206 @@
+// ©AngelaMos | 2026
+// jwk.go
+
+package idp
+
+import (
+ "crypto/ecdsa"
+ "crypto/rsa"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "math"
+ "math/big"
+)
+
+const (
+ errNoSuchKid = "%w: %q (published: %s)"
+ errIncompleteJWK = "%w: the key %q does not carry %s"
+ errUnusableJWK = "%w: the key %q carries material khaos will not build a verifier from"
+ errUnsupportedCurve = "%w: the key %q is on curve %q and khaos only serves %s"
+ kidSep = ", "
+)
+
+var ErrNoSuchKey = errors.New("that key is not in the published JWK Set")
+
+// JWK is the published form of one key: the thumbprint-derived `kid`, the
+// algorithm it is bound to, and the public material. The `alg` member is present
+// deliberately, because RFC 8725 rule 2 says a verifier MUST cross-check it
+// against the token header, and a provider that omits it denies the verifier the
+// means to comply.
+type JWK struct {
+ Kid string `json:"kid"`
+ Kty string `json:"kty"`
+ Use string `json:"use"`
+ Alg string `json:"alg"`
+
+ N string `json:"n,omitempty"`
+ E string `json:"e,omitempty"`
+
+ Crv string `json:"crv,omitempty"`
+ X string `json:"x,omitempty"`
+ Y string `json:"y,omitempty"`
+}
+
+type JWKSet struct {
+ Keys []JWK `json:"keys"`
+}
+
+func (s JWKSet) Kids() []string {
+ out := make([]string, 0, len(s.Keys))
+ for _, key := range s.Keys {
+ out = append(out, key.Kid)
+ }
+ return out
+}
+
+func (s JWKSet) Has(kid string) bool {
+ for _, key := range s.Keys {
+ if key.Kid == kid {
+ return true
+ }
+ }
+ return false
+}
+
+func (s JWKSet) Lookup(kid string) (JWK, error) {
+ for _, key := range s.Keys {
+ if key.Kid == kid {
+ return key, nil
+ }
+ }
+ return JWK{}, fmt.Errorf(errNoSuchKid, ErrNoSuchKey, kid, joinKids(s.Kids()))
+}
+
+func joinKids(kids []string) string {
+ if len(kids) == 0 {
+ return "nothing"
+ }
+ out := kids[0]
+ for _, kid := range kids[1:] {
+ out += kidSep + kid
+ }
+ return out
+}
+
+// PublicJWK is the published form. It carries the full member set a verifier
+// needs, which is a superset of the required members the thumbprint is computed
+// over, so the two forms are built separately on purpose.
+func (k *Key) PublicJWK() JWK {
+ out := JWK{
+ Kid: k.ID,
+ Use: useSignature,
+ Alg: string(k.Alg),
+ }
+
+ if k.rsaKey != nil {
+ out.Kty = keyTypeRSA
+ out.N = b64(k.rsaKey.N.Bytes())
+ out.E = b64(exponentBytes(k.rsaKey.E))
+ return out
+ }
+
+ name, size := curveShape(k.ecKey.Curve)
+ x, y := coordinates(&k.ecKey.PublicKey, size)
+
+ out.Kty = keyTypeEC
+ out.Crv = name
+ out.X = b64(x)
+ out.Y = b64(y)
+
+ return out
+}
+
+// RSAPublicKey rebuilds a verifiable key from a published JWK. khaos needs it on
+// its own confirmation path: proving a minted token actually verifies under the
+// key it names is what makes a target's acceptance of a DIFFERENT token mean
+// something.
+// RSAPublicKey rebuilds a verifiable key from a published JWK, and refuses
+// material that is absent or does not fit.
+//
+// Both refusals were missing. decodeBigInt("") returns zero with a nil error, so
+// an empty JWK yielded `{N: 0, E: 0}` and no error at all; and `int(e.Int64())`
+// is documented as undefined on overflow, which measured as `"e":"AQAAAAAAAAAB"`
+// (2^64 + 1) silently becoming e = 1 — the one exponent that makes the signature
+// and the encoded message the same bytes.
+func (j JWK) RSAPublicKey() (*rsa.PublicKey, error) {
+ if j.N == "" || j.E == "" {
+ return nil, fmt.Errorf(errIncompleteJWK, ErrNoSuchKey, j.Kid, "n and e")
+ }
+
+ n, err := decodeBigInt(j.N)
+ if err != nil {
+ return nil, err
+ }
+
+ e, err := decodeBigInt(j.E)
+ if err != nil {
+ return nil, err
+ }
+
+ if n.Sign() <= 0 || !e.IsInt64() || e.Int64() < 3 || e.Int64() > math.MaxInt32 {
+ return nil, fmt.Errorf(errUnusableJWK, ErrNoSuchKey, j.Kid)
+ }
+
+ return &rsa.PublicKey{N: n, E: int(e.Int64())}, nil
+}
+
+// ECPublicKey rebuilds an EC key, and refuses a curve khaos does not serve.
+//
+// It used to ignore `crv` entirely and always build a P-256 key, so a P-224 JWK
+// came back labelled P-256 with a zero-padded coordinate, which is a different
+// key under a different thumbprint.
+func (j JWK) ECPublicKey() (*ecdsa.PublicKey, error) {
+ if j.Crv != curveP256 {
+ return nil, fmt.Errorf(errUnsupportedCurve, ErrNoSuchKey, j.Kid, j.Crv, curveP256)
+ }
+
+ if j.X == "" || j.Y == "" {
+ return nil, fmt.Errorf(errIncompleteJWK, ErrNoSuchKey, j.Kid, "x and y")
+ }
+
+ x, err := decodeBigInt(j.X)
+ if err != nil {
+ return nil, err
+ }
+
+ y, err := decodeBigInt(j.Y)
+ if err != nil {
+ return nil, err
+ }
+
+ if !p256().IsOnCurve(x, y) {
+ return nil, fmt.Errorf(errUnusableJWK, ErrNoSuchKey, j.Kid)
+ }
+
+ return &ecdsa.PublicKey{Curve: p256(), X: x, Y: y}, nil
+}
+
+func decodeBigInt(encoded string) (*big.Int, error) {
+ raw, err := decodeSegment(encoded)
+ if err != nil {
+ return nil, err
+ }
+ return new(big.Int).SetBytes(raw), nil
+}
+
+func (s JWKSet) Encode() ([]byte, error) {
+ return json.Marshal(s)
+}
+
+func ParseJWKSet(raw []byte) (JWKSet, error) {
+ var set JWKSet
+ if err := json.Unmarshal(raw, &set); err != nil {
+ return JWKSet{}, err
+ }
+ return set, nil
+}
+
+func bigFromBytes(raw []byte) *big.Int {
+ return new(big.Int).SetBytes(raw)
+}
+
+// UseSignature is RFC 7517's `use` value for a signing key. It is exported so
+// the range's verifier can check it against the same constant khaos publishes,
+// rather than against a string literal that could drift from it silently.
+const UseSignature = useSignature
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/key.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/key.go
new file mode 100644
index 00000000..d08433df
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/key.go
@@ -0,0 +1,515 @@
+// ©AngelaMos | 2026
+// key.go
+
+package idp
+
+import (
+ "crypto"
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/sha256"
+ "crypto/x509"
+ "encoding/base64"
+ "encoding/pem"
+ "errors"
+ "fmt"
+ "math/big"
+)
+
+// Algorithm is the JWS `alg` header value. khaos needs to mint tokens under
+// algorithms a correct provider would refuse to use at all, so this is the set
+// khaos can SIGN with rather than a set it endorses.
+type Algorithm string
+
+const (
+ RS256 Algorithm = "RS256"
+ ES256 Algorithm = "ES256"
+ HS256 Algorithm = "HS256"
+
+ // None is RFC 7515's "No digital signature or MAC performed". It exists here
+ // because a verifier that routes on it is trap T1, and khaos cannot test for
+ // that without being willing to emit it.
+ None Algorithm = "none"
+)
+
+const (
+ keyTypeRSA = "RSA"
+ keyTypeEC = "EC"
+
+ curveP256 = "P-256"
+
+ useSignature = "sig"
+
+ // The modulus size khaos signs its honest tokens with. 2048 is what a real
+ // provider uses and it is comfortably above the floor crypto/rsa enforces.
+ strongBits = 2048
+
+ // Deliberately below that floor. Measured on go1.26.5: crypto/rsa refuses a
+ // key this size at GenerateKey, at SignPKCS1v15 AND at VerifyPKCS1v15, all
+ // three through checkKeySize, so a token signed under it cannot be produced
+ // or checked by the standard library at all. That is the whole point of the
+ // weak-key probe, and it is why khaos carries its own PKCS#1 v1.5 path below.
+ weakBits = 512
+
+ // Each prime is half the modulus. Assembling the key by hand is what lets
+ // khaos hold one crypto/rsa will not generate.
+ weakPrimeBits = weakBits / 2
+
+ // crypto/rsa's own floor, read from the source rather than remembered:
+ // checkKeySize returns nil at or above this and an error below it unless the
+ // rsa1024min GODEBUG is set. khaos never sets that GODEBUG, because doing so
+ // to enable one probe would weaken every other thing khaos does with RSA.
+ libraryFloorBits = 1024
+
+ primeAttempts = 64
+
+ // ES256 signatures are the fixed-width concatenation of r and s, each padded
+ // to the curve's coordinate size. RFC 7518 section 3.4.
+ p256CoordinateBytes = 32
+
+ errNoPrime = "khaos could not find a %d-bit prime for its deliberately weak signing key after %d attempts"
+ errBadAlg = "%w: %q"
+ errNotRSA = "%w: %s is not an RSA key, and %s is an RSA algorithm"
+ errNotEC = "%w: %s is not an EC key, and %s is an EC algorithm"
+ errCoprime = "khaos could not invert the public exponent against its chosen primes"
+ errShortModulus = "the digest and PKCS#1 v1.5 framing need %d bytes and this key's modulus is only %d"
+)
+
+var ErrUnsupportedAlgorithm = errors.New("khaos cannot sign under that algorithm")
+
+var ErrKeyAlgorithmMismatch = errors.New("that key cannot sign under that algorithm")
+
+// Key is one signing key the provider publishes and signs with. The thumbprint
+// is its `kid`, so a rotated key's identity is derived from the key material
+// rather than from a counter khaos increments: two providers holding the same
+// key agree on its name, and a `kid` cannot be reused for different bytes.
+type Key struct {
+ ID string
+ Alg Algorithm
+
+ rsaKey *rsa.PrivateKey
+ ecKey *ecdsa.PrivateKey
+}
+
+// NewRSAKey mints a key a real provider would use.
+func NewRSAKey() (*Key, error) {
+ return newRSAKey(strongBits)
+}
+
+// NewWeakRSAKey mints a key below the size crypto/rsa will accept, by assembling
+// the struct from primes khaos finds itself. Nothing in crypto/rsa validates a
+// PrivateKey on construction, so the value is well-formed and usable right up to
+// the moment a crypto/rsa function is called with it, which is exactly the
+// position a verifier that reimplements RSA puts itself in.
+func NewWeakRSAKey() (*Key, error) {
+ key, err := weakRSAPrivateKey()
+ if err != nil {
+ return nil, err
+ }
+ return keyFromRSA(key)
+}
+
+func newRSAKey(bits int) (*Key, error) {
+ key, err := rsa.GenerateKey(rand.Reader, bits)
+ if err != nil {
+ return nil, err
+ }
+ return keyFromRSA(key)
+}
+
+func keyFromRSA(key *rsa.PrivateKey) (*Key, error) {
+ out := &Key{Alg: RS256, rsaKey: key}
+
+ id, err := out.Thumbprint()
+ if err != nil {
+ return nil, err
+ }
+
+ out.ID = id
+
+ return out, nil
+}
+
+// NewECKey mints khaos's ES256 key. The curve is P-256 because ES256 is defined
+// over it, and the surrounding code derives its widths from the curve rather
+// than from that fact, so adding ES384 later is a constructor and not a hunt.
+func NewECKey() (*Key, error) {
+ key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ if err != nil {
+ return nil, err
+ }
+
+ out := &Key{Alg: ES256, ecKey: key}
+
+ id, err := out.Thumbprint()
+ if err != nil {
+ return nil, err
+ }
+
+ out.ID = id
+
+ return out, nil
+}
+
+// Weak reports whether this key is below the size crypto/rsa accepts. It is
+// derived from the modulus rather than remembered from construction, so a key
+// cannot claim to be strong while carrying weak material.
+func (k *Key) Weak() bool {
+ return k.rsaKey != nil && k.rsaKey.N.BitLen() < libraryFloorBits
+}
+
+// PublicDER is the SPKI encoding of the public key. It is what the algorithm
+// confusion probe uses as its HMAC secret, because an RSA public key loaded from
+// a PEM file is these bytes and a verifier that hands them to an HMAC function
+// is trap T2.
+func (k *Key) PublicDER() ([]byte, error) {
+ return x509.MarshalPKIXPublicKey(k.public())
+}
+
+func (k *Key) public() crypto.PublicKey {
+ if k.rsaKey != nil {
+ return &k.rsaKey.PublicKey
+ }
+ return &k.ecKey.PublicKey
+}
+
+// Sign produces the JWS signature over the signing input. It refuses an
+// algorithm the key cannot carry rather than silently producing a signature
+// nothing can check, because a probe that fails for the wrong reason reads
+// exactly like a target that refused it.
+func (k *Key) Sign(alg Algorithm, signingInput []byte) ([]byte, error) {
+ switch alg {
+ case RS256:
+ if k.rsaKey == nil {
+ return nil, fmt.Errorf(errNotRSA, ErrKeyAlgorithmMismatch, k.ID, alg)
+ }
+ return k.signRSA(signingInput)
+ case ES256:
+ if k.ecKey == nil {
+ return nil, fmt.Errorf(errNotEC, ErrKeyAlgorithmMismatch, k.ID, alg)
+ }
+ return k.signEC(signingInput)
+ default:
+ return nil, fmt.Errorf(errBadAlg, ErrUnsupportedAlgorithm, alg)
+ }
+}
+
+// signRSA uses the standard library wherever the standard library will accept
+// the key, and khaos's own PKCS#1 v1.5 path only where it will not. The two are
+// pinned to agree byte for byte at a legal key size, so the weak-key path is
+// exercised against a known-good reference rather than trusted on its own.
+func (k *Key) signRSA(signingInput []byte) ([]byte, error) {
+ digest := sha256.Sum256(signingInput)
+
+ if !k.Weak() {
+ return rsa.SignPKCS1v15(rand.Reader, k.rsaKey, crypto.SHA256, digest[:])
+ }
+
+ return signPKCS1v15Raw(k.rsaKey, digest[:])
+}
+
+func (k *Key) signEC(signingInput []byte) ([]byte, error) {
+ digest := sha256.Sum256(signingInput)
+
+ r, s, err := ecdsa.Sign(rand.Reader, k.ecKey, digest[:])
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]byte, 2*p256CoordinateBytes)
+ r.FillBytes(out[:p256CoordinateBytes])
+ s.FillBytes(out[p256CoordinateBytes:])
+
+ return out, nil
+}
+
+// Thumbprint is the RFC 7638 JWK thumbprint: the required members only, in
+// lexicographic order, with no whitespace, SHA-256, base64url with no padding.
+// It is implemented from the specification text and pinned by the worked example
+// in RFC 7638 section 3.1.
+func (k *Key) Thumbprint() (string, error) {
+ canonical, err := k.canonicalJWK()
+ if err != nil {
+ return "", err
+ }
+
+ sum := sha256.Sum256(canonical)
+
+ return base64.RawURLEncoding.EncodeToString(sum[:]), nil
+}
+
+func (k *Key) canonicalJWK() ([]byte, error) {
+ if k.rsaKey != nil {
+ return canonicalRSAJWK(&k.rsaKey.PublicKey), nil
+ }
+ if k.ecKey != nil {
+ return canonicalECJWK(&k.ecKey.PublicKey), nil
+ }
+ return nil, fmt.Errorf(errBadAlg, ErrUnsupportedAlgorithm, k.Alg)
+}
+
+// canonicalRSAJWK is the required-members-only RSA form. The member order is
+// e, kty, n, which is lexicographic, and it is written literally rather than
+// marshalled, because encoding/json orders struct fields by declaration and a
+// field reorder would silently change every kid khaos derives.
+func canonicalRSAJWK(pub *rsa.PublicKey) []byte {
+ return []byte(fmt.Sprintf(
+ `{"e":%q,"kty":%q,"n":%q}`,
+ b64(exponentBytes(pub.E)),
+ keyTypeRSA,
+ b64(pub.N.Bytes()),
+ ))
+}
+
+// exponentBytes is the base64url-ready form of the public exponent: big-endian,
+// with no leading zero octets. RFC 7518 section 6.3.1.2 calls for the
+// "base64urlUInt-encoded value", and a leading zero would change the thumbprint
+// for the same key.
+func exponentBytes(e int) []byte {
+ return new(big.Int).SetInt64(int64(e)).Bytes()
+}
+
+// canonicalECJWK is the required-members-only EC form, in the lexicographic
+// order crv, kty, x, y.
+func canonicalECJWK(pub *ecdsa.PublicKey) []byte {
+ name, size := curveShape(pub.Curve)
+ x, y := coordinates(pub, size)
+
+ return []byte(fmt.Sprintf(
+ `{"crv":%q,"kty":%q,"x":%q,"y":%q}`,
+ name,
+ keyTypeEC,
+ b64(x),
+ b64(y),
+ ))
+}
+
+// curveShape is the curve's JWK name and its coordinate width, DERIVED from the
+// curve rather than hardcoded.
+//
+// Hardcoding P-256 in three places made a larger curve PANIC inside the `jwks`
+// HTTP handler — math/big's FillBytes refuses a buffer too small for the value —
+// and made a smaller one serialize as a zero-padded 28-byte coordinate labelled
+// P-256, which is a different key under a different thumbprint. Neither was
+// reachable through NewECKey, which is why neither was noticed.
+func curveShape(curve elliptic.Curve) (string, int) {
+ switch curve {
+ case elliptic.P384():
+ return "P-384", 48
+ case elliptic.P521():
+ return "P-521", 66
+ default:
+ return curveP256, p256CoordinateBytes
+ }
+}
+
+// coordinates is the fixed-width big-endian pair RFC 7518 section 6.2.1.2 calls
+// for. The width comes from the curve, so a short coordinate is padded and a
+// long one cannot overflow the buffer.
+func coordinates(pub *ecdsa.PublicKey, size int) ([]byte, []byte) {
+ x := make([]byte, size)
+ y := make([]byte, size)
+
+ if pub.X.BitLen() > size*8 || pub.Y.BitLen() > size*8 {
+ return x, y
+ }
+
+ pub.X.FillBytes(x)
+ pub.Y.FillBytes(y)
+
+ return x, y
+}
+
+func b64(raw []byte) string {
+ return base64.RawURLEncoding.EncodeToString(raw)
+}
+
+// weakRSAPrivateKey assembles a key crypto/rsa will not generate. It is built
+// the long way round rather than with GenerateKey because GenerateKey enforces
+// the same floor as the signing and verifying paths.
+func weakRSAPrivateKey() (*rsa.PrivateKey, error) {
+ exponent := big.NewInt(defaultExponent)
+
+ for range primeAttempts {
+ p, err := rand.Prime(rand.Reader, weakPrimeBits)
+ if err != nil {
+ return nil, err
+ }
+
+ q, err := rand.Prime(rand.Reader, weakPrimeBits)
+ if err != nil {
+ return nil, err
+ }
+
+ if p.Cmp(q) == 0 {
+ continue
+ }
+
+ n := new(big.Int).Mul(p, q)
+ if n.BitLen() != weakBits {
+ continue
+ }
+
+ // Carmichael's totient, which is what PKCS#1 actually requires. Using
+ // Euler's would also work but produces a larger private exponent for no
+ // benefit.
+ pMinus := new(big.Int).Sub(p, one)
+ qMinus := new(big.Int).Sub(q, one)
+ totient := new(big.Int).Div(new(big.Int).Mul(pMinus, qMinus), new(big.Int).GCD(nil, nil, pMinus, qMinus))
+
+ d := new(big.Int).ModInverse(exponent, totient)
+ if d == nil {
+ continue
+ }
+
+ key := &rsa.PrivateKey{
+ PublicKey: rsa.PublicKey{N: n, E: defaultExponent},
+ D: d,
+ Primes: []*big.Int{p, q},
+ }
+
+ key.Precompute()
+
+ return key, nil
+ }
+
+ return nil, fmt.Errorf(errNoPrime, weakPrimeBits, primeAttempts)
+}
+
+const defaultExponent = 65537
+
+var one = big.NewInt(1)
+
+// sha256DigestInfo is the DER-encoded DigestInfo prefix for SHA-256, from
+// RFC 8017 section 9.2 note 1. PKCS#1 v1.5 signs this prefix followed by the
+// digest, so the value is fixed and not computed.
+var sha256DigestInfo = []byte{
+ 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
+ 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20,
+}
+
+// signPKCS1v15Raw is RFC 8017's RSASSA-PKCS1-v1_5 over a modulus crypto/rsa
+// refuses to touch. It exists for exactly one probe and it is pinned against
+// rsa.SignPKCS1v15 at a legal key size, so the arithmetic is verified against
+// the standard library rather than asserted.
+//
+// There is no constant-time concern here: the key is a throwaway khaos generated
+// to be weak on purpose, published in khaos's own JWK Set, and discarded with the
+// run.
+func signPKCS1v15Raw(key *rsa.PrivateKey, digest []byte) ([]byte, error) {
+ size := (key.N.BitLen() + 7) / 8
+ framed := len(sha256DigestInfo) + len(digest)
+
+ // EM = 0x00 || 0x01 || PS || 0x00 || T, and PS must be at least 8 bytes.
+ if size < framed+11 {
+ return nil, fmt.Errorf(errShortModulus, framed+11, size)
+ }
+
+ block := make([]byte, size)
+ block[0] = 0x00
+ block[1] = 0x01
+ for i := 2; i < size-framed-1; i++ {
+ block[i] = 0xff
+ }
+ block[size-framed-1] = 0x00
+ copy(block[size-framed:], sha256DigestInfo)
+ copy(block[size-len(digest):], digest)
+
+ m := new(big.Int).SetBytes(block)
+ c := new(big.Int).Exp(m, key.D, key.N)
+
+ out := make([]byte, size)
+ c.FillBytes(out)
+
+ return out, nil
+}
+
+// verifyPKCS1v15Raw is the mirror, and it is here rather than in the range
+// fixture because the confirmation oracle needs it: khaos has to be able to
+// prove the token it just minted really does verify under the weak key it
+// published, or "the target accepted it" says nothing about what khaos sent.
+func verifyPKCS1v15Raw(pub *rsa.PublicKey, digest, signature []byte) bool {
+ // The exponent is validated because this function is EXPORTED with a comment
+ // inviting reuse, and a caller who built the key from an attacker-supplied
+ // JWK can hand it anything.
+ //
+ // Measured: with e = 1 the modular exponentiation is the identity, so the
+ // signature IS the encoded message and khaos accepted a forgery that
+ // rsa.VerifyPKCS1v15 refused. The standard library's FIPS path rejects e < 2
+ // and even exponents; this path rejected neither. An `"e":"AQAAAAAAAAAB"`
+ // (2^64+1) also truncated to 1 through big.Int.Int64, whose behaviour is
+ // documented as undefined on overflow.
+ if pub.N == nil || pub.E < 3 || pub.E%2 == 0 {
+ return false
+ }
+
+ size := (pub.N.BitLen() + 7) / 8
+ if len(signature) != size {
+ return false
+ }
+
+ c := new(big.Int).SetBytes(signature)
+ if c.Cmp(pub.N) >= 0 {
+ return false
+ }
+
+ m := new(big.Int).Exp(c, big.NewInt(int64(pub.E)), pub.N)
+
+ block := make([]byte, size)
+ m.FillBytes(block)
+
+ framed := len(sha256DigestInfo) + len(digest)
+ if size < framed+11 {
+ return false
+ }
+
+ if block[0] != 0x00 || block[1] != 0x01 || block[size-framed-1] != 0x00 {
+ return false
+ }
+
+ for i := 2; i < size-framed-1; i++ {
+ if block[i] != 0xff {
+ return false
+ }
+ }
+
+ expected := make([]byte, 0, framed)
+ expected = append(expected, sha256DigestInfo...)
+ expected = append(expected, digest...)
+
+ for i := range expected {
+ if block[size-framed+i] != expected[i] {
+ return false
+ }
+ }
+
+ return true
+}
+
+// VerifyPKCS1v15Raw is khaos's own PKCS#1 v1.5 verification, exported for the
+// range's reference verifier. The fixture needs it to express the one bug Go's
+// standard library makes unreachable: crypto/rsa enforces a 1024-bit floor, so a
+// verifier that accepts an undersized key has to have reimplemented RSA, which
+// is what most libraries in other ecosystems have done.
+//
+// It is pinned against rsa.SignPKCS1v15 at a legal key size by this package's
+// own tests, in both directions.
+func VerifyPKCS1v15Raw(pub *rsa.PublicKey, digest, signature []byte) bool {
+ return verifyPKCS1v15Raw(pub, digest, signature)
+}
+
+// SPKIPEMBytes is the serialization a verifier is overwhelmingly holding when it
+// was handed "the issuer's public key": a PEM file read off disk. It is exported
+// because the algorithm-confusion bug is only reachable when the forged MAC and
+// the verifier's secret are the SAME bytes, so khaos and the fixture have to
+// agree on which bytes those are.
+func SPKIPEMBytes(pub *rsa.PublicKey) ([]byte, error) {
+ der, err := x509.MarshalPKIXPublicKey(pub)
+ if err != nil {
+ return nil, err
+ }
+
+ return pem.EncodeToMemory(&pem.Block{Type: pemTypePublic, Bytes: der}), nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/key_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/key_test.go
new file mode 100644
index 00000000..8a3a0439
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/key_test.go
@@ -0,0 +1,275 @@
+// ©AngelaMos | 2026
+// key_test.go
+
+package idp
+
+import (
+ "crypto"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/sha256"
+ "testing"
+)
+
+// The worked example from RFC 7638 section 3.1. The specification publishes both
+// the JWK and the thumbprint it must produce, which makes this a real
+// known-answer test rather than a self-consistency check: khaos's
+// canonicalization is implemented from the spec text and this is the spec's own
+// answer.
+const (
+ rfc7638Modulus = "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFF" +
+ "xuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4" +
+ "QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD0" +
+ "8qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awa" +
+ "pJzKnqDKgw"
+
+ rfc7638Thumbprint = "NzbLsXh8uDCcd-6MNwXF4W_7noWXFZAfHkxZsRGC9Xs"
+)
+
+func TestThumbprintMatchesRFC7638WorkedExample(t *testing.T) {
+ jwk := JWK{Kty: keyTypeRSA, N: rfc7638Modulus, E: "AQAB"}
+
+ pub, err := jwk.RSAPublicKey()
+ if err != nil {
+ t.Fatalf("rebuild the RFC's key: %v", err)
+ }
+
+ got := thumbprintOf(t, canonicalRSAJWK(pub))
+
+ if got != rfc7638Thumbprint {
+ t.Fatalf("RFC 7638 section 3.1 publishes %q, khaos computed %q", rfc7638Thumbprint, got)
+ }
+}
+
+// The canonical form is written literally rather than marshalled, so a test has
+// to pin the exact bytes: encoding/json would order members by struct
+// declaration, and a field reorder would silently change every kid khaos derives
+// without changing any key material.
+func TestCanonicalRSAJWKIsExactlyTheRequiredMembersInLexicographicOrder(t *testing.T) {
+ jwk := JWK{Kty: keyTypeRSA, N: rfc7638Modulus, E: "AQAB"}
+
+ pub, err := jwk.RSAPublicKey()
+ if err != nil {
+ t.Fatalf("rebuild the RFC's key: %v", err)
+ }
+
+ want := `{"e":"AQAB","kty":"RSA","n":"` + rfc7638Modulus + `"}`
+
+ if got := string(canonicalRSAJWK(pub)); got != want {
+ t.Fatalf("canonical form drifted\n got: %s\nwant: %s", got, want)
+ }
+}
+
+func TestThumbprintIsStableAndKeyDependent(t *testing.T) {
+ first, err := NewRSAKey()
+ if err != nil {
+ t.Fatalf("mint a key: %v", err)
+ }
+
+ second, err := NewRSAKey()
+ if err != nil {
+ t.Fatalf("mint a second key: %v", err)
+ }
+
+ again, err := first.Thumbprint()
+ if err != nil {
+ t.Fatalf("recompute: %v", err)
+ }
+
+ if again != first.ID {
+ t.Fatalf("the same key produced two kids: %q then %q", first.ID, again)
+ }
+
+ if first.ID == second.ID {
+ t.Fatal("two different keys produced the same kid, so a rotation would be invisible")
+ }
+}
+
+func TestECThumbprintUsesTheRequiredMembers(t *testing.T) {
+ key, err := NewECKey()
+ if err != nil {
+ t.Fatalf("mint an EC key: %v", err)
+ }
+
+ canonical := string(canonicalECJWK(&key.ecKey.PublicKey))
+
+ // crv, kty, x, y is lexicographic, and the coordinates are fixed-width per
+ // RFC 7518 section 6.2.1.2: a short coordinate left unpadded would change the
+ // thumbprint for the same key roughly one time in 256.
+ for _, member := range []string{`{"crv":"P-256"`, `"kty":"EC"`, `"x":"`, `"y":"`} {
+ if !contains(canonical, member) {
+ t.Fatalf("canonical EC form is missing %s: %s", member, canonical)
+ }
+ }
+
+ jwk := key.PublicJWK()
+ if decodedLen(t, jwk.X) != p256CoordinateBytes || decodedLen(t, jwk.Y) != p256CoordinateBytes {
+ t.Fatalf("coordinates are not fixed-width: x=%d y=%d, want %d each",
+ decodedLen(t, jwk.X), decodedLen(t, jwk.Y), p256CoordinateBytes)
+ }
+}
+
+// This is the differential that makes the weak-key trap trustworthy. khaos
+// carries its own PKCS#1 v1.5 implementation because crypto/rsa refuses to sign
+// or verify below 1024 bits, so the arithmetic has nowhere to be checked except
+// against the standard library at a size the standard library will accept.
+func TestRawPKCS1v15AgreesWithTheStandardLibrary(t *testing.T) {
+ key, err := rsa.GenerateKey(rand.Reader, strongBits)
+ if err != nil {
+ t.Fatalf("generate: %v", err)
+ }
+
+ for _, message := range []string{"", "khaos", "a much longer signing input than the first two"} {
+ digest := sha256.Sum256([]byte(message))
+
+ library, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, digest[:])
+ if err != nil {
+ t.Fatalf("library sign %q: %v", message, err)
+ }
+
+ mine, err := signPKCS1v15Raw(key, digest[:])
+ if err != nil {
+ t.Fatalf("khaos sign %q: %v", message, err)
+ }
+
+ if string(library) != string(mine) {
+ t.Fatalf("khaos's PKCS#1 v1.5 disagrees with crypto/rsa on %q", message)
+ }
+
+ if !verifyPKCS1v15Raw(&key.PublicKey, digest[:], library) {
+ t.Fatalf("khaos's verify rejected a signature crypto/rsa produced for %q", message)
+ }
+ }
+}
+
+// The negative control for that verifier. A verify that always said yes would
+// pass the agreement test above, because the agreement test only ever hands it
+// correct signatures.
+func TestRawPKCS1v15VerifyRejectsWrongSignatures(t *testing.T) {
+ key, err := rsa.GenerateKey(rand.Reader, strongBits)
+ if err != nil {
+ t.Fatalf("generate: %v", err)
+ }
+
+ digest := sha256.Sum256([]byte("khaos"))
+
+ signature, err := signPKCS1v15Raw(key, digest[:])
+ if err != nil {
+ t.Fatalf("sign: %v", err)
+ }
+
+ other := sha256.Sum256([]byte("khaos "))
+
+ cases := map[string][]byte{
+ "a flipped bit": flip(signature, 0),
+ "a flipped last bit": flip(signature, len(signature)-1),
+ "a truncated value": signature[:len(signature)-1],
+ "an empty value": {},
+ "all zeroes": make([]byte, len(signature)),
+ }
+
+ for name, bad := range cases {
+ if verifyPKCS1v15Raw(&key.PublicKey, digest[:], bad) {
+ t.Errorf("khaos's verify accepted %s", name)
+ }
+ }
+
+ if verifyPKCS1v15Raw(&key.PublicKey, other[:], signature) {
+ t.Error("khaos's verify accepted a valid signature over a different digest")
+ }
+}
+
+// The whole premise of the weak-key trap, asserted rather than remembered: the
+// standard library will not touch this key, and khaos can.
+func TestWeakKeyIsBelowTheLibraryFloorAndStillSigns(t *testing.T) {
+ key, err := NewWeakRSAKey()
+ if err != nil {
+ t.Fatalf("assemble a weak key: %v", err)
+ }
+
+ if !key.Weak() {
+ t.Fatalf("a %d-bit key did not report as weak", key.rsaKey.N.BitLen())
+ }
+
+ if bits := key.rsaKey.N.BitLen(); bits != weakBits {
+ t.Fatalf("modulus is %d bits, want exactly %d", bits, weakBits)
+ }
+
+ digest := sha256.Sum256([]byte("khaos"))
+
+ if _, err := rsa.SignPKCS1v15(rand.Reader, key.rsaKey, crypto.SHA256, digest[:]); err == nil {
+ t.Fatal("crypto/rsa signed below its own floor; the weak-key trap's premise is gone")
+ }
+
+ signature, err := key.Sign(RS256, []byte("khaos"))
+ if err != nil {
+ t.Fatalf("khaos could not sign with its own weak key: %v", err)
+ }
+
+ if err := rsa.VerifyPKCS1v15(&key.rsaKey.PublicKey, crypto.SHA256, digest[:], signature); err == nil {
+ t.Fatal("crypto/rsa verified below its own floor; the weak-key trap would find nothing")
+ }
+
+ if !verifyPKCS1v15Raw(&key.rsaKey.PublicKey, digest[:], signature) {
+ t.Fatal("khaos's own weak signature does not verify under khaos's own arithmetic")
+ }
+}
+
+func TestSignRefusesAnAlgorithmTheKeyCannotCarry(t *testing.T) {
+ rsaKey, err := NewRSAKey()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ ecKey, err := NewECKey()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ if _, err := rsaKey.Sign(ES256, []byte("x")); err == nil {
+ t.Error("an RSA key signed under ES256")
+ }
+
+ if _, err := ecKey.Sign(RS256, []byte("x")); err == nil {
+ t.Error("an EC key signed under RS256")
+ }
+
+ if _, err := rsaKey.Sign(HS256, []byte("x")); err == nil {
+ t.Error("a key signed under HS256, which takes no key of its own")
+ }
+}
+
+func thumbprintOf(t *testing.T, canonical []byte) string {
+ t.Helper()
+ sum := sha256.Sum256(canonical)
+ return b64(sum[:])
+}
+
+func flip(raw []byte, at int) []byte {
+ out := make([]byte, len(raw))
+ copy(out, raw)
+ out[at] ^= 0xff
+ return out
+}
+
+func contains(haystack, needle string) bool {
+ return len(haystack) >= len(needle) && indexOf(haystack, needle) >= 0
+}
+
+func indexOf(haystack, needle string) int {
+ for i := 0; i+len(needle) <= len(haystack); i++ {
+ if haystack[i:i+len(needle)] == needle {
+ return i
+ }
+ }
+ return -1
+}
+
+func decodedLen(t *testing.T, encoded string) int {
+ t.Helper()
+ raw, err := decodeSegment(encoded)
+ if err != nil {
+ t.Fatalf("decode %q: %v", encoded, err)
+ }
+ return len(raw)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/mint.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/mint.go
new file mode 100644
index 00000000..3a9bacc0
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/mint.go
@@ -0,0 +1,379 @@
+// ©AngelaMos | 2026
+// mint.go
+
+package idp
+
+import (
+ "crypto/hmac"
+ "crypto/rand"
+ "crypto/sha256"
+ "crypto/x509"
+ "encoding/json"
+ "encoding/pem"
+ "errors"
+ "fmt"
+ "time"
+)
+
+// KeyForm is how a verifier loaded the issuer's public key, which decides what
+// bytes the algorithm-confusion attack has to use as its HMAC secret. Research 07
+// is explicit that this varies by target and that khaos should be able to try
+// each, because the same bug is unreachable under the wrong serialization.
+type KeyForm string
+
+const (
+ // SPKIPEM is the default because it is overwhelmingly how an RSA public key
+ // reaches a verifier: a `.pem` file read off disk and passed through as a
+ // string.
+ SPKIPEM KeyForm = "spki-pem"
+ SPKIDER KeyForm = "spki-der"
+ PKCS1PEM KeyForm = "pkcs1-pem"
+
+ // Modulus is the raw big-endian modulus, which is what a verifier that
+ // reconstructed the key from the JWK's `n` member is holding.
+ Modulus KeyForm = "modulus"
+)
+
+const (
+ pemTypePublic = "PUBLIC KEY"
+ pemTypeRSAPublic = "RSA PUBLIC KEY"
+
+ truncatedLength = 1
+
+ errUnknownKeyForm = "%w: %q (known forms: %s, %s, %s, %s)"
+ errNoKey = "%w: khaos's identity provider holds no %s key"
+ errNothingToCorrupt = "%w: that token has no signature to corrupt"
+)
+
+var ErrUnknownKeyForm = errors.New("khaos cannot serialize a public key that way")
+
+var ErrNoSigningKey = errors.New("khaos's identity provider is missing a key a trap needs")
+
+func KeyForms() []string {
+ return []string{string(SPKIPEM), string(SPKIDER), string(PKCS1PEM), string(Modulus)}
+}
+
+// grantRequest is what a token is minted for. The provider fills it from its own
+// configuration so a caller never chooses the issuer or audience: a probe that
+// could pick its own `aud` would make the wrong-audience trap meaningless.
+type grantRequest struct {
+ subject string
+ lifetime time.Duration
+}
+
+// mintToken produces one credential under the currently armed trap. Every branch
+// corrupts EXACTLY ONE property and leaves the rest correct, which is what makes
+// a 2xx isolate a single missing check rather than report that something,
+// somewhere, is wrong.
+//
+// `typ` is the property that keeps each trap isolated: every token khaos mints
+// carries `at+jwt`, including the forged ones, so a verifier that implements
+// RFC 8725 section 3.11's explicit typing refuses each forgery for its OWN
+// defect rather than for its type. Setting `typ: JWT` on the three header traps,
+// which is what a careless forger would do, would get them refused on the type
+// instead, and a token rejected for the wrong reason reads exactly like a target
+// that caught the trap.
+//
+// An earlier version of this comment claimed that refusal had been OBSERVED. It
+// had not: the explicit-typing check did not exist anywhere in the repository at
+// the time, so the property this defends was inert against khaos's own fixture
+// and the comment described a measurement nobody took. The check is now
+// implemented in internal/verifier and `TestEveryTrapIsCaught...` exercises it.
+func (p *Provider) mintToken(trap Trap, request grantRequest) (string, error) {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+
+ now := p.clock()
+
+ header := Header{Alg: string(RS256), Typ: typAccessJWT, Kid: p.signing.ID}
+ claims := Claims{
+ Issuer: p.issuer,
+ Subject: request.subject,
+ Audience: Audience{p.audience},
+ Expires: unix(now.Add(request.lifetime)),
+ NotBefore: unix(now),
+ IssuedAt: unix(now),
+ ID: rand.Text(),
+ }
+
+ switch trap.Kind {
+ case AlgNone:
+ header.Alg = string(None)
+ return p.serialize(header, claims, nil, nil)
+
+ case AlgConfusion:
+ header.Alg = string(HS256)
+ secret, err := publicKeyBytes(p.signing, trap.KeyForm)
+ if err != nil {
+ return "", err
+ }
+ return p.serialize(header, claims, nil, func(input []byte) ([]byte, error) {
+ mac := hmac.New(sha256.New, secret)
+ mac.Write(input)
+ return mac.Sum(nil), nil
+ })
+
+ case EmbeddedJWK:
+ if p.attacker == nil {
+ return "", fmt.Errorf(errNoKey, ErrNoSigningKey, "attacker")
+ }
+ embedded := p.attacker.PublicJWK()
+ header.Kid = p.attacker.ID
+ header.JWK = &embedded
+ return p.serialize(header, claims, p.attacker, nil)
+
+ case ClaimExpired:
+ // Internally consistent on purpose: the token is expired and nothing
+ // else. A token that was also not-yet-valid would trip two checks and
+ // isolate neither.
+ claims.IssuedAt = unix(now.Add(-trap.Delay - request.lifetime))
+ claims.NotBefore = unix(now.Add(-trap.Delay - request.lifetime))
+ claims.Expires = unix(now.Add(-trap.Delay))
+ return p.serialize(header, claims, nil, nil)
+
+ case ClaimNotYetValid:
+ claims.NotBefore = unix(now.Add(trap.Delay))
+ claims.Expires = unix(now.Add(trap.Delay + request.lifetime))
+ return p.serialize(header, claims, nil, nil)
+
+ case ClaimWrongAudience:
+ claims.Audience = Audience{p.foreignAudience}
+ return p.serialize(header, claims, nil, nil)
+
+ case ClaimForeignIssuer:
+ // Signed by a key khaos DOES publish, in the same JWK Set. That is the
+ // multi-tenant shape, and the teaching point is that the key set is not
+ // the trust boundary: the signature verifies and only the `iss` check
+ // can refuse it.
+ if p.foreign == nil {
+ return "", fmt.Errorf(errNoKey, ErrNoSigningKey, "second issuer")
+ }
+ claims.Issuer = p.foreignIssuer
+ header.Kid = p.foreign.ID
+ return p.serialize(header, claims, p.foreign, nil)
+
+ case SignatureTruncated:
+ return p.serializeThen(header, claims, nil, func(signature []byte) []byte {
+ return signature[:truncatedLength]
+ })
+
+ case SignatureZero:
+ if p.ec == nil {
+ return "", fmt.Errorf(errNoKey, ErrNoSigningKey, "EC")
+ }
+ header.Alg = string(ES256)
+ header.Kid = p.ec.ID
+ return p.serializeThen(header, claims, p.ec, func([]byte) []byte {
+ return make([]byte, 2*p256CoordinateBytes)
+ })
+
+ case SignatureWeakKey:
+ if p.weak == nil {
+ return "", fmt.Errorf(errNoKey, ErrNoSigningKey, "weak RSA")
+ }
+ header.Kid = p.weak.ID
+ return p.serialize(header, claims, p.weak, nil)
+
+ default:
+ return p.serialize(header, claims, nil, nil)
+ }
+}
+
+// serialize marshals, signs and assembles. A nil key means the provider's
+// current signing key; a nil signer means the key's own algorithm.
+func (p *Provider) serialize(
+ header Header,
+ claims Claims,
+ key *Key,
+ signer func([]byte) ([]byte, error),
+) (string, error) {
+ return p.serializeWith(header, claims, key, signer, nil)
+}
+
+func (p *Provider) serializeThen(
+ header Header,
+ claims Claims,
+ key *Key,
+ corrupt func([]byte) []byte,
+) (string, error) {
+ return p.serializeWith(header, claims, key, nil, corrupt)
+}
+
+func (p *Provider) serializeWith(
+ header Header,
+ claims Claims,
+ key *Key,
+ signer func([]byte) ([]byte, error),
+ corrupt func([]byte) []byte,
+) (string, error) {
+ headerJSON, err := json.Marshal(header)
+ if err != nil {
+ return "", err
+ }
+
+ claimsJSON, err := json.Marshal(claims)
+ if err != nil {
+ return "", err
+ }
+
+ input := signingInput(headerJSON, claimsJSON)
+
+ signature, err := p.produceSignature(Algorithm(header.Alg), input, key, signer)
+ if err != nil {
+ return "", err
+ }
+
+ if corrupt != nil {
+ signature = corrupt(signature)
+ }
+
+ return assemble(headerJSON, claimsJSON, signature), nil
+}
+
+func (p *Provider) produceSignature(
+ alg Algorithm,
+ input []byte,
+ key *Key,
+ signer func([]byte) ([]byte, error),
+) ([]byte, error) {
+ if signer != nil {
+ return signer(input)
+ }
+
+ if alg == None {
+ return nil, nil
+ }
+
+ if key == nil {
+ key = p.signing
+ }
+
+ return key.Sign(alg, input)
+}
+
+// publicKeyBytes is the bytes a verifier is holding for this key, under one of
+// the four serializations research 07 says to try. It is the HMAC secret for the
+// algorithm-confusion probe, and it is the whole reason that probe needs a
+// parameter: the bug is unreachable if khaos guesses the wrong form.
+func publicKeyBytes(key *Key, form KeyForm) ([]byte, error) {
+ der, err := key.PublicDER()
+ if err != nil {
+ return nil, err
+ }
+
+ switch form {
+ case SPKIDER:
+ return der, nil
+
+ case SPKIPEM:
+ return pem.EncodeToMemory(&pem.Block{Type: pemTypePublic, Bytes: der}), nil
+
+ case PKCS1PEM:
+ if key.rsaKey == nil {
+ return nil, fmt.Errorf(errNoKey, ErrNoSigningKey, "RSA")
+ }
+ pkcs1 := x509.MarshalPKCS1PublicKey(&key.rsaKey.PublicKey)
+ return pem.EncodeToMemory(&pem.Block{Type: pemTypeRSAPublic, Bytes: pkcs1}), nil
+
+ case Modulus:
+ if key.rsaKey == nil {
+ return nil, fmt.Errorf(errNoKey, ErrNoSigningKey, "RSA")
+ }
+ return key.rsaKey.N.Bytes(), nil
+
+ default:
+ return nil, fmt.Errorf(errUnknownKeyForm, ErrUnknownKeyForm,
+ form, SPKIPEM, SPKIDER, PKCS1PEM, Modulus)
+ }
+}
+
+// corruptSignature replaces a token's signature with bytes no key produced. It
+// is not a trap: it is what the JWKS-unreachable experiment presents, because
+// that trap asks whether a verifier that cannot fetch keys checks the signature
+// at all, and the token has to be one a reachable verifier would refuse.
+func CorruptSignature(raw string) (string, error) {
+ token, err := Parse(raw)
+ if err != nil {
+ return "", err
+ }
+
+ if len(token.Signature) == 0 {
+ return "", fmt.Errorf(errNothingToCorrupt, ErrNotAJWS)
+ }
+
+ // Flipping a bit of the real signature, rather than substituting random
+ // bytes, keeps the length and the algorithm exactly right. The only thing
+ // wrong with the token is that the signature does not verify, which is what
+ // makes a 2xx mean the verifier never checked it.
+ forged := make([]byte, len(token.Signature))
+ copy(forged, token.Signature)
+ forged[0] ^= 0xff
+
+ return assemble(token.HeaderJSON, token.ClaimsJSON, forged), nil
+}
+
+// mintSignedBy produces a correct token under a named key rather than under the
+// provider's current signing key.
+func (p *Provider) mintSignedBy(key *Key) (string, error) {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+
+ now := p.clock()
+
+ header := Header{Alg: string(RS256), Typ: typAccessJWT, Kid: key.ID}
+ claims := Claims{
+ Issuer: p.issuer,
+ Subject: DefaultSubject,
+ Audience: Audience{p.audience},
+ Expires: unix(now.Add(p.lifetime)),
+ NotBefore: unix(now),
+ IssuedAt: unix(now),
+ ID: rand.Text(),
+ }
+
+ return p.serialize(header, claims, key, nil)
+}
+
+// reheader and reclaim mint a correctly signed token with one header or claim
+// rewritten. They are the seam the range verifier's tests need to reach the
+// checks khaos's own published material can never trip, and they sign properly
+// so the token is refused by the check under test rather than by the signature.
+func (p *Provider) reheader(edit func(*Header)) string {
+ return p.remint(edit, nil)
+}
+
+func (p *Provider) reclaim(edit func(*Claims)) string {
+ return p.remint(nil, edit)
+}
+
+func (p *Provider) remint(header func(*Header), claims func(*Claims)) string {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+
+ now := p.clock()
+
+ head := Header{Alg: string(RS256), Typ: typAccessJWT, Kid: p.signing.ID}
+ body := Claims{
+ Issuer: p.issuer,
+ Subject: DefaultSubject,
+ Audience: Audience{p.audience},
+ Expires: unix(now.Add(p.lifetime)),
+ NotBefore: unix(now),
+ IssuedAt: unix(now),
+ ID: rand.Text(),
+ }
+
+ if header != nil {
+ header(&head)
+ }
+ if claims != nil {
+ claims(&body)
+ }
+
+ raw, err := p.serialize(head, body, nil, nil)
+ if err != nil {
+ return ""
+ }
+
+ return raw
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/observe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/observe.go
new file mode 100644
index 00000000..218b95eb
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/observe.go
@@ -0,0 +1,532 @@
+// ©AngelaMos | 2026
+// observe.go
+
+package idp
+
+import (
+ "context"
+ "crypto"
+ "crypto/ecdsa"
+ "crypto/hmac"
+ "crypto/rsa"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "time"
+)
+
+const (
+ observeTimeout = 5 * time.Second
+ observeBudget = 4
+
+ // Slack on a measured hold, for the same reason the fault proxy carries it:
+ // the threshold is a lower bound with no tolerance otherwise, and on a host
+ // that also serves real traffic CPU contention routinely shaves a few
+ // percent off a sleep.
+ slackNumerator = 9
+ slackDenominator = 10
+
+ observedAlgNone = "khaos's provider mints a token whose header declares alg %q with an empty signature, where its honest token carries %q"
+ observedAlgConfusion = "khaos's provider mints an HS256 token whose MAC verifies under its own published RSA public key as %s bytes"
+ observedEmbeddedJWK = "khaos's provider mints a token signed by kid %q, which it carries in its own `jwk` header and does NOT publish"
+ observedExpired = "khaos's provider mints a correctly signed token that expired %s ago"
+ observedNotYetValid = "khaos's provider mints a correctly signed token not valid for another %s"
+ observedWrongAud = "khaos's provider mints a correctly signed token addressed to %q rather than %q"
+ observedForeignIss = "khaos's provider mints a token from issuer %q, signed by a key it publishes, where its honest token says %q"
+ observedTruncated = "khaos's provider mints a token whose signature is %d byte where its honest token signs %d"
+ observedZero = "khaos's provider mints an %s token whose %d-byte signature is entirely zero"
+ observedWeakKey = "khaos's provider mints a token under a %d-bit modulus that crypto/rsa refuses to verify and khaos's own PKCS#1 v1.5 path accepts"
+ observedJWKSGone = "khaos's provider answers %d on its health route and %d on its JWK Set"
+ observedRollover = "kid %q is gone from khaos's published JWK Set and %q is signing in its place"
+ observedRotation = "khaos publishes both %q and %q, and mints under %q"
+ observedIntroError = "khaos's provider answers %d on its health route and %d on introspection"
+ observedIntroBad = "khaos's provider answers introspection with %d and a body that is not JSON"
+ observedIntroSlow = "introspection takes %s where the health route answers in %s"
+ observedRevoked = "khaos's provider reports the credential it revoked inactive and a second credential still active"
+
+ errNothingArmed = "%w: nothing is armed on khaos's identity provider, so there is nothing to confirm"
+ errWrongExpected = "%w: the provider holds %q but %q was expected"
+ errNoListener = "%w: this trap is confirmed over the provider's own HTTP surface, and it is not listening"
+ errProviderDown = "%w: khaos cannot reach its own identity provider's health route (%v), so nothing it sees there can be attributed to the armed trap"
+ errHonestToken = "%w: khaos's own provider cannot mint an honest token (%v), so nothing it mints under a trap can be compared with one"
+)
+
+// Observe confirms the armed trap through the provider's OWN public surface and
+// against a control, never by reading the flag that was set.
+//
+// The discipline is the fault proxy's, and it was expensive to learn: a
+// confirmation that can only succeed is not an oracle. Gutting the proxy's
+// equivalent to `return true` left every package and the Docker gate green, and
+// the fix was a way to arm a fault that does not happen. Every branch below
+// therefore has a control that must disagree with the fault, and `Inert` makes
+// the fault not happen so a test can prove the branch says NO.
+func (p *Provider) Observe(ctx context.Context, expected Kind) (string, bool, error) {
+ select {
+ case <-p.closed:
+ return "", false, ErrClosed
+ default:
+ }
+
+ trap := p.Trap()
+
+ if trap.Kind == PassThrough {
+ return "", false, fmt.Errorf(errNothingArmed, ErrTrapNotObserved)
+ }
+
+ if expected != "" && trap.Kind != expected {
+ return "", false, fmt.Errorf(errWrongExpected, ErrTrapNotObserved, trap.Kind, expected)
+ }
+
+ if trap.Forgery() {
+ return p.observeForgery(trap)
+ }
+
+ return p.observeEndpoint(ctx, trap)
+}
+
+// observeForgery mints on BOTH paths and requires them to differ. The honest
+// token is the control: if khaos's own provider cannot produce a correct token,
+// nothing it produces under a trap can be attributed to the trap.
+func (p *Provider) observeForgery(trap Trap) (string, bool, error) {
+ honest, err := p.mintHonestForControl()
+ if err != nil {
+ return "", false, fmt.Errorf(errHonestToken, ErrTrapNotObserved, err)
+ }
+
+ forged, err := p.mintTrappedForControl()
+ if err != nil {
+ return "", false, err
+ }
+
+ published := p.PublishedSet()
+ now := p.clock()
+
+ switch trap.Kind {
+ case AlgNone:
+ if forged.Header.Alg == string(None) && len(forged.Signature) == 0 && honest.Header.Alg != string(None) {
+ return sprintf(observedAlgNone, forged.Header.Alg, honest.Header.Alg), true, nil
+ }
+
+ case AlgConfusion:
+ // Both halves. The header has to say HS256, and the MAC has to actually
+ // verify under the bytes a verifier would be holding, or khaos is
+ // sending a token no target could accept and an acceptance would mean
+ // something else entirely.
+ if forged.Header.Alg != string(HS256) {
+ return "", false, nil
+ }
+ secret, err := p.confusionSecret(trap.KeyForm)
+ if err != nil {
+ return "", false, err
+ }
+ mac := hmac.New(sha256.New, secret)
+ mac.Write(forged.SigningInput)
+ if hmac.Equal(mac.Sum(nil), forged.Signature) {
+ return sprintf(observedAlgConfusion, trap.KeyForm), true, nil
+ }
+
+ case EmbeddedJWK:
+ if forged.Header.JWK == nil || published.Has(forged.Header.JWK.Kid) {
+ return "", false, nil
+ }
+ if verifiesUnder(*forged.Header.JWK, forged) {
+ return sprintf(observedEmbeddedJWK, forged.Header.JWK.Kid), true, nil
+ }
+
+ case ClaimExpired:
+ if forged.Claims.Expires == nil || !verifiesUnderSet(published, forged) {
+ return "", false, nil
+ }
+ expiry := time.Unix(*forged.Claims.Expires, 0)
+ if expiry.Before(now) {
+ return sprintf(observedExpired, now.Sub(expiry).Round(time.Second)), true, nil
+ }
+
+ case ClaimNotYetValid:
+ if forged.Claims.NotBefore == nil || !verifiesUnderSet(published, forged) {
+ return "", false, nil
+ }
+ start := time.Unix(*forged.Claims.NotBefore, 0)
+ if start.After(now) {
+ return sprintf(observedNotYetValid, start.Sub(now).Round(time.Second)), true, nil
+ }
+
+ case ClaimWrongAudience:
+ if !verifiesUnderSet(published, forged) {
+ return "", false, nil
+ }
+ if !forged.Claims.Audience.Contains(p.audience) && len(forged.Claims.Audience) > 0 {
+ return sprintf(observedWrongAud, forged.Claims.Audience, honest.Claims.Audience), true, nil
+ }
+
+ case ClaimForeignIssuer:
+ // The key IS published, deliberately. That is the multi-tenant shape and
+ // the teaching point: the key set is not the trust boundary, so the
+ // signature verifies and only the `iss` check can refuse it.
+ if !verifiesUnderSet(published, forged) {
+ return "", false, nil
+ }
+ if forged.Claims.Issuer != honest.Claims.Issuer && forged.Claims.Issuer != "" {
+ return sprintf(observedForeignIss, forged.Claims.Issuer, honest.Claims.Issuer), true, nil
+ }
+
+ case SignatureTruncated:
+ if len(forged.Signature) == truncatedLength && len(honest.Signature) > truncatedLength {
+ return sprintf(observedTruncated, len(forged.Signature), len(honest.Signature)), true, nil
+ }
+
+ case SignatureZero:
+ if forged.Header.Alg == string(ES256) && len(forged.Signature) > 0 && allZero(forged.Signature) {
+ return sprintf(observedZero, forged.Header.Alg, len(forged.Signature)), true, nil
+ }
+
+ case SignatureWeakKey:
+ // Three assertions, and none of them is redundant. The modulus has to be
+ // below the library floor, khaos's own arithmetic has to ACCEPT the
+ // signature, or khaos is sending an unverifiable token and an acceptance
+ // means something else, and the standard library has to REFUSE it.
+ //
+ // That last one looks implied by the first and is not: the floor is a
+ // GODEBUG. Under `rsa1024min=0` crypto/rsa verifies below it, so khaos's
+ // claim that a correct verifier cannot accept this token would be false
+ // in that environment, and the trap declines to confirm rather than
+ // asserting it anyway.
+ key, err := published.Lookup(forged.Header.Kid)
+ if err != nil {
+ return "", false, nil
+ }
+ pub, err := key.RSAPublicKey()
+ if err != nil {
+ return "", false, nil
+ }
+ if pub.N.BitLen() >= libraryFloorBits {
+ return "", false, nil
+ }
+ digest := sha256.Sum256(forged.SigningInput)
+ libraryRefuses := rsa.VerifyPKCS1v15(pub, crypto.SHA256, digest[:], forged.Signature) != nil
+ khaosAccepts := verifyPKCS1v15Raw(pub, digest[:], forged.Signature)
+ if libraryRefuses && khaosAccepts {
+ return sprintf(observedWeakKey, pub.N.BitLen()), true, nil
+ }
+ }
+
+ return "", false, nil
+}
+
+// mintHonestForControl and mintTrappedForControl mint WITHOUT counting, because
+// the mint counter is a measurement and khaos's own confirmation must not move
+// the thing it is confirming.
+//
+// Both are needed. An earlier version had only the honest one and then called
+// the counting TrappedToken on the next line, so the comment on this function
+// described a property the function beside it broke.
+func (p *Provider) mintHonestForControl() (Token, error) {
+ return p.parseMint(Trap{})
+}
+
+func (p *Provider) mintTrappedForControl() (Token, error) {
+ return p.parseMint(p.effective())
+}
+
+func (p *Provider) parseMint(trap Trap) (Token, error) {
+ raw, err := p.mintToken(trap, grantRequest{subject: DefaultSubject, lifetime: p.lifetime})
+ if err != nil {
+ return Token{}, err
+ }
+ return Parse(raw)
+}
+
+// confusionSecret reads the TRAP's key form, and so does mintToken. They used to
+// read two different fields that Arm copied one into the other, which agreed
+// only by that copy and which Disarm never cleared.
+func (p *Provider) confusionSecret(form KeyForm) ([]byte, error) {
+ p.mu.RLock()
+ signing := p.signing
+ p.mu.RUnlock()
+
+ return publicKeyBytes(signing, form)
+}
+
+// observeEndpoint confirms the traps that change what the provider SERVES rather
+// than what it mints. Every one of them is measured over the provider's own HTTP
+// surface, with the health route as the control: it is never faulted, so an
+// unhealthy control means khaos's provider is broken rather than lying and
+// nothing seen elsewhere can be attributed to the trap.
+func (p *Provider) observeEndpoint(ctx context.Context, trap Trap) (string, bool, error) {
+ base, err := p.Addr()
+ if err != nil {
+ return "", false, fmt.Errorf(errNoListener, ErrTrapNotObserved)
+ }
+
+ root := "http://" + base
+ client := &http.Client{Timeout: observeTimeout + trap.Delay*observeBudget}
+
+ healthStatus, healthTook, err := probeRoute(ctx, client, root+RouteHealth)
+ if err != nil || healthStatus != http.StatusOK {
+ return "", false, fmt.Errorf(errProviderDown, ErrTrapNotObserved, orStatus(err, healthStatus))
+ }
+
+ switch trap.Kind {
+ case JWKSUnreachable:
+ p.ownFetches.Add(1)
+ status, _, err := probeRoute(ctx, client, root+RouteJWKS)
+ if err == nil && status != http.StatusOK {
+ return sprintf(observedJWKSGone, healthStatus, status), true, nil
+ }
+
+ case KidRollover:
+ retired, ok := p.RetiredKid()
+ if !ok {
+ return "", false, nil
+ }
+ set, err := p.fetchSet(ctx, client, root+RouteJWKS)
+ if err != nil {
+ return "", false, err
+ }
+ // `retired != p.SigningKid()` used to be a third conjunct here and it was
+ // INERT: it is implied by the other two, because `!Has(retired)` and
+ // `Has(SigningKid)` cannot both hold when the two are the same kid.
+ //
+ // Worse, it ALIBIED the live one. Measured: with it present, replacing
+ // `!set.Has(retired)` with `true` left both the positive and the inert
+ // case green and the whole suite passing, because the implied conjunct
+ // was carrying the negative control on its own. Withdrawing the retired
+ // key is the entire claim this branch makes, so that assertion has to be
+ // the one the control exercises.
+ if !set.Has(retired) && set.Has(p.SigningKid()) {
+ return sprintf(observedRollover, retired, p.SigningKid()), true, nil
+ }
+
+ case KeyRotation:
+ retired, ok := p.RetiredKid()
+ if !ok {
+ return "", false, nil
+ }
+ set, err := p.fetchSet(ctx, client, root+RouteJWKS)
+ if err != nil {
+ return "", false, err
+ }
+ if retired == p.SigningKid() || !set.Has(retired) || !set.Has(p.SigningKid()) {
+ return "", false, nil
+ }
+ // The three assertions above are the live ones, and the first carries the
+ // negative control: an inert arm never rotates, so retired and signing
+ // are the same kid.
+ //
+ // A fourth, `minted.Header.Kid == p.SigningKid()`, used to follow and was
+ // unconditionally true: mintToken's default branch always stamps the
+ // current signing kid and KeyRotation is not a forgery trap. It is gone
+ // rather than kept for symmetry, because an assertion that cannot fail
+ // reads in a review as evidence and is not.
+ minted, err := p.mintTrappedForControl()
+ if err != nil {
+ return "", false, err
+ }
+
+ return sprintf(observedRotation, retired, p.SigningKid(), minted.Header.Kid), true, nil
+
+ case IntrospectionError:
+ status, _, err := probeRoute(ctx, client, p.introspectURL(root, p.Opaque()))
+ if err == nil && status != http.StatusOK {
+ return sprintf(observedIntroError, healthStatus, status), true, nil
+ }
+
+ case IntrospectionMalformed:
+ status, body, err := readRoute(ctx, client, p.introspectURL(root, p.Opaque()))
+ if err != nil {
+ return "", false, err
+ }
+ if status == http.StatusOK && !json.Valid(body) {
+ return sprintf(observedIntroBad, status), true, nil
+ }
+
+ case IntrospectionSlow:
+ status, took, err := probeRoute(ctx, client, p.introspectURL(root, p.Opaque()))
+ if err != nil || status != http.StatusOK {
+ return "", false, nil
+ }
+ // A differential, not a threshold. The health route is the control and it
+ // is never faulted, so the comparison separates khaos's own hold from a
+ // generally slow machine.
+ if took-healthTook >= slack(trap.Delay) {
+ return sprintf(observedIntroSlow, took.Round(time.Millisecond), healthTook.Round(time.Millisecond)), true, nil
+ }
+
+ case Revoke:
+ // A control credential, issued now and never revoked. Without it,
+ // "inactive" is also what a provider that lost every grant would say, and
+ // the finding would be khaos's own bug reported as the target's.
+ control := p.Opaque()
+
+ revoked, err := fetchIntrospection(ctx, client, p.introspectURL(root, trap.Credential))
+ if err != nil {
+ return "", false, err
+ }
+
+ alive, err := fetchIntrospection(ctx, client, p.introspectURL(root, control))
+ if err != nil {
+ return "", false, err
+ }
+
+ if !revoked.Active && alive.Active {
+ return observedRevoked, true, nil
+ }
+ }
+
+ return "", false, nil
+}
+
+// introspectURL records the call as khaos's own, so the exported count stays a
+// measurement of the TARGET's behaviour. Every caller of this is a confirmation
+// probe.
+// introspectURL is khaos's own read-back path. It keeps the query form on
+// purpose: these probes exist to observe the endpoint's BEHAVIOUR, and reading
+// the token the same way a conforming client does is the fixture's job to prove,
+// not the oracle's. The contract test covers the specified wire format directly.
+func (p *Provider) introspectURL(root, credential string) string {
+ p.ownIntrospects.Add(1)
+ return root + RouteIntrospect + "?" + ParamToken + "=" + credential
+}
+
+func probeRoute(ctx context.Context, client *http.Client, url string) (int, time.Duration, error) {
+ started := time.Now()
+
+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return 0, 0, err
+ }
+
+ response, err := client.Do(request)
+ if err != nil {
+ return 0, time.Since(started), err
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ _, _ = io.Copy(io.Discard, response.Body)
+
+ return response.StatusCode, time.Since(started), nil
+}
+
+func readRoute(ctx context.Context, client *http.Client, url string) (int, []byte, error) {
+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return 0, nil, err
+ }
+
+ response, err := client.Do(request)
+ if err != nil {
+ return 0, nil, err
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ return 0, nil, err
+ }
+
+ return response.StatusCode, body, nil
+}
+
+// fetchSet is khaos's own read-back of the key set, so it records the fetch as
+// khaos's and the exported count stays a measurement of the TARGET's behaviour.
+func (p *Provider) fetchSet(ctx context.Context, client *http.Client, url string) (JWKSet, error) {
+ p.ownFetches.Add(1)
+ return fetchSet(ctx, client, url)
+}
+
+func fetchSet(ctx context.Context, client *http.Client, url string) (JWKSet, error) {
+ status, body, err := readRoute(ctx, client, url)
+ if err != nil {
+ return JWKSet{}, err
+ }
+
+ if status != http.StatusOK {
+ return JWKSet{}, fmt.Errorf(errProviderDown, ErrTrapNotObserved, status)
+ }
+
+ return ParseJWKSet(body)
+}
+
+func fetchIntrospection(ctx context.Context, client *http.Client, url string) (Introspection, error) {
+ status, body, err := readRoute(ctx, client, url)
+ if err != nil {
+ return Introspection{}, err
+ }
+
+ if status != http.StatusOK {
+ return Introspection{}, fmt.Errorf(errProviderDown, ErrTrapNotObserved, status)
+ }
+
+ var out Introspection
+ if err := json.Unmarshal(body, &out); err != nil {
+ return Introspection{}, err
+ }
+
+ return out, nil
+}
+
+// verifiesUnderSet is the control half of every claim trap. A claim trap's whole
+// claim is that the SIGNATURE is correct and exactly one claim is not, so a
+// token whose signature does not verify is not the token the trap describes.
+func verifiesUnderSet(set JWKSet, token Token) bool {
+ key, err := set.Lookup(token.Header.Kid)
+ if err != nil {
+ return false
+ }
+ return verifiesUnder(key, token)
+}
+
+func verifiesUnder(key JWK, token Token) bool {
+ digest := sha256.Sum256(token.SigningInput)
+
+ switch Algorithm(token.Header.Alg) {
+ case RS256:
+ pub, err := key.RSAPublicKey()
+ if err != nil {
+ return false
+ }
+ if pub.N.BitLen() < libraryFloorBits {
+ return verifyPKCS1v15Raw(pub, digest[:], token.Signature)
+ }
+ return rsa.VerifyPKCS1v15(pub, crypto.SHA256, digest[:], token.Signature) == nil
+
+ case ES256:
+ pub, err := key.ECPublicKey()
+ if err != nil || len(token.Signature) != 2*p256CoordinateBytes {
+ return false
+ }
+ return ecdsa.Verify(pub,
+ digest[:],
+ bigFromBytes(token.Signature[:p256CoordinateBytes]),
+ bigFromBytes(token.Signature[p256CoordinateBytes:]))
+
+ default:
+ return false
+ }
+}
+
+func allZero(raw []byte) bool {
+ for _, b := range raw {
+ if b != 0 {
+ return false
+ }
+ }
+ return true
+}
+
+func slack(delay time.Duration) time.Duration {
+ return delay * slackNumerator / slackDenominator
+}
+
+func orStatus(err error, status int) any {
+ if err != nil {
+ return err
+ }
+ return status
+}
+
+func sprintf(format string, args ...any) string {
+ return fmt.Sprintf(format, args...)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/observe_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/observe_test.go
new file mode 100644
index 00000000..23768470
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/observe_test.go
@@ -0,0 +1,615 @@
+// ©AngelaMos | 2026
+// observe_test.go
+
+package idp
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/url"
+ "testing"
+ "time"
+)
+
+const (
+ testDelay = 300 * time.Millisecond
+ testIssuer = "http://khaos.test:1234"
+)
+
+// armable is one trap plus whatever has to be true before it can be armed. The
+// two traps that need a subject supply it here, because a trap armed without one
+// is refused by Validate and would never reach the oracle.
+type armable struct {
+ kind Kind
+ prepare func(*Provider) Trap
+}
+
+func everyTrap() []armable {
+ return []armable{
+ {kind: AlgNone},
+ {kind: AlgConfusion, prepare: func(*Provider) Trap {
+ return Trap{Kind: AlgConfusion, KeyForm: SPKIPEM}
+ }},
+ {kind: EmbeddedJWK},
+ {kind: ClaimExpired, prepare: func(*Provider) Trap {
+ return Trap{Kind: ClaimExpired, Delay: time.Hour}
+ }},
+ {kind: ClaimNotYetValid, prepare: func(*Provider) Trap {
+ return Trap{Kind: ClaimNotYetValid, Delay: time.Hour}
+ }},
+ {kind: ClaimWrongAudience},
+ {kind: ClaimForeignIssuer},
+ {kind: SignatureTruncated},
+ {kind: SignatureZero},
+ {kind: SignatureWeakKey},
+ {kind: JWKSUnreachable},
+ {kind: KidRollover},
+ {kind: KeyRotation},
+ {kind: IntrospectionError},
+ {kind: IntrospectionMalformed},
+ {kind: IntrospectionSlow, prepare: func(*Provider) Trap {
+ return Trap{Kind: IntrospectionSlow, Delay: testDelay}
+ }},
+ {kind: Revoke, prepare: func(p *Provider) Trap {
+ return Trap{Kind: Revoke, Credential: p.Revocable()}
+ }},
+ }
+}
+
+func (a armable) trap(p *Provider) Trap {
+ if a.prepare != nil {
+ return a.prepare(p)
+ }
+ return Trap{Kind: a.kind}
+}
+
+// The census: every trap khaos advertises has to be armable and confirmable.
+// A trap that cannot be confirmed is a fault khaos would apply and then measure
+// without knowing it landed, which is the one thing the engine refuses to do.
+func TestEveryTrapIsConfirmedByItsOwnObservable(t *testing.T) {
+ cases := everyTrap()
+
+ if len(cases) != len(Kinds()) {
+ t.Fatalf("this test covers %d traps and khaos advertises %d; a new trap needs a case here",
+ len(cases), len(Kinds()))
+ }
+
+ for _, each := range cases {
+ t.Run(string(each.kind), func(t *testing.T) {
+ p := listening(t)
+
+ trap := each.trap(p)
+
+ if err := p.Arm(trap); err != nil {
+ t.Fatalf("arm %s: %v", each.kind, err)
+ }
+
+ observable, confirmed, err := p.Observe(context.Background(), each.kind)
+ if err != nil {
+ t.Fatalf("observe %s: %v", each.kind, err)
+ }
+
+ if !confirmed {
+ t.Fatalf("%s was armed and its own observable did not appear", each.kind)
+ }
+
+ if observable == "" {
+ t.Fatalf("%s confirmed with no observable; the run record would carry no evidence", each.kind)
+ }
+ })
+ }
+}
+
+// The negative control, and the reason Inert exists at all. Without a way to arm
+// a trap that does NOT happen, every assertion above is satisfied by an oracle
+// gutted to `return true`. Measured on the fault proxy: exactly that gut left
+// every package and the Docker gate green.
+func TestEveryTrapRefusesConfirmationWhenItDidNotHappen(t *testing.T) {
+ for _, each := range everyTrap() {
+ t.Run(string(each.kind), func(t *testing.T) {
+ p := listening(t)
+
+ // Inert BEFORE arming, so the state-changing traps never perform
+ // their change and the oracle has a real nothing to look at.
+ p.Inert(true)
+
+ trap := each.trap(p)
+
+ if err := p.Arm(trap); err != nil {
+ t.Fatalf("arm %s inert: %v", each.kind, err)
+ }
+
+ _, confirmed, _ := p.Observe(context.Background(), each.kind)
+
+ if confirmed {
+ t.Fatalf("%s confirmed itself while inert, so its oracle cannot tell a landed trap from one that did nothing", each.kind)
+ }
+ })
+ }
+}
+
+func TestObserveRefusesWhenNothingIsArmed(t *testing.T) {
+ p := listening(t)
+
+ if _, confirmed, err := p.Observe(context.Background(), AlgNone); confirmed || err == nil {
+ t.Fatalf("an unarmed provider confirmed a trap: confirmed=%v err=%v", confirmed, err)
+ }
+}
+
+func TestObserveRefusesADifferentTrapThanTheOneArmed(t *testing.T) {
+ p := listening(t)
+
+ if err := p.Arm(Trap{Kind: AlgNone}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ if _, confirmed, err := p.Observe(context.Background(), SignatureTruncated); confirmed || err == nil {
+ t.Fatalf("the provider confirmed %s while holding %s", SignatureTruncated, AlgNone)
+ }
+}
+
+// Disarm has to put the key material back, or the recovery phase measures a
+// provider the fault permanently changed and no rotation experiment can ever
+// report that the target came back.
+func TestDisarmRestoresTheSigningKeyARotationReplaced(t *testing.T) {
+ p := listening(t)
+
+ before := p.SigningKid()
+
+ if err := p.Arm(Trap{Kind: KidRollover}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ during := p.SigningKid()
+ if during == before {
+ t.Fatal("arming a rollover did not change the signing key")
+ }
+
+ if p.PublishedSet().Has(before) {
+ t.Fatal("the retired key is still published, so nothing was withdrawn")
+ }
+
+ p.Disarm()
+
+ if p.SigningKid() != before {
+ t.Fatalf("after disarm the signing key is %q, want the original %q", p.SigningKid(), before)
+ }
+
+ if !p.PublishedSet().Has(before) {
+ t.Fatal("after disarm the original key is still not published")
+ }
+}
+
+func TestDisarmRestoresARevokedGrant(t *testing.T) {
+ p := listening(t)
+
+ credential := p.Revocable()
+
+ if err := p.Arm(Trap{Kind: Revoke, Credential: credential}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ if active := p.active(t, credential); active {
+ t.Fatal("the credential is still active after being revoked")
+ }
+
+ p.Disarm()
+
+ if active := p.active(t, credential); !active {
+ t.Fatal("disarm did not restore the revoked grant, so recovery cannot be measured")
+ }
+}
+
+// A rotation with overlap must keep the old key published. Without this the
+// availability-preserving rotation is indistinguishable from the withdrawing
+// one, and the two experiments built on them would measure the same thing.
+func TestRotationWithOverlapKeepsBothKeysPublished(t *testing.T) {
+ p := listening(t)
+
+ before := p.SigningKid()
+
+ if err := p.Arm(Trap{Kind: KeyRotation}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ set := p.PublishedSet()
+
+ if !set.Has(before) {
+ t.Fatal("an overlapping rotation withdrew the old key")
+ }
+
+ if !set.Has(p.SigningKid()) {
+ t.Fatal("an overlapping rotation did not publish the new key")
+ }
+
+ if p.SigningKid() == before {
+ t.Fatal("an overlapping rotation did not change the signing key")
+ }
+}
+
+// Revoking something the provider never issued has to fail. A revocation that
+// quietly succeeded would make every later acceptance correct behaviour while
+// khaos reported it as a finding.
+func TestArmRefusesToRevokeACredentialItNeverIssued(t *testing.T) {
+ p := listening(t)
+
+ if err := p.Arm(Trap{Kind: Revoke, Credential: "khaos-opaque-never-issued"}); err == nil {
+ t.Fatal("the provider revoked a credential it never issued")
+ }
+}
+
+// The provider has to be indistinguishable from a correct one when nothing is
+// armed, because every experiment's baseline is measured against it.
+func TestAnUnarmedProviderMintsOnlyCorrectTokens(t *testing.T) {
+ p := listening(t)
+ p.SetIssuer(testIssuer)
+
+ for _, mint := range []func() (string, error){p.HonestToken, p.TrappedToken} {
+ raw, err := mint()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ token, err := Parse(raw)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ if token.Header.Alg != string(RS256) {
+ t.Errorf("alg is %q, want %q", token.Header.Alg, RS256)
+ }
+
+ if token.Header.JWK != nil {
+ t.Error("an unarmed provider embedded a key in the header")
+ }
+
+ if token.Claims.Issuer != testIssuer {
+ t.Errorf("iss is %q, want %q", token.Claims.Issuer, testIssuer)
+ }
+
+ if !token.Claims.Audience.Contains(DefaultAudience) {
+ t.Errorf("aud is %q, want %q", token.Claims.Audience, DefaultAudience)
+ }
+
+ if !verifiesUnderSet(p.PublishedSet(), token) {
+ t.Error("an unarmed provider minted a token that does not verify under its own published keys")
+ }
+
+ if token.Claims.Expires == nil || time.Unix(*token.Claims.Expires, 0).Before(time.Now()) {
+ t.Error("an unarmed provider minted an already-expired token")
+ }
+ }
+}
+
+// The valid stream carries HonestToken and must keep carrying a usable
+// credential whatever is armed. A stream that drifted into forged tokens would
+// make every verdict read as the target having crashed.
+func TestHonestTokenIgnoresEveryArmedForgery(t *testing.T) {
+ p := listening(t)
+ p.SetIssuer(testIssuer)
+
+ for _, each := range everyTrap() {
+ trap := each.trap(p)
+ if !trap.Forgery() {
+ continue
+ }
+
+ if err := p.Arm(trap); err != nil {
+ t.Fatalf("arm %s: %v", each.kind, err)
+ }
+
+ raw, err := p.HonestToken()
+ if err != nil {
+ t.Fatalf("honest mint under %s: %v", each.kind, err)
+ }
+
+ token, err := Parse(raw)
+ if err != nil {
+ t.Fatalf("parse under %s: %v", each.kind, err)
+ }
+
+ if token.Header.Alg != string(RS256) || token.Claims.Issuer != testIssuer {
+ t.Errorf("%s leaked into the honest token: alg=%q iss=%q", each.kind, token.Header.Alg, token.Claims.Issuer)
+ }
+
+ if !verifiesUnderSet(p.PublishedSet(), token) {
+ t.Errorf("%s left the honest token unverifiable", each.kind)
+ }
+
+ p.Disarm()
+ }
+}
+
+// The attacker key is the one key the provider must never advertise. If it were
+// published, a verifier resolving it would be doing the correct thing and the
+// trap would report a finding that is not one.
+func TestTheAttackerKeyIsNeverPublished(t *testing.T) {
+ p := listening(t)
+
+ if err := p.Arm(Trap{Kind: EmbeddedJWK}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ raw, err := p.TrappedToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ token, err := Parse(raw)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ if token.Header.JWK == nil {
+ t.Fatal("the embedded-key trap minted a token with no `jwk` header")
+ }
+
+ if p.PublishedSet().Has(token.Header.JWK.Kid) {
+ t.Fatal("khaos published the attacker key, so resolving it would be correct behaviour")
+ }
+
+ if !verifiesUnder(*token.Header.JWK, token) {
+ t.Fatal("the embedded key does not verify the token it is embedded in, so no verifier could accept it for the reason the trap claims")
+ }
+}
+
+func TestJWKSFetchesAreCounted(t *testing.T) {
+ p := listening(t)
+
+ addr, err := p.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ before := p.JWKSFetches()
+
+ for range 3 {
+ if _, _, err := probeRoute(context.Background(), httpClient(), "http://"+addr+RouteJWKS); err != nil {
+ t.Fatalf("fetch: %v", err)
+ }
+ }
+
+ if got := p.JWKSFetches() - before; got != 3 {
+ t.Fatalf("counted %d JWK Set fetches, want 3", got)
+ }
+}
+
+// The health route is the control for every endpoint trap, so it must never
+// participate in one. If it did, khaos could not tell its own provider being
+// gone from its own provider lying.
+func TestHealthIsNeverFaulted(t *testing.T) {
+ for _, kind := range []Kind{JWKSUnreachable, IntrospectionError, IntrospectionMalformed} {
+ p := listening(t)
+
+ if err := p.Arm(Trap{Kind: kind}); err != nil {
+ t.Fatalf("arm %s: %v", kind, err)
+ }
+
+ addr, err := p.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ status, _, err := probeRoute(context.Background(), httpClient(), "http://"+addr+RouteHealth)
+ if err != nil || status != 200 {
+ t.Fatalf("health answered %d (%v) with %s armed", status, err, kind)
+ }
+
+ _ = p.Close()
+ }
+}
+
+// The production ordering: pin the issuer, THEN bind. khaos always does it that
+// way round, because it learns the issuer from the target's own environment
+// before it knows where it will be listening.
+//
+// Listen used to overwrite the pinned issuer with its own listener address, so
+// every token khaos minted was refused by the target on the issuer check and the
+// run reported `baseline-not-met / "the valid credential was never accepted"` —
+// a true statement that blamed the target for khaos sending the wrong thing.
+//
+// No existing test saw it: a test that reads Issuer() after Listen() agrees with
+// itself whatever Listen did. This one states the expected string up front.
+func TestListenDoesNotOverwriteAPinnedIssuer(t *testing.T) {
+ p, err := New(Config{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+ t.Cleanup(func() { _ = p.Close() })
+
+ p.SetIssuer(testIssuer)
+
+ if err := p.Listen("127.0.0.1:0"); err != nil {
+ t.Fatalf("bind: %v", err)
+ }
+
+ addr, err := p.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ if p.Issuer() != testIssuer {
+ t.Fatalf("after binding on %s the issuer is %q, want the pinned %q", addr, p.Issuer(), testIssuer)
+ }
+
+ raw, err := p.HonestToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ token, err := Parse(raw)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ if token.Claims.Issuer != testIssuer {
+ t.Fatalf("the minted token says iss %q, want the pinned %q; a target configured for "+
+ "the pinned issuer refuses this and the run blames the target", token.Claims.Issuer, testIssuer)
+ }
+}
+
+// The other half: a provider nobody pinned still names itself, so it is usable
+// without the range and `khaos plan` can describe it.
+func TestAnUnpinnedProviderNamesItselfAfterBinding(t *testing.T) {
+ p, err := New(Config{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+ t.Cleanup(func() { _ = p.Close() })
+
+ if p.Issuer() != "" {
+ t.Fatalf("an unbound, unpinned provider claims the issuer %q", p.Issuer())
+ }
+
+ if err := p.Listen("127.0.0.1:0"); err != nil {
+ t.Fatalf("bind: %v", err)
+ }
+
+ addr, err := p.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ if want := "http://" + addr; p.Issuer() != want {
+ t.Fatalf("issuer = %q, want %q", p.Issuer(), want)
+ }
+}
+
+// The key-set fetch count is the measurement research 07 requirement R5 exists
+// for: how often a verifier refetches, which no third-party provider will tell
+// you. khaos's own read-back goes over the same HTTP surface, so without the
+// correction khaos's instrumentation would appear in the number khaos reports.
+func TestKhaosOwnConfirmationDoesNotInflateTheFetchCount(t *testing.T) {
+ for _, kind := range []Kind{JWKSUnreachable, KidRollover, KeyRotation} {
+ t.Run(string(kind), func(t *testing.T) {
+ p := listening(t)
+
+ before := p.JWKSFetches()
+
+ if err := p.Arm(Trap{Kind: kind}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ if _, confirmed, err := p.Observe(context.Background(), kind); err != nil || !confirmed {
+ t.Fatalf("confirm said %v (%v)", confirmed, err)
+ }
+
+ if after := p.JWKSFetches(); after != before {
+ t.Fatalf("confirming %s moved the reported fetch count from %d to %d; the number has "+
+ "to describe the TARGET's refetch behaviour, not khaos's own probing", kind, before, after)
+ }
+ })
+ }
+}
+
+// And the correction must not hide a real fetch.
+func TestATargetsFetchIsStillCounted(t *testing.T) {
+ p := listening(t)
+
+ addr, err := p.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ if err := p.Arm(Trap{Kind: KidRollover}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ if _, confirmed, err := p.Observe(context.Background(), KidRollover); err != nil || !confirmed {
+ t.Fatalf("confirm said %v (%v)", confirmed, err)
+ }
+
+ before := p.JWKSFetches()
+
+ for range 4 {
+ if _, _, err := probeRoute(context.Background(), httpClient(), "http://"+addr+RouteJWKS); err != nil {
+ t.Fatalf("fetch: %v", err)
+ }
+ }
+
+ if got := p.JWKSFetches() - before; got != 4 {
+ t.Fatalf("counted %d target fetches, want 4; the correction is swallowing real ones", got)
+ }
+}
+
+// A conforming RFC 7662 client has to work. khaos read the token only from the
+// query string and its own client sent it only that way, so the two agreed with
+// each other and with nothing else: an external client following the
+// specification got `active: false` for a perfectly valid credential.
+//
+// This is a REQUEST-level contract test on purpose. Going through khaos's own
+// helper would exercise the same private convention on both sides and prove
+// nothing about interoperability.
+func TestIntrospectionAnswersTheWireFormatTheRFCSpecifies(t *testing.T) {
+ p := listening(t)
+
+ addr, err := p.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ credential := p.Revocable()
+ endpoint := "http://" + addr + RouteIntrospect
+
+ post := func() Introspection {
+ t.Helper()
+
+ response, err := httpClient().PostForm(endpoint, url.Values{ParamToken: {credential}})
+ if err != nil {
+ t.Fatalf("post: %v", err)
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ if response.StatusCode != http.StatusOK {
+ t.Fatalf("introspection answered %d", response.StatusCode)
+ }
+
+ var body Introspection
+ if err := json.NewDecoder(response.Body).Decode(&body); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+
+ return body
+ }
+
+ if body := post(); !body.Active {
+ t.Fatal("a form-encoded POST for an active credential reported it inactive; no conforming " +
+ "client can establish a baseline against this provider")
+ }
+
+ if err := p.Arm(Trap{Kind: Revoke, Credential: credential}); err != nil {
+ t.Fatalf("revoke: %v", err)
+ }
+
+ if body := post(); body.Active {
+ t.Fatal("a form-encoded POST reported a revoked credential active")
+ }
+}
+
+// Revocation takes the same wire format, per RFC 7009 section 2.1.
+func TestRevocationAcceptsTheFormEncodedPost(t *testing.T) {
+ p := listening(t)
+
+ addr, err := p.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ credential := p.Opaque()
+
+ response, err := httpClient().PostForm("http://"+addr+RouteRevoke, url.Values{ParamToken: {credential}})
+ if err != nil {
+ t.Fatalf("post: %v", err)
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ if response.StatusCode != http.StatusOK {
+ t.Fatalf("a form-encoded revocation answered %d, so a conforming client cannot revoke",
+ response.StatusCode)
+ }
+
+ if p.active(t, credential) {
+ t.Fatal("the credential is still active after a conforming revocation")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go
new file mode 100644
index 00000000..8d337eeb
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go
@@ -0,0 +1,646 @@
+// ©AngelaMos | 2026
+// provider.go
+
+package idp
+
+import (
+ "crypto/rand"
+ "errors"
+ "fmt"
+ "net"
+ "net/http"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+const (
+ RouteDiscovery = "/.well-known/openid-configuration"
+ RouteJWKS = "/jwks"
+ RouteToken = "/token"
+ RouteIntrospect = "/introspect"
+ RouteRevoke = "/revoke"
+ RouteHealth = "/health"
+
+ ParamToken = "token"
+
+ headerContentType = "Content-Type"
+ mimeJSON = "application/json"
+ mimeText = "text/plain"
+
+ // DefaultAudience and DefaultSubject are what khaos's provider puts in every
+ // honest token. They are constants rather than parameters because the target
+ // has to be configured with the same audience at container creation, and a
+ // knob on both ends that must agree is a knob that will disagree.
+ DefaultAudience = "khaos-range"
+ DefaultSubject = "khaos-operator"
+
+ // ForeignAudience and ForeignIssuer are the second tenant. The wrong-audience
+ // and foreign-issuer traps address tokens to these, and a verifier that
+ // accepts either has stopped treating its own identity as a constraint.
+ ForeignAudience = "khaos-range-other"
+ ForeignIssuerID = "https://khaos.invalid/second-tenant"
+
+ // DefaultLifetime is long enough that no run expires a token it is using and
+ // short enough to be a plausible access-token lifetime.
+ DefaultLifetime = 10 * time.Minute
+
+ opaquePrefix = "khaos-opaque-"
+
+ malformedBody = "{\"active\": tru"
+
+ shutdownGrace = 2 * time.Second
+
+ errAlreadyListening = "%w: khaos's identity provider is already listening on %s"
+ errNotListening = "%w: khaos's identity provider is not listening, so it has no address"
+ errNoRevocable = "%w: this run never issued a revocable credential, so there is nothing to revoke"
+ errWrongTrap = "%w: the provider holds %q but %q was expected"
+ errUnknownGrant = "%w: khaos's identity provider never issued %q"
+)
+
+var ErrListening = errors.New("khaos's identity provider cannot be bound twice")
+
+var ErrNotListening = errors.New("khaos's identity provider has no listener")
+
+var ErrClosed = errors.New("khaos's identity provider is closed")
+
+var ErrNothingToRevoke = errors.New("khaos has no credential to revoke")
+
+var ErrTrapNotObserved = errors.New("the armed trap did not produce its own observable")
+
+var ErrNoSuchGrant = errors.New("that credential was never issued here")
+
+// armedTrap is the whole mutable fault state, swapped as one value so a handler
+// can never read a trap from one revert and a generation from another. It is the
+// same discipline the fault proxy uses, and for the same reason: arming has to
+// be indivisible or a probe can observe a half-applied fault.
+type armedTrap struct {
+ trap Trap
+ generation uint64
+}
+
+// keySnapshot is the key state as it stood before a rotating trap was armed.
+// Reverting restores it, so the recovery phase measures the provider the
+// baseline measured rather than a provider the fault permanently changed.
+type keySnapshot struct {
+ signing *Key
+ retired []*Key
+}
+
+// grant is one opaque credential the provider issued. Introspection answers from
+// these, and revoking one flips active rather than deleting it, so a revert can
+// put it back and the recovery phase means something.
+type grant struct {
+ subject string
+ active bool
+}
+
+// Provider is khaos's own OIDC-shaped identity provider: a programmable signing
+// oracle with a JWK Set, an introspection endpoint and a kill switch.
+//
+// It is hand-built rather than adopted, and research 07 section 4.2 is blunt
+// about why: ten of the traps need a provider that will deliberately emit a
+// malformed token, and four more need it to break its own endpoints on command
+// and report how many times it was fetched. A correct provider does none of
+// that, and that is not a gap in Keycloak or Dex, it is the definition of their
+// being correct.
+//
+// The cost is stated out loud: this is not a conformance-tested OP and khaos
+// must never claim it validates OIDC conformance. It validates VERIFIERS.
+//
+// Minting needs no listener, which is what lets an experiment build its probes
+// before khaos has discovered the address the target is already addressing.
+type Provider struct {
+ audience string
+ foreignAudience string
+ foreignIssuer string
+ lifetime time.Duration
+
+ mu sync.RWMutex
+ issuer string
+ signing *Key
+ retired []*Key
+ foreign *Key
+ stable *Key
+ weak *Key
+ ec *Key
+ attacker *Key
+ grants map[string]*grant
+ snapshot *keySnapshot
+
+ // revocable is the credential THIS run intends to revoke. The mutator reads
+ // it at inject time rather than at build time, because the probe that issues
+ // it has not fired yet when the experiment is built.
+ revocable string
+
+ state atomic.Pointer[armedTrap]
+ inert atomic.Bool
+ closed chan struct{}
+ once sync.Once
+
+ listener net.Listener
+ server *http.Server
+ wg sync.WaitGroup
+
+ // jwksFetches counts every fetch of the key set, and ownFetches counts the
+ // subset khaos's own confirmation probes made. The accessor reports the
+ // difference, because this number is the measurement research 07 requirement
+ // R5 exists for — how often a verifier refetches, which no third-party
+ // provider exposes — and khaos's instrumentation must not appear in the
+ // measurement khaos is taking.
+ //
+ // Subtracting is used rather than a header the handler skips, because a
+ // header is spoofable and the arithmetic is not.
+ jwksFetches atomic.Int64
+ ownFetches atomic.Int64
+
+ // The same arrangement for introspection. This counter is not decoration:
+ // three experiments fault the introspection endpoint, and without a way to
+ // see that the target ever CALLED it they can only report that the target
+ // survived a fault it never experienced.
+ introspects atomic.Int64
+ ownIntrospects atomic.Int64
+
+ now func() time.Time
+}
+
+type Config struct {
+ Audience string
+ Lifetime time.Duration
+
+ // Now is the clock. Tests control it so a claim trap can be asserted against
+ // a known instant rather than against whatever the wall clock said.
+ Now func() time.Time
+}
+
+// New builds a provider with every key it might ever sign with, and no listener.
+// All five keys are generated up front so arming a trap is a pointer swap rather
+// than a key generation, which keeps the fault indivisible.
+func New(config Config) (*Provider, error) {
+ if config.Audience == "" {
+ config.Audience = DefaultAudience
+ }
+ if config.Lifetime <= 0 {
+ config.Lifetime = DefaultLifetime
+ }
+ if config.Now == nil {
+ config.Now = func() time.Time { return time.Now().UTC() }
+ }
+
+ signing, err := NewRSAKey()
+ if err != nil {
+ return nil, err
+ }
+
+ foreign, err := NewRSAKey()
+ if err != nil {
+ return nil, err
+ }
+
+ // Published, and never rotated. It exists so an experiment that measures key
+ // ROTATION can still prove the target is serving without the proof itself
+ // disturbing what is being measured: a control token under the current
+ // signing key becomes new-key traffic the moment the key rotates, and that
+ // traffic is exactly what evicts the retired key.
+ stable, err := NewRSAKey()
+ if err != nil {
+ return nil, err
+ }
+
+ // NOT published, ever. The whole claim of the embedded-key trap is that the
+ // verifier resolved a key the provider never advertised, so publishing this
+ // one would make an acceptance unremarkable.
+ attacker, err := NewRSAKey()
+ if err != nil {
+ return nil, err
+ }
+
+ weak, err := NewWeakRSAKey()
+ if err != nil {
+ return nil, err
+ }
+
+ ec, err := NewECKey()
+ if err != nil {
+ return nil, err
+ }
+
+ p := &Provider{
+ audience: config.Audience,
+ foreignAudience: ForeignAudience,
+ foreignIssuer: ForeignIssuerID,
+ lifetime: config.Lifetime,
+ signing: signing,
+ foreign: foreign,
+ stable: stable,
+ attacker: attacker,
+ weak: weak,
+ ec: ec,
+ grants: map[string]*grant{},
+ closed: make(chan struct{}),
+ now: config.Now,
+ }
+
+ p.state.Store(&armedTrap{})
+
+ return p, nil
+}
+
+func (p *Provider) clock() time.Time {
+ return p.now()
+}
+
+// Listen binds the provider where the target is already looking for it. The
+// address is not khaos's choice: the verifier gateway's environment is fixed at
+// container creation, so khaos binds the port that container is addressing
+// rather than one khaos would have picked.
+func (p *Provider) Listen(addr string) error {
+ // The guard and the bind are under ONE lock acquisition. Releasing between
+ // them let two concurrent calls both pass the guard and both bind, with the
+ // second orphaning the first listener and its goroutine. Holding the lock
+ // across net.Listen is fine: it is a local syscall, not a round trip, and
+ // nothing else can usefully run until the provider has an address.
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ if p.listener != nil {
+ return fmt.Errorf(errAlreadyListening, ErrListening, p.listener.Addr().String())
+ }
+
+ listener, err := net.Listen("tcp", addr)
+ if err != nil {
+ return err
+ }
+
+ server := &http.Server{
+ Handler: p.routes(),
+ ReadHeaderTimeout: shutdownGrace,
+ }
+
+ p.listener = listener
+ p.server = server
+
+ // Only when nothing pinned one. An explicitly set issuer is the string the
+ // TARGET was configured to trust, and the address khaos listens on is never
+ // that string: khaos binds the bridge gateway's numeric address while the
+ // container addresses `host.docker.internal`.
+ //
+ // Overwriting it here minted every token with `iss` set to khaos's own
+ // listener, and the target refused all of them on the issuer check. The
+ // symptom was `baseline-not-met / "the valid credential was never accepted"`,
+ // which reads as a broken target rather than as khaos sending the wrong
+ // thing, and no unit test saw it because a test that reads Issuer() after
+ // Listen() agrees with itself by construction.
+ if p.issuer == "" {
+ p.issuer = "http://" + listener.Addr().String()
+ }
+
+ // wg.Add inside the lock, so a Listen racing a Close cannot add to a
+ // WaitGroup that Close is already waiting on — the one misuse the
+ // sync.WaitGroup documentation calls out by name.
+ p.wg.Add(1)
+
+ go func() {
+ defer p.wg.Done()
+ _ = server.Serve(listener)
+ }()
+
+ return nil
+}
+
+// Issuer is the `iss` claim and the base of the JWKS URI. Before Listen it is
+// whatever SetIssuer was told, because the target is configured with the issuer
+// at container creation and khaos has to mint tokens that match it.
+func (p *Provider) Issuer() string {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+ return p.issuer
+}
+
+// SetIssuer pins the issuer khaos mints under, to the value the target was built
+// expecting. A token whose `iss` disagrees with the target's configuration is
+// rejected for a reason that has nothing to do with the trap under test, which
+// reads exactly like a target that caught the trap.
+func (p *Provider) SetIssuer(issuer string) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.issuer = issuer
+}
+
+func (p *Provider) Addr() (string, error) {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+
+ if p.listener == nil {
+ return "", fmt.Errorf(errNotListening, ErrNotListening)
+ }
+
+ return p.listener.Addr().String(), nil
+}
+
+func (p *Provider) Close() error {
+ var err error
+
+ p.once.Do(func() {
+ close(p.closed)
+
+ p.mu.RLock()
+ server := p.server
+ p.mu.RUnlock()
+
+ if server != nil {
+ err = server.Close()
+ }
+
+ p.wg.Wait()
+ })
+
+ return err
+}
+
+// Arm swaps the trap in as one value and snapshots any key state the trap is
+// about to change, so Disarm can put it back.
+func (p *Provider) Arm(trap Trap) error {
+ select {
+ case <-p.closed:
+ return ErrClosed
+ default:
+ }
+
+ if err := trap.Validate(); err != nil {
+ return err
+ }
+
+ // The snapshot is taken whether or not the state change happens, because it
+ // is what the confirmation oracle compares the published set against. An
+ // inert arm has to leave the oracle able to say NO, and an oracle with
+ // nothing to compare cannot say anything.
+ p.snap()
+
+ // Inert skips the state change and nothing else. That is what makes it a
+ // negative control for the two traps that mutate key material and the one
+ // that mutates a grant: the trap reads as armed, the observable never
+ // appears, and confirmation must refuse.
+ if !p.inert.Load() {
+ switch trap.Kind {
+ case KidRollover:
+ if err := p.rotate(false); err != nil {
+ return err
+ }
+ case KeyRotation:
+ if err := p.rotate(true); err != nil {
+ return err
+ }
+ case Revoke:
+ if err := p.revoke(trap.Credential); err != nil {
+ return err
+ }
+ }
+ }
+
+ previous := p.state.Load()
+ p.state.Store(&armedTrap{trap: trap, generation: previous.generation + 1})
+
+ return nil
+}
+
+// snap records the key state as it stands now, once per arming cycle. Disarm
+// restores it and clears it, so a second arming snapshots afresh.
+// snap records the key state as it stands now. Disarm restores it and clears it.
+//
+// It OVERWRITES rather than keeping the first snapshot. The previous version
+// returned early when one existed, so a second Arm without an intervening
+// Disarm left RetiredKid naming the first retired key and made Disarm restore
+// the wrong one — the confirmation oracle would then compare the published set
+// against a kid two rotations old. No path arms twice today; the mutator arms in
+// Apply and disarms in Revert. This is the behaviour that is correct whether or
+// not that stays true.
+func (p *Provider) snap() {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ p.snapshot = &keySnapshot{signing: p.signing, retired: append([]*Key{}, p.retired...)}
+}
+
+// RetiredKid is the key that was signing before the current arming cycle. The
+// confirmation oracle asserts against the PUBLISHED SET rather than against this
+// value, so reading it here cannot make a failed rotation look like a successful
+// one: a rotation that did not happen leaves this kid still published.
+func (p *Provider) RetiredKid() (string, bool) {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+
+ if p.snapshot == nil || p.snapshot.signing == nil {
+ return "", false
+ }
+
+ return p.snapshot.signing.ID, true
+}
+
+func (p *Provider) Disarm() {
+ previous := p.state.Load()
+ p.state.Store(&armedTrap{generation: previous.generation + 1})
+
+ p.restoreKeys()
+ p.restoreGrants()
+}
+
+func (p *Provider) Trap() Trap {
+ return p.state.Load().trap
+}
+
+// Inert makes the provider behave correctly while a trap is still armed. It
+// exists so the confirmation oracle has a negative control.
+//
+// Without a way to arm a trap that does NOT happen, every test can only assert
+// that confirmation SUCCEEDS, and a confirmation gutted to `return true`
+// satisfies all of them. The fault proxy paid for that lesson: measured, that
+// gut left all seventeen packages and the Docker gate green.
+func (p *Provider) Inert(inert bool) {
+ p.inert.Store(inert)
+}
+
+func (p *Provider) effective() Trap {
+ if p.inert.Load() {
+ return Trap{}
+ }
+ return p.Trap()
+}
+
+// rotate installs a fresh signing key. With overlap the retired key stays
+// published, which is the availability-preserving rotation every provider is
+// supposed to perform. Without it the retired key is withdrawn immediately,
+// which is what makes a previously valid token unverifiable to anyone who
+// refetches and still valid to anyone who caches forever.
+func (p *Provider) rotate(overlap bool) error {
+ fresh, err := NewRSAKey()
+ if err != nil {
+ return err
+ }
+
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ if overlap {
+ p.retired = append(p.retired, p.signing)
+ } else {
+ p.retired = nil
+ }
+
+ p.signing = fresh
+
+ return nil
+}
+
+func (p *Provider) restoreKeys() {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ if p.snapshot == nil {
+ return
+ }
+
+ p.signing = p.snapshot.signing
+ p.retired = p.snapshot.retired
+ p.snapshot = nil
+}
+
+// revoke flips a grant inactive. It refuses a credential the provider never
+// issued rather than silently succeeding, because a revocation that did not
+// happen makes every later acceptance correct behaviour and the run's finding
+// unreachable.
+func (p *Provider) revoke(credential string) error {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ held, ok := p.grants[credential]
+ if !ok {
+ return fmt.Errorf(errUnknownGrant, ErrNoSuchGrant, credential)
+ }
+
+ held.active = false
+
+ return nil
+}
+
+func (p *Provider) restoreGrants() {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ for _, held := range p.grants {
+ held.active = true
+ }
+}
+
+// HonestToken is always a correct token, whatever is armed. The valid stream
+// carries it, and a stream that drifted into carrying forged tokens would make
+// every verdict read as a crash.
+func (p *Provider) HonestToken() (string, error) {
+ return p.mintToken(Trap{}, grantRequest{subject: DefaultSubject, lifetime: p.lifetime})
+}
+
+// TrappedToken honours whatever is armed. It is what the forged stream carries,
+// and it is also what the rotation experiment's VALID stream carries, because
+// there the question is whether the target accepts a correctly signed token under
+// the provider's new key.
+func (p *Provider) TrappedToken() (string, error) {
+ return p.mintToken(p.effective(), grantRequest{subject: DefaultSubject, lifetime: p.lifetime})
+}
+
+// StableToken is a correct JWT signed by a key that is always published and
+// never rotated.
+//
+// It is the valid-stream credential for the rotation experiments, and the reason
+// is an observer effect khaos measured on itself. A valid stream minting under
+// the CURRENT signing key supplies new-key traffic after a rotation, and research
+// 07 section 2.3 establishes that the eviction of a retired key is a side effect
+// of exactly that traffic — so khaos was causing the recovery it then reported.
+//
+// An opaque credential also avoids it, and was the first fix, but it requires
+// the target to support introspection, which a JWT-only verifier does not. This
+// one is a JWT, so it works against any verifier, and its kid is in the key set
+// from the baseline onwards, so resolving it never forces a refetch.
+func (p *Provider) StableToken() (string, error) {
+ p.mu.RLock()
+ stable := p.stable
+ p.mu.RUnlock()
+
+ return p.mintSignedBy(stable)
+}
+
+// Opaque issues a credential that carries no claims, so the only way a verifier
+// can decide anything about it is to ask. It is what the introspection traps are
+// measured with.
+func (p *Provider) Opaque() string {
+ credential := opaquePrefix + rand.Text()
+
+ p.mu.Lock()
+ p.grants[credential] = &grant{subject: DefaultSubject, active: true}
+ p.mu.Unlock()
+
+ return credential
+}
+
+// Revocable issues an opaque credential and records it as the one this run
+// intends to revoke, so the mutator can find it at inject time. The probe that
+// calls this has not fired when the experiment is built, which is why the
+// credential cannot travel on the trap from the start.
+func (p *Provider) Revocable() string {
+ credential := p.Opaque()
+
+ p.mu.Lock()
+ p.revocable = credential
+ p.mu.Unlock()
+
+ return credential
+}
+
+func (p *Provider) RevocableCredential() (string, error) {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+
+ if p.revocable == "" {
+ return "", fmt.Errorf(errNoRevocable, ErrNothingToRevoke)
+ }
+
+ return p.revocable, nil
+}
+
+// PublishedSet is every key a verifier may resolve, which is every key khaos
+// signs with EXCEPT the attacker key. The weak and EC keys are always published
+// rather than published on arming, so that a forgery trap changes only what the
+// provider MINTS and a key trap changes only what it SERVES. Confusing the two
+// would make each one's read-back ambiguous.
+func (p *Provider) PublishedSet() JWKSet {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+
+ set := JWKSet{Keys: make([]JWK, 0, len(p.retired)+4)}
+ set.Keys = append(set.Keys, p.signing.PublicJWK())
+
+ for _, key := range p.retired {
+ set.Keys = append(set.Keys, key.PublicJWK())
+ }
+
+ set.Keys = append(set.Keys,
+ p.foreign.PublicJWK(), p.stable.PublicJWK(), p.weak.PublicJWK(), p.ec.PublicJWK())
+
+ return set
+}
+
+func (p *Provider) SigningKid() string {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+ return p.signing.ID
+}
+
+// JWKSFetches is how many times something OTHER than khaos fetched the key set.
+func (p *Provider) JWKSFetches() int64 {
+ return p.jwksFetches.Load() - p.ownFetches.Load()
+}
+
+// Introspections is how many times something OTHER than khaos asked about a
+// credential.
+func (p *Provider) Introspections() int64 {
+ return p.introspects.Load() - p.ownIntrospects.Load()
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/routes.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/routes.go
new file mode 100644
index 00000000..ccfd7cec
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/routes.go
@@ -0,0 +1,210 @@
+// ©AngelaMos | 2026
+// routes.go
+
+package idp
+
+import (
+ "encoding/json"
+ "net/http"
+ "time"
+)
+
+const (
+ fieldIssuer = "issuer"
+ fieldJWKSURI = "jwks_uri"
+ fieldTokenEndpoint = "token_endpoint"
+ fieldIntrospection = "introspection_endpoint"
+ fieldRevocation = "revocation_endpoint"
+ fieldAlgsSupported = "id_token_signing_alg_values_supported"
+ fieldToken = "token"
+ fieldSubject = "sub"
+
+ // Enough for a token and its form framing, and nowhere near enough to be a
+ // memory-exhaustion surface on an endpoint that takes unauthenticated
+ // requests by design.
+ maxFormBytes = 64 << 10
+)
+
+// Introspection is RFC 7662's response. `active` is the only member a verifier
+// may decide on, and the specification is explicit that the ABSENCE of a
+// response is not `active: false`. A caller that invents one from a timeout or a
+// 503 has made up a value the provider never gave it, which is taxonomy row F20
+// and which three of this package's traps exist to provoke.
+type Introspection struct {
+ Active bool `json:"active"`
+ Subject string `json:"sub,omitempty"`
+ Scope string `json:"scope,omitempty"`
+}
+
+func (p *Provider) routes() *http.ServeMux {
+ mux := http.NewServeMux()
+ mux.HandleFunc(RouteHealth, p.health)
+ mux.HandleFunc(RouteDiscovery, p.discovery)
+ mux.HandleFunc(RouteJWKS, p.jwks)
+ mux.HandleFunc(RouteToken, p.token)
+ mux.HandleFunc(RouteIntrospect, p.introspect)
+ mux.HandleFunc(RouteRevoke, p.revokeRoute)
+ return mux
+}
+
+// health is never faulted. A target or an operator has to be able to tell
+// "khaos's provider is gone" from "khaos's provider is lying", and a health
+// endpoint that participates in the fault destroys that distinction.
+func (p *Provider) health(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+}
+
+// discovery is the OIDC Discovery document. It is served so a target can be
+// pointed at the issuer alone, and it is deliberately NOT faulted: research 07
+// found that OIDC Discovery says nothing about rotation, nothing about `kid` and
+// nothing about refetching, so every interesting fault lives at the JWK Set
+// rather than at the document that points to it.
+func (p *Provider) discovery(w http.ResponseWriter, _ *http.Request) {
+ issuer := p.Issuer()
+
+ writeJSON(w, http.StatusOK, map[string]any{
+ fieldIssuer: issuer,
+ fieldJWKSURI: issuer + RouteJWKS,
+ fieldTokenEndpoint: issuer + RouteToken,
+ fieldIntrospection: issuer + RouteIntrospect,
+ fieldRevocation: issuer + RouteRevoke,
+ fieldAlgsSupported: []string{string(RS256), string(ES256)},
+ })
+}
+
+// jwks serves the published key set, and counts every fetch. The count is the
+// measurement the refetch-amplification question needs and that no third-party
+// provider exposes: an unknown `kid` is an attacker-controlled header field, so
+// an unbounded refetch is a denial-of-service amplifier, and the only way to
+// know whether a verifier bounds it is to count from the provider's side.
+func (p *Provider) jwks(w http.ResponseWriter, _ *http.Request) {
+ p.jwksFetches.Add(1)
+
+ if p.effective().Kind == JWKSUnreachable {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ return
+ }
+
+ writeJSON(w, http.StatusOK, p.PublishedSet())
+}
+
+// token mints under whatever is armed. khaos's own probes call mintToken
+// in-process and never come through here; this exists so an operator can point
+// their own client at khaos's provider, which is what makes the provider an
+// instrument rather than a range fixture.
+func (p *Provider) token(w http.ResponseWriter, _ *http.Request) {
+ minted, err := p.TrappedToken()
+ if err != nil {
+ writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
+ return
+ }
+
+ writeJSON(w, http.StatusOK, map[string]string{
+ fieldToken: minted,
+ fieldSubject: DefaultSubject,
+ })
+}
+
+// introspect answers about an opaque credential, and carries three of the traps.
+// Each one leaves the provider REACHABLE and answering, which is the whole
+// point: a caller's `err != nil` branch is never taken, so only its status
+// handling, its decoder's error branch, or its timeout decides what happens.
+// That is a different code path from an outage and it is reached by different
+// bugs.
+func (p *Provider) introspect(w http.ResponseWriter, r *http.Request) {
+ p.introspects.Add(1)
+
+ trap := p.effective()
+
+ switch trap.Kind {
+ case IntrospectionError:
+ w.WriteHeader(http.StatusServiceUnavailable)
+ return
+
+ case IntrospectionMalformed:
+ // A 200 with a body that starts out looking like the real thing and then
+ // stops being JSON. A caller whose decode error branch returns "active"
+ // has invented a value RFC 7662 never provides.
+ w.Header().Set(headerContentType, mimeJSON)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(malformedBody))
+ return
+
+ case IntrospectionSlow:
+ select {
+ case <-time.After(trap.Delay):
+ case <-r.Context().Done():
+ return
+ case <-p.closed:
+ return
+ }
+ }
+
+ credential := credentialFrom(r)
+
+ p.mu.RLock()
+ held, known := p.grants[credential]
+ active := known && held.active
+ subject := ""
+ if known {
+ subject = held.subject
+ }
+ p.mu.RUnlock()
+
+ body := Introspection{Active: active}
+ if active {
+ body.Subject = subject
+ }
+
+ writeJSON(w, http.StatusOK, body)
+}
+
+// revokeRoute is the operator-facing revocation. It reports whether it actually
+// revoked anything: a 404 for a credential this provider never issued, because
+// answering 200 to a revocation that did not happen is how a run goes on to
+// measure a credential nobody took away.
+func (p *Provider) revokeRoute(w http.ResponseWriter, r *http.Request) {
+ credential := credentialFrom(r)
+
+ if err := p.revoke(credential); err != nil {
+ writeJSON(w, http.StatusNotFound, map[string]string{"error": err.Error()})
+ return
+ }
+
+ w.WriteHeader(http.StatusOK)
+}
+
+// credentialFrom reads the token the way the specifications say it is sent, and
+// also the way khaos used to send it.
+//
+// RFC 7662 section 2.1 defines the introspection request as a POST with an
+// `application/x-www-form-urlencoded` body, and RFC 7009 section 2.1 says the
+// same for revocation. khaos read the token ONLY from the query string, and its
+// own client sent it only that way, so the two agreed with each other and with
+// nothing else: a conforming external client got `active: false` for a perfectly
+// valid credential and could not establish a baseline at all.
+//
+// The query form is still accepted. It costs nothing, it keeps khaos's own
+// read-back probes working, and a provider that is deliberately willing to be
+// wrong has no reason to be strict about where it reads an input from.
+func credentialFrom(r *http.Request) string {
+ if token := r.URL.Query().Get(ParamToken); token != "" {
+ return token
+ }
+
+ // Bounded. This is a fixture that accepts unauthenticated requests, so an
+ // unbounded ParseForm is a trivial way to exhaust its memory.
+ r.Body = http.MaxBytesReader(nil, r.Body, maxFormBytes)
+
+ if err := r.ParseForm(); err != nil {
+ return ""
+ }
+
+ return r.PostFormValue(ParamToken)
+}
+
+func writeJSON(w http.ResponseWriter, status int, body any) {
+ w.Header().Set(headerContentType, mimeJSON)
+ w.WriteHeader(status)
+ _ = json.NewEncoder(w).Encode(body)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/token.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/token.go
new file mode 100644
index 00000000..8b934b7f
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/token.go
@@ -0,0 +1,271 @@
+// ©AngelaMos | 2026
+// token.go
+
+package idp
+
+import (
+ "crypto/elliptic"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+)
+
+const (
+ segmentSep = "."
+ jwsSegments = 3
+ typJWT = "JWT"
+ typAccessJWT = "at+jwt"
+
+ errSegments = "%w: a JWS has %d dot-separated segments and this has %d"
+ errSegment = "%w: segment %d is not base64url (%v)"
+ errHeader = "%w: the header segment is not JSON (%v)"
+ errClaims = "%w: the claims segment is not JSON (%v)"
+)
+
+var ErrNotAJWS = errors.New("that is not a JWS compact serialization")
+
+// Header is the JOSE header khaos emits. Every field a trap needs to corrupt is
+// here, including the ones a correct provider never sets: `jwk` exists so khaos
+// can embed an attacker key, which is trap T2b, and `alg` is a plain string
+// rather than an Algorithm so khaos can emit a value it does not itself support.
+type Header struct {
+ Alg string `json:"alg"`
+ Typ string `json:"typ,omitempty"`
+ Kid string `json:"kid,omitempty"`
+
+ // JWK is RFC 7515's in-token key. khaos sets it only for the key-injection
+ // probe. A verifier that resolves a key from here instead of from the trusted
+ // JWK Set has violated RFC 8725 rule 4, and that is the finding.
+ JWK *JWK `json:"jwk,omitempty"`
+
+ // JKU is the in-token key-set URL, and khaos does not set it.
+ //
+ // An earlier comment here claimed khaos pointed it at its own provider and
+ // counted the hit. It did not: nothing in the repository ever assigned this
+ // field. Research 07's trap T2b does ask for that probe — a verifier that
+ // FETCHES an attacker-supplied URL is a finding even when the token is then
+ // rejected — but the fetch is not an outcome in any probe stream, so
+ // surfacing it needs an observation channel khaos does not have. The
+ // omission is recorded in the design contract's fault catalogue rather than
+ // described here as a feature.
+ //
+ // The field stays so a verifier's non-following of it is a property khaos can
+ // express later without reshaping the header.
+ JKU string `json:"jku,omitempty"`
+}
+
+// Claims is the payload. Every time field is a pointer so khaos can emit a token
+// with a claim genuinely ABSENT rather than present and zero: a verifier that
+// rejects `exp: 0` while accepting a missing `exp` is a different bug, and
+// encoding/json cannot express the difference without this.
+// Audience is RFC 7519 section 4.1.3's `aud`, which is "an array of
+// case-sensitive strings" in the general case and "MAY be a single
+// case-sensitive string" when there is one audience.
+//
+// It is a type rather than a plain string because a plain string made the
+// general case fail to unmarshal, and a verifier that dispatched on parse
+// success then treated a perfectly legal multi-audience JWT as an opaque
+// credential.
+type Audience []string
+
+func (a *Audience) UnmarshalJSON(raw []byte) error {
+ var one string
+ if err := json.Unmarshal(raw, &one); err == nil {
+ *a = Audience{one}
+ return nil
+ }
+
+ var many []string
+ if err := json.Unmarshal(raw, &many); err != nil {
+ return err
+ }
+
+ *a = many
+
+ return nil
+}
+
+// MarshalJSON emits the single-string form for one audience, because that is
+// what a real provider emits and khaos's forged tokens have to be ordinary in
+// every respect except the one property each trap corrupts.
+func (a Audience) MarshalJSON() ([]byte, error) {
+ if len(a) == 1 {
+ return json.Marshal(a[0])
+ }
+ return json.Marshal([]string(a))
+}
+
+func (a Audience) Contains(want string) bool {
+ for _, each := range a {
+ if each == want {
+ return true
+ }
+ }
+ return false
+}
+
+func (a Audience) String() string {
+ return strings.Join(a, ", ")
+}
+
+type Claims struct {
+ Issuer string `json:"iss,omitempty"`
+ Subject string `json:"sub,omitempty"`
+ Audience Audience `json:"aud,omitempty"`
+ Expires *int64 `json:"exp,omitempty"`
+ NotBefore *int64 `json:"nbf,omitempty"`
+ IssuedAt *int64 `json:"iat,omitempty"`
+ ID string `json:"jti,omitempty"`
+ Scope string `json:"scope,omitempty"`
+}
+
+// Token is a parsed JWS with its segments kept verbatim. The raw signing input
+// is retained because re-deriving it from the decoded parts would re-serialize
+// the JSON, and a signature is over the bytes that were actually sent.
+type Token struct {
+ Raw string
+ SigningInput []byte
+ Header Header
+ Claims Claims
+ Signature []byte
+ HeaderJSON []byte
+ ClaimsJSON []byte
+}
+
+// assemble builds the compact serialization. It takes the already-marshalled
+// header and claims so a trap can corrupt either one before signing, and it
+// takes the signature as bytes so a trap can replace it after signing.
+func assemble(headerJSON, claimsJSON, signature []byte) string {
+ return b64(headerJSON) + segmentSep + b64(claimsJSON) + segmentSep + b64(signature)
+}
+
+func signingInput(headerJSON, claimsJSON []byte) []byte {
+ return []byte(b64(headerJSON) + segmentSep + b64(claimsJSON))
+}
+
+// Parse reads a compact JWS without verifying anything. khaos uses it on its own
+// confirmation path, where the question is what khaos actually emitted rather
+// than whether it is acceptable.
+func Parse(raw string) (Token, error) {
+ parts := strings.Split(raw, segmentSep)
+ if len(parts) != jwsSegments {
+ return Token{}, fmt.Errorf(errSegments, ErrNotAJWS, jwsSegments, len(parts))
+ }
+
+ headerJSON, err := decodeSegment(parts[0])
+ if err != nil {
+ return Token{}, fmt.Errorf(errSegment, ErrNotAJWS, 1, err)
+ }
+
+ claimsJSON, err := decodeSegment(parts[1])
+ if err != nil {
+ return Token{}, fmt.Errorf(errSegment, ErrNotAJWS, 2, err)
+ }
+
+ signature, err := decodeSegment(parts[2])
+ if err != nil {
+ return Token{}, fmt.Errorf(errSegment, ErrNotAJWS, 3, err)
+ }
+
+ out := Token{
+ Raw: raw,
+ SigningInput: []byte(parts[0] + segmentSep + parts[1]),
+ Signature: signature,
+ HeaderJSON: headerJSON,
+ ClaimsJSON: claimsJSON,
+ }
+
+ if err := json.Unmarshal(headerJSON, &out.Header); err != nil {
+ return Token{}, fmt.Errorf(errHeader, ErrNotAJWS, err)
+ }
+
+ if err := json.Unmarshal(claimsJSON, &out.Claims); err != nil {
+ return Token{}, fmt.Errorf(errClaims, ErrNotAJWS, err)
+ }
+
+ return out, nil
+}
+
+// decodeSegment is strict base64url with no padding, which is what RFC 7515
+// section 2 specifies. A lenient decoder would let khaos's own confirmation read
+// a token a verifier would reject for a different reason.
+func decodeSegment(encoded string) ([]byte, error) {
+ return base64.RawURLEncoding.DecodeString(encoded)
+}
+
+func p256() elliptic.Curve {
+ return elliptic.P256()
+}
+
+func unix(t time.Time) *int64 {
+ seconds := t.Unix()
+ return &seconds
+}
+
+// Retarget rewrites a token's `kid` and nothing else, leaving its signature
+// intact and therefore unverifiable under the key it now names.
+//
+// It exists for the range verifier's own tests rather than for any khaos
+// experiment: a header that disagrees with its key about the algorithm is
+// forbidden by RFC 8725 rule 2, and no token an attacker could actually forge
+// isolates that rule, because anything that trips it also fails the signature
+// check. The fixture still has to carry the rule, so the rule still has to be
+// exercised.
+func Retarget(token Token, kid string) string {
+ header := token.Header
+ header.Kid = kid
+
+ headerJSON, err := json.Marshal(header)
+ if err != nil {
+ return token.Raw
+ }
+
+ return assemble(headerJSON, token.ClaimsJSON, token.Signature)
+}
+
+// LooksLikeJWS reports whether a credential has the SHAPE of a compact JWS:
+// three dot-separated segments whose first decodes to JSON with an `alg`.
+//
+// A verifier has to choose between local validation and introspection before it
+// can parse anything, and choosing on "did my parser succeed" is wrong in a way
+// that matters: a token carrying a legal array `aud` fails a parser whose field
+// is a string, and would then be introspected as though it were opaque.
+func LooksLikeJWS(credential string) bool {
+ parts := strings.Split(credential, segmentSep)
+ if len(parts) != jwsSegments {
+ return false
+ }
+
+ header, err := decodeSegment(parts[0])
+ if err != nil {
+ return false
+ }
+
+ var probe struct {
+ Alg string `json:"alg"`
+ }
+
+ if err := json.Unmarshal(header, &probe); err != nil {
+ return false
+ }
+
+ return probe.Alg != ""
+}
+
+// Retyped re-mints an honest token with a different `typ` and a valid signature.
+// It exists so the range verifier's explicit-typing check can be exercised: a
+// token whose signature does not verify would be refused before the claim checks
+// are reached, and the typ check sits among them.
+func Retyped(p *Provider, typ string) string {
+ return p.reheader(func(header *Header) { header.Typ = typ })
+}
+
+// WithoutExpiry re-mints an honest token with no `exp` member at all, correctly
+// signed. The pointer field is what makes an ABSENT claim expressible rather
+// than a present zero.
+func WithoutExpiry(p *Provider) string {
+ return p.reclaim(func(claims *Claims) { claims.Expires = nil })
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/trap.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/trap.go
new file mode 100644
index 00000000..737bc9ed
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/trap.go
@@ -0,0 +1,262 @@
+// ©AngelaMos | 2026
+// trap.go
+
+package idp
+
+import (
+ "errors"
+ "fmt"
+ "slices"
+ "time"
+)
+
+// Kind names one way khaos makes its own identity provider wrong. Every kind
+// maps to a verifier defect with a traceable anchor in research 07, and the
+// mapping is one-to-one with the experiments so `khaos catalog` names the actual
+// trap rather than a generic entry.
+type Kind string
+
+const (
+ // PassThrough is the zero value: the provider behaves correctly. A provider
+ // with nothing armed must be indistinguishable from a real one, because every
+ // experiment's baseline is measured against it.
+ PassThrough Kind = ""
+
+ // AlgNone emits `{"alg":"none"}` with an empty signature. Trap T1,
+ // CVE-2022-23540.
+ AlgNone Kind = "alg-none"
+
+ // AlgConfusion emits HS256 keyed on the bytes of the published RSA public
+ // key. Trap T2, CVE-2015-9235 and three others.
+ AlgConfusion Kind = "alg-confusion"
+
+ // EmbeddedJWK emits a token carrying its own signing key in the `jwk` header,
+ // signed by a key khaos never publishes. Trap T2b, CVE-2018-0114.
+ EmbeddedJWK Kind = "embedded-jwk"
+
+ // ClaimExpired through ClaimForeignIssuer each emit a CORRECTLY SIGNED token
+ // with exactly one wrong claim. Research 07 calls this the cleanest
+ // experiment design in the tool: a 2xx isolates precisely which check is
+ // missing, with no ambiguity about the signature.
+ ClaimExpired Kind = "claim-expired"
+ ClaimNotYetValid Kind = "claim-not-yet-valid"
+ ClaimWrongAudience Kind = "claim-wrong-audience"
+ ClaimForeignIssuer Kind = "claim-foreign-issuer"
+
+ // SignatureTruncated cuts a valid signature to one byte. Trap T6b,
+ // CVE-2020-28042.
+ SignatureTruncated Kind = "signature-truncated"
+
+ // SignatureZero emits ES256 with an all-zero signature. Trap T6c, the shape
+ // of CVE-2022-21449. Measured on go1.26.5: ecdsa.Verify rejects r = s = 0, so
+ // a Go verifier is only vulnerable if it skips the check, which is what the
+ // range fixture does on purpose.
+ SignatureZero Kind = "signature-zero"
+
+ // SignatureWeakKey signs with a modulus below the size crypto/rsa accepts.
+ // Trap T6a's class, CVE-2022-23539: a key type or size not validated against
+ // the algorithm.
+ SignatureWeakKey Kind = "signature-weak-key"
+
+ // JWKSUnreachable stops serving the JWK Set. Trap T3 and taxonomy row F2,
+ // and research 07 calls it the highest-value trap in the archive because the
+ // code path is reachable only while the provider is down.
+ JWKSUnreachable Kind = "jwks-unreachable"
+
+ // KidRollover rotates the signing key and REMOVES the retired one from the
+ // JWK Set, so a token signed by it is no longer verifiable by anyone who
+ // refetches. Trap T4 and taxonomy row F3.
+ KidRollover Kind = "kid-rollover"
+
+ // KeyRotation rotates WITH an overlap window: both keys stay published and
+ // new tokens carry the new one. This is the availability mirror of
+ // KidRollover, and it is the one trap in the family where DENIED is the
+ // defect, because refusing a correctly signed current token is the failure.
+ KeyRotation Kind = "key-rotation"
+
+ // IntrospectionError answers introspection with a 503. It is deliberately not
+ // an outage: the provider is reachable and answering, so a caller's
+ // `err != nil` branch is never taken and only its status handling decides.
+ // Taxonomy row F20, where collapsing deny into unknown is the defect.
+ IntrospectionError Kind = "introspection-error"
+
+ // IntrospectionMalformed answers 200 with a body that is not JSON, so the
+ // caller's decode-error branch decides. RFC 7662 is explicit that the absence
+ // of a usable response is not `active: true`.
+ IntrospectionMalformed Kind = "introspection-malformed"
+
+ // IntrospectionSlow holds introspection past the caller's timeout while
+ // answering everything else promptly. A slow authority is not an unreachable
+ // one and a caller's code tells them apart.
+ IntrospectionSlow Kind = "introspection-slow"
+
+ // Revoke revokes a credential at a provider the target can still reach. The
+ // contrast with `identity.idp_outage` is the whole point: there the authority
+ // is unreachable, here it is answering `active: false` and the target is
+ // simply not asking.
+ Revoke Kind = "revoke"
+)
+
+const (
+ describeAlgNone = "make khaos's identity provider mint %s an unsigned token that declares `alg: none`"
+ describeAlgConfusion = "make khaos's identity provider mint %s an HS256 token keyed on its own published RSA public key, as %s bytes"
+ describeEmbeddedJWK = "make khaos's identity provider mint %s a token carrying its own signing key in the `jwk` header"
+ describeClaimExpired = "make khaos's identity provider mint %s a correctly signed token that expired %s ago"
+ describeClaimNotYetValid = "make khaos's identity provider mint %s a correctly signed token that is not valid for another %s"
+ describeClaimWrongAudience = "make khaos's identity provider mint %s a correctly signed token addressed to a different service"
+ describeClaimForeignIssuer = "make khaos's identity provider mint %s a correctly signed token from a second issuer it also publishes keys for"
+ describeSignatureTruncated = "make khaos's identity provider mint %s a token whose valid signature is cut to one byte"
+ describeSignatureZero = "make khaos's identity provider mint %s an ES256 token whose signature is all zero bytes"
+ describeSignatureWeakKey = "make khaos's identity provider mint %s a token signed with a modulus below the size crypto/rsa will accept"
+ describeJWKSUnreachable = "stop khaos's identity provider from serving its JWK Set to %s, leaving every other endpoint up"
+ describeKidRollover = "rotate khaos's identity provider's signing key and withdraw the retired one from the JWK Set %s reads"
+ describeKeyRotation = "rotate khaos's identity provider's signing key with both keys published, so %s has to pick up the new one"
+ describeIntrospectionError = "make khaos's identity provider answer %s's introspection calls with a 503 while staying reachable"
+ describeIntrospectionBad = "make khaos's identity provider answer %s's introspection calls with a 200 and a body that is not JSON"
+ describeIntrospectionSlow = "hold %s's introspection calls for %s while answering everything else promptly"
+ describeRevoke = "revoke a credential at khaos's identity provider while %s can still reach it"
+
+ errNeedsDelay = "%w: %s has no magnitude, and a hold of zero is indistinguishable from no hold at all"
+ errNeedsCredential = "%w: %s has to name the credential it revokes"
+ errUnknownTrap = "%w: %q (known traps: %s)"
+ trapSep = ", "
+)
+
+var ErrInvalidTrap = errors.New("that trap cannot be armed as specified")
+
+var ErrUnknownTrap = errors.New("khaos's identity provider has no such trap")
+
+// Trap is one armed misbehaviour and its magnitude. It is a value rather than a
+// closure so it can be journaled: the provider dies with the process, so the
+// journal record is the only durable account of what a run actually did.
+type Trap struct {
+ Kind Kind `json:"kind"`
+
+ // Delay is how long IntrospectionSlow holds a call, and how far ClaimExpired
+ // and ClaimNotYetValid move the claim they corrupt.
+ Delay time.Duration `json:"delay,omitempty"`
+
+ // Credential is the opaque token Revoke acts on.
+ Credential string `json:"credential,omitempty"`
+
+ // KeyForm is the serialization AlgConfusion keys its HMAC on. It is a
+ // parameter rather than a fixed choice because research 07 is explicit that
+ // the same verifier bug is unreachable under the wrong form: the attack has
+ // to use the exact bytes the target loaded its public key from.
+ KeyForm KeyForm `json:"key_form,omitempty"`
+}
+
+func (t Trap) Validate() error {
+ switch t.Kind {
+ case IntrospectionSlow, ClaimExpired, ClaimNotYetValid:
+ if t.Delay <= 0 {
+ return fmt.Errorf(errNeedsDelay, ErrInvalidTrap, t.Kind)
+ }
+ case Revoke:
+ if t.Credential == "" {
+ return fmt.Errorf(errNeedsCredential, ErrInvalidTrap, t.Kind)
+ }
+ case AlgConfusion:
+ if !slices.Contains(KeyForms(), string(t.KeyForm)) {
+ return fmt.Errorf(errUnknownKeyForm, ErrUnknownKeyForm,
+ t.KeyForm, SPKIPEM, SPKIDER, PKCS1PEM, Modulus)
+ }
+ }
+
+ if t.Kind != PassThrough && !slices.Contains(Kinds(), string(t.Kind)) {
+ return fmt.Errorf(errUnknownTrap, ErrUnknownTrap, t.Kind, joinKinds())
+ }
+
+ return nil
+}
+
+// Describe states the trap with its actual magnitude. The proxy tier paid for
+// this lesson: reading the kind alone and leaving the parameters at zero printed
+// a plan line that understated the fault it was asking consent for.
+func (t Trap) Describe(target string) string {
+ switch t.Kind {
+ case AlgNone:
+ return fmt.Sprintf(describeAlgNone, target)
+ case AlgConfusion:
+ return fmt.Sprintf(describeAlgConfusion, target, t.KeyForm)
+ case EmbeddedJWK:
+ return fmt.Sprintf(describeEmbeddedJWK, target)
+ case ClaimExpired:
+ return fmt.Sprintf(describeClaimExpired, target, t.Delay)
+ case ClaimNotYetValid:
+ return fmt.Sprintf(describeClaimNotYetValid, target, t.Delay)
+ case ClaimWrongAudience:
+ return fmt.Sprintf(describeClaimWrongAudience, target)
+ case ClaimForeignIssuer:
+ return fmt.Sprintf(describeClaimForeignIssuer, target)
+ case SignatureTruncated:
+ return fmt.Sprintf(describeSignatureTruncated, target)
+ case SignatureZero:
+ return fmt.Sprintf(describeSignatureZero, target)
+ case SignatureWeakKey:
+ return fmt.Sprintf(describeSignatureWeakKey, target)
+ case JWKSUnreachable:
+ return fmt.Sprintf(describeJWKSUnreachable, target)
+ case KidRollover:
+ return fmt.Sprintf(describeKidRollover, target)
+ case KeyRotation:
+ return fmt.Sprintf(describeKeyRotation, target)
+ case IntrospectionError:
+ return fmt.Sprintf(describeIntrospectionError, target)
+ case IntrospectionMalformed:
+ return fmt.Sprintf(describeIntrospectionBad, target)
+ case IntrospectionSlow:
+ return fmt.Sprintf(describeIntrospectionSlow, target, t.Delay)
+ case Revoke:
+ return fmt.Sprintf(describeRevoke, target)
+ default:
+ return string(t.Kind)
+ }
+}
+
+// Forgery reports whether the trap corrupts what the provider MINTS, as opposed
+// to what it serves or how it answers. It decides which probe stream an
+// experiment needs, and it is derived here so an experiment cannot disagree with
+// the trap about its own shape.
+func (t Trap) Forgery() bool {
+ switch t.Kind {
+ case AlgNone, AlgConfusion, EmbeddedJWK,
+ ClaimExpired, ClaimNotYetValid, ClaimWrongAudience, ClaimForeignIssuer,
+ SignatureTruncated, SignatureZero, SignatureWeakKey:
+ return true
+ default:
+ return false
+ }
+}
+
+// Kinds is every trap an operator can arm, in a stable order.
+func Kinds() []string {
+ return []string{
+ string(AlgNone),
+ string(AlgConfusion),
+ string(EmbeddedJWK),
+ string(ClaimExpired),
+ string(ClaimNotYetValid),
+ string(ClaimWrongAudience),
+ string(ClaimForeignIssuer),
+ string(SignatureTruncated),
+ string(SignatureZero),
+ string(SignatureWeakKey),
+ string(JWKSUnreachable),
+ string(KidRollover),
+ string(KeyRotation),
+ string(IntrospectionError),
+ string(IntrospectionMalformed),
+ string(IntrospectionSlow),
+ string(Revoke),
+ }
+}
+
+func joinKinds() string {
+ kinds := Kinds()
+ out := kinds[0]
+ for _, kind := range kinds[1:] {
+ out += trapSep + kind
+ }
+ return out
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault.go
new file mode 100644
index 00000000..f3248ca3
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault.go
@@ -0,0 +1,139 @@
+// ©AngelaMos | 2026
+// idpfault.go
+
+package inject
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const (
+ KindIdPPrefix = "idp-"
+
+ errNoIdP = "%w: %s needs khaos's own identity provider, and this run did not start one"
+ errWrongTrapKind = "%w: this run armed %q but the mutation is %q"
+
+ describeIdP = "%s, at khaos's own identity provider, which holds no capabilities and serves nothing but the target under test"
+)
+
+var ErrNoIdP = errors.New("this mutation needs khaos's identity provider, which is not running")
+
+var ErrTrapMismatch = errors.New("the armed trap does not match the mutation being applied")
+
+// IdPKind names the mutation that arms one trap on khaos's identity provider.
+// The trap kinds and the mutation kinds are one-to-one, so `khaos catalog` names
+// the actual defect being provoked rather than a generic "identity provider"
+// entry.
+func IdPKind(kind idp.Kind) string {
+ return KindIdPPrefix + string(kind)
+}
+
+// idpMutator arms a trap on khaos's in-process identity provider. This is a
+// THIRD injection tier alongside the Docker-native one and the fault proxy, and
+// it exists because the other two cannot express the fault at all: pausing a
+// provider or severing the path to it makes it unreachable, where every trap in
+// this tier leaves it REACHABLE and wrong. A verifier's behaviour on an
+// unreachable authority and its behaviour on a lying one are different code
+// paths reached by different bugs.
+type idpMutator struct {
+ kind idp.Kind
+}
+
+func init() {
+ for _, name := range idp.Kinds() {
+ kind := idp.Kind(name)
+ registry[IdPKind(kind)] = idpMutator{kind: kind}
+ }
+}
+
+func (m idpMutator) Describe(env Env, target dockerx.Target) string {
+ trap := env.IdPTrap
+ if trap.Kind != m.kind {
+ trap = idp.Trap{Kind: m.kind}
+ }
+
+ return fmt.Sprintf(describeIdP, trap.Describe(target.Name))
+}
+
+// Plan journals the trap that is about to be armed, and resolves the one piece
+// of it that cannot be known at build time.
+//
+// The revocation's credential is issued by a PROBE, and no probe has fired when
+// the experiment is built, so the trap travels without it and picks it up here.
+// Inject runs after the baseline, by which time the credential exists and the
+// target has been honouring it, which is exactly the state the finding needs.
+func (m idpMutator) Plan(_ context.Context, env Env, _ dockerx.Target) (json.RawMessage, error) {
+ if env.IdP == nil {
+ return nil, fmt.Errorf(errNoIdP, ErrNoIdP, IdPKind(m.kind))
+ }
+
+ if env.IdPTrap.Kind != m.kind {
+ return nil, fmt.Errorf(errWrongTrapKind, ErrTrapMismatch, env.IdPTrap.Kind, m.kind)
+ }
+
+ trap := env.IdPTrap
+
+ if trap.Kind == idp.Revoke && trap.Credential == "" {
+ credential, err := env.IdP.RevocableCredential()
+ if err != nil {
+ return nil, err
+ }
+ trap.Credential = credential
+ }
+
+ if err := trap.Validate(); err != nil {
+ return nil, err
+ }
+
+ return journal.EncodePayload(trap)
+}
+
+func (m idpMutator) Apply(_ context.Context, env Env, _ dockerx.Target, raw json.RawMessage) error {
+ if env.IdP == nil {
+ return fmt.Errorf(errNoIdP, ErrNoIdP, IdPKind(m.kind))
+ }
+
+ var trap idp.Trap
+ if err := json.Unmarshal(raw, &trap); err != nil {
+ return err
+ }
+
+ return env.IdP.Arm(trap)
+}
+
+// Revert disarms, which also restores any key material or grant the trap
+// changed. A cold start has no provider to disarm and that is RESOLVED rather
+// than failed: the provider died with the process that held it, so the fault
+// provably went with it. Reporting that as a failure would wedge the journal
+// record forever and, because `khaos reap` stops on a sweep error, would disable
+// the dead man's switch for every other fault on the machine.
+func (m idpMutator) Revert(_ context.Context, env Env, _ journal.Record) error {
+ if env.IdP == nil {
+ return nil
+ }
+
+ env.IdP.Disarm()
+
+ return nil
+}
+
+// Confirm delegates to the provider's own read-back, which measures the trap
+// through the provider's public surface against a control and returns the
+// specific observable it saw. It deliberately does NOT read the flag that was
+// set: a flag records which branch was chosen, not what a verifier would
+// actually receive, and a provider whose minting broke would confirm a trap it
+// is not producing.
+func (m idpMutator) Confirm(ctx context.Context, env Env, _ dockerx.Target) (string, bool, error) {
+ if env.IdP == nil {
+ return "", false, fmt.Errorf(errNoIdP, ErrNoIdP, IdPKind(m.kind))
+ }
+
+ return env.IdP.Observe(ctx, m.kind)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault_test.go
new file mode 100644
index 00000000..071e54a3
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault_test.go
@@ -0,0 +1,246 @@
+// ©AngelaMos | 2026
+// idpfault_test.go
+
+package inject
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const idpTestDelay = 300 * time.Millisecond
+
+func traps(p *idp.Provider) []idp.Trap {
+ return []idp.Trap{
+ {Kind: idp.AlgNone},
+ {Kind: idp.AlgConfusion, KeyForm: idp.SPKIPEM},
+ {Kind: idp.EmbeddedJWK},
+ {Kind: idp.ClaimExpired, Delay: time.Hour},
+ {Kind: idp.ClaimNotYetValid, Delay: time.Hour},
+ {Kind: idp.ClaimWrongAudience},
+ {Kind: idp.ClaimForeignIssuer},
+ {Kind: idp.SignatureTruncated},
+ {Kind: idp.SignatureZero},
+ {Kind: idp.SignatureWeakKey},
+ {Kind: idp.JWKSUnreachable},
+ {Kind: idp.KidRollover},
+ {Kind: idp.KeyRotation},
+ {Kind: idp.IntrospectionError},
+ {Kind: idp.IntrospectionMalformed},
+ {Kind: idp.IntrospectionSlow, Delay: idpTestDelay},
+ {Kind: idp.Revoke, Credential: p.Revocable()},
+ }
+}
+
+// The whole mutation, end to end, for every registered identity trap: journal
+// it, arm it, confirm it by read-back on the provider's own surface, revert it,
+// and prove the provider is correct again afterwards.
+//
+// The census is counted by the identity prefix specifically, because counting
+// "not direct" would let the proxy tier's registrations satisfy it.
+func TestEveryIdentityTrapAppliesConfirmsAndReverts(t *testing.T) {
+ provider := provider(t)
+ all := traps(provider)
+
+ registered := 0
+ for _, kind := range Kinds() {
+ if strings.HasPrefix(kind, KindIdPPrefix) {
+ registered++
+ }
+ }
+
+ if registered != len(all) {
+ t.Fatalf("this test covers %d identity traps but %d are registered", len(all), registered)
+ }
+
+ ctx := context.Background()
+ target := dockerx.Target{ID: "abc", Name: "a-verifier"}
+
+ for _, trap := range all {
+ t.Run(string(trap.Kind), func(t *testing.T) {
+ env := Env{IdP: provider, IdPTrap: trap}
+
+ mutator, err := Lookup(IdPKind(trap.Kind))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ if describe := mutator.Describe(env, target); !strings.Contains(describe, target.Name) {
+ t.Errorf("the plan line does not name the target: %s", describe)
+ }
+
+ payload, err := mutator.Plan(ctx, env, target)
+ if err != nil {
+ t.Fatalf("plan: %v", err)
+ }
+
+ if err := mutator.Apply(ctx, env, target, payload); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+
+ observable, confirmed, err := mutator.Confirm(ctx, env, target)
+ if err != nil || !confirmed {
+ t.Fatalf("confirm said %v (%v) with %s armed", confirmed, err, trap.Kind)
+ }
+
+ if observable == "" {
+ t.Fatal("confirmed with no observable, so the run record would carry no evidence")
+ }
+
+ if err := mutator.Revert(ctx, env, journal.Record{}); err != nil {
+ t.Fatalf("revert: %v", err)
+ }
+
+ if _, confirmed, _ := mutator.Confirm(ctx, env, target); confirmed {
+ t.Fatal("the trap still confirms itself after being reverted")
+ }
+ })
+ }
+}
+
+// The journal is the only durable account of what an identity run did, because
+// the provider dies with the process. A payload that does not round-trip means a
+// process killed mid-apply leaves a record that cannot describe the fault.
+func TestTheJournalledTrapRoundTrips(t *testing.T) {
+ provider := provider(t)
+
+ for _, trap := range traps(provider) {
+ env := Env{IdP: provider, IdPTrap: trap}
+
+ mutator, err := Lookup(IdPKind(trap.Kind))
+ if err != nil {
+ t.Fatalf("lookup %s: %v", trap.Kind, err)
+ }
+
+ payload, err := mutator.Plan(context.Background(), env, dockerx.Target{})
+ if err != nil {
+ t.Fatalf("plan %s: %v", trap.Kind, err)
+ }
+
+ record := journal.Record{Payload: payload}
+
+ var back idp.Trap
+ if err := record.DecodePayload(&back); err != nil {
+ t.Fatalf("decode %s: %v", trap.Kind, err)
+ }
+
+ if back.Kind != trap.Kind || back.Delay != trap.Delay || back.KeyForm != trap.KeyForm {
+ t.Errorf("%s did not round-trip: got %+v want %+v", trap.Kind, back, trap)
+ }
+ }
+}
+
+// A mutation whose run never started a provider has to refuse by name rather
+// than arm nothing and report success.
+func TestApplyingAnIdentityTrapWithNoProviderIsRefused(t *testing.T) {
+ mutator, err := Lookup(IdPKind(idp.AlgNone))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ if _, err := mutator.Plan(context.Background(), Env{}, dockerx.Target{}); err == nil {
+ t.Fatal("planning an identity trap with no provider succeeded")
+ }
+
+ if err := mutator.Apply(context.Background(), Env{}, dockerx.Target{}, nil); err == nil {
+ t.Fatal("applying an identity trap with no provider succeeded")
+ }
+
+ if _, confirmed, err := mutator.Confirm(context.Background(), Env{}, dockerx.Target{}); confirmed || err == nil {
+ t.Fatal("confirming an identity trap with no provider succeeded")
+ }
+}
+
+// Reverting on a cold start must RESOLVE. The provider died with the process
+// that held it, so the fault provably went with it, and reporting a failure
+// would wedge the journal record forever and stop `khaos reap` sweeping anything
+// else on the machine.
+func TestRevertingWithoutAProviderIsResolvedRatherThanFailed(t *testing.T) {
+ mutator, err := Lookup(IdPKind(idp.AlgNone))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ if err := mutator.Revert(context.Background(), Env{}, journal.Record{}); err != nil {
+ t.Fatalf("revert on a cold start returned %v; it must resolve", err)
+ }
+}
+
+// A run that armed one trap must not be able to apply a different one. Without
+// this, the trap an operator consented to on the plan line and the trap that
+// actually lands can differ.
+func TestAMutationRefusesATrapTheRunDidNotArm(t *testing.T) {
+ provider := provider(t)
+
+ mutator, err := Lookup(IdPKind(idp.AlgNone))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ env := Env{IdP: provider, IdPTrap: idp.Trap{Kind: idp.SignatureTruncated}}
+
+ if _, err := mutator.Plan(context.Background(), Env(env), dockerx.Target{}); err == nil {
+ t.Fatal("a run holding signature-truncated planned an alg-none mutation")
+ }
+}
+
+// The revocation's credential cannot be known when the experiment is built, so
+// the trap travels without it. A run whose probe never issued one has nothing to
+// revoke, and that must be a refusal rather than a revocation of nothing.
+func TestRevokeResolvesItsCredentialAtInjectTime(t *testing.T) {
+ fresh, err := idp.New(idp.Config{})
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ mutator, err := Lookup(IdPKind(idp.Revoke))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ bare := Env{IdP: fresh, IdPTrap: idp.Trap{Kind: idp.Revoke}}
+
+ if _, err := mutator.Plan(context.Background(), bare, dockerx.Target{}); err == nil {
+ t.Fatal("planning a revocation succeeded before any credential was issued")
+ }
+
+ credential := fresh.Revocable()
+
+ payload, err := mutator.Plan(context.Background(), bare, dockerx.Target{})
+ if err != nil {
+ t.Fatalf("plan after issuing: %v", err)
+ }
+
+ var resolved idp.Trap
+ if err := (journal.Record{Payload: payload}).DecodePayload(&resolved); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+
+ if resolved.Credential != credential {
+ t.Fatalf("the journalled trap revokes %q, want the credential the probe issued, %q",
+ resolved.Credential, credential)
+ }
+}
+
+func provider(t *testing.T) *idp.Provider {
+ t.Helper()
+
+ p, err := idp.New(idp.Config{})
+ if err != nil {
+ t.Fatalf("build a provider: %v", err)
+ }
+
+ if err := p.Listen("127.0.0.1:0"); err != nil {
+ t.Fatalf("bind a provider: %v", err)
+ }
+
+ t.Cleanup(func() { _ = p.Close() })
+
+ return p
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/deadlock_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/deadlock_test.go
new file mode 100644
index 00000000..575f93f2
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/deadlock_test.go
@@ -0,0 +1,54 @@
+package proxy
+
+import (
+ "io"
+ "net"
+ "testing"
+ "time"
+)
+
+// A dependency that accepts and never answers. This is what research 06 section
+// 6.1 measured for both `docker pause` and `docker network disconnect`: a silent
+// hang, no RST, no FIN, no error.
+func TestCloseDoesNotDeadlockAgainstAWedgedDependency(t *testing.T) {
+ wedged, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+ defer func() { _ = wedged.Close() }()
+
+ go func() {
+ for {
+ conn, err := wedged.Accept()
+ if err != nil {
+ return
+ }
+ _ = conn
+ }
+ }()
+
+ p, err := Open("127.0.0.1:0", wedged.Addr().String())
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+
+ client, err := net.Dial("tcp", p.Addr())
+ if err != nil {
+ t.Fatalf("dial: %v", err)
+ }
+ defer func() { _ = client.Close() }()
+
+ _, _ = io.WriteString(client, "GET / HTTP/1.1\r\nHost: x\r\n\r\n")
+ time.Sleep(300 * time.Millisecond)
+
+ done := make(chan error, 1)
+ go func() { done <- p.Close() }()
+
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("Close() did not return within 5s against a dependency that accepts and never " +
+ "answers. khaos run defers this Close, so the whole command hangs with a fault's " +
+ "listener still bound, and the operator reaches for kill -9")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/fault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/fault.go
new file mode 100644
index 00000000..eef782f9
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/fault.go
@@ -0,0 +1,157 @@
+// ©AngelaMos | 2026
+// fault.go
+
+package proxy
+
+import (
+ "errors"
+ "fmt"
+ "slices"
+ "strings"
+ "time"
+)
+
+type Kind string
+
+const (
+ // PassThrough is not a fault. It is what the proxy does when nothing is
+ // armed, and it has to be a real mode rather than the absence of one,
+ // because the baseline is measured through the proxy and a baseline
+ // measured through a different code path than the fault window proves
+ // nothing about the difference between them.
+ PassThrough Kind = "pass-through"
+
+ // Hang accepts the connection and never answers. The research found this
+ // and Reset to be the two shapes that most reliably separate a fail-closed
+ // implementation from a fail-open one, because a refused connection and a
+ // hung one hit different branches in almost every HTTP client, and the hang
+ // branch is the one carrying the badly-chosen default timeout.
+ Hang Kind = "hang"
+
+ // Reset answers with RST rather than FIN. Reaching this needs SO_LINGER 0,
+ // because an ordinary close with an empty receive queue sends FIN.
+ Reset Kind = "reset"
+
+ // EOF accepts and closes cleanly, which is what a stopped or killed
+ // dependency looks like to its peer. Measured in research 06 section 6.1:
+ // even SIGKILL produces an orderly FIN.
+ EOF Kind = "eof"
+
+ // Latency delays both directions. It is the graduated fault, for finding a
+ // timeout that is too generous rather than one that is missing.
+ Latency Kind = "latency"
+
+ // Partial forwards a prefix of the dependency's answer and then closes, so
+ // the target receives a truncated body with a valid status line. This is the
+ // shape that finds a parser treating a short read as an empty-but-successful
+ // response, which an outage-shaped fault never reaches.
+ Partial Kind = "partial"
+
+ // SlowClose forwards the whole answer and then holds the connection open.
+ // A client that waits for close rather than for Content-Length stalls.
+ SlowClose Kind = "slow-close"
+)
+
+const (
+ errUnknownKind = "%w: %q (known proxy faults: %s)"
+ errNeedDelay = "%w: %s needs a positive delay"
+ errNeedBytes = "%w: %s needs a positive byte count"
+
+ kindSep = ", "
+
+ describePassThrough = "forward every byte between %s and its dependency, unchanged"
+ describeHang = "accept the target's connections to its dependency and never answer them"
+ describeReset = "answer the target's connections to its dependency with an immediate RST"
+ describeEOF = "accept the target's connections to its dependency and close them cleanly"
+ describeLatency = "delay every byte between %s and its dependency by %s in each direction"
+ describePartial = "forward only the first %d bytes of the dependency's answer, then close"
+ describeSlowClose = "forward the dependency's answer, then hold the connection open for %s"
+)
+
+var ErrUnknownKind = errors.New("no proxy fault is registered under that name")
+
+var ErrFaultIncomplete = errors.New("that proxy fault needs a parameter it was not given")
+
+// Fault is the whole mutable state of an armed proxy. Reverting is assigning
+// PassThrough over it, which is why the tier rolls back by freeing a struct
+// rather than by undoing anything.
+type Fault struct {
+ Kind Kind
+ Delay time.Duration
+ Bytes int
+}
+
+func Kinds() []string {
+ kinds := []string{
+ string(Hang), string(Reset), string(EOF),
+ string(Latency), string(Partial), string(SlowClose),
+ }
+ slices.Sort(kinds)
+ return kinds
+}
+
+func ParseKind(name string) (Kind, error) {
+ kind := Kind(name)
+ if kind == PassThrough || !slices.Contains(Kinds(), name) {
+ return "", fmt.Errorf(errUnknownKind, ErrUnknownKind, name, strings.Join(Kinds(), kindSep))
+ }
+ return kind, nil
+}
+
+// Validate refuses a fault that cannot express itself. A latency fault with no
+// delay and a partial fault with no byte count are both indistinguishable from
+// pass-through at runtime, which is a run that measures nothing and reports a
+// clean survival.
+func (f Fault) Validate() error {
+ switch f.Kind {
+ case Latency, SlowClose:
+ if f.Delay <= 0 {
+ return fmt.Errorf(errNeedDelay, ErrFaultIncomplete, f.Kind)
+ }
+ case Partial:
+ if f.Bytes <= 0 {
+ return fmt.Errorf(errNeedBytes, ErrFaultIncomplete, f.Kind)
+ }
+ case Hang, Reset, EOF:
+ default:
+ // The whitelist is the point. A switch that only names the parameterised
+ // kinds accepts the zero Fault, a misspelling and a trailing space, all of
+ // which forward every byte untouched while reporting as armed. An
+ // unmarshalled journal payload reaches this with no other guard.
+ return fmt.Errorf(errUnknownKind, ErrUnknownKind, f.Kind, strings.Join(Kinds(), kindSep))
+ }
+
+ return nil
+}
+
+func (f Fault) Describe(target string) string {
+ switch f.Kind {
+ case Hang:
+ return describeHang
+ case Reset:
+ return describeReset
+ case EOF:
+ return describeEOF
+ case Latency:
+ return fmt.Sprintf(describeLatency, target, f.Delay)
+ case Partial:
+ return fmt.Sprintf(describePartial, f.Bytes)
+ case SlowClose:
+ return fmt.Sprintf(describeSlowClose, f.Delay)
+ default:
+ return fmt.Sprintf(describePassThrough, target)
+ }
+}
+
+// Severs reports whether the fault prevents the dependency from being reached at
+// all. khaos confirms a severing fault by proving the dependency is unreachable
+// THROUGH the proxy, and a forwarding fault by proving the proxy altered the
+// stream, because those are different observations and one cannot stand in for
+// the other.
+func (f Fault) Severs() bool {
+ return f.Kind == Hang || f.Kind == Reset || f.Kind == EOF
+}
+
+func sprintf(format string, args ...any) string {
+ return fmt.Sprintf(format, args...)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe.go
new file mode 100644
index 00000000..86d67328
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe.go
@@ -0,0 +1,191 @@
+// ©AngelaMos | 2026
+// observe.go
+
+package proxy
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "syscall"
+ "time"
+)
+
+const (
+ observeGrace = 2 * time.Second
+ observeBuffer = 64 * 1024
+
+ observeDelayFactor = 4
+
+ // Slack on the measured magnitude. The thresholds are lower bounds with no
+ // tolerance otherwise, and on a host that also serves real traffic CPU
+ // contention routinely shaves a few percent off a sleep. Measured: under
+ // 64-goroutine load a 250ms slow-close refused 6 of 25 times, voiding a
+ // quarter of its runs.
+ observeSlackNumerator = 9
+ observeSlackDenominator = 10
+
+ errNoControl = "%w: %v"
+)
+
+var ErrNothingArmed = errors.New("nothing is armed on the fault proxy, so there is nothing to confirm")
+
+var ErrDependencyUnreachable = errors.New("khaos cannot reach the dependency directly, so it cannot attribute what it sees through the proxy to its own fault")
+
+var ErrFaultNotObserved = errors.New("the armed fault did not produce its own observable")
+
+// reading is what khaos's confirmation probe measured on one path.
+type reading struct {
+ firstByte time.Duration
+ total int64
+ closed time.Duration
+}
+
+// Observe confirms the armed fault by measuring the dependency on TWO paths and
+// requiring them to disagree: once directly, bypassing the proxy, and once
+// through it. It never consults an internal flag or a branch counter, because a
+// counter incremented by the dispatcher records which code path was chosen rather
+// than what reached the wire.
+//
+// The differential is what makes the oracle sound. An absolute threshold cannot
+// tell khaos's fault from a slow dependency: measured, a dependency answering in
+// four seconds confirmed a `hang` that was not applied at all, with the target
+// served in full, and a dependency taking one second confirmed a `latency` whose
+// delay had been removed. The direct path is the control, and the design contract
+// calls for read-back rather than an API return value.
+func (p *Proxy) Observe(ctx context.Context, expected Kind) (string, bool, error) {
+ // Closed is checked FIRST. Close reverts to pass-through, so reading the fault
+ // before this would report "nothing armed" for a proxy that is gone, and a
+ // dial to a listener that no longer exists is ECONNREFUSED, which a reset
+ // oracle must not accept as its own observable.
+ select {
+ case <-p.closed:
+ return "", false, ErrClosed
+ default:
+ }
+
+ fault := p.Fault()
+
+ if fault.Kind == PassThrough {
+ return "", false, ErrNothingArmed
+ }
+
+ if expected != "" && fault.Kind != expected {
+ return "", false, fmt.Errorf(errWrongArmed, ErrFaultNotObserved, fault.Kind, expected)
+ }
+
+ p.probes.Add(1)
+
+ budget := observeGrace + fault.Delay*observeDelayFactor
+
+ // The control first. If the dependency cannot be reached without the proxy in
+ // the way, nothing seen through the proxy can be attributed to khaos's fault,
+ // and saying so voids the run instead of crediting khaos for someone else's
+ // outage.
+ // A complete answer always ends in an error, because that is what the peer
+ // closing looks like to a reader. Only the absence of BYTES means khaos could
+ // not reach the dependency.
+ direct, directErr := p.measure(ctx, p.upstream, observeGrace)
+ if direct.total == 0 {
+ return "", false, fmt.Errorf(errNoControl, ErrDependencyUnreachable, directErr)
+ }
+
+ through, throughErr := p.measure(ctx, p.Addr(), budget)
+
+ switch fault.Kind {
+ case Hang:
+ // The dependency answered on the control path and answered nothing here.
+ if through.total == 0 && isTimeout(throughErr) {
+ return observedHang, true, nil
+ }
+ case Reset:
+ // ECONNREFUSED is deliberately NOT accepted. Dialling a listener that is
+ // gone produces exactly that, so accepting it let a CLOSED proxy confirm
+ // its own fault.
+ if isReset(throughErr) {
+ return observedReset, true, nil
+ }
+ case EOF:
+ if through.total == 0 && errors.Is(throughErr, io.EOF) {
+ return observedEOF, true, nil
+ }
+ case Latency:
+ added := through.firstByte - direct.firstByte
+ if added >= slack(fault.Delay) {
+ return sprintf(observedLatency, added.Round(time.Millisecond)), true, nil
+ }
+ case Partial:
+ // Both halves. A byte count that merely MATCHES the budget is also what
+ // pass-through delivers when the answer happens to be that long, so the
+ // control's answer has to be longer than what came through.
+ if through.total == int64(fault.Bytes) && direct.total > through.total {
+ return sprintf(observedPartial, through.total, direct.total), true, nil
+ }
+ case SlowClose:
+ held := through.closed - through.firstByte
+ reference := direct.closed - direct.firstByte
+ if held-reference >= slack(fault.Delay) {
+ return sprintf(observedSlowClose, held.Round(time.Millisecond)), true, nil
+ }
+ }
+
+ return "", false, nil
+}
+
+func slack(delay time.Duration) time.Duration {
+ return delay * observeSlackNumerator / observeSlackDenominator
+}
+
+// measure sends one probe to an address and records when the first byte arrived,
+// how much arrived in total, and when the peer closed.
+func (p *Proxy) measure(ctx context.Context, addr string, budget time.Duration) (reading, error) {
+ dialer := net.Dialer{Timeout: budget}
+
+ conn, err := dialer.DialContext(ctx, network, addr)
+ if err != nil {
+ return reading{}, err
+ }
+ defer func() { _ = conn.Close() }()
+
+ if err := conn.SetDeadline(time.Now().Add(budget)); err != nil {
+ return reading{}, err
+ }
+
+ started := time.Now()
+
+ if _, err := io.WriteString(conn, probePayload); err != nil {
+ return reading{}, err
+ }
+
+ out := reading{}
+ buffer := make([]byte, observeBuffer)
+
+ for {
+ read, err := conn.Read(buffer)
+ if read > 0 {
+ if out.total == 0 {
+ out.firstByte = time.Since(started)
+ }
+ out.total += int64(read)
+ out.closed = time.Since(started)
+ }
+ if err != nil {
+ out.closed = time.Since(started)
+ return out, err
+ }
+ }
+}
+
+// isReset accepts ECONNRESET and deliberately NOT ECONNREFUSED. Dialling a
+// listener that is gone produces refused, and accepting it let a CLOSED proxy
+// confirm the fault it was holding when it died.
+func isReset(err error) bool {
+ return errors.Is(err, syscall.ECONNRESET)
+}
+
+func isTimeout(err error) bool {
+ var timeout net.Error
+ return errors.As(err, &timeout) && timeout.Timeout()
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe_test.go
new file mode 100644
index 00000000..12a785d0
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/observe_test.go
@@ -0,0 +1,319 @@
+// ©AngelaMos | 2026
+// observe_test.go
+
+package proxy
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+)
+
+func armable() []Fault {
+ return []Fault{
+ {Kind: Hang},
+ {Kind: Reset},
+ {Kind: EOF},
+ {Kind: Latency, Delay: 300 * time.Millisecond},
+ {Kind: Partial, Bytes: 9},
+ {Kind: SlowClose, Delay: 300 * time.Millisecond},
+ }
+}
+
+// The failure this replaced: confirmation compared branch counters, so it
+// reported a latency fault confirmed while the proxy was in pass-through and
+// nothing was armed at all.
+func TestObserveRefusesToConfirmWhenNothingIsArmed(t *testing.T) {
+ p, _ := through(t)
+
+ method, confirmed, err := p.Observe(context.Background(), "")
+
+ if confirmed {
+ t.Fatalf("a pass-through proxy confirmed a fault (%q)", method)
+ }
+
+ if !errors.Is(err, ErrNothingArmed) {
+ t.Fatalf("err = %v, want ErrNothingArmed", err)
+ }
+}
+
+func TestObserveConfirmsEveryFaultAndStopsAfterTheRevert(t *testing.T) {
+ if len(armable()) != len(Kinds()) {
+ t.Fatalf("this test covers %d faults but %d are registered", len(armable()), len(Kinds()))
+ }
+
+ for _, fault := range armable() {
+ t.Run(string(fault.Kind), func(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(fault); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ method, confirmed, err := p.Observe(context.Background(), fault.Kind)
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+ if !confirmed {
+ t.Fatalf("%s was armed and the proxy could not observe it; a fault khaos "+
+ "cannot confirm voids the run", fault.Kind)
+ }
+ if method == "" {
+ t.Fatal("the confirmation reported no method, so the run record cannot say how")
+ }
+
+ p.Disarm()
+
+ if _, confirmed, _ := p.Observe(context.Background(), ""); confirmed {
+ t.Fatalf("%s still confirmed after the revert", fault.Kind)
+ }
+ })
+ }
+}
+
+// THE negative control for the oracle. Every other test asserts confirmation
+// SUCCEEDS for a correctly armed fault, which a confirmation gutted to
+// `return true` also satisfies — measured, that gut left all seventeen packages
+// and the Docker gate green. This arms each fault and makes the proxy forward
+// anyway, so confirmation MUST refuse.
+func TestObserveRefusesAFaultThatIsArmedButDoesNotHappen(t *testing.T) {
+ for _, fault := range armable() {
+ t.Run(string(fault.Kind), func(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(fault); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ p.inert.Store(true)
+
+ method, confirmed, _ := p.Observe(context.Background(), fault.Kind)
+
+ if confirmed {
+ t.Fatalf("%s was confirmed (%q) while the proxy forwarded normally. The oracle "+
+ "cannot tell an applied fault from an unapplied one, so a run would record "+
+ "that the fault landed, the probes would see a healthy dependency, and the "+
+ "verdict would be a clean survival", fault.Kind, method)
+ }
+ })
+ }
+}
+
+// An absolute threshold cannot tell khaos's fault from a slow dependency. The
+// oracle measures the dependency on both paths and requires them to disagree.
+func TestASlowDependencyDoesNotConfirmAFaultThatIsNotHappening(t *testing.T) {
+ const dependencyDelay = 1200 * time.Millisecond
+
+ slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ time.Sleep(dependencyDelay)
+ _, _ = io.WriteString(w, body)
+ }))
+ defer slow.Close()
+
+ for _, fault := range []Fault{
+ {Kind: Hang},
+ {Kind: Latency, Delay: 300 * time.Millisecond},
+ } {
+ t.Run(string(fault.Kind), func(t *testing.T) {
+ p, err := Open(loopback, strings.TrimPrefix(slow.URL, "http://"))
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ defer func() { _ = p.Close() }()
+
+ if err := p.Arm(fault); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ p.inert.Store(true)
+
+ _, confirmed, _ := p.Observe(context.Background(), fault.Kind)
+
+ if confirmed {
+ t.Fatalf("a dependency that takes %s to answer confirmed %s while the proxy "+
+ "forwarded it untouched; khaos would credit itself for someone else's "+
+ "slowness", dependencyDelay, fault.Kind)
+ }
+ })
+ }
+}
+
+// A dependency khaos cannot reach at all is not a confirmation, it is an
+// unattributable observation, and saying so voids the run instead of crediting
+// khaos for an outage it did not cause.
+func TestAnUnreachableDependencyCannotConfirmAnything(t *testing.T) {
+ p, err := Open(loopback, "127.0.0.1:1")
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ defer func() { _ = p.Close() }()
+
+ for _, fault := range armable() {
+ t.Run(string(fault.Kind), func(t *testing.T) {
+ if err := p.Arm(fault); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ _, confirmed, err := p.Observe(context.Background(), fault.Kind)
+
+ if confirmed {
+ t.Fatalf("%s was confirmed against a dependency khaos cannot reach", fault.Kind)
+ }
+
+ if !errors.Is(err, ErrDependencyUnreachable) {
+ t.Fatalf("err = %v, want ErrDependencyUnreachable so the operator learns why "+
+ "the run is void", err)
+ }
+ })
+ }
+}
+
+// A proxy that no longer exists must not confirm its own fault. Dialling a closed
+// listener gives ECONNREFUSED, and accepting that as a reset let a dead proxy
+// report the fault it was holding when it died.
+func TestAClosedProxyDoesNotConfirmItsFault(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Reset}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ if err := p.Close(); err != nil {
+ t.Fatalf("close: %v", err)
+ }
+
+ method, confirmed, err := p.Observe(context.Background(), Reset)
+
+ if confirmed {
+ t.Fatalf("a closed proxy confirmed %q", method)
+ }
+
+ if !errors.Is(err, ErrClosed) {
+ t.Fatalf("err = %v, want ErrClosed", err)
+ }
+}
+
+// A byte count that merely MATCHES the budget is what pass-through delivers when
+// the answer happens to be that long. Truncation means the answer was CUT.
+func TestPartialDoesNotConfirmWhenNothingWasTruncated(t *testing.T) {
+ const answer = "exactly-this"
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Length", fmt.Sprint(len(answer)))
+ _, _ = io.WriteString(w, answer)
+ }))
+ defer server.Close()
+
+ p, err := Open(loopback, strings.TrimPrefix(server.URL, "http://"))
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ defer func() { _ = p.Close() }()
+
+ // A complete answer always ends in an error, because that is what the peer
+ // closing looks like to a reader. The byte count is the measurement.
+ full, _ := p.measure(context.Background(), p.upstream, observeGrace)
+ if full.total == 0 {
+ t.Fatal("the control path measured nothing")
+ }
+
+ // A budget equal to the dependency's whole answer truncates nothing.
+ if err := p.Arm(Fault{Kind: Partial, Bytes: int(full.total)}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ _, confirmed, _ := p.Observe(context.Background(), Partial)
+
+ if confirmed {
+ t.Fatalf("a budget of %d bytes against a %d byte answer confirmed truncation; the "+
+ "target received the dependency's complete answer", full.total, full.total)
+ }
+}
+
+// Confirmation must respect cancellation. A long delay otherwise blocks khaos
+// uninterruptibly, and the design doc is explicit that a tool which LOOKS hung is
+// what makes an operator reach for kill -9.
+func TestObserveHonoursCancellation(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Hang}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ started := time.Now()
+ _, confirmed, _ := p.Observe(ctx, Hang)
+ took := time.Since(started)
+
+ if confirmed {
+ t.Fatal("a cancelled confirmation still reported a verdict")
+ }
+
+ if took >= observeGrace {
+ t.Fatalf("a cancelled confirmation took %s; it ignored the context", took)
+ }
+}
+
+// The oracle must also catch the wrong fault being armed, which is the other
+// thing a confirmation step is for.
+func TestObserveRefusesWhenADifferentFaultIsArmed(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Reset}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ _, confirmed, err := p.Observe(context.Background(), Hang)
+
+ if confirmed {
+ t.Fatal("the oracle confirmed a hang while a reset was armed")
+ }
+
+ if !errors.Is(err, ErrFaultNotObserved) {
+ t.Fatalf("err = %v, want ErrFaultNotObserved", err)
+ }
+}
+
+// Probe traffic is khaos's own and must not be reported as the target's.
+func TestConfirmationTrafficIsCountedSeparately(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Hang}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ before := p.Counts()
+
+ if _, _, err := p.Observe(context.Background(), Hang); err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+
+ if after := p.Counts(); after.Probes <= before.Probes {
+ t.Fatal("a confirmation dial was not recorded as khaos's own probe, so the run record " +
+ "reports it as the target's traffic")
+ }
+}
+
+func TestMeasureReadsBothPathsOfTheDifferential(t *testing.T) {
+ p, _ := through(t)
+
+ direct, _ := p.measure(context.Background(), p.upstream, observeGrace)
+ if direct.total == 0 {
+ t.Fatal("the control path measured nothing, so no differential is possible")
+ }
+
+ proxied, _ := p.measure(context.Background(), p.Addr(), observeGrace)
+
+ if proxied.total != direct.total {
+ t.Fatalf("pass-through delivered %d bytes where the dependency answers %d",
+ proxied.total, direct.total)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
new file mode 100644
index 00000000..f8cfbc16
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
@@ -0,0 +1,642 @@
+// ©AngelaMos | 2026
+// proxy.go
+
+package proxy
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "os"
+ "path/filepath"
+ "strconv"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+const (
+ dialTimeout = 5 * time.Second
+
+ // A hung connection is held open, not closed, because closing it is a
+ // different fault. The ceiling exists only so a proxy that is never reverted
+ // cannot leak the goroutine forever. It is not the rollback: Arm, Disarm and
+ // Close all free held connections immediately.
+ hangCeiling = 10 * time.Minute
+
+ noLinger = 0
+ network = "tcp"
+ copyBuffer = 32 * 1024
+
+ // How long EOF waits for the target's in-flight request before closing. The
+ // kernel sends FIN only when the receive queue is empty, so the queue has to
+ // be drained to the deadline rather than to the first short read.
+ drainWindow = 250 * time.Millisecond
+
+ errWildcard = "%w: %q binds every address on this host, and a fault proxy in front of a dependency under test must not be reachable off it; bind the Docker bridge gateway instead"
+ errPrivileged = "%w: %q is a privileged port"
+ errBadAddr = "%w: %q is not a host and port (%v)"
+
+ probePayload = "GET / HTTP/1.1\r\nHost: khaos.confirm\r\nConnection: close\r\n\r\n"
+
+ // Each of these names what khaos measured through its proxy AGAINST what the
+ // same dependency gave it directly, because only the difference between the
+ // two paths can be attributed to khaos's own fault.
+ observedHang = "the dependency answers khaos directly but never through its proxy"
+ observedReset = "the dependency answers khaos directly but resets through its proxy"
+ observedEOF = "the dependency answers khaos directly but closes without answering through its proxy"
+ observedLatency = "the first byte arrives %s later through khaos's proxy than directly"
+ observedPartial = "%d bytes arrive through khaos's proxy where the dependency answers %d directly"
+ observedSlowClose = "the close arrives %s after the last byte, which it does not do directly"
+ errWrongArmed = "%w: the proxy holds %q but %q was expected"
+
+ lowestUnprivilegedPort = 1024
+ ephemeralPort = 0
+
+ sysClassNet = "/sys/class/net"
+ bridgeMarker = "bridge"
+
+ errOffBox = "%w: %q is not on loopback or on a Docker bridge, so it is reachable from outside this host and a container cannot reach it anyway; bind the bridge gateway the daemon reports"
+)
+
+var ErrClosed = errors.New("the fault proxy is closed")
+
+var ErrUnsafeBind = errors.New("khaos will not bind the fault proxy there")
+
+// Proxy is a TCP man-in-the-middle between a target and one of its dependencies,
+// modelled on Toxiproxy's per-connection pipeline and credited as such. It holds
+// no capabilities and writes no kernel state, so the worst a total failure of it
+// can leak is a listener on a high port, and there is nothing for bad targeting
+// to reach.
+//
+// The armed fault is one value swapped atomically, so arming and disarming are
+// indivisible and cannot half-apply. That is the property that makes this the
+// primary tier rather than a convenience.
+// armed is the whole mutable state of the proxy: the fault and the generation it
+// belongs to, swapped as one value so a handler can never read a fault from one
+// revert and a generation from another.
+type armed struct {
+ fault Fault
+ generation uint64
+}
+
+type Proxy struct {
+ upstream string
+ listener net.Listener
+ state atomic.Pointer[armed]
+
+ // Both ends of every exchange are held, and the revert closes both. Closing
+ // only the client cannot unblock a read parked on the DEPENDENCY, and a
+ // dependency that accepts and never answers is precisely what research 06
+ // section 6.1 measured for `docker pause` and `docker network disconnect`.
+ mu sync.Mutex
+ held map[net.Conn]struct{}
+
+ // inert makes the proxy forward normally while a fault is still armed. It
+ // exists so the confirmation oracle has a negative control: without a way to
+ // arm a fault that does NOT happen, every test can only assert that
+ // confirmation SUCCEEDS, and a confirmation gutted to `return true` satisfies
+ // all of them. Measured: that gut left all seventeen packages and the Docker
+ // gate green.
+ inert atomic.Bool
+
+ wg sync.WaitGroup
+ closed chan struct{}
+ once sync.Once
+
+ accepted atomic.Int64
+ severed atomic.Int64
+ relayed atomic.Int64
+ probes atomic.Int64
+}
+
+// Open binds the proxy and refuses an address that would expose it. The
+// containment property of this tier is that the listener answers containers and
+// nothing else, and that is enforced here rather than left to callers: a
+// wildcard bind puts a fault proxy for a dependency under test on the operator's
+// network.
+func Open(listenAddr, upstream string) (*Proxy, error) {
+ if err := safeBind(listenAddr); err != nil {
+ return nil, err
+ }
+
+ listener, err := net.Listen(network, listenAddr)
+ if err != nil {
+ return nil, err
+ }
+
+ p := &Proxy{
+ upstream: upstream,
+ listener: listener,
+ held: map[net.Conn]struct{}{},
+ closed: make(chan struct{}),
+ }
+
+ p.state.Store(&armed{fault: Fault{Kind: PassThrough}})
+
+ p.wg.Add(1)
+ go p.serve()
+
+ return p, nil
+}
+
+// safeBind enforces the containment property instead of describing it. The
+// earlier version refused only a wildcard, which let the proxy bind this host's
+// own LAN address: exactly the off-box exposure its error message warns about,
+// and useless besides, because a container cannot reach the LAN address anyway.
+//
+// The test is the routing property, measured rather than guessed from the
+// address: the listener must sit on loopback, which is unreachable off the host by
+// construction, or on a BRIDGE interface, which is what a container reaches the
+// host through. A bridge is identified by its sysfs `bridge` directory, so this
+// does not depend on interface naming.
+func safeBind(listenAddr string) error {
+ host, portText, err := net.SplitHostPort(listenAddr)
+ if err != nil {
+ return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)
+ }
+
+ // No name resolution. A hostname would mean an unbounded resolver lookup on a
+ // safety check, and khaos is always handed a literal address the daemon
+ // reported.
+ ip := net.ParseIP(host)
+ if ip == nil || ip.IsUnspecified() {
+ return fmt.Errorf(errWildcard, ErrUnsafeBind, listenAddr)
+ }
+
+ port, err := strconv.Atoi(portText)
+ if err != nil {
+ return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)
+ }
+
+ if port != ephemeralPort && port < lowestUnprivilegedPort {
+ return fmt.Errorf(errPrivileged, ErrUnsafeBind, listenAddr)
+ }
+
+ if ip.IsLoopback() {
+ return nil
+ }
+
+ onBridge, err := bridgeHosts(ip)
+ if err != nil {
+ return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)
+ }
+
+ if !onBridge {
+ return fmt.Errorf(errOffBox, ErrUnsafeBind, listenAddr)
+ }
+
+ return nil
+}
+
+// bridgeHosts reports whether an address belongs to a bridge interface on this
+// host.
+func bridgeHosts(ip net.IP) (bool, error) {
+ interfaces, err := net.Interfaces()
+ if err != nil {
+ return false, err
+ }
+
+ for _, iface := range interfaces {
+ addrs, err := iface.Addrs()
+ if err != nil {
+ continue
+ }
+
+ for _, addr := range addrs {
+ network, ok := addr.(*net.IPNet)
+ if !ok || !network.IP.Equal(ip) {
+ continue
+ }
+
+ _, err := os.Stat(filepath.Join(sysClassNet, iface.Name, bridgeMarker))
+
+ return err == nil, nil
+ }
+ }
+
+ return false, nil
+}
+
+func (p *Proxy) Addr() string { return p.listener.Addr().String() }
+
+func (p *Proxy) Upstream() string { return p.upstream }
+
+func (p *Proxy) Fault() Fault { return p.state.Load().fault }
+
+// Arm swaps the fault in. It also drops every connection currently held open, so
+// a target sitting on a pooled keepalive connection established before the fault
+// does not keep sailing through it and report a clean survival.
+func (p *Proxy) Arm(fault Fault) error {
+ if err := fault.Validate(); err != nil {
+ return err
+ }
+
+ select {
+ case <-p.closed:
+ return ErrClosed
+ default:
+ }
+
+ p.swap(fault)
+
+ return nil
+}
+
+// Disarm returns the proxy to pass-through and releases every connection the
+// fault was holding. This is the whole revert.
+func (p *Proxy) Disarm() {
+ p.swap(Fault{Kind: PassThrough})
+}
+
+// swap replaces the armed state and releases everything held under the old one,
+// as ONE transaction under the mutex.
+//
+// The fault and the generation have to move together. Storing the fault first and
+// advancing the generation afterwards let a handler read the NEW fault with the
+// OLD generation, which track then refused, so hold closed the client instead of
+// holding it. Measured: 116 of 2000 armed hangs came out as a reset, which is the
+// other fault in the catalogue and the one the design doc exists to distinguish
+// from it.
+func (p *Proxy) swap(fault Fault) {
+ p.mu.Lock()
+
+ previous := p.state.Load()
+ p.state.Store(&armed{fault: fault, generation: previous.generation + 1})
+
+ conns := make([]net.Conn, 0, len(p.held))
+ for conn := range p.held {
+ conns = append(conns, conn)
+ }
+ p.held = map[net.Conn]struct{}{}
+
+ p.mu.Unlock()
+
+ for _, conn := range conns {
+ _ = conn.Close()
+ }
+}
+
+// Counts is telemetry for the run record, never evidence that a fault landed.
+// Probes are reported separately so khaos's own confirmation traffic is not
+// mistaken for the target's.
+type Counts struct {
+ Accepted int64
+ Severed int64
+ Relayed int64
+ Probes int64
+}
+
+func (p *Proxy) Counts() Counts {
+ return Counts{
+ Accepted: p.accepted.Load(),
+ Severed: p.severed.Load(),
+ Relayed: p.relayed.Load(),
+ Probes: p.probes.Load(),
+ }
+}
+
+func (p *Proxy) Close() error {
+ var err error
+
+ p.once.Do(func() {
+ close(p.closed)
+ err = p.listener.Close()
+ p.swap(Fault{Kind: PassThrough})
+ })
+
+ p.wg.Wait()
+
+ return err
+}
+
+func (p *Proxy) serve() {
+ defer p.wg.Done()
+
+ for {
+ conn, err := p.listener.Accept()
+ if err != nil {
+ select {
+ case <-p.closed:
+ return
+ default:
+ }
+ if errors.Is(err, net.ErrClosed) {
+ return
+ }
+ // Backing off matters: a persistent accept error that is neither of
+ // the above would otherwise spin a core for the whole run.
+ time.Sleep(drainWindow)
+ continue
+ }
+
+ p.accepted.Add(1)
+ p.wg.Add(1)
+
+ go func() {
+ defer p.wg.Done()
+ p.handle(conn)
+ }()
+ }
+}
+
+// handle owns the client connection and re-reads the armed state until it finds
+// one that is still current.
+//
+// A connection accepted just before a revert is handled under the state that was
+// armed when it was accepted, and refusing it for staleness is not enough: the
+// first attempt at this closed such a connection, and closing one with an unread
+// request in its receive queue IS a reset. Measured: 81 of 2000 armed hangs
+// arrived at the client as `connection reset by peer`, which is the other fault
+// in the catalogue and the one the design doc exists to keep distinguishable.
+// Retrying under the current state delivers the fault that is actually armed.
+func (p *Proxy) handle(client net.Conn) {
+ defer func() { _ = client.Close() }()
+
+ for {
+ select {
+ case <-p.closed:
+ return
+ default:
+ }
+
+ // One load. The fault and the generation it belongs to arrive together, so
+ // there is no interleaving that pairs a new fault with a stale generation.
+ state := p.state.Load()
+
+ if p.inert.Load() {
+ if p.forward(client, Fault{Kind: PassThrough}, state.generation) {
+ return
+ }
+ continue
+ }
+
+ if state.fault.Severs() {
+ if p.sever(client, state.fault, state.generation) {
+ return
+ }
+ continue
+ }
+
+ if p.forward(client, state.fault, state.generation) {
+ return
+ }
+ }
+}
+
+// sever reports whether it handled the connection. False means the armed state
+// moved underneath it and the caller must try again under the current one.
+func (p *Proxy) sever(client net.Conn, fault Fault, generation uint64) bool {
+ switch fault.Kind {
+ case Reset:
+ // SO_LINGER 0 is necessary AND sufficient: it forces RST whatever the
+ // receive queue holds, and the reset surfaces at the target's connect or
+ // at its first read, which are the same observable.
+ //
+ // This used to also wait for the target's request, on the theory that a
+ // non-empty receive queue was needed to force RST on an empty-queue close.
+ // Measured: deleting that wait leaves both reset controls green over 20
+ // runs, including a target that sends nothing, while deleting SO_LINGER 0
+ // turns them red immediately. The wait was inert, and an inert arm alibis
+ // the live one's control, so it is gone rather than kept for symmetry. It
+ // also cost every reset a quarter second it should not have.
+ if tcp, ok := client.(*net.TCPConn); ok {
+ _ = tcp.SetLinger(noLinger)
+ }
+ _ = client.Close()
+ p.severed.Add(1)
+ return true
+ case EOF:
+ // The queue has to be drained to the deadline first. The kernel sends FIN
+ // only when the receive queue is EMPTY and RST when it is not, so closing
+ // on an unread request yields a reset and makes this a nondeterministic
+ // duplicate of Reset. Measured: a request larger than one read produced
+ // ECONNRESET on every attempt until the drain ran to its deadline.
+ drain(client)
+ _ = client.Close()
+ p.severed.Add(1)
+ return true
+ default:
+ return p.hold(client, generation)
+ }
+}
+
+// drain empties the receive queue until the deadline expires. Returning on the
+// first short read is wrong: a short read means "that is what had arrived by
+// now", not "the queue is empty", so bytes still in flight land on a closed
+// socket and the kernel answers RST.
+func drain(client net.Conn) {
+ if err := client.SetReadDeadline(time.Now().Add(drainWindow)); err != nil {
+ return
+ }
+
+ buffer := make([]byte, copyBuffer)
+
+ for {
+ if _, err := client.Read(buffer); err != nil {
+ return
+ }
+ }
+}
+
+// hold implements Hang: keep the connection open and answer nothing, until the
+// revert drops it or the ceiling expires.
+func (p *Proxy) hold(client net.Conn, generation uint64) bool {
+ if !p.track(generation, client) {
+ return false
+ }
+ defer p.untrack(client)
+
+ p.severed.Add(1)
+
+ timer := time.NewTimer(hangCeiling)
+ defer timer.Stop()
+
+ gone := make(chan struct{})
+
+ go func() {
+ defer close(gone)
+ // The target may give up first. Reading tells us when, and a hang must
+ // not interpret the target's own timeout as something it has to answer.
+ _, _ = io.Copy(io.Discard, client)
+ }()
+
+ select {
+ case <-gone:
+ case <-timer.C:
+ case <-p.closed:
+ }
+
+ return true
+}
+
+// forward reports whether it handled the connection. False means the armed state
+// moved underneath it and the caller must try again under the current one.
+func (p *Proxy) forward(client net.Conn, fault Fault, generation uint64) bool {
+ upstream, err := net.DialTimeout(network, p.upstream, dialTimeout)
+ if err != nil {
+ // The dependency is genuinely unreachable. Passing that through as a
+ // reset rather than inventing an answer keeps the proxy transparent when
+ // the real fault is somewhere else, which is what lets a Docker-native
+ // fault be measured THROUGH a pass-through proxy.
+ if tcp, ok := client.(*net.TCPConn); ok {
+ _ = tcp.SetLinger(noLinger)
+ }
+ return true
+ }
+ defer func() { _ = upstream.Close() }()
+
+ // BOTH ends are tracked, so the revert can close both. Tracking only the
+ // client deadlocked against a dependency that accepts and never answers: the
+ // answer leg sat in a read on the dependency, closing the client could not
+ // unblock it, and nothing closed the dependency until this function returned.
+ // `khaos run` defers this proxy's Close, so the whole command hung with a
+ // fault's listener still bound.
+ if !p.track(generation, client, upstream) {
+ return false
+ }
+ defer p.untrack(client, upstream)
+
+ // Only LATENCY delays the bytes. slow-close passed fault.Delay to both pumps
+ // as well as holding the close, so a two second slow-close delayed the
+ // request two seconds and every response chunk two seconds more, on top of
+ // the hold. A client with a 500ms timeout then failed before the response
+ // arrived — and failed for latency while khaos attributed it to the close.
+ //
+ // The two faults are supposed to be distinguishable: slow-close exists to
+ // find a caller that waits for the connection to end instead of reading a
+ // Content-Length, and that caller must receive its payload promptly or the
+ // experiment is measuring `network.dependency_latency` under another name.
+ chunkDelay := time.Duration(0)
+ if fault.Kind == Latency {
+ chunkDelay = fault.Delay
+ }
+
+ var relay sync.WaitGroup
+ relay.Add(2)
+
+ go func() {
+ defer relay.Done()
+ p.pump(upstream, client, chunkDelay, 0, true)
+ }()
+
+ go func() {
+ defer relay.Done()
+
+ p.pump(client, upstream, chunkDelay, limitFor(fault), false)
+
+ // The hold belongs to the ANSWER, not to the exchange. Putting it after
+ // both legs meant a client that keeps its write side open never reached
+ // it, so the fault could not be confirmed at all; putting it before the
+ // answer's own half-close is what makes it a slow CLOSE rather than a
+ // slow everything.
+ if fault.Kind == SlowClose {
+ select {
+ case <-time.After(fault.Delay):
+ case <-p.closed:
+ }
+ }
+
+ if tcp, ok := client.(*net.TCPConn); ok {
+ _ = tcp.CloseWrite()
+ }
+ }()
+
+ // Both directions have to finish, which is ordinary proxying. Returning on
+ // the first one closed an answer still in flight, so a half-closing client
+ // received nothing even through a pass-through proxy, and abandoning the
+ // request leg delivered a TRUNCATED request to the dependency: measured, a
+ // target sent 307200 bytes and the dependency received none of them.
+ relay.Wait()
+
+ return true
+}
+
+// pump copies one direction, optionally delaying each chunk, optionally stopping
+// after a byte budget, and optionally half-closing the destination when the
+// source ends. The delay is per chunk rather than once at the start, because a
+// one-off delay is a slow connection and a per-chunk delay is a slow dependency,
+// and only the second survives a client that retries.
+func (p *Proxy) pump(dst, src net.Conn, delay time.Duration, limit int, halfClose bool) {
+ if halfClose {
+ defer func() {
+ if tcp, ok := dst.(*net.TCPConn); ok {
+ _ = tcp.CloseWrite()
+ }
+ }()
+ }
+
+ var reader io.Reader = src
+ if limit > 0 {
+ reader = io.LimitReader(src, int64(limit))
+ }
+
+ if delay <= 0 {
+ written, _ := io.Copy(dst, reader)
+ if written > 0 {
+ p.relayed.Add(1)
+ }
+ return
+ }
+
+ buffer := make([]byte, copyBuffer)
+
+ for {
+ read, err := reader.Read(buffer)
+ if read > 0 {
+ select {
+ case <-time.After(delay):
+ case <-p.closed:
+ return
+ }
+ if _, writeErr := dst.Write(buffer[:read]); writeErr != nil {
+ return
+ }
+ p.relayed.Add(1)
+ }
+ if err != nil {
+ return
+ }
+ }
+}
+
+// limitFor applies Partial to the DEPENDENCY's answer only. Truncating what the
+// target sends would be a different experiment: it would test the dependency,
+// which is not the thing under test.
+func limitFor(fault Fault) int {
+ if fault.Kind == Partial {
+ return fault.Bytes
+ }
+ return 0
+}
+
+func (p *Proxy) track(generation uint64, conns ...net.Conn) bool {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ select {
+ case <-p.closed:
+ return false
+ default:
+ }
+
+ if p.state.Load().generation != generation {
+ return false
+ }
+
+ for _, conn := range conns {
+ p.held[conn] = struct{}{}
+ }
+
+ return true
+}
+
+func (p *Proxy) untrack(conns ...net.Conn) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ for _, conn := range conns {
+ delete(p.held, conn)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go
new file mode 100644
index 00000000..6c8b74c4
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy_test.go
@@ -0,0 +1,791 @@
+// ©AngelaMos | 2026
+// proxy_test.go
+
+package proxy
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+)
+
+const (
+ loopback = "127.0.0.1:0"
+ body = "the-dependency-answered-in-full"
+ probeWait = 2 * time.Second
+)
+
+func upstreamServer(t *testing.T) *httptest.Server {
+ t.Helper()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Length", fmt.Sprint(len(body)))
+ _, _ = io.WriteString(w, body)
+ }))
+ t.Cleanup(server.Close)
+
+ return server
+}
+
+func through(t *testing.T) (*Proxy, *httptest.Server) {
+ t.Helper()
+
+ server := upstreamServer(t)
+
+ p, err := Open(loopback, strings.TrimPrefix(server.URL, "http://"))
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ t.Cleanup(func() { _ = p.Close() })
+
+ return p, server
+}
+
+// get returns the body, or a description of how the attempt failed. Both are
+// evidence: the whole point of these faults is that the failure SHAPE differs.
+func get(p *Proxy) (string, error) {
+ client := &http.Client{Timeout: probeWait}
+
+ response, err := client.Get("http://" + p.Addr())
+ if err != nil {
+ return "", err
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ raw, err := io.ReadAll(response.Body)
+
+ return string(raw), err
+}
+
+func TestPassThroughDeliversTheDependencyUntouched(t *testing.T) {
+ p, _ := through(t)
+
+ got, err := get(p)
+ if err != nil {
+ t.Fatalf("pass-through failed: %v", err)
+ }
+
+ if got != body {
+ t.Fatalf("pass-through delivered %q, want %q", got, body)
+ }
+}
+
+// Every fault has to be distinguishable from pass-through. A fault that is not
+// is a run that measures nothing and reports a clean survival, which is the
+// single worst outcome this tool can produce.
+func TestEveryFaultIsDistinguishableFromPassThrough(t *testing.T) {
+ faults := []Fault{
+ {Kind: Hang},
+ {Kind: Reset},
+ {Kind: EOF},
+ {Kind: Latency, Delay: 3 * time.Second},
+ {Kind: Partial, Bytes: 8},
+ {Kind: SlowClose, Delay: 3 * time.Second},
+ }
+
+ if len(faults) != len(Kinds()) {
+ t.Fatalf("this test covers %d faults but %d are registered: %v",
+ len(faults), len(Kinds()), Kinds())
+ }
+
+ for _, fault := range faults {
+ t.Run(string(fault.Kind), func(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(fault); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ started := time.Now()
+ got, err := get(p)
+ took := time.Since(started)
+
+ // slow-close is the one fault whose answer is supposed to arrive
+ // INTACT. It exists to find a caller that waits for the connection
+ // to end instead of reading a Content-Length, so the payload has to
+ // be delivered and only the close held back. It is distinguished by
+ // the clock, not by a damaged body.
+ //
+ // It used to be distinguishable by a damaged body, because the delay
+ // was applied to the relay as well as the close — which made it
+ // `network.dependency_latency` wearing a different name, and meant a
+ // client that timed out mid-payload was reported as one that waits
+ // for the close.
+ if fault.Kind == SlowClose {
+ if got != body {
+ t.Fatalf("slow-close delivered %q, want the answer intact; holding the "+
+ "PAYLOAD back makes it indistinguishable from a latency fault", got)
+ }
+
+ // Measured at the SOCKET, not through this HTTP client. Go's
+ // client reads a Content-Length body and returns without ever
+ // waiting for the close — measured here at 296µs against a three
+ // second hold — so an HTTP GET cannot see this fault at all.
+ //
+ // That is the fault behaving correctly: it exists to find a
+ // caller that waits for EOF instead of reading a length, and a
+ // caller that reads the length is supposed to be unaffected.
+ // Asserting otherwise would be asserting that slow-close breaks
+ // clients it is not meant to break.
+ observable, confirmed, err := p.Observe(t.Context(), SlowClose)
+ if err != nil || !confirmed {
+ t.Fatalf("slow-close is not observable at the socket either (%v), so nothing "+
+ "distinguishes it from no fault at all: %s", err, observable)
+ }
+
+ t.Logf("slow-close -> body intact after %s, and %s",
+ took.Round(time.Millisecond), observable)
+ return
+ }
+
+ if err == nil && got == body {
+ t.Fatalf("%s delivered the dependency's answer intact, so it is "+
+ "indistinguishable from no fault at all", fault.Kind)
+ }
+
+ t.Logf("%s -> body=%q err=%v", fault.Kind, got, err)
+ })
+ }
+}
+
+func TestResetIsRefusedWhileHangIsSilence(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Reset}); err != nil {
+ t.Fatalf("arm reset: %v", err)
+ }
+
+ start := time.Now()
+ _, resetErr := get(p)
+ resetTook := time.Since(start)
+
+ if resetErr == nil {
+ t.Fatal("reset was answered successfully")
+ }
+
+ if errors.Is(resetErr, http.ErrHandlerTimeout) || resetTook >= probeWait {
+ t.Fatalf("reset took %s, which is a timeout rather than a refusal; a client "+
+ "cannot tell it from a hang and the two hit different branches", resetTook)
+ }
+
+ if err := p.Arm(Fault{Kind: Hang}); err != nil {
+ t.Fatalf("arm hang: %v", err)
+ }
+
+ start = time.Now()
+ _, hangErr := get(p)
+ hangTook := time.Since(start)
+
+ if hangErr == nil {
+ t.Fatal("hang was answered successfully")
+ }
+
+ if hangTook < probeWait {
+ t.Fatalf("hang failed after %s, before the client's own timeout; a hang the "+
+ "client distinguishes from a refusal is the whole point of having both", hangTook)
+ }
+}
+
+// Reset and EOF are only two faults if the client can tell them apart. An
+// ordinary close with an empty receive queue sends FIN, so without SO_LINGER 0
+// "reset" is an EOF wearing a different name and the catalogue has two entries
+// for one behaviour. Measured at the socket, not through net/http, because the
+// transport rewrites both into its own errors.
+func TestResetIsAnRSTAndEOFIsAFIN(t *testing.T) {
+ cases := []struct {
+ fault Kind
+ wantReset bool
+ }{
+ {fault: Reset, wantReset: true},
+ {fault: EOF, wantReset: false},
+ }
+
+ for _, tc := range cases {
+ t.Run(string(tc.fault), func(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: tc.fault}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ // A reset can surface at connect OR at the first read depending on
+ // how fast the proxy gets to it, and both are the same observable.
+ // Insisting on the read would make this test flaky rather than
+ // strict.
+ var readErr error
+
+ conn, dialErr := net.DialTimeout(network, p.Addr(), probeWait)
+ if dialErr != nil {
+ readErr = dialErr
+ } else {
+ defer func() { _ = conn.Close() }()
+
+ _, _ = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: x\r\n\r\n")
+
+ if err := conn.SetReadDeadline(time.Now().Add(probeWait)); err != nil {
+ t.Fatalf("deadline: %v", err)
+ }
+
+ buffer := make([]byte, 64)
+ _, readErr = conn.Read(buffer)
+ }
+
+ gotReset := errors.Is(readErr, syscall.ECONNRESET)
+ gotEOF := errors.Is(readErr, io.EOF)
+
+ if tc.wantReset && !gotReset {
+ t.Fatalf("%s produced %v, not ECONNRESET; a client cannot tell it from an "+
+ "orderly close, and the two hit different branches in almost every "+
+ "HTTP client", tc.fault, readErr)
+ }
+
+ if !tc.wantReset && !gotEOF {
+ t.Fatalf("%s produced %v, not a clean EOF", tc.fault, readErr)
+ }
+ })
+ }
+}
+
+func TestPartialDeliversAPrefixAndThenStops(t *testing.T) {
+ const budget = 12
+
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Partial, Bytes: budget}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ got, _ := get(p)
+
+ if len(got) >= len(body) {
+ t.Fatalf("partial delivered %d bytes of a %d byte body, so nothing was truncated",
+ len(got), len(body))
+ }
+
+ if strings.Contains(got, body) {
+ t.Fatalf("partial delivered the whole body: %q", got)
+ }
+}
+
+func TestLatencyDelaysRatherThanFails(t *testing.T) {
+ const delay = 300 * time.Millisecond
+
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Latency, Delay: delay}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ start := time.Now()
+ got, err := get(p)
+ took := time.Since(start)
+
+ if err != nil {
+ t.Fatalf("latency broke the connection instead of slowing it: %v", err)
+ }
+
+ if got != body {
+ t.Fatalf("latency altered the body: %q", got)
+ }
+
+ if took < delay {
+ t.Fatalf("latency of %s took only %s, so the delay is not being applied", delay, took)
+ }
+}
+
+// Disarm is the whole revert. If it does not fully restore pass-through, a
+// recovery window reads as a target that never came back.
+func TestDisarmRestoresPassThroughCompletely(t *testing.T) {
+ for _, fault := range []Fault{
+ {Kind: Hang}, {Kind: Reset}, {Kind: EOF},
+ {Kind: Latency, Delay: 2 * time.Second},
+ {Kind: Partial, Bytes: 4},
+ {Kind: SlowClose, Delay: 2 * time.Second},
+ } {
+ t.Run(string(fault.Kind), func(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(fault); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+ _, _ = get(p)
+
+ p.Disarm()
+
+ if armed := p.Fault().Kind; armed != PassThrough {
+ t.Fatalf("after disarm the proxy still reads %s", armed)
+ }
+
+ got, err := get(p)
+ if err != nil {
+ t.Fatalf("after disarming %s the proxy still fails: %v", fault.Kind, err)
+ }
+ if got != body {
+ t.Fatalf("after disarming %s the proxy delivered %q", fault.Kind, got)
+ }
+ })
+ }
+}
+
+// A target holding a pooled keepalive connection established BEFORE the fault
+// would otherwise keep sailing through it, and a fault nothing experiences reads
+// as a clean survival.
+func TestArmingDropsConnectionsOpenedBeforeTheFault(t *testing.T) {
+ p, _ := through(t)
+
+ client, err := net.Dial("tcp", p.Addr())
+ if err != nil {
+ t.Fatalf("dial: %v", err)
+ }
+ defer func() { _ = client.Close() }()
+
+ if _, err := io.WriteString(client, "GET / HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ if err := client.SetReadDeadline(time.Now().Add(probeWait)); err != nil {
+ t.Fatalf("deadline: %v", err)
+ }
+
+ buffer := make([]byte, 128)
+ if _, err := client.Read(buffer); err != nil {
+ t.Fatalf("the established connection never answered before the fault: %v", err)
+ }
+
+ if err := p.Arm(Fault{Kind: Hang}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ // Reading once is not enough and the first attempt at this test was wrong
+ // for exactly that reason: the response above was only partly consumed, so a
+ // single Read returns the REMAINDER OF THE OLD ANSWER out of the socket
+ // buffer and looks indistinguishable from a connection still being served.
+ // The connection must be drained until it actually ends.
+ if err := client.SetReadDeadline(time.Now().Add(probeWait)); err != nil {
+ t.Fatalf("deadline: %v", err)
+ }
+
+ for {
+ _, err = client.Read(buffer)
+ if err == nil {
+ continue
+ }
+
+ var timeout net.Error
+ if errors.As(err, &timeout) && timeout.Timeout() {
+ t.Fatalf("a connection established before the fault was neither dropped nor ended " +
+ "after arming; a target on a pooled keepalive connection would sail straight " +
+ "through the fault and the run would report a clean survival")
+ }
+
+ return
+ }
+}
+
+// Closing the proxy IS the rollback of last resort. The fault has to die with the
+// process, because that is the tier's entire safety argument.
+func TestClosingTheProxyRemovesTheFaultEntirely(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Hang}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ addr := p.Addr()
+
+ if err := p.Close(); err != nil {
+ t.Fatalf("close: %v", err)
+ }
+
+ conn, err := net.DialTimeout("tcp", addr, probeWait)
+ if err == nil {
+ _ = conn.Close()
+ t.Fatal("the proxy still accepts connections after Close, so the fault outlived it")
+ }
+
+ if err := p.Arm(Fault{Kind: Reset}); !errors.Is(err, ErrClosed) {
+ t.Fatalf("arming a closed proxy returned %v, want ErrClosed", err)
+ }
+}
+
+// An incomplete fault is indistinguishable from pass-through at runtime, so it
+// has to be refused at arm time rather than silently measured.
+func TestAFaultThatCannotExpressItselfIsRefused(t *testing.T) {
+ p, _ := through(t)
+
+ for _, fault := range []Fault{
+ {Kind: Latency},
+ {Kind: SlowClose},
+ {Kind: Partial},
+ } {
+ t.Run(string(fault.Kind), func(t *testing.T) {
+ if err := p.Arm(fault); !errors.Is(err, ErrFaultIncomplete) {
+ t.Fatalf("arm(%+v) = %v, want ErrFaultIncomplete; without a parameter this "+
+ "fault forwards everything untouched and reports a clean survival", fault, err)
+ }
+
+ if armed := p.Fault().Kind; armed != PassThrough {
+ t.Fatalf("a refused fault still changed the armed state to %s", armed)
+ }
+ })
+ }
+}
+
+func TestAnUnreachableDependencyIsNotMistakenForAFault(t *testing.T) {
+ p, err := Open(loopback, "127.0.0.1:1")
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ defer func() { _ = p.Close() }()
+
+ if _, err := get(p); err == nil {
+ t.Fatal("the proxy invented an answer for a dependency it could not reach")
+ }
+
+ if counts := p.Counts(); counts.Relayed != 0 {
+ t.Fatalf("the proxy counted %d relayed chunks to a dead upstream", counts.Relayed)
+ }
+}
+
+func TestParseKindRefusesPassThroughAndNonsense(t *testing.T) {
+ for _, name := range []string{string(PassThrough), "", "latencyy", "RESET"} {
+ if _, err := ParseKind(name); !errors.Is(err, ErrUnknownKind) {
+ t.Fatalf("ParseKind(%q) = %v, want ErrUnknownKind", name, err)
+ }
+ }
+
+ for _, name := range Kinds() {
+ if _, err := ParseKind(name); err != nil {
+ t.Fatalf("ParseKind(%q) rejected a registered fault: %v", name, err)
+ }
+ }
+}
+
+func TestSeveringAndForwardingFaultsAreSeparated(t *testing.T) {
+ severing := map[Kind]bool{Hang: true, Reset: true, EOF: true}
+
+ for _, name := range Kinds() {
+ kind := Kind(name)
+ fault := Fault{Kind: kind, Delay: time.Second, Bytes: 1}
+
+ if fault.Severs() != severing[kind] {
+ t.Fatalf("%s reports Severs()=%v; khaos confirms a severing fault by proving the "+
+ "dependency is unreachable and a forwarding one by proving the stream was "+
+ "altered, and one cannot stand in for the other", kind, fault.Severs())
+ }
+ }
+}
+
+// Audit findings, reproduced as controls before being fixed.
+
+func TestSlowCloseHoldsTheConnectionOpenAfterTheAnswer(t *testing.T) {
+ const delay = 600 * time.Millisecond
+
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: SlowClose, Delay: delay}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ conn, err := net.DialTimeout(network, p.Addr(), probeWait)
+ if err != nil {
+ t.Fatalf("dial: %v", err)
+ }
+ defer func() { _ = conn.Close() }()
+
+ // Connection: close is load-bearing. Without it the upstream keeps the
+ // connection alive, the answer leg never finishes, and the "hold" being
+ // measured is the test's own read deadline rather than the fault.
+ _, _ = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n")
+ _ = conn.SetReadDeadline(time.Now().Add(probeWait * 4))
+
+ buffer := make([]byte, 4096)
+ var lastByte, ended time.Time
+
+ for {
+ read, err := conn.Read(buffer)
+ if read > 0 {
+ lastByte = time.Now()
+ }
+ if err != nil {
+ ended = time.Now()
+ break
+ }
+ }
+
+ if lastByte.IsZero() {
+ t.Fatal("slow-close delivered nothing at all, so it is not a slow CLOSE")
+ }
+
+ held := ended.Sub(lastByte)
+ if held < delay {
+ t.Fatalf("the stream ended %s after the last byte, want at least %s; slow-close is not "+
+ "holding the connection open and is only distinguishable from pass-through because "+
+ "it also delays every chunk, which is what latency does", held, delay)
+ }
+}
+
+func TestEOFStaysAFINWhenTheRequestIsLargerThanOneRead(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: EOF}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ for attempt := range 6 {
+ conn, err := net.DialTimeout(network, p.Addr(), probeWait)
+ if err != nil {
+ t.Fatalf("dial: %v", err)
+ }
+
+ body := strings.Repeat("x", 4<<20)
+ go func() {
+ _, _ = io.WriteString(conn, "POST / HTTP/1.1\r\nHost: x\r\n\r\n"+body)
+ }()
+
+ _ = conn.SetReadDeadline(time.Now().Add(probeWait))
+ _, readErr := conn.Read(make([]byte, 64))
+ _ = conn.Close()
+
+ if errors.Is(readErr, syscall.ECONNRESET) {
+ t.Fatalf("attempt %d: eof produced ECONNRESET on a request larger than one read, so "+
+ "it is a nondeterministic duplicate of reset after all", attempt)
+ }
+ }
+}
+
+func TestArmRefusesAFaultWhoseKindIsNotRegistered(t *testing.T) {
+ p, _ := through(t)
+
+ for _, fault := range []Fault{
+ {},
+ {Kind: "not-a-fault"},
+ {Kind: Kind(string(Hang) + " ")},
+ {Kind: PassThrough},
+ } {
+ if err := p.Arm(fault); err == nil {
+ t.Fatalf("Arm(%+v) was accepted; an unrecognised kind forwards every byte untouched "+
+ "and the run reports a clean survival", fault)
+ }
+
+ if armed := p.Fault().Kind; armed != PassThrough {
+ t.Fatalf("a refused fault changed the armed state to %q", armed)
+ }
+ }
+}
+
+// A client that half-closes after its request is an ordinary HTTP client shape.
+// The baseline is measured through the proxy, so getting this wrong makes a
+// healthy dependency read as a dead one before any fault is armed.
+func TestAHalfClosingClientStillReceivesTheWholeAnswer(t *testing.T) {
+ p, _ := through(t)
+
+ conn, err := net.DialTimeout(network, p.Addr(), probeWait)
+ if err != nil {
+ t.Fatalf("dial: %v", err)
+ }
+ defer func() { _ = conn.Close() }()
+
+ tcp, ok := conn.(*net.TCPConn)
+ if !ok {
+ t.Fatal("expected a TCP connection")
+ }
+
+ if _, err := io.WriteString(tcp, "GET / HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ if err := tcp.CloseWrite(); err != nil {
+ t.Fatalf("close write: %v", err)
+ }
+
+ _ = tcp.SetReadDeadline(time.Now().Add(probeWait * 2))
+
+ answer, err := io.ReadAll(tcp)
+ if err != nil {
+ t.Fatalf("read: %v", err)
+ }
+
+ if !strings.Contains(string(answer), body) {
+ t.Fatalf("a half-closing client received %d bytes and never the answer; pass-through "+
+ "dropped it: %q", len(answer), string(answer))
+ }
+}
+
+// The containment property is the tier's whole safety argument, so it needs an
+// input it MUST refuse rather than a comment saying what callers should pass.
+func TestOpenRefusesAnAddressThatWouldExposeTheProxy(t *testing.T) {
+ for _, addr := range []string{"0.0.0.0:0", ":0", "[::]:0", "0.0.0.0:45001"} {
+ p, err := Open(addr, "127.0.0.1:1")
+ if err == nil {
+ bound := p.Addr()
+ _ = p.Close()
+ t.Fatalf("Open(%q) bound %s; a fault proxy in front of a deliberately vulnerable "+
+ "dependency must not be reachable off the host", addr, bound)
+ }
+ }
+}
+
+func TestDisarmReleasesAConnectionThatRacedTheRevert(t *testing.T) {
+ p, _ := through(t)
+
+ for attempt := range 300 {
+ if err := p.Arm(Fault{Kind: Hang}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ conn, err := net.DialTimeout(network, p.Addr(), probeWait)
+ if err != nil {
+ t.Fatalf("dial: %v", err)
+ }
+
+ _, _ = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: x\r\n\r\n")
+ p.Disarm()
+
+ _ = conn.SetReadDeadline(time.Now().Add(400 * time.Millisecond))
+ _, readErr := conn.Read(make([]byte, 64))
+ _ = conn.Close()
+
+ var timeout net.Error
+ if errors.As(readErr, &timeout) && timeout.Timeout() {
+ t.Fatalf("attempt %d: a hung connection was still held 400ms after Disarm. The "+
+ "ceiling is %s against a %s recovery window, so the target reads as one that "+
+ "never came back and khaos manufactures the verdict itself",
+ attempt, hangCeiling, "5s")
+ }
+ }
+}
+
+// The LAN address is the finding this exists for: the first version of safeBind
+// refused only a wildcard, so it accepted this host's own routable address, which
+// is both the off-box exposure its error warns about and useless, because a
+// container cannot reach the LAN address at all.
+func TestOpenRefusesThisHostsRoutableAddress(t *testing.T) {
+ routable := ""
+
+ interfaces, err := net.Interfaces()
+ if err != nil {
+ t.Fatalf("interfaces: %v", err)
+ }
+
+ for _, iface := range interfaces {
+ if _, err := os.Stat(filepath.Join(sysClassNet, iface.Name, bridgeMarker)); err == nil {
+ continue
+ }
+
+ addrs, err := iface.Addrs()
+ if err != nil {
+ continue
+ }
+
+ for _, addr := range addrs {
+ network, ok := addr.(*net.IPNet)
+ if !ok || network.IP.IsLoopback() || network.IP.To4() == nil {
+ continue
+ }
+ routable = network.IP.String()
+ }
+ }
+
+ if routable == "" {
+ t.Skip("this host has no non-bridge routable IPv4 address to try")
+ }
+
+ p, err := Open(net.JoinHostPort(routable, "45999"), "127.0.0.1:1")
+ if err == nil {
+ bound := p.Addr()
+ _ = p.Close()
+ t.Fatalf("Open bound %s, which is reachable from outside this host", bound)
+ }
+
+ if !errors.Is(err, ErrUnsafeBind) {
+ t.Fatalf("err = %v, want ErrUnsafeBind", err)
+ }
+}
+
+// The bridge gateway MUST be accepted, or the only address a container can reach
+// khaos on is refused and the tier cannot run at all.
+func TestOpenAcceptsABridgeAddress(t *testing.T) {
+ interfaces, err := net.Interfaces()
+ if err != nil {
+ t.Fatalf("interfaces: %v", err)
+ }
+
+ for _, iface := range interfaces {
+ if _, err := os.Stat(filepath.Join(sysClassNet, iface.Name, bridgeMarker)); err != nil {
+ continue
+ }
+
+ addrs, err := iface.Addrs()
+ if err != nil {
+ continue
+ }
+
+ for _, addr := range addrs {
+ network, ok := addr.(*net.IPNet)
+ if !ok || network.IP.To4() == nil {
+ continue
+ }
+
+ p, err := Open(net.JoinHostPort(network.IP.String(), "0"), "127.0.0.1:1")
+ if err != nil {
+ t.Fatalf("Open refused the bridge address %s: %v", network.IP, err)
+ }
+ _ = p.Close()
+
+ return
+ }
+ }
+
+ t.Skip("this host has no bridge interface with an IPv4 address")
+}
+
+// Reset has two mechanisms and each covers a case the other cannot, so each needs
+// its own control or they alibi one another: with both present either alone
+// usually produces RST and the gate stops detecting the loss of either.
+//
+// This is the SO_LINGER case specifically. A target that connects and sends
+// nothing leaves an EMPTY receive queue, so awaitRequest has nothing to hold back
+// and an ordinary close would send FIN, which is the eof fault wearing the wrong
+// name.
+func TestResetIsStillAnRSTWhenTheTargetSendsNothing(t *testing.T) {
+ p, _ := through(t)
+
+ if err := p.Arm(Fault{Kind: Reset}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ for attempt := range 10 {
+ conn, err := net.DialTimeout(network, p.Addr(), probeWait)
+ if err != nil {
+ if isReset(err) {
+ continue
+ }
+ t.Fatalf("dial: %v", err)
+ }
+
+ // Deliberately no write. awaitRequest will time out on an empty queue and
+ // only SO_LINGER 0 can still force a reset.
+ _ = conn.SetReadDeadline(time.Now().Add(probeWait))
+ _, readErr := conn.Read(make([]byte, 64))
+ _ = conn.Close()
+
+ if errors.Is(readErr, io.EOF) {
+ t.Fatalf("attempt %d: a target that sent nothing got a clean EOF, so reset degraded "+
+ "into the eof fault", attempt)
+ }
+
+ if !isReset(readErr) {
+ t.Fatalf("attempt %d: got %v, want a reset", attempt, readErr)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/race_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/race_test.go
new file mode 100644
index 00000000..c43da647
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/race_test.go
@@ -0,0 +1,92 @@
+// ©AngelaMos | 2026
+// race_test.go
+
+package proxy
+
+import (
+ "errors"
+ "io"
+ "net"
+ "syscall"
+ "testing"
+ "time"
+)
+
+const (
+ raceAttempts = 2000
+ raceWindow = 150 * time.Millisecond
+)
+
+// An armed fault must arrive as ITSELF. Storing the fault and advancing the
+// generation as two steps let a handler read the new fault with the old
+// generation, which track refused, so hold closed the client instead of holding
+// it: an armed hang came out as a reset. Measured before the fix at 116 of 2000
+// attempts, and hang and reset are the two faults the design doc exists to keep
+// distinguishable because they hit different branches in almost every client.
+func TestAnArmedHangNeverArrivesAsAReset(t *testing.T) {
+ p, _ := through(t)
+
+ resets := 0
+
+ for range raceAttempts {
+ p.Disarm()
+
+ conn, err := net.DialTimeout(network, p.Addr(), probeWait)
+ if err != nil {
+ t.Fatalf("dial: %v", err)
+ }
+
+ if err := p.Arm(Fault{Kind: Hang}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ _, _ = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: x\r\n\r\n")
+ _ = conn.SetReadDeadline(time.Now().Add(raceWindow))
+
+ _, readErr := conn.Read(make([]byte, 64))
+ _ = conn.Close()
+
+ if errors.Is(readErr, syscall.ECONNRESET) {
+ resets++
+ }
+ }
+
+ if resets > 0 {
+ t.Fatalf("%d of %d armed hangs arrived as a reset. A handler read the new fault with a "+
+ "stale generation, so the connection was refused and closed rather than held, and "+
+ "the operator measured the other fault in the catalogue", resets, raceAttempts)
+ }
+}
+
+// The mirror of the above, for the property the generation counter exists to
+// provide: a connection accepted just before a revert must not be left held by
+// nothing once the revert has run.
+func TestNoConnectionOutlivesTheRevertThatShouldHaveFreedIt(t *testing.T) {
+ p, _ := through(t)
+
+ for attempt := range raceAttempts {
+ if err := p.Arm(Fault{Kind: Hang}); err != nil {
+ t.Fatalf("arm: %v", err)
+ }
+
+ conn, err := net.DialTimeout(network, p.Addr(), probeWait)
+ if err != nil {
+ t.Fatalf("dial: %v", err)
+ }
+
+ _, _ = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: x\r\n\r\n")
+ p.Disarm()
+
+ _ = conn.SetReadDeadline(time.Now().Add(raceWindow))
+ _, readErr := conn.Read(make([]byte, 64))
+ _ = conn.Close()
+
+ var timeout net.Error
+ if errors.As(readErr, &timeout) && timeout.Timeout() {
+ t.Fatalf("attempt %d: a connection was still held %s after the revert. The ceiling "+
+ "is %s against a five second recovery window, so the target reads as one that "+
+ "never came back and khaos manufactures that verdict itself",
+ attempt, raceWindow, hangCeiling)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault.go
new file mode 100644
index 00000000..312ac5d0
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault.go
@@ -0,0 +1,126 @@
+// ©AngelaMos | 2026
+// proxyfault.go
+
+package inject
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const (
+ KindProxyPrefix = "proxy-"
+
+ errNoProxy = "%w: %s needs khaos's fault proxy, and this run did not start one"
+ errWrongFault = "%w: this run armed %q but the mutation is %q"
+
+ confirmProxy = "dialling khaos's own fault proxy and asserting the observable the armed fault is"
+
+ describeProxy = "%s, through khaos's own proxy, which holds no capabilities and cannot reach the host"
+)
+
+var ErrNoProxy = errors.New("this mutation needs a fault proxy that is not running")
+
+var ErrProxyFaultMismatch = errors.New("the armed proxy fault does not match the mutation being applied")
+
+// ProxyKind names the mutation that arms one proxy fault. The proxy fault kinds
+// and the mutation kinds are deliberately one-to-one, so `khaos catalog` names
+// the actual fault rather than a generic "proxy" entry.
+func ProxyKind(kind proxy.Kind) string {
+ return KindProxyPrefix + string(kind)
+}
+
+// proxyMutator arms a fault on khaos's in-process proxy. The whole mutation is a
+// single atomic pointer swap, which is why this tier cannot half-apply and why
+// its rollback cannot partially fail.
+type proxyMutator struct {
+ kind proxy.Kind
+}
+
+func init() {
+ for _, name := range proxy.Kinds() {
+ kind := proxy.Kind(name)
+ registry[ProxyKind(kind)] = proxyMutator{kind: kind}
+ }
+}
+
+// Describe states the fault the operator is about to consent to, with its actual
+// magnitude. Reading the kind alone and leaving the parameters at zero printed
+// "delay every byte ... by 0s" for a three second latency fault, which is the
+// plan line understating the fault it is asking permission for.
+func (m proxyMutator) Describe(env Env, target dockerx.Target) string {
+ fault := env.ProxyFault
+ if fault.Kind != m.kind {
+ fault = proxy.Fault{Kind: m.kind}
+ }
+
+ return fmt.Sprintf(describeProxy, fault.Describe(target.Name))
+}
+
+// Plan journals the fault that is about to be armed. For every other tier the
+// payload is what the revert needs; here it is also the only durable record of
+// what the run actually did, since the proxy itself dies with the process.
+func (m proxyMutator) Plan(_ context.Context, env Env, _ dockerx.Target) (json.RawMessage, error) {
+ if env.Proxy == nil {
+ return nil, fmt.Errorf(errNoProxy, ErrNoProxy, ProxyKind(m.kind))
+ }
+
+ if env.ProxyFault.Kind != m.kind {
+ return nil, fmt.Errorf(errWrongFault, ErrProxyFaultMismatch, env.ProxyFault.Kind, m.kind)
+ }
+
+ if err := env.ProxyFault.Validate(); err != nil {
+ return nil, err
+ }
+
+ return journal.EncodePayload(env.ProxyFault)
+}
+
+func (m proxyMutator) Apply(_ context.Context, env Env, _ dockerx.Target, raw json.RawMessage) error {
+ if env.Proxy == nil {
+ return fmt.Errorf(errNoProxy, ErrNoProxy, ProxyKind(m.kind))
+ }
+
+ var fault proxy.Fault
+ if err := json.Unmarshal(raw, &fault); err != nil {
+ return err
+ }
+
+ return env.Proxy.Arm(fault)
+}
+
+// Revert disarms. A cold start has no proxy to disarm, and that is RESOLVED
+// rather than failed: the proxy died with the process that held it, so the fault
+// provably went with it. Reporting that as a failure would wedge the journal
+// record forever and, because `khaos reap` stops on a sweep error, would disable
+// the dead man's switch for every other fault on the machine.
+func (m proxyMutator) Revert(_ context.Context, env Env, _ journal.Record) error {
+ if env.Proxy == nil {
+ return nil
+ }
+
+ env.Proxy.Disarm()
+
+ return nil
+}
+
+// Confirm delegates to the proxy's own socket-level read-back and returns the
+// specific observable it measured, so the run record says what was actually seen
+// rather than a constant describing the technique. It deliberately does NOT
+// compare counters: a counter incremented by the dispatcher records which branch
+// was chosen, not what reached the wire, so counter comparison confirms a fault
+// that is not armed and confirms a severing fault while the dependency is fully
+// reachable.
+func (m proxyMutator) Confirm(ctx context.Context, env Env, _ dockerx.Target) (string, bool, error) {
+ if env.Proxy == nil {
+ return "", false, fmt.Errorf(errNoProxy, ErrNoProxy, ProxyKind(m.kind))
+ }
+
+ return env.Proxy.Observe(ctx, m.kind)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
new file mode 100644
index 00000000..858647c5
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
@@ -0,0 +1,270 @@
+// ©AngelaMos | 2026
+// proxyfault_test.go
+
+package inject
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const (
+ loopback = "127.0.0.1:0"
+ answer = "the-dependency-answered"
+)
+
+func armed(t *testing.T, fault proxy.Fault) Env {
+ t.Helper()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = io.WriteString(w, answer)
+ }))
+ t.Cleanup(server.Close)
+
+ p, err := proxy.Open(loopback, strings.TrimPrefix(server.URL, "http://"))
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ t.Cleanup(func() { _ = p.Close() })
+
+ return Env{Proxy: p, ProxyFault: fault}
+}
+
+func kinds() []proxy.Fault {
+ return []proxy.Fault{
+ {Kind: proxy.Hang},
+ {Kind: proxy.Reset},
+ {Kind: proxy.EOF},
+ {Kind: proxy.Latency, Delay: 300 * time.Millisecond},
+ {Kind: proxy.Partial, Bytes: 9},
+ {Kind: proxy.SlowClose, Delay: 300 * time.Millisecond},
+ }
+}
+
+// The whole mutation, end to end, for every registered proxy fault: journal the
+// fault, arm it, confirm it by socket read-back, revert it, and prove the
+// dependency is reachable again afterwards.
+func TestEveryProxyFaultAppliesConfirmsAndReverts(t *testing.T) {
+ // Counted by the PROXY prefix, not by "not direct". The identity provider
+ // tier is also not direct, so the looser count started reporting 23 the
+ // moment that tier landed and this census would have had to be weakened to
+ // pass rather than tightened.
+ registered := 0
+ for _, kind := range Kinds() {
+ if strings.HasPrefix(kind, KindProxyPrefix) {
+ registered++
+ }
+ }
+
+ if registered != len(kinds()) {
+ t.Fatalf("this test covers %d proxy faults but %d are registered", len(kinds()), registered)
+ }
+
+ for _, fault := range kinds() {
+ t.Run(string(fault.Kind), func(t *testing.T) {
+ env := armed(t, fault)
+ ctx := context.Background()
+ target := dockerx.Target{ID: "abc", Name: "a-dependency"}
+
+ mutator, err := Lookup(ProxyKind(fault.Kind))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ payload, err := mutator.Plan(ctx, env, target)
+ if err != nil {
+ t.Fatalf("plan: %v", err)
+ }
+
+ var journaled proxy.Fault
+ if err := json.Unmarshal(payload, &journaled); err != nil {
+ t.Fatalf("the journalled payload does not describe the fault: %v", err)
+ }
+ if journaled != fault {
+ t.Fatalf("journalled %+v, armed %+v; the journal is the only durable record of "+
+ "what this run did, because the proxy dies with the process", journaled, fault)
+ }
+
+ if err := mutator.Apply(ctx, env, target, payload); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+
+ if got := env.Proxy.Fault(); got != fault {
+ t.Fatalf("after apply the proxy holds %+v, want %+v", got, fault)
+ }
+
+ method, confirmed, err := mutator.Confirm(ctx, env, target)
+ if err != nil {
+ t.Fatalf("confirm: %v", err)
+ }
+ if !confirmed {
+ t.Fatalf("%s applied but could not be confirmed, so every run of it is void",
+ fault.Kind)
+ }
+
+ // The run record has to carry what was MEASURED. A constant naming the
+ // technique is not evidence a reader can audit.
+ if method == "" || method == confirmProxy {
+ t.Fatalf("%s confirmed with %q rather than the observable it measured",
+ fault.Kind, method)
+ }
+
+ if err := mutator.Revert(ctx, env, journal.Record{}); err != nil {
+ t.Fatalf("revert: %v", err)
+ }
+
+ if got := env.Proxy.Fault().Kind; got != proxy.PassThrough {
+ t.Fatalf("after revert the proxy holds %s", got)
+ }
+
+ if body := reach(t, env.Proxy.Addr()); !strings.Contains(body, answer) {
+ t.Fatalf("after reverting %s the dependency is still not reachable through the "+
+ "proxy, got %q", fault.Kind, body)
+ }
+ })
+ }
+}
+
+func reach(t *testing.T, addr string) string {
+ t.Helper()
+
+ client := &http.Client{Timeout: 3 * time.Second}
+
+ response, err := client.Get("http://" + addr)
+ if err != nil {
+ return err.Error()
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ raw, _ := io.ReadAll(response.Body)
+
+ return string(raw)
+}
+
+func TestAProxyMutationWithoutAProxyIsRefusedByName(t *testing.T) {
+ ctx := context.Background()
+ target := dockerx.Target{ID: "abc", Name: "dep"}
+
+ mutator, err := Lookup(ProxyKind(proxy.Hang))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ empty := Env{}
+
+ if _, err := mutator.Plan(ctx, empty, target); !errors.Is(err, ErrNoProxy) {
+ t.Fatalf("plan without a proxy returned %v, want ErrNoProxy", err)
+ }
+
+ if err := mutator.Apply(ctx, empty, target, nil); !errors.Is(err, ErrNoProxy) {
+ t.Fatalf("apply without a proxy returned %v, want ErrNoProxy", err)
+ }
+
+ if _, _, err := mutator.Confirm(ctx, empty, target); !errors.Is(err, ErrNoProxy) {
+ t.Fatalf("confirm without a proxy returned %v, want ErrNoProxy", err)
+ }
+}
+
+// A run cannot arm one fault and journal another.
+func TestPlanRefusesAFaultTheRunDidNotArm(t *testing.T) {
+ env := armed(t, proxy.Fault{Kind: proxy.Reset})
+
+ mutator, err := Lookup(ProxyKind(proxy.Hang))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ _, err = mutator.Plan(context.Background(), env, dockerx.Target{ID: "a", Name: "dep"})
+ if !errors.Is(err, ErrProxyFaultMismatch) {
+ t.Fatalf("err = %v, want ErrProxyFaultMismatch", err)
+ }
+}
+
+// A cold start has no proxy, and the fault provably died with the process that
+// held it. Reporting that as a failure would wedge the journal record forever,
+// and `khaos reap` stops on a sweep error, which disables the dead man's switch
+// for every other fault on the machine.
+func TestRevertingWithoutAProxyIsResolvedRatherThanFailed(t *testing.T) {
+ mutator, err := Lookup(ProxyKind(proxy.Hang))
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ if err := mutator.Revert(context.Background(), Env{}, journal.Record{}); err != nil {
+ t.Fatalf("revert on a cold start returned %v; it must resolve, because the proxy died "+
+ "with the process and the fault went with it", err)
+ }
+}
+
+// khaos inject acts on containers. A proxy fault acts on a path and an identity
+// provider trap acts on a goroutine inside the run; both exist only while the
+// run that owns them does, so neither may be offered there.
+//
+// The exclusion is asserted PER PREFIX rather than in aggregate. Checking only
+// that something is excluded lets one tier's exclusion alibi the other's: this
+// test passed the whole time the identity tier was unreachable from DirectKinds
+// by accident rather than by the registry knowing about it.
+func TestInProcessKindsAreNotOfferedForDirectInjection(t *testing.T) {
+ prefixes := []string{KindProxyPrefix, KindIdPPrefix}
+
+ for _, kind := range Kinds() {
+ wantDirect := true
+ for _, prefix := range prefixes {
+ if strings.HasPrefix(kind, prefix) {
+ wantDirect = false
+ }
+ }
+
+ if IsDirect(kind) != wantDirect {
+ t.Errorf("%s: IsDirect reports %v, want %v", kind, IsDirect(kind), wantDirect)
+ }
+ }
+
+ for _, prefix := range prefixes {
+ excluded := 0
+
+ for _, kind := range Kinds() {
+ if strings.HasPrefix(kind, prefix) {
+ excluded++
+ }
+ }
+
+ if excluded == 0 {
+ t.Errorf("no kind carries the %q prefix, so excluding it proves nothing", prefix)
+ }
+
+ for _, kind := range DirectKinds() {
+ if strings.HasPrefix(kind, prefix) {
+ t.Errorf("DirectKinds offers %s", kind)
+ }
+ }
+ }
+
+ if len(DirectKinds()) >= len(Kinds()) {
+ t.Fatal("DirectKinds excludes nothing, so the distinction is inert")
+ }
+}
+
+func TestTheProxyBindRefusesAnExposedAddress(t *testing.T) {
+ if _, err := proxy.Open("0.0.0.0:0", "127.0.0.1:1"); err == nil {
+ t.Fatal("the fault proxy bound every address on the host")
+ }
+
+ listener, err := net.Listen("tcp", loopback)
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+ _ = listener.Close()
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
new file mode 100644
index 00000000..a2f95541
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
@@ -0,0 +1,356 @@
+// ©AngelaMos | 2026
+// registry.go
+
+package inject
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "slices"
+ "strings"
+
+ "github.com/moby/moby/api/types/network"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/inject/proxy"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const (
+ KindPause = "container-pause"
+ KindDisconnect = "network-disconnect"
+
+ errUnknownKind = "%w: %q (known kinds: %s)"
+ errAlreadyPaused = "%w: %s was paused before khaos was asked to pause it, so khaos has no fault to apply and will not unpause a container it did not freeze"
+ errNoNetworks = "%s is not attached to any network, so there is nothing to disconnect"
+
+ confirmPause = "reading the container back as paused"
+ confirmDisconnect = "reading the container back as detached from every network"
+
+ describePause = "freeze every process in %s so it stops answering without closing its connections"
+ describeDisconnect = "detach %s from every network it is on, then reattach it with the same aliases, addresses and driver options"
+)
+
+var ErrAlreadyPaused = errors.New("that container is already paused")
+
+var ErrUnknownKind = errors.New("no injector is registered for this mutation kind")
+
+// Proxy, ProxyFault, IdP and IdPTrap are nil and zero for every tier that does
+// not use them. Both the fault proxy and khaos's identity provider are per-run,
+// in-process state rather than something addressable through the daemon, so they
+// travel with the environment a mutation is applied in. Nothing else can hand
+// either one to a mutator, because they do not exist until the run that owns
+// them starts and they stop existing when that run ends.
+type Env struct {
+ Docker *dockerx.Client
+ Proxy *proxy.Proxy
+ ProxyFault proxy.Fault
+ IdP *idp.Provider
+ IdPTrap idp.Trap
+}
+
+// Plan discovers everything the revert will need and returns it so the caller
+// can journal it BEFORE Apply runs. Apply consumes that payload rather than
+// re-deriving it, so a process killed mid-apply still leaves a journal record
+// that fully describes how to put the target back.
+type Mutator interface {
+ Describe(Env, dockerx.Target) string
+ Plan(context.Context, Env, dockerx.Target) (json.RawMessage, error)
+ Apply(context.Context, Env, dockerx.Target, json.RawMessage) error
+ // Confirm returns the observable it actually measured along with the verdict.
+ // A constant describing the technique is not evidence, and the run record has
+ // to carry what was seen or a reader cannot audit the run.
+ Confirm(context.Context, Env, dockerx.Target) (string, bool, error)
+ Revert(context.Context, Env, journal.Record) error
+}
+
+var registry = map[string]Mutator{
+ KindPause: pauseMutator{},
+ KindDisconnect: disconnectMutator{},
+}
+
+// Lookup names only the kinds the operator can ask for directly. Listing the
+// proxy kinds here sends a typo straight to a kind that a different message then
+// refuses, which is two refusals for one mistake.
+func Lookup(kind string) (Mutator, error) {
+ mutator, ok := registry[kind]
+ if !ok {
+ return nil, fmt.Errorf(errUnknownKind, ErrUnknownKind, kind, strings.Join(DirectKinds(), ", "))
+ }
+ return mutator, nil
+}
+
+func Kinds() []string {
+ kinds := make([]string, 0, len(registry))
+ for kind := range registry {
+ kinds = append(kinds, kind)
+ }
+ slices.Sort(kinds)
+ return kinds
+}
+
+// DirectKinds are the mutations `khaos inject` can apply on its own: the ones
+// whose whole subject is a container the daemon can be told about. A proxy fault
+// is not one, because it acts on the PATH between a target and its dependency
+// and only exists for as long as the run that owns the proxy. An identity
+// provider trap is not one either, for the same reason: the provider is a
+// goroutine inside the run. Advertising either under `khaos inject` would offer
+// a command that cannot succeed.
+func DirectKinds() []string {
+ kinds := make([]string, 0, len(registry))
+ for kind, mutator := range registry {
+ if inProcess(mutator) {
+ continue
+ }
+ kinds = append(kinds, kind)
+ }
+ slices.Sort(kinds)
+ return kinds
+}
+
+// inProcess reports whether a mutation acts on state that lives inside the run
+// rather than in the daemon. It is derived from the mutator's TYPE rather than
+// from the kind's spelling, so a mutation cannot be given an in-process-shaped
+// name and then quietly be offered under `khaos inject`, or the reverse.
+func inProcess(mutator Mutator) bool {
+ switch mutator.(type) {
+ case proxyMutator, idpMutator:
+ return true
+ default:
+ return false
+ }
+}
+
+func IsDirect(kind string) bool {
+ return slices.Contains(DirectKinds(), kind)
+}
+
+// NeedsProxy reports whether a mutation requires the run to stand up khaos's
+// in-process fault proxy. It is derived from the registry rather than from the
+// kind's spelling, so a mutation cannot be given a proxy-shaped name and then
+// quietly fail to get one, or the reverse.
+func NeedsProxy(kind string) bool {
+ mutator, ok := registry[kind]
+ if !ok {
+ return false
+ }
+ _, isProxy := mutator.(proxyMutator)
+ return isProxy
+}
+
+// NeedsIdP reports whether a mutation requires the run to stand up khaos's own
+// identity provider, derived from the registry rather than from the kind's
+// spelling for the same reason NeedsProxy is.
+func NeedsIdP(kind string) bool {
+ mutator, ok := registry[kind]
+ if !ok {
+ return false
+ }
+ _, isIdP := mutator.(idpMutator)
+ return isIdP
+}
+
+type pauseMutator struct{}
+
+func (pauseMutator) Describe(_ Env, t dockerx.Target) string {
+ return fmt.Sprintf(describePause, t.Name)
+}
+
+// pausePayload records the state the target was in BEFORE khaos touched it.
+type pausePayload struct {
+ AlreadyPaused bool `json:"already_paused"`
+}
+
+// Plan refuses a target that is already paused.
+//
+// A paused container still reports Running, so it passes every refusal
+// predicate. Docker then rejects the second pause, the session records a failed
+// mutation and immediately reverts it, and the revert sees a paused container
+// and unpauses it — khaos resuming a container the OPERATOR had paused, as
+// cleanup for a fault khaos never applied.
+//
+// Refusing here rather than recording the precondition is deliberate: khaos
+// cannot pause something already paused, so there is no experiment to run, and
+// a fault that cannot be applied should be refused by name before anything is
+// journaled.
+func (pauseMutator) Plan(ctx context.Context, env Env, t dockerx.Target) (json.RawMessage, error) {
+ paused, err := env.Docker.Paused(ctx, t.ID)
+ if err != nil {
+ return nil, err
+ }
+
+ if paused {
+ return nil, fmt.Errorf(errAlreadyPaused, ErrAlreadyPaused, t.Name)
+ }
+
+ return journal.EncodePayload(pausePayload{})
+}
+
+func (pauseMutator) Apply(ctx context.Context, env Env, t dockerx.Target, _ json.RawMessage) error {
+ return env.Docker.Pause(ctx, t.ID)
+}
+
+// Revert unpauses only what khaos paused. The journaled precondition is what
+// makes that knowable from a cold start, where khaos has no memory of the run.
+func (pauseMutator) Revert(ctx context.Context, env Env, record journal.Record) error {
+ var payload pausePayload
+ if len(record.Payload) > 0 {
+ if err := record.DecodePayload(&payload); err != nil {
+ return err
+ }
+ }
+
+ if payload.AlreadyPaused {
+ return nil
+ }
+
+ paused, err := env.Docker.Paused(ctx, record.TargetID)
+ if err != nil {
+ return err
+ }
+ if !paused {
+ return nil
+ }
+ return env.Docker.Unpause(ctx, record.TargetID)
+}
+
+type attachment struct {
+ Network string `json:"network"`
+ IPAMConfig *network.EndpointIPAMConfig `json:"ipam_config,omitempty"`
+ Aliases []string `json:"aliases,omitempty"`
+ Links []string `json:"links,omitempty"`
+ DriverOpts map[string]string `json:"driver_opts,omitempty"`
+ MacAddress string `json:"mac_address,omitempty"`
+ GwPriority int `json:"gw_priority,omitempty"`
+}
+
+type disconnectPayload struct {
+ Attachments []attachment `json:"attachments"`
+}
+
+func (a attachment) settings() *network.EndpointSettings {
+ settings := &network.EndpointSettings{
+ IPAMConfig: a.IPAMConfig,
+ Aliases: a.Aliases,
+ Links: a.Links,
+ DriverOpts: a.DriverOpts,
+ GwPriority: a.GwPriority,
+ }
+
+ if a.MacAddress != "" {
+ var mac network.HardwareAddr
+ if err := mac.UnmarshalText([]byte(a.MacAddress)); err == nil {
+ settings.MacAddress = mac
+ }
+ }
+
+ return settings
+}
+
+type disconnectMutator struct{}
+
+func (disconnectMutator) Describe(_ Env, t dockerx.Target) string {
+ return fmt.Sprintf(describeDisconnect, t.Name)
+}
+
+func (disconnectMutator) Plan(ctx context.Context, env Env, t dockerx.Target) (json.RawMessage, error) {
+ endpoints, err := env.Docker.Endpoints(ctx, t.ID)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(endpoints) == 0 {
+ return nil, fmt.Errorf(errNoNetworks, t.Name)
+ }
+
+ names := make([]string, 0, len(endpoints))
+ for name := range endpoints {
+ names = append(names, name)
+ }
+ slices.Sort(names)
+
+ payload := disconnectPayload{Attachments: make([]attachment, 0, len(names))}
+ for _, name := range names {
+ settings := endpoints[name]
+ entry := attachment{Network: name}
+ if settings != nil {
+ entry.IPAMConfig = settings.IPAMConfig
+ entry.Aliases = settings.Aliases
+ entry.Links = settings.Links
+ entry.DriverOpts = settings.DriverOpts
+ entry.GwPriority = settings.GwPriority
+ if mac := settings.MacAddress.String(); mac != "" {
+ entry.MacAddress = mac
+ }
+ }
+ payload.Attachments = append(payload.Attachments, entry)
+ }
+
+ return journal.EncodePayload(payload)
+}
+
+func (disconnectMutator) Apply(ctx context.Context, env Env, t dockerx.Target, raw json.RawMessage) error {
+ var payload disconnectPayload
+ if err := json.Unmarshal(raw, &payload); err != nil {
+ return err
+ }
+
+ for _, entry := range payload.Attachments {
+ if err := env.Docker.Disconnect(ctx, entry.Network, t.ID); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (disconnectMutator) Revert(ctx context.Context, env Env, record journal.Record) error {
+ var payload disconnectPayload
+ if err := record.DecodePayload(&payload); err != nil {
+ return err
+ }
+
+ endpoints, err := env.Docker.Endpoints(ctx, record.TargetID)
+ if err != nil {
+ return err
+ }
+
+ // Every attachment is attempted, and the failures are joined. Returning on
+ // the first one abandoned the rest: a container on three networks that lost
+ // one of them stayed detached from the other two as well, because the first
+ // failure ended the loop.
+ var failures []error
+
+ for _, entry := range payload.Attachments {
+ if _, attached := endpoints[entry.Network]; attached {
+ continue
+ }
+ if err := env.Docker.Connect(ctx, entry.Network, record.TargetID, entry.settings()); err != nil {
+ failures = append(failures, err)
+ }
+ }
+
+ return errors.Join(failures...)
+}
+
+func (pauseMutator) Confirm(ctx context.Context, env Env, t dockerx.Target) (string, bool, error) {
+ paused, err := env.Docker.Paused(ctx, t.ID)
+ if err != nil || !paused {
+ return "", false, err
+ }
+ return confirmPause, true, nil
+}
+
+func (disconnectMutator) Confirm(ctx context.Context, env Env, t dockerx.Target) (string, bool, error) {
+ endpoints, err := env.Docker.Endpoints(ctx, t.ID)
+ if err != nil {
+ return "", false, err
+ }
+ if len(endpoints) != 0 {
+ return "", false, nil
+ }
+ return confirmDisconnect, true, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go
new file mode 100644
index 00000000..b7ac0753
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go
@@ -0,0 +1,279 @@
+// ©AngelaMos | 2026
+// journal.go
+
+package journal
+
+import (
+ "bufio"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "time"
+)
+
+type Phase string
+
+const (
+ PhasePending Phase = "pending"
+ PhaseApplied Phase = "applied"
+ PhaseReverted Phase = "reverted"
+ PhaseFailed Phase = "failed"
+)
+
+const (
+ dirMode = 0o700
+ fileMode = 0o600
+
+ scanInitialBytes = 64 * 1024
+ scanMaxBytes = 1024 * 1024
+
+ tempSuffix = ".compacting"
+
+ errOpen = "open journal: %w"
+ errRead = "read journal: %w"
+ errAppend = "append to journal: %w"
+ errEncode = "encode journal record: %w"
+ errDamaged = "%w: %s line %d: %v"
+ errCompact = "compact journal: %w"
+ errPayload = "journal payload: %w"
+)
+
+var (
+ ErrDamaged = errors.New("the journal contains a line khaos cannot parse, so it may be hiding an outstanding fault")
+ ErrNoPayload = errors.New("the journal record carries no payload, so khaos cannot tell what to restore")
+)
+
+func NewID() string {
+ return rand.Text()
+}
+
+type Record struct {
+ MutationID string `json:"mutation_id"`
+ RunID string `json:"run_id"`
+ LeaseID string `json:"lease_id"`
+ Kind string `json:"kind"`
+
+ // Daemon is the Docker daemon this mutation lives on. Without it a container
+ // ID is ambiguous across machines: reclaiming against another daemon finds
+ // no such container, reads that as "the target is gone, the fault went with
+ // it", and marks the record REVERTED. The original fault stays applied on
+ // the original daemon and is now invisible to every future sweep.
+ //
+ // Empty means khaos could not establish it, which is a different state from
+ // a mismatch and must not be settled automatically.
+ Daemon string `json:"daemon,omitempty"`
+
+ TargetID string `json:"target_id"`
+ TargetName string `json:"target_name"`
+ Payload json.RawMessage `json:"payload,omitempty"`
+ Phase Phase `json:"phase"`
+ At time.Time `json:"at"`
+ ExpiresAt time.Time `json:"expires_at,omitempty"`
+ Detail string `json:"detail,omitempty"`
+}
+
+func (r Record) DecodePayload(into any) error {
+ if len(r.Payload) == 0 {
+ return ErrNoPayload
+ }
+ if err := json.Unmarshal(r.Payload, into); err != nil {
+ return fmt.Errorf(errPayload, err)
+ }
+ return nil
+}
+
+func EncodePayload(from any) (json.RawMessage, error) {
+ raw, err := json.Marshal(from)
+ if err != nil {
+ return nil, fmt.Errorf(errPayload, err)
+ }
+ return raw, nil
+}
+
+func (r Record) Expired(now time.Time) bool {
+ if r.ExpiresAt.IsZero() {
+ return true
+ }
+ return now.After(r.ExpiresAt)
+}
+
+func (r Record) Outstanding() bool {
+ return r.Phase == PhasePending || r.Phase == PhaseApplied || r.Phase == PhaseFailed
+}
+
+type Journal struct {
+ path string
+ file *os.File
+}
+
+func Open(path string) (*Journal, error) {
+ if err := os.MkdirAll(filepath.Dir(path), dirMode); err != nil {
+ return nil, fmt.Errorf(errOpen, err)
+ }
+
+ file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, fileMode)
+ if err != nil {
+ return nil, fmt.Errorf(errOpen, err)
+ }
+
+ return &Journal{path: path, file: file}, nil
+}
+
+func (j *Journal) Path() string {
+ return j.path
+}
+
+func (j *Journal) Append(record Record) error {
+ if record.At.IsZero() {
+ record.At = time.Now().UTC()
+ }
+
+ line, err := json.Marshal(record)
+ if err != nil {
+ return fmt.Errorf(errEncode, err)
+ }
+
+ if _, err := j.file.Write(append(line, '\n')); err != nil {
+ return fmt.Errorf(errAppend, err)
+ }
+
+ return j.file.Sync()
+}
+
+func (j *Journal) Close() error {
+ return j.file.Close()
+}
+
+func Replay(path string) ([]Record, error) {
+ file, err := os.Open(path)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf(errOpen, err)
+ }
+ defer func() { _ = file.Close() }()
+
+ var (
+ records []Record
+ damage error
+ )
+
+ scanner := bufio.NewScanner(file)
+ scanner.Buffer(make([]byte, 0, scanInitialBytes), scanMaxBytes)
+
+ for line := 1; scanner.Scan(); line++ {
+ raw := scanner.Bytes()
+ if len(raw) == 0 {
+ continue
+ }
+
+ var record Record
+ if err := json.Unmarshal(raw, &record); err != nil {
+ if damage == nil {
+ damage = fmt.Errorf(errDamaged, ErrDamaged, path, line, err)
+ }
+ continue
+ }
+
+ records = append(records, record)
+ }
+
+ if err := scanner.Err(); err != nil {
+ return records, fmt.Errorf(errRead, err)
+ }
+
+ return records, damage
+}
+
+func Outstanding(path string) ([]Record, error) {
+ records, damage := Replay(path)
+
+ latest := make(map[string]Record, len(records))
+ order := make([]string, 0, len(records))
+ settled := make(map[string]bool, len(records))
+
+ for _, record := range records {
+ if _, seen := latest[record.MutationID]; !seen {
+ order = append(order, record.MutationID)
+ }
+
+ // A completed reversion is TERMINAL. Plain last-write-wins let a record
+ // come back: the reaper reverts a fault and appends Reverted while a
+ // renewal that read the journal before that is still in flight, and the
+ // renewal then appends Applied with a fresh expiry from its stale
+ // snapshot. Replay believed the newer line, so khaos went on renewing a
+ // lease for a fault that no longer existed, and the next reaper sweep
+ // would try to revert a target nothing had touched.
+ //
+ // Nothing legitimately re-applies a mutation ID: a new fault gets a new
+ // one. So the first successful reversion ends that ID's life.
+ if record.Phase == PhaseReverted {
+ settled[record.MutationID] = true
+ }
+
+ latest[record.MutationID] = record
+ }
+
+ out := make([]Record, 0, len(order))
+ for _, id := range order {
+ if settled[id] {
+ continue
+ }
+ if record := latest[id]; record.Outstanding() {
+ out = append(out, record)
+ }
+ }
+
+ return out, damage
+}
+
+func Compact(path string) error {
+ outstanding, err := Outstanding(path)
+ if err != nil {
+ return err
+ }
+
+ temp := path + tempSuffix
+
+ file, err := os.OpenFile(temp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fileMode)
+ if err != nil {
+ return fmt.Errorf(errCompact, err)
+ }
+
+ encoder := json.NewEncoder(file)
+ for _, record := range outstanding {
+ if err := encoder.Encode(record); err != nil {
+ _ = file.Close()
+ _ = os.Remove(temp)
+ return fmt.Errorf(errCompact, err)
+ }
+ }
+
+ if err := file.Sync(); err != nil {
+ _ = file.Close()
+ _ = os.Remove(temp)
+ return fmt.Errorf(errCompact, err)
+ }
+
+ if err := file.Close(); err != nil {
+ _ = os.Remove(temp)
+ return fmt.Errorf(errCompact, err)
+ }
+
+ return os.Rename(temp, path)
+}
+
+// ShortDaemon trims a daemon ID for display, the way container IDs are trimmed.
+// The full value stays in the record; this is only for a human reading an error.
+func ShortDaemon(id string) string {
+ if len(id) <= shortDaemonLength {
+ return id
+ }
+ return id[:shortDaemonLength]
+}
+
+const shortDaemonLength = 12
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go
new file mode 100644
index 00000000..1cf2d61c
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go
@@ -0,0 +1,380 @@
+// ©AngelaMos | 2026
+// journal_test.go
+
+package journal
+
+import (
+ "errors"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+const (
+ testFile = "journal.jsonl"
+ wantDirMode = fs.FileMode(0o700)
+)
+
+func tempJournal(t *testing.T) *Journal {
+ t.Helper()
+
+ j, err := Open(filepath.Join(t.TempDir(), "nested", testFile))
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ t.Cleanup(func() { _ = j.Close() })
+
+ return j
+}
+
+func record(id string, phase Phase) Record {
+ return Record{
+ MutationID: id,
+ RunID: "run-1",
+ LeaseID: "lease-1",
+ Kind: "container-pause",
+ TargetID: "abc123",
+ TargetName: "shop-api-1",
+ Phase: phase,
+ }
+}
+
+type payload struct {
+ Networks []string `json:"networks"`
+}
+
+func TestAppendAndReplayRoundTrip(t *testing.T) {
+ j := tempJournal(t)
+
+ raw, err := EncodePayload(payload{Networks: []string{"shop_default", "shop_edge"}})
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+
+ want := record("m1", PhaseApplied)
+ want.Payload = raw
+ want.ExpiresAt = time.Now().UTC().Add(time.Minute).Truncate(time.Second)
+
+ if err := j.Append(want); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+
+ got, err := Replay(j.Path())
+ if err != nil {
+ t.Fatalf("replay: %v", err)
+ }
+
+ if len(got) != 1 {
+ t.Fatalf("replayed %d records, want 1", len(got))
+ }
+
+ var decoded payload
+ if err := got[0].DecodePayload(&decoded); err != nil {
+ t.Fatalf("decode payload: %v", err)
+ }
+
+ if len(decoded.Networks) != 2 || decoded.Networks[0] != "shop_default" {
+ t.Fatalf("payload did not survive the round trip: %+v", decoded)
+ }
+
+ if got[0].At.IsZero() {
+ t.Fatal("append did not stamp a time")
+ }
+}
+
+func TestDecodingAnAbsentPayloadIsNamed(t *testing.T) {
+ if err := record("m1", PhaseApplied).DecodePayload(&payload{}); !errors.Is(err, ErrNoPayload) {
+ t.Fatalf("got %v, want ErrNoPayload so a revert can tell empty from corrupt", err)
+ }
+}
+
+func TestOutstandingCollapsesToTheLatestPhaseAndKeepsOrder(t *testing.T) {
+ j := tempJournal(t)
+
+ for _, r := range []Record{
+ record("m1", PhasePending),
+ record("m1", PhaseApplied),
+ record("m2", PhasePending),
+ record("m2", PhaseApplied),
+ record("m2", PhaseReverted),
+ record("m3", PhasePending),
+ } {
+ if err := j.Append(r); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+ }
+
+ out, err := Outstanding(j.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(out) != 2 {
+ t.Fatalf("got %d outstanding, want 2 (m1 applied, m3 pending)", len(out))
+ }
+
+ if out[0].MutationID != "m1" || out[1].MutationID != "m3" {
+ t.Fatalf("outstanding came back as %s,%s; first-seen order must hold because "+
+ "reclaim reverts in reverse to undo most-recent-first",
+ out[0].MutationID, out[1].MutationID)
+ }
+}
+
+func TestOutstandingKeepsTheLatestPayload(t *testing.T) {
+ j := tempJournal(t)
+
+ raw, err := EncodePayload(payload{Networks: []string{"shop_default"}})
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+
+ pending := record("m1", PhasePending)
+ applied := record("m1", PhaseApplied)
+ applied.Payload = raw
+
+ for _, r := range []Record{pending, applied} {
+ if err := j.Append(r); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+ }
+
+ out, err := Outstanding(j.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ var decoded payload
+ if err := out[0].DecodePayload(&decoded); err != nil {
+ t.Fatalf("the collapse dropped the payload, so a revert would silently do nothing: %v", err)
+ }
+
+ if len(decoded.Networks) != 1 {
+ t.Fatalf("payload came back as %+v", decoded)
+ }
+}
+
+func TestPendingAndFailedAreBothOutstanding(t *testing.T) {
+ if !record("m1", PhasePending).Outstanding() {
+ t.Fatal("a mutation journaled before it was applied must still be reclaimed, " +
+ "because khaos may have died between the write and the apply")
+ }
+
+ if !record("m1", PhaseFailed).Outstanding() {
+ t.Fatal("a mutation whose apply failed may have partially landed and must stay reclaimable")
+ }
+
+ if record("m1", PhaseReverted).Outstanding() {
+ t.Fatal("a reverted mutation must not be reclaimed again")
+ }
+}
+
+func TestARecordWithNoLeaseIsTreatedAsExpired(t *testing.T) {
+ r := record("m1", PhaseApplied)
+
+ if !r.Expired(time.Now().UTC()) {
+ t.Fatal("a record carrying no lease was treated as immortal; every injected fault " +
+ "must carry a lease, so a record without one is malformed and must fail safe")
+ }
+
+ r.ExpiresAt = time.Now().UTC().Add(time.Hour)
+ if r.Expired(time.Now().UTC()) {
+ t.Fatal("a live lease was reported expired")
+ }
+}
+
+func TestReplayOfAMissingJournalIsEmptyNotAnError(t *testing.T) {
+ got, err := Replay(filepath.Join(t.TempDir(), "absent.jsonl"))
+ if err != nil {
+ t.Fatalf("replay of a missing journal errored: %v", err)
+ }
+ if got != nil {
+ t.Fatalf("got %d records from a missing journal", len(got))
+ }
+}
+
+func TestADamagedLineIsQuarantinedNotFatal(t *testing.T) {
+ path := filepath.Join(t.TempDir(), testFile)
+
+ body := "{\"mutation_id\":\"m1\",\"phase\":\"applied\"}\n" +
+ "{\"mutation_id\":\"m2\",\"phase\":\"appl\n" +
+ "{\"mutation_id\":\"m3\",\"phase\":\"pending\"}\n"
+
+ if err := os.WriteFile(path, []byte(body), fileMode); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ out, err := Outstanding(path)
+
+ if !errors.Is(err, ErrDamaged) {
+ t.Fatalf("got %v; a damaged line must be reported, or khaos silently claims completeness", err)
+ }
+
+ if len(out) != 2 {
+ t.Fatalf("got %d outstanding records; the readable records must still be acted on, "+
+ "or one bad line strands every fault on the machine forever", len(out))
+ }
+}
+
+func TestCompactKeepsOnlyOutstandingRecords(t *testing.T) {
+ j := tempJournal(t)
+
+ for _, r := range []Record{
+ record("m1", PhasePending),
+ record("m1", PhaseApplied),
+ record("m1", PhaseReverted),
+ record("m2", PhasePending),
+ record("m2", PhaseApplied),
+ } {
+ if err := j.Append(r); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+ }
+
+ before, err := os.Stat(j.Path())
+ if err != nil {
+ t.Fatalf("stat: %v", err)
+ }
+
+ if err := Compact(j.Path()); err != nil {
+ t.Fatalf("compact: %v", err)
+ }
+
+ after, err := os.Stat(j.Path())
+ if err != nil {
+ t.Fatalf("stat: %v", err)
+ }
+
+ if after.Size() >= before.Size() {
+ t.Fatalf("compaction grew the journal from %d to %d bytes", before.Size(), after.Size())
+ }
+
+ out, err := Outstanding(j.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(out) != 1 || out[0].MutationID != "m2" {
+ t.Fatalf("compaction changed what is outstanding: %+v", out)
+ }
+}
+
+func TestTheJournalIsNotWorldReadable(t *testing.T) {
+ j := tempJournal(t)
+
+ if err := j.Append(record("m1", PhaseApplied)); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+
+ info, err := os.Stat(j.Path())
+ if err != nil {
+ t.Fatalf("stat: %v", err)
+ }
+
+ if info.Mode().Perm() != fileMode {
+ t.Fatalf("journal mode is %v, want %v; it records container names and network topology",
+ info.Mode().Perm(), fs.FileMode(fileMode))
+ }
+
+ dir, err := os.Stat(filepath.Dir(j.Path()))
+ if err != nil {
+ t.Fatalf("stat dir: %v", err)
+ }
+
+ if dir.Mode().Perm() != wantDirMode {
+ t.Fatalf("state directory mode is %v, want %v", dir.Mode().Perm(), wantDirMode)
+ }
+}
+
+func TestAppendSurfacesAWriteFailure(t *testing.T) {
+ j := tempJournal(t)
+
+ if err := j.Close(); err != nil {
+ t.Fatalf("close: %v", err)
+ }
+
+ if err := j.Append(record("m1", PhaseApplied)); err == nil {
+ t.Fatal("append on a closed journal reported success, so a durability failure " +
+ "would be swallowed while the fault stays applied")
+ }
+}
+
+func TestIDsAreDistinct(t *testing.T) {
+ seen := make(map[string]bool, 128)
+
+ for range 128 {
+ id := NewID()
+ if seen[id] {
+ t.Fatalf("duplicate id %q; two mutations would collapse into one journal entry", id)
+ }
+ seen[id] = true
+ }
+}
+
+// A completed reversion is terminal. Plain last-write-wins let a stale renewal
+// resurrect a fault the reaper had already undone, and khaos then renewed a
+// lease for a target nothing was holding.
+func TestAReversionCannotBeUndoneByALaterApplied(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "journal.ndjson")
+
+ log, err := Open(path)
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ defer func() { _ = log.Close() }()
+
+ const id = "mutation-1"
+
+ for _, record := range []Record{
+ {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhasePending},
+ {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseApplied},
+ // The reaper wins the race and reverts.
+ {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseReverted},
+ // A renewal that read the journal BEFORE that reversion lands late.
+ {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseApplied},
+ } {
+ if err := log.Append(record); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+ }
+
+ outstanding, err := Outstanding(path)
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(outstanding) != 0 {
+ t.Fatalf("a reverted mutation came back as outstanding: %+v", outstanding)
+ }
+}
+
+// And the control: a mutation that was never reverted is still outstanding, or
+// the rule above would simply have hidden every fault.
+func TestAnAppliedMutationIsStillOutstanding(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "journal.ndjson")
+
+ log, err := Open(path)
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ defer func() { _ = log.Close() }()
+
+ for _, record := range []Record{
+ {MutationID: "m", Kind: "container-pause", TargetID: "abc", Phase: PhasePending},
+ {MutationID: "m", Kind: "container-pause", TargetID: "abc", Phase: PhaseApplied},
+ } {
+ if err := log.Append(record); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+ }
+
+ outstanding, err := Outstanding(path)
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(outstanding) != 1 {
+ t.Fatalf("an applied mutation is not outstanding: %+v", outstanding)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/lease/lease.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/lease/lease.go
new file mode 100644
index 00000000..0be3f9bf
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/lease/lease.go
@@ -0,0 +1,91 @@
+// ©AngelaMos | 2026
+// lease.go
+
+package lease
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const (
+ DefaultTTL = 30 * time.Second
+ MinimumTTL = 2 * time.Second
+ renewalDivisor = 3
+ minimumTick = 250 * time.Millisecond
+)
+
+type Lease struct {
+ id string
+ ttl time.Duration
+
+ mu sync.RWMutex
+ expiresAt time.Time
+}
+
+func New(ttl time.Duration) *Lease {
+ if ttl < MinimumTTL {
+ ttl = MinimumTTL
+ }
+
+ l := &Lease{id: journal.NewID(), ttl: ttl}
+ l.Renew()
+
+ return l
+}
+
+func (l *Lease) ID() string {
+ return l.id
+}
+
+func (l *Lease) TTL() time.Duration {
+ return l.ttl
+}
+
+func (l *Lease) ExpiresAt() time.Time {
+ l.mu.RLock()
+ defer l.mu.RUnlock()
+ return l.expiresAt
+}
+
+func (l *Lease) Renew() time.Time {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.expiresAt = time.Now().UTC().Add(l.ttl)
+ return l.expiresAt
+}
+
+func (l *Lease) Expired(now time.Time) bool {
+ return now.After(l.ExpiresAt())
+}
+
+func (l *Lease) RenewalInterval() time.Duration {
+ interval := l.ttl / renewalDivisor
+ if interval < minimumTick {
+ interval = minimumTick
+ }
+ return interval
+}
+
+func (l *Lease) Keep(ctx context.Context, onRenew func(time.Time) error) error {
+ ticker := time.NewTicker(l.RenewalInterval())
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-ticker.C:
+ expiry := l.Renew()
+ if onRenew == nil {
+ continue
+ }
+ if err := onRenew(expiry); err != nil {
+ return err
+ }
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/lease/lease_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/lease/lease_test.go
new file mode 100644
index 00000000..ea1d4558
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/lease/lease_test.go
@@ -0,0 +1,150 @@
+// ©AngelaMos | 2026
+// lease_test.go
+
+package lease
+
+import (
+ "context"
+ "errors"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func TestNewLeaseIsAliveImmediately(t *testing.T) {
+ l := New(time.Minute)
+
+ if l.Expired(time.Now().UTC()) {
+ t.Fatal("a freshly issued lease was already expired")
+ }
+
+ if l.ID() == "" {
+ t.Fatal("a lease was issued with no identifier, so the journal cannot attribute a fault to it")
+ }
+}
+
+func TestTTLIsClampedToAFloor(t *testing.T) {
+ l := New(time.Millisecond)
+
+ if l.TTL() != MinimumTTL {
+ t.Fatalf("TTL %v was accepted; a lease shorter than %v would expire mid-injection", l.TTL(), MinimumTTL)
+ }
+}
+
+func TestLeaseExpiresWithoutRenewal(t *testing.T) {
+ l := New(MinimumTTL)
+
+ if l.Expired(time.Now().UTC()) {
+ t.Fatal("expired too early")
+ }
+
+ if !l.Expired(time.Now().UTC().Add(MinimumTTL + time.Second)) {
+ t.Fatal("a lease that is never renewed must expire, or the dead man's switch never fires")
+ }
+}
+
+func TestRenewPushesExpiryOut(t *testing.T) {
+ l := New(time.Minute)
+ first := l.ExpiresAt()
+
+ time.Sleep(10 * time.Millisecond)
+
+ if second := l.Renew(); !second.After(first) {
+ t.Fatalf("renew did not extend the lease: %v then %v", first, second)
+ }
+}
+
+func TestRenewalAlwaysFitsTwiceInsideTheTTL(t *testing.T) {
+ ttls := []time.Duration{
+ time.Millisecond, MinimumTTL, 3 * time.Second, 10 * time.Second,
+ DefaultTTL, time.Minute, 10 * time.Minute, time.Hour,
+ }
+
+ for _, ttl := range ttls {
+ t.Run(ttl.String(), func(t *testing.T) {
+ l := New(ttl)
+
+ if l.RenewalInterval() >= l.TTL() {
+ t.Fatalf("renewal interval %v is not shorter than TTL %v, so one missed tick drops the fault",
+ l.RenewalInterval(), l.TTL())
+ }
+
+ if l.TTL()/l.RenewalInterval() < 2 {
+ t.Fatalf("only %v fits inside TTL %v; a single hiccup would expire a live fault",
+ l.TTL()/l.RenewalInterval(), l.TTL())
+ }
+ })
+ }
+}
+
+func TestKeepRenewsUntilCancelled(t *testing.T) {
+ l := New(MinimumTTL)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ var renewals atomic.Int64
+ done := make(chan error, 1)
+
+ go func() {
+ done <- l.Keep(ctx, func(time.Time) error {
+ renewals.Add(1)
+ return nil
+ })
+ }()
+
+ // Read the lease concurrently with the keeper, so -race is the negative
+ // control for the mutex guarding expiresAt.
+ deadline := time.Now().Add(l.RenewalInterval() * 2)
+ for time.Now().Before(deadline) {
+ _ = l.ExpiresAt()
+ _ = l.Expired(time.Now().UTC())
+ }
+ cancel()
+
+ if err := <-done; err != context.Canceled {
+ t.Fatalf("Keep returned %v, want context.Canceled", err)
+ }
+
+ if renewals.Load() == 0 {
+ t.Fatal("Keep never renewed, so a held fault would expire under a live khaos")
+ }
+}
+
+func TestKeepStopsWhenRenewalFails(t *testing.T) {
+ l := New(MinimumTTL)
+
+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+ defer cancel()
+
+ // The sentinel must be distinct from any error the context itself can
+ // produce. An earlier version of this test returned context.DeadlineExceeded
+ // from the callback under a short deadline, so a Keep that ignored the
+ // callback entirely still returned the expected value and the test passed.
+ want := errors.New("journal write failed")
+
+ started := time.Now()
+ got := l.Keep(ctx, func(time.Time) error { return want })
+ elapsed := time.Since(started)
+
+ if !errors.Is(got, want) {
+ t.Fatalf("Keep returned %v; a journal write failure during renewal must surface, "+
+ "not be swallowed while the fault stays applied", got)
+ }
+
+ if elapsed > l.RenewalInterval()*3 {
+ t.Fatalf("Keep took %v to notice a failing renewal, which is longer than it should wait", elapsed)
+ }
+}
+
+func TestLeaseIDsAreDistinct(t *testing.T) {
+ seen := make(map[string]bool, 64)
+
+ for range 64 {
+ id := New(time.Minute).ID()
+ if seen[id] {
+ t.Fatalf("duplicate lease id %q; two runs would reclaim each other's faults", id)
+ }
+ seen[id] = true
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/lock/lock_unix.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/lock/lock_unix.go
new file mode 100644
index 00000000..a482eed7
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/lock/lock_unix.go
@@ -0,0 +1,116 @@
+// ©AngelaMos | 2026
+// lock_unix.go
+
+package lock
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "golang.org/x/sys/unix"
+)
+
+const (
+ dirMode = 0o700
+ fileMode = 0o600
+
+ holderBufBytes = 32
+
+ errOpen = "open lock file: %w"
+ errWrite = "record lock holder: %w"
+ errHeldBy = "%w (pid %s). wait for it to finish, or stop it and run `khaos reclaim` to revert anything it left behind"
+ errUnlockable = "%w: %v"
+ fixUnsupported = ". set KHAOS_STATE_DIR to a local filesystem that supports file locking"
+)
+
+var (
+ ErrHeld = errors.New("another khaos instance is already running")
+ ErrUnlockable = errors.New("khaos could not take its instance lock, so it cannot be sure another run is not already holding a fault")
+ unsupportedFS = []error{unix.ENOLCK, unix.ENOSYS, unix.EOPNOTSUPP}
+ ErrUnsupported = errors.New("this filesystem does not support file locking" + fixUnsupported)
+)
+
+type Lock struct {
+ file *os.File
+}
+
+func Acquire(path string) (*Lock, error) {
+ if err := os.MkdirAll(filepath.Dir(path), dirMode); err != nil {
+ return nil, fmt.Errorf(errOpen, err)
+ }
+
+ file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, fileMode)
+ if err != nil {
+ return nil, fmt.Errorf(errOpen, err)
+ }
+
+ if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
+ holder := readHolder(file)
+ _ = file.Close()
+ return nil, classify(err, holder)
+ }
+
+ if err := recordHolder(file); err != nil {
+ _ = unix.Flock(int(file.Fd()), unix.LOCK_UN)
+ _ = file.Close()
+ return nil, err
+ }
+
+ return &Lock{file: file}, nil
+}
+
+func (l *Lock) Release() error {
+ if err := unix.Flock(int(l.file.Fd()), unix.LOCK_UN); err != nil {
+ _ = l.file.Close()
+ return err
+ }
+ return l.file.Close()
+}
+
+func classify(err error, holder string) error {
+ if errors.Is(err, unix.EWOULDBLOCK) {
+ return fmt.Errorf(errHeldBy, ErrHeld, orUnknown(holder))
+ }
+
+ for _, unsupported := range unsupportedFS {
+ if errors.Is(err, unsupported) {
+ return fmt.Errorf(errUnlockable, ErrUnsupported, err)
+ }
+ }
+
+ return fmt.Errorf(errUnlockable, ErrUnlockable, err)
+}
+
+func recordHolder(file *os.File) error {
+ if err := file.Truncate(0); err != nil {
+ return fmt.Errorf(errWrite, err)
+ }
+
+ if _, err := file.WriteAt([]byte(strconv.Itoa(os.Getpid())), 0); err != nil {
+ return fmt.Errorf(errWrite, err)
+ }
+
+ return file.Sync()
+}
+
+func readHolder(file *os.File) string {
+ buf := make([]byte, holderBufBytes)
+
+ n, _ := file.ReadAt(buf, 0)
+ if n == 0 {
+ return ""
+ }
+
+ return strings.TrimSpace(string(buf[:n]))
+}
+
+func orUnknown(holder string) string {
+ if holder == "" {
+ return "unknown"
+ }
+ return holder
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/lock/lock_unix_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/lock/lock_unix_test.go
new file mode 100644
index 00000000..72b55631
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/lock/lock_unix_test.go
@@ -0,0 +1,187 @@
+// ©AngelaMos | 2026
+// lock_unix_test.go
+
+package lock
+
+import (
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+)
+
+const (
+ envHolder = "KHAOS_TEST_LOCK_HOLDER"
+ envPath = "KHAOS_TEST_LOCK_PATH"
+ holdFor = 10 * time.Second
+ readyMarker = "HELD"
+)
+
+func TestMain(m *testing.M) {
+ if os.Getenv(envHolder) == "" {
+ os.Exit(m.Run())
+ }
+
+ held, err := Acquire(os.Getenv(envPath))
+ if err != nil {
+ os.Exit(1)
+ }
+
+ _, _ = os.Stdout.WriteString(readyMarker + "\n")
+ time.Sleep(holdFor)
+ _ = held.Release()
+ os.Exit(0)
+}
+
+func lockPath(t *testing.T) string {
+ t.Helper()
+ return filepath.Join(t.TempDir(), "khaos.lock")
+}
+
+func holderProcess(t *testing.T, path string) *exec.Cmd {
+ t.Helper()
+
+ cmd := exec.Command(os.Args[0])
+ cmd.Env = append(os.Environ(), envHolder+"=1", envPath+"="+path)
+
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ t.Fatalf("stdout pipe: %v", err)
+ }
+
+ if err := cmd.Start(); err != nil {
+ t.Fatalf("start holder: %v", err)
+ }
+
+ t.Cleanup(func() {
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ })
+
+ buf := make([]byte, len(readyMarker)+1)
+ if _, err := stdout.Read(buf); err != nil {
+ t.Fatalf("holder never signalled it had the lock: %v", err)
+ }
+
+ return cmd
+}
+
+func TestASecondAcquireIsRefused(t *testing.T) {
+ path := lockPath(t)
+
+ first, err := Acquire(path)
+ if err != nil {
+ t.Fatalf("first acquire: %v", err)
+ }
+ defer func() { _ = first.Release() }()
+
+ second, err := Acquire(path)
+ if err == nil {
+ _ = second.Release()
+ t.Fatal("two khaos instances held the lock at once, so both could inject into the same target")
+ }
+
+ if !errors.Is(err, ErrHeld) {
+ t.Fatalf("got %v, want an error wrapping ErrHeld", err)
+ }
+}
+
+func TestTheRefusalNamesTheHoldingProcessAndTheFix(t *testing.T) {
+ path := lockPath(t)
+ holder := holderProcess(t, path)
+
+ _, err := Acquire(path)
+ if err == nil {
+ t.Fatal("acquired a lock another process was holding")
+ }
+
+ wantPid := strconv.Itoa(holder.Process.Pid)
+ if !strings.Contains(err.Error(), wantPid) {
+ t.Fatalf("the refusal %q does not name the holding pid %s, so the operator cannot find it",
+ err, wantPid)
+ }
+
+ if !strings.Contains(err.Error(), "khaos reclaim") {
+ t.Fatalf("the refusal %q names a cause but no fix", err)
+ }
+}
+
+func TestTheLockIsReleasedWhenTheHolderIsKilled(t *testing.T) {
+ path := lockPath(t)
+ holder := holderProcess(t, path)
+
+ if _, err := Acquire(path); err == nil {
+ t.Fatal("the lock was free while a holder was alive")
+ }
+
+ if err := holder.Process.Kill(); err != nil {
+ t.Fatalf("kill holder: %v", err)
+ }
+ _ = holder.Wait()
+
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ if held, err := Acquire(path); err == nil {
+ _ = held.Release()
+ return
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+
+ t.Fatal("the lock stayed held after its owner was killed; flock was chosen precisely " +
+ "so the kernel releases it, and a stale lock would block every later khaos run")
+}
+
+func TestReleaseAllowsReacquisition(t *testing.T) {
+ path := lockPath(t)
+
+ first, err := Acquire(path)
+ if err != nil {
+ t.Fatalf("first acquire: %v", err)
+ }
+
+ if err := first.Release(); err != nil {
+ t.Fatalf("release: %v", err)
+ }
+
+ second, err := Acquire(path)
+ if err != nil {
+ t.Fatalf("the lock was not released: %v", err)
+ }
+
+ if err := second.Release(); err != nil {
+ t.Fatalf("second release: %v", err)
+ }
+}
+
+func TestAcquireCreatesTheStateDirectoryPrivately(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "nested", "deeper", "khaos.lock")
+
+ held, err := Acquire(path)
+ if err != nil {
+ t.Fatalf("acquire: %v", err)
+ }
+ defer func() { _ = held.Release() }()
+
+ info, err := os.Stat(path)
+ if err != nil {
+ t.Fatalf("lock file was not created: %v", err)
+ }
+
+ if info.Mode().Perm() != fileMode {
+ t.Fatalf("lock mode is %v, want %v", info.Mode().Perm(), os.FileMode(fileMode))
+ }
+
+ dir, err := os.Stat(filepath.Dir(path))
+ if err != nil {
+ t.Fatalf("stat dir: %v", err)
+ }
+
+ if dir.Mode().Perm() != dirMode {
+ t.Fatalf("state directory mode is %v, want %v", dir.Mode().Perm(), os.FileMode(dirMode))
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go
new file mode 100644
index 00000000..58b6ac54
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go
@@ -0,0 +1,211 @@
+// ©AngelaMos | 2026
+// http.go
+
+package probe
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "time"
+)
+
+const (
+ DefaultTimeout = 5 * time.Second
+
+ headerAuthorization = "Authorization"
+ bearerPrefix = "Bearer "
+ headerUserAgent = "User-Agent"
+ userAgentValue = "khaos"
+
+ describeHTTP = "%s %s with a %s credential"
+ describeHTTPMint = "%s %s with a %s credential minted per request"
+ detailError = "%v"
+ detailStatus = "HTTP %d"
+ detailMint = "khaos could not mint the credential for this probe: %v"
+ nonceSep = "."
+)
+
+type HTTPOptions struct {
+ Name string
+ Method string
+ URL string
+ Token string
+ Credential Credential
+ Timeout time.Duration
+
+ // Unique appends a per-request nonce to the token. A negative control that
+ // reuses one fixed credential keeps the target's decision cache warm, so
+ // the cached refusal answers every probe and a control that stops
+ // enforcing during the fault is never exercised.
+ Unique bool
+
+ // Mint supplies the credential per request instead of Token. A forged stream
+ // needs it: the provider's arming changes between the baseline and the fault,
+ // so a credential captured once would carry the baseline's honest token into
+ // the fault window and the trap would never be presented at all.
+ //
+ // A mint failure is khaos's own fault and is reported as ProbeFailed, which
+ // every rate already excludes. Folding it into a target outcome would
+ // manufacture a finding out of khaos's own bug.
+ Mint func(context.Context) (string, error)
+}
+
+type HTTPProbe struct {
+ options HTTPOptions
+ client *http.Client
+}
+
+func NewHTTP(options HTTPOptions) *HTTPProbe {
+ if options.Method == "" {
+ options.Method = http.MethodGet
+ }
+ if options.Timeout <= 0 {
+ options.Timeout = DefaultTimeout
+ }
+
+ return &HTTPProbe{
+ options: options,
+ client: &http.Client{
+ Timeout: options.Timeout,
+ Transport: &http.Transport{
+ // TLS session resumption does not repeat chain verification, so
+ // a probe that resumes passes against a certificate that has
+ // since expired. A nil cache is what actually disables it:
+ // crypto/tls only stores and loads sessions when one is set.
+ TLSClientConfig: &tls.Config{ClientSessionCache: nil, MinVersion: tls.VersionTLS12},
+ DisableKeepAlives: true,
+ // Setting TLSClientConfig disables automatic HTTP/2 unless this
+ // is set, and h2 connection reuse is itself a fail-open khaos
+ // needs to be able to observe.
+ ForceAttemptHTTP2: true,
+ },
+ CheckRedirect: func(*http.Request, []*http.Request) error {
+ return http.ErrUseLastResponse
+ },
+ },
+ }
+}
+
+func (p *HTTPProbe) Name() string {
+ return p.options.Name
+}
+
+func (p *HTTPProbe) Credential() Credential {
+ return p.options.Credential
+}
+
+func (p *HTTPProbe) Describe() string {
+ if p.options.Mint != nil {
+ return fmt.Sprintf(describeHTTPMint, p.options.Method, p.options.URL, p.options.Credential)
+ }
+ return fmt.Sprintf(describeHTTP, p.options.Method, p.options.URL, p.options.Credential)
+}
+
+func (p *HTTPProbe) Run(ctx context.Context) Outcome {
+ started := time.Now().UTC()
+
+ outcome := Outcome{
+ At: started,
+ Credential: p.options.Credential,
+ Disposition: Unknown,
+ }
+
+ request, err := http.NewRequestWithContext(ctx, p.options.Method, p.options.URL, nil)
+ if err != nil {
+ outcome.Disposition = ProbeFailed
+ outcome.Detail = fmt.Sprintf(detailError, err)
+ return outcome
+ }
+
+ request.Header.Set(headerUserAgent, userAgentValue)
+
+ token, err := p.credential(ctx)
+ if err != nil {
+ outcome.Disposition = ProbeFailed
+ outcome.Detail = fmt.Sprintf(detailMint, err)
+ return outcome
+ }
+
+ if token != "" {
+ request.Header.Set(headerAuthorization, bearerPrefix+token)
+ }
+
+ response, err := p.client.Do(request)
+ outcome.SetLatency(time.Since(started))
+
+ if err != nil {
+ outcome.Disposition = transportDisposition(err)
+ outcome.Detail = fmt.Sprintf(detailError, err)
+ return outcome
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ // The body must be read. Under TLS 1.3 a handshake rejection surfaces on
+ // the first read rather than at dial, so a probe that never reads reports
+ // success against a server that refused its identity.
+ if _, err := io.Copy(io.Discard, response.Body); err != nil {
+ outcome.Disposition = transportDisposition(err)
+ outcome.Detail = fmt.Sprintf(detailError, err)
+ return outcome
+ }
+
+ outcome.SetLatency(time.Since(started))
+ outcome.Status = response.StatusCode
+ outcome.Disposition = statusDisposition(response.StatusCode)
+ outcome.Detail = fmt.Sprintf(detailStatus, response.StatusCode)
+
+ return outcome
+}
+
+// credential resolves what this request carries. A minted credential is never
+// given a nonce: the mint already produced a fresh one, and appending to a
+// signed token would corrupt it into a different fault than the one armed.
+func (p *HTTPProbe) credential(ctx context.Context) (string, error) {
+ if p.options.Mint != nil {
+ return p.options.Mint(ctx)
+ }
+
+ if p.options.Token == "" || !p.options.Unique {
+ return p.options.Token, nil
+ }
+
+ return p.options.Token + nonceSep + rand.Text(), nil
+}
+
+func transportDisposition(err error) Disposition {
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, os.ErrDeadlineExceeded) {
+ return TimedOut
+ }
+ if errors.Is(err, context.Canceled) {
+ return ProbeFailed
+ }
+ return Unavailable
+}
+
+// statusDisposition maps a status to what it says about the CREDENTIAL, which
+// is not the same question as whether the request succeeded.
+func statusDisposition(status int) Disposition {
+ switch {
+ case status == http.StatusTooManyRequests:
+ return Throttled
+ case status == http.StatusUnauthorized, status == http.StatusForbidden:
+ return Rejected
+ case status >= http.StatusInternalServerError:
+ return Unavailable
+ case status >= http.StatusMultipleChoices && status < http.StatusBadRequest:
+ // A redirect is how most browser-facing gateways refuse an
+ // unauthenticated request, and it is also how they route an
+ // authenticated one. It decides nothing about the credential.
+ return Redirected
+ case status >= http.StatusOK && status < http.StatusMultipleChoices:
+ return Accepted
+ default:
+ return Rejected
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go
new file mode 100644
index 00000000..fe527a12
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go
@@ -0,0 +1,154 @@
+// ©AngelaMos | 2026
+// probe.go
+
+package probe
+
+import (
+ "context"
+ "time"
+)
+
+type Disposition string
+
+const (
+ // Unknown is the zero value on purpose: a half-built Outcome must read as
+ // obviously wrong rather than as a plausible target failure.
+ Unknown Disposition = "unknown"
+
+ Accepted Disposition = "accepted"
+ Rejected Disposition = "rejected"
+ Redirected Disposition = "redirected"
+ Throttled Disposition = "throttled"
+ Unavailable Disposition = "unavailable"
+ TimedOut Disposition = "timed-out"
+
+ // ProbeFailed is khaos's own fault, never the target's. It is excluded from
+ // every rate, because folding a tool bug into a target outcome manufactures
+ // findings out of a typo in --url.
+ ProbeFailed Disposition = "probe-failed"
+)
+
+func (d Disposition) String() string {
+ if d == "" {
+ return string(Unknown)
+ }
+ return string(d)
+}
+
+// Answered reports whether the target responded at all, whatever it said.
+func (d Disposition) Answered() bool {
+ switch d {
+ case Accepted, Rejected, Redirected, Throttled:
+ return true
+ default:
+ return false
+ }
+}
+
+// Countable excludes khaos's own failures from every rate computation.
+func (d Disposition) Countable() bool {
+ return d != ProbeFailed && d != Unknown
+}
+
+type Credential string
+
+const (
+ Valid Credential = "valid"
+ Invalid Credential = "invalid"
+
+ // Revoked is a credential that was LEGITIMATELY accepted and that khaos then
+ // revoked at the authority during the fault. It is a third kind rather than a
+ // flavour of Invalid because the two make different claims: an invalid
+ // credential was never acceptable, and a revoked one was and then stopped
+ // being. Reporting the second as the first would say the target honoured
+ // something it had already rejected, which is not what happened.
+ Revoked Credential = "revoked"
+
+ // Forged is a credential khaos MINTED to be wrong in exactly one named way,
+ // and confirmed wrong at the provider before presenting it. It is a fourth
+ // kind rather than a flavour of Invalid because the claims differ again, and
+ // because the baseline requirement is different: the invalid stream has to be
+ // REJECTED during the baseline to prove the target enforces anything at all,
+ // while a forged stream's baseline requirement depends on the trap and is
+ // declared by the experiment.
+ //
+ // Folding it into Invalid also breaks the run outright for the traps whose
+ // forged credential is accepted at steady state: the baseline check would
+ // report "the invalid credential was never rejected" and void the run, which
+ // is khaos refusing to report the very finding it went looking for.
+ Forged Credential = "forged"
+)
+
+// Expectation is what a stream must do during the baseline for the fault window
+// to mean anything. It is DECLARED per experiment rather than derived, because
+// it points in opposite directions for traps that look alike: a token forged to
+// be malformed has to be refused before the fault, and a token that is correct
+// until the provider is armed has to be ACCEPTED before it.
+type Expectation string
+
+const (
+ // ExpectAccepted is the liveness guard for a trap whose credential is
+ // honest until the fault lands. If the target never accepted one, it was not
+ // trusting khaos's provider and its later refusals say nothing.
+ ExpectAccepted Expectation = "accepted"
+
+ // ExpectRejected is the enforcement guard for a trap whose credential is
+ // already malformed. If the target never refused one, it was not checking
+ // the thing the fault is about to break.
+ ExpectRejected Expectation = "rejected"
+)
+
+func (c Credential) String() string {
+ return string(c)
+}
+
+type Outcome struct {
+ At time.Time `json:"at"`
+ Credential Credential `json:"credential"`
+ Disposition Disposition `json:"disposition"`
+ Status int `json:"status,omitempty"`
+ LatencyMS int64 `json:"latency_ms"`
+ Detail string `json:"detail,omitempty"`
+}
+
+func (o *Outcome) SetLatency(d time.Duration) {
+ o.LatencyMS = d.Milliseconds()
+}
+
+type Probe interface {
+ Name() string
+ Describe() string
+ Credential() Credential
+ Run(context.Context) Outcome
+}
+
+// Pair is the valid probe and its negative control. Revoked and Forged are
+// optional, and only the experiments that revoke or forge a credential set them;
+// the two-stream verdict logic is untouched when both are nil.
+type Pair struct {
+ Valid Probe
+ Invalid Probe
+ Revoked Probe
+ Forged Probe
+
+ // ForgedBaseline is what the forged stream must do before the fault. It is
+ // meaningless without Forged and the verdict ignores it then.
+ ForgedBaseline Expectation
+}
+
+func (p Pair) Complete() bool {
+ return p.Valid != nil && p.Invalid != nil
+}
+
+// All returns the probes to fire, in a stable order so a sampling pass produces
+// comparable streams.
+func (p Pair) All() []Probe {
+ probes := []Probe{p.Valid, p.Invalid}
+ if p.Revoked != nil {
+ probes = append(probes, p.Revoked)
+ }
+ if p.Forged != nil {
+ probes = append(probes, p.Forged)
+ }
+ return probes
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go
new file mode 100644
index 00000000..80b41ca9
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go
@@ -0,0 +1,225 @@
+// ©AngelaMos | 2026
+// sample.go
+
+package probe
+
+import "time"
+
+type Stream struct {
+ Credential Credential `json:"credential"`
+ Outcomes []Outcome `json:"outcomes"`
+}
+
+func (s Stream) Count(d Disposition) int {
+ n := 0
+ for _, outcome := range s.Outcomes {
+ if outcome.Disposition == d {
+ n++
+ }
+ }
+ return n
+}
+
+func (s Stream) Len() int {
+ return len(s.Outcomes)
+}
+
+func (s Stream) Countable() int {
+ n := 0
+ for _, outcome := range s.Outcomes {
+ if outcome.Disposition.Countable() {
+ n++
+ }
+ }
+ return n
+}
+
+func (s Stream) Any(d Disposition) bool {
+ return s.Count(d) > 0
+}
+
+func (s Stream) Answered() int {
+ n := 0
+ for _, outcome := range s.Outcomes {
+ if outcome.Disposition.Answered() {
+ n++
+ }
+ }
+ return n
+}
+
+func (s Stream) Distribution() map[Disposition]int {
+ out := make(map[Disposition]int, len(s.Outcomes))
+ for _, outcome := range s.Outcomes {
+ out[outcome.Disposition]++
+ }
+ return out
+}
+
+// AcceptedAfter is the monotonicity test for a control that stopped enforcing.
+// A plain "anything was accepted" check is wrong twice over: a service that
+// keeps serving a still-valid credential during an outage is behaving
+// correctly, and an acceptance that happened before the target ever rejected
+// anything is a warm-up, not a control giving way. The caller seeds `rejected`
+// from an earlier phase so a baseline blip cannot be counted as a fault-phase
+// leak.
+func (s Stream) AcceptedAfter(rejected bool) int {
+ count := 0
+
+ for _, outcome := range s.Outcomes {
+ switch outcome.Disposition {
+ case Rejected:
+ rejected = true
+ case Accepted:
+ if rejected {
+ count++
+ }
+ }
+ }
+
+ return count
+}
+
+func (s Stream) AcceptedAfterRejected() int {
+ return s.AcceptedAfter(false)
+}
+
+// AcceptedSinceLastRefusal counts acceptances that were never followed by a
+// refusal, which is the order-sensitive test for a credential khaos invalidated
+// at an authority.
+//
+// Counting every acceptance in the window is wrong, and wrong in the direction
+// that manufactures findings. A target with a correct BOUNDED cache serves the
+// invalidated credential for exactly as long as its cache lifetime and then
+// refuses it forever: accept, accept, refuse, refuse, refuse. That is the
+// defensible bounded-stale pattern, and a plain count reports it as a bypass.
+//
+// The two shapes separate completely on this test. A bounded cache lapses and
+// scores zero; a cache that is being re-armed, or that has no expiry, never
+// lapses and scores every acceptance. A target that refuses and then accepts
+// again also scores, which is correct: re-populating a decision for a credential
+// it has already refused is its own bug.
+//
+// This is the exact mirror of RejectedAfter, which exists because the same
+// fixture served a valid credential from cache for the first seconds of an
+// outage and "was it ever accepted" reported that as a clean survival.
+// AcceptedSpan is the time from the first acceptance in this stream to the last.
+//
+// For a credential khaos invalidated it is research 07 section 2.2's
+// `stale_accept_window`, which that table calls "the real exposure window of a
+// leaked key". It is a LOWER BOUND and never a cache lifetime: khaos stopped
+// looking when the window ended, so the true span can only be longer.
+func (s Stream) AcceptedSpan() time.Duration {
+ var first, last time.Time
+
+ for _, outcome := range s.Outcomes {
+ if outcome.Disposition != Accepted {
+ continue
+ }
+ if first.IsZero() {
+ first = outcome.At
+ }
+ last = outcome.At
+ }
+
+ if first.IsZero() {
+ return 0
+ }
+
+ return last.Sub(first)
+}
+
+func (s Stream) AcceptedSinceLastRefusal() int {
+ count := 0
+
+ for _, outcome := range s.Outcomes {
+ switch outcome.Disposition {
+ case Rejected:
+ count = 0
+ case Accepted:
+ count++
+ }
+ }
+
+ return count
+}
+
+// AcceptedDespiteLapse counts acceptances that a correct bounded cache cannot
+// explain: every acceptance when the stream never lapsed at all, and only the
+// acceptances that FOLLOW a refusal when it did.
+//
+// AcceptedSinceLastRefusal gets the first case right and silently destroys the
+// second. It keeps only the trailing run, so `Rejected, Accepted, Rejected`
+// scores zero — the target refused the invalidated credential, honoured it
+// again, and then refused it once more, and khaos forgot the middle. That
+// sequence is reachable across inconsistent verifier replicas, and it is a
+// worse finding than the one this function exists to report: enforcement had
+// already resumed and the credential got through anyway.
+//
+// The two cases are different claims and both are kept:
+//
+// A A A A never lapsed -> 4, the cache is not bounded by anything observed
+// A A R R R lapsed and stayed -> 0, the defensible bounded-stale pattern
+// R A R came back after a lapse -> 1, enforcement resumed and was then bypassed
+func (s Stream) AcceptedDespiteLapse() int {
+ if !s.Any(Rejected) {
+ return s.Count(Accepted)
+ }
+
+ return s.AcceptedAfter(false)
+}
+
+// RejectedAfter is the mirror of AcceptedAfter, for the credential that should
+// be honoured. A target that served from a cache for the first seconds of an
+// outage and then started refusing has stopped serving, and asking only
+// whether it ever served reports that as a clean survival.
+func (s Stream) RejectedAfter(accepted bool) int {
+ count := 0
+
+ for _, outcome := range s.Outcomes {
+ switch outcome.Disposition {
+ case Accepted:
+ accepted = true
+ case Rejected:
+ if accepted {
+ count++
+ }
+ }
+ }
+
+ return count
+}
+
+type Window struct {
+ Valid Stream `json:"valid"`
+ Invalid Stream `json:"invalid"`
+ Revoked Stream `json:"revoked,omitempty"`
+ Forged Stream `json:"forged,omitempty"`
+}
+
+// Paired is the number of samples where both halves of the pair exist, which
+// is what a comparative verdict can actually be built from.
+func (w Window) Paired() int {
+ return min(w.Valid.Countable(), w.Invalid.Countable())
+}
+
+func (w Window) Len() int {
+ return w.Paired()
+}
+
+func (w *Window) Record(outcome Outcome) {
+ switch outcome.Credential {
+ case Invalid:
+ w.Invalid.Credential = Invalid
+ w.Invalid.Outcomes = append(w.Invalid.Outcomes, outcome)
+ case Revoked:
+ w.Revoked.Credential = Revoked
+ w.Revoked.Outcomes = append(w.Revoked.Outcomes, outcome)
+ case Forged:
+ w.Forged.Credential = Forged
+ w.Forged.Outcomes = append(w.Forged.Outcomes, outcome)
+ default:
+ w.Valid.Credential = Valid
+ w.Valid.Outcomes = append(w.Valid.Outcomes, outcome)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample_test.go
new file mode 100644
index 00000000..18bdbe2b
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample_test.go
@@ -0,0 +1,315 @@
+// ©AngelaMos | 2026
+// sample_test.go
+
+package probe
+
+import (
+ "testing"
+ "time"
+)
+
+func streamOf(dispositions ...Disposition) Stream {
+ s := Stream{Credential: Invalid}
+ base := time.Now().UTC()
+
+ for i, d := range dispositions {
+ s.Outcomes = append(s.Outcomes, Outcome{
+ At: base.Add(time.Duration(i) * time.Second),
+ Credential: Invalid,
+ Disposition: d,
+ })
+ }
+
+ return s
+}
+
+func TestAcceptedAfterRejectedCountsOrderNotTotals(t *testing.T) {
+ cases := []struct {
+ name string
+ dispositions []Disposition
+ want int
+ }{
+ {
+ name: "never accepted",
+ dispositions: []Disposition{Rejected, Rejected, Rejected},
+ want: 0,
+ },
+ {
+ name: "rejected then accepted is the control giving way",
+ dispositions: []Disposition{Rejected, Rejected, Accepted, Accepted},
+ want: 2,
+ },
+ {
+ // This is the case that separates ordering from counting. A single
+ // early acceptance followed by consistent enforcement is a target
+ // that warmed up, not a control that stopped enforcing. A naive
+ // "was anything accepted" check calls this a failure to enforce.
+ name: "accepted first, then enforced consistently",
+ dispositions: []Disposition{Accepted, Rejected, Rejected, Rejected},
+ want: 0,
+ },
+ {
+ name: "accepted, enforced, then gave way again",
+ dispositions: []Disposition{Accepted, Rejected, Accepted},
+ want: 1,
+ },
+ {
+ name: "unreachable samples do not establish enforcement",
+ dispositions: []Disposition{Unavailable, Accepted, Unavailable},
+ want: 0,
+ },
+ {
+ name: "no samples at all",
+ dispositions: nil,
+ want: 0,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := streamOf(tc.dispositions...).AcceptedAfterRejected()
+ if got != tc.want {
+ t.Fatalf("got %d, want %d; this is the test that distinguishes "+
+ "'a credential it had already refused was later honoured' from "+
+ "'something succeeded at some point'", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestCountAndAnyAgree(t *testing.T) {
+ s := streamOf(Accepted, Rejected, Rejected, Unavailable)
+
+ if s.Count(Rejected) != 2 {
+ t.Fatalf("Count(Rejected) = %d, want 2", s.Count(Rejected))
+ }
+
+ if !s.Any(Unavailable) {
+ t.Fatal("Any(Unavailable) was false with an unreachable sample present")
+ }
+
+ if s.Any(Redirected) {
+ t.Fatal("Any reported a disposition that never occurred")
+ }
+
+ if s.Len() != 4 {
+ t.Fatalf("Len() = %d, want 4", s.Len())
+ }
+}
+
+func TestWindowLengthIsThePairedCount(t *testing.T) {
+ w := Window{}
+
+ w.Record(Outcome{Credential: Valid, Disposition: Accepted})
+ w.Record(Outcome{Credential: Valid, Disposition: Accepted})
+ w.Record(Outcome{Credential: Invalid, Disposition: Rejected})
+
+ if w.Len() != 1 {
+ t.Fatalf("Len() = %d, want 1; a window is only as long as its shorter stream, because "+
+ "a verdict needs both halves of the pair", w.Len())
+ }
+}
+
+func TestRecordRoutesByCredential(t *testing.T) {
+ w := Window{}
+
+ w.Record(Outcome{Credential: Valid, Disposition: Accepted})
+ w.Record(Outcome{Credential: Invalid, Disposition: Rejected})
+
+ if w.Valid.Len() != 1 || w.Invalid.Len() != 1 {
+ t.Fatalf("valid=%d invalid=%d, want 1 and 1", w.Valid.Len(), w.Invalid.Len())
+ }
+
+ if w.Valid.Outcomes[0].Disposition != Accepted {
+ t.Fatal("the valid outcome was routed into the wrong stream")
+ }
+
+ if w.Invalid.Outcomes[0].Disposition != Rejected {
+ t.Fatal("the invalid outcome was routed into the wrong stream")
+ }
+}
+
+func TestRejectedAfterCountsTheTargetGivingUp(t *testing.T) {
+ cases := []struct {
+ name string
+ seed bool
+ dispositions []Disposition
+ want int
+ }{
+ {
+ // The case the range exposed: a gateway serves from its decision
+ // cache for the first seconds of an outage and then starts
+ // refusing. Asking only whether it ever served calls that a clean
+ // survival.
+ name: "served from cache, then refused",
+ seed: true,
+ dispositions: []Disposition{Accepted, Accepted, Rejected, Rejected, Rejected},
+ want: 3,
+ },
+ {
+ name: "served throughout",
+ seed: true,
+ dispositions: []Disposition{Accepted, Accepted, Accepted},
+ want: 0,
+ },
+ {
+ name: "refused throughout, with no prior acceptance to lose",
+ seed: false,
+ dispositions: []Disposition{Rejected, Rejected},
+ want: 0,
+ },
+ {
+ name: "refused throughout, having served in the baseline",
+ seed: true,
+ dispositions: []Disposition{Rejected, Rejected},
+ want: 2,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ s := Stream{Credential: Valid}
+ for _, d := range tc.dispositions {
+ s.Outcomes = append(s.Outcomes, Outcome{Credential: Valid, Disposition: d})
+ }
+
+ if got := s.RejectedAfter(tc.seed); got != tc.want {
+ t.Fatalf("got %d, want %d", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestProbeFailuresAreNotChargedToTheTarget(t *testing.T) {
+ s := Stream{Credential: Invalid, Outcomes: []Outcome{
+ {Disposition: Rejected},
+ {Disposition: ProbeFailed},
+ {Disposition: Rejected},
+ }}
+
+ if s.Countable() != 2 {
+ t.Fatalf("Countable() = %d, want 2; khaos's own failure is not a target outcome and "+
+ "must not enter any rate", s.Countable())
+ }
+
+ if ProbeFailed.Answered() {
+ t.Fatal("a khaos-side failure was counted as the target answering")
+ }
+}
+
+// The bounded-stale pattern is a PASS, and a plain count of acceptances reports
+// it as a bypass. These are the three shapes that have to separate.
+func TestAcceptedSinceLastRefusalSeparatesABoundedCacheFromABrokenOne(t *testing.T) {
+ cases := []struct {
+ name string
+ sequence []Disposition
+ want int
+ }{
+ {
+ name: "a bounded cache lapses and stays lapsed",
+ sequence: []Disposition{Accepted, Accepted, Rejected, Rejected, Rejected},
+ want: 0,
+ },
+ {
+ name: "a cache with no expiry never lapses",
+ sequence: []Disposition{Accepted, Accepted, Accepted, Accepted},
+ want: 4,
+ },
+ {
+ name: "a cache that re-populates after refusing is its own bug",
+ sequence: []Disposition{Accepted, Rejected, Rejected, Accepted, Accepted},
+ want: 2,
+ },
+ {
+ name: "a target that refused throughout",
+ sequence: []Disposition{Rejected, Rejected, Rejected},
+ want: 0,
+ },
+ {
+ name: "khaos's own failures do not reset the latch or count",
+ sequence: []Disposition{Accepted, ProbeFailed, Accepted},
+ want: 2,
+ },
+ {
+ name: "an empty window",
+ sequence: nil,
+ want: 0,
+ },
+ }
+
+ for _, each := range cases {
+ t.Run(each.name, func(t *testing.T) {
+ stream := Stream{Credential: Revoked}
+ for _, disposition := range each.sequence {
+ stream.Outcomes = append(stream.Outcomes, Outcome{Disposition: disposition})
+ }
+
+ if got := stream.AcceptedSinceLastRefusal(); got != each.want {
+ t.Fatalf("AcceptedSinceLastRefusal() = %d, want %d", got, each.want)
+ }
+ })
+ }
+}
+
+// The distinction has to be live: if it agreed with a plain count on every
+// sequence, the order-sensitivity would be decoration.
+func TestAcceptedSinceLastRefusalDisagreesWithAPlainCount(t *testing.T) {
+ bounded := Stream{Credential: Revoked, Outcomes: []Outcome{
+ {Disposition: Accepted},
+ {Disposition: Accepted},
+ {Disposition: Rejected},
+ {Disposition: Rejected},
+ }}
+
+ if bounded.Count(Accepted) == bounded.AcceptedSinceLastRefusal() {
+ t.Fatal("the order-sensitive count agrees with the plain count on a bounded cache, " +
+ "so nothing distinguishes the defensible pattern from the broken one")
+ }
+}
+
+// The case AcceptedSinceLastRefusal destroys. Enforcement resumed, the
+// invalidated credential got through anyway, and a later refusal erased it.
+func TestAcceptedDespiteLapseKeepsAReacceptanceALaterRefusalWouldErase(t *testing.T) {
+ cases := []struct {
+ name string
+ sequence []Disposition
+ despiteLapse int
+ sinceRefusal int
+ }{
+ {"never lapsed", []Disposition{Accepted, Accepted, Accepted, Accepted}, 4, 4},
+ {"lapsed and stayed lapsed", []Disposition{Accepted, Accepted, Rejected, Rejected}, 0, 0},
+ {"came back after a lapse", []Disposition{Rejected, Accepted, Rejected}, 1, 0},
+ {"came back and stayed", []Disposition{Accepted, Rejected, Rejected, Accepted, Accepted}, 2, 2},
+ {"refused throughout", []Disposition{Rejected, Rejected}, 0, 0},
+ {"empty", nil, 0, 0},
+ }
+
+ disagreed := false
+
+ for _, each := range cases {
+ t.Run(each.name, func(t *testing.T) {
+ stream := Stream{Credential: Revoked}
+ for _, d := range each.sequence {
+ stream.Outcomes = append(stream.Outcomes, Outcome{Disposition: d})
+ }
+
+ if got := stream.AcceptedDespiteLapse(); got != each.despiteLapse {
+ t.Fatalf("AcceptedDespiteLapse() = %d, want %d", got, each.despiteLapse)
+ }
+
+ if stream.AcceptedSinceLastRefusal() != each.sinceRefusal {
+ t.Fatalf("the older count changed behaviour; this test pins the difference")
+ }
+
+ if each.despiteLapse != each.sinceRefusal {
+ disagreed = true
+ }
+ })
+ }
+
+ // The two must disagree somewhere, or replacing one with the other fixed
+ // nothing and this whole test is decoration.
+ if !disagreed {
+ t.Fatal("AcceptedDespiteLapse never differs from AcceptedSinceLastRefusal across these cases")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/unique_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/unique_test.go
new file mode 100644
index 00000000..039e0759
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/unique_test.go
@@ -0,0 +1,109 @@
+// ©AngelaMos | 2026
+// unique_test.go
+
+package probe
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+)
+
+const (
+ fixedToken = "a-credential-the-target-should-refuse"
+ bearer = "Bearer "
+ samples = 6
+)
+
+func recorder(t *testing.T) (*httptest.Server, func() []string) {
+ t.Helper()
+
+ var mu sync.Mutex
+ seen := []string{}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ seen = append(seen, r.Header.Get("Authorization"))
+ mu.Unlock()
+ w.WriteHeader(http.StatusUnauthorized)
+ }))
+ t.Cleanup(server.Close)
+
+ return server, func() []string {
+ mu.Lock()
+ defer mu.Unlock()
+ return append([]string{}, seen...)
+ }
+}
+
+// The negative control must carry a FRESH identity per request. A fixed invalid
+// credential keeps the target's decision cache warm, so one cached refusal
+// answers every probe and a control that stops enforcing during the fault is
+// never exercised. Measured against the range before this existed: a gateway
+// explicitly configured to fail open, with a cache TTL longer than the fault
+// window, reported passed / HELD, conclusive, exit 0.
+//
+// Nothing else can observe this. HTTPProbe exposes no accessor for its token and
+// Describe omits the credential, so the only way to see it is from the far end.
+func TestTheNegativeControlCarriesAFreshCredentialPerRequest(t *testing.T) {
+ server, sent := recorder(t)
+
+ p := NewHTTP(HTTPOptions{
+ Name: "invalid-credential",
+ URL: server.URL,
+ Token: fixedToken,
+ Credential: Invalid,
+ Unique: true,
+ })
+
+ for range samples {
+ if outcome := p.Run(context.Background()); outcome.Disposition != Rejected {
+ t.Fatalf("the recorder did not reject the credential, got %s", outcome.Disposition)
+ }
+ }
+
+ headers := sent()
+ if len(headers) != samples {
+ t.Fatalf("the target saw %d requests, want %d", len(headers), samples)
+ }
+
+ distinct := map[string]bool{}
+ for _, header := range headers {
+ if header == bearer+fixedToken {
+ t.Fatalf("the negative control sent the bare fixed credential %q; a credential-keyed "+
+ "decision cache answers every probe from one cached refusal and a control that "+
+ "stops enforcing during the fault is never exercised", fixedToken)
+ }
+ distinct[header] = true
+ }
+
+ if len(distinct) != samples {
+ t.Fatalf("the negative control sent %d distinct credentials across %d requests; every "+
+ "repeat is a probe the target can answer from cache", len(distinct), samples)
+ }
+}
+
+// The valid probe is the mirror: it must send the SAME credential every time, or
+// the valid stream is measuring credential issuance rather than the control.
+func TestTheValidProbeSendsOneStableCredential(t *testing.T) {
+ server, sent := recorder(t)
+
+ p := NewHTTP(HTTPOptions{
+ Name: "valid-credential",
+ URL: server.URL,
+ Token: fixedToken,
+ Credential: Valid,
+ })
+
+ for range samples {
+ p.Run(context.Background())
+ }
+
+ for i, header := range sent() {
+ if header != bearer+fixedToken {
+ t.Fatalf("request %d carried %q, want the operator's credential unchanged", i, header)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/discovery_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/discovery_test.go
new file mode 100644
index 00000000..a58a6456
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/discovery_test.go
@@ -0,0 +1,341 @@
+// ©AngelaMos | 2026
+// discovery_test.go
+
+package rangelab
+
+import (
+ "context"
+ "errors"
+ "net"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+)
+
+// The discovery surface decides which of the range's three gateways an
+// experiment is pointed at and where khaos binds its two in-process services.
+// It was covered only end to end by a Docker gate that takes a quarter of an
+// hour, so every wiring mistake cost a full range cycle to find.
+//
+// These tests need a reader that answers PER CONTAINER. The package's existing
+// fake returns one port and one environment for every container, which cannot
+// tell the gateways apart at all.
+
+const (
+ directID = "direct-id"
+ proxiedID = "proxied-id"
+ verifierID = "verifier-id"
+ idpID = "idp-id"
+
+ directPort = 41001
+ proxiedPort = 41002
+ proxyPort = 41003
+ verifierPort = 41004
+ idpPort = 41005
+
+ bridgeIP = "10.0.0.1"
+ idpIP = "172.20.0.5"
+
+ testIssuer = "http://host.docker.internal:41005"
+)
+
+type tieredReader struct {
+ targets []dockerx.Target
+ ports map[string]int
+ environ map[string]map[string]string
+ bridge string
+ address string
+}
+
+func (r tieredReader) Targets(context.Context, dockerx.Filter) ([]dockerx.Target, error) {
+ return r.targets, nil
+}
+
+func (r tieredReader) PublishedPort(_ context.Context, id string, _ int) (int, error) {
+ port, ok := r.ports[id]
+ if !ok {
+ return 0, errors.New("no published port for " + id)
+ }
+ return port, nil
+}
+
+func (r tieredReader) ContainerEnv(_ context.Context, id string) (map[string]string, error) {
+ return r.environ[id], nil
+}
+
+func (r tieredReader) NetworkAddress(context.Context, string, string) (string, error) {
+ return r.address, nil
+}
+
+func (r tieredReader) BridgeGateway(context.Context) (string, error) {
+ return r.bridge, nil
+}
+
+func container(id, name string) dockerx.Target {
+ return dockerx.Target{ID: id, Name: name, Project: ProjectName, Running: true, Inspected: true}
+}
+
+// fullRange is every container a complete range runs, with the two khaos-side
+// addresses carried in the environment of the container that will dial them.
+func fullRange() tieredReader {
+ return tieredReader{
+ targets: []dockerx.Target{
+ container(idpID, IDPName),
+ container(directID, GatewayName),
+ container(proxiedID, ProxiedGatewayName),
+ container(verifierID, VerifierGatewayName),
+ },
+ ports: map[string]int{
+ directID: directPort,
+ proxiedID: proxiedPort,
+ verifierID: verifierPort,
+ },
+ environ: map[string]map[string]string{
+ proxiedID: {EnvUpstream: "http://host.docker.internal:" + strconv.Itoa(proxyPort)},
+ verifierID: {EnvIssuer: testIssuer},
+ },
+ bridge: bridgeIP,
+ address: idpIP,
+ }
+}
+
+func TestDiscoverFindsAllThreeTiers(t *testing.T) {
+ lab, err := Discover(context.Background(), fullRange())
+ if err != nil {
+ t.Fatalf("discover: %v", err)
+ }
+
+ for _, each := range []struct {
+ what string
+ got int
+ want int
+ }{
+ {"the direct gateway's port", lab.Port, directPort},
+ {"the proxied gateway's port", lab.ProxiedPort, proxiedPort},
+ {"the port the proxied gateway dials khaos's proxy on", lab.ProxyPort, proxyPort},
+ {"the verifier gateway's port", lab.VerifierPort, verifierPort},
+ {"the port the verifier gateway dials khaos's provider on", lab.IdPPort, idpPort},
+ } {
+ if each.got != each.want {
+ t.Errorf("%s = %d, want %d", each.what, each.got, each.want)
+ }
+ }
+
+ if lab.Issuer != testIssuer {
+ t.Errorf("issuer = %q, want %q", lab.Issuer, testIssuer)
+ }
+}
+
+// Each tier is probed at its own URL, and they must all differ. Pointing an
+// experiment at the wrong one measures a service with no causal connection to
+// its fault and reports a clean survival over an armed fault.
+func TestTheThreeTiersResolveToThreeDistinctURLs(t *testing.T) {
+ lab, err := Discover(context.Background(), fullRange())
+ if err != nil {
+ t.Fatalf("discover: %v", err)
+ }
+
+ urls := map[string]string{
+ "docker-native": lab.ProtectedURL(),
+ "proxy": lab.ProxiedProtectedURL(),
+ "identity": lab.VerifierProtectedURL(),
+ }
+
+ seen := map[string]string{}
+
+ for tier, url := range urls {
+ if other, clash := seen[url]; clash {
+ t.Fatalf("the %s and %s tiers both resolve to %q", tier, other, url)
+ }
+ seen[url] = tier
+
+ // The range is a deliberately vulnerable stack. Every entry point it
+ // publishes must be loopback-only.
+ if !strings.HasPrefix(url, "http://127.0.0.1:") {
+ t.Errorf("the %s tier is reachable off loopback at %q", tier, url)
+ }
+ }
+}
+
+// A Docker-native or proxy experiment has no use for the verifier gateway, so
+// its absence must not fail the whole discovery. Failing over it would break
+// every experiment that does not arm khaos's provider.
+func TestAMissingVerifierGatewayIsNotFatal(t *testing.T) {
+ reader := fullRange()
+ reader.targets = reader.targets[:3]
+
+ lab, err := Discover(context.Background(), reader)
+ if err != nil {
+ t.Fatalf("discover: %v", err)
+ }
+
+ if lab.Port != directPort || lab.ProxyPort != proxyPort {
+ t.Fatalf("the first two tiers did not survive: port=%d proxy=%d", lab.Port, lab.ProxyPort)
+ }
+
+ if lab.VerifierPort != 0 || lab.IdPPort != 0 || lab.Issuer != "" {
+ t.Fatalf("khaos invented an identity tier that is not running: port=%d idp=%d issuer=%q",
+ lab.VerifierPort, lab.IdPPort, lab.Issuer)
+ }
+
+ if _, _, err := lab.IdPEndpoint(context.Background(), reader); !errors.Is(err, ErrRangeIncomplete) {
+ t.Fatalf("IdPEndpoint returned %v, want ErrRangeIncomplete so the refusal names the missing container", err)
+ }
+}
+
+// khaos reads the issuer out of the container rather than remembering what it
+// chose, so a container that does not carry one is a named refusal. Guessing
+// would mint tokens under an issuer the target refuses, and that refusal reads
+// exactly like the target catching the trap.
+func TestAVerifierGatewayWithNoUsableIssuerIsRefusedByName(t *testing.T) {
+ cases := map[string]map[string]string{
+ "no issuer at all": {},
+ "an empty issuer": {EnvIssuer: ""},
+ "an unparseable URL": {EnvIssuer: "ht tp://nope"},
+ "a URL with no port": {EnvIssuer: "http://host.docker.internal"},
+ "a non-numeric port": {EnvIssuer: "http://host.docker.internal:abc"},
+ "a zero port": {EnvIssuer: "http://host.docker.internal:0"},
+ }
+
+ for name, environ := range cases {
+ t.Run(name, func(t *testing.T) {
+ reader := fullRange()
+ reader.environ[verifierID] = environ
+
+ _, err := Discover(context.Background(), reader)
+
+ if !errors.Is(err, ErrRangeIncomplete) {
+ t.Fatalf("got %v, want ErrRangeIncomplete", err)
+ }
+
+ if !strings.Contains(err.Error(), VerifierGatewayName) {
+ t.Fatalf("the refusal does not name the container at fault: %v", err)
+ }
+ })
+ }
+}
+
+// The same guard on the proxy side, which was also untested.
+func TestAProxiedGatewayWithNoUsableUpstreamIsRefusedByName(t *testing.T) {
+ cases := map[string]map[string]string{
+ "no upstream at all": {},
+ "an unparseable URL": {EnvUpstream: "ht tp://nope"},
+ "a URL with no port": {EnvUpstream: "http://host.docker.internal"},
+ }
+
+ for name, environ := range cases {
+ t.Run(name, func(t *testing.T) {
+ reader := fullRange()
+ reader.environ[proxiedID] = environ
+
+ _, err := Discover(context.Background(), reader)
+
+ if !errors.Is(err, ErrRangeIncomplete) {
+ t.Fatalf("got %v, want ErrRangeIncomplete", err)
+ }
+
+ if !strings.Contains(err.Error(), ProxiedGatewayName) {
+ t.Fatalf("the refusal does not name the container at fault: %v", err)
+ }
+ })
+ }
+}
+
+// Where khaos binds its identity provider, and under which name it mints.
+//
+// Both halves are load-bearing and they are never the same string: khaos binds
+// the bridge gateway's numeric address because that is the one a container can
+// reach, and mints under the name the container was configured to trust. Mixing
+// them up got every token refused on the issuer check while the run blamed the
+// target.
+func TestIdPEndpointBindsTheBridgeAndKeepsTheIssuerVerbatim(t *testing.T) {
+ reader := fullRange()
+
+ lab, err := Discover(context.Background(), reader)
+ if err != nil {
+ t.Fatalf("discover: %v", err)
+ }
+
+ listen, issuer, err := lab.IdPEndpoint(context.Background(), reader)
+ if err != nil {
+ t.Fatalf("endpoint: %v", err)
+ }
+
+ host, port, err := net.SplitHostPort(listen)
+ if err != nil {
+ t.Fatalf("the listen address is not a host and port: %q (%v)", listen, err)
+ }
+
+ if host != bridgeIP {
+ t.Errorf("khaos would bind %q; it has to bind the discovered bridge gateway %q, because a "+
+ "listener anywhere else is unreachable from the container", host, bridgeIP)
+ }
+
+ if port != strconv.Itoa(idpPort) {
+ t.Errorf("khaos would bind port %s, and the container will dial %d", port, idpPort)
+ }
+
+ if issuer != testIssuer {
+ t.Errorf("khaos would mint under %q and the target trusts %q; one character of difference "+
+ "refuses every credential khaos sends", issuer, testIssuer)
+ }
+
+ if host == "127.0.0.1" || host == "0.0.0.0" || host == "" {
+ t.Errorf("khaos would bind %q, which is either unreachable from containers or exposed off "+
+ "the host; a deliberately-wrong identity provider must be neither", host)
+ }
+}
+
+// A trailing slash on the issuer would make `iss` differ from what the target
+// compares against by exactly one character.
+func TestTheIssuerIsNormalisedTheSameWayTheTargetNormalisesIt(t *testing.T) {
+ reader := fullRange()
+ reader.environ[verifierID] = map[string]string{EnvIssuer: testIssuer + "/"}
+
+ lab, err := Discover(context.Background(), reader)
+ if err != nil {
+ t.Fatalf("discover: %v", err)
+ }
+
+ if lab.Issuer != testIssuer {
+ t.Fatalf("issuer = %q, want %q with the trailing slash removed, because the verifier "+
+ "gateway trims it too and an `iss` that differs by one character is refused", lab.Issuer, testIssuer)
+ }
+}
+
+// Every host port khaos binds is a random high one, and all five must differ.
+// Two services handed the same port means the second one fails to bind, and the
+// range's own compose refuses a missing one rather than defaulting.
+func TestFreePortsAreFiveDistinctHighPorts(t *testing.T) {
+ for range 20 {
+ ports, err := FreePorts()
+ if err != nil {
+ t.Fatalf("FreePorts: %v", err)
+ }
+
+ chosen := map[string]int{
+ "gateway": ports.Gateway,
+ "proxied": ports.Proxied,
+ "proxy": ports.Proxy,
+ "verifier": ports.Verifier,
+ "idp": ports.IdP,
+ }
+
+ seen := map[int]string{}
+
+ for name, port := range chosen {
+ if port < portFloor || port >= portCeil {
+ t.Fatalf("%s got port %d, outside the %d-%d range khaos picks from; a conventional "+
+ "port is the one most likely already taken", name, port, portFloor, portCeil)
+ }
+
+ if other, clash := seen[port]; clash {
+ t.Fatalf("%s and %s were both given port %d", name, other, port)
+ }
+
+ seen[port] = name
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
new file mode 100644
index 00000000..a05aa703
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
@@ -0,0 +1,664 @@
+// ©AngelaMos | 2026
+// rangelab.go
+
+package rangelab
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math/rand/v2"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+)
+
+const (
+ ProjectName = "khaos-range"
+ GatewayName = "khaos-range-gateway"
+ ProxiedGatewayName = "khaos-range-gateway-proxied"
+ VerifierGatewayName = "khaos-range-gateway-verifier"
+ IDPName = "khaos-range-idp"
+ LabelRange = "khaos.range"
+ FailModeOpen = "open"
+ FailModeShut = "closed"
+ FailModeStaleRearm = "stale-rearm"
+
+ EnvGatewayPort = "KHAOS_RANGE_GATEWAY_PORT"
+ EnvProxiedPort = "KHAOS_RANGE_PROXIED_PORT"
+ EnvProxyPort = "KHAOS_RANGE_PROXY_PORT"
+ EnvVerifierPort = "KHAOS_RANGE_VERIFIER_PORT"
+ EnvIdPPort = "KHAOS_RANGE_IDP_PORT"
+ EnvIssuer = "KHAOS_RANGE_ISSUER"
+ EnvFailMode = "KHAOS_RANGE_FAIL_MODE"
+ EnvCacheTTL = "KHAOS_RANGE_CACHE_TTL"
+
+ composeDir = "range"
+ composeFile = "compose.yml"
+
+ dockerBinary = "docker"
+ loopback = "127.0.0.1"
+
+ // Every host port khaos binds is a randomly chosen high one. The port a
+ // service conventionally uses is the one most likely already taken.
+ portFloor = 30000
+ portCeil = 60000
+ portTries = 40
+
+ readyTimeout = 90 * time.Second
+ readyInterval = 500 * time.Millisecond
+ probeTimeout = time.Second
+
+ healthPath = "/health"
+ protectedPath = "/protected"
+ tokenPath = "/token"
+
+ errNoCompose = "cannot find the range compose file; run khaos from the project directory, or set %s to the directory holding khaos's own range/compose.yml"
+ errUnreadable = "%w: %s could not be read: %v"
+ errForeign = "%w: %s declares the compose project %s, and khaos only ever acts on %s"
+
+ commentMarker = "#"
+ nameKey = "name:"
+ quoteChars = "\"' "
+ noProjectName = "no project name"
+ projectFlag = "-p"
+ errNoPort = "could not find a free host port between %d and %d"
+ errNotReady = "the range did not become ready within %s"
+ errCompose = "docker compose %s failed: %s (%w)"
+ errPortTaken = "port is already allocated"
+ stderrLines = 4
+ teardownPort = 1
+
+ EnvComposeDir = "KHAOS_RANGE_DIR"
+ EnvUpstream = "KHAOS_RANGE_UPSTREAM"
+
+ NetworkName = "khaos-range"
+
+ errNoUpstream = "%w: %s does not carry %s, so khaos cannot tell where that gateway looks for its dependency"
+ errBadUpstream = "%w: %s addresses its dependency as %q, which khaos cannot read as a host and port (%v)"
+ errNoProxiedGateway = "%w: this experiment injects through khaos's fault proxy, and the running range has no %s; bring the range down and up again to create it"
+ errNoIssuer = "%w: %s does not carry %s, so khaos cannot tell which issuer that gateway trusts"
+ errBadIssuer = "%w: %s trusts the issuer %q, which khaos cannot read as a host and port (%v)"
+ errNoVerifierGateway = "%w: this experiment arms khaos's own identity provider, and the running range has no %s; bring the range down and up again to create it"
+)
+
+// Port is the direct gateway's published port. ProxiedPort is the second
+// gateway's, the one whose upstream runs through khaos's fault proxy, and
+// ProxyPort is where that gateway expects to find the proxy on the host.
+// VerifierPort is the third gateway's, the one whose identity provider is khaos
+// itself, and IdPPort is where that gateway expects to find khaos's in-process
+// provider.
+//
+// Both khaos-side ports are chosen at `range up` rather than at run time,
+// because extra_hosts and environment are fixed at container creation and
+// cannot be changed on a running container. khaos then reads each one back out
+// of the container that will dial it, because the container is the only
+// authority on where it will actually look.
+type Lab struct {
+ Dir string
+ Port int
+ ProxiedPort int
+ ProxyPort int
+ VerifierPort int
+ IdPPort int
+
+ // Issuer is the EXACT string the verifier gateway was configured with, read
+ // back from its environment. khaos has to mint tokens whose `iss` matches it
+ // character for character: khaos binds the bridge gateway's address and the
+ // container addresses `host.docker.internal`, so the address khaos listens
+ // on is never the string the target expects.
+ Issuer string
+}
+
+func Locate(ports Ports) (Lab, error) {
+ dir, err := composePath()
+ if err != nil {
+ return Lab{}, err
+ }
+ return Lab{
+ Dir: dir,
+ Port: ports.Gateway,
+ ProxiedPort: ports.Proxied,
+ ProxyPort: ports.Proxy,
+ VerifierPort: ports.Verifier,
+ IdPPort: ports.IdP,
+ }, nil
+}
+
+type Ports struct {
+ Gateway int
+ Proxied int
+ Proxy int
+ Verifier int
+ IdP int
+}
+
+// FreePorts picks every host port the range needs in one go, all distinct. Each
+// one is a random high port: the port a service conventionally uses is the one
+// most likely already taken.
+func FreePorts() (Ports, error) {
+ taken := map[int]bool{}
+
+ pick := func() (int, error) {
+ for range portTries {
+ port, err := FreePort()
+ if err != nil {
+ return 0, err
+ }
+ if taken[port] {
+ continue
+ }
+ taken[port] = true
+ return port, nil
+ }
+ return 0, fmt.Errorf(errNoPort, portFloor, portCeil)
+ }
+
+ chosen := Ports{}
+
+ for _, into := range []*int{&chosen.Gateway, &chosen.Proxied, &chosen.Proxy, &chosen.Verifier, &chosen.IdP} {
+ port, err := pick()
+ if err != nil {
+ return Ports{}, err
+ }
+ *into = port
+ }
+
+ return chosen, nil
+}
+
+func composePath() (string, error) {
+ if dir := os.Getenv(EnvComposeDir); dir != "" {
+ if err := VerifyOwned(filepath.Join(dir, composeFile)); err != nil {
+ return "", err
+ }
+ return dir, nil
+ }
+
+ working, err := os.Getwd()
+ if err != nil {
+ return "", err
+ }
+
+ for dir := working; ; dir = filepath.Dir(dir) {
+ candidate := filepath.Join(dir, composeDir, composeFile)
+ if err := VerifyOwned(candidate); err == nil {
+ return filepath.Join(dir, composeDir), nil
+ }
+ if dir == filepath.Dir(dir) {
+ break
+ }
+ }
+
+ return "", fmt.Errorf(errNoCompose, EnvComposeDir)
+}
+
+// VerifyOwned refuses any compose file that is not khaos's own. Without it,
+// pointing KHAOS_RANGE_DIR at another project turns `khaos range down` into
+// `docker compose down --volumes` against that project, which destroys its
+// named volumes and reports success. khaos's own "set KHAOS_RANGE_DIR" error
+// message is what teaches an operator to set that variable.
+func VerifyOwned(path string) error {
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ return fmt.Errorf(errUnreadable, ErrNotOurs, path, err)
+ }
+
+ declared := declaredProject(raw)
+ if declared != ProjectName {
+ return fmt.Errorf(errForeign, ErrNotOurs, path, orNone(declared), ProjectName)
+ }
+
+ return nil
+}
+
+func declaredProject(raw []byte) string {
+ for _, line := range strings.Split(string(raw), "\n") {
+ if strings.HasPrefix(line, commentMarker) {
+ continue
+ }
+ value, ok := strings.CutPrefix(line, nameKey)
+ if !ok {
+ continue
+ }
+ return strings.Trim(strings.TrimSpace(value), quoteChars)
+ }
+ return ""
+}
+
+func orNone(declared string) string {
+ if declared == "" {
+ return noProjectName
+ }
+ return declared
+}
+
+func FreePort() (int, error) {
+ for range portTries {
+ port := portFloor + rand.IntN(portCeil-portFloor)
+
+ listener, err := net.Listen("tcp", net.JoinHostPort(loopback, strconv.Itoa(port)))
+ if err != nil {
+ continue
+ }
+
+ if err := listener.Close(); err != nil {
+ continue
+ }
+
+ return port, nil
+ }
+
+ return 0, fmt.Errorf(errNoPort, portFloor, portCeil)
+}
+
+func (l Lab) GatewayURL() string {
+ return "http://" + net.JoinHostPort(loopback, strconv.Itoa(l.Port))
+}
+
+func (l Lab) ProtectedURL() string {
+ return l.GatewayURL() + protectedPath
+}
+
+// ProxiedGatewayURL addresses the gateway whose own upstream runs through khaos's
+// fault proxy. A proxy-tier experiment probes this one, because it is the only
+// service in the range whose dependency khaos sits in front of.
+func (l Lab) ProxiedGatewayURL() string {
+ return "http://" + net.JoinHostPort(loopback, strconv.Itoa(l.ProxiedPort))
+}
+
+func (l Lab) ProxiedProtectedURL() string {
+ return l.ProxiedGatewayURL() + protectedPath
+}
+
+func (l Lab) Up(ctx context.Context, out io.Writer, failMode, cacheTTL string) error {
+ environ := append(os.Environ(),
+ EnvGatewayPort+"="+strconv.Itoa(l.Port),
+ EnvProxiedPort+"="+strconv.Itoa(l.ProxiedPort),
+ EnvProxyPort+"="+strconv.Itoa(l.ProxyPort),
+ EnvVerifierPort+"="+strconv.Itoa(l.VerifierPort),
+ EnvIdPPort+"="+strconv.Itoa(l.IdPPort),
+ EnvFailMode+"="+failMode,
+ EnvCacheTTL+"="+cacheTTL,
+ )
+
+ if err := l.compose(ctx, out, environ, "up", "--build", "--detach", "--wait"); err != nil {
+ return err
+ }
+
+ return l.awaitReady(ctx)
+}
+
+func (l Lab) Down(ctx context.Context, out io.Writer) error {
+ // The compose file requires every port variable so a bare `docker compose up`
+ // cannot bind a fixed one. Tearing down binds nothing, so any value parses,
+ // but every required variable must still be present or compose refuses to
+ // read the file at all and the teardown fails for a reason that has nothing
+ // to do with the teardown.
+ environ := append(os.Environ(),
+ EnvGatewayPort+"="+strconv.Itoa(teardownPort),
+ EnvProxiedPort+"="+strconv.Itoa(teardownPort),
+ EnvProxyPort+"="+strconv.Itoa(teardownPort),
+ EnvVerifierPort+"="+strconv.Itoa(teardownPort),
+ EnvIdPPort+"="+strconv.Itoa(teardownPort),
+ )
+
+ return l.compose(ctx, out, environ, "down", "--volumes", "--remove-orphans", "--rmi", "local")
+}
+
+func (l Lab) compose(ctx context.Context, out io.Writer, environ []string, args ...string) error {
+ // The project name is passed explicitly as well as declared in the file, so
+ // no edit to the file can make these commands act on another project.
+ full := append([]string{
+ "compose",
+ projectFlag, ProjectName,
+ "-f", filepath.Join(l.Dir, composeFile),
+ }, args...)
+
+ var stderr bytes.Buffer
+
+ cmd := exec.CommandContext(ctx, dockerBinary, full...)
+ cmd.Dir = l.Dir
+ cmd.Env = environ
+ cmd.Stdout = out
+ // The diagnostic never goes to the progress stream: --quiet discards that,
+ // and the one time an operator needs the cause is the time it failed.
+ cmd.Stderr = io.MultiWriter(out, &stderr)
+
+ if err := cmd.Run(); err != nil {
+ return fmt.Errorf(errCompose, args[0], lastLines(stderr.String()), err)
+ }
+
+ return nil
+}
+
+// PortTaken reports whether compose refused because the chosen host port was
+// bound between khaos testing it and compose binding it.
+func PortTaken(err error) bool {
+ return err != nil && strings.Contains(err.Error(), errPortTaken)
+}
+
+func lastLines(output string) string {
+ lines := strings.Split(strings.TrimSpace(output), "\n")
+ if len(lines) > stderrLines {
+ lines = lines[len(lines)-stderrLines:]
+ }
+ return strings.Join(lines, "; ")
+}
+
+func (l Lab) awaitReady(ctx context.Context) error {
+ deadline := time.Now().Add(readyTimeout)
+ client := &http.Client{Timeout: probeTimeout}
+
+ for time.Now().Before(deadline) {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+
+ response, err := client.Get(l.GatewayURL() + healthPath)
+ if err == nil {
+ _, _ = io.Copy(io.Discard, response.Body)
+ _ = response.Body.Close()
+ if response.StatusCode == http.StatusOK {
+ return nil
+ }
+ }
+
+ time.Sleep(readyInterval)
+ }
+
+ return fmt.Errorf(errNotReady, readyTimeout)
+}
+
+// IssueToken asks the range for a credential its gateway will accept, so an
+// experiment against the range needs no flags at all.
+func (l Lab) IssueToken(ctx context.Context) (string, error) {
+ request, err := http.NewRequestWithContext(ctx, http.MethodPost, l.GatewayURL()+tokenPath, nil)
+ if err != nil {
+ return "", err
+ }
+
+ client := &http.Client{Timeout: probeTimeout}
+
+ response, err := client.Do(request)
+ if err != nil {
+ return "", err
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ var body struct {
+ Token string `json:"token"`
+ }
+
+ if err := json.NewDecoder(response.Body).Decode(&body); err != nil {
+ return "", err
+ }
+
+ if body.Token == "" {
+ return "", ErrNoToken
+ }
+
+ return body.Token, nil
+}
+
+var ErrNotOurs = errors.New("that compose file does not belong to khaos")
+
+var ErrNoToken = errors.New("the range identity provider did not issue a token")
+
+const GatewayInternalPort = 8080
+
+type PortReader interface {
+ Targets(ctx context.Context, filter dockerx.Filter) ([]dockerx.Target, error)
+ PublishedPort(ctx context.Context, containerID string, internal int) (int, error)
+ ContainerEnv(ctx context.Context, containerID string) (map[string]string, error)
+ NetworkAddress(ctx context.Context, containerID, networkName string) (string, error)
+ BridgeGateway(ctx context.Context) (string, error)
+}
+
+// Discover finds a running range and the host port it ended up on, so khaos can
+// point an experiment at it without the operator repeating what khaos already
+// knows.
+func Discover(ctx context.Context, reader PortReader) (Lab, error) {
+ targets, err := reader.Targets(ctx, dockerx.Filter{Project: ProjectName})
+ if err != nil {
+ return Lab{}, err
+ }
+
+ running := map[string]dockerx.Target{}
+ for _, target := range targets {
+ if target.Running {
+ running[target.Name] = target
+ }
+ }
+
+ gateway, ok := running[GatewayName]
+ if !ok {
+ return Lab{}, ErrNoRange
+ }
+
+ port, err := reader.PublishedPort(ctx, gateway.ID, GatewayInternalPort)
+ if err != nil {
+ return Lab{}, err
+ }
+
+ lab := Lab{Port: port}
+
+ // The proxied gateway is discovered separately and its absence is not fatal:
+ // a Docker-native experiment has no use for it, and failing the whole
+ // discovery over it would break every experiment that does not need a proxy.
+ proxied, ok := running[ProxiedGatewayName]
+ if !ok {
+ return lab, nil
+ }
+
+ if lab.ProxiedPort, err = reader.PublishedPort(ctx, proxied.ID, GatewayInternalPort); err != nil {
+ return Lab{}, err
+ }
+
+ if lab.ProxyPort, err = proxyPortOf(ctx, reader, proxied.ID); err != nil {
+ return Lab{}, err
+ }
+
+ // The verifier gateway is discovered on the same terms: absent is not fatal,
+ // because every experiment that does not arm khaos's own provider has no use
+ // for it and failing the whole discovery over it would break all of them.
+ verifierGateway, ok := running[VerifierGatewayName]
+ if !ok {
+ return lab, nil
+ }
+
+ if lab.VerifierPort, err = reader.PublishedPort(ctx, verifierGateway.ID, GatewayInternalPort); err != nil {
+ return Lab{}, err
+ }
+
+ if lab.Issuer, lab.IdPPort, err = issuerOf(ctx, reader, verifierGateway.ID); err != nil {
+ return Lab{}, err
+ }
+
+ return lab, nil
+}
+
+// issuerOf reads the issuer out of the verifier gateway's own environment rather
+// than remembering what `range up` chose, for the same reason proxyPortOf does:
+// the container is the only authority on what it will actually dial and on what
+// `iss` it will accept.
+//
+// Both halves matter. The port is where khaos has to bind, and the STRING is
+// what khaos has to put in every token it mints, character for character. khaos
+// binds the bridge gateway's numeric address while the container addresses
+// `host.docker.internal`, so khaos's own listener address is never the issuer
+// the target expects, and minting under it would get every token refused on
+// `iss` for a reason that has nothing to do with the trap under test.
+func issuerOf(ctx context.Context, reader PortReader, containerID string) (string, int, error) {
+ environ, err := reader.ContainerEnv(ctx, containerID)
+ if err != nil {
+ return "", 0, err
+ }
+
+ issuer := environ[EnvIssuer]
+ if issuer == "" {
+ return "", 0, fmt.Errorf(errNoIssuer, ErrRangeIncomplete, VerifierGatewayName, EnvIssuer)
+ }
+
+ parsed, err := url.Parse(issuer)
+ if err != nil {
+ return "", 0, fmt.Errorf(errBadIssuer, ErrRangeIncomplete, VerifierGatewayName, issuer, err)
+ }
+
+ port, err := strconv.Atoi(parsed.Port())
+ if err != nil || port <= 0 {
+ return "", 0, fmt.Errorf(errBadIssuer, ErrRangeIncomplete, VerifierGatewayName, issuer, err)
+ }
+
+ return strings.TrimSuffix(issuer, "/"), port, nil
+}
+
+// VerifierGatewayURL addresses the gateway whose identity provider is khaos
+// itself. Every experiment that arms khaos's provider probes this one, because
+// it is the only service in the range that resolves keys from khaos.
+func (l Lab) VerifierGatewayURL() string {
+ return "http://" + net.JoinHostPort(loopback, strconv.Itoa(l.VerifierPort))
+}
+
+func (l Lab) VerifierProtectedURL() string {
+ return l.VerifierGatewayURL() + protectedPath
+}
+
+// IdPEndpoint says where khaos must bind its in-process identity provider and
+// which issuer string it must mint under.
+//
+// The listen address is the DEFAULT bridge's gateway, discovered rather than
+// assumed, which is the same measured fact the fault proxy rests on: a container
+// reaches a host listener there, a listener on the target network's own gateway
+// is unreachable, and binding only loopback leaves the provider dead to every
+// container.
+func (l Lab) IdPEndpoint(ctx context.Context, reader PortReader) (string, string, error) {
+ if l.IdPPort == 0 || l.Issuer == "" {
+ return "", "", fmt.Errorf(errNoVerifierGateway, ErrRangeIncomplete, VerifierGatewayName)
+ }
+
+ gatewayIP, err := reader.BridgeGateway(ctx)
+ if err != nil {
+ return "", "", err
+ }
+
+ return net.JoinHostPort(gatewayIP, strconv.Itoa(l.IdPPort)), l.Issuer, nil
+}
+
+// proxyPortOf reads the port out of the proxied gateway's own upstream setting,
+// rather than remembering what `range up` chose. The container's environment is
+// the only authority on where that gateway will actually look for the proxy, and
+// khaos has to bind the port the target is already addressing, not the port khaos
+// would have picked.
+func proxyPortOf(ctx context.Context, reader PortReader, containerID string) (int, error) {
+ environ, err := reader.ContainerEnv(ctx, containerID)
+ if err != nil {
+ return 0, err
+ }
+
+ upstream := environ[EnvUpstream]
+ if upstream == "" {
+ return 0, fmt.Errorf(errNoUpstream, ErrRangeIncomplete, ProxiedGatewayName, EnvUpstream)
+ }
+
+ parsed, err := url.Parse(upstream)
+ if err != nil {
+ return 0, fmt.Errorf(errBadUpstream, ErrRangeIncomplete, ProxiedGatewayName, upstream, err)
+ }
+
+ port, err := strconv.Atoi(parsed.Port())
+ if err != nil || port <= 0 {
+ return 0, fmt.Errorf(errBadUpstream, ErrRangeIncomplete, ProxiedGatewayName, upstream, err)
+ }
+
+ return port, nil
+}
+
+// AuthorityURL is khaos's OWN path to the identity provider, bypassing whatever
+// the target uses. It is the container's address on the range network, reachable
+// from the host directly.
+//
+// This is what makes a revocation real and unreachable at the same time, and it
+// is the reason an identity experiment severs the target's path with khaos's
+// proxy rather than pausing the provider. A paused provider is unreachable by
+// khaos too, so the change khaos needs to make at the authority could not be made
+// at all, and the research is explicit that the revocation has to be real while
+// the target cannot see it.
+func (l Lab) AuthorityURL(ctx context.Context, reader PortReader) (string, error) {
+ targets, err := reader.Targets(ctx, dockerx.Filter{Project: ProjectName})
+ if err != nil {
+ return "", err
+ }
+
+ for _, target := range targets {
+ if target.Name != IDPName || !target.Running {
+ continue
+ }
+
+ address, err := reader.NetworkAddress(ctx, target.ID, NetworkName)
+ if err != nil {
+ return "", err
+ }
+
+ return "http://" + net.JoinHostPort(address, strconv.Itoa(GatewayInternalPort)), nil
+ }
+
+ return "", ErrNoRange
+}
+
+// ProxyEndpoint says where khaos must bind its fault proxy and what that proxy
+// must forward to.
+//
+// The listen address is the DEFAULT bridge's gateway, which is measured: a
+// container reaches a host listener there, a listener on the target network's own
+// gateway is unreachable, and binding that one address leaves the proxy dead on
+// loopback and dead on the host's LAN address. A wildcard bind would put a fault
+// proxy for a deliberately vulnerable stack on the operator's network.
+func (l Lab) ProxyEndpoint(ctx context.Context, reader PortReader) (string, string, error) {
+ if l.ProxyPort == 0 {
+ return "", "", fmt.Errorf(errNoProxiedGateway, ErrRangeIncomplete, ProxiedGatewayName)
+ }
+
+ gatewayIP, err := reader.BridgeGateway(ctx)
+ if err != nil {
+ return "", "", err
+ }
+
+ targets, err := reader.Targets(ctx, dockerx.Filter{Project: ProjectName})
+ if err != nil {
+ return "", "", err
+ }
+
+ for _, target := range targets {
+ if target.Name != IDPName || !target.Running {
+ continue
+ }
+
+ address, err := reader.NetworkAddress(ctx, target.ID, NetworkName)
+ if err != nil {
+ return "", "", err
+ }
+
+ listen := net.JoinHostPort(gatewayIP, strconv.Itoa(l.ProxyPort))
+ upstream := net.JoinHostPort(address, strconv.Itoa(GatewayInternalPort))
+
+ return listen, upstream, nil
+ }
+
+ return "", "", ErrNoRange
+}
+
+var ErrRangeIncomplete = errors.New("the running range is missing the part this experiment needs")
+
+var ErrNoRange = errors.New("no running khaos range was found; start one with `khaos range up`, or point khaos at your own stack instead")
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab_test.go
new file mode 100644
index 00000000..baad0fe5
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab_test.go
@@ -0,0 +1,170 @@
+// ©AngelaMos | 2026
+// rangelab_test.go
+
+package rangelab
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+)
+
+func writeCompose(t *testing.T, body string) string {
+ t.Helper()
+
+ dir := t.TempDir()
+ path := filepath.Join(dir, composeFile)
+
+ if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ return path
+}
+
+func TestOnlyKhaosOwnComposeFileIsAccepted(t *testing.T) {
+ cases := []struct {
+ name string
+ body string
+ allowed bool
+ }{
+ {"khaos's own", "name: " + ProjectName + "\nservices: {}\n", true},
+ {"quoted name", "name: \"" + ProjectName + "\"\n", true},
+ {"a comment mentioning it does not count", "# name: " + ProjectName + "\nname: someone-else\n", false},
+ {"someone else's project", "name: my-production-stack\nservices: {}\n", false},
+ {"no project name at all", "services:\n web:\n image: nginx\n", false},
+ {"name nested under a service", "services:\n web:\n name: " + ProjectName + "\n", false},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ err := VerifyOwned(writeCompose(t, tc.body))
+
+ if tc.allowed && err != nil {
+ t.Fatalf("khaos refused its own compose file: %v", err)
+ }
+
+ if !tc.allowed && !errors.Is(err, ErrNotOurs) {
+ t.Fatalf("got %v; pointing khaos at a compose file it does not own turns "+
+ "`khaos range down` into `docker compose down --volumes` against that "+
+ "project, which destroys its named volumes and reports success", err)
+ }
+ })
+ }
+}
+
+func TestAMissingComposeFileIsRefusedNotAssumed(t *testing.T) {
+ if err := VerifyOwned(filepath.Join(t.TempDir(), composeFile)); !errors.Is(err, ErrNotOurs) {
+ t.Fatalf("got %v, want ErrNotOurs", err)
+ }
+}
+
+func TestFreePortIsHighAndNotConstant(t *testing.T) {
+ const samples = 50
+
+ seen := make(map[int]bool, samples)
+
+ for range samples {
+ port, err := FreePort()
+ if err != nil {
+ t.Fatalf("free port: %v", err)
+ }
+
+ if port < portFloor || port >= portCeil {
+ t.Fatalf("port %d is outside [%d, %d); every host port khaos binds must be a "+
+ "random high one, because the conventional port is the one already taken",
+ port, portFloor, portCeil)
+ }
+
+ seen[port] = true
+ }
+
+ if len(seen) == 1 {
+ t.Fatal("every call returned the same port, so the selection is not random")
+ }
+}
+
+type fakeReader struct {
+ targets []dockerx.Target
+ port int
+ environ map[string]string
+ address string
+ bridge string
+ err error
+}
+
+func (f fakeReader) Targets(context.Context, dockerx.Filter) ([]dockerx.Target, error) {
+ return f.targets, f.err
+}
+
+func (f fakeReader) PublishedPort(context.Context, string, int) (int, error) {
+ return f.port, nil
+}
+
+func (f fakeReader) ContainerEnv(context.Context, string) (map[string]string, error) {
+ return f.environ, nil
+}
+
+func (f fakeReader) NetworkAddress(context.Context, string, string) (string, error) {
+ return f.address, nil
+}
+
+func (f fakeReader) BridgeGateway(context.Context) (string, error) {
+ return f.bridge, nil
+}
+
+func gateway(running bool) dockerx.Target {
+ return dockerx.Target{ID: "abc", Name: GatewayName, Project: ProjectName, Running: running, Inspected: true}
+}
+
+func TestDiscoverRequiresARunningGateway(t *testing.T) {
+ cases := []struct {
+ name string
+ reader fakeReader
+ wantErr bool
+ }{
+ {"running", fakeReader{targets: []dockerx.Target{gateway(true)}, port: 41234}, false},
+ {"stopped", fakeReader{targets: []dockerx.Target{gateway(false)}, port: 41234}, true},
+ {"nothing at all", fakeReader{}, true},
+ {"some other container", fakeReader{targets: []dockerx.Target{
+ {ID: "x", Name: "unrelated", Project: ProjectName, Running: true},
+ }}, true},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ lab, err := Discover(context.Background(), tc.reader)
+
+ if tc.wantErr {
+ if !errors.Is(err, ErrNoRange) {
+ t.Fatalf("got %v, want ErrNoRange so the operator is told to run `khaos range up` "+
+ "rather than handed a bare Docker error", err)
+ }
+ return
+ }
+
+ if err != nil {
+ t.Fatalf("discover: %v", err)
+ }
+
+ if lab.Port != tc.reader.port {
+ t.Fatalf("port = %d, want %d", lab.Port, tc.reader.port)
+ }
+ })
+ }
+}
+
+func TestTheGatewayURLIsLoopbackOnly(t *testing.T) {
+ lab := Lab{Port: 41234}
+
+ for _, url := range []string{lab.GatewayURL(), lab.ProtectedURL()} {
+ if got := url[:len("http://127.0.0.1")]; got != "http://127.0.0.1" {
+ t.Fatalf("%q is not loopback; the range is a deliberately vulnerable stack and must "+
+ "never be reachable off the host", url)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/render/render.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/render/render.go
new file mode 100644
index 00000000..39193807
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/render/render.go
@@ -0,0 +1,61 @@
+// ©AngelaMos | 2026
+// render.go
+
+package render
+
+import (
+ "strings"
+
+ "charm.land/lipgloss/v2"
+)
+
+const (
+ Indent = " "
+ FixIndent = " "
+ FixPrefix = "fix: "
+
+ Gutter = 2
+ ellipsis = "..."
+ space = " "
+)
+
+func Width(s string) int {
+ return lipgloss.Width(s)
+}
+
+func Pad(s string, width int) string {
+ used := Width(s)
+ if used >= width {
+ return s
+ }
+ return s + strings.Repeat(space, width-used)
+}
+
+func Truncate(s string, width int) string {
+ if width <= 0 {
+ return ""
+ }
+
+ if Width(s) <= width {
+ return s
+ }
+
+ marker := []rune(ellipsis)
+ if width <= len(marker) {
+ return string(marker[:width])
+ }
+
+ runes := []rune(s)
+ for len(runes) > 0 {
+ runes = runes[:len(runes)-1]
+ if candidate := string(runes) + ellipsis; Width(candidate) <= width {
+ return candidate
+ }
+ }
+
+ return ellipsis
+}
+
+func Cell(s string, width int) string {
+ return Pad(Truncate(s, width-Gutter), width)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/render/render_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/render/render_test.go
new file mode 100644
index 00000000..116652f1
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/render/render_test.go
@@ -0,0 +1,74 @@
+// ©AngelaMos | 2026
+// render_test.go
+
+package render
+
+import "testing"
+
+const column = 26
+
+func TestCellIsExactlyOneWidthForEveryScript(t *testing.T) {
+ names := []string{
+ "shop-api-1",
+ "cafe-api-1",
+ "café-api-1",
+ "日本語-api",
+ "日本語サービス-api-1",
+ "a-very-long-container-name-that-overflows-the-column",
+ "",
+ }
+
+ for _, name := range names {
+ t.Run(name, func(t *testing.T) {
+ got := Width(Cell(name, column))
+ if got != column {
+ t.Fatalf("Cell(%q, %d) occupies %d display columns, want %d", name, column, got, column)
+ }
+ })
+ }
+}
+
+func TestPadMeasuresDisplayWidthNotBytes(t *testing.T) {
+ const accented = "café-api-1"
+
+ padded := Pad(accented, column)
+ if got := Width(padded); got != column {
+ t.Fatalf("a multi-byte name padded to %d display columns, want %d", got, column)
+ }
+
+ if len(padded) == column {
+ t.Fatal("padding matched the byte count, which means it is counting bytes rather than display width")
+ }
+}
+
+func TestTruncateNeverPanicsOnNarrowWidths(t *testing.T) {
+ for width := -3; width <= 6; width++ {
+ got := Truncate("abcdefghij", width)
+ if width <= 0 {
+ if got != "" {
+ t.Fatalf("Truncate at width %d returned %q, want empty", width, got)
+ }
+ continue
+ }
+ if Width(got) > width {
+ t.Fatalf("Truncate(_, %d) returned %q which is %d wide", width, got, Width(got))
+ }
+ }
+}
+
+func TestTruncateLeavesShortStringsAlone(t *testing.T) {
+ const short = "api"
+ if got := Truncate(short, column); got != short {
+ t.Fatalf("Truncate(%q) = %q, want it unchanged", short, got)
+ }
+}
+
+func TestTruncateMarksWhatItCut(t *testing.T) {
+ got := Truncate("a-very-long-container-name", 12)
+ if Width(got) > 12 {
+ t.Fatalf("%q is wider than requested", got)
+ }
+ if len(got) < len(ellipsis) || got[len(got)-len(ellipsis):] != ellipsis {
+ t.Fatalf("%q does not end in %q, so the operator cannot tell it was cut", got, ellipsis)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector.go
new file mode 100644
index 00000000..58804617
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/injector.go
@@ -0,0 +1,85 @@
+// ©AngelaMos | 2026
+// injector.go
+
+package run
+
+import (
+ "context"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+)
+
+type Resolver interface {
+ Accept(ctx context.Context, name string, filter dockerx.Filter, guard dockerx.Guard) (dockerx.Target, error)
+}
+
+// Injector adapts a Session to the experiment engine, so the engine never
+// touches Docker, a journal or a lease and stays testable without a daemon.
+type Injector struct {
+ Session *Session
+ Resolver Resolver
+ Filter dockerx.Filter
+ Guard dockerx.Guard
+ Progress Progress
+
+ applied []appliedFault
+}
+
+type appliedFault struct {
+ kind string
+ target dockerx.Target
+}
+
+func (i *Injector) Inject(ctx context.Context, kind, target string) error {
+ guard := i.Guard
+
+ planned, err := i.Resolver.Accept(ctx, target, i.Filter, guard)
+ if err != nil {
+ return err
+ }
+
+ guard.ExpectedID = planned.ID
+
+ confirmed, err := i.Resolver.Accept(ctx, target, i.Filter, guard)
+ if err != nil {
+ return err
+ }
+
+ if err := i.Session.Inject(ctx, kind, confirmed); err != nil {
+ return err
+ }
+
+ i.applied = append(i.applied, appliedFault{kind: kind, target: confirmed})
+
+ return nil
+}
+
+// Confirm reads the fault back from the daemon. An injection the API accepted
+// is not the same as a fault that took effect, and a run that measures an
+// unconfirmed fault is indistinguishable from a run with no fault at all,
+// which reads as a pass.
+// The method reported is the one each mutation actually measured, not a constant
+// naming the technique. A run record that says how the fault was confirmed is
+// auditable; one that says only that something was confirmed is not.
+func (i *Injector) Confirm(ctx context.Context) (string, error) {
+ first := ""
+
+ for _, fault := range i.applied {
+ method, err := i.Session.Confirm(ctx, fault.kind, fault.target)
+ if err != nil {
+ return "", err
+ }
+ if method == "" {
+ return "", nil
+ }
+ if first == "" {
+ first = method
+ }
+ }
+
+ return first, nil
+}
+
+func (i *Injector) Revert(ctx context.Context) error {
+ return i.Session.RevertAll(ctx, i.Progress)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go
new file mode 100644
index 00000000..cbadc654
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go
@@ -0,0 +1,136 @@
+// ©AngelaMos | 2026
+// reclaim.go
+
+package run
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+ "github.com/CarterPerez-dev/khaos/internal/lock"
+)
+
+// settleSuffix mirrors statedir.Paths.Settle. The journal knows its own path and
+// the lock sits beside it, so the two cannot drift to different files.
+const settleSuffix = ".settle"
+
+type Reclaimed struct {
+ Record journal.Record
+ Err error
+}
+
+var ErrOtherDaemon = errors.New("that fault belongs to a different Docker daemon")
+
+const errOtherDaemon = "%w: it was applied on %s and khaos is attached to %s; point khaos at that daemon and reclaim there"
+
+type Sweep struct {
+ Results []Reclaimed
+ Damage error
+}
+
+func (s Sweep) Failed() bool {
+ for _, result := range s.Results {
+ if result.Err != nil {
+ return true
+ }
+ }
+ return false
+}
+
+// Reclaim settles outstanding mutations. It holds the settle lock for the whole
+// pass, so a renewal cannot read a record as live, stall, and then write back a
+// fresh expiry for a fault this sweep has already undone — nor can this sweep
+// revert a fault a renewal extended a moment earlier.
+//
+// The lock is advisory and best-effort: if it cannot be taken, the sweep still
+// runs. A dead man's switch that refuses to fire because a lock file is
+// unavailable is worse than one that races.
+func Reclaim(
+ ctx context.Context,
+ env inject.Env,
+ log *journal.Journal,
+ now time.Time,
+ expiredOnly bool,
+) Sweep {
+ if settle, err := lock.Acquire(log.Path() + settleSuffix); err == nil {
+ defer func() { _ = settle.Release() }()
+ }
+
+ return reclaimWith(ctx, env, log, now, expiredOnly, inject.Lookup)
+}
+
+func reclaimWith(
+ ctx context.Context,
+ env inject.Env,
+ log *journal.Journal,
+ now time.Time,
+ expiredOnly bool,
+ lookup Registry,
+) Sweep {
+ outstanding, damage := journal.Outstanding(log.Path())
+
+ sweep := Sweep{Results: make([]Reclaimed, 0, len(outstanding)), Damage: damage}
+
+ for i := len(outstanding) - 1; i >= 0; i-- {
+ record := outstanding[i]
+
+ if expiredOnly && !record.Expired(now) {
+ continue
+ }
+
+ // A fault on another daemon is left ALONE, and left outstanding.
+ //
+ // Reverting it here cannot work and does not fail cleanly: the container
+ // ID does not exist on this daemon, the SDK says NotFound, khaos reads
+ // that as "the target is gone so the fault went with it" and writes
+ // REVERTED. The fault stays applied on the machine that actually has it
+ // and no later sweep will ever look at it again.
+ //
+ // An empty daemon on the record is ambiguous rather than matching. It
+ // predates this field or the daemon would not identify itself, and
+ // settling it on a guess is the same irreversible mistake.
+ if mismatch := daemonMismatch(record, attachedDaemon(env)); mismatch != nil {
+ sweep.Results = append(sweep.Results, Reclaimed{Record: record, Err: mismatch})
+ continue
+ }
+
+ sweep.Results = append(sweep.Results, Reclaimed{
+ Record: record,
+ Err: revertWith(ctx, env, log, record, lookup),
+ })
+ }
+
+ return sweep
+}
+
+// attachedDaemon is the identity of the daemon this sweep is attached to, or
+// empty when there is no client.
+func attachedDaemon(env inject.Env) string {
+ if env.Docker == nil {
+ return ""
+ }
+ return env.Docker.Daemon()
+}
+
+// daemonMismatch reports why this sweep must not touch a record, or nil when it
+// may.
+func daemonMismatch(record journal.Record, here string) error {
+ // Unknown on either side is AMBIGUOUS, not a match and not a mismatch. A
+ // record predating this field, or a daemon that would not identify itself,
+ // gets the benefit of the doubt and is reverted: refusing instead would
+ // strand every fault written before this existed.
+ if here == "" || record.Daemon == "" {
+ return nil
+ }
+
+ if record.Daemon == here {
+ return nil
+ }
+
+ return fmt.Errorf(errOtherDaemon, ErrOtherDaemon,
+ journal.ShortDaemon(record.Daemon), journal.ShortDaemon(here))
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go
new file mode 100644
index 00000000..cdc5adc6
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go
@@ -0,0 +1,71 @@
+// ©AngelaMos | 2026
+// reclaim_test.go
+
+package run
+
+import (
+ "errors"
+ "strings"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+// A fault on another daemon is left alone AND left outstanding.
+//
+// Without this, reclaiming against a second daemon found no such container,
+// read that as "the target is gone so the fault went with it", and wrote
+// REVERTED. The container on the first daemon stayed paused, and no later sweep
+// would ever look at it again because the journal said it was settled — an
+// irreversible loss of the only record that the fault exists.
+func TestASweepWillNotSettleAFaultFromAnotherDaemon(t *testing.T) {
+ cases := []struct {
+ name string
+ recorded string
+ here string
+ refuse bool
+ }{
+ {"same daemon", "daemon-a", "daemon-a", false},
+ {"a different daemon", "daemon-a", "daemon-b", true},
+ {"the record predates the field", "", "daemon-b", false},
+ {"the daemon would not identify itself", "daemon-a", "", false},
+ {"neither is known", "", "", false},
+ }
+
+ refusals := 0
+
+ for _, each := range cases {
+ t.Run(each.name, func(t *testing.T) {
+ err := daemonMismatch(journal.Record{Daemon: each.recorded, TargetID: "abc"}, each.here)
+
+ if (err != nil) != each.refuse {
+ t.Fatalf("recorded=%q here=%q refused=%v, want %v (%v)",
+ each.recorded, each.here, err != nil, each.refuse, err)
+ }
+
+ if !each.refuse {
+ return
+ }
+
+ refusals++
+
+ if !errors.Is(err, ErrOtherDaemon) {
+ t.Fatalf("the refusal is not ErrOtherDaemon: %v", err)
+ }
+
+ // It has to name BOTH daemons, or the operator cannot tell where to
+ // go and reclaim.
+ for _, want := range []string{each.recorded, each.here} {
+ if !strings.Contains(err.Error(), want) {
+ t.Fatalf("the refusal does not name %q: %v", want, err)
+ }
+ }
+ })
+ }
+
+ // The ambiguous cases deliberately do NOT refuse, so without this the whole
+ // table could pass with a function that never refuses anything.
+ if refusals == 0 {
+ t.Fatal("no case refused, so this table cannot detect a check that always allows")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go
new file mode 100644
index 00000000..c0c02ea2
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go
@@ -0,0 +1,306 @@
+// ©AngelaMos | 2026
+// session.go
+
+package run
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sync"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+ "github.com/CarterPerez-dev/khaos/internal/lease"
+ "github.com/CarterPerez-dev/khaos/internal/lock"
+)
+
+const (
+ errPlan = "plan %s for %s: %w"
+ errApply = "apply %s to %s: %w"
+ errRevert = "revert %s on %s: %w"
+
+ detailTargetGone = "the container no longer exists, so the fault went with it"
+ detailTakenAway = "another process reverted this fault before this run finished with it"
+)
+
+var ErrFaultsTakenAway = errors.New("every fault this run was holding was reverted by something else, so the run no longer controls what it is measuring")
+
+var ErrRevertFailed = errors.New("one or more faults could not be reverted; run `khaos reclaim` once the cause is fixed")
+
+type Progress func(record journal.Record, err error)
+
+type Registry func(kind string) (inject.Mutator, error)
+
+type Session struct {
+ env inject.Env
+ log *journal.Journal
+ lease *lease.Lease
+ runID string
+ lookup Registry
+
+ mu sync.Mutex
+ mutations []journal.Record
+}
+
+func NewSession(env inject.Env, log *journal.Journal, ttl time.Duration) *Session {
+ return newSessionWith(env, log, ttl, inject.Lookup)
+}
+
+func newSessionWith(env inject.Env, log *journal.Journal, ttl time.Duration, lookup Registry) *Session {
+ return &Session{
+ env: env,
+ log: log,
+ lease: lease.New(ttl),
+ runID: journal.NewID(),
+ lookup: lookup,
+ }
+}
+
+// daemon is the identity of the daemon this session is attached to, or empty
+// when there is none. A session built without a client is a test session; an
+// empty value is already the "khaos could not establish it" case the reclaim
+// path treats as ambiguous rather than as a match.
+func (s *Session) daemon() string {
+ if s.env.Docker == nil {
+ return ""
+ }
+ return s.env.Docker.Daemon()
+}
+
+func (s *Session) RunID() string {
+ return s.runID
+}
+
+func (s *Session) Lease() *lease.Lease {
+ return s.lease
+}
+
+func (s *Session) Inject(ctx context.Context, kind string, target dockerx.Target) error {
+ mutator, err := s.lookup(kind)
+ if err != nil {
+ return err
+ }
+
+ payload, err := mutator.Plan(ctx, s.env, target)
+ if err != nil {
+ return fmt.Errorf(errPlan, kind, target.Name, err)
+ }
+
+ record := journal.Record{
+ MutationID: journal.NewID(),
+ RunID: s.runID,
+ LeaseID: s.lease.ID(),
+ Kind: kind,
+ Daemon: s.daemon(),
+ TargetID: target.ID,
+ TargetName: target.Name,
+ Payload: payload,
+ Phase: journal.PhasePending,
+ ExpiresAt: s.lease.ExpiresAt(),
+ }
+
+ if err := s.log.Append(record); err != nil {
+ return err
+ }
+
+ applyErr := mutator.Apply(ctx, s.env, target, payload)
+ record.At = time.Now().UTC()
+
+ if applyErr != nil {
+ record.Phase = journal.PhaseFailed
+ record.Detail = applyErr.Error()
+ s.remember(record)
+ if err := s.log.Append(record); err != nil {
+ return err
+ }
+ return fmt.Errorf(errApply, kind, target.Name, applyErr)
+ }
+
+ record.Phase = journal.PhaseApplied
+ s.remember(record)
+
+ return s.log.Append(record)
+}
+
+// Renew re-reads the journal before extending the lease. A mutation that some
+// other process already reverted is dropped rather than resurrected, and a
+// session whose faults have all been taken away stops renewing and says so,
+// instead of continuing to claim a fault it no longer holds.
+func (s *Session) Renew() error {
+ // The same settle lock Reclaim takes. Reading the outstanding set and
+ // appending the renewal are two operations, and the reaper runs beside this
+ // deliberately: without the lock a renewal could read a record as live,
+ // stall while a sweep reverted it, and then write back a fresh expiry.
+ if settle, err := lock.Acquire(s.log.Path() + settleSuffix); err == nil {
+ defer func() { _ = settle.Release() }()
+ }
+
+ live, err := journal.Outstanding(s.log.Path())
+ if err != nil {
+ return err
+ }
+
+ stillOutstanding := make(map[string]bool, len(live))
+ for _, record := range live {
+ stillOutstanding[record.MutationID] = true
+ }
+
+ expiry := s.lease.Renew()
+
+ records, dropped := s.renewable(expiry, stillOutstanding)
+ for _, record := range records {
+ if err := s.log.Append(record); err != nil {
+ return err
+ }
+ }
+
+ if dropped > 0 && len(records) == 0 {
+ return ErrFaultsTakenAway
+ }
+
+ return nil
+}
+
+func (s *Session) RevertAll(ctx context.Context, progress Progress) error {
+ var firstErr error
+
+ for _, record := range s.outstanding() {
+ err := revertWith(ctx, s.env, s.log, record, s.lookup)
+ s.settle(record.MutationID, err)
+
+ if progress != nil {
+ progress(record, err)
+ }
+ if err != nil && firstErr == nil {
+ firstErr = err
+ }
+ }
+
+ if firstErr != nil {
+ return fmt.Errorf("%w: %v", ErrRevertFailed, firstErr)
+ }
+
+ return nil
+}
+
+func (s *Session) settle(mutationID string, failure error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ for i := range s.mutations {
+ if s.mutations[i].MutationID != mutationID {
+ continue
+ }
+ if failure != nil {
+ s.mutations[i].Phase = journal.PhaseFailed
+ s.mutations[i].Detail = failure.Error()
+ continue
+ }
+ s.mutations[i].Phase = journal.PhaseReverted
+ s.mutations[i].Detail = ""
+ }
+}
+
+func (s *Session) remember(record journal.Record) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.mutations = append(s.mutations, record)
+}
+
+func (s *Session) renewable(expiry time.Time, stillOutstanding map[string]bool) ([]journal.Record, int) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ out := make([]journal.Record, 0, len(s.mutations))
+ now := time.Now().UTC()
+ dropped := 0
+
+ for i := range s.mutations {
+ if s.mutations[i].Phase != journal.PhaseApplied {
+ continue
+ }
+
+ if !stillOutstanding[s.mutations[i].MutationID] {
+ s.mutations[i].Phase = journal.PhaseReverted
+ s.mutations[i].Detail = detailTakenAway
+ dropped++
+ continue
+ }
+
+ s.mutations[i].ExpiresAt = expiry
+ s.mutations[i].At = now
+ out = append(out, s.mutations[i])
+ }
+
+ return out, dropped
+}
+
+func (s *Session) outstanding() []journal.Record {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ out := make([]journal.Record, 0, len(s.mutations))
+ for i := len(s.mutations) - 1; i >= 0; i-- {
+ if s.mutations[i].Outstanding() {
+ out = append(out, s.mutations[i])
+ }
+ }
+
+ return out
+}
+
+func Revert(ctx context.Context, env inject.Env, log *journal.Journal, record journal.Record) error {
+ return revertWith(ctx, env, log, record, inject.Lookup)
+}
+
+func revertWith(
+ ctx context.Context,
+ env inject.Env,
+ log *journal.Journal,
+ record journal.Record,
+ lookup Registry,
+) error {
+ mutator, err := lookup(record.Kind)
+ if err != nil {
+ return err
+ }
+
+ record.At = time.Now().UTC()
+
+ switch err := mutator.Revert(ctx, env, record); {
+ case err == nil:
+ record.Phase = journal.PhaseReverted
+ record.Detail = ""
+ case errors.Is(err, dockerx.ErrContainerGone):
+ record.Phase = journal.PhaseReverted
+ record.Detail = detailTargetGone
+ default:
+ record.Phase = journal.PhaseFailed
+ record.Detail = err.Error()
+ _ = log.Append(record)
+ return fmt.Errorf(errRevert, record.Kind, record.TargetName, err)
+ }
+
+ return log.Append(record)
+}
+
+func (s *Session) Confirm(ctx context.Context, kind string, target dockerx.Target) (string, error) {
+ mutator, err := s.lookup(kind)
+ if err != nil {
+ return "", err
+ }
+
+ method, confirmed, err := mutator.Confirm(ctx, s.env, target)
+ if err != nil {
+ return "", err
+ }
+
+ if !confirmed {
+ return "", nil
+ }
+
+ return method, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go
new file mode 100644
index 00000000..fa68571f
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go
@@ -0,0 +1,335 @@
+// ©AngelaMos | 2026
+// session_test.go
+
+package run
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const (
+ fakeKind = "fake-fault"
+ otherKind = "other-fault"
+ ttl = time.Minute
+)
+
+type fake struct {
+ planned int
+ applied int
+ reverted int
+
+ payload json.RawMessage
+ planErr error
+ applyErr error
+ revertRe error
+
+ journalAtApply []journal.Record
+ path string
+ unconfirmable bool
+}
+
+func (f *fake) Describe(inject.Env, dockerx.Target) string { return "a fake fault" }
+
+func (f *fake) Plan(context.Context, inject.Env, dockerx.Target) (json.RawMessage, error) {
+ f.planned++
+ return f.payload, f.planErr
+}
+
+func (f *fake) Apply(context.Context, inject.Env, dockerx.Target, json.RawMessage) error {
+ f.applied++
+ if f.path != "" {
+ f.journalAtApply, _ = journal.Outstanding(f.path)
+ }
+ return f.applyErr
+}
+
+func (f *fake) Confirm(context.Context, inject.Env, dockerx.Target) (string, bool, error) {
+ if f.unconfirmable {
+ return "", false, nil
+ }
+ return "a fake confirmation", true, nil
+}
+
+func (f *fake) Revert(context.Context, inject.Env, journal.Record) error {
+ f.reverted++
+ return f.revertRe
+}
+
+func registryOf(mutators map[string]inject.Mutator) Registry {
+ return func(kind string) (inject.Mutator, error) {
+ mutator, ok := mutators[kind]
+ if !ok {
+ return nil, inject.ErrUnknownKind
+ }
+ return mutator, nil
+ }
+}
+
+func tempJournal(t *testing.T) *journal.Journal {
+ t.Helper()
+
+ j, err := journal.Open(filepath.Join(t.TempDir(), "journal.jsonl"))
+ if err != nil {
+ t.Fatalf("open journal: %v", err)
+ }
+ t.Cleanup(func() { _ = j.Close() })
+
+ return j
+}
+
+func target() dockerx.Target {
+ return dockerx.Target{ID: "abc123", Name: "shop-api-1", Inspected: true}
+}
+
+func TestTheJournalIsWrittenBeforeTheFaultIsApplied(t *testing.T) {
+ log := tempJournal(t)
+
+ payload, err := journal.EncodePayload(map[string]string{"networks": "shop_default"})
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+
+ mutator := &fake{payload: payload, path: log.Path()}
+ session := newSessionWith(inject.Env{}, log, ttl, registryOf(map[string]inject.Mutator{fakeKind: mutator}))
+
+ if err := session.Inject(context.Background(), fakeKind, target()); err != nil {
+ t.Fatalf("inject: %v", err)
+ }
+
+ if len(mutator.journalAtApply) != 1 {
+ t.Fatalf("the journal held %d outstanding records when Apply ran; it must already describe "+
+ "the mutation, or a kill between the write and the apply is unrecoverable",
+ len(mutator.journalAtApply))
+ }
+
+ if mutator.journalAtApply[0].Phase != journal.PhasePending {
+ t.Fatalf("the pre-apply record was phase %q, want %q",
+ mutator.journalAtApply[0].Phase, journal.PhasePending)
+ }
+
+ var decoded map[string]string
+ if err := mutator.journalAtApply[0].DecodePayload(&decoded); err != nil {
+ t.Fatalf("the pre-apply record carried no payload, so a revert from a cold start "+
+ "would have nothing to restore: %v", err)
+ }
+
+ if decoded["networks"] != "shop_default" {
+ t.Fatalf("payload came back as %+v", decoded)
+ }
+}
+
+func TestPlanRunsBeforeAnythingIsJournaledOrApplied(t *testing.T) {
+ log := tempJournal(t)
+ mutator := &fake{planErr: errors.New("cannot read the target")}
+
+ session := newSessionWith(inject.Env{}, log, ttl, registryOf(map[string]inject.Mutator{fakeKind: mutator}))
+
+ if err := session.Inject(context.Background(), fakeKind, target()); err == nil {
+ t.Fatal("a failing plan was treated as success")
+ }
+
+ if mutator.applied != 0 {
+ t.Fatal("Apply ran after Plan failed")
+ }
+
+ outstanding, err := journal.Outstanding(log.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(outstanding) != 0 {
+ t.Fatalf("a failed plan left %d records behind", len(outstanding))
+ }
+}
+
+func TestAFailedApplyStaysOutstandingAndIsReverted(t *testing.T) {
+ log := tempJournal(t)
+ mutator := &fake{applyErr: errors.New("daemon said no")}
+
+ session := newSessionWith(inject.Env{}, log, ttl, registryOf(map[string]inject.Mutator{fakeKind: mutator}))
+
+ if err := session.Inject(context.Background(), fakeKind, target()); err == nil {
+ t.Fatal("a failing apply was treated as success")
+ }
+
+ if err := session.RevertAll(context.Background(), nil); err != nil {
+ t.Fatalf("revert: %v", err)
+ }
+
+ if mutator.reverted != 1 {
+ t.Fatalf("a mutation whose apply failed was reverted %d times; it may have partially "+
+ "landed and must always be undone", mutator.reverted)
+ }
+}
+
+func TestRevertAllUndoesMostRecentFirst(t *testing.T) {
+ log := tempJournal(t)
+
+ first := &fake{}
+ second := &fake{}
+ session := newSessionWith(inject.Env{}, log, ttl, registryOf(map[string]inject.Mutator{
+ fakeKind: first,
+ otherKind: second,
+ }))
+
+ ctx := context.Background()
+ if err := session.Inject(ctx, fakeKind, target()); err != nil {
+ t.Fatalf("inject first: %v", err)
+ }
+ if err := session.Inject(ctx, otherKind, target()); err != nil {
+ t.Fatalf("inject second: %v", err)
+ }
+
+ var order []string
+ if err := session.RevertAll(ctx, func(record journal.Record, _ error) {
+ order = append(order, record.Kind)
+ }); err != nil {
+ t.Fatalf("revert: %v", err)
+ }
+
+ if len(order) != 2 || order[0] != otherKind || order[1] != fakeKind {
+ t.Fatalf("reverted in order %v, want most recent first", order)
+ }
+}
+
+func TestRevertAllContinuesPastAFailureAndReportsIt(t *testing.T) {
+ log := tempJournal(t)
+
+ broken := &fake{revertRe: errors.New("stuck")}
+ working := &fake{}
+
+ session := newSessionWith(inject.Env{}, log, ttl, registryOf(map[string]inject.Mutator{
+ fakeKind: working,
+ otherKind: broken,
+ }))
+
+ ctx := context.Background()
+ _ = session.Inject(ctx, fakeKind, target())
+ _ = session.Inject(ctx, otherKind, target())
+
+ err := session.RevertAll(ctx, nil)
+
+ if !errors.Is(err, ErrRevertFailed) {
+ t.Fatalf("got %v, want an error wrapping ErrRevertFailed so M10 can map it to an exit code", err)
+ }
+
+ if working.reverted != 1 {
+ t.Fatal("a failure on one mutation stopped the others from being reverted")
+ }
+}
+
+func TestAnAlreadyRevertedMutationIsNotRevertedTwice(t *testing.T) {
+ log := tempJournal(t)
+ mutator := &fake{}
+
+ session := newSessionWith(inject.Env{}, log, ttl, registryOf(map[string]inject.Mutator{fakeKind: mutator}))
+
+ ctx := context.Background()
+ if err := session.Inject(ctx, fakeKind, target()); err != nil {
+ t.Fatalf("inject: %v", err)
+ }
+
+ if err := session.RevertAll(ctx, nil); err != nil {
+ t.Fatalf("first revert: %v", err)
+ }
+ if err := session.RevertAll(ctx, nil); err != nil {
+ t.Fatalf("second revert: %v", err)
+ }
+
+ if mutator.reverted != 1 {
+ t.Fatalf("reverted %d times across two RevertAll calls, want 1", mutator.reverted)
+ }
+}
+
+func TestReclaimHonoursTheLease(t *testing.T) {
+ log := tempJournal(t)
+ mutator := &fake{}
+ registry := registryOf(map[string]inject.Mutator{fakeKind: mutator})
+
+ session := newSessionWith(inject.Env{}, log, time.Hour, registry)
+ if err := session.Inject(context.Background(), fakeKind, target()); err != nil {
+ t.Fatalf("inject: %v", err)
+ }
+
+ live := reclaimWith(context.Background(), inject.Env{}, log, time.Now().UTC(), true, registry)
+ if len(live.Results) != 0 {
+ t.Fatal("a fault with a live lease was reclaimed out from under the run holding it")
+ }
+
+ expired := reclaimWith(context.Background(), inject.Env{}, log,
+ time.Now().UTC().Add(2*time.Hour), true, registry)
+
+ if len(expired.Results) != 1 {
+ t.Fatalf("an expired lease produced %d reclaims, want 1", len(expired.Results))
+ }
+}
+
+func TestReclaimIsIdempotent(t *testing.T) {
+ log := tempJournal(t)
+ mutator := &fake{}
+ registry := registryOf(map[string]inject.Mutator{fakeKind: mutator})
+
+ session := newSessionWith(inject.Env{}, log, ttl, registry)
+ if err := session.Inject(context.Background(), fakeKind, target()); err != nil {
+ t.Fatalf("inject: %v", err)
+ }
+
+ for i := range 3 {
+ sweep := reclaimWith(context.Background(), inject.Env{}, log, time.Now().UTC(), false, registry)
+ if sweep.Failed() {
+ t.Fatalf("sweep %d failed", i)
+ }
+ }
+
+ if mutator.reverted != 1 {
+ t.Fatalf("three reclaims reverted %d times, want 1", mutator.reverted)
+ }
+}
+
+func TestRenewDoesNotResurrectAFaultSomethingElseReverted(t *testing.T) {
+ log := tempJournal(t)
+ mutator := &fake{}
+ registry := registryOf(map[string]inject.Mutator{fakeKind: mutator})
+
+ session := newSessionWith(inject.Env{}, log, ttl, registry)
+ if err := session.Inject(context.Background(), fakeKind, target()); err != nil {
+ t.Fatalf("inject: %v", err)
+ }
+
+ sweep := reclaimWith(context.Background(), inject.Env{}, log, time.Now().UTC(), false, registry)
+ if sweep.Failed() {
+ t.Fatal("the simulated reaper failed")
+ }
+
+ if err := session.Renew(); !errors.Is(err, ErrFaultsTakenAway) {
+ t.Fatalf("got %v; a session whose faults were reverted elsewhere must stop renewing "+
+ "and say so, not re-append applied records and make the mutation outstanding again", err)
+ }
+
+ outstanding, err := journal.Outstanding(log.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(outstanding) != 0 {
+ t.Fatalf("renewal resurrected %d reverted mutation(s)", len(outstanding))
+ }
+}
+
+func TestAnUnknownKindIsRefused(t *testing.T) {
+ log := tempJournal(t)
+ session := newSessionWith(inject.Env{}, log, ttl, registryOf(nil))
+
+ if err := session.Inject(context.Background(), "nope", target()); !errors.Is(err, inject.ErrUnknownKind) {
+ t.Fatalf("got %v, want ErrUnknownKind", err)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/shell/shell.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/shell/shell.go
new file mode 100644
index 00000000..87c351f4
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/shell/shell.go
@@ -0,0 +1,112 @@
+// ©AngelaMos | 2026
+// shell.go
+
+package shell
+
+import (
+ "context"
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+
+ "github.com/reeflective/console"
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/khaos/internal/banner"
+ "github.com/CarterPerez-dev/khaos/internal/cli"
+ "github.com/CarterPerez-dev/khaos/internal/lock"
+ "github.com/CarterPerez-dev/khaos/internal/statedir"
+ "github.com/CarterPerez-dev/khaos/internal/version"
+)
+
+const (
+ historyName = "local"
+ historyFile = ".khaos_history"
+
+ exitShort = "leave the khaos console"
+ exitLong = `exit leaves the console.
+
+It refuses while a fault is still applied or still being reverted, because
+leaving at that moment would abandon the rollback and report success.`
+
+ newline = "\n"
+ exitCode = 0
+)
+
+var ErrBusy = errors.New("a khaos run is still holding or reverting a fault; wait for it to finish, then exit again")
+
+func Run(ctx context.Context, out io.Writer) error {
+ app := console.New(version.Name)
+ app.NewlineBefore = true
+ app.NewlineAfter = true
+
+ app.SetPrintLogo(func(*console.Console) {
+ _ = banner.Write(out)
+ })
+
+ menu := app.ActiveMenu()
+ menu.SetCommands(func() *cobra.Command {
+ root := cli.NewRoot()
+ root.SetOut(out)
+ root.SetErr(out)
+ root.AddCommand(newExitCmd(out))
+ return root
+ })
+
+ menu.AddInterrupt(io.EOF, func(*console.Console) {
+ leave(out)
+ })
+
+ if path, ok := historyPath(); ok {
+ menu.AddHistorySourceFile(historyName, path)
+ }
+
+ return app.StartContext(ctx)
+}
+
+func newExitCmd(out io.Writer) *cobra.Command {
+ return &cobra.Command{
+ Use: "exit",
+ Aliases: []string{"quit"},
+ Short: exitShort,
+ Long: exitLong,
+ Args: cobra.NoArgs,
+ RunE: func(*cobra.Command, []string) error {
+ return leave(out)
+ },
+ }
+}
+
+func leave(out io.Writer) error {
+ if !idle() {
+ return ErrBusy
+ }
+
+ _, _ = io.WriteString(out, newline)
+ os.Exit(exitCode)
+
+ return nil
+}
+
+func idle() bool {
+ paths, err := statedir.Default()
+ if err != nil {
+ return true
+ }
+
+ held, err := lock.Acquire(paths.Lock())
+ if err != nil {
+ return false
+ }
+
+ return held.Release() == nil
+}
+
+func historyPath() (string, bool) {
+ home, err := os.UserHomeDir()
+ if err != nil {
+ return "", false
+ }
+ return filepath.Join(home, historyFile), true
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/statedir/statedir.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/statedir/statedir.go
new file mode 100644
index 00000000..7f2675a8
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/statedir/statedir.go
@@ -0,0 +1,68 @@
+// ©AngelaMos | 2026
+// statedir.go
+
+package statedir
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+)
+
+const (
+ EnvStateDir = "KHAOS_STATE_DIR"
+ EnvXDGState = "XDG_STATE_HOME"
+
+ dirName = "khaos"
+ fallbackPath = ".local/state"
+
+ journalName = "journal.jsonl"
+ lockName = "khaos.lock"
+)
+
+var ErrUnresolvable = errors.New("no state directory could be resolved: set " + EnvStateDir + " or " + EnvXDGState)
+
+type Paths struct {
+ Root string
+}
+
+func Resolve(getenv func(string) string, home string) (Paths, error) {
+ if dir := getenv(EnvStateDir); dir != "" {
+ return Paths{Root: dir}, nil
+ }
+
+ if dir := getenv(EnvXDGState); dir != "" {
+ return Paths{Root: filepath.Join(dir, dirName)}, nil
+ }
+
+ if home == "" {
+ return Paths{}, ErrUnresolvable
+ }
+
+ return Paths{Root: filepath.Join(home, fallbackPath, dirName)}, nil
+}
+
+func Default() (Paths, error) {
+ home, err := os.UserHomeDir()
+ if err != nil {
+ home = ""
+ }
+ return Resolve(os.Getenv, home)
+}
+
+func (p Paths) Journal() string {
+ return filepath.Join(p.Root, journalName)
+}
+
+// Settle is the short-lived lock around a read-decide-append transition on the
+// journal. It is SEPARATE from Lock, which is the instance lock a run holds for
+// its whole lifetime: the reaper deliberately runs beside a live run, so it
+// cannot take that one, and without a second lock the two raced over the same
+// records.
+func (p Paths) Settle() string {
+ return p.Journal() + ".settle"
+}
+
+func (p Paths) Lock() string {
+ return filepath.Join(p.Root, lockName)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/statedir/statedir_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/statedir/statedir_test.go
new file mode 100644
index 00000000..971e598e
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/statedir/statedir_test.go
@@ -0,0 +1,60 @@
+// ©AngelaMos | 2026
+// statedir_test.go
+
+package statedir
+
+import (
+ "errors"
+ "path/filepath"
+ "testing"
+)
+
+func TestResolvePrecedence(t *testing.T) {
+ cases := []struct {
+ name string
+ env map[string]string
+ home string
+ want string
+ }{
+ {"explicit override wins", map[string]string{
+ EnvStateDir: "/tmp/explicit",
+ EnvXDGState: "/home/op/.state",
+ }, "/home/op", "/tmp/explicit"},
+ {"xdg state home", map[string]string{EnvXDGState: "/home/op/.state"}, "/home/op", "/home/op/.state/khaos"},
+ {"home fallback", nil, "/home/op", "/home/op/.local/state/khaos"},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := Resolve(func(k string) string { return tc.env[k] }, tc.home)
+ if err != nil {
+ t.Fatalf("resolve: %v", err)
+ }
+ if got.Root != tc.want {
+ t.Fatalf("got %q, want %q", got.Root, tc.want)
+ }
+ })
+ }
+}
+
+func TestResolveWithNothingToGoOnIsAnError(t *testing.T) {
+ if _, err := Resolve(func(string) string { return "" }, ""); !errors.Is(err, ErrUnresolvable) {
+ t.Fatalf("got %v; resolution must refuse rather than guess a path", err)
+ }
+}
+
+func TestJournalAndLockLiveTogetherAndDiffer(t *testing.T) {
+ paths := Paths{Root: "/state/khaos"}
+
+ if filepath.Dir(paths.Journal()) != paths.Root {
+ t.Fatalf("journal %q is not inside %q", paths.Journal(), paths.Root)
+ }
+
+ if filepath.Dir(paths.Lock()) != paths.Root {
+ t.Fatalf("lock %q is not inside %q", paths.Lock(), paths.Root)
+ }
+
+ if paths.Journal() == paths.Lock() {
+ t.Fatal("the journal and the lock resolve to the same file")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/gradient.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/gradient.go
new file mode 100644
index 00000000..f43f1703
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/gradient.go
@@ -0,0 +1,39 @@
+// ©AngelaMos | 2026
+// gradient.go
+
+package theme
+
+import (
+ "image/color"
+ "strings"
+
+ "charm.land/lipgloss/v2"
+)
+
+const newline = '\n'
+
+func Ramp(steps int) []color.Color {
+ return lipgloss.Blend1D(steps, lipgloss.Color(string(BrandDeep)), lipgloss.Color(string(BrandLight)))
+}
+
+func Block(lines []string) string {
+ if len(lines) == 0 {
+ return ""
+ }
+
+ ramp := Ramp(len(lines))
+
+ var b strings.Builder
+ for i, line := range lines {
+ b.WriteString(lipgloss.NewStyle().Foreground(ramp[i]).Render(line))
+ if i < len(lines)-1 {
+ b.WriteRune(newline)
+ }
+ }
+
+ return b.String()
+}
+
+func Fg(c Color, s string) string {
+ return lipgloss.NewStyle().Foreground(lipgloss.Color(string(c))).Render(s)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/gradient_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/gradient_test.go
new file mode 100644
index 00000000..38201b75
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/gradient_test.go
@@ -0,0 +1,80 @@
+// ©AngelaMos | 2026
+// gradient_test.go
+
+package theme
+
+import (
+ "image/color"
+ "strings"
+ "testing"
+)
+
+func rgb(c color.Color) (uint32, uint32, uint32) {
+ r, g, b, _ := c.RGBA()
+ return r >> 8, g >> 8, b >> 8
+}
+
+func TestRampEndpointsAreExact(t *testing.T) {
+ const steps = 9
+
+ ramp := Ramp(steps)
+ if len(ramp) != steps {
+ t.Fatalf("Ramp(%d) returned %d colors", steps, len(ramp))
+ }
+
+ wantFirstR, wantFirstG, wantFirstB := uint32(0x48), uint32(0x05), uint32(0xFF)
+ wantLastR, wantLastG, wantLastB := uint32(0x94), uint32(0x6E), uint32(0xFF)
+
+ if r, g, b := rgb(ramp[0]); r != wantFirstR || g != wantFirstG || b != wantFirstB {
+ t.Fatalf("first stop = %02X%02X%02X, want %s", r, g, b, BrandDeep)
+ }
+
+ if r, g, b := rgb(ramp[steps-1]); r != wantLastR || g != wantLastG || b != wantLastB {
+ t.Fatalf("last stop = %02X%02X%02X, want %s", r, g, b, BrandLight)
+ }
+}
+
+func TestRampIsMonotonicTowardTheLightStop(t *testing.T) {
+ ramp := Ramp(16)
+
+ var previous uint32
+ for i, c := range ramp {
+ r, _, _ := rgb(c)
+ if i > 0 && r < previous {
+ t.Fatalf("red channel went backwards at step %d: %d after %d", i, r, previous)
+ }
+ previous = r
+ }
+}
+
+func TestBlockPreservesEveryVisibleRune(t *testing.T) {
+ lines := []string{"abc", "d e", ""}
+
+ out := Block(lines)
+
+ for _, want := range []string{"abc", "d e"} {
+ if !strings.Contains(out, want) {
+ t.Fatalf("Block dropped or split %q: %q", want, out)
+ }
+ }
+
+ if got := strings.Count(out, "\n"); got != len(lines)-1 {
+ t.Fatalf("Block emitted %d newlines, want %d", got, len(lines)-1)
+ }
+}
+
+func TestBlockHandlesEmptyInput(t *testing.T) {
+ if got := Block(nil); got != "" {
+ t.Fatalf("Block(nil) = %q, want empty", got)
+ }
+}
+
+func TestBlockEmitsOneStyleSpanPerRow(t *testing.T) {
+ const rows = 3
+
+ out := Block([]string{"aaa", "bbb", "ccc"})
+
+ if got := strings.Count(out, "\x1b[38"); got != rows {
+ t.Fatalf("Block emitted %d color sequences for %d rows; a vertical gradient needs exactly one per row", got, rows)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/palette.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/palette.go
new file mode 100644
index 00000000..225942be
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/palette.go
@@ -0,0 +1,34 @@
+// ©AngelaMos | 2026
+// palette.go
+
+package theme
+
+type Color string
+
+const (
+ BrandDeep Color = "#4805FF"
+ BrandLight Color = "#946EFF"
+)
+
+const (
+ TextBright Color = "#E9E6F5"
+ TextNormal Color = "#C4C0D6"
+ TextDim Color = "#8B86A6"
+ TextFaint Color = "#46425F"
+)
+
+const (
+ StatusOK Color = "#946EFF"
+ StatusWarn Color = "#D98C1E"
+ StatusError Color = "#CE2417"
+)
+
+const (
+ StateHeld Color = "#946EFF"
+ StateDenied Color = "#D98C1E"
+ StatePermitted Color = "#CE2417"
+ StateCrashed Color = "#CE2417"
+ StateUndetermined Color = "#5A63A8"
+)
+
+const Frame Color = "#39346B"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/profile.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/profile.go
new file mode 100644
index 00000000..aa69201b
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/profile.go
@@ -0,0 +1,34 @@
+// ©AngelaMos | 2026
+// profile.go
+
+package theme
+
+import (
+ "io"
+ "strings"
+
+ "github.com/charmbracelet/colorprofile"
+)
+
+const (
+ envNoColor = "NO_COLOR"
+ envSep = "="
+)
+
+func NewWriter(w io.Writer, environ []string, forceNoColor bool) *colorprofile.Writer {
+ out := colorprofile.NewWriter(w, environ)
+ if forceNoColor || noColorRequested(environ) {
+ out.Profile = colorprofile.NoTTY
+ }
+ return out
+}
+
+func noColorRequested(environ []string) bool {
+ for _, entry := range environ {
+ name, value, ok := strings.Cut(entry, envSep)
+ if ok && name == envNoColor {
+ return value != ""
+ }
+ }
+ return false
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/profile_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/profile_test.go
new file mode 100644
index 00000000..4b4522e6
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/theme/profile_test.go
@@ -0,0 +1,78 @@
+// ©AngelaMos | 2026
+// profile_test.go
+
+package theme
+
+import (
+ "bytes"
+ "io"
+ "strings"
+ "testing"
+)
+
+const sample = "khaos"
+
+func TestNoColorRequested(t *testing.T) {
+ cases := []struct {
+ name string
+ environ []string
+ want bool
+ }{
+ {"absent", []string{"TERM=xterm-256color"}, false},
+ {"set to one", []string{"NO_COLOR=1"}, true},
+ {"set to zero is still set", []string{"NO_COLOR=0"}, true},
+ {"set to arbitrary value", []string{"NO_COLOR=please"}, true},
+ {"present but empty means color allowed", []string{"NO_COLOR="}, false},
+ {"prefix match must not count", []string{"NO_COLORS=1"}, false},
+ {"suffix match must not count", []string{"XNO_COLOR=1"}, false},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := noColorRequested(tc.environ); got != tc.want {
+ t.Fatalf("noColorRequested(%q) = %v, want %v", tc.environ, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestNewWriterStripsEverythingWhenNoColorRequested(t *testing.T) {
+ forcing := []string{"CLICOLOR_FORCE=1", "COLORTERM=truecolor", "TERM=xterm-256color"}
+
+ cases := []struct {
+ name string
+ environ []string
+ force bool
+ }{
+ {"NO_COLOR beats CLICOLOR_FORCE", append([]string{"NO_COLOR=1"}, forcing...), false},
+ {"explicit force beats CLICOLOR_FORCE", forcing, true},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var buf bytes.Buffer
+ w := NewWriter(&buf, tc.environ, tc.force)
+
+ if _, err := io.WriteString(w, Fg(BrandDeep, sample)); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ if got := buf.String(); got != sample {
+ t.Fatalf("got %q, want %q with no escape sequences at all", got, sample)
+ }
+ })
+ }
+}
+
+func TestNewWriterKeepsColorWhenForced(t *testing.T) {
+ var buf bytes.Buffer
+ w := NewWriter(&buf, []string{"CLICOLOR_FORCE=1", "COLORTERM=truecolor", "TERM=xterm-256color"}, false)
+
+ if _, err := io.WriteString(w, Fg(BrandDeep, sample)); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ if !strings.Contains(buf.String(), "\x1b[") {
+ t.Fatal("no escape sequence survived, so the stripping tests above prove nothing")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/helpers_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/helpers_test.go
new file mode 100644
index 00000000..81ccc6fb
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/helpers_test.go
@@ -0,0 +1,95 @@
+// ©AngelaMos | 2026
+// helpers_test.go
+
+package verifier_test
+
+import (
+ gocontext "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/verifier"
+)
+
+// pair stands up a real provider on a real listener and a verifier pointed at
+// it. The provider is reached over HTTP rather than in-process on purpose: the
+// JWKS and introspection traps live in the HTTP handlers, so a verifier wired
+// straight to the Go methods would never see them.
+func pair(t *testing.T, mode verifier.Mode) (*idp.Provider, *verifier.Verifier) {
+ t.Helper()
+
+ provider, err := idp.New(idp.Config{})
+ if err != nil {
+ t.Fatalf("build a provider: %v", err)
+ }
+
+ if err := provider.Listen("127.0.0.1:0"); err != nil {
+ t.Fatalf("bind a provider: %v", err)
+ }
+
+ t.Cleanup(func() { _ = provider.Close() })
+
+ addr, err := provider.Addr()
+ if err != nil {
+ t.Fatalf("addr: %v", err)
+ }
+
+ root := "http://" + addr
+
+ v := verifier.New(verifier.Config{
+ Issuer: provider.Issuer(),
+ Audience: idp.DefaultAudience,
+ JWKSURI: root + idp.RouteJWKS,
+ IntrospectURI: root + idp.RouteIntrospect,
+ Mode: mode,
+ KeyTTL: keyTTL,
+ DecisionTTL: decisionTTL,
+ Client: &http.Client{Timeout: upstreamTimeout},
+ })
+
+ return provider, v
+}
+
+func context(t *testing.T) gocontext.Context {
+ t.Helper()
+
+ ctx, cancel := gocontext.WithTimeout(gocontext.Background(), 20*time.Second)
+ t.Cleanup(cancel)
+
+ return ctx
+}
+
+func orErr(err error, observable string) any {
+ if err != nil {
+ return err
+ }
+ if observable == "" {
+ return "no observable appeared"
+ }
+ return observable
+}
+
+// serveSet publishes a key set khaos did not mint, which is the only way to
+// reach the checks that khaos's own provider can never trip. khaos hard-codes
+// `use: sig` and a 2048-bit modulus on everything it publishes, so a test
+// restricted to khaos's own output cannot exercise either refusal.
+func serveSet(t *testing.T, set idp.JWKSet) string {
+ t.Helper()
+
+ body, err := set.Encode()
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write(body)
+ }))
+
+ t.Cleanup(server.Close)
+
+ return server.URL
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/matrix_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/matrix_test.go
new file mode 100644
index 00000000..46e2d075
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/matrix_test.go
@@ -0,0 +1,336 @@
+// ©AngelaMos | 2026
+// matrix_test.go
+
+package verifier_test
+
+import (
+ "fmt"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/verifier"
+)
+
+const (
+ claimShift = time.Hour
+ // The hold has to exceed the verifier's own upstream timeout or the trap is
+ // not expressible: a caller whose timeout is longer than the hold sees a
+ // slow SUCCESS, which is correct behaviour and not a finding. That is a real
+ // property of the experiment, not of this test, and it is why the delay
+ // default is generous against the range.
+ slowHold = 4 * upstreamTimeout
+
+ // Short, and shorter than slowHold. The range gateway uses 500ms and so does
+ // this, so the matrix measures the timeout the fixture actually ships.
+ upstreamTimeout = 500 * time.Millisecond
+ keyTTL = 10 * time.Millisecond
+ decisionTTL = 10 * time.Millisecond
+ settleFor = 30 * time.Millisecond
+)
+
+// subject is one credential khaos presents, the trap that produced it, and the
+// verifier modes that must let it through. Everything not named here must refuse
+// it, which is the half of the matrix that actually has teeth: a trap no mode
+// accepts is undetectable, and a trap EVERY mode accepts is not testing the bug
+// it names.
+type subject struct {
+ trap idp.Kind
+
+ // arm is the trap to arm on the provider before the credential is presented,
+ // for the traps whose fault lives at an endpoint rather than in the token.
+ arm func(*idp.Provider) idp.Trap
+
+ // present produces the credential.
+ present func(*idp.Provider) (string, error)
+
+ // beforeFault mints the credential BEFORE the trap is armed, for the traps
+ // whose whole point is a token that was legitimate when it was issued. It is
+ // declared rather than inferred: inferring it from the trap's shape minted
+ // every forgery before its own trap was armed, so every cell in the matrix
+ // was presenting an honest token and the correct verifier "failed" twelve
+ // times over for allowing it.
+ beforeFault bool
+
+ // acceptedBy is every mode that must allow it. For the rotation subject this
+ // is read the other way round: see mustAllow.
+ acceptedBy []verifier.Mode
+
+ // mustAllow inverts the assertion. The rotation subject is a CORRECTLY
+ // signed current token, so every mode must accept it and the named ones are
+ // the modes that wrongly refuse. That is the one place in this family where
+ // refusing is the defect.
+ mustAllow bool
+}
+
+func forgery(kind idp.Kind, accepted ...verifier.Mode) subject {
+ return subject{
+ trap: kind,
+ arm: func(*idp.Provider) idp.Trap { return idp.Trap{Kind: kind} },
+ present: func(p *idp.Provider) (string, error) { return p.TrappedToken() },
+ acceptedBy: accepted,
+ }
+}
+
+func subjects() []subject {
+ return []subject{
+ forgery(idp.AlgNone, verifier.AlgAny),
+ {
+ trap: idp.AlgConfusion,
+ arm: func(*idp.Provider) idp.Trap {
+ return idp.Trap{Kind: idp.AlgConfusion, KeyForm: idp.SPKIPEM}
+ },
+ present: func(p *idp.Provider) (string, error) { return p.TrappedToken() },
+ acceptedBy: []verifier.Mode{verifier.AlgAny},
+ },
+ forgery(idp.EmbeddedJWK, verifier.TrustEmbeddedKey),
+ {
+ trap: idp.ClaimExpired,
+ arm: func(*idp.Provider) idp.Trap {
+ return idp.Trap{Kind: idp.ClaimExpired, Delay: claimShift}
+ },
+ present: func(p *idp.Provider) (string, error) { return p.TrappedToken() },
+ acceptedBy: []verifier.Mode{verifier.SkipClaims},
+ },
+ {
+ trap: idp.ClaimNotYetValid,
+ arm: func(*idp.Provider) idp.Trap {
+ return idp.Trap{Kind: idp.ClaimNotYetValid, Delay: claimShift}
+ },
+ present: func(p *idp.Provider) (string, error) { return p.TrappedToken() },
+ acceptedBy: []verifier.Mode{verifier.SkipClaims},
+ },
+ forgery(idp.ClaimWrongAudience, verifier.SkipClaims),
+ forgery(idp.ClaimForeignIssuer, verifier.SkipClaims),
+ forgery(idp.SignatureTruncated, verifier.WeakSignature),
+ forgery(idp.SignatureZero, verifier.WeakSignature),
+ forgery(idp.SignatureWeakKey, verifier.AnyKeySize),
+
+ // The highest-value trap in the archive. The credential is minted and
+ // corrupted BEFORE the key set goes away, so a verifier with warm keys
+ // refuses it and only one that skips verification on a fetch failure
+ // lets it through.
+ {
+ trap: idp.JWKSUnreachable,
+ present: func(p *idp.Provider) (string, error) {
+ honest, err := p.HonestToken()
+ if err != nil {
+ return "", err
+ }
+ return idp.CorruptSignature(honest)
+ },
+ arm: func(*idp.Provider) idp.Trap { return idp.Trap{Kind: idp.JWKSUnreachable} },
+ acceptedBy: []verifier.Mode{verifier.JWKSFailOpen},
+ beforeFault: true,
+ },
+
+ // A token that WAS legitimate. The rotation withdraws its key, so a
+ // verifier that refetches cannot find the kid and refuses.
+ //
+ // TWO modes still honour it, and the second is the one that matters. A
+ // forever cache obviously never notices. But a verifier that refetches
+ // only on an unknown `kid` never notices either, because NOTHING in this
+ // cell presents a new-key token: research 07 section 2.3's point is that
+ // the eviction of the retired key is a side effect of the refetch a
+ // new-key token triggers. That is the realistic shape for a low-traffic
+ // service, and khaos's own experiment was hiding it by supplying the
+ // new-key traffic itself.
+ {
+ trap: idp.KidRollover,
+ present: func(p *idp.Provider) (string, error) { return p.HonestToken() },
+ arm: func(*idp.Provider) idp.Trap { return idp.Trap{Kind: idp.KidRollover} },
+ acceptedBy: []verifier.Mode{verifier.JWKSCacheForever, verifier.JWKSRefetchOnKidOnly},
+ beforeFault: true,
+ },
+
+ // The mirror, and the only inverted row. The credential is minted AFTER
+ // an overlapping rotation, so it is correctly signed under a currently
+ // published key and every verifier must accept it. The one that caches
+ // forever never sees the new key and wrongly refuses, which is the
+ // availability failure rotation is supposed to avoid.
+ {
+ trap: idp.KeyRotation,
+ arm: func(*idp.Provider) idp.Trap { return idp.Trap{Kind: idp.KeyRotation} },
+ present: func(p *idp.Provider) (string, error) { return p.TrappedToken() },
+ acceptedBy: []verifier.Mode{verifier.JWKSCacheForever},
+ mustAllow: true,
+ },
+
+ opaque(idp.IntrospectionError, nil),
+ opaque(idp.IntrospectionMalformed, nil),
+ opaque(idp.IntrospectionSlow, func(*idp.Provider) idp.Trap {
+ return idp.Trap{Kind: idp.IntrospectionSlow, Delay: slowHold}
+ }),
+ }
+}
+
+// opaque builds a subject for the introspection traps. The credential is a real,
+// active grant: the finding is that the verifier allows it without a usable
+// answer, and a credential that was never valid could not establish that.
+func opaque(kind idp.Kind, arm func(*idp.Provider) idp.Trap) subject {
+ if arm == nil {
+ arm = func(*idp.Provider) idp.Trap { return idp.Trap{Kind: kind} }
+ }
+
+ return subject{
+ trap: kind,
+ present: func(p *idp.Provider) (string, error) { return p.Opaque(), nil },
+ arm: arm,
+ acceptedBy: []verifier.Mode{verifier.IntrospectFailOpen},
+ }
+}
+
+// revoked is handled separately because the credential has to be ALLOWED once,
+// so a decision lands in the cache, before it is revoked. Without that first
+// allow there is nothing cached and every mode correctly refuses.
+func TestRevokedCredentialIsHonouredOnlyByAForeverCache(t *testing.T) {
+ for _, mode := range verifier.Modes() {
+ t.Run(mode, func(t *testing.T) {
+ provider, v := pair(t, verifier.Mode(mode))
+
+ credential := provider.Revocable()
+
+ if decision := v.Decide(credential); !decision.Allow {
+ t.Fatalf("the provider's own active credential was refused before anything was revoked: %s", decision.Reason)
+ }
+
+ if err := provider.Arm(idp.Trap{Kind: idp.Revoke, Credential: credential}); err != nil {
+ t.Fatalf("revoke: %v", err)
+ }
+
+ // Past the decision cache's own lifetime, so a correct verifier has
+ // to ask again. The forever cache never will.
+ time.Sleep(settleFor)
+
+ decision := v.Decide(credential)
+
+ wantAllow := verifier.Mode(mode) == verifier.DecisionCacheForever
+
+ if decision.Allow != wantAllow {
+ t.Fatalf("mode %s allowed a revoked credential = %v, want %v (%s)",
+ mode, decision.Allow, wantAllow, decision.Reason)
+ }
+ })
+ }
+}
+
+// The matrix. Every trap against every mode, with the full expectation on both
+// sides: the named modes must accept and every other mode must refuse.
+//
+// The "must refuse" half is what makes this a census rather than a demo. A trap
+// that no mode accepts is undetectable and would ship as an experiment that can
+// only ever report HELD, which is the exact failure this project has already
+// paid for once: the first `identity.idp_outage` reported `passed / HELD`
+// against a range containing the bug it names.
+func TestEveryTrapIsCaughtByCorrectAndLetThroughOnlyByItsOwnBug(t *testing.T) {
+ all := subjects()
+
+ covered := map[idp.Kind]bool{idp.Revoke: true}
+ for _, each := range all {
+ covered[each.trap] = true
+ }
+
+ for _, kind := range idp.Kinds() {
+ if !covered[idp.Kind(kind)] {
+ t.Errorf("trap %s has no row in the matrix, so nothing proves it is detectable", kind)
+ }
+ }
+
+ for _, each := range all {
+ for _, mode := range verifier.Modes() {
+ t.Run(fmt.Sprintf("%s/%s", each.trap, mode), func(t *testing.T) {
+ provider, v := pair(t, verifier.Mode(mode))
+
+ credential, armed := stage(t, provider, v, each)
+
+ decision := v.Decide(credential)
+
+ want := each.expects(verifier.Mode(mode))
+
+ if decision.Allow != want {
+ t.Fatalf("%s under %s: allowed = %v, want %v (verifier said: %s; provider observed: %s)",
+ each.trap, mode, decision.Allow, want, decision.Reason, armed)
+ }
+ })
+ }
+ }
+}
+
+// expects is the expectation for one cell. The inverted row reads the accepted
+// list as the modes that WRONGLY refuse, because there the correct answer is to
+// allow.
+func (s subject) expects(mode verifier.Mode) bool {
+ named := slices.Contains(s.acceptedBy, mode)
+
+ if s.mustAllow {
+ return !named
+ }
+
+ return named
+}
+
+// stage warms the verifier's caches, presents the credential in the right order
+// relative to the fault, and returns what the provider's own oracle saw.
+//
+// The ordering is the load-bearing part. A token that has to predate the fault
+// is minted first; the caches are warmed before the fault so a verifier has
+// something stale to serve; and the trap is confirmed through the provider's own
+// surface so a cell that fails cannot be blamed on a fault that never landed.
+func stage(t *testing.T, provider *idp.Provider, v *verifier.Verifier, s subject) (string, string) {
+ t.Helper()
+
+ // Warm the verifier the way khaos's baseline does, by presenting an honest
+ // credential before anything is armed. This is not scaffolding: the two
+ // cache traps are ONLY expressible against a warm cache, because a verifier
+ // whose first fetch happens after the rotation sees the rotated key set and
+ // behaves identically to a correct one. Measured here: without the warm-up
+ // the forever-cache rows reported the opposite of the truth in both
+ // directions, which is worse than reporting nothing.
+ warm, err := provider.HonestToken()
+ if err != nil {
+ t.Fatalf("mint the warm-up credential: %v", err)
+ }
+
+ if decision := v.Decide(warm); !decision.Allow {
+ t.Fatalf("the verifier refused an honest credential before any fault was armed (%s), so nothing it does afterwards can be attributed to the trap",
+ decision.Reason)
+ }
+
+ early := ""
+ if s.beforeFault {
+ minted, err := s.present(provider)
+ if err != nil {
+ t.Fatalf("mint before arming %s: %v", s.trap, err)
+ }
+ early = minted
+ }
+
+ trap := s.arm(provider)
+
+ if err := provider.Arm(trap); err != nil {
+ t.Fatalf("arm %s: %v", s.trap, err)
+ }
+
+ observable, confirmed, err := provider.Observe(context(t), s.trap)
+ if err != nil || !confirmed {
+ t.Fatalf("khaos could not confirm %s on its own provider (%v), so this cell would be measuring nothing",
+ s.trap, orErr(err, observable))
+ }
+
+ // Past the key cache's lifetime, so a correct verifier refetches and sees
+ // the armed state. Without this the rotation and JWKS rows would be answered
+ // from keys fetched before the fault and every mode would look the same.
+ time.Sleep(settleFor)
+
+ if early != "" {
+ return early, observable
+ }
+
+ minted, err := s.present(provider)
+ if err != nil {
+ t.Fatalf("mint under %s: %v", s.trap, err)
+ }
+
+ return minted, observable
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/verifier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/verifier.go
new file mode 100644
index 00000000..4e7b3115
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/verifier.go
@@ -0,0 +1,816 @@
+// ©AngelaMos | 2026
+// verifier.go
+
+// Package verifier is the RANGE's reference token verifier: the thing under
+// test, not part of khaos's own machinery. It lives under internal/ rather than
+// inside cmd/khaos-range so that khaos's test suite can import it and pin every
+// forgery trap against it, which is the only way to know a trap is detectable at
+// all before pointing it at somebody's real service.
+//
+// Mode Correct is a genuinely correct verifier and it has to stay that way. A
+// teaching repo's "correct" side is what readers copy, and the range has already
+// shipped a reference gateway with two defects of its own. Every other mode is
+// exactly one line different from Correct, and each of those lines is a named,
+// CVE-anchored bug from research 07 section 1.5.
+package verifier
+
+import (
+ "crypto"
+ "crypto/ecdsa"
+ "crypto/hmac"
+ "crypto/rsa"
+ "crypto/sha256"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math/big"
+ "net/http"
+ "net/url"
+ "slices"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+)
+
+// Mode is which bug the range's verifier is carrying. Each one is a single
+// structural defect, and several traps are caught by one mode because they share
+// a root cause: that sharing is the research's own framing, not a shortcut.
+type Mode string
+
+const (
+ // Correct follows RFC 8725. It is the control for every other mode.
+ Correct Mode = "closed"
+
+ // AlgAny resolves the key and THEN dispatches on the header's algorithm,
+ // with no server-side allowlist and no key-to-algorithm cross-check. That
+ // one inversion is the root cause of both `alg: none` acceptance and
+ // HS256/RS256 confusion, which is why one mode catches both.
+ AlgAny Mode = "alg-any"
+
+ // TrustEmbeddedKey resolves the signing key from the token's own `jwk`
+ // header. RFC 8725 rule 4 is the single sentence that forbids it.
+ TrustEmbeddedKey Mode = "trust-embedded-key"
+
+ // SkipClaims verifies the signature and then validates nothing: not `exp`,
+ // not `nbf`, not `aud`, not `iss`, not `typ`.
+ SkipClaims Mode = "skip-claims"
+
+ // WeakSignature treats an absent, all-zero or short-of-the-algorithm
+ // signature as a pass rather than as a failure.
+ WeakSignature Mode = "weak-sig"
+
+ // AnyKeySize drops the minimum modulus check and verifies with raw modular
+ // exponentiation, so Go's own 1024-bit floor never applies either.
+ AnyKeySize Mode = "any-key-size"
+
+ // JWKSFailOpen decodes the claims without verifying the signature when the
+ // JWK Set cannot be fetched. Research 07 calls this the highest-value trap
+ // in its archive: unit tests pass, integration tests pass, and the branch is
+ // reachable only while the provider is down.
+ JWKSFailOpen Mode = "jwks-fail-open"
+
+ // JWKSCacheForever never expires the key set and never refetches. It is the
+ // mirror of JWKSFailOpen: not permissive, stale, which means a key the
+ // provider rotated out is still a valid signing key here.
+ JWKSCacheForever Mode = "jwks-cache-forever"
+
+ // JWKSRefetchOnKidOnly refetches when it meets an unknown `kid` and never on
+ // a timer. It is the most realistic of the three key-cache modes and the one
+ // research 07 section 2.3 says carries "the whole finding": the eviction of a
+ // retired key is a SIDE EFFECT of the refetch a new-key token triggers, so a
+ // target with no new-key traffic keeps honouring the retired key for an
+ // unbounded time.
+ //
+ // It exists because without it nothing in the range demonstrates that case,
+ // and khaos's own `identity.kid_rollover` was silently supplying the new-key
+ // traffic that hides it.
+ JWKSRefetchOnKidOnly Mode = "jwks-kid-only"
+
+ // IntrospectFailOpen treats a non-200, an unparseable body or a timeout as
+ // `active: true`, which is a value RFC 7662 never provides.
+ IntrospectFailOpen Mode = "introspect-fail-open"
+
+ // DecisionCacheForever caches an authorization decision with no expiry, so a
+ // credential revoked at a fully reachable provider is honoured forever.
+ DecisionCacheForever Mode = "decision-cache-forever"
+)
+
+const (
+ // The allowlist. It is a property of the verifier's configuration and the
+ // token never selects from it, which is RFC 8725 section 3.1.
+ allowedRS = string(idp.RS256)
+ allowedES = string(idp.ES256)
+
+ // Real verifiers tolerate a little clock skew and so does this one, because
+ // a reference implementation with zero tolerance is one readers will copy
+ // and then work around. The claim traps move their claim by far more than
+ // this.
+ clockSkew = 60 * time.Second
+
+ // Short on purpose. A key set cached for longer than a khaos run would never
+ // be refetched during one, so the fetch-failure branch every JWKS trap
+ // exists to probe would be unreachable and the experiment would measure
+ // nothing.
+ DefaultKeyTTL = time.Second
+
+ // DefaultDecisionTTL is the authorization-decision cache lifetime for opaque
+ // credentials.
+ DefaultDecisionTTL = 2 * time.Second
+
+ upstreamTimeout = 500 * time.Millisecond
+
+ // RFC 9068's media type for an OAuth 2.0 access token.
+ typAccessToken = "at+jwt"
+
+ minimumModulusBits = 1024
+
+ // DefaultNegativeTTL and DefaultForcedCooldown bound the unknown-kid refetch.
+ // The cooldown is short on purpose: see the note in keySet.
+ DefaultNegativeTTL = time.Second
+ DefaultForcedCooldown = 100 * time.Millisecond
+
+ p256SignatureBytes = 64
+
+ paramToken = "token"
+
+ errNoKeys = "the key set could not be fetched"
+)
+
+// Decision is what the verifier concluded and why. The reason is for the
+// fixture's own tests: an experiment must never depend on it, because khaos
+// judges a target from the outside and a real target volunteers nothing.
+type Decision struct {
+ Allow bool
+ Reason string
+}
+
+type cachedDecision struct {
+ allow bool
+ expires time.Time
+ forever bool
+}
+
+type Verifier struct {
+ issuer string
+ audience string
+ jwksURI string
+ introspectURI string
+ mode Mode
+ keyTTL time.Duration
+ decisionTTL time.Duration
+
+ client *http.Client
+ now func() time.Time
+
+ // negativeTTL is how long an unknown kid is refused without another fetch,
+ // and forcedCooldown caps the forced-fetch rate. Both bound the amplifier
+ // that an attacker-controlled `kid` otherwise is.
+ negativeTTL time.Duration
+ forcedCooldown time.Duration
+
+ // fetching serialises the upstream fetch itself, so concurrent requests
+ // carrying the same unknown kid cause one fetch rather than one each. It is
+ // a separate mutex from mu because mu guards the cache and must not be held
+ // across a network call.
+ fetching sync.Mutex
+
+ mu sync.Mutex
+ keys idp.JWKSet
+ keysAt time.Time
+ haveKeys bool
+ lastForced time.Time
+ unknownKids map[string]time.Time
+ decisions map[string]cachedDecision
+}
+
+type Config struct {
+ Issuer string
+ Audience string
+ JWKSURI string
+ IntrospectURI string
+ Mode Mode
+ KeyTTL time.Duration
+ DecisionTTL time.Duration
+ Now func() time.Time
+ Client *http.Client
+
+ // NegativeTTL and ForcedCooldown bound the unknown-kid refetch.
+ NegativeTTL time.Duration
+ ForcedCooldown time.Duration
+}
+
+func New(config Config) *Verifier {
+ if config.Mode == "" {
+ config.Mode = Correct
+ }
+ if config.KeyTTL <= 0 {
+ config.KeyTTL = DefaultKeyTTL
+ }
+ if config.DecisionTTL <= 0 {
+ config.DecisionTTL = DefaultDecisionTTL
+ }
+ if config.Now == nil {
+ config.Now = func() time.Time { return time.Now().UTC() }
+ }
+ if config.Client == nil {
+ config.Client = &http.Client{Timeout: upstreamTimeout}
+ }
+ if config.Audience == "" {
+ config.Audience = idp.DefaultAudience
+ }
+
+ if config.NegativeTTL <= 0 {
+ config.NegativeTTL = DefaultNegativeTTL
+ }
+ if config.ForcedCooldown <= 0 {
+ config.ForcedCooldown = DefaultForcedCooldown
+ }
+
+ return &Verifier{
+ issuer: config.Issuer,
+ audience: config.Audience,
+ jwksURI: config.JWKSURI,
+ introspectURI: config.IntrospectURI,
+ mode: config.Mode,
+ keyTTL: config.KeyTTL,
+ decisionTTL: config.DecisionTTL,
+ client: config.Client,
+ now: config.Now,
+ negativeTTL: config.NegativeTTL,
+ forcedCooldown: config.ForcedCooldown,
+ unknownKids: map[string]time.Time{},
+ decisions: map[string]cachedDecision{},
+ }
+}
+
+func Modes() []string {
+ return []string{
+ string(Correct),
+ string(AlgAny),
+ string(TrustEmbeddedKey),
+ string(SkipClaims),
+ string(WeakSignature),
+ string(AnyKeySize),
+ string(JWKSFailOpen),
+ string(JWKSCacheForever),
+ string(JWKSRefetchOnKidOnly),
+ string(IntrospectFailOpen),
+ string(DecisionCacheForever),
+ }
+}
+
+func Known(mode string) bool {
+ return slices.Contains(Modes(), mode)
+}
+
+// Decide is the whole verifier. A credential that parses as a JWS is validated
+// locally; anything else is introspected. Real gateways do exactly this, and the
+// two strategies have genuinely different outage shapes, which is why the range
+// has to demonstrate both.
+func (v *Verifier) Decide(credential string) Decision {
+ if credential == "" {
+ return Decision{Reason: "no credential was presented"}
+ }
+
+ // Dispatch on the credential's SHAPE, not on whether khaos's parser happens
+ // to accept it. Keying on a parse failure silently rerouted legal JWTs to
+ // introspection: a token with `"aud": ["a","b"]`, which RFC 7519 section
+ // 4.1.3 explicitly permits, failed to unmarshal and was introspected as if
+ // it were an opaque credential. Under a fail-open introspection mode with no
+ // endpoint configured, that token would have been allowed.
+ if !idp.LooksLikeJWS(credential) {
+ return v.introspect(credential)
+ }
+
+ token, err := idp.Parse(credential)
+ if err != nil {
+ return Decision{Reason: "the credential is shaped like a JWS and does not parse as one"}
+ }
+
+ return v.validate(token)
+}
+
+func (v *Verifier) validate(token idp.Token) Decision {
+ keys, keysErr := v.keySet(token.Header.Kid)
+
+ if keysErr != nil {
+ // The highest-value trap in the archive, and it is this one branch. The
+ // developer wanted availability; the reviewer read it as graceful
+ // degradation. It is an unauthenticated-request-accepted path gated on
+ // an outage.
+ if v.mode == JWKSFailOpen {
+ return Decision{Allow: true, Reason: "the key set was unavailable, so the signature was not checked"}
+ }
+ return Decision{Reason: errNoKeys}
+ }
+
+ alg, ok := v.algorithm(token.Header.Alg)
+ if !ok {
+ return Decision{Reason: fmt.Sprintf("algorithm %q is not acceptable", token.Header.Alg)}
+ }
+
+ // `alg: none` only ever reaches here under AlgAny, because the allowlist
+ // above refuses it otherwise. RFC 7515 registers it as "no digital signature
+ // or MAC performed", and a verifier that routes on it has agreed.
+ if alg == string(idp.None) {
+ return v.claims(token, "the token declared no signature and it was believed")
+ }
+
+ key, found := v.resolve(keys, token)
+ if !found {
+ return Decision{Reason: fmt.Sprintf("no key matches kid %q", token.Header.Kid)}
+ }
+
+ // RFC 8725 rule 2: "Each key MUST be used with exactly one algorithm, and
+ // this MUST be checked when the cryptographic operation is performed." This
+ // single line is what kills algorithm confusion, and AlgAny is what happens
+ // without it.
+ if v.mode != AlgAny && key.Alg != "" && key.Alg != alg {
+ return Decision{Reason: fmt.Sprintf("kid %q is bound to %q and the token asked for %q", key.Kid, key.Alg, alg)}
+ }
+
+ if key.Use != "" && key.Use != idp.UseSignature {
+ return Decision{Reason: fmt.Sprintf("kid %q is not a signing key", key.Kid)}
+ }
+
+ if reason, ok := v.signature(key, token, alg); !ok {
+ return Decision{Reason: reason}
+ }
+
+ return v.claims(token, "the signature verified")
+}
+
+// algorithm is the allowlist, and under AlgAny it is the token's own choice.
+// The whole difference between the two is whether the server or the attacker
+// picks.
+func (v *Verifier) algorithm(declared string) (string, bool) {
+ if v.mode == AlgAny {
+ return declared, declared != ""
+ }
+
+ if declared == allowedRS || declared == allowedES {
+ return declared, true
+ }
+
+ return "", false
+}
+
+// resolve finds the key. Correct mode resolves ONLY from the fetched key set;
+// TrustEmbeddedKey prefers the one the token brought with it, which is RFC 8725
+// rule 4 violated and CVE-2018-0114 in the wild.
+func (v *Verifier) resolve(keys idp.JWKSet, token idp.Token) (idp.JWK, bool) {
+ if v.mode == TrustEmbeddedKey && token.Header.JWK != nil {
+ return *token.Header.JWK, true
+ }
+
+ key, err := keys.Lookup(token.Header.Kid)
+ if err != nil {
+ return idp.JWK{}, false
+ }
+
+ return key, true
+}
+
+func (v *Verifier) signature(key idp.JWK, token idp.Token, alg string) (string, bool) {
+ // A signature that is absent, all zero, or shorter than THIS KEY produces is
+ // treated as a pass. CVE-2020-28042 is the no-minimum-length shape and
+ // CVE-2022-21449 is the all-zero shape; measured on go1.26.5, ecdsa.Verify
+ // rejects r = s = 0, so a Go verifier only falls for the second one by
+ // skipping the check, which is precisely what this does.
+ //
+ // The length is measured against the resolved key rather than against a
+ // constant, which is what the CVE actually describes: "a valid minimum
+ // length for a signature" is a property of the key, not of the algorithm
+ // name. A constant floor also made this mode accept the weak-key token,
+ // whose signature is legitimately short because its modulus is, and that
+ // overlap would have let one mode alibi another in the matrix below.
+ if v.mode == WeakSignature && weakSignature(key, token.Signature, alg) {
+ return "", true
+ }
+
+ digest := sha256.Sum256(token.SigningInput)
+
+ switch alg {
+ case allowedRS:
+ pub, err := key.RSAPublicKey()
+ if err != nil {
+ return "the key is not usable as RSA", false
+ }
+
+ // An explicit floor, rather than relying on crypto/rsa to have one. The
+ // standard library does enforce 1024 bits, but a reference
+ // implementation that depends on that is correct by luck, and the floor
+ // is a GODEBUG away from being gone.
+ if v.mode != AnyKeySize && pub.N.BitLen() < minimumModulusBits {
+ return fmt.Sprintf("kid %q has a %d-bit modulus", key.Kid, pub.N.BitLen()), false
+ }
+
+ if v.mode == AnyKeySize {
+ if !verifyRaw(pub, digest[:], token.Signature) {
+ return "the signature does not verify", false
+ }
+ return "", true
+ }
+
+ if err := rsa.VerifyPKCS1v15(pub, crypto.SHA256, digest[:], token.Signature); err != nil {
+ return "the signature does not verify", false
+ }
+
+ return "", true
+
+ case allowedES:
+ pub, err := key.ECPublicKey()
+ if err != nil || len(token.Signature) != p256SignatureBytes {
+ return "the signature is not a P-256 signature", false
+ }
+
+ r := new(big.Int).SetBytes(token.Signature[:p256SignatureBytes/2])
+ s := new(big.Int).SetBytes(token.Signature[p256SignatureBytes/2:])
+
+ if !ecdsa.Verify(pub, digest[:], r, s) {
+ return "the signature does not verify", false
+ }
+
+ return "", true
+
+ case string(idp.HS256):
+ // Reachable only under AlgAny. The verifier is configured with an RSA
+ // public key and has just been told to use it as an HMAC secret, which
+ // it can do, because the bytes are bytes. The attacker had those bytes
+ // all along: they are public.
+ secret, err := publicBytes(key)
+ if err != nil {
+ return "the key cannot be used as a MAC secret", false
+ }
+
+ mac := hmac.New(sha256.New, secret)
+ mac.Write(token.SigningInput)
+
+ if !hmac.Equal(mac.Sum(nil), token.Signature) {
+ return "the MAC does not verify", false
+ }
+
+ return "", true
+
+ default:
+ return fmt.Sprintf("algorithm %q cannot be verified", alg), false
+ }
+}
+
+func weakSignature(key idp.JWK, signature []byte, alg string) bool {
+ if len(signature) == 0 || allZero(signature) {
+ return true
+ }
+
+ switch alg {
+ case allowedES:
+ return len(signature) < p256SignatureBytes
+ case allowedRS:
+ pub, err := key.RSAPublicKey()
+ if err != nil {
+ return false
+ }
+ return len(signature) < (pub.N.BitLen()+7)/8
+ default:
+ return false
+ }
+}
+
+// claims is the four checks a stateless verifier has, and SkipClaims is all four
+// missing at once. `exp` is the one that matters most: it is the ONLY revocation
+// mechanism a locally validating verifier has, so a verifier that ignores it has
+// no revocation at all.
+func (v *Verifier) claims(token idp.Token, reason string) Decision {
+ if v.mode == SkipClaims {
+ return Decision{Allow: true, Reason: reason + ", and no claim was checked"}
+ }
+
+ // RFC 8725 section 3.11, Use Explicit Typing, and RFC 9068 which requires
+ // `at+jwt` on an OAuth 2.0 access token. Section 3.12 is the reason it
+ // matters: without it this verifier cannot refuse an ID token presented
+ // where an access token belongs, and research 07 section 1.2 calls that
+ // substitution "cheap to build and catches real bugs".
+ //
+ // This check was MISSING, and `internal/idp/mint.go` cited it to justify a
+ // design decision — the comment claimed the three header traps had been
+ // refused by it. Nothing in the repository implemented it, so the comment
+ // described a refusal that never happened and the property it defends (every
+ // forged token carrying `at+jwt`) was inert against this fixture.
+ if token.Header.Typ != typAccessToken {
+ return Decision{Reason: fmt.Sprintf("typ %q is not %q", token.Header.Typ, typAccessToken)}
+ }
+
+ now := v.now()
+
+ if token.Claims.Expires == nil {
+ return Decision{Reason: "the token carries no `exp`"}
+ }
+
+ if time.Unix(*token.Claims.Expires, 0).Add(clockSkew).Before(now) {
+ return Decision{Reason: "the token has expired"}
+ }
+
+ if token.Claims.NotBefore != nil && time.Unix(*token.Claims.NotBefore, 0).Add(-clockSkew).After(now) {
+ return Decision{Reason: "the token is not valid yet"}
+ }
+
+ if v.issuer != "" && token.Claims.Issuer != v.issuer {
+ return Decision{Reason: fmt.Sprintf("issuer %q is not %q", token.Claims.Issuer, v.issuer)}
+ }
+
+ // RFC 9700: "every resource server is obliged to verify, for every request,
+ // whether the access token sent with that request was meant to be used for
+ // that particular resource server."
+ // Membership, not equality. A token legitimately addressed to several
+ // resource servers carries all of them, and RFC 9700's obligation is that
+ // this one "was meant to be used for that particular resource server".
+ if !token.Claims.Audience.Contains(v.audience) {
+ return Decision{Reason: fmt.Sprintf("audience %q does not include %q", token.Claims.Audience, v.audience)}
+ }
+
+ return Decision{Allow: true, Reason: reason}
+}
+
+// keySet returns the trusted keys, refetching when the cache is cold, stale, or
+// does not hold the kid the token names. The unknown-kid refetch is convention
+// rather than specification: research 07 established that OIDC Discovery says
+// nothing about rotation, nothing about `kid` and nothing about refetching, which
+// is exactly why it is implemented inconsistently and worth a chaos fault.
+func (v *Verifier) keySet(kid string) (idp.JWKSet, error) {
+ v.mu.Lock()
+ cached := v.keys
+ have := v.haveKeys
+ fresh := v.now().Sub(v.keysAt) < v.keyTTL
+ v.mu.Unlock()
+
+ // Not permissive, stale. Key rotation is the control that bounds the damage
+ // of a leaked signing key, and a verifier that never expires its cache has
+ // turned rotation into something that is not a revocation mechanism.
+ if v.mode == JWKSCacheForever && have {
+ return cached, nil
+ }
+
+ // Freshness is ignored while the kid is known, so the only thing that ever
+ // triggers a fetch is meeting a kid this verifier has not seen. That is one
+ // line different from correct and it is the difference between a bounded
+ // stale window and an unbounded one.
+ if v.mode == JWKSRefetchOnKidOnly && have && cached.Has(kid) {
+ return cached, nil
+ }
+
+ if have && fresh && cached.Has(kid) {
+ return cached, nil
+ }
+
+ // A kid this verifier already looked for and did not find is refused without
+ // another fetch. Research 07 section 2.3's own recovery trace ends on exactly
+ // this line: "refetch already attempted, negative cached -> 401".
+ if have && fresh && v.negativelyCached(kid) {
+ return cached, nil
+ }
+
+ // Single-flight. The fetch happens under its own mutex and the cache is
+ // re-checked after acquiring it, so N concurrent requests carrying the same
+ // unknown kid cause ONE upstream fetch rather than N. Measured before this
+ // existed: 200 requests carrying 200 attacker-chosen kids produced 200
+ // upstream fetches, a ratio of 1.00, which research 07 section 1.4 calls
+ // verbatim "a denial-of-service amplifier if unbounded".
+ //
+ // `kid` is an attacker-controlled header field. A reference implementation
+ // that readers copy cannot leave this unbounded, least of all in the one file
+ // whose job is to be the correct side of the comparison.
+ v.fetching.Lock()
+ defer v.fetching.Unlock()
+
+ if set, ok := v.freshWith(kid); ok {
+ return set, nil
+ }
+
+ // A rate cap on the FORCED path only. The periodic staleness refresh above is
+ // untouched, so a rotation is still picked up within one key lifetime even
+ // when a burst of unknown kids has the forced path on cooldown.
+ //
+ // The cap is deliberately short relative to a probe interval: a cooldown long
+ // enough to be a real brake also delays a legitimate rotation, and a verifier
+ // that refuses correctly signed new-key tokens for a second is the
+ // availability failure research 07 section 2.2 counts as `wrongly_rejected`.
+ // The residual amplification is inherent to the unknown-kid convention, which
+ // is why section 2.4 treats it as a fault worth measuring rather than a bug
+ // to be eliminated.
+ if have && fresh && !v.forcedFetchAllowed() {
+ return cached, nil
+ }
+
+ fetched, err := v.fetchKeys()
+ if err != nil {
+ if have && fresh {
+ return cached, nil
+ }
+ return idp.JWKSet{}, err
+ }
+
+ v.mu.Lock()
+ v.keys = fetched
+ v.keysAt = v.now()
+ v.haveKeys = true
+ v.lastForced = v.now()
+ if !fetched.Has(kid) && kid != "" {
+ v.unknownKids[kid] = v.now()
+ }
+ v.mu.Unlock()
+
+ return fetched, nil
+}
+
+// freshWith is the post-lock re-check that makes the fetch single-flight: a
+// waiter that blocked while another goroutine fetched finds the answer already
+// in the cache and does not fetch again.
+func (v *Verifier) freshWith(kid string) (idp.JWKSet, bool) {
+ v.mu.Lock()
+ defer v.mu.Unlock()
+
+ if !v.haveKeys || v.now().Sub(v.keysAt) >= v.keyTTL {
+ return idp.JWKSet{}, false
+ }
+
+ return v.keys, v.keys.Has(kid)
+}
+
+func (v *Verifier) negativelyCached(kid string) bool {
+ v.mu.Lock()
+ defer v.mu.Unlock()
+
+ at, known := v.unknownKids[kid]
+ if !known {
+ return false
+ }
+
+ if v.now().Sub(at) >= v.negativeTTL {
+ delete(v.unknownKids, kid)
+ return false
+ }
+
+ return true
+}
+
+func (v *Verifier) forcedFetchAllowed() bool {
+ v.mu.Lock()
+ defer v.mu.Unlock()
+
+ return v.now().Sub(v.lastForced) >= v.forcedCooldown
+}
+
+func (v *Verifier) fetchKeys() (idp.JWKSet, error) {
+ if v.jwksURI == "" {
+ return idp.JWKSet{}, errors.New("no key set is configured")
+ }
+
+ response, err := v.client.Get(v.jwksURI)
+ if err != nil {
+ return idp.JWKSet{}, err
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ if response.StatusCode != http.StatusOK {
+ return idp.JWKSet{}, fmt.Errorf("the key set answered %d", response.StatusCode)
+ }
+
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ return idp.JWKSet{}, err
+ }
+
+ return idp.ParseJWKSet(body)
+}
+
+// introspect is the opaque-credential path. RFC 7662 is explicit that the
+// ABSENCE of a response is not `active: false`, and a caller that invents one
+// from a timeout, a 503 or an unparseable body has made up a value the provider
+// never gave it.
+func (v *Verifier) introspect(credential string) Decision {
+ if decision, ok := v.cachedDecision(credential); ok {
+ return Decision{Allow: decision, Reason: "a cached decision answered"}
+ }
+
+ active, answered := v.ask(credential)
+
+ if !answered {
+ if v.mode == IntrospectFailOpen {
+ return Decision{Allow: true, Reason: "introspection did not answer usefully, so the credential was allowed"}
+ }
+ return Decision{Reason: "introspection did not answer"}
+ }
+
+ v.remember(credential, active)
+
+ return Decision{Allow: active, Reason: "introspection answered"}
+}
+
+func (v *Verifier) ask(credential string) (bool, bool) {
+ if v.introspectURI == "" {
+ return false, false
+ }
+
+ // RFC 7662 section 2.1: a POST with an `application/x-www-form-urlencoded`
+ // body. This used to send the token as a query parameter, which is khaos's
+ // own private convention and not the specification — so the fixture and the
+ // provider agreed with each other and neither would interoperate with
+ // anything real.
+ response, err := v.client.PostForm(v.introspectURI, url.Values{paramToken: {credential}})
+ if err != nil {
+ return false, false
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ // A 401 from an introspection endpoint is a verdict on the CALLER, not on
+ // the token. RFC 7662 section 2.3 defines it as the response when the
+ // protected resource's own credentials to the authorization server are
+ // invalid, and the RFC defines no 403 at all.
+ //
+ // This used to be folded into "the provider answered, the token is inactive",
+ // which is a conflation and the opposite of what the cited RFC says: it turns
+ // "my credentials to the authority are wrong" into a flood of token denials
+ // with no distinguishable signal. Both paths deny under this mode, so nothing
+ // about the range's behaviour changes; what changes is that the file readers
+ // copy no longer teaches the conflation.
+ if response.StatusCode != http.StatusOK {
+ return false, false
+ }
+
+ var body idp.Introspection
+ if err := json.NewDecoder(response.Body).Decode(&body); err != nil {
+ return false, false
+ }
+
+ return body.Active, true
+}
+
+func (v *Verifier) cachedDecision(credential string) (bool, bool) {
+ v.mu.Lock()
+ defer v.mu.Unlock()
+
+ held, ok := v.decisions[credential]
+ if !ok {
+ return false, false
+ }
+
+ if held.forever {
+ return held.allow, true
+ }
+
+ if v.now().After(held.expires) {
+ return false, false
+ }
+
+ return held.allow, true
+}
+
+func (v *Verifier) remember(credential string, allow bool) {
+ v.mu.Lock()
+ defer v.mu.Unlock()
+
+ v.decisions[credential] = cachedDecision{
+ allow: allow,
+ expires: v.now().Add(v.decisionTTL),
+ forever: v.mode == DecisionCacheForever && allow,
+ }
+}
+
+func publicBytes(key idp.JWK) ([]byte, error) {
+ pub, err := key.RSAPublicKey()
+ if err != nil {
+ return nil, err
+ }
+
+ return idp.SPKIPEMBytes(pub)
+}
+
+func verifyRaw(pub *rsa.PublicKey, digest, signature []byte) bool {
+ return idp.VerifyPKCS1v15Raw(pub, digest, signature)
+}
+
+func allZero(raw []byte) bool {
+ for _, b := range raw {
+ if b != 0 {
+ return false
+ }
+ }
+ return true
+}
+
+// Resolve maps an operator's string onto a mode, and falls back to Correct for
+// anything it does not recognise. That fallback is deliberate: one `--fail-mode`
+// drives the whole range, and a mode meant for a different gateway must leave
+// this one behaving correctly rather than refusing to start.
+func Resolve(mode string) Mode {
+ trimmed := strings.TrimSpace(mode)
+
+ if Known(trimmed) {
+ return Mode(trimmed)
+ }
+
+ return Correct
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/verifier_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/verifier_test.go
new file mode 100644
index 00000000..ca4727db
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/verifier/verifier_test.go
@@ -0,0 +1,376 @@
+// ©AngelaMos | 2026
+// verifier_test.go
+
+package verifier_test
+
+import (
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/idp"
+ "github.com/CarterPerez-dev/khaos/internal/verifier"
+)
+
+// RFC 8725 rule 2, exercised directly, because no forged token can isolate it.
+//
+// The matrix proves alg-confusion is refused by Correct mode, but deleting this
+// check leaves the matrix green: the allowlist refuses HS256 first. And a token
+// that trips the cross-check any OTHER way, such as ES256 against an RSA kid,
+// also carries a signature that cannot verify, so the cross-check is never the
+// only thing standing between the token and acceptance.
+//
+// It stays in the reference implementation anyway, because it is normative and
+// because this is the file readers copy. This test is what stops it being a
+// branch nothing runs.
+func TestKeyAlgorithmCrossCheckRefusesAHeaderThatDisagreesWithTheKey(t *testing.T) {
+ provider, v := pair(t, verifier.Correct)
+
+ honest, err := provider.HonestToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ if decision := v.Decide(honest); !decision.Allow {
+ t.Fatalf("the honest token was refused: %s", decision.Reason)
+ }
+
+ token, err := idp.Parse(honest)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ // The EC key khaos publishes is bound to ES256. Naming it from an RS256
+ // header is the mismatch the rule forbids.
+ ec, ok := signingKidFor(provider, string(idp.ES256))
+ if !ok {
+ t.Fatal("khaos's provider publishes no ES256 key, so this mismatch is not expressible")
+ }
+
+ mismatched := idp.Retarget(token, ec)
+
+ decision := v.Decide(mismatched)
+
+ if decision.Allow {
+ t.Fatal("an RS256 header naming a key bound to ES256 was accepted")
+ }
+
+ if !containsText(decision.Reason, "is bound to") {
+ t.Fatalf("the refusal was not the cross-check: %s", decision.Reason)
+ }
+}
+
+// A key published for encryption must not verify signatures. It is cheap, it is
+// normative, and a verifier that ignores `use` will happily check a signature
+// with a key its provider never intended for signing.
+//
+// This needs a DOCTORED key set, and that is the whole point. An earlier version
+// of this test asserted only that every key khaos publishes carries `use: sig` —
+// which is true, and which is why the branch was statically unreachable from the
+// entire suite. Measured: gutting the `use` check left every package green.
+// khaos's provider hard-codes `use: sig` in PublicJWK, so nothing khaos serves
+// can ever reach that branch; only a key set khaos did not mint can.
+func TestAKeyNotMarkedForSigningIsRefused(t *testing.T) {
+ provider, _ := pair(t, verifier.Correct)
+
+ honest, err := provider.HonestToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ token, err := idp.Parse(honest)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ // The real key set, with `use` rewritten to encryption. Everything else
+ // about the token and the key is correct, so `use` is the only thing that
+ // can refuse it.
+ doctored := provider.PublishedSet()
+ for i := range doctored.Keys {
+ if doctored.Keys[i].Kid == token.Header.Kid {
+ doctored.Keys[i].Use = "enc"
+ }
+ }
+
+ served := serveSet(t, doctored)
+
+ strict := verifier.New(verifier.Config{
+ Issuer: provider.Issuer(),
+ Audience: idp.DefaultAudience,
+ JWKSURI: served,
+ Mode: verifier.Correct,
+ KeyTTL: keyTTL,
+ })
+
+ decision := strict.Decide(honest)
+
+ if decision.Allow {
+ t.Fatal("a signature was verified with a key published for encryption")
+ }
+
+ if !containsText(decision.Reason, "not a signing key") {
+ t.Fatalf("the refusal was not the `use` check: %s", decision.Reason)
+ }
+
+ // The control: the same verifier against the UNDOCTORED set accepts. Without
+ // it, a verifier that refused everything would pass the assertion above.
+ loose := verifier.New(verifier.Config{
+ Issuer: provider.Issuer(),
+ Audience: idp.DefaultAudience,
+ JWKSURI: serveSet(t, provider.PublishedSet()),
+ Mode: verifier.Correct,
+ KeyTTL: keyTTL,
+ })
+
+ if decision := loose.Decide(honest); !decision.Allow {
+ t.Fatalf("the same token was refused against the real key set, so the test above proves "+
+ "nothing about `use`: %s", decision.Reason)
+ }
+}
+
+// One `--fail-mode` drives the whole range, and most of its values are meant for
+// a different gateway. An unrecognised value has to leave this verifier CORRECT,
+// or `khaos range up --fail-mode stale-rearm` would silently also break the
+// verifier and every forgery experiment would report a finding it did not cause.
+func TestAnUnrecognisedModeLeavesTheVerifierCorrect(t *testing.T) {
+ for _, mode := range []string{"stale-rearm", "open", "", "nonsense", "ALG-ANY"} {
+ if got := verifier.Resolve(mode); got != verifier.Correct {
+ t.Errorf("Resolve(%q) = %q, want %q", mode, got, verifier.Correct)
+ }
+ }
+
+ for _, mode := range verifier.Modes() {
+ if got := verifier.Resolve(mode); string(got) != mode {
+ t.Errorf("Resolve(%q) = %q, want it unchanged", mode, got)
+ }
+ }
+}
+
+// The decision cache has to expire, or Correct mode is DecisionCacheForever
+// wearing a different name and the revocation experiment would find the same
+// bug in both.
+func TestTheDecisionCacheExpiresInCorrectMode(t *testing.T) {
+ provider, v := pair(t, verifier.Correct)
+
+ credential := provider.Opaque()
+
+ if decision := v.Decide(credential); !decision.Allow {
+ t.Fatalf("an active credential was refused: %s", decision.Reason)
+ }
+
+ if err := provider.Arm(idp.Trap{Kind: idp.Revoke, Credential: credential}); err != nil {
+ t.Fatalf("revoke: %v", err)
+ }
+
+ time.Sleep(decisionTTL * 3)
+
+ if decision := v.Decide(credential); decision.Allow {
+ t.Fatalf("a revoked credential was still honoured after the cache lifetime: %s", decision.Reason)
+ }
+}
+
+// An unknown kid has to trigger a refetch, or an overlapping rotation breaks
+// every verifier rather than none of them. Research 07 established that this
+// behaviour is convention and not specification, which is exactly why it is
+// implemented inconsistently and worth measuring.
+func TestAnUnknownKidTriggersARefetch(t *testing.T) {
+ provider, v := pair(t, verifier.Correct)
+
+ honest, err := provider.HonestToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ if decision := v.Decide(honest); !decision.Allow {
+ t.Fatalf("warm-up refused: %s", decision.Reason)
+ }
+
+ before := provider.JWKSFetches()
+
+ if err := provider.Arm(idp.Trap{Kind: idp.KeyRotation}); err != nil {
+ t.Fatalf("rotate: %v", err)
+ }
+
+ rotated, err := provider.TrappedToken()
+ if err != nil {
+ t.Fatalf("mint under the new key: %v", err)
+ }
+
+ if decision := v.Decide(rotated); !decision.Allow {
+ t.Fatalf("a correctly signed token under a published new key was refused: %s", decision.Reason)
+ }
+
+ if provider.JWKSFetches() <= before {
+ t.Fatal("the verifier accepted a token under an unseen kid without refetching the key set")
+ }
+}
+
+func signingKidFor(provider *idp.Provider, alg string) (string, bool) {
+ for _, key := range provider.PublishedSet().Keys {
+ if key.Alg == alg {
+ return key.Kid, true
+ }
+ }
+ return "", false
+}
+
+func containsText(haystack, needle string) bool {
+ for i := 0; i+len(needle) <= len(haystack); i++ {
+ if haystack[i:i+len(needle)] == needle {
+ return true
+ }
+ }
+ return false
+}
+
+// A token with no `exp` must be refused, and the branch was unreachable from the
+// suite: khaos's provider always sets one. Measured — flipping that refusal to
+// an allow left every package green.
+//
+// `exp` is the ONLY revocation mechanism a locally validating verifier has, so a
+// token without one is a credential that can never be taken away.
+func TestATokenWithNoExpiryIsRefused(t *testing.T) {
+ provider, v := pair(t, verifier.Correct)
+
+ honest, err := provider.HonestToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ if decision := v.Decide(honest); !decision.Allow {
+ t.Fatalf("the honest token was refused: %s", decision.Reason)
+ }
+
+ // The signature will not verify once `exp` is removed, so this asserts the
+ // CLAIM check specifically rather than the signature check: the claim branch
+ // runs after the signature one, so a token stripped of `exp` has to reach it
+ // another way.
+ stripped := idp.WithoutExpiry(provider)
+
+ decision := v.Decide(stripped)
+
+ if decision.Allow {
+ t.Fatal("a token carrying no `exp` was accepted, so nothing can ever revoke it")
+ }
+
+ if !containsText(decision.Reason, "no `exp`") {
+ t.Fatalf("the refusal was not the expiry check: %s", decision.Reason)
+ }
+}
+
+// RFC 8725 section 3.11 explicit typing, and RFC 9068's `at+jwt`. Section 3.12
+// is why it matters: without it this verifier cannot refuse an ID token
+// presented where an access token belongs.
+//
+// The check did not exist, and `internal/idp/mint.go` cited it to justify every
+// forged token carrying `at+jwt`. Measured before it was implemented: `typ` of
+// `id_token`, `JWT`, `""` and `anything-at-all` were all accepted.
+func TestATokenOfTheWrongTypeIsRefused(t *testing.T) {
+ provider, v := pair(t, verifier.Correct)
+
+ for _, typ := range []string{"JWT", "id_token", "dpop+jwt", ""} {
+ retyped := idp.Retyped(provider, typ)
+
+ decision := v.Decide(retyped)
+
+ if decision.Allow {
+ t.Errorf("a token with typ %q was accepted where an access token belongs", typ)
+ continue
+ }
+
+ if !containsText(decision.Reason, "typ") {
+ t.Errorf("typ %q was refused for the wrong reason: %s", typ, decision.Reason)
+ }
+ }
+
+ // The control. Every forged token khaos mints carries `at+jwt` precisely so
+ // this check does not pre-empt the one each trap probes.
+ honest, err := provider.HonestToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ if decision := v.Decide(honest); !decision.Allow {
+ t.Fatalf("the explicit-typing check refuses khaos's own correct token: %s", decision.Reason)
+ }
+}
+
+// The amplifier. `kid` is an attacker-controlled header field, and research 07
+// section 1.4 calls an unbounded refetch "a denial-of-service amplifier".
+//
+// Measured before the fix: 200 requests carrying 200 attacker-chosen kids
+// produced 200 upstream fetches, a ratio of 1.00.
+func TestUnknownKidsDoNotAmplifyIntoOneFetchEach(t *testing.T) {
+ provider, v := pair(t, verifier.Correct)
+
+ honest, err := provider.HonestToken()
+ if err != nil {
+ t.Fatalf("mint: %v", err)
+ }
+
+ if decision := v.Decide(honest); !decision.Allow {
+ t.Fatalf("warm-up refused: %s", decision.Reason)
+ }
+
+ token, err := idp.Parse(honest)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ const attempts = 200
+
+ before := provider.JWKSFetches()
+
+ for i := range attempts {
+ _ = v.Decide(idp.Retarget(token, "attacker-chosen-kid-"+strconv.Itoa(i)))
+ }
+
+ fetches := provider.JWKSFetches() - before
+
+ if fetches >= attempts {
+ t.Fatalf("%d requests carrying unknown kids caused %d upstream fetches; an "+
+ "attacker-controlled header field must not be a one-to-one amplifier", attempts, fetches)
+ }
+
+ // The control. Bounding the refetch must not stop a legitimate rotation being
+ // picked up, which is the availability failure research 07 section 2.2 counts
+ // as `wrongly_rejected`.
+ //
+ // BOUNDED pickup, not instant. The burst above is an attack, and the cooldown
+ // it trips is the whole point: research 07 section 1.4 lists a refetch
+ // cooldown as a mitigation precisely because the unknown-kid path is
+ // attacker-driven, and every mitigation there trades a little rotation
+ // latency for it. Demanding instant pickup immediately after two hundred
+ // forged kids would be asserting that the brake does not work.
+ if err := provider.Arm(idp.Trap{Kind: idp.KeyRotation}); err != nil {
+ t.Fatalf("rotate: %v", err)
+ }
+
+ rotated, err := provider.TrappedToken()
+ if err != nil {
+ t.Fatalf("mint under the new key: %v", err)
+ }
+
+ deadline := time.Now().Add(rotationPickupBudget)
+ var last string
+
+ for time.Now().Before(deadline) {
+ decision := v.Decide(rotated)
+ if decision.Allow {
+ return
+ }
+ last = decision.Reason
+ time.Sleep(10 * time.Millisecond)
+ }
+
+ t.Fatalf("a correctly signed token under a newly published key was still refused after %s, "+
+ "so the amplifier fix turned a routine rotation into an outage: %s",
+ rotationPickupBudget, last)
+}
+
+// How long a legitimate rotation may take to be picked up after an unknown-kid
+// burst has tripped the refetch cooldown. It is comfortably above both the
+// cooldown and the key lifetime these tests use, so a failure here means the
+// rotation is never picked up rather than merely delayed.
+const rotationPickupBudget = 2 * time.Second
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/version/version.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/version/version.go
new file mode 100644
index 00000000..bc5ff918
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/version/version.go
@@ -0,0 +1,73 @@
+// ©AngelaMos | 2026
+// version.go
+
+package version
+
+import "runtime/debug"
+
+const (
+ Name = "khaos"
+ Tagline = "security chaos engineering"
+ Author = "carterperez.dev"
+)
+
+const (
+ fallback = "dev"
+ develMarker = "(devel)"
+ keyRevision = "vcs.revision"
+ keyModified = "vcs.modified"
+ shortSHALen = 12
+ devPrefix = "dev+"
+ dirtySuffix = "+dirty"
+ trueValue = "true"
+)
+
+var ldflagsVersion string
+
+func Version() string {
+ if ldflagsVersion != "" {
+ return ldflagsVersion
+ }
+
+ info, ok := debug.ReadBuildInfo()
+ if !ok {
+ return fallback
+ }
+
+ if v := info.Main.Version; v != "" && v != develMarker {
+ return v
+ }
+
+ return fromVCS(info)
+}
+
+func fromVCS(info *debug.BuildInfo) string {
+ var (
+ revision string
+ dirty bool
+ )
+
+ for _, setting := range info.Settings {
+ switch setting.Key {
+ case keyRevision:
+ revision = setting.Value
+ case keyModified:
+ dirty = setting.Value == trueValue
+ }
+ }
+
+ if revision == "" {
+ return fallback
+ }
+
+ if len(revision) > shortSHALen {
+ revision = revision[:shortSHALen]
+ }
+
+ out := devPrefix + revision
+ if dirty {
+ out += dirtySuffix
+ }
+
+ return out
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/workspace/workspace.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/workspace/workspace.go
new file mode 100644
index 00000000..bb13c9ba
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/workspace/workspace.go
@@ -0,0 +1,101 @@
+// ©AngelaMos | 2026
+// workspace.go
+
+package workspace
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+ "github.com/CarterPerez-dev/khaos/internal/lock"
+ "github.com/CarterPerez-dev/khaos/internal/run"
+ "github.com/CarterPerez-dev/khaos/internal/statedir"
+)
+
+const errDockerHint = "%w (run `khaos doctor` for the fix)"
+
+type Workspace struct {
+ // Reconciled is what the startup sweep of EXPIRED orphans did, so a command
+ // can tell the operator that khaos cleaned up before it started rather than
+ // leaving it to be discovered later.
+ Reconciled run.Sweep
+
+ Docker *dockerx.Client
+ Journal *journal.Journal
+ Paths statedir.Paths
+
+ held *lock.Lock
+}
+
+func OpenExclusive(ctx context.Context) (*Workspace, error) {
+ return open(ctx, true)
+}
+
+func OpenShared(ctx context.Context) (*Workspace, error) {
+ return open(ctx, false)
+}
+
+func open(ctx context.Context, exclusive bool) (*Workspace, error) {
+ paths, err := statedir.Default()
+ if err != nil {
+ return nil, err
+ }
+
+ var held *lock.Lock
+ if exclusive {
+ if held, err = lock.Acquire(paths.Lock()); err != nil {
+ return nil, err
+ }
+ }
+
+ log, err := journal.Open(paths.Journal())
+ if err != nil {
+ release(held)
+ return nil, err
+ }
+
+ client, err := dockerx.Open(ctx)
+ if err != nil {
+ _ = log.Close()
+ release(held)
+ return nil, fmt.Errorf(errDockerHint, err)
+ }
+
+ work := &Workspace{Docker: client, Journal: log, Paths: paths, held: held}
+
+ // The promise `khaos inject --ttl` makes is "before `khaos reap` or the next
+ // khaos invocation reclaims it". Nothing implemented the second half: only
+ // the explicit reclaim command and the reap loop ever called run.Reclaim, so
+ // a fault whose controller was killed outlived its TTL indefinitely unless
+ // the operator happened to run one of those two commands.
+ //
+ // EXPIRED leases only, and only on an exclusive open. A live lease belongs to
+ // a session that is still running and reverting it would steal another run's
+ // fault out from under it; a shared open is a read-only command and must not
+ // mutate anything. That is the same split `khaos reap` already uses.
+ if exclusive {
+ work.Reconciled = run.Reclaim(ctx, work.Env(), log, time.Now().UTC(), true)
+ }
+
+ return work, nil
+}
+
+func (w *Workspace) Env() inject.Env {
+ return inject.Env{Docker: w.Docker}
+}
+
+func (w *Workspace) Close() {
+ _ = w.Docker.Close()
+ _ = w.Journal.Close()
+ release(w.held)
+}
+
+func release(held *lock.Lock) {
+ if held != nil {
+ _ = held.Release()
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/justfile b/PROJECTS/intermediate/chaos-engineering-tool/justfile
new file mode 100644
index 00000000..7e137ecf
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/justfile
@@ -0,0 +1,67 @@
+# ©AngelaMos | 2026
+# justfile
+
+set shell := ["bash", "-cu"]
+
+binary := "khaos"
+
+default:
+ @just --list
+
+build:
+ go build -o {{binary}} ./cmd/{{binary}}
+
+run *args:
+ go run ./cmd/{{binary}} {{args}}
+
+banner: build
+ ./{{binary}} banner
+
+test:
+ go test ./...
+
+race:
+ go test -race ./...
+
+vet:
+ go vet ./...
+
+fmt:
+ gofmt -w .
+
+lint: vet
+ test -z "$(gofmt -l .)"
+
+tidy:
+ go mod tidy
+
+tidy-check:
+ go mod tidy -diff
+
+check: lint tidy-check test colors
+
+gate: check race smoke survive range-gate proxy-gate identity-gate family-gate
+
+colors: build
+ ./scripts/colors.sh
+
+smoke: build
+ ./scripts/smoke.sh
+
+survive: build
+ ./scripts/survive.sh
+
+range-gate: build
+ ./scripts/range.sh
+
+proxy-gate: build
+ ./scripts/proxy.sh
+
+identity-gate: build
+ ./scripts/identity.sh
+
+family-gate: build
+ ./scripts/family.sh
+
+clean:
+ rm -f {{binary}}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/Dockerfile b/PROJECTS/intermediate/chaos-engineering-tool/range/Dockerfile
new file mode 100644
index 00000000..171bc4d5
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/Dockerfile
@@ -0,0 +1,20 @@
+# ©AngelaMos | 2026
+# Dockerfile
+
+FROM golang:1.26-alpine AS build
+
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY cmd ./cmd
+COPY internal ./internal
+RUN CGO_ENABLED=0 go build -trimpath -o /out/khaos-range ./cmd/khaos-range
+
+FROM alpine:3.22
+
+RUN adduser -D -u 10001 range
+COPY --from=build /out/khaos-range /usr/local/bin/khaos-range
+
+USER range
+ENTRYPOINT ["/usr/local/bin/khaos-range"]
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
new file mode 100644
index 00000000..c6962da9
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
@@ -0,0 +1,114 @@
+# ©AngelaMos | 2026
+# compose.yml
+
+name: khaos-range
+
+services:
+ idp:
+ container_name: khaos-range-idp
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["idp"]
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ networks: [range]
+ healthcheck:
+ test: ["CMD", "/usr/local/bin/khaos-range", "healthcheck"]
+ interval: 2s
+ timeout: 2s
+ retries: 5
+ restart: "no"
+
+ gateway:
+ container_name: khaos-range-gateway
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["gateway"]
+ environment:
+ KHAOS_RANGE_UPSTREAM: "http://idp:8080"
+ KHAOS_RANGE_FAIL_MODE: "${KHAOS_RANGE_FAIL_MODE:-closed}"
+ KHAOS_RANGE_CACHE_TTL: "${KHAOS_RANGE_CACHE_TTL:-2s}"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ ports:
+ - "127.0.0.1:${KHAOS_RANGE_GATEWAY_PORT:?khaos chooses this port; run `khaos range up`}:8080"
+ depends_on:
+ - idp
+ networks: [range]
+ restart: "no"
+
+ # The same gateway, reaching the same identity provider, but addressing it
+ # through khaos's own in-process fault proxy on the host instead of directly.
+ # It is a SECOND service rather than a rewiring of the first so that the
+ # Docker-native tier keeps being measured on a direct path: an experiment that
+ # pauses a container is not measuring the Docker-native tier if its traffic
+ # runs through a proxy. Running the same experiment against both and comparing
+ # is what proves the proxy tier adds nothing of its own.
+ gateway-proxied:
+ container_name: khaos-range-gateway-proxied
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["gateway"]
+ environment:
+ KHAOS_RANGE_UPSTREAM: "http://host.docker.internal:${KHAOS_RANGE_PROXY_PORT:?khaos chooses this port; run `khaos range up`}"
+ KHAOS_RANGE_FAIL_MODE: "${KHAOS_RANGE_FAIL_MODE:-closed}"
+ KHAOS_RANGE_CACHE_TTL: "${KHAOS_RANGE_CACHE_TTL:-2s}"
+ # host-gateway is the DEFAULT bridge's gateway, measured, not this network's
+ # gateway: a listener on this network's own gateway address is unreachable.
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ ports:
+ - "127.0.0.1:${KHAOS_RANGE_PROXIED_PORT:?khaos chooses this port; run `khaos range up`}:8080"
+ depends_on:
+ - idp
+ networks: [range]
+ restart: "no"
+
+ # The third gateway, and the only one whose identity provider is khaos itself
+ # rather than the idp container. khaos cannot make a correct provider mint an
+ # `alg: none` token or withdraw its own signing key on command, so the whole
+ # JWT family needs a target that resolves keys from khaos's in-process
+ # provider. It is a THIRD service for the same reason the proxied one is a
+ # second: repointing an existing gateway would change what the suites that
+ # already measure it are asserting.
+ #
+ # It verifies JWTs locally AND introspects opaque credentials, because the two
+ # strategies have different outage shapes and the family spans both.
+ gateway-verifier:
+ container_name: khaos-range-gateway-verifier
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["verifier"]
+ environment:
+ KHAOS_RANGE_ISSUER: "http://host.docker.internal:${KHAOS_RANGE_IDP_PORT:?khaos chooses this port; run `khaos range up`}"
+ KHAOS_RANGE_FAIL_MODE: "${KHAOS_RANGE_FAIL_MODE:-closed}"
+ KHAOS_RANGE_CACHE_TTL: "${KHAOS_RANGE_CACHE_TTL:-2s}"
+ KHAOS_RANGE_KEY_TTL: "${KHAOS_RANGE_KEY_TTL:-1s}"
+ # host-gateway is the DEFAULT bridge's gateway, measured, not this network's
+ # gateway: a listener on this network's own gateway address is unreachable.
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ ports:
+ - "127.0.0.1:${KHAOS_RANGE_VERIFIER_PORT:?khaos chooses this port; run `khaos range up`}:8080"
+ networks: [range]
+ restart: "no"
+
+networks:
+ range:
+ name: khaos-range
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/colors.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/colors.sh
new file mode 100755
index 00000000..c1899095
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/colors.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# colors.sh
+
+set -euo pipefail
+
+BIN="${BIN:-./khaos}"
+ART="${ART:-internal/banner/khaos.ascii}"
+
+failures=0
+
+check_bare() {
+ local label="$1" command="$2" count
+ count="$(eval "$command" | grep -c $'\x1b' || true)"
+
+ if [[ "$count" -ne 0 ]]; then
+ echo "FAIL ${label}: ${count} escape sequence(s), want 0"
+ failures=$((failures + 1))
+ return
+ fi
+
+ echo "ok ${label}"
+}
+
+check_bare "clean env, piped" "env -i PATH=\$PATH $BIN banner"
+check_bare "NO_COLOR=1" "NO_COLOR=1 $BIN banner"
+check_bare "NO_COLOR + FORCE" "NO_COLOR=1 CLICOLOR_FORCE=1 $BIN banner"
+check_bare "--no-color" "$BIN banner --no-color"
+check_bare "--no-color=true" "$BIN banner --no-color=true"
+check_bare "--no-color=1" "$BIN banner --no-color=1"
+check_bare "--no-color=T" "$BIN banner --no-color=T"
+check_bare "TERM=dumb" "env -i PATH=\$PATH TERM=dumb $BIN banner"
+
+expected="$(grep -o $'█' "$ART" | wc -l)"
+rendered="$($BIN banner --no-color | grep -o $'█' | wc -l)"
+
+if [[ "$rendered" -ne "$expected" ]]; then
+ echo "FAIL glyph survival: rendered ${rendered} full blocks, the asset has ${expected}"
+ failures=$((failures + 1))
+else
+ echo "ok glyph survival (${expected} full blocks, derived from the asset)"
+fi
+
+if [[ $failures -ne 0 ]]; then
+ echo "${failures} color assertion(s) failed"
+ exit 1
+fi
+
+echo "colors: all assertions passed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh
new file mode 100755
index 00000000..2e450e43
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh
@@ -0,0 +1,410 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# family.sh
+
+# The `identity.` family's exit gate, covering the seventeen experiments built on
+# khaos's own in-process identity provider. `scripts/identity.sh` keeps the three
+# `identity.idp_outage` cases, because those measure the PROXY tier and the
+# idleness discriminator and they are a separate claim.
+#
+# Two halves, and the second one is what has teeth. Every experiment is run
+# against a range carrying the bug it names, where it must find a defect, AND
+# against a CORRECT verifier, where it must find nothing. An experiment that only
+# ever reported findings would pass the first half while being useless, and one
+# that can never report a finding would pass the second half while being inert.
+# This project has already shipped the second kind once: the first
+# `identity.idp_outage` returned `passed / HELD` against a range containing
+# exactly the bug it names.
+
+set -euo pipefail
+
+BIN="${BIN:-$PWD/khaos}"
+
+KHAOS_STATE_DIR="$(mktemp -d)"
+export KHAOS_STATE_DIR
+
+# The fault window has to clear MinimumToClaimClosed (30 paired samples) on the
+# correct-verifier half, or every control run comes back `passed / HELD` with a
+# thin-evidence warning and the gate is asserting less than it looks like it is.
+TIMING=(--baseline-for 3s --fault-for 6s --recover-for 2s --sample-every 150ms)
+
+VERIFIER_GATEWAY=khaos-range-gateway-verifier
+
+# Each row is a fail mode followed by the experiments that mode must let through.
+# The mapping is deliberately many-to-one in places: `alg-any` is a single
+# structural defect, resolving the key and THEN dispatching on the token's
+# algorithm, and it is the root cause of both `alg: none` acceptance and
+# HS256/RS256 confusion. Research 07 frames them that way and so does the range.
+CATCHING=(
+ "alg-any|identity.alg_none|identity.alg_confusion"
+ "trust-embedded-key|identity.embedded_jwk"
+ "skip-claims|identity.claim_expired|identity.claim_not_yet_valid|identity.claim_wrong_audience|identity.claim_foreign_issuer"
+ "weak-sig|identity.signature_truncated|identity.signature_zero"
+ "any-key-size|identity.signature_weak_key"
+ "jwks-fail-open|identity.jwks_unreachable"
+ # One experiment per cycle, deliberately. A verifier whose key cache NEVER
+ # expires keeps the first run's key set forever, and khaos mints a fresh one
+ # every run, so a second consecutive run against the same container cannot
+ # establish a baseline at all. Measured: `kid_rollover` passed and
+ # `key_rotation` then came back `baseline-not-met`. That is a true property
+ # of the bug rather than a gate artifact, so each of these gets a fresh
+ # container instead of the gate working around it.
+ "jwks-cache-forever|identity.kid_rollover"
+ "jwks-cache-forever|identity.key_rotation"
+ # Refetch only on an unknown kid, never on a timer. Research 07 section 2.3
+ # calls this "the whole finding": the eviction of a retired key is a side
+ # effect of the refetch a NEW-key token triggers, so a target with no
+ # new-key traffic honours the retired key for an unbounded time. khaos's own
+ # valid stream used to supply that traffic and hide it.
+ "jwks-kid-only|identity.kid_rollover"
+ "introspect-fail-open|identity.introspection_error|identity.malformed_response|identity.idp_latency"
+ "decision-cache-forever|identity.revoked_token"
+)
+
+# What a CORRECT verifier must produce, per experiment. Anything not listed must
+# reach `passed / HELD`.
+#
+# Asserting only "exit 0, no PERMITTED" was too weak: `identity.jwks_unreachable`
+# against a correct verifier legitimately lands `deviated / DENIED` once its key
+# cache lapses and it refuses everything, so that row passed half two while
+# asserting nothing about what khaos actually concluded.
+declare -A CORRECT_VERDICT=(
+ # A correct verifier that cannot fetch its key set refuses everything,
+ # including the valid credential, once the cached set lapses.
+ [identity.jwks_unreachable]="deviated / DENIED"
+
+ # Same shape, and it is the point of these three. Their valid control is an
+ # OPAQUE credential, so it travels the introspection path the fault breaks —
+ # a locally validated JWT control never reaches the fault and would report a
+ # clean survival of something the target never experienced. A correct
+ # verifier with no usable introspection therefore denies the control too,
+ # which is failing CLOSED and is not the defect these experiments declare.
+ [identity.introspection_error]="deviated / DENIED"
+ [identity.malformed_response]="deviated / DENIED"
+ [identity.idp_latency]="deviated / DENIED"
+)
+
+# The one experiment in the whole catalogue where REFUSING is the defect. An
+# overlapping rotation publishes both keys precisely so nothing breaks, so a
+# verifier that will not pick the new one up has turned a routine key rotation
+# into an outage. If this ever stops being the exception, nothing else in khaos
+# exercises the directional axis's other branch.
+INVERTED=identity.key_rotation
+
+failures=0
+report=""
+last_code=0
+
+cleanup() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ rm -rf "$KHAOS_STATE_DIR"
+}
+trap cleanup EXIT INT TERM HUP
+
+pass() { echo "ok $1"; }
+fail() { echo "FAIL $1"; failures=$((failures + 1)); }
+
+start_range() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ if ! "$BIN" range up --fail-mode "$1" --quiet --no-color >/dev/null 2>&1; then
+ fail "range up --fail-mode $1"
+ return 1
+ fi
+ return 0
+}
+
+# Sets the globals `report` and `last_code`. Never call it through a command
+# substitution: that runs it in a subshell and the report is lost while the exit
+# code still comes back, which looks like a passing gate with no evidence.
+run_experiment() {
+ set +e
+ report="$("$BIN" run "$1" --no-color "${TIMING[@]}" "${@:2}" 2>&1)"
+ last_code=$?
+ set -e
+}
+
+verdict_line() { grep -E '^ verdict' <<<"$report" || echo '<none>'; }
+
+assert_verdict() {
+ local label="$1" want="$2"
+ if grep -qE "^ verdict .*${want}" <<<"$report"; then
+ pass "${label} verdict is ${want}"
+ else
+ fail "${label} expected ${want}, got: $(verdict_line)"
+ fi
+}
+
+assert_exit() {
+ local label="$1" got="$2" want="$3"
+ if [[ "$got" == "$want" ]]; then
+ pass "${label} exit code ${want}"
+ else
+ fail "${label} exit code ${got}, want ${want}"
+ fi
+}
+
+# The fault has to be confirmed by something khaos MEASURED, not by a constant
+# describing the technique. The proxy tier paid for this one: a single grepped
+# string turned out to be the constant, so every fault "confirmed" identically
+# and a confirmation gutted to always-succeed went unnoticed.
+assert_confirmed_by_an_observable() {
+ local label="$1" experiment="$2"
+ local line
+ line="$(grep -E '^ plan confirm' <<<"$report" || true)"
+
+ if [[ -z "$line" ]]; then
+ fail "${label} never confirmed its fault, so nothing it measured is attributable"
+ return
+ fi
+
+ if ! grep -q "fault confirmed by" <<<"$line"; then
+ fail "${label} confirm line is not a confirmation: ${line}"
+ return
+ fi
+
+ # Every observable names khaos's own provider or a measured quantity. A line
+ # that is just the experiment's own name would be the kind restated.
+ if grep -qE "fault confirmed by (${experiment}|idp-)" <<<"$line"; then
+ fail "${label} confirmed itself by restating its own kind: ${line}"
+ return
+ fi
+
+ pass "${label} confirmed by a measured observable"
+}
+
+# Which CLASS of finding the verdict states. A forged credential and a revoked
+# one are different observations and the verdict has to make the one that
+# happened: "a credential khaos revoked was still accepted" is not "a credential
+# khaos minted with a deliberate defect was accepted".
+# Per CLASS, not an OR over every phrase khaos can print. An OR let a forgery
+# experiment satisfy this with the generic monotonic reason, which says a
+# credential the target had already rejected got through and says nothing about
+# the trap the experiment is named for.
+assert_states_a_finding() {
+ local label="$1" want="$2"
+
+ if grep -qE "^ why .*${want}" <<<"$report"; then
+ pass "${label} states the observation its own stream carried"
+ else
+ fail "${label} did not state ${want}: $(grep -E '^ why' <<<"$report" | head -1)"
+ fi
+}
+
+# The exposure as a NUMBER. A count of acceptances with no span leaves the reader
+# inferring the window from the timing flags, and research 07 section 2.2 calls
+# this the real exposure window of a leaked key.
+assert_reports_the_exposure_bound() {
+ local label="$1"
+ if grep -qE "^ why .*accepted over a span of at least" <<<"$report"; then
+ pass "${label} reports the exposure as a bounded span"
+ else
+ fail "${label} reported a revocation finding with no exposure bound: $(grep -E '^ why' <<<"$report" | head -1)"
+ fi
+}
+
+# khaos must never claim a cache has no expiry from a bounded observation.
+# Research 07's open question 5 bans exactly that: "label it as a bound, never as
+# caches forever".
+assert_never_claims_an_unbounded_cache() {
+ local label="$1"
+ if grep -qiE "no expiry at all|caches forever" <<<"$report"; then
+ fail "${label} claimed an unbounded cache from a bounded observation: $(grep -E '^ why' <<<"$report" | head -1)"
+ return
+ fi
+ pass "${label} did not overstate what its observation bounds"
+}
+
+# A forged finding must never be reported in the revoked stream's words, or the
+# verdict describes an observation khaos never made.
+assert_does_not_claim_a_revocation() {
+ local label="$1"
+ if grep -q "invalidated at the authority" <<<"$report"; then
+ fail "${label} claims a credential was invalidated at the authority, which this experiment never does"
+ return
+ fi
+ pass "${label} does not claim an observation it never made"
+}
+
+# The census. A new experiment in the family that nobody added here would be
+# shipped with no end-to-end coverage at all, and the structural tests cannot
+# catch that because they do not know what this script runs.
+assert_every_family_experiment_is_covered() {
+ local covered=" ${INVERTED} "
+ local row rest
+
+ for row in "${CATCHING[@]}"; do
+ rest="${row#*|}"
+ covered+="${rest//|/ } "
+ done
+
+ local catalogued missing=0
+ catalogued="$("$BIN" catalog --no-color | grep -oE '^ identity\.[a-z_]+' | tr -d ' ')"
+
+ local name
+ for name in $catalogued; do
+ # idp_outage is covered by scripts/identity.sh, which measures the proxy
+ # tier and the idleness discriminator.
+ [[ "$name" == "identity.idp_outage" ]] && continue
+
+ if [[ "$covered" != *" ${name} "* ]]; then
+ fail "${name} is in the catalogue and this gate never runs it"
+ missing=$((missing + 1))
+ fi
+ done
+
+ if [[ "$missing" -eq 0 ]]; then
+ pass "every catalogued identity experiment is run by this gate"
+ fi
+}
+
+# The other half of the census. The verifier gateway understands ten modes; a
+# mode with no row here is a bug the range can carry that nothing ever points an
+# experiment at, and the experiment census cannot see that.
+assert_every_broken_mode_is_exercised() {
+ local covered=" closed "
+ local row
+
+ for row in "${CATCHING[@]}"; do
+ covered+="${row%%|*} "
+ done
+
+ # `set +e` around it on purpose: this reads the mode list out of a REFUSAL,
+ # so the command is supposed to exit non-zero, and under `set -euo pipefail`
+ # that aborted the whole script before any experiment ran.
+ local advertised missing=0
+
+ set +e
+ # The closing paren goes too. khaos's refusal ends `..., decision-cache-forever)`,
+ # so stripping only commas left the LAST mode carrying a `)` and the census
+ # reported that mode as uncovered while every other one parsed clean.
+ advertised="$("$BIN" range up --fail-mode not-a-mode --no-color 2>&1 |
+ grep -oE 'known modes: .*' | sed 's/known modes: //' | tr ',)' ' ')"
+ set -e
+
+ if [[ -z "$advertised" ]]; then
+ fail "khaos did not list its fail modes, so this census cannot run"
+ return
+ fi
+
+ local mode
+ for mode in $advertised; do
+ # The three the INTROSPECTING gateways understand are covered by
+ # scripts/identity.sh, which measures the proxy tier.
+ case "$mode" in
+ closed | open | stale-rearm) continue ;;
+ esac
+
+ if [[ "$covered" != *" ${mode} "* ]]; then
+ fail "the range offers --fail-mode ${mode} and this gate never points an experiment at it"
+ missing=$((missing + 1))
+ fi
+ done
+
+ if [[ "$missing" -eq 0 ]]; then
+ pass "every verifier fail mode is exercised by this gate"
+ fi
+}
+
+# An unrecognised fail mode leaves EVERY gateway behaving correctly, so a typo
+# would bring up a range carrying no bug at all and all seventeen experiments
+# would report a clean survival. The refusal has to name the modes.
+assert_an_unknown_fail_mode_is_refused() {
+ set +e
+ local out
+ out="$("$BIN" range up --fail-mode not-a-mode --no-color 2>&1)"
+ local code=$?
+ set -e
+
+ if [[ "$code" -eq 0 ]]; then
+ fail "the range came up with an unrecognised fail mode, so a typo yields a range with no bug"
+ return
+ fi
+
+ local mode
+ for mode in alg-any skip-claims weak-sig jwks-fail-open decision-cache-forever; do
+ if ! grep -q "$mode" <<<"$out"; then
+ fail "the refusal does not name the ${mode} mode, so nobody can discover it"
+ return
+ fi
+ done
+
+ pass "an unknown fail mode is refused by name and the known ones are listed"
+}
+
+assert_every_family_experiment_is_covered
+assert_every_broken_mode_is_exercised
+assert_an_unknown_fail_mode_is_refused
+
+# Half one: against the bug it names, every experiment must find a defect.
+for row in "${CATCHING[@]}"; do
+ IFS='|' read -r -a parts <<<"$row"
+ mode="${parts[0]}"
+
+ if ! start_range "$mode"; then
+ continue
+ fi
+
+ for experiment in "${parts[@]:1}"; do
+ label="${experiment} vs ${mode}"
+
+ if [[ "$experiment" == "$INVERTED" ]]; then
+ run_experiment "$experiment"
+ assert_verdict "$label" "deviated / DENIED"
+ assert_exit "$label" "$last_code" 2
+ assert_confirmed_by_an_observable "$label" "$experiment"
+ continue
+ fi
+
+ run_experiment "$experiment"
+ assert_verdict "$label" "deviated / PERMITTED"
+ assert_exit "$label" "$last_code" 2
+ assert_confirmed_by_an_observable "$label" "$experiment"
+ assert_never_claims_an_unbounded_cache "$label"
+
+ case "$experiment" in
+ identity.kid_rollover | identity.revoked_token)
+ assert_states_a_finding "$label" "invalidated at the authority"
+ assert_reports_the_exposure_bound "$label"
+ ;;
+ identity.introspection_error | identity.malformed_response | identity.idp_latency)
+ assert_states_a_finding "$label" "had already rejected"
+ assert_does_not_claim_a_revocation "$label"
+ ;;
+ *)
+ assert_states_a_finding "$label" "khaos minted with"
+ assert_does_not_claim_a_revocation "$label"
+ ;;
+ esac
+ done
+done
+
+# Half two, and the half that makes this a gate rather than a demo. The same
+# seventeen experiments against a CORRECT verifier must clear it. An experiment
+# that reports a finding here is a false positive, which is worse than a missing
+# one: it sends somebody to fix a bug that is not there.
+if start_range closed; then
+ for row in "${CATCHING[@]}"; do
+ IFS='|' read -r -a parts <<<"$row"
+
+ for experiment in "${parts[@]:1}"; do
+ label="${experiment} vs a correct verifier"
+
+ run_experiment "$experiment"
+ assert_exit "$label" "$last_code" 0
+ assert_verdict "$label" "${CORRECT_VERDICT[$experiment]:-passed / HELD}"
+
+ if grep -qE "^ verdict .*PERMITTED" <<<"$report"; then
+ fail "${label} found a bypass in a correct verifier: $(verdict_line)"
+ else
+ pass "${label} found no bypass"
+ fi
+ done
+ done
+fi
+
+if [[ $failures -ne 0 ]]; then
+ echo "${failures} identity-family assertion(s) failed"
+ exit 1
+fi
+
+echo "identity family: all assertions passed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh
new file mode 100755
index 00000000..bbb22bc2
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh
@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# identity.sh
+
+set -euo pipefail
+
+BIN="${BIN:-$PWD/khaos}"
+
+KHAOS_STATE_DIR="$(mktemp -d)"
+export KHAOS_STATE_DIR
+
+# The pause has to outlast the range's staleness ceiling, which is a multiple of
+# the cache lifetime. At a 2s lifetime the ceiling is 8s, so 14s is comfortably
+# past it without making the gate crawl.
+#
+# The fault window then has to be AT LEAST the pause, and this gate shipped with
+# 8s against a 14s pause — the exact configuration under which the TTL-extension
+# diagnosis is unsound, because a fixed lifetime between those two durations
+# produces the same observation. khaos now refuses the causal claim there, so a
+# gate that still asked for it would be asserting what the tool declines to say.
+CACHE_TTL=2s
+TIMING=(--baseline-for 4s --fault-for 14s --recover-for 3s --sample-every 200ms --idle-for 14s)
+
+EXPERIMENT=identity.idp_outage
+
+failures=0
+report=""
+last_code=0
+
+cleanup() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ rm -rf "$KHAOS_STATE_DIR"
+}
+trap cleanup EXIT INT TERM HUP
+
+pass() { echo "ok $1"; }
+fail() { echo "FAIL $1"; failures=$((failures + 1)); }
+
+start_range() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ if ! "$BIN" range up --fail-mode "$1" --cache-ttl "$CACHE_TTL" --quiet --no-color >/dev/null 2>&1; then
+ fail "range up --fail-mode $1"
+ return 1
+ fi
+ return 0
+}
+
+# Sets the globals `report` and `last_code`. Never call it through a command
+# substitution: that runs it in a subshell and the report is lost while the exit
+# code still comes back, which looks like a passing gate with no evidence.
+run_experiment() {
+ set +e
+ report="$("$BIN" run "$EXPERIMENT" --no-color "${TIMING[@]}" "$@" 2>&1)"
+ last_code=$?
+ set -e
+}
+
+assert_verdict() {
+ local label="$1" want="$2"
+ if grep -qE "^ verdict .*${want}" <<<"$report"; then
+ pass "${label} verdict is ${want}"
+ else
+ fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"
+ fi
+}
+
+assert_exit() {
+ local label="$1" got="$2" want="$3"
+ if [[ "$got" == "$want" ]]; then
+ pass "${label} exit code ${want}"
+ else
+ fail "${label} exit code ${got}, want ${want}"
+ fi
+}
+
+# THE milestone assertion. Two targets that are indistinguishable under steady
+# traffic must come back with DIFFERENT causes, because they need different fixes.
+assert_cause() {
+ local label="$1" want="$2"
+ if grep -qE "^ cause .*${want}" <<<"$report"; then
+ pass "${label} cause is ${want}"
+ else
+ fail "${label} expected cause ${want}, got: $(grep -E '^ cause' <<<"$report" || echo '<none>')"
+ fi
+}
+
+assert_no_cause() {
+ local label="$1"
+ if grep -qE "^ cause " <<<"$report"; then
+ fail "${label} reported a cause on a run with no leak to diagnose: $(grep -E '^ cause' <<<"$report")"
+ return
+ fi
+ pass "${label} reported no cause, because there was nothing to diagnose"
+}
+
+# The two extra phases are what make this experiment different from severing a
+# dependency, so the gate has to see both of them happen.
+assert_phase() {
+ local label="$1" stage="$2" because="$3"
+ if grep -qE "^ plan ${stage}" <<<"$report"; then
+ pass "${label} ran the ${stage} phase"
+ else
+ fail "${label} never ran the ${stage} phase, ${because}"
+ fi
+}
+
+# Two halves. The verdict has to say a REVOKED credential was honoured, and it
+# has to say HOW khaos invalidated it in THIS experiment's own words.
+#
+# The second half exists because one shared sentence previously claimed every
+# revocation happened "while that authority was unreachable", which is true only
+# of this experiment: a key rollover and a plain revocation both leave the
+# provider fully reachable, so two of the three experiments with a revoked stream
+# were asserting something false about their own run.
+assert_states_the_revocation() {
+ local label="$1"
+
+ if ! grep -q "invalidated at the authority" <<<"$report"; then
+ fail "${label} did not state that a REVOKED credential was honoured: $(grep -E '^ why' <<<"$report" | head -1)"
+ return
+ fi
+
+ if ! grep -q "an authority the target could no longer reach" <<<"$report"; then
+ fail "${label} did not say HOW khaos invalidated the credential, so the verdict would read "\
+"identically for every experiment with a revoked stream: $(grep -E '^ why' <<<"$report" | head -1)"
+ return
+ fi
+
+ pass "${label} states what khaos actually observed, and how"
+}
+
+# The gate's negative control: khaos must not clear a target whose credential it
+# never managed to revoke. Pointing the authority at a closed port makes the
+# mid-run action fail for real.
+assert_a_failed_revocation_voids_the_run() {
+ run_experiment --authority http://127.0.0.1:1 \
+ --target khaos-range-idp \
+ --url "$(grep -oE 'http://127\.0\.0\.1:[0-9]+/protected' <<<"$report" | head -1)" \
+ --token placeholder --revocable-token placeholder
+
+ if [[ "$last_code" -eq 0 ]]; then
+ fail "a run whose revocation failed exited 0, so khaos cleared a target it never tested"
+ return
+ fi
+
+ if grep -qE "^ verdict .*deviated" <<<"$report"; then
+ fail "a run whose revocation failed still reported a deviation: $(grep -E '^ verdict' <<<"$report")"
+ return
+ fi
+
+ pass "a failed revocation voids the run instead of clearing the target"
+}
+
+# A cached allow being re-armed by every failed check. Steady traffic keeps it
+# alive; the pause lets it lapse.
+if start_range stale-rearm; then
+ run_experiment
+ assert_verdict "re-armed cache" "deviated / PERMITTED"
+ assert_cause "re-armed cache" "TTL-EXTENSION"
+ assert_exit "re-armed cache" "$last_code" 2
+ assert_phase "re-armed cache" "act" "so nothing it measured had been revoked"
+ assert_phase "re-armed cache" "idle" "so the two causes were never separated"
+ assert_phase "re-armed cache" "resume" "so nothing was measured after the pause"
+ assert_states_the_revocation "re-armed cache"
+ assert_a_failed_revocation_voids_the_run
+fi
+
+# The other bug. Identical to the above under steady traffic, and the pause is
+# the only thing that tells them apart.
+if start_range open; then
+ run_experiment
+ assert_verdict "stale-past-pause cache" "deviated / PERMITTED"
+ assert_cause "stale-past-pause cache" "STALE-PAST-PAUSE"
+ assert_exit "stale-past-pause cache" "$last_code" 2
+ assert_states_the_revocation "stale-past-pause cache"
+fi
+
+# The reference implementation. Refusing a credential it can no longer verify is
+# correct, it is not the declared defect, and there is no cause to diagnose.
+if start_range closed; then
+ run_experiment
+ assert_verdict "fail-closed" "deviated / DENIED"
+ assert_exit "fail-closed" "$last_code" 0
+ assert_no_cause "fail-closed"
+
+ if grep -q "acceptable for this control" <<<"$report"; then
+ pass "fail-closed polarity spared the safe direction"
+ else
+ fail "fail-closed polarity did not spare the safe direction"
+ fi
+fi
+
+if [[ $failures -ne 0 ]]; then
+ echo "${failures} identity assertion(s) failed"
+ exit 1
+fi
+
+echo "identity: all assertions passed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh
new file mode 100755
index 00000000..fec7b6c6
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh
@@ -0,0 +1,337 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# proxy.sh
+
+set -euo pipefail
+
+BIN="${BIN:-$PWD/khaos}"
+
+KHAOS_STATE_DIR="$(mktemp -d)"
+export KHAOS_STATE_DIR
+
+# The fault window has to clear MinimumToClaimClosed (30 paired samples) or a
+# HELD verdict carries `Conclusive=false` and now exits 4: khaos stopped treating
+# "no bypass observed, but the sample cannot rule one out" as a success.
+TIMING=(--baseline-for 2s --fault-for 8s --recover-for 2s --sample-every 200ms)
+
+# slow-close is the one fault this range cannot be made to feel, and the reason
+# is a property of the protocol rather than a gap in the fixture.
+#
+# It exists to find a caller that waits for the connection to END instead of
+# reading a Content-Length, and every service in the range is a Go HTTP/1.1
+# server talking to a Go HTTP client. Go's server always frames a response with
+# either Content-Length or chunked encoding, so its client returns the moment the
+# body is complete and never waits for the close. Measured through the proxy: the
+# body arrives intact in 296 microseconds against a three second hold.
+#
+# This gate used to assert a bypass here and it PASSED, for the wrong reason: the
+# fault applied its delay to the relay as well as to the close, so it was finding
+# what `network.dependency_latency` finds and reporting it as slow closure. The
+# fix removed the payload delay, and this is what was underneath.
+#
+# The fault is still confirmed at the socket, which is where it lives, and that
+# assertion is kept below. What cannot be asserted end to end is a VERDICT, so
+# this gate asserts the truth instead of the thing it would prefer to see.
+EOF_DEPENDENT_FAULT=network.dependency_slow_close
+
+# These mirror portFloor and portCeil in internal/rangelab, and the container name
+# in internal/rangelab. A shell gate cannot import them; naming them here at least
+# keeps the duplication in one place per file.
+PORT_FLOOR=30000
+PORT_CEIL=60000
+PROXIED_GATEWAY=khaos-range-gateway-proxied
+
+PROXY_EXPERIMENTS=(
+ network.dependency_hang
+ network.dependency_reset
+ network.dependency_eof
+ network.dependency_latency
+ network.dependency_partial
+ network.dependency_slow_close
+)
+
+failures=0
+report=""
+last_code=0
+
+cleanup() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ rm -rf "$KHAOS_STATE_DIR"
+}
+trap cleanup EXIT INT TERM HUP
+
+pass() { echo "ok $1"; }
+fail() { echo "FAIL $1"; failures=$((failures + 1)); }
+
+start_range() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ if ! "$BIN" range up --fail-mode "$1" --quiet --no-color >/dev/null 2>&1; then
+ fail "range up --fail-mode $1"
+ return 1
+ fi
+ return 0
+}
+
+# Sets the globals `report` and `last_code`. Never call it through a command
+# substitution: that runs it in a subshell and the report is lost while the exit
+# code still comes back, which looks like a passing gate with no evidence.
+run_experiment() {
+ set +e
+ report="$("$BIN" run "$1" --no-color "${TIMING[@]}" 2>&1)"
+ last_code=$?
+ set -e
+}
+
+assert_verdict() {
+ local label="$1" want="$2"
+ if grep -qE "^ verdict .*${want}" <<<"$report"; then
+ pass "${label} verdict is ${want}"
+ else
+ fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"
+ fi
+}
+
+assert_exit() {
+ local label="$1" got="$2" want="$3"
+ if [[ "$got" == "$want" ]]; then
+ pass "${label} exit code ${want}"
+ else
+ fail "${label} exit code ${got}, want ${want}"
+ fi
+}
+
+# The run record must name the observable khaos MEASURED, and a different one per
+# fault. The first version of this grepped a single shared substring, which turned
+# out to be a constant describing the technique rather than anything measured:
+# replacing the socket read-back with a counter comparison, and deleting it
+# outright so confirmation always said yes, both left this gate 35/35 green.
+# A shared substring regresses straight back to decorative, so each fault gets its
+# own expected string and nothing else can satisfy it.
+expected_observable() {
+ case "$1" in
+ network.dependency_hang) echo "never through its proxy" ;;
+ network.dependency_reset) echo "resets through its proxy" ;;
+ network.dependency_eof) echo "closes without answering through its proxy" ;;
+ network.dependency_latency) echo "later through khaos's proxy than directly" ;;
+ network.dependency_partial) echo "arrive through khaos's proxy where the dependency answers" ;;
+ network.dependency_slow_close) echo "after the last byte, which it does not do directly" ;;
+ *) echo "" ;;
+ esac
+}
+
+assert_confirmed_on_the_socket() {
+ local label="$1" want
+ want="$(expected_observable "$1")"
+
+ if [[ -z "$want" ]]; then
+ fail "${label} has no expected observable, so this gate cannot check it"
+ return
+ fi
+
+ if grep -qF "$want" <<<"$report"; then
+ pass "${label} confirmed by its own measured observable"
+ else
+ fail "${label} did not report ${want}: $(grep -E '^ plan confirm' <<<"$report" || echo '<none>')"
+ fi
+}
+
+# Every fault's observable must be DISTINCT, or one constant satisfies them all.
+assert_observables_are_distinct() {
+ local -a seen=()
+ local experiment want
+
+ for experiment in "${PROXY_EXPERIMENTS[@]}"; do
+ want="$(expected_observable "$experiment")"
+ seen+=("$want")
+ done
+
+ local unique
+ unique="$(printf '%s\n' "${seen[@]}" | sort -u | wc -l)"
+
+ if [[ "$unique" -ne "${#PROXY_EXPERIMENTS[@]}" ]]; then
+ fail "only ${unique} distinct observables for ${#PROXY_EXPERIMENTS[@]} faults; a shared string makes this gate decorative"
+ return
+ fi
+
+ pass "every fault has its own distinct observable (${unique})"
+}
+
+# A run that measured nothing must not exit 0. A fault too large to confirm used
+# to abort before measuring, report that honestly in the verdict text, and still
+# exit success, so pointing khaos at a fail-open target in CI was a green build.
+assert_an_unconfirmable_fault_is_not_success() {
+ set +e
+ report="$("$BIN" run network.dependency_partial --no-color "${TIMING[@]}" --bytes 999999 2>&1)"
+ last_code=$?
+ set -e
+
+ if [[ "$last_code" -eq 0 ]]; then
+ fail "a fault khaos could not confirm exited 0 against a target that fails open"
+ return
+ fi
+
+ if ! grep -qE "^ verdict .*(aborted|probe-error|baseline-not-met)" <<<"$report"; then
+ fail "expected an inconclusive verdict, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"
+ return
+ fi
+
+ assert_exit "an unconfirmable fault" "$last_code" 4
+}
+
+assert_ran_with_no_flags() {
+ local label="$1"
+ if grep -q "using the running range" <<<"$report"; then
+ pass "${label} ran with no target flags at all"
+ else
+ fail "${label} did not discover the range on its own"
+ fi
+}
+
+# The proxy must be reachable by the container and by nothing else. A wildcard
+# bind would put a fault proxy, in front of a stack built to fail open, on the
+# operator's network.
+assert_proxy_bound_where_the_container_looks() {
+ local bound expected
+ bound="$(grep -oE "fault proxy is listening on [0-9.]+:[0-9]+" <<<"$report" | awk '{print $NF}')"
+
+ if [[ -z "$bound" ]]; then
+ fail "the run never said where it bound its fault proxy"
+ return
+ fi
+
+ # Not through a bare command substitution under `set -e`: a docker hiccup
+ # would kill the whole script instead of reporting one failed assertion.
+ set +e
+ expected="$(docker network inspect bridge -f '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null)"
+ set -e
+
+ if [[ -z "$expected" ]]; then
+ fail "could not read the Docker bridge gateway, so this assertion cannot check anything"
+ return
+ fi
+
+ if [[ "${bound%:*}" != "$expected" ]]; then
+ fail "the proxy bound ${bound}, but containers reach this host at ${expected}"
+ return
+ fi
+
+ # The wildcard case is covered in Go by TestOpenRefusesAnAddressThatWouldExposeTheProxy
+ # and TestOpenRefusesThisHostsRoutableAddress. It is unreachable here: a
+ # wildcard is refused before the proxy ever prints an address, so `bound`
+ # would be empty and the check above fires instead.
+ local port="${bound##*:}"
+ if (( port < PORT_FLOOR || port >= PORT_CEIL )); then
+ fail "the proxy bound host port ${port}, outside the random high range"
+ return
+ fi
+
+ pass "the proxy bound the bridge gateway on a random high port (${bound})"
+}
+
+# There is deliberately NO assertion here that the listener is gone after the run.
+# It was tried and it was a kernel test wearing a khaos label: the listener belongs
+# to a process that always exits, so the OS closes it no matter what khaos does,
+# and deleting khaos's own teardown left that assertion green. It also passed
+# silently on a host without `ss`. The property it claimed to check is real but it
+# is only falsifiable in-process, and
+# `TestClosingTheProxyRemovesTheFaultEntirely` in internal/inject/proxy does check
+# it: it asserts the listener refuses connections and that arming a closed proxy
+# returns ErrClosed.
+
+# The gate's negative control: a proxy-tier experiment against a range that has no
+# proxied gateway must be refused by name, not silently run against a path khaos
+# does not sit in front of.
+assert_refuses_without_a_proxied_gateway() {
+ local out code
+ docker rm -f "$PROXIED_GATEWAY" >/dev/null 2>&1 || true
+
+ set +e
+ out="$("$BIN" run network.dependency_reset --no-color "${TIMING[@]}" 2>&1)"
+ code=$?
+ set -e
+
+ if [[ $code -eq 0 ]]; then
+ fail "a proxy experiment ran with no proxied gateway in the range"
+ return
+ fi
+
+ if ! grep -qE "${PROXIED_GATEWAY}|no running khaos range" <<<"$out"; then
+ fail "the refusal did not name the missing part: $(tail -2 <<<"$out")"
+ return
+ fi
+
+ pass "a proxy experiment is refused when the range has no proxied gateway"
+}
+
+# A Docker-native experiment must NOT be routed through the proxy. If it were, it
+# would stop being a measurement of the Docker-native tier.
+assert_direct_tier_is_not_proxied() {
+ run_experiment network.dependency_outage
+
+ if grep -q "fault proxy is listening" <<<"$report"; then
+ fail "a container-pause experiment stood up a fault proxy"
+ return
+ fi
+
+ # assert_verdict reports for itself. An unconditional pass here printed `ok`
+ # even on the run where the verdict check had just failed.
+ assert_verdict "direct tier" "deviated / PERMITTED"
+ pass "the Docker-native tier ran without standing up a proxy"
+}
+
+if start_range open; then
+ for experiment in "${PROXY_EXPERIMENTS[@]}"; do
+ run_experiment "$experiment"
+ assert_ran_with_no_flags "$experiment"
+ assert_confirmed_on_the_socket "$experiment"
+
+ if [[ "$experiment" == "$EOF_DEPENDENT_FAULT" ]]; then
+ # Confirmed at the socket, and correctly finding nothing: this
+ # target reads a Content-Length and is not the kind of caller this
+ # fault breaks. Asserting a bypass here is what hid the payload
+ # delay for an entire milestone.
+ assert_verdict "$experiment" "passed / HELD"
+ assert_exit "$experiment" "$last_code" 0
+ continue
+ fi
+
+ assert_verdict "$experiment" "deviated / PERMITTED"
+ assert_exit "$experiment" "$last_code" 2
+ done
+
+ assert_observables_are_distinct
+ assert_proxy_bound_where_the_container_looks
+ assert_direct_tier_is_not_proxied
+ assert_an_unconfirmable_fault_is_not_success
+ assert_refuses_without_a_proxied_gateway
+fi
+
+# The mirror. A control that fails CLOSED refuses the credential it had been
+# honouring, and that is not a defect for this polarity, so it must not exit 2.
+if start_range closed; then
+ for experiment in "${PROXY_EXPERIMENTS[@]}"; do
+ run_experiment "$experiment"
+
+ if [[ "$experiment" == "$EOF_DEPENDENT_FAULT" ]]; then
+ assert_verdict "${experiment} fail-closed" "passed / HELD"
+ assert_exit "${experiment} fail-closed" "$last_code" 0
+ continue
+ fi
+
+ assert_verdict "${experiment} fail-closed" "deviated / DENIED"
+ assert_exit "${experiment} fail-closed" "$last_code" 0
+
+ if grep -q "acceptable for this control" <<<"$report"; then
+ pass "${experiment} fail-closed polarity spared the safe direction"
+ else
+ fail "${experiment} fail-closed polarity did not spare the safe direction"
+ fi
+ done
+fi
+
+if [[ $failures -ne 0 ]]; then
+ echo "${failures} proxy assertion(s) failed"
+ exit 1
+fi
+
+echo "proxy: all assertions passed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh
new file mode 100755
index 00000000..72c77f74
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh
@@ -0,0 +1,242 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# range.sh
+
+set -euo pipefail
+
+BIN="${BIN:-$PWD/khaos}"
+
+KHAOS_STATE_DIR="$(mktemp -d)"
+export KHAOS_STATE_DIR
+
+TIMING=(--baseline-for 3s --fault-for 6s --recover-for 3s --sample-every 200ms)
+
+failures=0
+report=""
+last_code=0
+
+cleanup() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ rm -rf "$KHAOS_STATE_DIR"
+}
+trap cleanup EXIT INT TERM HUP
+
+pass() { echo "ok $1"; }
+fail() { echo "FAIL $1"; failures=$((failures + 1)); }
+
+start_range() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ if ! "$BIN" range up --fail-mode "$1" --cache-ttl "${2:-2s}" --quiet --no-color >/dev/null 2>&1; then
+ fail "range up --fail-mode $1 --cache-ttl ${2:-2s}"
+ return 1
+ fi
+ return 0
+}
+
+# Sets the globals `report` and `last_code`. It must not be called through a
+# command substitution: that runs it in a subshell and the report is lost while
+# the exit code still comes back, which looks like a passing gate with no
+# evidence behind it.
+run_experiment() {
+ set +e
+ report="$("$BIN" run network.dependency_outage --no-color "${TIMING[@]}" 2>&1)"
+ last_code=$?
+ set -e
+}
+
+assert_verdict() {
+ local label="$1" want="$2"
+ if grep -qE "^ verdict .*${want}" <<<"$report"; then
+ pass "${label} verdict is ${want}"
+ else
+ fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"
+ fi
+}
+
+assert_exit() {
+ local label="$1" got="$2" want="$3"
+ if [[ "$got" == "$want" ]]; then
+ pass "${label} exit code ${want}"
+ else
+ fail "${label} exit code ${got}, want ${want}"
+ fi
+}
+
+# The gate's negative control: inputs it MUST reject, and the one safety
+# property of a deliberately-vulnerable stack that nothing else checks.
+assert_loopback_only() {
+ local binding port
+ binding="$(docker inspect -f '{{range $p, $conf := .NetworkSettings.Ports}}{{range $conf}}{{.HostIp}} {{.HostPort}}{{end}}{{end}}' khaos-range-gateway 2>/dev/null)"
+ port="${binding##* }"
+
+ if [[ "$binding" != "127.0.0.1 "* ]]; then
+ fail "the range gateway is published on ${binding:-nothing}, not loopback; a stack built to fail open must never be reachable off the host"
+ return
+ fi
+
+ if (( port < 30000 || port >= 60000 )); then
+ fail "the range gateway is on host port ${port}, outside the random high range"
+ return
+ fi
+
+ pass "the range is loopback-only on a random high port (${port})"
+}
+
+assert_refuses_foreign_compose() {
+ local foreign out
+ foreign="$(mktemp -d)"
+ printf 'name: not-khaos\nservices: {}\n' > "${foreign}/compose.yml"
+
+ set +e
+ out="$(KHAOS_RANGE_DIR="$foreign" "$BIN" range down --no-color 2>&1)"
+ local code=$?
+ set -e
+ rm -rf "$foreign"
+
+ if [[ $code -eq 0 ]]; then
+ fail "khaos ran compose down against a project it does not own"
+ return
+ fi
+
+ if ! grep -q "does not belong to khaos" <<<"$out"; then
+ fail "the refusal did not name the cause: ${out}"
+ return
+ fi
+
+ pass "khaos refuses a compose file it does not own"
+}
+
+assert_refuses_half_specified() {
+ local label="$1" out code
+ shift
+
+ set +e
+ out="$("$BIN" plan network.dependency_outage --no-color "$@" 2>&1)"
+ code=$?
+ set -e
+
+ if [[ $code -eq 0 ]] || ! grep -q "will not fill one half" <<<"$out"; then
+ fail "a half-specified invocation (${label}) was silently completed from the range: ${out}"
+ return
+ fi
+
+ pass "a half-specified invocation is refused rather than fused (${label})"
+}
+
+# A parameter no experiment declares must not be settable on it. The alternative
+# is registering every experiment's parameters on one command, which at twelve
+# experiments shows a beginner twenty flags that do nothing for the one they ran.
+assert_rejects_undeclared_flag() {
+ local out code
+ set +e
+ out="$("$BIN" run network.dependency_outage --ca-cert /dev/null --no-color 2>&1)"
+ code=$?
+ set -e
+
+ if [[ $code -eq 0 ]] || ! grep -q "unknown flag" <<<"$out"; then
+ fail "a flag this experiment never declared was accepted: ${out}"
+ return
+ fi
+
+ pass "a flag the experiment never declared is rejected"
+}
+
+# Supplying everything the range could have answered means the operator is
+# pointing khaos at their own stack, so khaos must not require a range at all.
+assert_operator_stack_needs_no_range() {
+ local out code
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+
+ set +e
+ out="$("$BIN" plan network.dependency_outage --no-color \
+ --target khaos-no-such-container --url http://127.0.0.1:1/ --token t 2>&1)"
+ code=$?
+ set -e
+
+ if [[ $code -eq 0 ]]; then
+ fail "khaos planned against a container that does not exist"
+ return
+ fi
+
+ if grep -q "no running khaos range" <<<"$out"; then
+ fail "a fully specified invocation still went looking for a range: ${out}"
+ return
+ fi
+
+ pass "a fully specified invocation never looks for a range"
+}
+
+assert_zero_flags() {
+ if grep -q "using the running range" <<<"$report"; then
+ pass "$1 ran with no target flags at all"
+ else
+ fail "$1 did not discover the range on its own"
+ fi
+}
+
+if start_range open; then
+ run_experiment
+ assert_zero_flags "fail-open"
+ assert_verdict "fail-open" "deviated / PERMITTED"
+ assert_exit "fail-open" "$last_code" 2
+ assert_loopback_only
+ assert_refuses_foreign_compose
+ assert_refuses_half_specified "target only" --target something
+ assert_refuses_half_specified "target and url, no token" --target t --url http://127.0.0.1:1/
+ assert_rejects_undeclared_flag
+
+ if grep -q "fault confirmed by" <<<"$report"; then
+ pass "fail-open the fault was confirmed by read-back, not by an API return value"
+ else
+ fail "fail-open the fault was never confirmed"
+ fi
+fi
+
+if start_range closed; then
+ run_experiment
+ assert_zero_flags "fail-closed"
+ assert_verdict "fail-closed" "deviated / DENIED"
+ assert_exit "fail-closed" "$last_code" 0
+
+ if grep -q "acceptable for this control" <<<"$report"; then
+ pass "fail-closed the deviation was recorded without calling it a defect"
+ else
+ fail "fail-closed polarity did not spare the safe direction"
+ fi
+fi
+
+# The observer-effect cycle. Every other cycle runs the range at its default 2s
+# decision cache against a 6s fault window, so a cached refusal expires INSIDE the
+# fault and the hazard is never exercised. With the cache outliving the fault, a
+# negative control that reuses one credential is answered entirely from that cache
+# and a gateway explicitly configured to fail open reports `passed / HELD`,
+# conclusive, exit 0 — the worst outcome this tool can produce. Measured: that is
+# exactly what happens when the per-request nonce is removed.
+if start_range open 30s; then
+ run_experiment
+ assert_verdict "fail-open with a cache outliving the fault" "deviated / PERMITTED"
+ assert_exit "fail-open with a cache outliving the fault" "$last_code" 2
+fi
+
+assert_operator_stack_needs_no_range
+
+if "$BIN" range down --no-color >/dev/null 2>&1; then
+ remaining="$(docker ps -aq --filter label=khaos.range | wc -l)"
+ nets="$(docker network ls -q --filter name=^khaos-range$ | wc -l)"
+ images="$(docker images -q khaos-range:latest | wc -l)"
+
+ if [[ "$remaining" -eq 0 && "$nets" -eq 0 && "$images" -eq 0 ]]; then
+ pass "range down removed everything it created"
+ else
+ fail "range down left ${remaining} container(s), ${nets} network(s), ${images} image(s) behind"
+ fi
+else
+ fail "range down errored"
+fi
+
+if [[ $failures -ne 0 ]]; then
+ echo "${failures} range assertion(s) failed"
+ exit 1
+fi
+
+echo "range: all assertions passed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/smoke.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/smoke.sh
new file mode 100755
index 00000000..a9ee0f82
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/smoke.sh
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# smoke.sh
+
+set -euo pipefail
+
+BIN="${BIN:-./khaos}"
+IMAGE="${SMOKE_IMAGE:-alpine:3.22}"
+
+RUN_ID="$$"
+OWNER_LABEL="khaos.smoke=${RUN_ID}"
+PREFIX="ks-${RUN_ID}"
+WORKDIR="$(mktemp -d)"
+
+failures=0
+
+cleanup() {
+ local owned
+ owned="$(docker ps -aq --filter "label=${OWNER_LABEL}" 2>/dev/null || true)"
+ if [[ -n "$owned" ]]; then
+ docker rm -f $owned >/dev/null 2>&1 || true
+ fi
+ rm -rf "$WORKDIR"
+}
+trap cleanup EXIT INT TERM HUP
+
+spawn() {
+ local name="$1"
+ shift
+ docker run -d --name "${PREFIX}-${name}" --label "$OWNER_LABEL" "$@" "$IMAGE" sleep 600 >/dev/null
+}
+
+assert_status() {
+ local name="$1" want="$2" line
+ line="$(grep -E "^[[:space:]]+${PREFIX}-${name}[[:space:]]" <<<"$OUTPUT" || true)"
+
+ if [[ -z "$line" ]]; then
+ echo "FAIL ${name}: absent from the listing, expected status ${want}"
+ failures=$((failures + 1))
+ return
+ fi
+
+ if [[ "$line" != *"$want"* ]]; then
+ echo "FAIL ${name}: expected ${want}, got: ${line}"
+ failures=$((failures + 1))
+ return
+ fi
+
+ echo "ok ${name} -> ${want}"
+}
+
+assert_absent() {
+ local name="$1"
+ if grep -qE "^[[:space:]]+${PREFIX}-${name}[[:space:]]" <<<"$OUTPUT"; then
+ echo "FAIL ${name}: a container without the consent label was enumerated"
+ failures=$((failures + 1))
+ return
+ fi
+ echo "ok ${name} -> never enumerated"
+}
+
+if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
+ echo "pulling ${IMAGE}"
+ docker pull "$IMAGE" >/dev/null
+fi
+
+# A decoy socket. The predicate reasons about the mount path, never the file,
+# so the live daemon socket is never handed to a test fixture.
+mkdir -p "${WORKDIR}/run"
+touch "${WORKDIR}/run/docker.sock"
+
+spawn ok --label khaos.target=true
+spawn badval --label khaos.target=yes
+spawn sock --label khaos.target=true -v "${WORKDIR}/run/docker.sock:/var/run/docker.sock:ro"
+spawn sockdir --label khaos.target=true -v "/var/run:/hostrun:ro"
+spawn hostnet --label khaos.target=true --network host
+spawn privileged --label khaos.target=true --privileged
+spawn netadmin --label khaos.target=true --cap-add NET_ADMIN
+spawn hostpid --label khaos.target=true --pid host
+spawn nolabel
+
+docker create --name "${PREFIX}-stopped" --label "$OWNER_LABEL" \
+ --label khaos.target=true "$IMAGE" sleep 600 >/dev/null
+
+peer="${PREFIX}-ok"
+docker run -d --name "${PREFIX}-joiner" --label "$OWNER_LABEL" \
+ --network "container:${peer}" "$IMAGE" sleep 600 >/dev/null
+
+OUTPUT="$("$BIN" targets --all --no-color)"
+
+assert_status ok "namespace-joined-by-peer"
+assert_status badval "consent-label-not-true"
+assert_status sock "docker-socket-reachable"
+assert_status sockdir "docker-socket-reachable"
+assert_status hostnet "host-network-mode"
+assert_status privileged "privileged-container"
+assert_status netadmin "dangerous-capability"
+assert_status hostpid "host-namespace"
+assert_status stopped "container-not-running"
+assert_absent nolabel
+assert_absent joiner
+
+docker rm -f "${PREFIX}-joiner" >/dev/null
+OUTPUT="$("$BIN" targets --all --no-color)"
+assert_status ok "ready"
+
+READY_ONLY="$("$BIN" targets --no-color)"
+if grep -qE "^[[:space:]]+${PREFIX}-(badval|sock|sockdir|hostnet|privileged|netadmin|hostpid|stopped)[[:space:]]" <<<"$READY_ONLY"; then
+ echo "FAIL default view listed a refused target"
+ failures=$((failures + 1))
+else
+ echo "ok default view hides refused targets"
+fi
+
+if [[ $failures -ne 0 ]]; then
+ echo "${failures} smoke assertion(s) failed"
+ exit 1
+fi
+
+echo "smoke: all assertions passed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/survive.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/survive.sh
new file mode 100755
index 00000000..5ae06fce
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/survive.sh
@@ -0,0 +1,240 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# survive.sh
+
+set -euo pipefail
+
+BIN="${BIN:-$PWD/khaos}"
+IMAGE="${SMOKE_IMAGE:-alpine:3.22}"
+
+RUN_ID="$$"
+OWNER_LABEL="khaos.survive=${RUN_ID}"
+TARGET="ks-surv-${RUN_ID}"
+DECOY="ks-decoy-${RUN_ID}"
+NET="ksnet-${RUN_ID}"
+ALIAS="api-under-test"
+
+KHAOS_STATE_DIR="$(mktemp -d)"
+export KHAOS_STATE_DIR
+JOURNAL="${KHAOS_STATE_DIR}/journal.jsonl"
+
+failures=0
+children=()
+
+cleanup() {
+ local pid owned
+ for pid in "${children[@]:-}"; do
+ [[ -n "$pid" ]] && kill -9 "$pid" 2>/dev/null || true
+ done
+ owned="$(docker ps -aq --filter "label=${OWNER_LABEL}" 2>/dev/null || true)"
+ if [[ -n "$owned" ]]; then
+ docker unpause $owned >/dev/null 2>&1 || true
+ docker rm -f $owned >/dev/null 2>&1 || true
+ fi
+ docker network rm "$NET" >/dev/null 2>&1 || true
+ rm -rf "$KHAOS_STATE_DIR"
+}
+trap cleanup EXIT INT TERM HUP
+
+pass() { echo "ok $1"; }
+fail() { echo "FAIL $1"; failures=$((failures + 1)); }
+
+paused() { [[ "$(docker inspect -f '{{.State.Paused}}' "$1" 2>/dev/null)" == "true" ]]; }
+
+networks_of() {
+ docker inspect -f '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}
+{{end}}' "$1" 2>/dev/null
+}
+
+attached() { networks_of "$1" | grep -qx "$NET"; }
+
+aliases_of() {
+ docker inspect -f "{{range \$n, \$cfg := .NetworkSettings.Networks}}{{if eq \$n \"$NET\"}}{{\$cfg.Aliases}}{{end}}{{end}}" "$1" 2>/dev/null
+}
+
+await() {
+ local predicate="$1" limit="${2:-15}" waited=0
+ while (( waited < limit * 10 )); do
+ eval "$predicate" && return 0
+ sleep 0.1
+ waited=$((waited + 1))
+ done
+ return 1
+}
+
+inject_bg() {
+ "$BIN" inject "$1" "$TARGET" --ttl "$2" --no-color >/dev/null 2>&1 &
+ local pid=$!
+ children+=("$pid")
+ echo "$pid"
+}
+
+assert_no_outstanding() {
+ local out
+ out="$("$BIN" reclaim --no-color 2>&1 || true)"
+ if grep -q "nothing outstanding" <<<"$out"; then
+ pass "$1"
+ else
+ fail "$1 (reclaim still found work: ${out//$'\n'/ })"
+ fi
+}
+
+assert_decoy_untouched() {
+ if paused "$DECOY" || ! attached "$DECOY"; then
+ fail "$1 khaos touched a container it was never pointed at"
+ return
+ fi
+ pass "$1"
+}
+
+docker image inspect "$IMAGE" >/dev/null 2>&1 || docker pull "$IMAGE" >/dev/null
+docker network create "$NET" >/dev/null
+
+docker run -d --name "$TARGET" --label "$OWNER_LABEL" --label khaos.target=true \
+ --network "$NET" --network-alias "$ALIAS" "$IMAGE" sleep 900 >/dev/null
+docker run -d --name "$DECOY" --label "$OWNER_LABEL" --label khaos.target=true \
+ --network "$NET" "$IMAGE" sleep 900 >/dev/null
+
+pid="$(inject_bg container-pause 60s)"
+if await "paused $TARGET"; then
+ kill -TERM "$pid" 2>/dev/null || true
+ wait "$pid" 2>/dev/null || true
+ if await "! paused $TARGET"; then
+ pass "L1 in-process revert on signal"
+ else
+ fail "L1 the fault survived a graceful stop"
+ fi
+else
+ fail "L1 the fault was never applied"
+fi
+assert_no_outstanding "L1 the journal records the revert"
+assert_decoy_untouched "L1 decoy"
+
+pid="$(inject_bg container-pause 60s)"
+if await "paused $TARGET"; then
+ refusal="$("$BIN" inject container-pause "$TARGET" --no-color 2>&1 || true)"
+ if grep -qi "already running" <<<"$refusal"; then
+ pass "lock second instance refused by the lock"
+ else
+ fail "lock a second instance was not stopped by the lock, it said: ${refusal}"
+ fi
+else
+ fail "lock the fault was never applied"
+fi
+kill -9 "$pid" 2>/dev/null || true
+wait "$pid" 2>/dev/null || true
+
+if paused "$TARGET"; then
+ pass "L2 the fault outlived kill -9, as designed"
+else
+ fail "L2 the fault vanished on its own, so nothing was really applied"
+fi
+
+if "$BIN" reclaim --no-color >/dev/null 2>&1 && await "! paused $TARGET"; then
+ pass "L2 journal replay reverted a fault khaos never got to revert"
+else
+ fail "L2 reclaim did not revert the orphaned fault"
+fi
+assert_no_outstanding "L2 the journal is clean after reclaim"
+
+pid="$(inject_bg container-pause 3s)"
+if await "paused $TARGET"; then
+ kill -9 "$pid" 2>/dev/null || true
+ wait "$pid" 2>/dev/null || true
+
+ "$BIN" reap --interval 1s --no-color >/dev/null 2>&1 &
+ reaper=$!
+ children+=("$reaper")
+
+ if await "! paused $TARGET" 20; then
+ pass "L3 the lease expired and the reaper reverted it unattended"
+ else
+ fail "L3 the fault outlived its lease"
+ fi
+
+ kill -TERM "$reaper" 2>/dev/null || true
+ wait "$reaper" 2>/dev/null || true
+else
+ fail "L3 the fault was never applied"
+fi
+
+pid="$(inject_bg container-pause 120s)"
+if await "paused $TARGET"; then
+ kill -9 "$pid" 2>/dev/null || true
+ wait "$pid" 2>/dev/null || true
+
+ "$BIN" reclaim --expired-only --no-color >/dev/null 2>&1 || true
+ sleep 1
+
+ if paused "$TARGET"; then
+ pass "lease an unexpired fault was left alone by --expired-only"
+ else
+ fail "lease --expired-only reverted a fault whose lease was still live"
+ fi
+
+ "$BIN" reclaim --no-color >/dev/null 2>&1 || true
+ await "! paused $TARGET" || fail "final reclaim did not restore the target"
+fi
+
+before_aliases="$(aliases_of "$TARGET")"
+if "$BIN" inject network-disconnect "$TARGET" --hold 2s --no-color >/dev/null 2>&1; then
+ if attached "$TARGET"; then
+ pass "disconnect the target was reattached"
+ else
+ fail "disconnect the target was left detached"
+ fi
+
+ after_aliases="$(aliases_of "$TARGET")"
+ if [[ "$after_aliases" == "$before_aliases" ]] && grep -q "$ALIAS" <<<"$after_aliases"; then
+ pass "disconnect the network alias survived the revert"
+ else
+ fail "disconnect aliases went from ${before_aliases} to ${after_aliases}; peers addressing it by name are now broken"
+ fi
+else
+ fail "disconnect the clean inject/revert cycle errored"
+fi
+assert_no_outstanding "disconnect the journal is clean"
+assert_decoy_untouched "disconnect decoy"
+
+pid="$(inject_bg network-disconnect 120s)"
+if await "! attached $TARGET"; then
+ kill -9 "$pid" 2>/dev/null || true
+ wait "$pid" 2>/dev/null || true
+
+ if "$BIN" reclaim --no-color >/dev/null 2>&1 && attached "$TARGET"; then
+ recovered="$(aliases_of "$TARGET")"
+ if grep -q "$ALIAS" <<<"$recovered"; then
+ pass "disconnect kill -9 then reclaim restored the alias too"
+ else
+ fail "disconnect reclaim reattached but lost the alias: ${recovered}"
+ fi
+ else
+ fail "disconnect reclaim did not reattach the target after kill -9"
+ fi
+else
+ fail "disconnect the fault was never applied"
+fi
+assert_no_outstanding "disconnect the journal is clean after kill -9"
+
+docker rm -f "$DECOY" >/dev/null 2>&1 || true
+pid="$(inject_bg container-pause 3s)"
+if await "paused $TARGET"; then
+ kill -9 "$pid" 2>/dev/null || true
+ wait "$pid" 2>/dev/null || true
+ docker rm -f "$TARGET" >/dev/null 2>&1 || true
+
+ if "$BIN" reclaim --no-color >/dev/null 2>&1; then
+ pass "gone a fault on a deleted container resolves instead of wedging"
+ else
+ fail "gone a deleted container left a permanently unrevertable record"
+ fi
+else
+ fail "gone the fault was never applied"
+fi
+
+if [[ $failures -ne 0 ]]; then
+ echo "${failures} survival assertion(s) failed"
+ exit 1
+fi
+
+echo "survive: all assertions passed"