Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(minikube): preserve recovery for profiles with unobservable identity

Alfredo Lopez66f441aceaaf
diff --git a/docs/deploy/minikube.md b/docs/deploy/minikube.md
index 90b1a352a..d68f1db62 100644
--- a/docs/deploy/minikube.md
+++ b/docs/deploy/minikube.md
@@ -229,6 +229,22 @@ labelled `minikube.k8s.io/name=<profile>` whose InternalIP is the address
`kubectl --context=<profile> cluster-info` succeeds: a stopped profile cannot be
asked who it is, so `stop` and `delete` still run for it.
+For `stop` and `delete`, the guard row above means that the identity check runs
+only when the API answers, and a refusal requires a readable contradicting node
+identity. `start`, `status`, `pf`, and `health` require a complete identity
+whenever their strict check runs. A failed or timed-out IP/node observation is
+unknown, not a foreign match, so the recovery action may continue.
+
+**Recovery identity decision table.**
+
+| API reachability | Minikube IP | Node identity | Result |
+|---|---|---|---|
+| unreachable | not called | not called | continue profile stop or teardown |
+| reachable | fails or times out | not called | warn that identity is unobservable and continue |
+| reachable | readable | fails or times out | warn that identity is unobservable and continue |
+| reachable | readable | matches this profile | continue |
+| reachable | readable | names another profile | `BRANCH_PROFILE_CONTEXT_IDENTITY`; no records are cleared and the profile command does not run |
+
**Kubeconfig read.** The context guards read the kubeconfig only; they never
contact a cluster.
@@ -270,7 +286,7 @@ An unreadable list is a refusal, never an empty list.
| `BRANCH_PROFILE_REMOTE_CONTEXT` | the profile's context points at a non-local API server |
| `BRANCH_PROFILE_MINIKUBE_PROFILES_UNREADABLE` | `minikube profile list` failed or printed no readable lists |
| `BRANCH_PROFILE_UNKNOWN_MINIKUBE_PROFILE` | a context named after the profile exists, but minikube lists no such profile |
-| `BRANCH_PROFILE_CONTEXT_IDENTITY` | the cluster behind the context does not identify the profile, or its identity could not be read |
+| `BRANCH_PROFILE_CONTEXT_IDENTITY` | strict actions refuse an unreadable identity or one that names another profile; recovery actions refuse only a readable identity that names another profile |
| `CONTEXT_IDENTITY_UNAVAILABLE`, `CONTEXT_IDENTITY_API_INVALID` | `scripts/minikube/context-identity.sh` is missing or lacks a predicate |
| `PORT_FORWARD_OWNER_UNAVAILABLE`, `PORT_FORWARD_OWNER_API_INVALID`, `PORT_FORWARD_OWNER_BINDING_INVALID` | `scripts/minikube/port-forward-owner.sh` is missing or incomplete, or a record would bind another host |
| `PORT_FORWARD_OWNERSHIP_ERROR` | a port-forward process does not match its record; it is never signalled |
diff --git a/scripts/minikube-profiles/branch-profile.sh b/scripts/minikube-profiles/branch-profile.sh
index 529a87ace..7bc272745 100755
--- a/scripts/minikube-profiles/branch-profile.sh
+++ b/scripts/minikube-profiles/branch-profile.sh
@@ -732,17 +732,42 @@ require_context_profile_known_to_minikube() {
# another local Minikube profile, or a LAN cluster on a private address. The
# cluster must have a node labelled minikube.k8s.io/name=<profile> at the IP
# `minikube -p <profile> ip` reports.
-require_profile_cluster_identity() {
+profile_cluster_identity() {
+ local allow_unobservable="${1:-false}" expected_ip nodes_json
load_context_identity
- local expected_ip nodes_json
- expected_ip="$(run_bounded minikube-ip "${MINIKUBE_STATUS_TIMEOUT_SECONDS}" minikube -p "${PROFILE}" ip)" ||
+ if ! expected_ip="$(run_bounded minikube-ip "${MINIKUBE_STATUS_TIMEOUT_SECONDS}" minikube -p "${PROFILE}" ip)"; then
+ if [[ "${allow_unobservable}" == true ]]; then
+ printf 'WARN: cluster identity is unobservable for profile %s: minikube did not report an IP; continuing recovery\n' \
+ "${PROFILE}" >&2
+ return 2
+ fi
die "BRANCH_PROFILE_CONTEXT_IDENTITY: minikube did not report an IP for profile ${PROFILE}"
- nodes_json="$(kubectl "--context=${PROFILE}" "--request-timeout=${KUBECTL_REQUEST_TIMEOUT}" get nodes -o json)" ||
+ fi
+ if ! nodes_json="$(kubectl "--context=${PROFILE}" "--request-timeout=${KUBECTL_REQUEST_TIMEOUT}" get nodes -o json)"; then
+ if [[ "${allow_unobservable}" == true ]]; then
+ printf 'WARN: cluster identity is unobservable for profile %s: unable to read its nodes; continuing recovery\n' \
+ "${PROFILE}" >&2
+ return 2
+ fi
die "BRANCH_PROFILE_CONTEXT_IDENTITY: unable to read the nodes of kube context ${PROFILE}"
+ fi
minikube_nodes_identify_profile "${nodes_json}" "${PROFILE}" "${expected_ip}" ||
die "BRANCH_PROFILE_CONTEXT_IDENTITY: kube context ${PROFILE} does not identify Minikube profile ${PROFILE} at ${expected_ip:-<no IP>}; refusing to use it"
}
+require_profile_cluster_identity() {
+ profile_cluster_identity false
+}
+
+# Stop and delete are recovery actions for a profile minikube already lists.
+# A readable node identity that names another profile still refuses, but an
+# IP or node observation that fails within its deadline is unknown, not foreign.
+refuse_readable_foreign_profile_cluster_identity() {
+ local status=0
+ profile_cluster_identity true || status=$?
+ (( status == 2 )) || return "${status}"
+}
+
probe_service() {
local namespace="$1" service="$2" output
if ! output="$(kubectl "--context=${PROFILE}" "--request-timeout=${KUBECTL_REQUEST_TIMEOUT}" \
@@ -1361,10 +1386,12 @@ cmd_stop() {
require_local_context_endpoint
require_context_profile_known_to_minikube
# minikube listing the profile does not prove the context reaches its
- # cluster. A cluster that answers must identify the profile before any
- # record is cleared or minikube -p runs; a stopped one cannot be asked.
+ # cluster. A cluster that answers with another readable identity refuses
+ # before any record is cleared or minikube -p runs. A stopped profile, or
+ # one whose identity cannot be observed within its deadline, can still be
+ # recovered because minikube already proved it owns that profile name.
if cluster_reachable; then
- require_profile_cluster_identity
+ refuse_readable_foreign_profile_cluster_identity
fi
# Clear the verified port-forward records first: a stopped cluster leaves
# records naming dead kubectl processes, and the next T2 preflight refuses
@@ -1415,9 +1442,11 @@ cmd_delete() {
fi
require_local_context_endpoint
require_context_profile_known_to_minikube
- # As in cmd_stop: a cluster that answers must identify the profile.
+ # As in cmd_stop: only a readable contradicting identity refuses. An
+ # unobservable identity is recovery evidence for an invalid profile, not a
+ # foreign-cluster match.
if cluster_reachable; then
- require_profile_cluster_identity
+ refuse_readable_foreign_profile_cluster_identity
fi
# The registry (pids/*.pid) outlives the cluster; clear the port-forward
# records so the deleted profile does not leave PORT_FORWARD_CONFLICT behind.
diff --git a/scripts/tests/test-branch-profile-lifecycle.sh b/scripts/tests/test-branch-profile-lifecycle.sh
index afb89b955..29ab97d75 100755
--- a/scripts/tests/test-branch-profile-lifecycle.sh
+++ b/scripts/tests/test-branch-profile-lifecycle.sh
@@ -244,6 +244,10 @@ case "${verb}" in
;;
nodes)
if [[ "${output}" == json ]]; then
+ [[ ! -e "${state}/fail-get-nodes" ]] || {
+ printf 'stub: get nodes failed\n' >&2
+ exit 1
+ }
# The node carries the minikube.k8s.io/name label of the context it
# was asked through, unless node-label names another cluster.
label="${context}"
@@ -317,7 +321,16 @@ case " $* " in
[[ ! -e "${state}/minikube-start-fails" ]] || exit 7
;;
*" status "*) exit "$(cat "${state}/minikube-status-rc")" ;;
- *" ip "*) cat "${state}/minikube-ip" ;;
+ *" ip "*)
+ [[ ! -e "${state}/minikube-ip-fails" ]] || {
+ printf 'stub: minikube ip failed\n' >&2
+ exit 6
+ }
+ if [[ -e "${state}/minikube-ip-hang-seconds" ]]; then
+ sleep "$(cat "${state}/minikube-ip-hang-seconds")"
+ fi
+ cat "${state}/minikube-ip"
+ ;;
*" stop "*) [[ ! -e "${state}/minikube-stop-fails" ]] || exit 9 ;;
*" delete "*) [[ ! -e "${state}/minikube-delete-fails" ]] || exit 11 ;;
esac
@@ -376,7 +389,8 @@ reset_state() {
rm -f "${state}/fail-get-deploy" "${state}/curl-fail" "${state}/start-switches-context" \
"${state}/use-context-fails" "${state}/minikube-start-fails" "${state}/get-contexts-fails" \
"${state}/pf-ignores-term" "${state}/minikube-stop-fails" "${state}/minikube-delete-fails" \
- "${state}/node-label" "${state}/minikube-profile-list-fails" "${state}/docker-info-fails"
+ "${state}/node-label" "${state}/minikube-profile-list-fails" "${state}/docker-info-fails" \
+ "${state}/fail-get-nodes" "${state}/minikube-ip-fails" "${state}/minikube-ip-hang-seconds"
# By default the branch profile's context is this local Minikube, minikube
# lists the profile, and its node carries the profile label at the address
# `minikube ip` reports.
@@ -967,10 +981,13 @@ done
reset_state
printf '{"invalid":[{"Name":"%s","Status":"Unknown"}],"valid":[]}\n' "${profile}" \
>"${state}/minikube-profile-list"
-bp invalid-known-delete delete "CONFIRM_DELETE=${profile}"
-assert_rc 0 'delete of a profile minikube lists as invalid'
-assert_log_has 'minikube profile list -o json' "delete of an invalid profile read minikube's profiles"
-assert_log_has "minikube -p ${profile} delete pidfiles=0" 'delete of a profile minikube lists as invalid ran'
+for action in stop delete; do
+ : >"${state}/calls.log"
+ bp "invalid-known-${action}" "${action}" "CONFIRM_DELETE=${profile}"
+ assert_rc 0 "${action} of a profile minikube lists as invalid"
+ assert_log_has 'minikube profile list -o json' "${action} of an invalid profile read minikube's profiles"
+ assert_log_has "minikube -p ${profile} ${action} pidfiles=0" "${action} of a profile minikube lists as invalid ran"
+done
# Without a context named after the profile there is nothing of another
# cluster's to remove, so minikube is not asked and delete runs.
@@ -996,6 +1013,7 @@ for action in stop delete; do
bp "foreign-identity-${action}-pf" pf
assert_rc 0 "pf before ${action} of a profile whose reachable cluster is another"
printf 'lan-cluster\n' >"${state}/node-label"
+ foreign_pid_record_before="$(shasum -a 256 "${pids_dir}/control-ui.pid" | awk '{print $1}')"
: >"${state}/calls.log"
bp "foreign-identity-${action}" "${action}" "CONFIRM_DELETE=${profile}"
assert_output_has 'BRANCH_PROFILE_CONTEXT_IDENTITY' "${action} of a profile whose reachable cluster is another"
@@ -1013,11 +1031,75 @@ for action in stop delete; do
"${action} of a profile whose reachable cluster is another must not run minikube ${action}"
assert_file "${pids_dir}/control-ui.pid" \
"${action} of a profile whose reachable cluster is another keeps the port-forward records"
+ [[ "$(shasum -a 256 "${pids_dir}/control-ui.pid" | awk '{print $1}')" == "${foreign_pid_record_before}" ]] ||
+ fail "${action} of a foreign cluster changed the port-forward record"
+ ok
+ assert_log_lacks 'port-forward' \
+ "${action} of a profile whose reachable cluster is another must not stop a forward through the API"
rm -f "${state}/node-label"
bp "foreign-identity-${action}-stop-pf" stop-pf
assert_rc 0 "stop-pf after the refused ${action}"
done
+# An invalid profile can still have a reachable API context. If minikube
+# cannot report its IP, the identity is unobservable rather than foreign:
+# recovery continues to the confirmed stop/delete after a bounded attempt.
+for action in stop delete; do
+ reset_state
+ printf '{"invalid":[{"Name":"%s","Status":"Unknown"}],"valid":[]}\n' "${profile}" \
+ >"${state}/minikube-profile-list"
+ : >"${state}/minikube-ip-fails"
+ bp "invalid-ip-fails-${action}" "${action}" "CONFIRM_DELETE=${profile}"
+ assert_rc 0 "${action} of an invalid profile whose IP cannot be read"
+ assert_output_has "cluster identity is unobservable for profile ${profile}" \
+ "${action} reports that unreadable identity is not a foreign-cluster refusal"
+ assert_log_has "kubectl --context=${profile} --request-timeout=10s cluster-info" \
+ "${action} proved the API was reachable before treating identity as unobservable"
+ assert_log_has "minikube -p ${profile} ip" "${action} attempted to read the invalid profile's IP"
+ assert_log_lacks 'get nodes -o json' "${action} must not invent a readable node identity without an IP"
+ assert_log_has "minikube -p ${profile} ${action} pidfiles=0" \
+ "${action} recovered an invalid profile whose IP cannot be read"
+done
+
+# A failed node inspection is also unobservable. It must not be collapsed into
+# a foreign-cluster mismatch: no readable node evidence contradicts the profile.
+for action in stop delete; do
+ reset_state
+ : >"${state}/fail-get-nodes"
+ bp "identity-nodes-fail-${action}" "${action}" "CONFIRM_DELETE=${profile}"
+ assert_rc 0 "${action} when the node identity inspection fails"
+ assert_output_has 'cluster identity is unobservable' \
+ "${action} reports the failed inspection without calling it foreign"
+ assert_output_lacks 'does not identify Minikube profile' \
+ "${action} does not hide a failed inspection as a foreign mismatch"
+ assert_log_has "minikube -p ${profile} ip" "${action} read the expected profile IP"
+ assert_log_has "kubectl --context=${profile} --request-timeout=10s get nodes -o json" \
+ "${action} attempted the node inspection"
+ assert_log_has "minikube -p ${profile} ${action} pidfiles=0" \
+ "${action} recovers when the node inspection fails"
+done
+
+# The identity attempt itself is finite: a hanging `minikube ip` is cut off by
+# the bounded runner, then recovery reaches stop/delete instead of stranding
+# the invalid profile until an unbounded child exits.
+for action in stop delete; do
+ reset_state
+ printf '{"invalid":[{"Name":"%s","Status":"Unknown"}],"valid":[]}\n' "${profile}" \
+ >"${state}/minikube-profile-list"
+ printf '30\n' >"${state}/minikube-ip-hang-seconds"
+ timeout_started="${SECONDS}"
+ bp "invalid-ip-timeout-${action}" "${action}" "CONFIRM_DELETE=${profile}" \
+ "BRANCH_PROFILE_MINIKUBE_STATUS_TIMEOUT_SECONDS=1"
+ timeout_elapsed=$((SECONDS - timeout_started))
+ assert_rc 0 "${action} after a finite minikube-ip timeout"
+ (( timeout_elapsed <= 8 )) ||
+ fail "${action} took ${timeout_elapsed}s to bound a 30s minikube-ip hang"
+ ok
+ assert_log_has "minikube -p ${profile} ip" "${action} entered the hanging IP read"
+ assert_log_has "minikube -p ${profile} ${action} pidfiles=0" \
+ "${action} recovered after the bounded IP timeout"
+done
+
for action in stop delete; do
reset_state
bp "own-identity-${action}" "${action}" "CONFIRM_DELETE=${profile}"
@@ -1517,5 +1599,8 @@ assert_output_lacks 'tree=A' 'setup on tree B must not run the tree A copy'
cp "${ROOT}/scripts/minikube/full-setup.sh" "${repo}/scripts/minikube/full-setup.sh"
printf 'fixture deploy file\n' >"${repo}/deploy/minikube/fixture.txt"
+minikube_test_assert_host_unchanged ||
+ fail 'the lifecycle fixture changed the host checkout HEAD, branch, or working tree'
+ok
(( ASSERTIONS >= 100 )) || fail "expected at least 100 assertions, ran ${ASSERTIONS}"
printf 'PASS: branch-profile lifecycle scenarios (%s assertions)\n' "${ASSERTIONS}"