Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(egress): close CI log and journal race regressions
Alfredo Lopeze17c4d39f367
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.mdindex 910a6999d..eba2a7896 100644--- a/docs/deploy/workflow-recipes-guide.md+++ b/docs/deploy/workflow-recipes-guide.md@@ -311,9 +311,11 @@ the write entirely when the existing object's stamped hash matches**. This keeps**Operational consequences:**-- **Logs:** expect "unchanged (spec-hash match); skipping update" instead of a per-loop- "Updated …". Per-loop write/`Updated` log volume drops sharply by design — re-baseline- any alert keyed on it.+- **Logs:** inspect the structured `msg` field for+ `Resource spec hash unchanged; skipping update` (resource identity in `label`) or+ `StatefulSet spec hash unchanged; skipping update` (identity in `name` and+ `namespace`). Per-loop write log volume drops sharply by design — re-baseline+ any alert keyed on the former unstructured `Updated …` text.- **Drift is no longer auto-reverted.** If an operator hand-edits a WRC-managed object(e.g. `kubectl edit`/`scale` on a recipe Deployment), WRC will **not** revert it untilthe recipe spec changes (which flips the hash). The change is still **visible**: thediff --git a/scripts/e2e/_lib/wrc-egress-lifecycle.cjs b/scripts/e2e/_lib/wrc-egress-lifecycle.cjsindex 7eb938f2b..bed20f684 100644--- a/scripts/e2e/_lib/wrc-egress-lifecycle.cjs+++ b/scripts/e2e/_lib/wrc-egress-lifecycle.cjs@@ -50,18 +50,53 @@ function safeParents(file, create = false, allowMissing = false) {function readJournal(file) {safeParents(file)- const info = fs.lstatSync(file)- invariant(info.isFile() && info.nlink === 1 && (info.mode & 0o077) === 0, 'UNSAFE_JOURNAL_FILE')- invariant(info.size > 0 && info.size <= 1024 * 1024, 'INVALID_JOURNAL_SIZE')- const state = JSON.parse(fs.readFileSync(file, 'utf8'))- invariant(- state.version === 1 &&- typeof state.runId === 'string' &&- /^[a-f0-9-]{36}$/.test(state.runId) &&- Array.isArray(state.resources),- 'INVALID_JOURNAL'- )- return state+ let descriptor+ try {+ // The local harness runs on POSIX. NOFOLLOW binds validation and reading+ // to one opened file; NONBLOCK prevents a substituted FIFO from hanging+ // before fstat can reject it as a non-regular file.+ descriptor = fs.openSync(+ file,+ fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK+ )+ } catch (error) {+ if (error.code === 'ELOOP') throw new Error('UNSAFE_JOURNAL_FILE', { cause: error })+ throw error+ }+ try {+ const checkedStat = () => {+ const info = fs.fstatSync(descriptor)+ invariant(+ info.isFile() && info.nlink === 1 && (info.mode & 0o077) === 0,+ 'UNSAFE_JOURNAL_FILE'+ )+ invariant(info.size > 0 && info.size <= 1024 * 1024, 'INVALID_JOURNAL_SIZE')+ return info+ }+ const before = checkedStat()+ // One extra byte detects growth without an unbounded read-to-EOF. Normal+ // journal writers replace atomically; in-place size changes are refused.+ const buffer = Buffer.alloc(before.size + 1)+ let length = 0+ while (length < buffer.length) {+ const count = fs.readSync(descriptor, buffer, length, buffer.length - length, null)+ if (count === 0) break+ length += count+ }+ const after = checkedStat()+ invariant(length === before.size && after.size === before.size, 'JOURNAL_CHANGED_DURING_READ')+ const state = JSON.parse(buffer.subarray(0, length).toString('utf8'))+ invariant(+ state.version === 1 &&+ typeof state.runId === 'string' &&+ /^[a-f0-9-]{36}$/.test(state.runId) &&+ Array.isArray(state.resources),+ 'INVALID_JOURNAL'+ )+ return state+ } finally {+ fs.closeSync(descriptor)+ }}function assertJournalBinding(state, expected) {diff --git a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh b/scripts/e2e/e2e-hcc-mcp-context-readiness.shindex 2527cd075..61078958b 100755--- a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh+++ b/scripts/e2e/e2e-hcc-mcp-context-readiness.sh@@ -210,6 +210,21 @@ hcc_log_contains() {grep -Fq "$marker" <<<"$logs"}+hcc_initial_empty_context_observed() {+ local logs+ [ -n "$NEW_HCC_POD" ] || return 1+ logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \+ -c host-context-controller 2>/dev/null)" || return 1+ # This pod still mixes legacy text with structured events. Only a complete+ # JSON event for our exact Context with an explicitly empty allowlist counts.+ jq -Rse --arg context_id "$CONTEXT_ID" '+ any(split("\n")[] | fromjson? | select(type == "object");+ .svc == "host-context-controller" and+ .msg == "reconciling context network policies" and+ .contextId == $context_id and .allowedServers == [])+ ' <<<"$logs" >/dev/null+}+dns_log_contains() {local marker=$1 logslogs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||@@ -1714,9 +1729,8 @@ ok "/ready is 200 while the divergent stale policy and affected runtime remain aok "safety_pass_* metrics corroborate an authoritative pass revoked the stale policy"ok "baseline NetworkPolicies remain present throughout fail-closed safety convergence"-initial_empty_context_log="[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []"wait_until 120 "initial NetworkPolicy pass to reconcile the fixture's empty Context snapshot" \- hcc_log_contains "$initial_empty_context_log" ||+ hcc_initial_empty_context_observed ||die "initial NetworkPolicy pass never reconciled the fixture's empty Context"wait_until 120 "a real divergent external-egress retry to be scheduled" \external_egress_retry_progress_is_observed ||diff --git a/scripts/tests/test-hcc-watch-recovery-gate.sh b/scripts/tests/test-hcc-watch-recovery-gate.shindex 4f711a573..a97110722 100755--- a/scripts/tests/test-hcc-watch-recovery-gate.sh+++ b/scripts/tests/test-hcc-watch-recovery-gate.sh@@ -498,8 +498,8 @@ marker_bindings=("$MCP_READINESS_GATE" 'Context watch event: MODIFIED for ${CONTEXT_NAME}'"$HCC_K8S_CLIENT" 'Context watch event: ${type} for ${context.name}'- "$MCP_READINESS_GATE" '[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []'- "$HCC_NETWORK_POLICY_RECONCILER" 'Reconciling context "${contextId}" — allowed servers: [${allowedServers.join('+ "$MCP_READINESS_GATE" '.msg == "reconciling context network policies"'+ "$HCC_NETWORK_POLICY_RECONCILER" "hccLogger.info('reconciling context network policies', { contextId, allowedServers })""$MCP_READINESS_GATE" "NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'""$HCC_K8S_CLIENT" 'Initial NetworkPolicy background reconciliation failed:'@@ -579,6 +579,75 @@ elsefail "an e2e gate defines a *_MARKER absent from marker_bindings (drift fence would miss it)"fi+# The initial Context witness must come from the structured HCC event for the+# exact fixture Context, not a substring in an unrelated event or old text log.+# These are local log fixtures; kctl never contacts a cluster.+initial_empty_context_function="$(+ sed -n '/^hcc_initial_empty_context_observed() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+run_initial_empty_context_case() (+ local mode=$1+ NEW_HCC_POD=hcc-pod+ HCC_NS=control-plane+ CONTEXT_ID='fixture-context'+ MOCK_CONTEXT_LOG="$(jq -cn --arg context "$CONTEXT_ID" '{+ svc:"host-context-controller",level:"info",+ msg:"reconciling context network policies",contextId:$context,allowedServers:[]+ }')"+ case "$mode" in+ exact) ;;+ mixed) MOCK_CONTEXT_LOG="unstructured bootstrap line+not-json+null+42+[]+$MOCK_CONTEXT_LOG+trailing diagnostic" ;;+ reordered) MOCK_CONTEXT_LOG="$(jq '{allowedServers,contextId,msg,level,svc}' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;+ wrong-context) MOCK_CONTEXT_LOG="$(jq '.contextId="another-context"' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;+ nonempty) MOCK_CONTEXT_LOG="$(jq '.allowedServers=["server-a"]' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;+ missing-servers) MOCK_CONTEXT_LOG="$(jq 'del(.allowedServers)' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;+ null-servers) MOCK_CONTEXT_LOG="$(jq '.allowedServers=null' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;+ wrong-service) MOCK_CONTEXT_LOG="$(jq '.svc="another-service"' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;+ wrong-event) MOCK_CONTEXT_LOG="$(jq '.msg="context reconciliation failed"' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;+ nested) MOCK_CONTEXT_LOG="$(jq '{detail:.}' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;+ legacy) MOCK_CONTEXT_LOG="[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []" ;;+ empty) MOCK_CONTEXT_LOG='' ;;+ api-error) ;;+ no-pod) NEW_HCC_POD='' ;;+ *) return 1 ;;+ esac+ kctl() {+ [[ "$*" == 'logs pod/hcc-pod -n control-plane -c host-context-controller' ]] || return 1+ printf '%s\n' "$MOCK_CONTEXT_LOG"+ [ "$mode" != api-error ]+ }+ [ -n "$initial_empty_context_function" ] || return 1+ eval "$initial_empty_context_function"+ hcc_initial_empty_context_observed+)+for context_log_case in exact mixed reordered; do+ if run_initial_empty_context_case "$context_log_case"; then+ pass "initial Context witness accepts $context_log_case structured HCC evidence"+ else+ fail "initial Context witness rejects $context_log_case structured HCC evidence"+ fi+done+for context_log_case in wrong-context nonempty missing-servers null-servers wrong-service wrong-event nested legacy empty api-error no-pod; do+ if run_initial_empty_context_case "$context_log_case"; then+ fail "initial Context witness accepts $context_log_case evidence"+ else+ pass "initial Context witness rejects $context_log_case evidence"+ fi+done+initial_context_wait="$(sed -n '/initial NetworkPolicy pass to reconcile the fixture/,/initial NetworkPolicy pass never reconciled/p' "$MCP_READINESS_GATE")"+if [[ "$initial_context_wait" == *'hcc_initial_empty_context_observed'* ]] &&+ [[ "$initial_context_wait" != *'hcc_log_contains'* ]]; then+ pass "initial Context readiness wait uses the structured identity-bound witness"+else+ fail "initial Context readiness wait does not use the structured identity-bound witness"+fi+fixture_mcp_runtime_absent_function="$(sed -n '/^fixture_mcp_runtime_absent() {$/,/^}$/p' "$MCP_READINESS_GATE")"diff --git a/scripts/tests/wrc-egress-lifecycle.test.cjs b/scripts/tests/wrc-egress-lifecycle.test.cjsindex 4755699d4..25f6800da 100644--- a/scripts/tests/wrc-egress-lifecycle.test.cjs+++ b/scripts/tests/wrc-egress-lifecycle.test.cjs@@ -62,6 +62,49 @@ function apiError(code) {return Object.assign(new Error(`API ${code}`), { code })}+function trackJournalDescriptors(t, file) {+ const open = fs.openSync+ const fstat = fs.fstatSync+ const descriptors = []+ t.mock.method(fs, 'openSync', (filename, flags, ...args) => {+ const descriptor = open(filename, flags, ...args)+ if (+ filename === file &&+ (flags === 'r' ||+ (typeof flags === 'number' &&+ (flags & (fs.constants.O_WRONLY | fs.constants.O_RDWR)) === 0))+ ) {+ descriptors.push(descriptor)+ }+ return descriptor+ })+ return { descriptors, fstat }+}++// Exercise the same interleaving with the old pathname check and with the new+// descriptor check. No production seam is needed: only the test mocks fs.+function afterJournalMetadata(t, file, tracked, action) {+ const lstat = fs.lstatSync+ let changed = false+ const once = () => {+ if (!changed) {+ changed = true+ action()+ }+ }+ t.mock.method(fs, 'lstatSync', (filename, ...args) => {+ const info = lstat(filename, ...args)+ if (filename === file) once()+ return info+ })+ t.mock.method(fs, 'fstatSync', (descriptor, ...args) => {+ const info = tracked.fstat(descriptor, ...args)+ if (tracked.descriptors.includes(descriptor)) once()+ return info+ })+ return () => changed+}+test('journal creates private durable state, saves atomically, and removes only its own run', t => {const { journal, file, state } = fixture(t)assert.equal(fs.statSync(file).mode & 0o777, 0o600)@@ -108,6 +151,108 @@ test('journal refuses replacement, hard links, public permissions, invalid JSONassert.throws(() => readJournal(file), /INVALID_JOURNAL_SIZE/)})+for (const replacementKind of ['regular file', 'symlink']) {+ test(`journal reads the checked descriptor when its pathname is replaced with a ${replacementKind}`, t => {+ const { directory, file, state } = fixture(t)+ const retained = path.join(directory, 'retained-journal.json')+ const replacement = path.join(directory, 'replacement-journal.json')+ fs.writeFileSync(replacement, JSON.stringify({ ...state, runId: randomUUID() }), {+ mode: 0o644,+ })+ const tracked = trackJournalDescriptors(t, file)+ const changed = afterJournalMetadata(t, file, tracked, () => {+ fs.renameSync(file, retained)+ if (replacementKind === 'symlink') fs.symlinkSync(replacement, file)+ else fs.renameSync(replacement, file)+ })+ assert.deepEqual(+ readJournal(file),+ state,+ 'never read a replacement whose permissions were not validated'+ )+ assert.equal(changed(), true, 'the check/use interleaving must actually execute')+ assert.equal(tracked.descriptors.length, 1)+ assert.throws(() => tracked.fstat(tracked.descriptors[0]), { code: 'EBADF' })+ })+}++for (const failureAt of ['fstat', 'read', 'parse']) {+ test(`journal closes its descriptor when ${failureAt} fails`, t => {+ const { file } = fixture(t)+ if (failureAt === 'parse') fs.writeFileSync(file, '{')+ const tracked = trackJournalDescriptors(t, file)+ const failure = new Error(`injected ${failureAt} failure`)+ if (failureAt === 'fstat') {+ t.mock.method(fs, 'fstatSync', descriptor => {+ if (tracked.descriptors.includes(descriptor)) throw failure+ return tracked.fstat(descriptor)+ })+ }+ if (failureAt === 'read') {+ const read = fs.readSync+ t.mock.method(fs, 'readSync', (descriptor, ...args) => {+ if (tracked.descriptors.includes(descriptor)) throw failure+ return read(descriptor, ...args)+ })+ }+ assert.throws(+ () => readJournal(file),+ error => (failureAt === 'parse' ? error instanceof SyntaxError : error === failure)+ )+ assert.equal(tracked.descriptors.length, 1)+ assert.throws(() => tracked.fstat(tracked.descriptors[0]), { code: 'EBADF' })+ })+}++test('journal rejects concurrent growth without reading beyond its checked size plus one byte', t => {+ const { file } = fixture(t)+ const checkedSize = fs.statSync(file).size+ const tracked = trackJournalDescriptors(t, file)+ const changed = afterJournalMetadata(t, file, tracked, () => {+ fs.appendFileSync(file, ' '.repeat(1024 * 1024))+ })+ const read = fs.readSync+ let bytesRead = 0+ t.mock.method(fs, 'readSync', (descriptor, ...args) => {+ const bytes = read(descriptor, ...args)+ if (tracked.descriptors.includes(descriptor)) bytesRead += bytes+ return bytes+ })+ assert.throws(() => readJournal(file), /INVALID_JOURNAL_SIZE|JOURNAL_CHANGED_DURING_READ/)+ assert.equal(changed(), true)+ assert(bytesRead > 0 && bytesRead <= checkedSize + 1)+ assert.throws(() => tracked.fstat(tracked.descriptors[0]), { code: 'EBADF' })+})++test('journal accepts exactly the size limit and rejects a FIFO without waiting for a writer', t => {+ const { file, state } = fixture(t)+ const json = JSON.stringify(state)+ fs.writeFileSync(file, json + ' '.repeat(1024 * 1024 - Buffer.byteLength(json)))+ assert.deepEqual(readJournal(file), state)+ fs.unlinkSync(file)+ const fifo = spawnSync('mkfifo', ['-m', '600', file], { encoding: 'utf8', timeout: 3000 })+ assert.ifError(fifo.error)+ assert.equal(fifo.status, 0, fifo.stderr)+ // A synchronous FIFO open cannot be interrupted by node:test's timer, so use+ // a separate bounded process. It must reject before any writer connects.+ const result = spawnSync(+ process.execPath,+ [+ '-e',+ `+ const assert = require('node:assert/strict');+ const { readJournal } = require(process.argv[1]);+ assert.throws(() => readJournal(process.argv[2]), /UNSAFE_JOURNAL_FILE/);+ `,+ lifecycle,+ file,+ ],+ { encoding: 'utf8', timeout: 3000 }+ )+ assert.ifError(result.error)+ assert.equal(result.status, 0, result.stderr)+})+test('journal paths reject traversal, noncanonical roots, and symbolic files or ancestors', t => {const { directory, file, state } = fixture(t)for (const profile of ['../escape', '/absolute', 'nested/profile', '', '..']) {