Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(egress): close CI log and journal race regressions

Alfredo Lopeze17c4d39f367
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.md
index 910a6999d..eba2a7896 100644
--- a/docs/deploy/workflow-recipes-guide.md
+++ b/docs/deploy/workflow-recipes-guide.md
@@ -311,9 +311,11 @@ the write entirely when the existing object's stamped hash matches**. This keeps
**Operational consequences:**
-- **Logs:** expect "unchanged (spec-hash match); skipping update" instead of a per-loop
- "Updated …". Per-loop write/`Updated` log volume drops sharply by design — re-baseline
- any alert keyed on it.
+- **Logs:** inspect the structured `msg` field for
+ `Resource spec hash unchanged; skipping update` (resource identity in `label`) or
+ `StatefulSet spec hash unchanged; skipping update` (identity in `name` and
+ `namespace`). Per-loop write log volume drops sharply by design — re-baseline
+ any alert keyed on the former unstructured `Updated …` text.
- **Drift is no longer auto-reverted.** If an operator hand-edits a WRC-managed object
(e.g. `kubectl edit`/`scale` on a recipe Deployment), WRC will **not** revert it until
the recipe spec changes (which flips the hash). The change is still **visible**: the
diff --git a/scripts/e2e/_lib/wrc-egress-lifecycle.cjs b/scripts/e2e/_lib/wrc-egress-lifecycle.cjs
index 7eb938f2b..bed20f684 100644
--- a/scripts/e2e/_lib/wrc-egress-lifecycle.cjs
+++ b/scripts/e2e/_lib/wrc-egress-lifecycle.cjs
@@ -50,18 +50,53 @@ function safeParents(file, create = false, allowMissing = false) {
function readJournal(file) {
safeParents(file)
- const info = fs.lstatSync(file)
- invariant(info.isFile() && info.nlink === 1 && (info.mode & 0o077) === 0, 'UNSAFE_JOURNAL_FILE')
- invariant(info.size > 0 && info.size <= 1024 * 1024, 'INVALID_JOURNAL_SIZE')
- const state = JSON.parse(fs.readFileSync(file, 'utf8'))
- invariant(
- state.version === 1 &&
- typeof state.runId === 'string' &&
- /^[a-f0-9-]{36}$/.test(state.runId) &&
- Array.isArray(state.resources),
- 'INVALID_JOURNAL'
- )
- return state
+ let descriptor
+ try {
+ // The local harness runs on POSIX. NOFOLLOW binds validation and reading
+ // to one opened file; NONBLOCK prevents a substituted FIFO from hanging
+ // before fstat can reject it as a non-regular file.
+ descriptor = fs.openSync(
+ file,
+ fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK
+ )
+ } catch (error) {
+ if (error.code === 'ELOOP') throw new Error('UNSAFE_JOURNAL_FILE', { cause: error })
+ throw error
+ }
+ try {
+ const checkedStat = () => {
+ const info = fs.fstatSync(descriptor)
+ invariant(
+ info.isFile() && info.nlink === 1 && (info.mode & 0o077) === 0,
+ 'UNSAFE_JOURNAL_FILE'
+ )
+ invariant(info.size > 0 && info.size <= 1024 * 1024, 'INVALID_JOURNAL_SIZE')
+ return info
+ }
+ const before = checkedStat()
+ // One extra byte detects growth without an unbounded read-to-EOF. Normal
+ // journal writers replace atomically; in-place size changes are refused.
+ const buffer = Buffer.alloc(before.size + 1)
+ let length = 0
+ while (length < buffer.length) {
+ const count = fs.readSync(descriptor, buffer, length, buffer.length - length, null)
+ if (count === 0) break
+ length += count
+ }
+ const after = checkedStat()
+ invariant(length === before.size && after.size === before.size, 'JOURNAL_CHANGED_DURING_READ')
+ const state = JSON.parse(buffer.subarray(0, length).toString('utf8'))
+ invariant(
+ state.version === 1 &&
+ typeof state.runId === 'string' &&
+ /^[a-f0-9-]{36}$/.test(state.runId) &&
+ Array.isArray(state.resources),
+ 'INVALID_JOURNAL'
+ )
+ return state
+ } finally {
+ fs.closeSync(descriptor)
+ }
}
function assertJournalBinding(state, expected) {
diff --git a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh b/scripts/e2e/e2e-hcc-mcp-context-readiness.sh
index 2527cd075..61078958b 100755
--- a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh
+++ b/scripts/e2e/e2e-hcc-mcp-context-readiness.sh
@@ -210,6 +210,21 @@ hcc_log_contains() {
grep -Fq "$marker" <<<"$logs"
}
+hcc_initial_empty_context_observed() {
+ local logs
+ [ -n "$NEW_HCC_POD" ] || return 1
+ logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \
+ -c host-context-controller 2>/dev/null)" || return 1
+ # This pod still mixes legacy text with structured events. Only a complete
+ # JSON event for our exact Context with an explicitly empty allowlist counts.
+ jq -Rse --arg context_id "$CONTEXT_ID" '
+ any(split("\n")[] | fromjson? | select(type == "object");
+ .svc == "host-context-controller" and
+ .msg == "reconciling context network policies" and
+ .contextId == $context_id and .allowedServers == [])
+ ' <<<"$logs" >/dev/null
+}
+
dns_log_contains() {
local marker=$1 logs
logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||
@@ -1714,9 +1729,8 @@ ok "/ready is 200 while the divergent stale policy and affected runtime remain a
ok "safety_pass_* metrics corroborate an authoritative pass revoked the stale policy"
ok "baseline NetworkPolicies remain present throughout fail-closed safety convergence"
-initial_empty_context_log="[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []"
wait_until 120 "initial NetworkPolicy pass to reconcile the fixture's empty Context snapshot" \
- hcc_log_contains "$initial_empty_context_log" ||
+ hcc_initial_empty_context_observed ||
die "initial NetworkPolicy pass never reconciled the fixture's empty Context"
wait_until 120 "a real divergent external-egress retry to be scheduled" \
external_egress_retry_progress_is_observed ||
diff --git a/scripts/tests/test-hcc-watch-recovery-gate.sh b/scripts/tests/test-hcc-watch-recovery-gate.sh
index 4f711a573..a97110722 100755
--- a/scripts/tests/test-hcc-watch-recovery-gate.sh
+++ b/scripts/tests/test-hcc-watch-recovery-gate.sh
@@ -498,8 +498,8 @@ marker_bindings=(
"$MCP_READINESS_GATE" 'Context watch event: MODIFIED for ${CONTEXT_NAME}'
"$HCC_K8S_CLIENT" 'Context watch event: ${type} for ${context.name}'
- "$MCP_READINESS_GATE" '[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []'
- "$HCC_NETWORK_POLICY_RECONCILER" 'Reconciling context "${contextId}" — allowed servers: [${allowedServers.join('
+ "$MCP_READINESS_GATE" '.msg == "reconciling context network policies"'
+ "$HCC_NETWORK_POLICY_RECONCILER" "hccLogger.info('reconciling context network policies', { contextId, allowedServers })"
"$MCP_READINESS_GATE" "NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'"
"$HCC_K8S_CLIENT" 'Initial NetworkPolicy background reconciliation failed:'
@@ -579,6 +579,75 @@ else
fail "an e2e gate defines a *_MARKER absent from marker_bindings (drift fence would miss it)"
fi
+# The initial Context witness must come from the structured HCC event for the
+# exact fixture Context, not a substring in an unrelated event or old text log.
+# These are local log fixtures; kctl never contacts a cluster.
+initial_empty_context_function="$(
+ sed -n '/^hcc_initial_empty_context_observed() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+run_initial_empty_context_case() (
+ local mode=$1
+ NEW_HCC_POD=hcc-pod
+ HCC_NS=control-plane
+ CONTEXT_ID='fixture-context'
+ MOCK_CONTEXT_LOG="$(jq -cn --arg context "$CONTEXT_ID" '{
+ svc:"host-context-controller",level:"info",
+ msg:"reconciling context network policies",contextId:$context,allowedServers:[]
+ }')"
+ case "$mode" in
+ exact) ;;
+ mixed) MOCK_CONTEXT_LOG="unstructured bootstrap line
+not-json
+null
+42
+[]
+$MOCK_CONTEXT_LOG
+trailing diagnostic" ;;
+ reordered) MOCK_CONTEXT_LOG="$(jq '{allowedServers,contextId,msg,level,svc}' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;
+ wrong-context) MOCK_CONTEXT_LOG="$(jq '.contextId="another-context"' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;
+ nonempty) MOCK_CONTEXT_LOG="$(jq '.allowedServers=["server-a"]' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;
+ missing-servers) MOCK_CONTEXT_LOG="$(jq 'del(.allowedServers)' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;
+ null-servers) MOCK_CONTEXT_LOG="$(jq '.allowedServers=null' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;
+ wrong-service) MOCK_CONTEXT_LOG="$(jq '.svc="another-service"' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;
+ wrong-event) MOCK_CONTEXT_LOG="$(jq '.msg="context reconciliation failed"' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;
+ nested) MOCK_CONTEXT_LOG="$(jq '{detail:.}' <<<"$MOCK_CONTEXT_LOG" | jq -c .)" ;;
+ legacy) MOCK_CONTEXT_LOG="[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []" ;;
+ empty) MOCK_CONTEXT_LOG='' ;;
+ api-error) ;;
+ no-pod) NEW_HCC_POD='' ;;
+ *) return 1 ;;
+ esac
+ kctl() {
+ [[ "$*" == 'logs pod/hcc-pod -n control-plane -c host-context-controller' ]] || return 1
+ printf '%s\n' "$MOCK_CONTEXT_LOG"
+ [ "$mode" != api-error ]
+ }
+ [ -n "$initial_empty_context_function" ] || return 1
+ eval "$initial_empty_context_function"
+ hcc_initial_empty_context_observed
+)
+for context_log_case in exact mixed reordered; do
+ if run_initial_empty_context_case "$context_log_case"; then
+ pass "initial Context witness accepts $context_log_case structured HCC evidence"
+ else
+ fail "initial Context witness rejects $context_log_case structured HCC evidence"
+ fi
+done
+for context_log_case in wrong-context nonempty missing-servers null-servers wrong-service wrong-event nested legacy empty api-error no-pod; do
+ if run_initial_empty_context_case "$context_log_case"; then
+ fail "initial Context witness accepts $context_log_case evidence"
+ else
+ pass "initial Context witness rejects $context_log_case evidence"
+ fi
+done
+initial_context_wait="$(sed -n '/initial NetworkPolicy pass to reconcile the fixture/,/initial NetworkPolicy pass never reconciled/p' "$MCP_READINESS_GATE")"
+if [[ "$initial_context_wait" == *'hcc_initial_empty_context_observed'* ]] &&
+ [[ "$initial_context_wait" != *'hcc_log_contains'* ]]; then
+ pass "initial Context readiness wait uses the structured identity-bound witness"
+else
+ fail "initial Context readiness wait does not use the structured identity-bound witness"
+fi
+
fixture_mcp_runtime_absent_function="$(
sed -n '/^fixture_mcp_runtime_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
)"
diff --git a/scripts/tests/wrc-egress-lifecycle.test.cjs b/scripts/tests/wrc-egress-lifecycle.test.cjs
index 4755699d4..25f6800da 100644
--- a/scripts/tests/wrc-egress-lifecycle.test.cjs
+++ b/scripts/tests/wrc-egress-lifecycle.test.cjs
@@ -62,6 +62,49 @@ function apiError(code) {
return Object.assign(new Error(`API ${code}`), { code })
}
+function trackJournalDescriptors(t, file) {
+ const open = fs.openSync
+ const fstat = fs.fstatSync
+ const descriptors = []
+ t.mock.method(fs, 'openSync', (filename, flags, ...args) => {
+ const descriptor = open(filename, flags, ...args)
+ if (
+ filename === file &&
+ (flags === 'r' ||
+ (typeof flags === 'number' &&
+ (flags & (fs.constants.O_WRONLY | fs.constants.O_RDWR)) === 0))
+ ) {
+ descriptors.push(descriptor)
+ }
+ return descriptor
+ })
+ return { descriptors, fstat }
+}
+
+// Exercise the same interleaving with the old pathname check and with the new
+// descriptor check. No production seam is needed: only the test mocks fs.
+function afterJournalMetadata(t, file, tracked, action) {
+ const lstat = fs.lstatSync
+ let changed = false
+ const once = () => {
+ if (!changed) {
+ changed = true
+ action()
+ }
+ }
+ t.mock.method(fs, 'lstatSync', (filename, ...args) => {
+ const info = lstat(filename, ...args)
+ if (filename === file) once()
+ return info
+ })
+ t.mock.method(fs, 'fstatSync', (descriptor, ...args) => {
+ const info = tracked.fstat(descriptor, ...args)
+ if (tracked.descriptors.includes(descriptor)) once()
+ return info
+ })
+ return () => changed
+}
+
test('journal creates private durable state, saves atomically, and removes only its own run', t => {
const { journal, file, state } = fixture(t)
assert.equal(fs.statSync(file).mode & 0o777, 0o600)
@@ -108,6 +151,108 @@ test('journal refuses replacement, hard links, public permissions, invalid JSON
assert.throws(() => readJournal(file), /INVALID_JOURNAL_SIZE/)
})
+for (const replacementKind of ['regular file', 'symlink']) {
+ test(`journal reads the checked descriptor when its pathname is replaced with a ${replacementKind}`, t => {
+ const { directory, file, state } = fixture(t)
+ const retained = path.join(directory, 'retained-journal.json')
+ const replacement = path.join(directory, 'replacement-journal.json')
+ fs.writeFileSync(replacement, JSON.stringify({ ...state, runId: randomUUID() }), {
+ mode: 0o644,
+ })
+ const tracked = trackJournalDescriptors(t, file)
+ const changed = afterJournalMetadata(t, file, tracked, () => {
+ fs.renameSync(file, retained)
+ if (replacementKind === 'symlink') fs.symlinkSync(replacement, file)
+ else fs.renameSync(replacement, file)
+ })
+ assert.deepEqual(
+ readJournal(file),
+ state,
+ 'never read a replacement whose permissions were not validated'
+ )
+ assert.equal(changed(), true, 'the check/use interleaving must actually execute')
+ assert.equal(tracked.descriptors.length, 1)
+ assert.throws(() => tracked.fstat(tracked.descriptors[0]), { code: 'EBADF' })
+ })
+}
+
+for (const failureAt of ['fstat', 'read', 'parse']) {
+ test(`journal closes its descriptor when ${failureAt} fails`, t => {
+ const { file } = fixture(t)
+ if (failureAt === 'parse') fs.writeFileSync(file, '{')
+ const tracked = trackJournalDescriptors(t, file)
+ const failure = new Error(`injected ${failureAt} failure`)
+ if (failureAt === 'fstat') {
+ t.mock.method(fs, 'fstatSync', descriptor => {
+ if (tracked.descriptors.includes(descriptor)) throw failure
+ return tracked.fstat(descriptor)
+ })
+ }
+ if (failureAt === 'read') {
+ const read = fs.readSync
+ t.mock.method(fs, 'readSync', (descriptor, ...args) => {
+ if (tracked.descriptors.includes(descriptor)) throw failure
+ return read(descriptor, ...args)
+ })
+ }
+ assert.throws(
+ () => readJournal(file),
+ error => (failureAt === 'parse' ? error instanceof SyntaxError : error === failure)
+ )
+ assert.equal(tracked.descriptors.length, 1)
+ assert.throws(() => tracked.fstat(tracked.descriptors[0]), { code: 'EBADF' })
+ })
+}
+
+test('journal rejects concurrent growth without reading beyond its checked size plus one byte', t => {
+ const { file } = fixture(t)
+ const checkedSize = fs.statSync(file).size
+ const tracked = trackJournalDescriptors(t, file)
+ const changed = afterJournalMetadata(t, file, tracked, () => {
+ fs.appendFileSync(file, ' '.repeat(1024 * 1024))
+ })
+ const read = fs.readSync
+ let bytesRead = 0
+ t.mock.method(fs, 'readSync', (descriptor, ...args) => {
+ const bytes = read(descriptor, ...args)
+ if (tracked.descriptors.includes(descriptor)) bytesRead += bytes
+ return bytes
+ })
+ assert.throws(() => readJournal(file), /INVALID_JOURNAL_SIZE|JOURNAL_CHANGED_DURING_READ/)
+ assert.equal(changed(), true)
+ assert(bytesRead > 0 && bytesRead <= checkedSize + 1)
+ assert.throws(() => tracked.fstat(tracked.descriptors[0]), { code: 'EBADF' })
+})
+
+test('journal accepts exactly the size limit and rejects a FIFO without waiting for a writer', t => {
+ const { file, state } = fixture(t)
+ const json = JSON.stringify(state)
+ fs.writeFileSync(file, json + ' '.repeat(1024 * 1024 - Buffer.byteLength(json)))
+ assert.deepEqual(readJournal(file), state)
+ fs.unlinkSync(file)
+ const fifo = spawnSync('mkfifo', ['-m', '600', file], { encoding: 'utf8', timeout: 3000 })
+ assert.ifError(fifo.error)
+ assert.equal(fifo.status, 0, fifo.stderr)
+ // A synchronous FIFO open cannot be interrupted by node:test's timer, so use
+ // a separate bounded process. It must reject before any writer connects.
+ const result = spawnSync(
+ process.execPath,
+ [
+ '-e',
+ `
+ const assert = require('node:assert/strict');
+ const { readJournal } = require(process.argv[1]);
+ assert.throws(() => readJournal(process.argv[2]), /UNSAFE_JOURNAL_FILE/);
+ `,
+ lifecycle,
+ file,
+ ],
+ { encoding: 'utf8', timeout: 3000 }
+ )
+ assert.ifError(result.error)
+ assert.equal(result.status, 0, result.stderr)
+})
+
test('journal paths reject traversal, noncanonical roots, and symbolic files or ancestors', t => {
const { directory, file, state } = fixture(t)
for (const profile of ['../escape', '/absolute', 'nested/profile', '', '..']) {