Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(t1): report failed file messages and redact escaped DSNs in the T1 reporter

Alfredo Lopezc5c82c8a8dab
diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.sh
index 0b2b2b410..6f661282f 100755
--- a/scripts/e2e/minikube-real-postgres.sh
+++ b/scripts/e2e/minikube-real-postgres.sh
@@ -266,6 +266,8 @@ sanitize_file() {
[ -f "$file" ] || return 0
T1_REDACT_PASSWORD="${T1_REDACT_PASSWORD:-${PG_PASSWORD:-}}" python3 - "$file" <<'PY'
from pathlib import Path
+from urllib.parse import quote
+import json
import os
import re
import sys
@@ -273,8 +275,13 @@ path = Path(sys.argv[1])
text = path.read_text(errors="replace")
password = os.environ.get("T1_REDACT_PASSWORD", "")
if password:
- text = text.replace(password, "<password-redacted>")
-text = re.sub(r"postgres(?:ql)?://[^\s\"'<>]+", "<minikube-postgres-dsn-redacted>", text)
+ # The Vitest JSON reporter stores messages as JSON strings and a DSN
+ # builder percent-encodes the password, so redact those spellings too.
+ for spelling in (json.dumps(password)[1:-1], quote(password, safe=""), password):
+ text = text.replace(spelling, "<password-redacted>")
+# Stop at a backslash: inside the JSON reporter a DSN is followed by the escape
+# that closes its string (\"), and consuming it leaves invalid JSON.
+text = re.sub(r"postgres(?:ql)?://[^\s\"'<>\\]+", "<minikube-postgres-dsn-redacted>", text)
text = re.sub(r"(?i)(authorization:\s*bearer\s+)[^\s]+", r"\1<token-redacted>", text)
path.write_text(text)
PY
@@ -359,6 +366,49 @@ print("postgresql://[email protected]:" + port + "/postgres", end="")
')"
}
+# Print the reporter counters and every non-passing file and test. The file must
+# already have gone through sanitize_file. A suite that fails before any test
+# runs (beforeAll, import error) has no assertionResults: its only diagnosis is
+# the file-level `message`, so it is printed right after the FAILED FILE line.
+print_reporter_failure_details() {
+ python3 - "$1" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+result = json.loads(Path(sys.argv[1]).read_text())
+print(
+ "T1 reporter counters:",
+ {
+ key: result.get(key)
+ for key in (
+ "success",
+ "numTotalTestSuites",
+ "numPassedTestSuites",
+ "numFailedTestSuites",
+ "numPendingTestSuites",
+ "numTotalTests",
+ "numPassedTests",
+ "numFailedTests",
+ "numPendingTests",
+ )
+ },
+)
+for test_result in result.get("testResults", []):
+ if test_result.get("status") == "passed":
+ continue
+ print(f"FAILED FILE: {test_result.get('name')}")
+ if test_result.get("message"):
+ print(test_result["message"])
+ for assertion in test_result.get("assertionResults", []):
+ if assertion.get("status") == "passed":
+ continue
+ print(f"FAILED TEST: {assertion.get('fullName') or assertion.get('title')}")
+ for message in assertion.get("failureMessages", []):
+ print(message)
+PY
+}
+
run_suite() {
local package="$1" lane="$2" admin_url="$3"
local expected_files=0 reported_files=0 file log_file json_file stats
@@ -474,40 +524,7 @@ PY
[ "$passed_suites" -ne "$total_suites" ] || [ "$failed_suites" -ne 0 ] || \
[ "$failed_tests" -ne 0 ]; then
cat "$log_file" >&2 || true
- python3 - "$json_file" <<'PY' >&2 || true
-import json
-import sys
-from pathlib import Path
-
-result = json.loads(Path(sys.argv[1]).read_text())
-print(
- "T1 reporter counters:",
- {
- key: result.get(key)
- for key in (
- "success",
- "numTotalTestSuites",
- "numPassedTestSuites",
- "numFailedTestSuites",
- "numPendingTestSuites",
- "numTotalTests",
- "numPassedTests",
- "numFailedTests",
- "numPendingTests",
- )
- },
-)
-for test_result in result.get("testResults", []):
- if test_result.get("status") == "passed":
- continue
- print(f"FAILED FILE: {test_result.get('name')}")
- for assertion in test_result.get("assertionResults", []):
- if assertion.get("status") == "passed":
- continue
- print(f"FAILED TEST: {assertion.get('fullName') or assertion.get('title')}")
- for message in assertion.get("failureMessages", []):
- print(message)
-PY
+ print_reporter_failure_details "$json_file" >&2 || true
T1_NEXT_COMMAND='repair the failed or incomplete Real PostgreSQL lane, then re-run T1'
die_t1 REAL_PG_SUITE_FAILED "Real PostgreSQL reporter did not pass every suite in $package ($lane)"
fi
diff --git a/scripts/tests/test-minikube-t1-reporter-report.sh b/scripts/tests/test-minikube-t1-reporter-report.sh
new file mode 100755
index 000000000..83aef9a48
--- /dev/null
+++ b/scripts/tests/test-minikube-t1-reporter-report.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# T1 must keep the Vitest JSON reporter parseable after redaction and must print
+# the file-level message of a suite that failed before any test ran. Only the
+# T1 function definitions are sourced; nothing here touches a cluster or Docker.
+set -euo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
+
+TEST_DIR="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-t1-reporter.XXXXXX")"
+trap 'rm -rf -- "$TEST_DIR"' EXIT
+
+(
+ set +e
+ export MINIKUBE_PROFILE=t1-reporter-contract
+ export T2_CONTEXT=t1-reporter-contract
+ source "$ROOT/scripts/e2e/minikube-real-postgres.sh" >/dev/null 2>&1
+ trap - EXIT INT TERM
+ set +e
+ declare -F sanitize_file >/dev/null || fail 'sanitize_file is not defined'
+ declare -F print_reporter_failure_details >/dev/null || fail 'print_reporter_failure_details is not defined'
+
+ assert_valid_json() {
+ python3 -c 'import json, sys; json.load(open(sys.argv[1]))' "$1" \
+ || fail "$2: sanitized reporter is not valid JSON"
+ }
+
+ # 1. A DSN followed by an escaped quote must not swallow the escape.
+ dsn_json="$TEST_DIR/dsn.json"
+ printf '%s' '{"failureMessages":["expected \"postgresql://u:p@h/db\" to be undefined"]}' >"$dsn_json"
+ T1_REDACT_PASSWORD='' sanitize_file "$dsn_json"
+ assert_valid_json "$dsn_json" 'DSN inside a JSON string'
+ grep -Fq '<minikube-postgres-dsn-redacted>' "$dsn_json" \
+ || fail 'DSN was not redacted (liveness witness missing)'
+ grep -Fq 'u:p@h' "$dsn_json" && fail 'DSN credential survived redaction'
+
+ # 2. A password with JSON-special characters is redacted in every spelling.
+ pw_json="$TEST_DIR/password.json"
+ password='s3cr"et\pw'
+ python3 - "$pw_json" "$password" <<'PY'
+import json
+import sys
+from urllib.parse import quote
+
+path, password = sys.argv[1], sys.argv[2]
+message = (
+ f"auth failed for {password} "
+ f"see http://example.invalid/?p={quote(password, safe='')}"
+)
+with open(path, "w") as handle:
+ json.dump({"testResults": [{"message": message}]}, handle)
+PY
+ grep -Fq 's3cr' "$pw_json" || fail 'fixture does not contain the password (liveness witness missing)'
+ T1_REDACT_PASSWORD="$password" sanitize_file "$pw_json"
+ assert_valid_json "$pw_json" 'JSON-escaped password'
+ grep -Fq '<password-redacted>' "$pw_json" || fail 'password was not redacted'
+ grep -Fq 's3cr' "$pw_json" && fail 'a spelling of the password survived redaction'
+
+ # 3. A suite that failed before any test has no assertionResults; its reason
+ # is the file-level message and must be printed after the FAILED FILE line.
+ report="$TEST_DIR/report.json"
+ cat >"$report" <<'JSON'
+{
+ "success": false,
+ "numTotalTestSuites": 3,
+ "numPassedTestSuites": 1,
+ "numFailedTestSuites": 2,
+ "numTotalTests": 1,
+ "numPassedTests": 1,
+ "numFailedTests": 0,
+ "testResults": [
+ {"name": "/repo/control-api/ok.realPostgres.test.ts", "status": "passed",
+ "message": "", "assertionResults": [{"status": "passed", "title": "works"}]},
+ {"name": "/repo/control-api/early.realPostgres.test.ts", "status": "failed",
+ "message": "beforeAll failed: connect ECONNREFUSED postgresql://admin:[email protected]:5432/postgres",
+ "assertionResults": []},
+ {"name": "/repo/control-api/late.realPostgres.test.ts", "status": "failed",
+ "message": "",
+ "assertionResults": [{"status": "failed", "fullName": "late fails",
+ "failureMessages": ["assertion detail LATE-DETAIL"]}]}
+ ]
+}
+JSON
+ T1_REDACT_PASSWORD='hunter2' sanitize_file "$report"
+ assert_valid_json "$report" 'reporter with a pre-test failure'
+ details="$TEST_DIR/details.out"
+ print_reporter_failure_details "$report" >"$details" 2>&1 || fail 'print_reporter_failure_details failed'
+ grep -Fq 'FAILED FILE: /repo/control-api/early.realPostgres.test.ts' "$details" \
+ || fail 'the failed file was not printed (liveness witness missing)'
+ grep -Fq 'FAILED FILE: /repo/control-api/late.realPostgres.test.ts' "$details" \
+ || fail 'the assertion-failure file was not printed'
+ grep -Fq 'LATE-DETAIL' "$details" || fail 'assertion failure messages are no longer printed'
+ grep -Fq 'ok.realPostgres' "$details" && fail 'a passing file was printed as failed'
+ early_next="$(grep -A1 -F 'FAILED FILE: /repo/control-api/early.realPostgres.test.ts' "$details" | tail -n 1)"
+ case "$early_next" in
+ 'beforeAll failed: connect ECONNREFUSED '*) ;;
+ *) fail "pre-test failure reason was not printed after FAILED FILE: '$early_next'" ;;
+ esac
+ grep -Fq 'hunter2' "$details" && fail 'a credential reached the failure dump'
+ grep -Fq '<minikube-postgres-dsn-redacted>' "$details" \
+ || fail 'the dumped message was not sanitized'
+ exit 0
+) || fail 'T1 reporter sanitization or failure dump contract broken'
+
+printf 'PASS: T1 reporter stays valid JSON after redaction and prints pre-test failure reasons\n'
diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.sh
index 24cb4921e..874f6ff11 100755
--- a/scripts/tests/test-minikube-t2-contract.sh
+++ b/scripts/tests/test-minikube-t2-contract.sh
@@ -51,6 +51,7 @@ for file in "$MINIKUBE_DIR/profile-readiness.sh" "$ROOT/scripts/tests/test-minik
"$ROOT/scripts/tests/test-minikube-pre-gate-shadow.sh" \
"$ROOT/scripts/tests/test-minikube-fenced-recovery-render.sh" \
"$ROOT/scripts/tests/test-minikube-t2-process-owner.sh" \
+ "$ROOT/scripts/tests/test-minikube-t1-reporter-report.sh" \
"$ROOT/scripts/tests/test-minikube-explicit-context.sh" \
"$ROOT/scripts/tests/test-minikube-t2-evidence.sh" \
"$ROOT/scripts/tests/test-minikube-targeted-health.sh" \
@@ -59,6 +60,7 @@ for file in "$MINIKUBE_DIR/profile-readiness.sh" "$ROOT/scripts/tests/test-minik
done
"$ROOT/scripts/tests/test-minikube-t1-port-forward-owner.sh"
"$ROOT/scripts/tests/test-minikube-t2-process-owner.sh"
+"$ROOT/scripts/tests/test-minikube-t1-reporter-report.sh"
"$ROOT/scripts/tests/test-minikube-explicit-context.sh"
"$ROOT/scripts/tests/test-minikube-t2-evidence.sh"
"$ROOT/scripts/tests/test-minikube-targeted-health.sh"
diff --git a/scripts/tests/test-minikube-t2-process-owner.sh b/scripts/tests/test-minikube-t2-process-owner.sh
index 595c27b69..370466c6d 100755
--- a/scripts/tests/test-minikube-t2-process-owner.sh
+++ b/scripts/tests/test-minikube-t2-process-owner.sh
@@ -149,5 +149,34 @@ if run_process_check "$TEST_DIR/duplicate.out"; then
exit 1
fi
grep -Fq PORT_FORWARD_CONFLICT "$TEST_DIR/duplicate.out"
+rm -f -- "$SECOND_RECORD"
+
+# A record whose first line is not a PID is an ownership failure and must stay
+# on disk. The baseline run proves the fixture alone is accepted, so the refusal
+# below can only come from the malformed record.
+run_process_check "$TEST_DIR/malformed-baseline.out" || {
+ printf 'FAIL: exact ownership fixture was rejected before the malformed record existed\n' >&2
+ exit 1
+}
+MALFORMED_RECORD="$PID_DIR/control-ui-malformed.pid"
+printf '%s\n' 'not-a-pid' 'trailing line' >"$MALFORMED_RECORD"
+malformed_before="$(cat "$MALFORMED_RECORD")"
+if run_process_check "$TEST_DIR/malformed.out"; then
+ printf 'FAIL: a port-forward record whose first line is not a PID was accepted\n' >&2
+ exit 1
+fi
+grep -Fq PORT_FORWARD_CONFLICT "$TEST_DIR/malformed.out"
+grep -Fq "$MALFORMED_RECORD" "$TEST_DIR/malformed.out" || {
+ printf 'FAIL: the conflict did not name the malformed record\n' >&2
+ exit 1
+}
+[ -f "$MALFORMED_RECORD" ] && [ "$(cat "$MALFORMED_RECORD")" = "$malformed_before" ] || {
+ printf 'FAIL: the process check removed or rewrote the malformed record\n' >&2
+ exit 1
+}
+[ -f "$RECORD" ] || {
+ printf 'FAIL: the process check removed the valid owner record\n' >&2
+ exit 1
+}
printf 'PASS: T2 process inventory requires one exact structured port-forward owner\n'