Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# E2E — Workflow Usage Attribution
# ═══════════════════════════════════════════════════════════════════════
#
# Run after executing the workflow-recipes E2E fixtures. It validates the
# business signal that powers /usage: new workflow LLM usage must carry a
# canonical run-id task_id plus non-null Team and LLM Secret dimensions.
#
# Environment:
# K8S_CONTEXT (default: clerum-test)
# E2E_POSTGRES_NAMESPACE (default: control-plane)
# E2E_POSTGRES_POD_SELECTOR (default: app=control-postgres)
# E2E_POSTGRES_DB (default: profiles)
# E2E_USAGE_RECIPE_LIKE (default: e2e-%)
# E2E_USAGE_LOOKBACK_HOURS (default: 24)
# E2E_USAGE_MIN_REQUESTS (default: 1)
# E2E_USAGE_ROLLUP_WAIT_SECONDS (default: 90)
# E2E_USAGE_TRIGGER_FIXTURES (default: false; trigger seeded recipes first)
# E2E_USAGE_EVENT_WAIT_SECONDS (default: 300 when triggering fixtures)
# E2E_EXTERNAL_REST_API_URL (default: http://localhost:8091)
# E2E_TEST_EMAIL (default: [email protected])
# E2E_USAGE_TRIGGER_RECIPES (default: e2e-ondemand-simple e2e-retention-recipe e2e-scheduled-recipe)
# E2E_USAGE_SCHEDULE_RECIPE optional; only set when the schedule suite seeded that fixture
# ═══════════════════════════════════════════════════════════════════════
set -euo pipefail
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
CYAN='\033[0;36m'; NC='\033[0m'; BOLD='\033[1m'
log() { echo -e "${CYAN}[usage-attribution-e2e]${NC} $*"; }
pass() { echo -e " ${GREEN}PASS${NC} $*"; }
warn() { echo -e " ${YELLOW}WARN${NC} $*"; }
fail() { echo -e " ${RED}FAIL${NC} $*"; echo -e "${RED}ABORT${NC}"; exit 1; }
K8S_CONTEXT="${K8S_CONTEXT:-clerum-test}"
KC="kubectl --context=${K8S_CONTEXT}"
PG_NS="${E2E_POSTGRES_NAMESPACE:-control-plane}"
PG_SEL="${E2E_POSTGRES_POD_SELECTOR:-app=control-postgres}"
PG_DB="${E2E_POSTGRES_DB:-profiles}"
RECIPE_LIKE="${E2E_USAGE_RECIPE_LIKE:-e2e-%}"
LOOKBACK_HOURS="${E2E_USAGE_LOOKBACK_HOURS:-24}"
MIN_REQUESTS="${E2E_USAGE_MIN_REQUESTS:-1}"
ROLLUP_WAIT_SECONDS="${E2E_USAGE_ROLLUP_WAIT_SECONDS:-90}"
TRIGGER_FIXTURES="${E2E_USAGE_TRIGGER_FIXTURES:-false}"
EVENT_WAIT_SECONDS="${E2E_USAGE_EVENT_WAIT_SECONDS:-300}"
EXT_URL="${E2E_EXTERNAL_REST_API_URL:-http://localhost:8091}"
TEST_EMAIL="${E2E_TEST_EMAIL:-test@clerum.io}"
ADMIN_PASSWORD="${ADMIN_PASSWORD:-changeme123!}"
TRIGGER_RECIPES="${E2E_USAGE_TRIGGER_RECIPES:-e2e-ondemand-simple e2e-retention-recipe e2e-scheduled-recipe}"
SCHEDULE_RECIPE="${E2E_USAGE_SCHEDULE_RECIPE:-}"
[[ "$LOOKBACK_HOURS" =~ ^[0-9]+$ && "$LOOKBACK_HOURS" -gt 0 ]] \
|| fail "E2E_USAGE_LOOKBACK_HOURS must be a positive integer"
[[ "$MIN_REQUESTS" =~ ^[0-9]+$ && "$MIN_REQUESTS" -gt 0 ]] \
|| fail "E2E_USAGE_MIN_REQUESTS must be a positive integer"
[[ "$ROLLUP_WAIT_SECONDS" =~ ^[0-9]+$ ]] \
|| fail "E2E_USAGE_ROLLUP_WAIT_SECONDS must be a non-negative integer"
[[ "$EVENT_WAIT_SECONDS" =~ ^[0-9]+$ ]] \
|| fail "E2E_USAGE_EVENT_WAIT_SECONDS must be a non-negative integer"
sql_literal() {
local value="${1//\'/\'\'}"
printf "'%s'" "$value"
}
RECIPE_LIKE_SQL=$(sql_literal "$RECIPE_LIKE")
UUID_RE="'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'"
pg_psql() {
local sql="$1"
local pod
pod=$($KC -n "$PG_NS" get pod -l "$PG_SEL" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
[[ -n "$pod" ]] || fail "postgres pod not found (ns=$PG_NS sel=$PG_SEL)"
$KC -n "$PG_NS" exec "$pod" -- psql -U postgres -d "$PG_DB" -v ON_ERROR_STOP=1 -tAc "$sql"
}
curl_config_quote() {
local value="$1"
value="${value//\\/\\\\}"
value="${value//\"/\\\"}"
printf '%s' "$value"
}
HTTP_STATUS=""
HTTP_BODY=""
http_json() {
local method="$1"
local url="$2"
local body="${3:-}"
shift 3 2>/dev/null || shift $#
local cfg body_file raw rc
cfg=$(mktemp "${TMPDIR:-/tmp}/clerum-usage-curl.XXXXXX")
chmod 600 "$cfg"
body_file=""
if [[ -n "$body" ]]; then
body_file=$(mktemp "${TMPDIR:-/tmp}/clerum-usage-body.XXXXXX")
chmod 600 "$body_file"
printf '%s' "$body" >"$body_file"
fi
{
printf 'silent\n'
printf 'write-out = "\\n%%{http_code}"\n'
printf 'max-time = 30\n'
printf 'request = "%s"\n' "$(curl_config_quote "$method")"
printf 'url = "%s"\n' "$(curl_config_quote "$url")"
printf 'header = "Content-Type: application/json"\n'
for hdr in "$@"; do
printf 'header = "%s"\n' "$(curl_config_quote "$hdr")"
done
[[ -n "$body_file" ]] && printf 'data-binary = "@%s"\n' "$(curl_config_quote "$body_file")"
} >"$cfg"
raw=$(curl --config "$cfg" 2>/dev/null)
rc=$?
rm -f "$cfg" "$body_file"
if [[ "$rc" -ne 0 ]]; then
HTTP_STATUS="000"
HTTP_BODY='{"error":"curl failed"}'
return 1
fi
HTTP_STATUS=$(echo "$raw" | tail -n1)
HTTP_BODY=$(echo "$raw" | sed '$d')
}
json_field() {
local json="$1"
local field="$2"
echo "$json" | node --no-warnings -e "
let d=''; process.stdin.on('data',c=>d+=c);
process.stdin.on('end',()=>{
try{
const o=JSON.parse(d);
const v=$field;
process.stdout.write(String(v===undefined||v===null?'':v));
}catch{process.stdout.write('');}
});
" 2>/dev/null
}
trigger_seeded_fixtures() {
log "Triggering seeded workflow recipes to generate fresh usage"
curl -sf --max-time 5 "${EXT_URL}/health" >/dev/null 2>&1 \
&& pass "external-rest-api reachable at $EXT_URL" \
|| fail "external-rest-api not reachable at $EXT_URL"
local login_body user_token user_id
login_body=$(TEST_EMAIL="$TEST_EMAIL" ADMIN_PASSWORD="$ADMIN_PASSWORD" node --no-warnings -e \
'process.stdout.write(JSON.stringify({email: process.env.TEST_EMAIL, password: process.env.ADMIN_PASSWORD}))')
http_json POST "${EXT_URL}/api/v1/auth/password-login" "$login_body"
[[ "$HTTP_STATUS" == "200" ]] || fail "password-login ${TEST_EMAIL} failed (HTTP $HTTP_STATUS): $HTTP_BODY"
user_token=$(json_field "$HTTP_BODY" "o.token")
user_id=$(json_field "$HTTP_BODY" "o.me && o.me.id")
[[ -n "$user_token" && -n "$user_id" ]] || fail "password-login response missing token/user id"
pass "password-login succeeded for usage trigger user"
local recipe idempotency body run_id status
for recipe in $TRIGGER_RECIPES; do
$KC -n sandbox-recipes get workflowrecipe "$recipe" >/dev/null 2>&1 \
|| fail "seeded WorkflowRecipe missing: sandbox-recipes/${recipe}"
idempotency="usage-attribution-${recipe}-$(date +%s)-${RANDOM}"
body='{"inputs":{"source":"usage-attribution-e2e"}}'
http_json POST "${EXT_URL}/api/v1/workflows/sandbox-recipes/${recipe}/trigger" "$body" \
"Authorization: Bearer ${user_token}" \
"Idempotency-Key: ${idempotency}"
status="$HTTP_STATUS"
[[ "$status" == "201" || "$status" == "200" ]] \
|| fail "trigger ${recipe} failed (HTTP $status): $HTTP_BODY"
run_id=$(json_field "$HTTP_BODY" "o.id")
[[ -n "$run_id" ]] || fail "trigger ${recipe} response missing run id"
pass "triggered ${recipe} (run_id=${run_id})"
done
# If SCHEDULE_RECIPE is also in TRIGGER_RECIPES, its live on-demand run makes
# the forced tick skip under concurrencyPolicy Forbid (the default).
if [[ -n "$SCHEDULE_RECIPE" ]]; then
local schedule_rows
schedule_rows=$(pg_psql "UPDATE workflow_schedules
SET next_fire_at = now() - interval '1 second', updated_at = now()
WHERE recipe_namespace='sandbox-recipes'
AND recipe_name='${SCHEDULE_RECIPE}'
AND team_id IS NOT NULL
RETURNING 1;" | wc -l | tr -d ' ')
[[ "$schedule_rows" -gt 0 ]] \
&& pass "forced scheduled fixture ${SCHEDULE_RECIPE} with non-null team snapshot" \
|| fail "scheduled fixture ${SCHEDULE_RECIPE} missing or has null team snapshot"
fi
}
wait_for_usage_events() {
local deadline event_count
deadline=$((SECONDS + EVENT_WAIT_SECONDS))
while true; do
event_count=$(pg_psql "SELECT COUNT(*)
FROM usage_events e
WHERE e.source_kind='workflow'
AND e.recipe_name LIKE ${RECIPE_LIKE_SQL}
AND e.ts >= now() - make_interval(hours => ${LOOKBACK_HOURS});" | tr -d ' \n')
if [[ "$event_count" -ge "$MIN_REQUESTS" || "$SECONDS" -ge "$deadline" ]]; then
EVENT_COUNT="$event_count"
return 0
fi
sleep 5
done
}
echo -e "${BOLD}=================================================================${NC}"
echo -e "${BOLD} Clerum E2E -- Workflow Usage Attribution${NC}"
echo -e "${BOLD}=================================================================${NC}"
log "Config: k8s_context=$K8S_CONTEXT postgres=$PG_NS/$PG_SEL db=$PG_DB"
log "Config: recipe_like=$RECIPE_LIKE lookback_hours=$LOOKBACK_HOURS min_requests=$MIN_REQUESTS"
echo ""
PG_PING=$(pg_psql "SELECT 1;" 2>/dev/null || echo "")
[[ "$PG_PING" == "1" ]] && pass "postgres reachable" || fail "postgres not reachable"
if [[ "$TRIGGER_FIXTURES" == "true" ]]; then
trigger_seeded_fixtures
wait_for_usage_events
else
EVENT_COUNT=$(pg_psql "SELECT COUNT(*)
FROM usage_events e
WHERE e.source_kind='workflow'
AND e.recipe_name LIKE ${RECIPE_LIKE_SQL}
AND e.ts >= now() - make_interval(hours => ${LOOKBACK_HOURS});" | tr -d ' \n')
fi
[[ "$EVENT_COUNT" -ge "$MIN_REQUESTS" ]] \
&& pass "workflow usage_events present ($EVENT_COUNT requests)" \
|| fail "workflow usage_events below minimum ($EVENT_COUNT < $MIN_REQUESTS)"
BAD_EVENT_COUNT=$(pg_psql "SELECT COUNT(*)
FROM usage_events e
WHERE e.source_kind='workflow'
AND e.recipe_name LIKE ${RECIPE_LIKE_SQL}
AND e.ts >= now() - make_interval(hours => ${LOOKBACK_HOURS})
AND (
e.team_id IS NULL
OR e.team_id !~* ${UUID_RE}
OR NOT EXISTS (SELECT 1 FROM teams t WHERE t.id::text = e.team_id)
OR e.llm_secret_name IS NULL
OR e.task_id IS NULL
OR split_part(e.task_id, ':', 1) !~* ${UUID_RE}
OR NOT (
EXISTS (SELECT 1 FROM workflow_runs wr WHERE wr.run_id::text = split_part(e.task_id, ':', 1))
OR EXISTS (SELECT 1 FROM workflow_runs_audit wra WHERE wra.run_id::text = split_part(e.task_id, ':', 1))
)
);" | tr -d ' \n')
[[ "$BAD_EVENT_COUNT" == "0" ]] \
&& pass "workflow usage_events have canonical team/secret/task attribution" \
|| fail "workflow usage_events with missing or invalid attribution: $BAD_EVENT_COUNT"
BAD_USER_EVENT_COUNT=$(pg_psql "WITH user_runs AS (
SELECT DISTINCT run_id::text AS run_id, actor_id::text AS actor_id
FROM workflow_runs
WHERE actor_type='user'
AND actor_id IS NOT NULL
UNION
SELECT DISTINCT run_id::text AS run_id, triggerer_user_id::text AS actor_id
FROM workflow_runs_audit
WHERE triggerer_actor_type='user'
AND triggerer_user_id IS NOT NULL
)
SELECT COUNT(*)
FROM usage_events e
JOIN user_runs r ON r.run_id = split_part(e.task_id, ':', 1)
WHERE e.source_kind='workflow'
AND e.recipe_name LIKE ${RECIPE_LIKE_SQL}
AND e.ts >= now() - make_interval(hours => ${LOOKBACK_HOURS})
AND COALESCE(e.user_id, '') <> r.actor_id;" | tr -d ' \n')
[[ "$BAD_USER_EVENT_COUNT" == "0" ]] \
&& pass "user-triggered workflow usage_events carry workflow actor user_id" \
|| fail "user-triggered workflow usage_events with missing or mismatched user_id: $BAD_USER_EVENT_COUNT"
ROLLUP_REQUESTS=0
deadline=$((SECONDS + ROLLUP_WAIT_SECONDS))
while true; do
ROLLUP_REQUESTS=$(pg_psql "SELECT COALESCE(SUM(request_count), 0)::bigint
FROM usage_5min
WHERE source_kind='workflow'
AND recipe_name LIKE ${RECIPE_LIKE_SQL}
AND bucket >= now() - make_interval(hours => ${LOOKBACK_HOURS});" | tr -d ' \n')
if [[ "$ROLLUP_REQUESTS" -ge "$MIN_REQUESTS" || "$SECONDS" -ge "$deadline" ]]; then
break
fi
sleep 5
done
[[ "$ROLLUP_REQUESTS" -ge "$MIN_REQUESTS" ]] \
&& pass "usage_5min rollup present ($ROLLUP_REQUESTS requests)" \
|| fail "usage_5min rollup did not materialize within ${ROLLUP_WAIT_SECONDS}s"
BAD_ROLLUP_COUNT=$(pg_psql "WITH bad AS (
SELECT request_count FROM usage_5min
WHERE source_kind='workflow'
AND recipe_name LIKE ${RECIPE_LIKE_SQL}
AND bucket >= now() - make_interval(hours => ${LOOKBACK_HOURS})
AND (team_id IS NULL OR llm_secret_name IS NULL)
UNION ALL
SELECT request_count FROM usage_hourly
WHERE source_kind='workflow'
AND recipe_name LIKE ${RECIPE_LIKE_SQL}
AND bucket >= now() - make_interval(hours => ${LOOKBACK_HOURS})
AND (team_id IS NULL OR llm_secret_name IS NULL)
UNION ALL
SELECT request_count FROM usage_daily
WHERE source_kind='workflow'
AND recipe_name LIKE ${RECIPE_LIKE_SQL}
AND bucket >= now() - make_interval(hours => ${LOOKBACK_HOURS})
AND (team_id IS NULL OR llm_secret_name IS NULL)
)
SELECT COALESCE(SUM(request_count), 0)::bigint FROM bad;" | tr -d ' \n')
[[ "$BAD_ROLLUP_COUNT" == "0" ]] \
&& pass "workflow rollups have non-null Team and LLM Secret dimensions" \
|| fail "workflow rollup requests with null Team or LLM Secret: $BAD_ROLLUP_COUNT"
log "Workflow usage attribution sample:"
pg_psql "SELECT COALESCE(t.name, e.team_id) AS team, e.llm_secret_name, e.model, e.recipe_name, COUNT(*) AS requests
FROM usage_events e
LEFT JOIN teams t ON t.id::text = e.team_id
WHERE e.source_kind='workflow'
AND e.recipe_name LIKE ${RECIPE_LIKE_SQL}
AND e.ts >= now() - make_interval(hours => ${LOOKBACK_HOURS})
GROUP BY 1, 2, 3, 4
ORDER BY requests DESC, team ASC, e.recipe_name ASC
LIMIT 20;"
echo ""
pass "workflow usage attribution gate complete"