Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Address E2E CI review: scan failures, cleanup errexit, issued credentials
Petr Bauchbe861e9dc22a
diff --git a/.github/workflows/e2e-scheduled.yml b/.github/workflows/e2e-scheduled.ymlindex d923cdeea..471588ceb 100644--- a/.github/workflows/e2e-scheduled.yml+++ b/.github/workflows/e2e-scheduled.yml@@ -8,9 +8,13 @@ name: E2E scheduled# nightly the provider lane again, and the fast lane against# dev, then one summary notification for the night#+# A failed run after a merge is reported too, since the lane is not a required+# check and nothing else would surface it before the night; a passing one is+# not, to keep the channel to one message a night.+## The fast lane on a throwaway stack is a PR check in pr-ci.yml.#-# The notification goes to the incoming webhook in the repository secret+# Notifications go to the incoming webhook in the repository secret# E2E_NOTIFY_WEBHOOK_URL (Slack and Mattermost both accept the payload). Without# it the run says so in a warning and sends nothing.@@ -19,9 +23,14 @@ on:- cron: '41 3 * * *'push:branches: [main]- paths-ignore:- - 'docs/**'- - '**/*.md'+ # Everything but documentation. The Switch skill is Markdown, but it is the+ # system context every real agent session runs with, so a change to it can+ # change whether the agent replies at all: later entries win.+ paths:+ - '**'+ - '!docs/**'+ - '!**/*.md'+ - 'console/packages/plugins/src/switch-skill/SKILL.md'workflow_dispatch:permissions:@@ -46,9 +55,9 @@ jobs:agent_type: claude-codenotify:- name: Nightly summary+ name: Reportneeds: [provider, fast-dev]- if: ${{ always() && github.event_name == 'schedule' }}+ if: ${{ always() && github.event_name != 'workflow_dispatch' }}runs-on: ubuntu-lateststeps:- name: Send the summary@@ -60,18 +69,28 @@ jobs:DEV_RESULT: ${{ needs.fast-dev.result }}DEV_SUMMARY: ${{ needs.fast-dev.outputs.summary }}run: |+ if [ "$GITHUB_EVENT_NAME" = push ] && [ "$PROVIDER_RESULT" = success ]; then+ echo "The provider lane passed after the merge; nothing to report."+ exit 0+ filine() {local mark=❌[ "$2" = success ] && mark=✅printf '%s %s — %s\n' "$mark" "$1" "${3:-$2, no scenario summary}"}- text="$(printf 'Switch E2E nightly\n%s\n%s\n%s' \- "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \- "$(line 'Fast lane (dev)' "$DEV_RESULT" "$DEV_SUMMARY")" \- "$RUN_URL")"+ if [ "$GITHUB_EVENT_NAME" = push ]; then+ text="$(printf 'Switch E2E after merging %s\n%s\n%s' "${GITHUB_SHA:0:9}" \+ "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \+ "$RUN_URL")"+ else+ text="$(printf 'Switch E2E nightly\n%s\n%s\n%s' \+ "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \+ "$(line 'Fast lane (dev)' "$DEV_RESULT" "$DEV_SUMMARY")" \+ "$RUN_URL")"+ fiecho "$text"if [ -z "$WEBHOOK" ]; then- echo "::warning::E2E_NOTIFY_WEBHOOK_URL is not set; the nightly summary was not sent anywhere."+ echo "::warning::E2E_NOTIFY_WEBHOOK_URL is not set; this report was not sent anywhere."exit 0ficurl -fsS -H 'Content-Type: application/json' -d "$(jq -n --arg text "$text" '{text: $text}')" "$WEBHOOK" >/dev/nulldiff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlindex 31a6eea3c..aaf9e5cc0 100644--- a/.github/workflows/e2e.yml+++ b/.github/workflows/e2e.yml@@ -167,6 +167,7 @@ jobs:MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}+ SWITCH_E2E_SECRETS_FILE: ${{ runner.temp }}/issued-secretsrun: |mkdir -p "$OUT"pnpm exec vitest run --reporter=verbose --reporter=junit --outputFile.junit="$OUT/junit.xml" \@@ -188,6 +189,7 @@ jobs:MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}+ SWITCH_E2E_SECRETS_FILE: ${{ runner.temp }}/issued-secretsrun: pnpm cleanup --older-than 180- name: Summarise the run@@ -207,12 +209,17 @@ jobs:run: |[ -d "$OUT" ] || { echo "No output to scan."; exit 1; }if [ "$TARGET" = ephemeral ]; then- # The stack script scans for the secrets it generated and the- # provider key file, and leaves this file only when none turned up.+ # The stack script scans for the secrets it generated, the provider+ # key file and every credential the server issued the harness, and+ # leaves this file only when none turned up.[ -e "$OUT/.secret-scan-passed" ] || { echo "::error::The stack's own secret scan did not pass."; exit 1; }else- printf '%s\n' "$SWITCH_GATEWAY_ADMIN_PASSWORD" "$SWITCH_AGENT_REGISTRATION_TOKEN" "$MATTERMOST_TOKEN" \- | console/tooling-e2e/scripts/scan-artifacts.sh "$OUT"+ # The environment's own secrets, and every credential the server+ # issued the harness during the run (agent keys, gateway sessions),+ # which stay valid on a shared deployment after the run ends.+ { printf '%s\n' "$SWITCH_GATEWAY_ADMIN_PASSWORD" "$SWITCH_AGENT_REGISTRATION_TOKEN" "$MATTERMOST_TOKEN"+ cat "$RUNNER_TEMP/issued-secrets" 2>/dev/null || true+ } | console/tooling-e2e/scripts/scan-artifacts.sh "$OUT"fidocker run --rm -v "$OUT:/out:ro" zricethezav/gitleaks:v8.30.1 dir /out --redact --no-bannerdiff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.mdindex 71b82c4b3..c090e3098 100644--- a/console/tooling-e2e/README.md+++ b/console/tooling-e2e/README.md@@ -151,6 +151,7 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.| `MATTERMOST_TEAM` | for Mattermost scenarios | Team **slug** (e.g. `switch`), not its display name. || `SWITCH_E2E_STRICT` | no | `1` turns every "cannot run" into a failure. CI sets it. || `SWITCH_E2E_KEEP` | no | `1` leaves agents, rooms and channels behind for inspection. |+| `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script and `e2e.yml` set it. || `SWITCH_E2E_AGENT_DIR` | provider lane | Working directory of the session under test. When set, the agent's credentials are written into it (see below) and it is sent to Switch as the agent's `repo_dir`. || `SWITCH_E2E_MANIFEST` | provider lane | File the setup records its agent and room in, so a second process can reuse them. See "Seeding first". || `SWITCH_E2E_REPLY_TIMEOUT_MS` | provider lane | How long a scenario waits for the agent (default 5 min). |@@ -236,13 +237,17 @@ dispatched by hand.behind. After every deployed run, `pnpm cleanup --older-than 180` removeswhat interrupted runs left.- **Artifacts.** The output directory is uploaded only after it passes two- scans: one for the exact secret values the run used (`scan-artifacts.sh`),- and gitleaks. On an ephemeral stack, the env file, the controller's state and+ scans. The first (`scan-artifacts.sh`) looks for the exact secret values the+ run used: the ones it was configured with, plus every credential the server+ issued the harness during the run (agent API keys, gateway sessions). The+ harness records the issued ones in `SWITCH_E2E_SECRETS_FILE`. The second is+ gitleaks. A file the first scan cannot read fails it. On an ephemeral stack, the env file, the controller's state andthe agent's directory live outside the output directory, so the output holdsonly logs and the JUnit report.-- **Reporting.** Each run writes a scenario table to the job summary. A- scheduled night sends one summary to the webhook in the repository secret- `E2E_NOTIFY_WEBHOOK_URL`.+- **Reporting.** Each run writes a scenario table to the job summary. Reports+ go to the webhook in the repository secret `E2E_NOTIFY_WEBHOOK_URL`:+ - one summary each night;+ - a report when the provider lane fails after a merge.## With a Switch Console sessiondiff --git a/console/tooling-e2e/scripts/ephemeral-stack.sh b/console/tooling-e2e/scripts/ephemeral-stack.shindex 2c149d8df..8fed28270 100755--- a/console/tooling-e2e/scripts/ephemeral-stack.sh+++ b/console/tooling-e2e/scripts/ephemeral-stack.sh@@ -67,6 +67,10 @@ SERVER_PID=""CONTROLLER_PID=""cleanup() {local status=$?+ # Every step runs whatever the one before it did: a process that exits+ # non-zero on SIGTERM must not skip the teardown, the scan or the removal+ # of $WORK.+ set +eif [ "$KEEP" = 0 ]; then[ -n "$CONTROLLER_PID" ] && kill "$CONTROLLER_PID" 2>/dev/null && wait "$CONTROLLER_PID" 2>/dev/null[ -n "$SERVER_PID" ] && kill "$SERVER_PID" 2>/dev/null && wait "$SERVER_PID" 2>/dev/null@@ -98,9 +102,15 @@ chmod 600 "$ENV_FILE"set -a; source "$ENV_FILE"; set +a# A secret's value, and each part of it, since SECRET_KEYS is `label:key,…`.+# One layer of surrounding quotes is dropped, as the env file's reader drops it:+# the logs hold the bare value.remember_secrets() {local valuefor value in "$@"; do+ case "$value" in+ \"*\") value="${value#\"}"; value="${value%\"}" ;;+ \'*\') value="${value#\'}"; value="${value%\'}" ;;+ esacprintf '%s\n' "$value" >>"$SECRETS"tr ',:' '\n\n' <<<"$value" >>"$SECRETS"done@@ -207,6 +217,8 @@ status=0(cd "$HARNESS"if [ ${#PROVIDER_ENV[@]} -gt 0 ]; then export "${PROVIDER_ENV[@]}"; fi+ # The credentials the harness is issued, for the scan on exit.+ export SWITCH_E2E_SECRETS_FILE="$SECRETS"SWITCH_E2E_LANE="$LANE" \SWITCH_E2E_ENVIRONMENT=ephemeral \SWITCH_E2E_STRICT=1 \diff --git a/console/tooling-e2e/scripts/scan-artifacts.sh b/console/tooling-e2e/scripts/scan-artifacts.shindex 8164c6499..c1499e0ed 100755--- a/console/tooling-e2e/scripts/scan-artifacts.sh+++ b/console/tooling-e2e/scripts/scan-artifacts.sh@@ -28,9 +28,20 @@ if [ ! -s "$PATTERNS" ]; thenexit 2fi-if LEAKS="$(grep -rlF -f "$PATTERNS" -- "$DIR")"; then- echo "scan-artifacts: these files contain a secret value from this run; nothing will be uploaded:" >&2- sed 's/^/ /' <<<"$LEAKS" >&2- exit 1-fi+# grep exits 1 for no match, but 2 for a read error even when it also matched,+# so only 1 passes: a file the scan could not read is not a file known clean.+set +e+LEAKS="$(grep -rlF -f "$PATTERNS" -- "$DIR")"+status=$?+set -e+case "$status" in+ 0)+ echo "scan-artifacts: these files contain a secret value from this run; nothing will be uploaded:" >&2+ sed 's/^/ /' <<<"$LEAKS" >&2+ exit 1 ;;+ 1) ;;+ *)+ echo "scan-artifacts: grep failed (exit $status) under $DIR; refusing to pass the scan" >&2+ exit 2 ;;+esacecho "scan-artifacts: $(grep -c '' "$PATTERNS") secret values, none found under $DIR."diff --git a/console/tooling-e2e/src/clients/switch-client.ts b/console/tooling-e2e/src/clients/switch-client.tsindex 1d42e853c..6cf445f99 100644--- a/console/tooling-e2e/src/clients/switch-client.ts+++ b/console/tooling-e2e/src/clients/switch-client.ts@@ -1,5 +1,6 @@import type { Environment, SwitchEnv } from '../core/env.ts';import { assertAllowed } from '../core/guard.ts';+import type { IssuedSecrets } from '../core/secrets.ts';/*** Thin client for the two Switch HTTP surfaces the harness talks to, both served@@ -144,14 +145,16 @@ export class SwitchClient {private readonly adminEmail: string;private readonly adminPassword: string;private readonly environment: Environment;+ private readonly issued: IssuedSecrets;private gatewayCookie: string | null = null;- constructor(env: SwitchEnv, environment: Environment) {+ constructor(env: SwitchEnv, environment: Environment, issued: IssuedSecrets) {this.apiUrl = env.apiUrl;this.registrationToken = env.agentRegistrationToken;this.adminEmail = env.gatewayAdminEmail;this.adminPassword = env.gatewayAdminPassword;this.environment = environment;+ this.issued = issued;}// ── plumbing ──────────────────────────────────────────────────────────────@@ -220,6 +223,7 @@ export class SwitchClient {'authenticates with a session cookie, not a bearer token.');}+ this.issued.record(cookie.slice('switch_auth='.length));this.gatewayCookie = cookie;return cookie;}@@ -436,6 +440,7 @@ export class SwitchClient {},});+ this.issued.record(body.api_key);return { id: body.id, name: params.name, apiKey: body.api_key };}diff --git a/console/tooling-e2e/src/core/core.test.ts b/console/tooling-e2e/src/core/core.test.tsindex 757bd3544..0912da1e2 100644--- a/console/tooling-e2e/src/core/core.test.ts+++ b/console/tooling-e2e/src/core/core.test.ts@@ -1,3 +1,6 @@+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import path from 'node:path';import { describe, expect, it } from 'vitest';import { olderThanFrom, timestampMs } from '../cleanup.ts';import { FAST_SCENARIOS } from '../fast/index.ts';@@ -12,6 +15,7 @@ import {laneOf,type ScenarioHeader,} from './scenario.ts';+import { IssuedSecrets } from './secrets.ts';const BASE = {SWITCH_API_URL: 'http://switch.invalid/',@@ -184,3 +188,20 @@ describe('cleanup age floor', () => {expect(() => olderThanFrom(['--older-than', '-1'])).toThrow(/--older-than/);});});++describe('issued secrets', () => {+ it('appends each credential on its own line, and records nothing without a file', () => {+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-secrets-'));+ try {+ const file = path.join(dir, 'issued');+ const issued = new IssuedSecrets(file);+ issued.record('agent-key-0123456789');+ issued.record('');+ issued.record('cookie-value-abcdef');+ expect(readFileSync(file, 'utf8')).toBe('agent-key-0123456789\ncookie-value-abcdef\n');+ expect(() => new IssuedSecrets(null).record('ignored-value')).not.toThrow();+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });+});diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.tsindex 8a869d1a2..04c096591 100644--- a/console/tooling-e2e/src/core/env.ts+++ b/console/tooling-e2e/src/core/env.ts@@ -95,6 +95,8 @@ export interface HarnessEnv {strict: boolean;/** Leave created agents, rooms and channels behind after the run (for debugging). */keepArtifacts: boolean;+ /** Where to record the credentials the server issues during the run; see `IssuedSecrets`. */+ secretsFile: string | null;switch: SwitchEnv;/** `null` when no Mattermost is configured; scenarios that need one then skip or fail. */mattermost: MattermostEnv | null;@@ -230,6 +232,7 @@ export function loadEnv(environment: environmentFrom(required.SWITCH_E2E_ENVIRONMENT as string),strict: strictFrom(source, file),keepArtifacts: get('SWITCH_E2E_KEEP') === '1',+ secretsFile: get('SWITCH_E2E_SECRETS_FILE') ?? null,switch: {apiUrl: stripTrailingSlash(required.SWITCH_API_URL as string),gatewayAdminEmail: required.SWITCH_GATEWAY_ADMIN_EMAIL as string,diff --git a/console/tooling-e2e/src/core/secrets.ts b/console/tooling-e2e/src/core/secrets.tsnew file mode 100644index 000000000..ae495bea3--- /dev/null+++ b/console/tooling-e2e/src/core/secrets.ts@@ -0,0 +1,21 @@+import { appendFileSync } from 'node:fs';++/**+ * Credentials the server issues during a run — agent API keys, gateway session+ * cookies — written one per line to `SWITCH_E2E_SECRETS_FILE`, so CI can scan+ * the run's output for them before uploading it. On a long-lived deployment+ * they outlive the run, and an opaque token is exactly what a pattern scanner+ * misses. Without the variable nothing is written: a local run uploads nothing.+ */+export class IssuedSecrets {+ private readonly path: string | null;++ constructor(path: string | null) {+ this.path = path;+ }++ record(value: string): void {+ if (this.path === null || value === '') return;+ appendFileSync(this.path, `${value}\n`, { encoding: 'utf8', mode: 0o600 });+ }+}diff --git a/console/tooling-e2e/src/core/suite.ts b/console/tooling-e2e/src/core/suite.tsindex 1fc95b682..01d61f3bd 100644--- a/console/tooling-e2e/src/core/suite.ts+++ b/console/tooling-e2e/src/core/suite.ts@@ -8,6 +8,7 @@ import {type HarnessEnv,type Lane,} from './env.ts';+import { IssuedSecrets } from './secrets.ts';/*** Whether this process runs a lane's suite at all, and the clients it runs with.@@ -51,7 +52,11 @@ export async function resolveLane(lane: Lane): Promise<LaneResolution> {return unable(message(error));}- const switchClient = new SwitchClient(env.switch, env.environment);+ const switchClient = new SwitchClient(+ env.switch,+ env.environment,+ new IssuedSecrets(env.secretsFile)+ );try {await switchClient.health();} catch (error) {