Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
ci: show skipped tests as skipped instead of green pass (#1486)
mattteufel-e2b42538836f3a8
diff --git a/.github/workflows/cli_tests.yml b/.github/workflows/cli_tests.ymlindex 0e7a12072..47fd73c8e 100644--- a/.github/workflows/cli_tests.yml+++ b/.github/workflows/cli_tests.yml@@ -7,11 +7,6 @@ on:required: falsetype: stringdefault: ''- run:- description: 'Whether to actually run the build and tests. When false the matrix jobs are skipped (reported as success to required checks).'- required: false- type: boolean- default: truesecrets:E2B_API_KEY:required: true@@ -21,8 +16,6 @@ permissions:jobs:test:- # Gate each step (not the job) on `inputs.run` so the matrix always expands- # and the per-OS required checks are created even when skipped.defaults:run:working-directory: ./packages/cli@@ -34,11 +27,9 @@ jobs:runs-on: ${{ matrix.os }}steps:- name: Checkout repository- if: ${{ inputs.run }}uses: actions/checkout@v4- name: Parse .tool-versions- if: ${{ inputs.run }}uses: wistia/[email protected]with:filename: '.tool-versions'@@ -46,13 +37,11 @@ jobs:prefix: 'tool_version_'- name: Install pnpm- if: ${{ inputs.run }}uses: pnpm/action-setup@v4with:version: '${{ env.TOOL_VERSION_PNPM }}'- name: Setup Node- if: ${{ inputs.run }}uses: actions/setup-node@v3with:node-version: '${{ env.TOOL_VERSION_NODEJS }}'@@ -61,26 +50,21 @@ jobs:cache-dependency-path: pnpm-lock.yaml- name: Configure pnpm- if: ${{ inputs.run }}run: |pnpm config set auto-install-peers true- name: Install dependencies- if: ${{ inputs.run }}run: pnpm install --frozen-lockfile- name: Build the SDK (pre-requisite for the tests)- if: ${{ inputs.run }}run: pnpm buildworking-directory: ./packages/js-sdk- name: Build the CLI- if: ${{ inputs.run }}run: pnpm buildworking-directory: ./packages/cli- name: Run tests- if: ${{ inputs.run }}run: pnpm testworking-directory: ./packages/clienv:diff --git a/.github/workflows/js_sdk_tests.yml b/.github/workflows/js_sdk_tests.ymlindex 8e1702a03..e235b9b22 100644--- a/.github/workflows/js_sdk_tests.yml+++ b/.github/workflows/js_sdk_tests.yml@@ -7,11 +7,6 @@ on:required: falsetype: stringdefault: ''- run:- description: 'Whether to actually run the build and tests. When false the matrix jobs are skipped (reported as success to required checks).'- required: false- type: boolean- default: truesecrets:E2B_API_KEY:required: true@@ -21,8 +16,6 @@ permissions:jobs:test:- # Gate each step (not the job) on `inputs.run` so the matrix always expands- # and the per-OS required checks are created even when skipped.defaults:run:working-directory: ./packages/js-sdk@@ -35,11 +28,9 @@ jobs:runs-on: ${{ matrix.os }}steps:- name: Checkout repository- if: ${{ inputs.run }}uses: actions/checkout@v4- name: Parse .tool-versions- if: ${{ inputs.run }}uses: wistia/[email protected]with:filename: '.tool-versions'@@ -47,14 +38,12 @@ jobs:prefix: 'tool_version_'- name: Install pnpm- if: ${{ inputs.run }}uses: pnpm/action-setup@v4id: pnpm-installwith:version: '${{ env.TOOL_VERSION_PNPM }}'- name: Setup Node- if: ${{ inputs.run }}uses: actions/setup-node@v3with:node-version: '${{ env.TOOL_VERSION_NODEJS }}'@@ -63,18 +52,16 @@ jobs:cache-dependency-path: pnpm-lock.yaml- name: Configure pnpm- if: ${{ inputs.run }}run: |pnpm config set auto-install-peers truepnpm config set exclude-links-from-lockfile true- name: Install dependencies- if: ${{ inputs.run }}run: |pnpm install --frozen-lockfile- name: Cache Playwright binaries (Linux)- if: ${{ inputs.run && matrix.os == 'ubuntu-22.04' }}+ if: matrix.os == 'ubuntu-22.04'uses: actions/cache@v4with:path: ~/.cache/ms-playwright@@ -83,7 +70,7 @@ jobs:playwright-${{ runner.os }}-${{ env.TOOL_VERSION_NODEJS }}-- name: Cache Playwright binaries (Windows)- if: ${{ inputs.run && matrix.os == 'windows-latest' }}+ if: matrix.os == 'windows-latest'uses: actions/cache@v4with:path: ~/AppData/Local/ms-playwright@@ -92,35 +79,29 @@ jobs:playwright-${{ runner.os }}-${{ env.TOOL_VERSION_NODEJS }}-- name: Test build- if: ${{ inputs.run }}run: pnpm build- name: Run Node tests- if: ${{ inputs.run }}run: pnpm testenv:E2B_API_KEY: ${{ secrets.E2B_API_KEY }}E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}- name: Install Bun- if: ${{ inputs.run }}uses: oven-sh/setup-bun@v2- name: Run Bun tests- if: ${{ inputs.run }}run: pnpm test:bunenv:E2B_API_KEY: ${{ secrets.E2B_API_KEY }}E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}- name: Install Deno- if: ${{ inputs.run }}uses: denoland/setup-deno@v1with:deno-version: v${{ env.TOOL_VERSION_DENO }}- name: Run Deno tests- if: ${{ inputs.run }}run: pnpm test:denoenv:E2B_API_KEY: ${{ secrets.E2B_API_KEY }}diff --git a/.github/workflows/python_sdk_tests.yml b/.github/workflows/python_sdk_tests.ymlindex 1a656622a..6c74f777b 100644--- a/.github/workflows/python_sdk_tests.yml+++ b/.github/workflows/python_sdk_tests.yml@@ -7,11 +7,6 @@ on:required: falsetype: stringdefault: ''- run:- description: 'Whether to actually run the build and tests. When false the matrix jobs are skipped (reported as success to required checks).'- required: false- type: boolean- default: truesecrets:E2B_API_KEY:required: true@@ -21,8 +16,6 @@ permissions:jobs:test:- # Gate each step (not the job) on `inputs.run` so the matrix always expands- # and the per-OS required checks are created even when skipped.defaults:run:working-directory: ./packages/python-sdk@@ -35,11 +28,9 @@ jobs:runs-on: ${{ matrix.os }}steps:- name: Checkout repository- if: ${{ inputs.run }}uses: actions/checkout@v4- name: Parse .tool-versions- if: ${{ inputs.run }}uses: wistia/[email protected]with:filename: '.tool-versions'@@ -47,13 +38,11 @@ jobs:prefix: 'tool_version_'- name: Set up Python- if: ${{ inputs.run }}uses: actions/setup-python@v4with:python-version: '${{ env.TOOL_VERSION_PYTHON }}'- name: Install and configure Poetry- if: ${{ inputs.run }}uses: snok/install-poetry@v1with:version: '${{ env.TOOL_VERSION_POETRY }}'@@ -62,15 +51,12 @@ jobs:installer-parallel: true- name: Install dependencies- if: ${{ inputs.run }}run: poetry install- name: Test build- if: ${{ inputs.run }}run: poetry build- name: Run tests- if: ${{ inputs.run }}run: poetry run pytest --verbose --numprocesses=4env:E2B_API_KEY: ${{ secrets.E2B_API_KEY }}diff --git a/.github/workflows/sdk_tests.yml b/.github/workflows/sdk_tests.ymlindex 6dcc8ee37..290ba3c89 100644--- a/.github/workflows/sdk_tests.yml+++ b/.github/workflows/sdk_tests.yml@@ -23,8 +23,8 @@ jobs:steps:- name: Filter changed paths# On workflow_dispatch there is no PR diff and paths-filter would fall- # back to git (failing without a checkout); the run inputs below force a- # full run for that event instead, so we only filter on pull_request.+ # back to git (failing without a checkout); the job-level `if`s below+ # force a full run for that event instead, so we only filter on pull_request.if: github.event_name == 'pull_request'uses: dorny/paths-filter@v3id: filter@@ -56,56 +56,78 @@ jobs:js-tests:name: Production / JS SDK Testsneeds: changes+ if: ${{ needs.changes.outputs.js == 'true' || github.event_name == 'workflow_dispatch' }}uses: ./.github/workflows/js_sdk_tests.yml- with:- run: ${{ needs.changes.outputs.js == 'true' || github.event_name == 'workflow_dispatch' }}secrets:E2B_API_KEY: ${{ secrets.E2B_API_KEY }}python-tests:name: Production / Python SDK Testsneeds: changes+ if: ${{ needs.changes.outputs.python == 'true' || github.event_name == 'workflow_dispatch' }}uses: ./.github/workflows/python_sdk_tests.yml- with:- run: ${{ needs.changes.outputs.python == 'true' || github.event_name == 'workflow_dispatch' }}secrets:E2B_API_KEY: ${{ secrets.E2B_API_KEY }}cli-tests:name: Production / CLI Testsneeds: changes+ if: ${{ needs.changes.outputs.cli == 'true' || github.event_name == 'workflow_dispatch' }}uses: ./.github/workflows/cli_tests.yml- with:- run: ${{ needs.changes.outputs.cli == 'true' || github.event_name == 'workflow_dispatch' }}secrets:E2B_API_KEY: ${{ secrets.E2B_API_KEY }}js-tests-staging:name: Staging / JS SDK Testsneeds: changes+ if: ${{ needs.changes.outputs.js == 'true' || github.event_name == 'workflow_dispatch' }}uses: ./.github/workflows/js_sdk_tests.ymlwith:E2B_DOMAIN: ${{ vars.E2B_DOMAIN_STAGING }}- run: ${{ needs.changes.outputs.js == 'true' || github.event_name == 'workflow_dispatch' }}secrets:E2B_API_KEY: ${{ secrets.E2B_API_KEY_STAGING }}python-tests-staging:name: Staging / Python SDK Testsneeds: changes+ if: ${{ needs.changes.outputs.python == 'true' || github.event_name == 'workflow_dispatch' }}uses: ./.github/workflows/python_sdk_tests.ymlwith:E2B_DOMAIN: ${{ vars.E2B_DOMAIN_STAGING }}- run: ${{ needs.changes.outputs.python == 'true' || github.event_name == 'workflow_dispatch' }}secrets:E2B_API_KEY: ${{ secrets.E2B_API_KEY_STAGING }}cli-tests-staging:name: Staging / CLI Testsneeds: changes+ if: ${{ needs.changes.outputs.cli == 'true' || github.event_name == 'workflow_dispatch' }}uses: ./.github/workflows/cli_tests.ymlwith:E2B_DOMAIN: ${{ vars.E2B_DOMAIN_STAGING }}- run: ${{ needs.changes.outputs.cli == 'true' || github.event_name == 'workflow_dispatch' }}secrets:E2B_API_KEY: ${{ secrets.E2B_API_KEY_STAGING }}++ status:+ name: SDK Tests Status+ # `changes` is included so a failure in change-detection (which would skip the+ # suites) surfaces as a failed required check instead of a false green.+ needs:+ - changes+ - js-tests+ - python-tests+ - cli-tests+ if: ${{ always() }}+ runs-on: ubuntu-latest+ steps:+ - name: Report aggregate result+ run: |+ echo "Job results: ${{ join(needs.*.result, ', ') }}"+ if ${{ contains(needs.*.result, 'failure') }}; then+ echo "::error::Change detection or a Production SDK test suite failed."+ exit 1+ fi+ if ${{ contains(needs.*.result, 'cancelled') }}; then+ echo "::error::Change detection or a Production SDK test suite was cancelled; tests were not verified — re-run before merging."+ exit 1+ fi+ echo "All Production SDK test suites passed or were skipped."