Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Prepare Shoppy Sample App GH job to run synthetic monitoring tests (#69492)

Timofey Kachalov890771e40eee
diff --git a/.github/actions/prepare-shoppy-e2e/action.yml b/.github/actions/prepare-shoppy-e2e/action.yml
new file mode 100644
index 00000000000..7ba2a954a65
--- /dev/null
+++ b/.github/actions/prepare-shoppy-e2e/action.yml
@@ -0,0 +1,132 @@
+name: Prepare Shoppy E2E environment
+description: Shared setup steps for Shoppy E2E tests (regular and synthetic)
+
+inputs:
+ commit:
+ description: "Commit SHA for artifact retrieval"
+ required: true
+ sample-app-branch-name:
+ description: "Branch name for the sample app"
+ required: true
+ sample-app-environment:
+ description: "Environment (production/development)"
+ required: true
+ staging-mb-all-features-token:
+ description: "Staging MB all features token"
+ required: true
+ shoppy-iam-role:
+ description: "Shoppy IAM role for AWS credentials"
+ required: true
+ shoppy-metabase-app-db-s3-bucket:
+ description: "S3 bucket for Shoppy app DB dump"
+ required: true
+ shoppy-metabase-app-db-s3-key:
+ description: "S3 key for Shoppy app DB dump"
+ required: true
+ shoppy-metabase-app-db-user:
+ description: "Shoppy Metabase app DB user"
+ required: true
+ shoppy-metabase-app-db-database:
+ description: "Shoppy Metabase app DB database"
+ required: true
+ metastore-dev-server-url:
+ description: "Metastore dev server URL"
+ required: true
+ shoppy-datadog-application-id:
+ description: "Shoppy Datadog application ID"
+ required: true
+ shoppy-datadog-client-token:
+ description: "Shoppy Datadog client token"
+ required: true
+
+outputs:
+ chrome-path:
+ description: "Path to Chrome executable"
+ value: ${{ steps.cypress-prep.outputs.chrome-path }}
+ sample_app_branch_exists:
+ description: "Whether the sample app branch exists"
+ value: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists }}
+
+runs:
+ using: "composite"
+ steps:
+ - uses: ./.github/actions/maximize-disk-space
+
+ # We have to use the latest version that does not have this issue https://github.com/docker/compose/pull/12752
+ - name: Set up Docker Compose
+ uses: docker/setup-compose-action@v1
+ with:
+ version: latest
+
+ - name: Retrieve uberjar artifact for ee
+ uses: ./.github/actions/fetch-artifact
+ with:
+ commit: ${{ inputs.commit }}
+ edition: ee
+ extract-path: "target/uberjar"
+ output-name: "metabase.jar"
+
+ - name: Retrieve Embedding SDK dist artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: embedding-sdk-${{ inputs.commit }}
+ path: ${{ github.workspace }}/resources/embedding-sdk
+
+ - name: Prepare front-end environment
+ uses: ./.github/actions/prepare-frontend
+
+ - name: Prepare Cypress environment
+ id: cypress-prep
+ uses: ./.github/actions/prepare-cypress
+
+ - name: Check if sample app branch exists
+ id: check-sample-app-branch
+ shell: bash
+ env:
+ SAMPLE_APP_BRANCH_NAME: ${{ inputs.sample-app-branch-name }}
+ run: |
+ REPO_URL="https://github.com/metabase/shoppy.git"
+ echo "Checking if branch '${SAMPLE_APP_BRANCH_NAME}' exists in ${REPO_URL}..."
+ if git ls-remote --exit-code --heads "$REPO_URL" "${SAMPLE_APP_BRANCH_NAME}"; then
+ echo "sample_app_branch_exists=true" >> $GITHUB_OUTPUT
+ else
+ echo "sample_app_branch_exists=false" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Configure AWS credentials
+ if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ role-to-assume: ${{ inputs.shoppy-iam-role }}
+ role-session-name: GitHub_to_AWS_via_FederatedOIDC
+ aws-region: us-east-1
+
+ - name: Download Shoppy's App DB PostgreSQL dump from S3
+ if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}
+ shell: bash
+ run: |
+ aws s3 cp s3://${{ inputs.shoppy-metabase-app-db-s3-bucket }}/${{ inputs.shoppy-metabase-app-db-s3-key }} ./e2e/tmp/db_dumps/shoppy_metabase_app_db_dump.sql
+
+ - name: Prepare and launch Shoppy containers
+ if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}
+ shell: bash
+ env:
+ SAMPLE_APP_BRANCH_NAME: ${{ inputs.sample-app-branch-name }}
+ SAMPLE_APP_ENVIRONMENT: ${{ inputs.sample-app-environment }}
+ CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ inputs.staging-mb-all-features-token }}
+ METASTORE_DEV_SERVER_URL: ${{ inputs.metastore-dev-server-url }}
+ METABASE_APP_DB_USER: ${{ inputs.shoppy-metabase-app-db-user }}
+ METABASE_APP_DB: ${{ inputs.shoppy-metabase-app-db-database }}
+ SHOPPY_DATADOG_APPLICATION_ID: ${{ inputs.shoppy-datadog-application-id }}
+ SHOPPY_DATADOG_CLIENT_TOKEN: ${{ inputs.shoppy-datadog-client-token }}
+ SHOPPY_DATADOG_SITE: "us5.datadoghq.com"
+ SHOPPY_DATADOG_ENV: "staging"
+ run: npx tsx ./e2e/runner/embedding-sdk/sample-apps/start-ci.ts shoppy-e2e
+
+ - name: Prune build cache
+ shell: bash
+ run: docker builder prune -af
+
+ - name: Log free disk space
+ shell: bash
+ run: df -h /
diff --git a/.github/workflows/embedding-sdk-host-sample-apps-tests.yml b/.github/workflows/embedding-sdk-host-sample-apps-tests.yml
index 4e3b06594b0..16efeca80a4 100644
--- a/.github/workflows/embedding-sdk-host-sample-apps-tests.yml
+++ b/.github/workflows/embedding-sdk-host-sample-apps-tests.yml
@@ -9,7 +9,6 @@ env:
TZ: US/Pacific # to make Node match the instance tz
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
HASH: ${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }}
- PR_NUMBER: ${{ github.event.pull_request.number || '' }}
MB_EDITION: ee
jobs:
get-sample-apps-data:
@@ -21,12 +20,7 @@ jobs:
TEST_SUITES: |
[
"metabase-nodejs-react-sdk-embedding-sample-e2e",
- "metabase-nextjs-sdk-embedding-sample-e2e",
- "shoppy-e2e"
- ]
- TEST_SUITES_TO_RUN_IN_MASTER_ONLY: |
- [
- "shoppy-e2e"
+ "metabase-nextjs-sdk-embedding-sample-e2e"
]
SAMPLE_APP_MAIN_BRANCH_NAME: "main"
SAMPLE_APP_STABLE_RELEASE_BRANCH_NAME_SUFFIX: "stable"
@@ -50,7 +44,6 @@ jobs:
} = require('${{ github.workspace }}/release/dist/index.cjs');
const testSuites = JSON.parse(process.env.TEST_SUITES);
- const testSuitesToRunInMasterOnly = JSON.parse(process.env.TEST_SUITES_TO_RUN_IN_MASTER_ONLY);
const ref = '${{ github.base_ref || github.ref }}';
const branchName = ref.replace(/^refs\/heads\//, '');
@@ -74,19 +67,16 @@ jobs:
const sampleAppBranchName = `${currentBranchVersion}-${{ env.SAMPLE_APP_STABLE_RELEASE_BRANCH_NAME_SUFFIX }}`
const shouldRunTestsForVersion = (currentReleaseVersion - currentBranchVersion) <= (supportedReleasesCount - 1);
- const releaseBranchTestSuites = testSuites.filter(testSuite => !testSuitesToRunInMasterOnly.includes(testSuite));
return {
branch_name: sampleAppBranchName,
- matrix: shouldRunTestsForVersion
- ? releaseBranchTestSuites
- : []
+ matrix: shouldRunTestsForVersion ? testSuites : [],
};
}
return {
branch_name: '',
- matrix: []
+ matrix: [],
};
# In order to check compatibility between Sample Apps and Embedding SDK, the job for each sample app:
@@ -168,28 +158,8 @@ jobs:
echo "sample_app_branch_exists=false" >> $GITHUB_OUTPUT
fi
- - name: Configure AWS credentials
- if: ${{ matrix.test_suite == 'shoppy-e2e' }}
- uses: aws-actions/configure-aws-credentials@v4
- with:
- role-to-assume: ${{ secrets.SHOPPY_IAM_ROLE }}
- role-session-name: GitHub_to_AWS_via_FederatedOIDC
- aws-region: us-east-1
-
- - name: Download Shoppy's App DB PostgreSQL dump from S3
- if: ${{ matrix.test_suite == 'shoppy-e2e' }}
- run: |
- aws s3 cp s3://${{ secrets.SHOPPY_METABASE_APP_DB_S3_BUCKET }}/${{ secrets.SHOPPY_METABASE_APP_DB_S3_KEY }} ./e2e/tmp/db_dumps/shoppy_metabase_app_db_dump.sql
-
- - name: Prepare and launch Sample Apps for shoppy-e2e test suite
- if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' && matrix.test_suite == 'shoppy-e2e' }}
- env:
- METABASE_APP_DB_USER: ${{ secrets.SHOPPY_METABASE_APP_DB_USER }}
- METABASE_APP_DB: ${{ secrets.SHOPPY_METABASE_APP_DB_DATABASE }}
- run: npx tsx ./e2e/runner/embedding-sdk/sample-apps/start-ci.ts shoppy-e2e
-
- - name: Prepare and launch Sample Apps for other test suites
- if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' && matrix.test_suite != 'shoppy-e2e' }}
+ - name: Prepare and launch Sample Apps
+ if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}
run: npx tsx ./e2e/runner/embedding-sdk/sample-apps/start-ci.ts ${{ matrix.test_suite }}
- name: Prune build cache
@@ -210,7 +180,7 @@ jobs:
uses: actions/upload-artifact@v4
if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
with:
- name: cypress-recording-sample-app-${{ matrix.test_suite }}-latest
+ name: cypress-recording-sample-app-${{ matrix.test_suite }}-${{ matrix.environment }}-latest
path: |
./e2e/tmp/${{ steps.get-sample-app-name.outputs.sample_app_name }}/cypress
if-no-files-found: ignore
@@ -220,6 +190,149 @@ jobs:
run: |
exit 1
+ # Shoppy e2e tests run separately using the shared composite action for setup.
+ e2e-shoppy-tests:
+ needs: [get-sample-apps-data]
+ strategy:
+ matrix:
+ environment: ["production", "development"]
+ if: |
+ !cancelled() && needs.get-sample-apps-data.result == 'success' &&
+ (github.ref == 'refs/heads/master' || github.base_ref == 'master')
+ runs-on: ubuntu-22.04
+ timeout-minutes: 45
+ name: e2e-sample-app-shoppy-e2e-tests-${{ matrix.environment }}-bundle
+ env:
+ MB_EDITION: ee
+ MB_RUN_MODE: e2e
+ METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
+ CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
+ SAMPLE_APP_BRANCH_NAME: ${{ needs.get-sample-apps-data.outputs.branch_name }}
+ SAMPLE_APP_ENVIRONMENT: ${{ matrix.environment }}
+ permissions:
+ id-token: write
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Prepare Shoppy E2E environment
+ uses: ./.github/actions/prepare-shoppy-e2e
+ id: shoppy-prep
+ with:
+ commit: ${{ github.event.pull_request.head.sha || github.sha }}
+ sample-app-branch-name: ${{ needs.get-sample-apps-data.outputs.branch_name }}
+ sample-app-environment: ${{ matrix.environment }}
+ staging-mb-all-features-token: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
+ shoppy-iam-role: ${{ secrets.SHOPPY_IAM_ROLE }}
+ shoppy-metabase-app-db-s3-bucket: ${{ secrets.SHOPPY_METABASE_APP_DB_S3_BUCKET }}
+ shoppy-metabase-app-db-s3-key: ${{ secrets.SHOPPY_METABASE_APP_DB_S3_KEY }}
+ shoppy-metabase-app-db-user: ${{ secrets.SHOPPY_METABASE_APP_DB_USER }}
+ shoppy-metabase-app-db-database: ${{ secrets.SHOPPY_METABASE_APP_DB_DATABASE }}
+ metastore-dev-server-url: ${{ secrets.METASTORE_DEV_SERVER_URL }}
+ shoppy-datadog-application-id: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_APPLICATION_ID }}
+ shoppy-datadog-client-token: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_CLIENT_TOKEN }}
+
+ - name: Run e2e tests for Shoppy
+ id: run-e2e-tests
+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}
+ continue-on-error: true
+ env:
+ CYPRESS_BROWSER: ${{ steps.shoppy-prep.outputs.chrome-path }}
+ run: node e2e/runner/run_cypress_ci.js shoppy-e2e
+
+ - name: Upload Cypress Artifacts upon failure
+ uses: actions/upload-artifact@v4
+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
+ with:
+ name: cypress-recording-sample-app-shoppy-e2e-${{ matrix.environment }}-latest
+ path: |
+ ./e2e/tmp/shoppy/cypress
+ if-no-files-found: ignore
+
+ - name: Fail job if tests failed
+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
+ run: |
+ exit 1
+
+ # Synthetic monitoring tests run only on master merges, in parallel with the regular e2e tests.
+ # They use the shoppy production environment and include performance measurement tests.
+ e2e-shoppy-synthetic-tests:
+ needs: [get-sample-apps-data]
+ if: |
+ !cancelled() && needs.get-sample-apps-data.result == 'success' &&
+ github.event_name == 'push' &&
+ github.ref == 'refs/heads/master'
+ runs-on: ubuntu-22.04
+ timeout-minutes: 45
+ name: e2e-shoppy-synthetic-monitoring-tests
+ env:
+ MB_EDITION: ee
+ MB_RUN_MODE: e2e
+ METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
+ CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
+ SAMPLE_APP_BRANCH_NAME: ${{ needs.get-sample-apps-data.outputs.branch_name }}
+ SAMPLE_APP_ENVIRONMENT: production
+ permissions:
+ id-token: write
+ contents: read
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Prepare Shoppy E2E environment
+ uses: ./.github/actions/prepare-shoppy-e2e
+ id: shoppy-prep
+ with:
+ commit: ${{ github.event.pull_request.head.sha || github.sha }}
+ sample-app-branch-name: ${{ needs.get-sample-apps-data.outputs.branch_name }}
+ sample-app-environment: production
+ staging-mb-all-features-token: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
+ shoppy-iam-role: ${{ secrets.SHOPPY_IAM_ROLE }}
+ shoppy-metabase-app-db-s3-bucket: ${{ secrets.SHOPPY_METABASE_APP_DB_S3_BUCKET }}
+ shoppy-metabase-app-db-s3-key: ${{ secrets.SHOPPY_METABASE_APP_DB_S3_KEY }}
+ shoppy-metabase-app-db-user: ${{ secrets.SHOPPY_METABASE_APP_DB_USER }}
+ shoppy-metabase-app-db-database: ${{ secrets.SHOPPY_METABASE_APP_DB_DATABASE }}
+ metastore-dev-server-url: ${{ secrets.METASTORE_DEV_SERVER_URL }}
+ shoppy-datadog-application-id: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_APPLICATION_ID }}
+ shoppy-datadog-client-token: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_CLIENT_TOKEN }}
+
+ - name: Get merged PR number
+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}
+ id: get-merged-pr-number
+ run: |
+ PR=$(gh api repos/${{ github.repository }}/commits/${{ github.sha }}/pulls --jq '.[0].number' 2>/dev/null || echo "")
+ if [ -z "$PR" ] || [ "$PR" = "null" ]; then
+ echo "number=" >> "$GITHUB_OUTPUT"
+ else
+ echo "number=$PR" >> "$GITHUB_OUTPUT"
+ fi
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Run synthetic monitoring tests
+ id: run-e2e-tests
+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}
+ continue-on-error: true
+ env:
+ CYPRESS_BROWSER: ${{ steps.shoppy-prep.outputs.chrome-path }}
+ PR_NUMBER: ${{ steps.get-merged-pr-number.outputs.number }}
+ INCLUDE_SYNTHETIC_MONITORING_TESTS: "true"
+ run: node e2e/runner/run_cypress_ci.js shoppy-e2e
+
+ - name: Upload Cypress Artifacts upon failure
+ uses: actions/upload-artifact@v4
+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
+ with:
+ name: cypress-recording-shoppy-synthetic-tests-latest
+ path: |
+ ./e2e/tmp/shoppy/cypress
+ if-no-files-found: ignore
+
+ - name: Fail job if tests failed
+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
+ run: |
+ exit 1
+
# The job checks integration of the Embedding SDK with consumer's apps that use different frameworks/bundlers,
# or when we want to test some tricky integration cases like conflicting types.
# The job
diff --git a/e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.js b/e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.js
index 78cd752fab7..db2f68e19dd 100644
--- a/e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.js
+++ b/e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.js
@@ -53,6 +53,14 @@ export const SAMPLE_APP_SETUP_CONFIGS = {
},
"shoppy-e2e": {
...BASE_SETUP_CONFIG,
+ env: {
+ ...BASE_SETUP_CONFIG.env,
+ DATADOG_APPLICATION_ID: process.env.SHOPPY_DATADOG_APPLICATION_ID,
+ DATADOG_CLIENT_TOKEN: process.env.SHOPPY_DATADOG_CLIENT_TOKEN,
+ DATADOG_SITE: process.env.SHOPPY_DATADOG_SITE,
+ DATADOG_SERVICE: "shoppy",
+ DATADOG_ENV: process.env.SHOPPY_DATADOG_ENV,
+ },
appName: "shoppy",
},
};