The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Prepare Shoppy Sample App GH job to run synthetic monitoring tests (#69492)
Timofey Kachalov890771e40eee
diff --git a/.github/actions/prepare-shoppy-e2e/action.yml b/.github/actions/prepare-shoppy-e2e/action.ymlnew file mode 100644index 00000000000..7ba2a954a65--- /dev/null+++ b/.github/actions/prepare-shoppy-e2e/action.yml@@ -0,0 +1,132 @@+name: Prepare Shoppy E2E environment+description: Shared setup steps for Shoppy E2E tests (regular and synthetic)++inputs:+ commit:+ description: "Commit SHA for artifact retrieval"+ required: true+ sample-app-branch-name:+ description: "Branch name for the sample app"+ required: true+ sample-app-environment:+ description: "Environment (production/development)"+ required: true+ staging-mb-all-features-token:+ description: "Staging MB all features token"+ required: true+ shoppy-iam-role:+ description: "Shoppy IAM role for AWS credentials"+ required: true+ shoppy-metabase-app-db-s3-bucket:+ description: "S3 bucket for Shoppy app DB dump"+ required: true+ shoppy-metabase-app-db-s3-key:+ description: "S3 key for Shoppy app DB dump"+ required: true+ shoppy-metabase-app-db-user:+ description: "Shoppy Metabase app DB user"+ required: true+ shoppy-metabase-app-db-database:+ description: "Shoppy Metabase app DB database"+ required: true+ metastore-dev-server-url:+ description: "Metastore dev server URL"+ required: true+ shoppy-datadog-application-id:+ description: "Shoppy Datadog application ID"+ required: true+ shoppy-datadog-client-token:+ description: "Shoppy Datadog client token"+ required: true++outputs:+ chrome-path:+ description: "Path to Chrome executable"+ value: ${{ steps.cypress-prep.outputs.chrome-path }}+ sample_app_branch_exists:+ description: "Whether the sample app branch exists"+ value: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists }}++runs:+ using: "composite"+ steps:+ - uses: ./.github/actions/maximize-disk-space++ # We have to use the latest version that does not have this issue https://github.com/docker/compose/pull/12752+ - name: Set up Docker Compose+ uses: docker/setup-compose-action@v1+ with:+ version: latest++ - name: Retrieve uberjar artifact for ee+ uses: ./.github/actions/fetch-artifact+ with:+ commit: ${{ inputs.commit }}+ edition: ee+ extract-path: "target/uberjar"+ output-name: "metabase.jar"++ - name: Retrieve Embedding SDK dist artifact+ uses: actions/download-artifact@v4+ with:+ name: embedding-sdk-${{ inputs.commit }}+ path: ${{ github.workspace }}/resources/embedding-sdk++ - name: Prepare front-end environment+ uses: ./.github/actions/prepare-frontend++ - name: Prepare Cypress environment+ id: cypress-prep+ uses: ./.github/actions/prepare-cypress++ - name: Check if sample app branch exists+ id: check-sample-app-branch+ shell: bash+ env:+ SAMPLE_APP_BRANCH_NAME: ${{ inputs.sample-app-branch-name }}+ run: |+ REPO_URL="https://github.com/metabase/shoppy.git"+ echo "Checking if branch '${SAMPLE_APP_BRANCH_NAME}' exists in ${REPO_URL}..."+ if git ls-remote --exit-code --heads "$REPO_URL" "${SAMPLE_APP_BRANCH_NAME}"; then+ echo "sample_app_branch_exists=true" >> $GITHUB_OUTPUT+ else+ echo "sample_app_branch_exists=false" >> $GITHUB_OUTPUT+ fi++ - name: Configure AWS credentials+ if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}+ uses: aws-actions/configure-aws-credentials@v4+ with:+ role-to-assume: ${{ inputs.shoppy-iam-role }}+ role-session-name: GitHub_to_AWS_via_FederatedOIDC+ aws-region: us-east-1++ - name: Download Shoppy's App DB PostgreSQL dump from S3+ if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}+ shell: bash+ run: |+ aws s3 cp s3://${{ inputs.shoppy-metabase-app-db-s3-bucket }}/${{ inputs.shoppy-metabase-app-db-s3-key }} ./e2e/tmp/db_dumps/shoppy_metabase_app_db_dump.sql++ - name: Prepare and launch Shoppy containers+ if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}+ shell: bash+ env:+ SAMPLE_APP_BRANCH_NAME: ${{ inputs.sample-app-branch-name }}+ SAMPLE_APP_ENVIRONMENT: ${{ inputs.sample-app-environment }}+ CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ inputs.staging-mb-all-features-token }}+ METASTORE_DEV_SERVER_URL: ${{ inputs.metastore-dev-server-url }}+ METABASE_APP_DB_USER: ${{ inputs.shoppy-metabase-app-db-user }}+ METABASE_APP_DB: ${{ inputs.shoppy-metabase-app-db-database }}+ SHOPPY_DATADOG_APPLICATION_ID: ${{ inputs.shoppy-datadog-application-id }}+ SHOPPY_DATADOG_CLIENT_TOKEN: ${{ inputs.shoppy-datadog-client-token }}+ SHOPPY_DATADOG_SITE: "us5.datadoghq.com"+ SHOPPY_DATADOG_ENV: "staging"+ run: npx tsx ./e2e/runner/embedding-sdk/sample-apps/start-ci.ts shoppy-e2e++ - name: Prune build cache+ shell: bash+ run: docker builder prune -af++ - name: Log free disk space+ shell: bash+ run: df -h /diff --git a/.github/workflows/embedding-sdk-host-sample-apps-tests.yml b/.github/workflows/embedding-sdk-host-sample-apps-tests.ymlindex 4e3b06594b0..16efeca80a4 100644--- a/.github/workflows/embedding-sdk-host-sample-apps-tests.yml+++ b/.github/workflows/embedding-sdk-host-sample-apps-tests.yml@@ -9,7 +9,6 @@ env:TZ: US/Pacific # to make Node match the instance tzGITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}HASH: ${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }}- PR_NUMBER: ${{ github.event.pull_request.number || '' }}MB_EDITION: eejobs:get-sample-apps-data:@@ -21,12 +20,7 @@ jobs:TEST_SUITES: |["metabase-nodejs-react-sdk-embedding-sample-e2e",- "metabase-nextjs-sdk-embedding-sample-e2e",- "shoppy-e2e"- ]- TEST_SUITES_TO_RUN_IN_MASTER_ONLY: |- [- "shoppy-e2e"+ "metabase-nextjs-sdk-embedding-sample-e2e"]SAMPLE_APP_MAIN_BRANCH_NAME: "main"SAMPLE_APP_STABLE_RELEASE_BRANCH_NAME_SUFFIX: "stable"@@ -50,7 +44,6 @@ jobs:} = require('${{ github.workspace }}/release/dist/index.cjs');const testSuites = JSON.parse(process.env.TEST_SUITES);- const testSuitesToRunInMasterOnly = JSON.parse(process.env.TEST_SUITES_TO_RUN_IN_MASTER_ONLY);const ref = '${{ github.base_ref || github.ref }}';const branchName = ref.replace(/^refs\/heads\//, '');@@ -74,19 +67,16 @@ jobs:const sampleAppBranchName = `${currentBranchVersion}-${{ env.SAMPLE_APP_STABLE_RELEASE_BRANCH_NAME_SUFFIX }}`const shouldRunTestsForVersion = (currentReleaseVersion - currentBranchVersion) <= (supportedReleasesCount - 1);- const releaseBranchTestSuites = testSuites.filter(testSuite => !testSuitesToRunInMasterOnly.includes(testSuite));return {branch_name: sampleAppBranchName,- matrix: shouldRunTestsForVersion- ? releaseBranchTestSuites- : []+ matrix: shouldRunTestsForVersion ? testSuites : [],};}return {branch_name: '',- matrix: []+ matrix: [],};# In order to check compatibility between Sample Apps and Embedding SDK, the job for each sample app:@@ -168,28 +158,8 @@ jobs:echo "sample_app_branch_exists=false" >> $GITHUB_OUTPUTfi- - name: Configure AWS credentials- if: ${{ matrix.test_suite == 'shoppy-e2e' }}- uses: aws-actions/configure-aws-credentials@v4- with:- role-to-assume: ${{ secrets.SHOPPY_IAM_ROLE }}- role-session-name: GitHub_to_AWS_via_FederatedOIDC- aws-region: us-east-1-- - name: Download Shoppy's App DB PostgreSQL dump from S3- if: ${{ matrix.test_suite == 'shoppy-e2e' }}- run: |- aws s3 cp s3://${{ secrets.SHOPPY_METABASE_APP_DB_S3_BUCKET }}/${{ secrets.SHOPPY_METABASE_APP_DB_S3_KEY }} ./e2e/tmp/db_dumps/shoppy_metabase_app_db_dump.sql-- - name: Prepare and launch Sample Apps for shoppy-e2e test suite- if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' && matrix.test_suite == 'shoppy-e2e' }}- env:- METABASE_APP_DB_USER: ${{ secrets.SHOPPY_METABASE_APP_DB_USER }}- METABASE_APP_DB: ${{ secrets.SHOPPY_METABASE_APP_DB_DATABASE }}- run: npx tsx ./e2e/runner/embedding-sdk/sample-apps/start-ci.ts shoppy-e2e-- - name: Prepare and launch Sample Apps for other test suites- if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' && matrix.test_suite != 'shoppy-e2e' }}+ - name: Prepare and launch Sample Apps+ if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}run: npx tsx ./e2e/runner/embedding-sdk/sample-apps/start-ci.ts ${{ matrix.test_suite }}- name: Prune build cache@@ -210,7 +180,7 @@ jobs:uses: actions/upload-artifact@v4if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}with:- name: cypress-recording-sample-app-${{ matrix.test_suite }}-latest+ name: cypress-recording-sample-app-${{ matrix.test_suite }}-${{ matrix.environment }}-latestpath: |./e2e/tmp/${{ steps.get-sample-app-name.outputs.sample_app_name }}/cypressif-no-files-found: ignore@@ -220,6 +190,149 @@ jobs:run: |exit 1+ # Shoppy e2e tests run separately using the shared composite action for setup.+ e2e-shoppy-tests:+ needs: [get-sample-apps-data]+ strategy:+ matrix:+ environment: ["production", "development"]+ if: |+ !cancelled() && needs.get-sample-apps-data.result == 'success' &&+ (github.ref == 'refs/heads/master' || github.base_ref == 'master')+ runs-on: ubuntu-22.04+ timeout-minutes: 45+ name: e2e-sample-app-shoppy-e2e-tests-${{ matrix.environment }}-bundle+ env:+ MB_EDITION: ee+ MB_RUN_MODE: e2e+ METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}+ CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}+ SAMPLE_APP_BRANCH_NAME: ${{ needs.get-sample-apps-data.outputs.branch_name }}+ SAMPLE_APP_ENVIRONMENT: ${{ matrix.environment }}+ permissions:+ id-token: write++ steps:+ - uses: actions/checkout@v4++ - name: Prepare Shoppy E2E environment+ uses: ./.github/actions/prepare-shoppy-e2e+ id: shoppy-prep+ with:+ commit: ${{ github.event.pull_request.head.sha || github.sha }}+ sample-app-branch-name: ${{ needs.get-sample-apps-data.outputs.branch_name }}+ sample-app-environment: ${{ matrix.environment }}+ staging-mb-all-features-token: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}+ shoppy-iam-role: ${{ secrets.SHOPPY_IAM_ROLE }}+ shoppy-metabase-app-db-s3-bucket: ${{ secrets.SHOPPY_METABASE_APP_DB_S3_BUCKET }}+ shoppy-metabase-app-db-s3-key: ${{ secrets.SHOPPY_METABASE_APP_DB_S3_KEY }}+ shoppy-metabase-app-db-user: ${{ secrets.SHOPPY_METABASE_APP_DB_USER }}+ shoppy-metabase-app-db-database: ${{ secrets.SHOPPY_METABASE_APP_DB_DATABASE }}+ metastore-dev-server-url: ${{ secrets.METASTORE_DEV_SERVER_URL }}+ shoppy-datadog-application-id: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_APPLICATION_ID }}+ shoppy-datadog-client-token: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_CLIENT_TOKEN }}++ - name: Run e2e tests for Shoppy+ id: run-e2e-tests+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}+ continue-on-error: true+ env:+ CYPRESS_BROWSER: ${{ steps.shoppy-prep.outputs.chrome-path }}+ run: node e2e/runner/run_cypress_ci.js shoppy-e2e++ - name: Upload Cypress Artifacts upon failure+ uses: actions/upload-artifact@v4+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}+ with:+ name: cypress-recording-sample-app-shoppy-e2e-${{ matrix.environment }}-latest+ path: |+ ./e2e/tmp/shoppy/cypress+ if-no-files-found: ignore++ - name: Fail job if tests failed+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}+ run: |+ exit 1++ # Synthetic monitoring tests run only on master merges, in parallel with the regular e2e tests.+ # They use the shoppy production environment and include performance measurement tests.+ e2e-shoppy-synthetic-tests:+ needs: [get-sample-apps-data]+ if: |+ !cancelled() && needs.get-sample-apps-data.result == 'success' &&+ github.event_name == 'push' &&+ github.ref == 'refs/heads/master'+ runs-on: ubuntu-22.04+ timeout-minutes: 45+ name: e2e-shoppy-synthetic-monitoring-tests+ env:+ MB_EDITION: ee+ MB_RUN_MODE: e2e+ METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}+ CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}+ SAMPLE_APP_BRANCH_NAME: ${{ needs.get-sample-apps-data.outputs.branch_name }}+ SAMPLE_APP_ENVIRONMENT: production+ permissions:+ id-token: write+ contents: read++ steps:+ - uses: actions/checkout@v4++ - name: Prepare Shoppy E2E environment+ uses: ./.github/actions/prepare-shoppy-e2e+ id: shoppy-prep+ with:+ commit: ${{ github.event.pull_request.head.sha || github.sha }}+ sample-app-branch-name: ${{ needs.get-sample-apps-data.outputs.branch_name }}+ sample-app-environment: production+ staging-mb-all-features-token: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}+ shoppy-iam-role: ${{ secrets.SHOPPY_IAM_ROLE }}+ shoppy-metabase-app-db-s3-bucket: ${{ secrets.SHOPPY_METABASE_APP_DB_S3_BUCKET }}+ shoppy-metabase-app-db-s3-key: ${{ secrets.SHOPPY_METABASE_APP_DB_S3_KEY }}+ shoppy-metabase-app-db-user: ${{ secrets.SHOPPY_METABASE_APP_DB_USER }}+ shoppy-metabase-app-db-database: ${{ secrets.SHOPPY_METABASE_APP_DB_DATABASE }}+ metastore-dev-server-url: ${{ secrets.METASTORE_DEV_SERVER_URL }}+ shoppy-datadog-application-id: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_APPLICATION_ID }}+ shoppy-datadog-client-token: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_CLIENT_TOKEN }}++ - name: Get merged PR number+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}+ id: get-merged-pr-number+ run: |+ PR=$(gh api repos/${{ github.repository }}/commits/${{ github.sha }}/pulls --jq '.[0].number' 2>/dev/null || echo "")+ if [ -z "$PR" ] || [ "$PR" = "null" ]; then+ echo "number=" >> "$GITHUB_OUTPUT"+ else+ echo "number=$PR" >> "$GITHUB_OUTPUT"+ fi+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}++ - name: Run synthetic monitoring tests+ id: run-e2e-tests+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}+ continue-on-error: true+ env:+ CYPRESS_BROWSER: ${{ steps.shoppy-prep.outputs.chrome-path }}+ PR_NUMBER: ${{ steps.get-merged-pr-number.outputs.number }}+ INCLUDE_SYNTHETIC_MONITORING_TESTS: "true"+ run: node e2e/runner/run_cypress_ci.js shoppy-e2e++ - name: Upload Cypress Artifacts upon failure+ uses: actions/upload-artifact@v4+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}+ with:+ name: cypress-recording-shoppy-synthetic-tests-latest+ path: |+ ./e2e/tmp/shoppy/cypress+ if-no-files-found: ignore++ - name: Fail job if tests failed+ if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}+ run: |+ exit 1+# The job checks integration of the Embedding SDK with consumer's apps that use different frameworks/bundlers,# or when we want to test some tricky integration cases like conflicting types.# The jobdiff --git a/e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.js b/e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.jsindex 78cd752fab7..db2f68e19dd 100644--- a/e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.js+++ b/e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.js@@ -53,6 +53,14 @@ export const SAMPLE_APP_SETUP_CONFIGS = {},"shoppy-e2e": {...BASE_SETUP_CONFIG,+ env: {+ ...BASE_SETUP_CONFIG.env,+ DATADOG_APPLICATION_ID: process.env.SHOPPY_DATADOG_APPLICATION_ID,+ DATADOG_CLIENT_TOKEN: process.env.SHOPPY_DATADOG_CLIENT_TOKEN,+ DATADOG_SITE: process.env.SHOPPY_DATADOG_SITE,+ DATADOG_SERVICE: "shoppy",+ DATADOG_ENV: process.env.SHOPPY_DATADOG_ENV,+ },appName: "shoppy",},};