The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
PR-Env-Stats-Dump creates PR env with stats db (#75367)
Oreon Lothamer890a48bc56d6
diff --git a/.github/workflows/pr-env-close-unlabel.yml b/.github/workflows/pr-env-close-unlabel.ymlindex 0d62d31155c..54ef964ab1c 100644--- a/.github/workflows/pr-env-close-unlabel.yml+++ b/.github/workflows/pr-env-close-unlabel.yml@@ -10,7 +10,7 @@ concurrency:jobs:pr-env:- if: ${{ (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'PR-Env') && !contains(github.event.pull_request.labels.*.name, 'PR-Env-self-hosting') && !contains(github.event.pull_request.labels.*.name, 'PR-Env-analytics-dev')) || github.event.label.name == 'PR-Env' }}+ if: ${{ (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'PR-Env') && !contains(github.event.pull_request.labels.*.name, 'PR-Env-self-hosting') && !contains(github.event.pull_request.labels.*.name, 'PR-Env-analytics-dev') && !contains(github.event.pull_request.labels.*.name, 'PR-Env-Stats-Dump')) || github.event.label.name == 'PR-Env' }}uses: ./.github/workflows/pr-env-destroy.ymlsecrets: inheritwith:@@ -29,3 +29,10 @@ jobs:secrets: inheritwith:self_hosting: false++ pr-env-stats-dump:+ if: ${{ (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'PR-Env-Stats-Dump')) || github.event.label.name == 'PR-Env-Stats-Dump' }}+ uses: ./.github/workflows/pr-env-destroy.yml+ secrets: inherit+ with:+ self_hosting: falsediff --git a/.github/workflows/pr-env-labeled.yml b/.github/workflows/pr-env-labeled.ymlindex d900eec9e1a..ce6a0ec8dd8 100644--- a/.github/workflows/pr-env-labeled.yml+++ b/.github/workflows/pr-env-labeled.yml@@ -10,7 +10,7 @@ concurrency:jobs:uberjar:- if: ${{ github.event.label.name == 'PR-Env' || github.event.label.name == 'PR-Env-self-hosting' || github.event.label.name == 'PR-Env-analytics-dev' }}+ if: ${{ github.event.label.name == 'PR-Env' || github.event.label.name == 'PR-Env-self-hosting' || github.event.label.name == 'PR-Env-analytics-dev' || github.event.label.name == 'PR-Env-Stats-Dump' }}uses: ./.github/workflows/uberjar.ymlsecrets: inherit@@ -40,3 +40,13 @@ jobs:with:self_hosting: falseanalytics_dev_mode: true++ pr-env-stats-dump:+ needs: [uberjar]+ if: ${{ github.event.label.name == 'PR-Env-Stats-Dump' }}+ uses: ./.github/workflows/pr-env.yml+ secrets: inherit+ with:+ self_hosting: false+ analytics_dev_mode: false+ stats_dump: truediff --git a/.github/workflows/pr-env.yml b/.github/workflows/pr-env.ymlindex 479bf2b1527..8fe425f7f8b 100644--- a/.github/workflows/pr-env.yml+++ b/.github/workflows/pr-env.yml@@ -14,6 +14,11 @@ on:required: falsetype: booleandefault: false+ stats_dump:+ description: "Load a recent app DB stats dump on first env creation"+ required: false+ type: boolean+ default: falsejobs:build:runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}@@ -74,39 +79,6 @@ jobs:tags: tag:civersion: 1.50.1sha256sum: d9fe6b480fb5078f0aa57dace686898dda7e2a768884271159faa74846bfb576- - name: Create OIDC Token- id: create-oidc-token- shell: bash- run: |- export OIDC_URL_WITH_AUDIENCE="$ACTIONS_ID_TOKEN_REQUEST_URL&audience=${PR_ENV_K8S_AUDIENCE}"- IDTOKEN=$(curl -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" -H "Accept: application/json; api-version=2.0" "$OIDC_URL_WITH_AUDIENCE" | jq -r .value)- echo "::add-mask::${IDTOKEN}"- echo "idToken=${IDTOKEN}" >>$GITHUB_OUTPUT- env:- PR_ENV_K8S_AUDIENCE: ${{ secrets.PR_ENV_K8S_AUDIENCE }}- - name: Setup Kube Context- uses: azure/k8s-set-context@89b837d75b40a7bd2ddafde837473c212db8b313 # v5.0.0- with:- method: kubeconfig- kubeconfig: |- kind: Config- apiVersion: v1- current-context: default- clusters:- - name: default- cluster:- certificate-authority-data: ${{ secrets.PR_ENV_K8S_CERTIFICATE_AUTHORITY_DATA }}- server: ${{ secrets.PR_ENV_K8S_SERVER }}- users:- - name: oidc-token- user:- token: ${{ steps.create-oidc-token.outputs.IDTOKEN }}- contexts:- - name: default- context:- cluster: default- namespace: default- user: oidc-token- name: Configure AWS credentialsuses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1with:@@ -118,19 +90,134 @@ jobs:sudo apt-get updatesudo apt-get install -y postgresql-client- name: Create app database if not exists+ id: create_app_dbrun: |export PGPASSWORD="${PR_ENV_DB_PASSWORD}"export PGUSER="${PR_ENV_DB_USER}"export PGHOST="${PR_ENV_DB_HOST}"export PGDATABASE="${PR_ENV_DB_NAME}"- psql -tc "SELECT 1 FROM pg_database WHERE datname = 'hosting_pr${{ github.event.number }}'" \- | grep -q 1 \- || psql -c "CREATE DATABASE hosting_pr${{ github.event.number }}"+ if psql -tc "SELECT 1 FROM pg_database WHERE datname = 'hosting_pr${{ github.event.number }}'" | grep -q 1; then+ echo "db_created=false" >> "$GITHUB_OUTPUT"+ else+ psql -c "CREATE DATABASE hosting_pr${{ github.event.number }}"+ echo "db_created=true" >> "$GITHUB_OUTPUT"+ fienv:PR_ENV_DB_PASSWORD: ${{ secrets.PR_ENV_DB_PASSWORD }}PR_ENV_DB_USER: ${{ secrets.PR_ENV_DB_USER }}PR_ENV_DB_HOST: ${{ secrets.PR_ENV_DB_HOST }}PR_ENV_DB_NAME: ${{ secrets.PR_ENV_DB_NAME }}+ - name: Verify stats dump load status+ id: verify_stats_dump+ if: ${{ inputs.stats_dump }}+ run: |+ export PGPASSWORD="${PR_ENV_DB_PASSWORD}"+ ADMIN="psql -h ${PR_ENV_DB_HOST} -U ${PR_ENV_DB_USER} -d ${PR_ENV_DB_NAME}"+ MARKED=$($ADMIN -tAc "SELECT 1 FROM pg_database d WHERE d.datname='hosting_pr${{ github.event.number }}' AND shobj_description(d.oid, 'pg_database') = 'metabase-pr-env stats-dump loaded';")+ if [ "$MARKED" = "1" ]; then+ echo "Stats dump already loaded; skipping reload."+ echo "should_load=false" >> "$GITHUB_OUTPUT"+ else+ echo "Stats dump not loaded (or previous load incomplete); recreating database."+ $ADMIN -c "ALTER DATABASE \"hosting_pr${{ github.event.number }}\" CONNECTION LIMIT 0;" || true+ $ADMIN -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='hosting_pr${{ github.event.number }}' AND pid <> pg_backend_pid();" || true+ $ADMIN -c "DROP DATABASE IF EXISTS \"hosting_pr${{ github.event.number }}\" WITH (FORCE);"+ $ADMIN -c "CREATE DATABASE \"hosting_pr${{ github.event.number }}\";"+ echo "should_load=true" >> "$GITHUB_OUTPUT"+ fi+ env:+ PR_ENV_DB_PASSWORD: ${{ secrets.PR_ENV_DB_PASSWORD }}+ PR_ENV_DB_USER: ${{ secrets.PR_ENV_DB_USER }}+ PR_ENV_DB_HOST: ${{ secrets.PR_ENV_DB_HOST }}+ PR_ENV_DB_NAME: ${{ secrets.PR_ENV_DB_NAME }}+ - name: Locate latest stats dump in S3+ id: locate_stats_dump+ if: ${{ inputs.stats_dump && steps.verify_stats_dump.outputs.should_load == 'true' }}+ env:+ STATS_DUMP_BUCKET: metabase-cloud-stats-backups-coredev+ run: |+ LATEST_KEY=$(aws s3api list-objects-v2 \+ --bucket "${STATS_DUMP_BUCKET}" \+ --query "reverse(sort_by(Contents[?ends_with(Key, '.dump')], &LastModified))[0].Key" \+ --output text)+ if [ -z "$LATEST_KEY" ] || [ "$LATEST_KEY" = "None" ]; then+ echo "No *.dump found in s3://${STATS_DUMP_BUCKET}" >&2+ exit 1+ fi+ echo "Latest dump: $LATEST_KEY"+ echo "key=$LATEST_KEY" >> "$GITHUB_OUTPUT"+ - name: Download stats dump+ if: ${{ inputs.stats_dump && steps.verify_stats_dump.outputs.should_load == 'true' }}+ env:+ STATS_DUMP_BUCKET: metabase-cloud-stats-backups-coredev+ run: |+ aws s3 cp --no-progress \+ "s3://${STATS_DUMP_BUCKET}/${{ steps.locate_stats_dump.outputs.key }}" \+ /tmp/stats-dump.dump+ echo "Downloaded $(stat -c%s /tmp/stats-dump.dump) bytes"+ file /tmp/stats-dump.dump+ - name: Prepare app database schema+ if: ${{ inputs.stats_dump && steps.verify_stats_dump.outputs.should_load == 'true' }}+ run: |+ PGPASSWORD="${PR_ENV_DB_PASSWORD}" psql \+ -h "${PR_ENV_DB_HOST}" \+ -U "${PR_ENV_DB_USER}" \+ -d "hosting_pr${{ github.event.number }}" \+ --no-align --tuples-only --expanded \+ --command='DROP SCHEMA public CASCADE; CREATE SCHEMA public; CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;'+ env:+ PR_ENV_DB_PASSWORD: ${{ secrets.PR_ENV_DB_PASSWORD }}+ PR_ENV_DB_USER: ${{ secrets.PR_ENV_DB_USER }}+ PR_ENV_DB_HOST: ${{ secrets.PR_ENV_DB_HOST }}+ - name: Restore stats dump into app database+ if: ${{ inputs.stats_dump && steps.verify_stats_dump.outputs.should_load == 'true' }}+ run: |+ set +e+ echo "::group::pg_restore / pg client versions"+ pg_restore --version+ psql --version+ echo "::endgroup::"++ echo "::group::Server version"+ PGPASSWORD="${PR_ENV_DB_PASSWORD}" psql \+ -h "${PR_ENV_DB_HOST}" -U "${PR_ENV_DB_USER}" -d "${PR_ENV_DB_NAME}" \+ -tAc 'SHOW server_version;'++ echo "::group::pg_restore"+ PGPASSWORD="${PR_ENV_DB_PASSWORD}" pg_restore \+ -h "${PR_ENV_DB_HOST}" \+ -U "${PR_ENV_DB_USER}" \+ -d "hosting_pr${{ github.event.number }}" \+ --single-transaction \+ --no-acl \+ --no-owner \+ --schema=public \+ /tmp/stats-dump.dump+ RESTORE_EXIT=$?+ echo "::endgroup::"+ echo "pg_restore exit code: ${RESTORE_EXIT}"++ if [ "${RESTORE_EXIT}" -ne 0 ]; then+ echo "Restore failed; the database will be rebuilt on the next run because it has not been marked as loaded."+ exit "${RESTORE_EXIT}"+ fi+ env:+ PR_ENV_DB_PASSWORD: ${{ secrets.PR_ENV_DB_PASSWORD }}+ PR_ENV_DB_USER: ${{ secrets.PR_ENV_DB_USER }}+ PR_ENV_DB_HOST: ${{ secrets.PR_ENV_DB_HOST }}+ PR_ENV_DB_NAME: ${{ secrets.PR_ENV_DB_NAME }}+ - name: Mark database as stats-dump loaded+ if: ${{ inputs.stats_dump && steps.verify_stats_dump.outputs.should_load == 'true' }}+ run: |+ PGPASSWORD="${PR_ENV_DB_PASSWORD}" psql \+ -h "${PR_ENV_DB_HOST}" \+ -U "${PR_ENV_DB_USER}" \+ -d "hosting_pr${{ github.event.number }}" \+ -c "COMMENT ON DATABASE \"hosting_pr${{ github.event.number }}\" IS 'metabase-pr-env stats-dump loaded';"+ env:+ PR_ENV_DB_PASSWORD: ${{ secrets.PR_ENV_DB_PASSWORD }}+ PR_ENV_DB_USER: ${{ secrets.PR_ENV_DB_USER }}+ PR_ENV_DB_HOST: ${{ secrets.PR_ENV_DB_HOST }}- name: Download Deployment YAML templateif: ${{ !inputs.self_hosting }}run: aws s3 cp s3://metabase-pr-env/metabase.yml.tmpl ./metabase.yml.tmpl@@ -146,6 +233,48 @@ jobs:MB_PREMIUM_EMBEDDING_TOKEN: ${{ secrets.PR_ENV_MB_PREMIUM_EMBEDDING_TOKEN }}SHA: ${{ github.event.pull_request.head.sha || github.sha }}MB_ANALYTICS_DEV_MODE: ${{ inputs.analytics_dev_mode && 'true' || 'false' }}+ - name: Inject stats-dump env vars into Deployment YAML+ if: ${{ !inputs.self_hosting && inputs.stats_dump }}+ env:+ MB_ENCRYPTION_SECRET_KEY: ${{ secrets.PR_ENV_STATS_DUMP_ENCRYPTION_KEY }}+ run: |+ yq -i 'select(.kind == "Metabase").spec.configuration.envVars += [+ {"name": "MB_ENCRYPTION_SECRET_KEY", "value": strenv(MB_ENCRYPTION_SECRET_KEY)},+ {"name": "MB_DISABLE_SCHEDULER", "value": "true"}+ ]' ./metabase.yml+ - name: Create OIDC Token+ id: create-oidc-token+ shell: bash+ run: |+ export OIDC_URL_WITH_AUDIENCE="$ACTIONS_ID_TOKEN_REQUEST_URL&audience=${PR_ENV_K8S_AUDIENCE}"+ IDTOKEN=$(curl -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" -H "Accept: application/json; api-version=2.0" "$OIDC_URL_WITH_AUDIENCE" | jq -r .value)+ echo "::add-mask::${IDTOKEN}"+ echo "idToken=${IDTOKEN}" >>$GITHUB_OUTPUT+ env:+ PR_ENV_K8S_AUDIENCE: ${{ secrets.PR_ENV_K8S_AUDIENCE }}+ - name: Setup Kube Context+ uses: azure/k8s-set-context@89b837d75b40a7bd2ddafde837473c212db8b313 # v5.0.0+ with:+ method: kubeconfig+ kubeconfig: |+ kind: Config+ apiVersion: v1+ current-context: default+ clusters:+ - name: default+ cluster:+ certificate-authority-data: ${{ secrets.PR_ENV_K8S_CERTIFICATE_AUTHORITY_DATA }}+ server: ${{ secrets.PR_ENV_K8S_SERVER }}+ users:+ - name: oidc-token+ user:+ token: ${{ steps.create-oidc-token.outputs.IDTOKEN }}+ contexts:+ - name: default+ context:+ cluster: default+ namespace: default+ user: oidc-token- name: Deploy PR Review ENVif: ${{ !inputs.self_hosting }}run: |@@ -209,4 +338,4 @@ jobs:✅ Preview:https://pr${{ github.event.number }}.coredev.metabase.com🗒️ [login instructions](https://www.notion.so/metabase/Ephemeral-PR-Environments-c74a3710c87a460dbf4fc0007a001458)- 📦 Deployment type: ${{ inputs.self_hosting && 'PR-Env-self-hosting' || (inputs.analytics_dev_mode && 'PR-Env-analytics-dev' || 'PR-Env') }}+ 📦 Deployment type: ${{ inputs.self_hosting && 'PR-Env-self-hosting' || (inputs.analytics_dev_mode && 'PR-Env-analytics-dev' || (inputs.stats_dump && 'PR-Env-Stats-Dump' || 'PR-Env')) }}