The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Parallelize embedding SDK package and uberjar builds (#66122)
Nemanja Glumac2e4b7adbcbdf
diff --git a/.github/file-paths.yaml b/.github/file-paths.yamlindex 6f287517b55..f4b4509a11e 100644--- a/.github/file-paths.yaml+++ b/.github/file-paths.yaml@@ -16,11 +16,6 @@ frontend_ci: &frontend_ci- ".github/actions/prepare-backend/**"- ".github/workflows/frontend.yml"-frontend_embedding_sdk_ci: &frontend_embedding_sdk_ci- - ".github/actions/prepare-frontend/**"- - ".github/actions/prepare-backend/**"- - ".github/workflows/embedding-sdk*.yml"-frontend_configs: &frontend_configs- "yarn.lock"- "**/tsconfig*.json"@@ -127,14 +122,19 @@ e2e_ci: &e2e_cie2e_specs: &e2e_specs- e2e/!(test-host-app)**/*.cy.*.{js,ts}-e2e_embedding_sdk:- - *frontend_embedding_sdk_ci+embedding_sdk_ci: &embedding_sdk_ci+ - ".github/actions/prepare-frontend/**"+ - ".github/actions/prepare-backend/**"+ - ".github/workflows/embedding-sdk*.yml"++embedding_sdk_components:+ - *embedding_sdk_ci- *sources- "e2e/component/**/*.cy.*.{js,ts}"-e2e_embedding_sdk_compatibility:+embedding_sdk_host_sample_apps:- *default- - *frontend_embedding_sdk_ci+ - *embedding_sdk_ci- *frontend_sources- *backend_sources- "e2e/embedding-sdk-host-apps/**"diff --git a/.github/workflows/embedding-sdk-component-tests.yml b/.github/workflows/embedding-sdk-component-tests.ymlindex eae471968de..4949ea82289 100644--- a/.github/workflows/embedding-sdk-component-tests.yml+++ b/.github/workflows/embedding-sdk-component-tests.yml@@ -2,10 +2,6 @@ name: Embedding SDK Component Testson:workflow_call:- inputs:- cached-embedding-sdk-dist-artifact-name:- required: true- type: stringjobs:sdk-component-tests:@@ -44,7 +40,7 @@ jobs:- name: Retrieve Embedding SDK dist artifactuses: actions/download-artifact@v4with:- name: ${{ inputs.cached-embedding-sdk-dist-artifact-name }}+ name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}path: ${{ github.workspace }}/resources/embedding-sdk- name: Prepare front-end environment@@ -192,7 +188,7 @@ jobs:- name: Retrieve Embedding SDK dist artifact built from the current commit Nuses: actions/download-artifact@v4with:- name: ${{ inputs.cached-embedding-sdk-dist-artifact-name }}+ name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}path: ${{ github.workspace }}/resources/embedding-sdk- name: Prepare Cypress environmentdiff --git a/.github/workflows/embedding-sdk-e2e-tests.yml b/.github/workflows/embedding-sdk-host-sample-apps-tests.ymlsimilarity index 99%rename from .github/workflows/embedding-sdk-e2e-tests.ymlrename to .github/workflows/embedding-sdk-host-sample-apps-tests.ymlindex be45ee5256d..d4a292eda2c 100644--- a/.github/workflows/embedding-sdk-e2e-tests.yml+++ b/.github/workflows/embedding-sdk-host-sample-apps-tests.yml@@ -2,10 +2,6 @@ name: "[NOT REQUIRED] SDK Compatibility tests"on:workflow_call:- inputs:- cached-embedding-sdk-dist-artifact-name:- required: true- type: stringjobs:get-sample-apps-data:diff --git a/.github/workflows/embedding-sdk-package-build.yml b/.github/workflows/embedding-sdk-package-build.ymlnew file mode 100644index 00000000000..f085f44370a--- /dev/null+++ b/.github/workflows/embedding-sdk-package-build.yml@@ -0,0 +1,31 @@+name: Embedding SDK Package Build++on:+ workflow_call:+ workflow_dispatch:++concurrency:+ group: embedding-sdk-package-${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}+ cancel-in-progress: true++jobs:+ build-sdk-package:+ runs-on: ubuntu-22.04+ timeout-minutes: 25+ steps:+ - uses: actions/checkout@v4+ - name: Prepare front-end environment+ uses: ./.github/actions/prepare-frontend+ - name: Prepare back-end environment+ uses: ./.github/actions/prepare-backend+ - name: Build Embedding SDK package+ run: |+ # Remove the matchers to not report false positives on every PR+ # unrelated to embedding (EMB-765)+ echo "::remove-matcher owner=tsc::"+ yarn build-embedding-sdk-package+ - name: Prepare Embedding SDK artifact+ uses: actions/upload-artifact@v4+ with:+ name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}+ path: resources/embedding-sdkdiff --git a/.github/workflows/embedding-sdk.yml b/.github/workflows/embedding-sdk.ymlindex 44eb57b2bb3..2cfe5fca472 100644--- a/.github/workflows/embedding-sdk.yml+++ b/.github/workflows/embedding-sdk.yml@@ -2,62 +2,35 @@ name: SDKon:workflow_call:+ inputs:+ cli-snippets-type-check:+ description: "Whether to run cli snippets type check."+ required: true+ type: boolean+ docs-snippets-type-check:+ description: "Whether to run docs snippets type check."+ required: true+ type: boolean+ documentation-validation:+ description: "Whether to run documentation generation validation check."+ required: true+ type: boolean+ component-tests:+ description: "Whether to run component tests."+ required: true+ type: boolean+ host-sample-apps-tests:+ description: "Whether to run host and sample apps compatibility tests."+ required: true+ type: booleanconcurrency:group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}cancel-in-progress: truejobs:- files-changed:- name: Check which files changed- runs-on: ubuntu-22.04- timeout-minutes: 3- outputs:- e2e_embedding_sdk: ${{ steps.changes.outputs.e2e_embedding_sdk }}- e2e_embedding_sdk_compatibility: ${{ steps.changes.outputs.e2e_embedding_sdk_compatibility }}- frontend_embedding_sdk_sources: ${{ steps.changes.outputs.frontend_embedding_sdk_sources }}- embedding_documentation: ${{ steps.changes.outputs.embedding_documentation }}- steps:- - uses: actions/checkout@v4- - name: Test which files changed- uses: dorny/[email protected]- id: changes- with:- token: ${{ github.token }}- filters: .github/file-paths.yaml-- build-sdk-package:- needs: [files-changed]- if: |- !cancelled() &&- (needs.files-changed.outputs.e2e_embedding_sdk == 'true' ||- needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' ||- needs.files-changed.outputs.embedding_documentation == 'true')- runs-on: ubuntu-22.04- timeout-minutes: 25- steps:- - uses: actions/checkout@v4- - name: Prepare front-end environment- uses: ./.github/actions/prepare-frontend- - name: Prepare back-end environment- uses: ./.github/actions/prepare-backend- - name: Build Embedding SDK package- run: |- # Remove the matchers to not report false positives on every PR- # unrelated to embedding (EMB-765)- echo "::remove-matcher owner=tsc::"- yarn build-embedding-sdk-package- - name: Prepare Embedding SDK artifact- uses: actions/upload-artifact@v4- with:- name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}- path: resources/embedding-sdk-embedding-sdk-cli-snippets-type-check:- needs: [files-changed, build-sdk-package]- if: |- needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' &&- needs.build-sdk-package.result == 'success'+ if: inputs.cli-snippets-type-checkruns-on: ubuntu-22.04timeout-minutes: 20steps:@@ -73,10 +46,7 @@ jobs:run: yarn run embedding-sdk:cli-snippets:type-checkembedding-sdk-docs-snippets-type-check:- needs: [files-changed, build-sdk-package]- if: |- needs.files-changed.outputs.embedding_documentation == 'true' &&- needs.build-sdk-package.result == 'success'+ if: inputs.docs-snippets-type-checkruns-on: ubuntu-22.04timeout-minutes: 20steps:@@ -92,10 +62,7 @@ jobs:run: yarn run embedding-sdk:docs-snippets:type-checkembedding-sdk-api-documentation-generation-validation:- needs: [files-changed, build-sdk-package]- if: |- needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' &&- needs.build-sdk-package.result == 'success'+ if: inputs.documentation-validationruns-on: ubuntu-22.04timeout-minutes: 20steps:@@ -143,21 +110,15 @@ jobs:# See `Component e2e tests` in `enterprise/frontend/src/embedding-sdk-package/dev.md`embedding-sdk-component-tests:name: "SDK component tests"- needs: [build-sdk-package, files-changed]- if: needs.files-changed.outputs.e2e_embedding_sdk == 'true'+ if: inputs.component-testsuses: ./.github/workflows/embedding-sdk-component-tests.ymlsecrets: inherit- with:- cached-embedding-sdk-dist-artifact-name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}not-required-embedding-sdk-apps-compatibility-tests:name: "SDK e2e tests for compatibility with (host/sample) apps"- needs: [build-sdk-package, files-changed]- if: needs.files-changed.outputs.e2e_embedding_sdk_compatibility == 'true'- uses: ./.github/workflows/embedding-sdk-e2e-tests.yml+ if: inputs.host-sample-apps-tests+ uses: ./.github/workflows/embedding-sdk-host-sample-apps-tests.ymlsecrets: inherit- with:- cached-embedding-sdk-dist-artifact-name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}embedding-sdk-tests-result:needs:diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.ymlindex 8c776e212f3..790395fe526 100644--- a/.github/workflows/run-tests.yml+++ b/.github/workflows/run-tests.yml@@ -25,6 +25,11 @@ jobs:frontend_all: ${{ steps.changes.outputs.frontend_all }}frontend_sources: ${{ steps.changes.outputs.frontend_sources }}documentation: ${{ steps.changes.outputs.documentation }}+ frontend_embedding_sdk_sources: ${{ steps.changes.outputs.frontend_embedding_sdk_sources }}+ embedding_documentation: ${{ steps.changes.outputs.embedding_documentation }}+ embedding_sdk_components: ${{ steps.changes.outputs.embedding_sdk_components }}+ embedding_sdk_host_sample_apps: ${{ steps.changes.outputs.embedding_sdk_host_sample_apps }}+ embedding_sdk_ci: ${{ steps.changes.outputs.embedding_sdk_ci }}steps:- uses: actions/checkout@v4- name: Test which files changed@@ -92,10 +97,30 @@ jobs:# list of changed specs files, then only specs are changed and we can run only those specsspecs: ${{ needs.files-changed.outputs.e2e_all_files == needs.files-changed.outputs.e2e_specs_files && needs.files-changed.outputs.e2e_specs_files || '' }}+ embedding-sdk-package:+ needs: [files-changed]+ if: |+ !cancelled() &&+ (needs.files-changed.outputs.embedding_sdk_components == 'true' ||+ needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' ||+ needs.files-changed.outputs.embedding_documentation == 'true' ||+ needs.files-changed.outputs.embedding_sdk_ci == 'true')+ uses: ./.github/workflows/embedding-sdk-package-build.yml+ secrets: inherit+sdk-tests:- needs: [uberjar]+ needs: [uberjar, embedding-sdk-package, files-changed]+ if: |+ needs.embedding-sdk-package.result == 'success' &&+ needs.uberjar.result == 'success'uses: ./.github/workflows/embedding-sdk.ymlsecrets: inherit+ with:+ cli-snippets-type-check: ${{ needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' }}+ docs-snippets-type-check: ${{ needs.files-changed.outputs.embedding_documentation == 'true' }}+ documentation-validation: ${{ needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' }}+ component-tests: ${{ needs.files-changed.outputs.embedding_sdk_components == 'true' }}+ host-sample-apps-tests: ${{ needs.files-changed.outputs.embedding_sdk_host_sample_apps == 'true' }}pr-env:needs: [uberjar]