Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Parallelize embedding SDK package and uberjar builds (#66122)

Nemanja Glumac2e4b7adbcbdf
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 6f287517b55..f4b4509a11e 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -16,11 +16,6 @@ frontend_ci: &frontend_ci
- ".github/actions/prepare-backend/**"
- ".github/workflows/frontend.yml"
-frontend_embedding_sdk_ci: &frontend_embedding_sdk_ci
- - ".github/actions/prepare-frontend/**"
- - ".github/actions/prepare-backend/**"
- - ".github/workflows/embedding-sdk*.yml"
-
frontend_configs: &frontend_configs
- "yarn.lock"
- "**/tsconfig*.json"
@@ -127,14 +122,19 @@ e2e_ci: &e2e_ci
e2e_specs: &e2e_specs
- e2e/!(test-host-app)**/*.cy.*.{js,ts}
-e2e_embedding_sdk:
- - *frontend_embedding_sdk_ci
+embedding_sdk_ci: &embedding_sdk_ci
+ - ".github/actions/prepare-frontend/**"
+ - ".github/actions/prepare-backend/**"
+ - ".github/workflows/embedding-sdk*.yml"
+
+embedding_sdk_components:
+ - *embedding_sdk_ci
- *sources
- "e2e/component/**/*.cy.*.{js,ts}"
-e2e_embedding_sdk_compatibility:
+embedding_sdk_host_sample_apps:
- *default
- - *frontend_embedding_sdk_ci
+ - *embedding_sdk_ci
- *frontend_sources
- *backend_sources
- "e2e/embedding-sdk-host-apps/**"
diff --git a/.github/workflows/embedding-sdk-component-tests.yml b/.github/workflows/embedding-sdk-component-tests.yml
index eae471968de..4949ea82289 100644
--- a/.github/workflows/embedding-sdk-component-tests.yml
+++ b/.github/workflows/embedding-sdk-component-tests.yml
@@ -2,10 +2,6 @@ name: Embedding SDK Component Tests
on:
workflow_call:
- inputs:
- cached-embedding-sdk-dist-artifact-name:
- required: true
- type: string
jobs:
sdk-component-tests:
@@ -44,7 +40,7 @@ jobs:
- name: Retrieve Embedding SDK dist artifact
uses: actions/download-artifact@v4
with:
- name: ${{ inputs.cached-embedding-sdk-dist-artifact-name }}
+ name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}
path: ${{ github.workspace }}/resources/embedding-sdk
- name: Prepare front-end environment
@@ -192,7 +188,7 @@ jobs:
- name: Retrieve Embedding SDK dist artifact built from the current commit N
uses: actions/download-artifact@v4
with:
- name: ${{ inputs.cached-embedding-sdk-dist-artifact-name }}
+ name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}
path: ${{ github.workspace }}/resources/embedding-sdk
- name: Prepare Cypress environment
diff --git a/.github/workflows/embedding-sdk-e2e-tests.yml b/.github/workflows/embedding-sdk-host-sample-apps-tests.yml
similarity index 99%
rename from .github/workflows/embedding-sdk-e2e-tests.yml
rename to .github/workflows/embedding-sdk-host-sample-apps-tests.yml
index be45ee5256d..d4a292eda2c 100644
--- a/.github/workflows/embedding-sdk-e2e-tests.yml
+++ b/.github/workflows/embedding-sdk-host-sample-apps-tests.yml
@@ -2,10 +2,6 @@ name: "[NOT REQUIRED] SDK Compatibility tests"
on:
workflow_call:
- inputs:
- cached-embedding-sdk-dist-artifact-name:
- required: true
- type: string
jobs:
get-sample-apps-data:
diff --git a/.github/workflows/embedding-sdk-package-build.yml b/.github/workflows/embedding-sdk-package-build.yml
new file mode 100644
index 00000000000..f085f44370a
--- /dev/null
+++ b/.github/workflows/embedding-sdk-package-build.yml
@@ -0,0 +1,31 @@
+name: Embedding SDK Package Build
+
+on:
+ workflow_call:
+ workflow_dispatch:
+
+concurrency:
+ group: embedding-sdk-package-${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ build-sdk-package:
+ runs-on: ubuntu-22.04
+ timeout-minutes: 25
+ steps:
+ - uses: actions/checkout@v4
+ - name: Prepare front-end environment
+ uses: ./.github/actions/prepare-frontend
+ - name: Prepare back-end environment
+ uses: ./.github/actions/prepare-backend
+ - name: Build Embedding SDK package
+ run: |
+ # Remove the matchers to not report false positives on every PR
+ # unrelated to embedding (EMB-765)
+ echo "::remove-matcher owner=tsc::"
+ yarn build-embedding-sdk-package
+ - name: Prepare Embedding SDK artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}
+ path: resources/embedding-sdk
diff --git a/.github/workflows/embedding-sdk.yml b/.github/workflows/embedding-sdk.yml
index 44eb57b2bb3..2cfe5fca472 100644
--- a/.github/workflows/embedding-sdk.yml
+++ b/.github/workflows/embedding-sdk.yml
@@ -2,62 +2,35 @@ name: SDK
on:
workflow_call:
+ inputs:
+ cli-snippets-type-check:
+ description: "Whether to run cli snippets type check."
+ required: true
+ type: boolean
+ docs-snippets-type-check:
+ description: "Whether to run docs snippets type check."
+ required: true
+ type: boolean
+ documentation-validation:
+ description: "Whether to run documentation generation validation check."
+ required: true
+ type: boolean
+ component-tests:
+ description: "Whether to run component tests."
+ required: true
+ type: boolean
+ host-sample-apps-tests:
+ description: "Whether to run host and sample apps compatibility tests."
+ required: true
+ type: boolean
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
jobs:
- files-changed:
- name: Check which files changed
- runs-on: ubuntu-22.04
- timeout-minutes: 3
- outputs:
- e2e_embedding_sdk: ${{ steps.changes.outputs.e2e_embedding_sdk }}
- e2e_embedding_sdk_compatibility: ${{ steps.changes.outputs.e2e_embedding_sdk_compatibility }}
- frontend_embedding_sdk_sources: ${{ steps.changes.outputs.frontend_embedding_sdk_sources }}
- embedding_documentation: ${{ steps.changes.outputs.embedding_documentation }}
- steps:
- - uses: actions/checkout@v4
- - name: Test which files changed
- uses: dorny/[email protected]
- id: changes
- with:
- token: ${{ github.token }}
- filters: .github/file-paths.yaml
-
- build-sdk-package:
- needs: [files-changed]
- if: |
- !cancelled() &&
- (needs.files-changed.outputs.e2e_embedding_sdk == 'true' ||
- needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' ||
- needs.files-changed.outputs.embedding_documentation == 'true')
- runs-on: ubuntu-22.04
- timeout-minutes: 25
- steps:
- - uses: actions/checkout@v4
- - name: Prepare front-end environment
- uses: ./.github/actions/prepare-frontend
- - name: Prepare back-end environment
- uses: ./.github/actions/prepare-backend
- - name: Build Embedding SDK package
- run: |
- # Remove the matchers to not report false positives on every PR
- # unrelated to embedding (EMB-765)
- echo "::remove-matcher owner=tsc::"
- yarn build-embedding-sdk-package
- - name: Prepare Embedding SDK artifact
- uses: actions/upload-artifact@v4
- with:
- name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}
- path: resources/embedding-sdk
-
embedding-sdk-cli-snippets-type-check:
- needs: [files-changed, build-sdk-package]
- if: |
- needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' &&
- needs.build-sdk-package.result == 'success'
+ if: inputs.cli-snippets-type-check
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
@@ -73,10 +46,7 @@ jobs:
run: yarn run embedding-sdk:cli-snippets:type-check
embedding-sdk-docs-snippets-type-check:
- needs: [files-changed, build-sdk-package]
- if: |
- needs.files-changed.outputs.embedding_documentation == 'true' &&
- needs.build-sdk-package.result == 'success'
+ if: inputs.docs-snippets-type-check
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
@@ -92,10 +62,7 @@ jobs:
run: yarn run embedding-sdk:docs-snippets:type-check
embedding-sdk-api-documentation-generation-validation:
- needs: [files-changed, build-sdk-package]
- if: |
- needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' &&
- needs.build-sdk-package.result == 'success'
+ if: inputs.documentation-validation
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
@@ -143,21 +110,15 @@ jobs:
# See `Component e2e tests` in `enterprise/frontend/src/embedding-sdk-package/dev.md`
embedding-sdk-component-tests:
name: "SDK component tests"
- needs: [build-sdk-package, files-changed]
- if: needs.files-changed.outputs.e2e_embedding_sdk == 'true'
+ if: inputs.component-tests
uses: ./.github/workflows/embedding-sdk-component-tests.yml
secrets: inherit
- with:
- cached-embedding-sdk-dist-artifact-name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}
not-required-embedding-sdk-apps-compatibility-tests:
name: "SDK e2e tests for compatibility with (host/sample) apps"
- needs: [build-sdk-package, files-changed]
- if: needs.files-changed.outputs.e2e_embedding_sdk_compatibility == 'true'
- uses: ./.github/workflows/embedding-sdk-e2e-tests.yml
+ if: inputs.host-sample-apps-tests
+ uses: ./.github/workflows/embedding-sdk-host-sample-apps-tests.yml
secrets: inherit
- with:
- cached-embedding-sdk-dist-artifact-name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}
embedding-sdk-tests-result:
needs:
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 8c776e212f3..790395fe526 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -25,6 +25,11 @@ jobs:
frontend_all: ${{ steps.changes.outputs.frontend_all }}
frontend_sources: ${{ steps.changes.outputs.frontend_sources }}
documentation: ${{ steps.changes.outputs.documentation }}
+ frontend_embedding_sdk_sources: ${{ steps.changes.outputs.frontend_embedding_sdk_sources }}
+ embedding_documentation: ${{ steps.changes.outputs.embedding_documentation }}
+ embedding_sdk_components: ${{ steps.changes.outputs.embedding_sdk_components }}
+ embedding_sdk_host_sample_apps: ${{ steps.changes.outputs.embedding_sdk_host_sample_apps }}
+ embedding_sdk_ci: ${{ steps.changes.outputs.embedding_sdk_ci }}
steps:
- uses: actions/checkout@v4
- name: Test which files changed
@@ -92,10 +97,30 @@ jobs:
# list of changed specs files, then only specs are changed and we can run only those specs
specs: ${{ needs.files-changed.outputs.e2e_all_files == needs.files-changed.outputs.e2e_specs_files && needs.files-changed.outputs.e2e_specs_files || '' }}
+ embedding-sdk-package:
+ needs: [files-changed]
+ if: |
+ !cancelled() &&
+ (needs.files-changed.outputs.embedding_sdk_components == 'true' ||
+ needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' ||
+ needs.files-changed.outputs.embedding_documentation == 'true' ||
+ needs.files-changed.outputs.embedding_sdk_ci == 'true')
+ uses: ./.github/workflows/embedding-sdk-package-build.yml
+ secrets: inherit
+
sdk-tests:
- needs: [uberjar]
+ needs: [uberjar, embedding-sdk-package, files-changed]
+ if: |
+ needs.embedding-sdk-package.result == 'success' &&
+ needs.uberjar.result == 'success'
uses: ./.github/workflows/embedding-sdk.yml
secrets: inherit
+ with:
+ cli-snippets-type-check: ${{ needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' }}
+ docs-snippets-type-check: ${{ needs.files-changed.outputs.embedding_documentation == 'true' }}
+ documentation-validation: ${{ needs.files-changed.outputs.frontend_embedding_sdk_sources == 'true' }}
+ component-tests: ${{ needs.files-changed.outputs.embedding_sdk_components == 'true' }}
+ host-sample-apps-tests: ${{ needs.files-changed.outputs.embedding_sdk_host_sample_apps == 'true' }}
pr-env:
needs: [uberjar]