Skip to content

The open alternative to Salesforce, designed for AI.

App media recording via standard web API polyfills (compare with #24053) (#24203)

Félix Malfait3ea46b5453bf
diff --git a/.github/workflows/ci-example-app-media-notes.yaml b/.github/workflows/ci-example-app-media-notes.yaml
new file mode 100644
index 00000000000..e2c24a5f842
--- /dev/null
+++ b/.github/workflows/ci-example-app-media-notes.yaml
@@ -0,0 +1,173 @@
+name: CI Example App Media Notes
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+ changed-files-check:
+ uses: ./.github/workflows/changed-files.yaml
+ with:
+ files: |
+ packages/twenty-apps/examples/media-notes/**
+ packages/twenty-sdk/**
+ packages/twenty-front/src/modules/front-components/**
+ packages/twenty-front-component-renderer/**
+ packages/twenty-shared/**
+ packages/twenty-server/src/engine/core-modules/file/**
+ !packages/twenty-sdk/package.json
+ !packages/twenty-shared/package.json
+
+ example-app-media-notes:
+ needs: changed-files-check
+ if: needs.changed-files-check.outputs.any_changed == 'true'
+ timeout-minutes: 30
+ runs-on: ubuntu-latest-8-cores
+ services:
+ postgres:
+ image: postgres:18
+ env:
+ POSTGRES_USER: postgres
+ POSTGRES_PASSWORD: postgres
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ redis:
+ image: redis
+ ports:
+ - 6379:6379
+ env:
+ NODE_OPTIONS: '--max-old-space-size=6144'
+ TWENTY_API_URL: http://localhost:3000
+ TWENTY_API_KEY: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIyMDIwMjAyMC1lNmI1LTQ2ODAtOGEzMi1iODIwOTczNzE1NmIiLCJ1c2VySWQiOiIyMDIwMjAyMC1lNmI1LTQ2ODAtOGEzMi1iODIwOTczNzE1NmIiLCJ3b3Jrc3BhY2VJZCI6IjIwMjAyMDIwLTFjMjUtNGQwMi1iZjI1LTZhZWNjZjdlYTQxOSIsIndvcmtzcGFjZU1lbWJlcklkIjoiMjAyMDIwMjAtNDYzZi00MzViLTgyOGMtMTA3ZTAwN2EyNzExIiwidXNlcldvcmtzcGFjZUlkIjoiMjAyMDIwMjAtMWU3Yy00M2Q5LWE1ZGItNjg1YjUwNjlkODE2IiwidHlwZSI6IkFDQ0VTUyIsImF1dGhQcm92aWRlciI6InBhc3N3b3JkIiwiaWF0IjoxNzUxMjgxNzA0LCJleHAiOjIwNjY4NTc3MDR9.HMGqCsVlOAPVUBhKSGlD1X86VoHKt4LIUtET3CGIdik
+ steps:
+ - name: Checkout
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+ - name: Install dependencies
+ uses: ./.github/actions/yarn-install
+
+ - name: Build SDK packages
+ run: npx nx build twenty-sdk
+
+ # Cheap checks first: a broken app never gets past publish, so failing
+ # here avoids paying for the server + browser boot below.
+ #
+ # The example app is outside the root workspaces list, so tsc and oxlint
+ # come from the root node_modules that yarn hoists them into. --no-install
+ # makes that dependency explicit: if hoisting ever stops providing them,
+ # this fails loudly instead of npx silently fetching an unrelated registry
+ # package and exiting 0 on a check that never ran.
+ - name: Typecheck and lint media-notes app
+ working-directory: packages/twenty-apps/examples/media-notes
+ run: |
+ npx --no-install tsc --noEmit -p tsconfig.json
+ npx --no-install oxlint -c .oxlintrc.json .
+
+ - name: Setup environment files
+ run: |
+ npx nx reset:env twenty-front
+ npx nx reset:env:e2e-testing-server twenty-server
+
+ # The shipped image compiles catalogs before building the front, so the
+ # generated ones are never committed. Without this the e2e would run
+ # against a build that shows message ids where the docker image shows text.
+ - name: Compile translations
+ run: |
+ npx nx run twenty-front:lingui:extract
+ npx nx run twenty-front:lingui:compile
+
+ - name: Build frontend
+ run: NODE_ENV=production npx nx build twenty-front
+
+ - name: Build server
+ run: npx nx build twenty-server
+
+ - name: Create databases
+ run: |
+ PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
+ PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
+
+ - name: Setup database
+ run: npx nx run twenty-server:database:reset
+
+ # Cookie sessions are credentialed, so the front only authenticates from
+ # the API's own origin — workspace subdomains included, which the login
+ # flow lands on. Serving the build from the server keeps every request
+ # same-origin, the same way the twenty docker image ships the front.
+ - name: Serve the frontend from the server
+ run: cp -r packages/twenty-front/build packages/twenty-server/dist/front
+
+ # start:ci runs nest start; the watch-mode target would rimraf dist and
+ # delete the front we just copied.
+ - name: Start server
+ run: nohup npx nx start:ci twenty-server &
+
+ - name: Wait for server to be ready
+ run: npx wait-on http://localhost:3000/healthz --timeout 120000 --interval 1000
+
+ - name: Configure remote for SDK CLI
+ run: |
+ mkdir -p ~/.twenty
+ cat > ~/.twenty/config.json <<EOF
+ {
+ "version": 1,
+ "remotes": {
+ "target": {
+ "apiUrl": "${TWENTY_API_URL}",
+ "apiKey": "${TWENTY_API_KEY}",
+ "accessToken": "${TWENTY_API_KEY}"
+ }
+ },
+ "defaultRemote": "target"
+ }
+ EOF
+
+ - name: Publish media-notes app
+ working-directory: packages/twenty-apps/examples/media-notes
+ run: node ${{ github.workspace }}/packages/twenty-sdk/dist/cli.cjs app:publish --private --remote target
+
+ - name: Install media-notes app
+ working-directory: packages/twenty-apps/examples/media-notes
+ run: node ${{ github.workspace }}/packages/twenty-sdk/dist/cli.cjs app:install --remote target
+
+ - name: Install Playwright chromium
+ run: npx playwright install --with-deps chromium
+
+ - name: Run media capture e2e test
+ working-directory: packages/twenty-apps/examples/media-notes
+ run: npx playwright test --project=setup --project=chromium
+ env:
+ FRONT_BASE_URL: http://localhost:3000
+
+ - name: Upload screenshots
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: media-notes-e2e-screenshots
+ path: packages/twenty-apps/examples/media-notes/e2e/.results/
+ retention-days: 7
+
+ ci-example-app-media-notes-status-check:
+ if: always() && !cancelled()
+ timeout-minutes: 5
+ runs-on: ubuntu-latest
+ needs: [changed-files-check, example-app-media-notes]
+ steps:
+ - name: Fail job if any needs failed
+ if: contains(needs.*.result, 'failure')
+ run: exit 1
diff --git a/packages/twenty-apps/examples/media-notes/.gitignore b/packages/twenty-apps/examples/media-notes/.gitignore
new file mode 100644
index 00000000000..3ce82946bab
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/.gitignore
@@ -0,0 +1,42 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn
+
+# codegen
+generated
+
+# testing
+/coverage
+/e2e/.auth
+/e2e/.results
+/playwright-report
+/test-results
+
+# dev
+/dist/
+
+.twenty
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# typescript
+*.tsbuildinfo
+*.d.ts
diff --git a/packages/twenty-apps/examples/media-notes/.oxlintrc.json b/packages/twenty-apps/examples/media-notes/.oxlintrc.json
new file mode 100644
index 00000000000..369b9cfd762
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/.oxlintrc.json
@@ -0,0 +1,37 @@
+{
+ "$schema": "./node_modules/oxlint/configuration_schema.json",
+ "plugins": ["typescript"],
+ "categories": {
+ "correctness": "off"
+ },
+ "ignorePatterns": ["node_modules", "dist"],
+ "rules": {
+ "no-unused-vars": "off",
+
+ "typescript/no-unused-vars": [
+ "warn",
+ {
+ "argsIgnorePattern": "^_"
+ }
+ ],
+ "typescript/no-explicit-any": "off"
+ },
+ "overrides": [
+ {
+ "files": ["**/*.logic-function.ts", "**/logic-functions/**/*.ts"],
+ "rules": {
+ "no-restricted-imports": [
+ "error",
+ {
+ "patterns": [
+ {
+ "group": ["twenty-shared", "twenty-shared/*"],
+ "message": "Logic functions must not import from twenty-shared directly. Import runtime types and helpers from `twenty-sdk/logic-function` instead so the logic-function bundle stays minimal."
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/packages/twenty-apps/examples/media-notes/.yarnrc.yml b/packages/twenty-apps/examples/media-notes/.yarnrc.yml
new file mode 100644
index 00000000000..3186f3f0795
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/.yarnrc.yml
@@ -0,0 +1 @@
+nodeLinker: node-modules
diff --git a/packages/twenty-apps/examples/media-notes/README.md b/packages/twenty-apps/examples/media-notes/README.md
new file mode 100644
index 00000000000..6a3095e1ce4
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/README.md
@@ -0,0 +1,42 @@
+# Media Notes
+
+Example Twenty app demonstrating media recording with the **standard web
+APIs**: the sandbox polyfills `navigator.mediaDevices.getUserMedia` and
+`MediaRecorder`, so the recording code reads like it would in any web page.
+The only Twenty-specific call is `uploadFile` from
+`twenty-sdk/front-component`, which stores the recorded blob in a FILES
+field.
+
+A pinned global command ("Record media note") opens a front component that
+records a voice or video note with its own UI, uploads it on stop, attaches
+it to a Media note record, then plays the stored file back from its signed
+URL.
+
+## Run the e2e regression test
+
+The test drives the full flow against a local Twenty instance with Chromium's
+fake media devices (no real microphone or camera needed):
+
+Cookie sessions are credentialed, so the front only authenticates from the
+API's own origin — workspace subdomains included, which the login flow lands
+on. Serve the front build from the server rather than on its own port:
+
+```sh
+# from the repo root
+npx nx build twenty-sdk
+NODE_ENV=production npx nx build twenty-front
+npx nx build twenty-server
+cp -r packages/twenty-front/build packages/twenty-server/dist/front
+
+# start:ci, not start: the watch target would rimraf dist and delete the front
+npx nx start:ci twenty-server &
+
+node packages/twenty-sdk/dist/cli.cjs app:publish --private && node packages/twenty-sdk/dist/cli.cjs app:install
+
+# then, from this directory
+FRONT_BASE_URL=http://localhost:3000 npx playwright test --project=setup --project=chromium
+```
+
+It asserts the app-owned recording timer, the playback preview, the uploaded
+file reference received by the component, and the cancel path — screenshots
+of each step land in `e2e/.results/screenshots/`.
diff --git a/packages/twenty-apps/examples/media-notes/e2e/auth.setup.ts b/packages/twenty-apps/examples/media-notes/e2e/auth.setup.ts
new file mode 100644
index 00000000000..637ed3f5b0f
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/e2e/auth.setup.ts
@@ -0,0 +1,68 @@
+import { type Locator, expect, test as setup } from '@playwright/test';
+import * as fs from 'fs';
+import * as path from 'path';
+
+const AUTH_DIR = path.resolve(__dirname, '.auth');
+const STORAGE_STATE = path.join(AUTH_DIR, 'user.json');
+const WORKSPACE_ORIGIN_FILE = path.join(AUTH_DIR, 'workspace-origin.txt');
+
+const LOGIN = process.env.E2E_LOGIN ?? '[email protected]';
+const PASSWORD = process.env.E2E_PASSWORD ?? '[email protected]';
+const WORKSPACE_NAME = process.env.E2E_WORKSPACE_NAME ?? 'Apple';
+
+const isVisible = async (locator: Locator) =>
+ locator.isVisible().catch(() => false);
+
+setup('authenticate', async ({ page }) => {
+ await page.goto('/');
+
+ // A fresh load shows the auth provider choice even when credentials are prefilled.
+ const continueWithEmail = page.getByRole('button', {
+ name: 'Continue with Email',
+ });
+ const emailField = page.getByPlaceholder('Email');
+
+ // Wait on the concrete auth UI rather than networkidle (flaky per Playwright).
+ await expect(continueWithEmail.or(emailField).first()).toBeVisible();
+
+ if (await isVisible(continueWithEmail)) {
+ await continueWithEmail.click();
+ }
+
+ if (await isVisible(emailField)) {
+ await emailField.fill(LOGIN);
+ await page.getByRole('button', { name: 'Continue', exact: true }).click();
+ }
+
+ const passwordField = page.getByPlaceholder('Password');
+ await passwordField.waitFor({ state: 'visible' });
+ await passwordField.fill(PASSWORD);
+
+ const signInButton = page.getByRole('button', { name: 'Sign in' });
+ await expect(signInButton).toBeEnabled();
+ await signInButton.click();
+
+ // Multi-workspace logins land on a picker; single-workspace logins skip it.
+ const workspacePicker = page.getByText('Choose a workspace');
+ const reachedPicker = await workspacePicker
+ .waitFor({ state: 'visible', timeout: 10_000 })
+ .then(() => true)
+ .catch(() => false);
+
+ if (reachedPicker) {
+ await page.getByText(WORKSPACE_NAME, { exact: true }).click();
+ await page.waitForFunction(() => window.location.href.includes('verify'));
+ await page.waitForFunction(() => !window.location.href.includes('verify'));
+ }
+
+ await page.waitForFunction(
+ () => window.localStorage.getItem('tokenPairState') !== null,
+ undefined,
+ { timeout: 15_000 },
+ );
+
+ fs.mkdirSync(AUTH_DIR, { recursive: true });
+ fs.writeFileSync(WORKSPACE_ORIGIN_FILE, new URL(page.url()).origin);
+
+ await page.context().storageState({ path: STORAGE_STATE });
+});
diff --git a/packages/twenty-apps/examples/media-notes/e2e/media-notes.spec.ts b/packages/twenty-apps/examples/media-notes/e2e/media-notes.spec.ts
new file mode 100644
index 00000000000..cf257ae1354
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/e2e/media-notes.spec.ts
@@ -0,0 +1,135 @@
+import { expect, test } from '@playwright/test';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { MEDIA_NOTES_TEST_IDS } from '../src/components/media-notes-test-ids';
+
+const WORKSPACE_ORIGIN_FILE = path.resolve(
+ __dirname,
+ '.auth',
+ 'workspace-origin.txt',
+);
+
+const SCREENSHOT_DIR = path.resolve(__dirname, '.results', 'screenshots');
+
+const resolveWorkspaceUrl = (): string => {
+ const fromEnv = process.env.E2E_WORKSPACE_URL;
+ if (fromEnv) {
+ return fromEnv.replace(/\/$/, '');
+ }
+
+ try {
+ return fs
+ .readFileSync(WORKSPACE_ORIGIN_FILE, 'utf8')
+ .trim()
+ .replace(/\/$/, '');
+ } catch {
+ return 'http://app.localhost:3001';
+ }
+};
+
+test.describe('Media notes capture flow', () => {
+ test.beforeAll(() => {
+ fs.mkdirSync(SCREENSHOT_DIR, { recursive: true });
+ });
+
+ // The front component runs in a sandboxed worker; surface browser output so
+ // failures inside it are diagnosable from the test log.
+ test.beforeEach(({ page }) => {
+ page.on('console', (message) => {
+ console.log(`[browser:${message.type()}] ${message.text()}`);
+ });
+ page.on('pageerror', (error) => {
+ console.log(`[pageerror] ${error.message}`);
+ });
+ page.on('response', (response) => {
+ if (response.status() >= 400) {
+ console.log(
+ `[response ${response.status()}] ${response.request().method()} ${response.url()}`,
+ );
+ }
+ });
+ });
+
+ const openMediaNotesComponent = async (
+ page: import('@playwright/test').Page,
+ ) => {
+ await page.goto(`${resolveWorkspaceUrl()}/`);
+
+ // The pinned global command renders as a top bar action button.
+ await page
+ .getByRole('button', { name: 'Record media note' })
+ .first()
+ .click();
+
+ await expect(page.getByTestId(MEDIA_NOTES_TEST_IDS.root)).toBeVisible();
+ };
+
+ test('records an audio note end to end', async ({ page }) => {
+ await openMediaNotesComponent(page);
+
+ await page.getByTestId(MEDIA_NOTES_TEST_IDS.recordAudioButton).click();
+
+ // The recording UX is the app's own UI: its timer and stop control.
+ const stopButton = page.getByTestId(
+ MEDIA_NOTES_TEST_IDS.stopRecordingButton,
+ );
+ await expect(stopButton).toBeVisible();
+ await expect(
+ page.getByTestId(MEDIA_NOTES_TEST_IDS.recordingTimer),
+ ).toBeVisible();
+
+ await page.waitForTimeout(2500);
+ await page.screenshot({
+ path: path.join(SCREENSHOT_DIR, '01-recording.png'),
+ });
+
+ await stopButton.click();
+
+ // Uploaded: the component receives a playable file reference.
+ await expect(
+ page.getByTestId(MEDIA_NOTES_TEST_IDS.captureStatus),
+ ).toHaveText('captured');
+ const capturedAudio = page.getByTestId(MEDIA_NOTES_TEST_IDS.capturedAudio);
+ await expect(capturedAudio).toBeVisible();
+
+ const audioSrc = await capturedAudio.getAttribute('src');
+ expect(audioSrc).toBeTruthy();
+ expect(audioSrc).toContain('/file');
+
+ // Actually fetch it: a signed url for a folder the file guard does not
+ // serve still looks correct in the src attribute but answers 401/403, so
+ // asserting on the string alone would not catch an unplayable recording.
+ const mediaResponse = await page.request.get(audioSrc as string);
+ expect(mediaResponse.status()).toBe(200);
+ // The body, not content-length: the file route streams the response, so
+ // the header is absent even on a perfectly good recording.
+ expect((await mediaResponse.body()).length).toBeGreaterThan(0);
+
+ // The recording is attached to a record, which is what makes the
+ // uploaded file permanent instead of a temporary orphan.
+ await expect(
+ page.getByTestId(MEDIA_NOTES_TEST_IDS.savedRecord),
+ ).toContainText('Attached to media note');
+
+ await page.screenshot({
+ path: path.join(SCREENSHOT_DIR, '02-captured.png'),
+ });
+ });
+
+ test('cancelling a recording resolves as cancelled', async ({ page }) => {
+ await openMediaNotesComponent(page);
+
+ await page.getByTestId(MEDIA_NOTES_TEST_IDS.recordAudioButton).click();
+
+ await expect(
+ page.getByTestId(MEDIA_NOTES_TEST_IDS.recordingTimer),
+ ).toBeVisible();
+
+ await page.getByTestId(MEDIA_NOTES_TEST_IDS.cancelRecordingButton).click();
+
+ await expect(
+ page.getByTestId(MEDIA_NOTES_TEST_IDS.captureStatus),
+ ).toHaveText('cancelled');
+ });
+});
diff --git a/packages/twenty-apps/examples/media-notes/package.json b/packages/twenty-apps/examples/media-notes/package.json
new file mode 100644
index 00000000000..574f5bc6c57
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/package.json
@@ -0,0 +1,34 @@
+{
+ "name": "media-notes",
+ "version": "0.1.0",
+ "license": "MIT",
+ "engines": {
+ "node": "^24.5.0",
+ "npm": "please-use-yarn",
+ "yarn": ">=4.0.2",
+ "twenty": ">=2.24.0"
+ },
+ "keywords": [
+ "twenty-app"
+ ],
+ "packageManager": "[email protected]",
+ "scripts": {
+ "twenty": "twenty",
+ "lint": "oxlint -c .oxlintrc.json .",
+ "lint:fix": "oxlint --fix -c .oxlintrc.json .",
+ "test:e2e": "playwright test",
+ "test:e2e:ui": "playwright test --ui"
+ },
+ "devDependencies": {
+ "@playwright/test": "^1.60.0",
+ "@types/node": "^24.7.2",
+ "@types/react": "^19.0.0",
+ "oxlint": "^0.16.0",
+ "playwright": "^1.60.0",
+ "react": "^19.0.0",
+ "react-dom": "^19.0.0",
+ "twenty-client-sdk": "2.32.0",
+ "twenty-sdk": "2.32.0",
+ "typescript": "^5.9.3"
+ }
+}
diff --git a/packages/twenty-apps/examples/media-notes/playwright.config.ts b/packages/twenty-apps/examples/media-notes/playwright.config.ts
new file mode 100644
index 00000000000..df73daf6a40
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/playwright.config.ts
@@ -0,0 +1,55 @@
+import { defineConfig, devices } from '@playwright/test';
+import * as path from 'path';
+
+// Front-end base URL of the running Twenty instance under test.
+const FRONT_BASE_URL = process.env.FRONT_BASE_URL ?? 'http://localhost:3001';
+
+export default defineConfig({
+ testDir: './e2e',
+ outputDir: './e2e/.results',
+ fullyParallel: false,
+ forbidOnly: !!process.env.CI,
+ retries: process.env.CI ? 2 : 0,
+ workers: 1,
+ timeout: 90 * 1000,
+ reporter: process.env.CI ? 'github' : 'list',
+ use: {
+ baseURL: FRONT_BASE_URL,
+ trace: 'retain-on-failure',
+ screenshot: 'only-on-failure',
+ headless: true,
+ testIdAttribute: 'data-testid',
+ launchOptions: {
+ executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH,
+ },
+ },
+ expect: {
+ timeout: 15_000,
+ },
+ projects: [
+ {
+ name: 'setup',
+ testMatch: /.*\.setup\.ts/,
+ },
+ // Chromium only: the fake media device flags that make getUserMedia and
+ // MediaRecorder deterministic in CI are Chromium-specific.
+ {
+ name: 'chromium',
+ use: {
+ ...devices['Desktop Chrome'],
+ storageState: path.resolve(__dirname, 'e2e/.auth/user.json'),
+ permissions: ['microphone', 'camera'],
+ launchOptions: {
+ // Lets sandboxed environments point at a preinstalled Chromium
+ // instead of downloading the pinned revision.
+ executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH,
+ args: [
+ '--use-fake-ui-for-media-stream',
+ '--use-fake-device-for-media-stream',
+ ],
+ },
+ },
+ dependencies: ['setup'],
+ },
+ ],
+});
diff --git a/packages/twenty-apps/examples/media-notes/src/application.config.ts b/packages/twenty-apps/examples/media-notes/src/application.config.ts
new file mode 100644
index 00000000000..d6a6a031dca
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/src/application.config.ts
@@ -0,0 +1,13 @@
+import { defineApplication } from 'twenty-sdk/define';
+import { DEFAULT_ROLE_UNIVERSAL_IDENTIFIER } from './roles/default.role';
+
+export const APPLICATION_UNIVERSAL_IDENTIFIER =
+ 'c832302c-e551-4b4f-b11c-19907888a284';
+
+export default defineApplication({
+ universalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
+ displayName: 'Media Notes',
+ description:
+ 'Example app demonstrating the recordAudio / recordVideo front component capability',
+ defaultRoleUniversalIdentifier: DEFAULT_ROLE_UNIVERSAL_IDENTIFIER,
+});
diff --git a/packages/twenty-apps/examples/media-notes/src/command-menu-items/open-media-notes.command-menu-item.ts b/packages/twenty-apps/examples/media-notes/src/command-menu-items/open-media-notes.command-menu-item.ts
new file mode 100644
index 00000000000..3760bf7fe1d
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/src/command-menu-items/open-media-notes.command-menu-item.ts
@@ -0,0 +1,13 @@
+import { defineCommandMenuItem } from 'twenty-sdk/define';
+import { MEDIA_NOTES_FRONT_COMPONENT_UNIVERSAL_IDENTIFIER } from '../components/media-notes.front-component';
+
+export default defineCommandMenuItem({
+ universalIdentifier: '83d9d1ba-b042-41c1-94e8-892931d8663f',
+ label: 'Record media note',
+ shortLabel: 'Media note',
+ icon: 'IconMicrophone',
+ isPinned: true,
+ availabilityType: 'GLOBAL',
+ frontComponentUniversalIdentifier:
+ MEDIA_NOTES_FRONT_COMPONENT_UNIVERSAL_IDENTIFIER,
+});
diff --git a/packages/twenty-apps/examples/media-notes/src/components/media-notes-test-ids.ts b/packages/twenty-apps/examples/media-notes/src/components/media-notes-test-ids.ts
new file mode 100644
index 00000000000..7acdda2dad5
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/src/components/media-notes-test-ids.ts
@@ -0,0 +1,15 @@
+export const MEDIA_NOTES_TEST_IDS = {
+ root: 'media-notes-root',
+ recordAudioButton: 'media-notes-record-audio',
+ recordVideoButton: 'media-notes-record-video',
+ recordingTimer: 'media-notes-recording-timer',
+ stopRecordingButton: 'media-notes-stop-recording',
+ cancelRecordingButton: 'media-notes-cancel-recording',
+ captureStatus: 'media-notes-capture-status',
+ capturedFile: 'media-notes-captured-file',
+ capturedAudio: 'media-notes-captured-audio',
+ capturedVideo: 'media-notes-captured-video',
+ savedRecord: 'media-notes-saved-record',
+ attachError: 'media-notes-attach-error',
+ retryAttachButton: 'media-notes-retry-attach',
+};
diff --git a/packages/twenty-apps/examples/media-notes/src/components/media-notes.front-component.tsx b/packages/twenty-apps/examples/media-notes/src/components/media-notes.front-component.tsx
new file mode 100644
index 00000000000..f8d11613e94
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/src/components/media-notes.front-component.tsx
@@ -0,0 +1,649 @@
+import { useEffect, useRef, useState } from 'react';
+import { CoreApiClient } from 'twenty-client-sdk/core';
+import { MetadataApiClient } from 'twenty-client-sdk/metadata';
+import { defineFrontComponent } from 'twenty-sdk/define';
+import {
+ type UploadedFrontComponentFile,
+ uploadFile,
+} from 'twenty-sdk/front-component';
+
+import { RECORDING_FIELD_UNIVERSAL_IDENTIFIER } from '../objects/media-note.object';
+import { MEDIA_NOTES_TEST_IDS } from './media-notes-test-ids';
+
+export const MEDIA_NOTES_FRONT_COMPONENT_UNIVERSAL_IDENTIFIER =
+ '14695714-35b5-4814-9489-f65c12368b5c';
+
+const buttonStyle: React.CSSProperties = {
+ backgroundColor: '#1961ed',
+ border: 'none',
+ borderRadius: '4px',
+ color: '#ffffff',
+ cursor: 'pointer',
+ fontSize: '13px',
+ fontWeight: 500,
+ marginRight: '8px',
+ padding: '8px 12px',
+};
+
+const OBJECTS_PAGE_SIZE = 100;
+
+const MAX_RECORDING_DURATION_SECONDS = 60;
+
+type ObjectsPage = {
+ fields: { id: string; universalIdentifier: string | null }[];
+ nextCursor: string | null;
+};
+
+const fetchObjectsPage = async (after: string | null): Promise<ObjectsPage> => {
+ const result = await new MetadataApiClient().query({
+ objects: {
+ __args: {
+ paging: { first: OBJECTS_PAGE_SIZE, ...(after ? { after } : {}) },
+ filter: {},
+ },
+ pageInfo: { hasNextPage: true, endCursor: true },
+ edges: {
+ node: { fieldsList: { id: true, universalIdentifier: true } },
+ },
+ },
+ });
+
+ const pageInfo = result.objects.pageInfo;
+
+ return {
+ fields: result.objects.edges.flatMap((edge) => edge.node.fieldsList ?? []),
+ nextCursor:
+ pageInfo.hasNextPage === true ? (pageInfo.endCursor ?? null) : null,
+ };
+};
+
+// An app declares its fields by universalIdentifier; the per-instance
+// fieldMetadataId the recording upload needs is resolved at runtime. The
+// metadata API cannot filter on universalIdentifier, so page through the
+// object list until the field turns up.
+const fetchRecordingFieldMetadataId = async (): Promise<string | null> => {
+ let cursor: string | null = null;
+
+ for (;;) {
+ const page: ObjectsPage = await fetchObjectsPage(cursor);
+
+ const recordingField = page.fields.find(
+ (field) =>
+ field.universalIdentifier === RECORDING_FIELD_UNIVERSAL_IDENTIFIER,
+ );
+
+ if (recordingField) {
+ return recordingField.id;
+ }
+
+ if (page.nextCursor === null) {
+ return null;
+ }
+
+ cursor = page.nextCursor;
+ }
+};
+
+const MIME_TYPE_TO_FILE_EXTENSION: Record<string, string> = {
+ 'audio/webm': 'webm',
+ 'video/webm': 'webm',
+ 'audio/mp4': 'm4a',
+ 'video/mp4': 'mp4',
+ 'audio/ogg': 'ogg',
+ 'video/ogg': 'ogg',
+};
+
+const getFileExtension = (mimeType: string): string =>
+ MIME_TYPE_TO_FILE_EXTENSION[mimeType.split(';')[0].trim().toLowerCase()] ??
+ 'webm';
+
+// The recording engine is the standard web platform: getUserMedia and
+// MediaRecorder, polyfilled into the sandbox by the host. Errors surface as
+// DOMException names exactly like in a regular page.
+const mapMediaErrorNameToReason = (errorName: string): string => {
+ switch (errorName) {
+ case 'NotAllowedError':
+ case 'SecurityError':
+ return 'permission-denied';
+ case 'NotFoundError':
+ case 'OverconstrainedError':
+ return 'no-device';
+ case 'NotReadableError':
+ return 'busy';
+ case 'NotSupportedError':
+ return 'blocked';
+ default:
+ return 'unknown';
+ }
+};
+
+const mapMediaErrorToReason = (error: unknown): string =>
+ error instanceof Error ? mapMediaErrorNameToReason(error.name) : 'unknown';
+
+type CapturedMediaFile = UploadedFrontComponentFile & {
+ durationSeconds: number;
+};
+
+type CaptureResult =
+ | { status: 'captured'; file: CapturedMediaFile }
+ | { status: 'cancelled' }
+ | { status: 'failed'; reason: string };
+
+const describeResult = (result: CaptureResult): string =>
+ result.status === 'failed' ? `failed:${result.reason}` : result.status;
+
+type PendingAttach = {
+ mediaType: 'audio' | 'video';
+ file: CapturedMediaFile;
+};
+
+type ActiveRecording = {
+ mediaType: 'audio' | 'video';
+ startedAt: number;
+ mediaStream: MediaStream;
+ mediaRecorder: MediaRecorder;
+};
+
+type RecordingSession = {
+ wasCancelled: boolean;
+ isUserStopping: boolean;
+ errorName: string | null;
+ collectRecordedBlob: Promise<Blob>;
+};
+
+const MediaNotes = () => {
+ const [recordingFieldMetadataId, setRecordingFieldMetadataId] = useState<
+ string | null
+ >(null);
+ const [captureResult, setCaptureResult] = useState<CaptureResult | null>(
+ null,
+ );
+ const [activeRecording, setActiveRecording] =
+ useState<ActiveRecording | null>(null);
+ const [elapsedSeconds, setElapsedSeconds] = useState(0);
+ const [savedRecordId, setSavedRecordId] = useState<string | null>(null);
+ // Set only when attaching failed, so it doubles as the retry payload.
+ const [failedAttach, setFailedAttach] = useState<PendingAttach | null>(null);
+ const [isAttaching, setIsAttaching] = useState(false);
+ // Covers the stop-and-upload window: without it the record buttons come
+ // back while the previous recording is still uploading, and two flows race.
+ const [isStopping, setIsStopping] = useState(false);
+
+ const recordingSessionRef = useRef<RecordingSession | null>(null);
+ const stopAndSaveRef = useRef<(() => void) | null>(null);
+ // Synchronous guard: a second click during the getUserMedia permission
+ // prompt would otherwise start a second capture.
+ const isStartingRef = useRef(false);
+
+ useEffect(() => {
+ fetchRecordingFieldMetadataId()
+ .then(setRecordingFieldMetadataId)
+ .catch(() => setRecordingFieldMetadataId(null));
+ }, []);
+
+ // The recording UX is the app's own: this timer is rendered and styled
+ // here, and the app also owns its duration ceiling — the host imposes
+ // none.
+ useEffect(() => {
+ if (activeRecording === null) {
+ return;
+ }
+
+ setElapsedSeconds(0);
+
+ const elapsedInterval = setInterval(() => {
+ const nextElapsedSeconds = Math.floor(
+ (Date.now() - activeRecording.startedAt) / 1000,
+ );
+
+ setElapsedSeconds(nextElapsedSeconds);
+
+ if (nextElapsedSeconds >= MAX_RECORDING_DURATION_SECONDS) {
+ stopAndSaveRef.current?.();
+ }
+ }, 1000);
+
+ return () => clearInterval(elapsedInterval);
+ }, [activeRecording]);
+
+ // Attaching the uploaded file to a record is what makes it permanent: until
+ // then it is a temporary file owned by the FILES field, so a failure here
+ // has to be recoverable rather than silent.
+ const attachToNewMediaNote = async (pendingAttach: PendingAttach) => {
+ setFailedAttach(null);
+ setIsAttaching(true);
+
+ try {
+ const created = await new CoreApiClient().mutation({
+ createMediaNote: {
+ __args: {
+ data: {
+ title: `${pendingAttach.mediaType} note`,
+ recording: [
+ {
+ fileId: pendingAttach.file.fileId,
+ label: pendingAttach.file.path,
+ },
+ ],
+ },
+ },
+ id: true,
+ },
+ });
+
+ setSavedRecordId(created.createMediaNote?.id ?? null);
+ } catch {
+ setFailedAttach(pendingAttach);
+ } finally {
+ setIsAttaching(false);
+ }
+ };
+
+ const handleStartRecording = async (mediaType: 'audio' | 'video') => {
+ // Starting a new recording mid-attach would let the in-flight attach
+ // settle against a recording the UI has already replaced, offering a
+ // retry for the wrong file.
+ if (
+ recordingFieldMetadataId === null ||
+ isAttaching ||
+ isStopping ||
+ isStartingRef.current ||
+ activeRecording !== null
+ ) {
+ return;
+ }
+
+ isStartingRef.current = true;
+ setCaptureResult(null);
+ setSavedRecordId(null);
+ setFailedAttach(null);
+
+ let acquiredMediaStream: MediaStream | null = null;
+
+ try {
+ const mediaStream = await navigator.mediaDevices.getUserMedia(
+ mediaType === 'audio' ? { audio: true } : { video: true, audio: true },
+ );
+
+ acquiredMediaStream = mediaStream;
+
+ const mediaRecorder = new MediaRecorder(mediaStream);
+
+ const recordedChunks: Blob[] = [];
+ const recordingSession: RecordingSession = {
+ wasCancelled: false,
+ isUserStopping: false,
+ errorName: null,
+ collectRecordedBlob: new Promise<Blob>((resolve) => {
+ mediaRecorder.onstop = () =>
+ resolve(
+ new Blob(recordedChunks, {
+ type:
+ mediaRecorder.mimeType ||
+ (mediaType === 'audio' ? 'audio/webm' : 'video/webm'),
+ }),
+ );
+ }),
+ };
+
+ mediaRecorder.ondataavailable = (event) => {
+ const dataEvent = event as Event & { data: Blob };
+
+ if (dataEvent.data.size > 0) {
+ recordedChunks.push(dataEvent.data);
+ }
+ };
+
+ mediaRecorder.onerror = (event) => {
+ const errorEvent = event as Event & { error?: Error };
+
+ recordingSession.errorName = errorEvent.error?.name ?? 'UnknownError';
+ };
+
+ // A stop the app did not ask for means the recorder failed to start on
+ // the host or died mid-recording (it fires error then stop, like the
+ // native API): release the devices and surface the failure.
+ mediaRecorder.addEventListener('stop', () => {
+ if (recordingSession.isUserStopping || recordingSession.wasCancelled) {
+ return;
+ }
+
+ recordingSession.wasCancelled = true;
+ recordingSessionRef.current = null;
+
+ for (const track of mediaStream.getTracks()) {
+ track.stop();
+ }
+
+ setActiveRecording(null);
+ setCaptureResult({
+ status: 'failed',
+ reason:
+ recordingSession.errorName === null
+ ? 'unknown'
+ : mapMediaErrorNameToReason(recordingSession.errorName),
+ });
+ });
+
+ // The host indicator's stop button (or a revoked device) surfaces as
+ // standard track ended events; the app reacts by discarding, without
+ // any bespoke callback from the host.
+ for (const track of mediaStream.getTracks()) {
+ track.onended = () => {
+ if (recordingSession.wasCancelled) {
+ return;
+ }
+
+ recordingSession.wasCancelled = true;
+ recordingSessionRef.current = null;
+
+ // One dead track ends the whole take: stop the recorder and the
+ // remaining tracks so nothing keeps capturing behind the
+ // cancelled state.
+ if (mediaRecorder.state !== 'inactive') {
+ mediaRecorder.stop();
+ }
+
+ for (const remainingTrack of mediaStream.getTracks()) {
+ if (remainingTrack.readyState === 'live') {
+ remainingTrack.stop();
+ }
+ }
+
+ setActiveRecording(null);
+ setCaptureResult({ status: 'cancelled' });
+ };
+ }
+
+ mediaRecorder.start();
+
+ recordingSessionRef.current = recordingSession;
+ setActiveRecording({
+ mediaType,
+ startedAt: Date.now(),
+ mediaStream,
+ mediaRecorder,
+ });
+ } catch (error) {
+ // The recorder can fail after the devices were granted; do not leave
+ // them capturing behind a failure message.
+ if (acquiredMediaStream !== null) {
+ for (const track of acquiredMediaStream.getTracks()) {
+ track.stop();
+ }
+ }
+
+ setCaptureResult({
+ status: 'failed',
+ reason: mapMediaErrorToReason(error),
+ });
+ } finally {
+ isStartingRef.current = false;
+ }
+ };
+
+ const handleStopRecording = async () => {
+ if (activeRecording === null) {
+ return;
+ }
+
+ const { mediaRecorder, mediaStream, mediaType, startedAt } =
+ activeRecording;
+ const recordingSession = recordingSessionRef.current;
+
+ if (recordingSession) {
+ recordingSession.isUserStopping = true;
+ }
+
+ setActiveRecording(null);
+ setIsStopping(true);
+
+ try {
+ if (mediaRecorder.state !== 'inactive') {
+ mediaRecorder.stop();
+ }
+
+ const recordedBlob = recordingSession
+ ? await recordingSession.collectRecordedBlob
+ : null;
+
+ for (const track of mediaStream.getTracks()) {
+ track.stop();
+ }
+
+ if (
+ recordedBlob === null ||
+ recordingSession === null ||
+ recordingSession.wasCancelled
+ ) {
+ setCaptureResult({ status: 'cancelled' });
+
+ return;
+ }
+
+ // An errored recorder still resolves the stop, but with nothing worth
+ // uploading.
+ if (recordingSession.errorName !== null || recordedBlob.size === 0) {
+ setCaptureResult({
+ status: 'failed',
+ reason:
+ recordingSession.errorName === null
+ ? 'unknown'
+ : mapMediaErrorNameToReason(recordingSession.errorName),
+ });
+
+ return;
+ }
+
+ recordingSessionRef.current = null;
+
+ const durationSeconds = Math.max(
+ 1,
+ Math.round((Date.now() - startedAt) / 1000),
+ );
+
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const uploadResult = await uploadFile(recordedBlob, {
+ fieldMetadataId: recordingFieldMetadataId ?? '',
+ fileName: `${mediaType}-recording-${timestamp}.${getFileExtension(recordedBlob.type)}`,
+ });
+
+ if (uploadResult.status === 'failed') {
+ setCaptureResult({ status: 'failed', reason: uploadResult.reason });
+
+ return;
+ }
+
+ const capturedFile: CapturedMediaFile = {
+ ...uploadResult.file,
+ durationSeconds,
+ };
+
+ setCaptureResult({ status: 'captured', file: capturedFile });
+
+ await attachToNewMediaNote({ mediaType, file: capturedFile });
+ } finally {
+ setIsStopping(false);
+ }
+ };
+
+ stopAndSaveRef.current = handleStopRecording;
+
+ const handleCancelRecording = async () => {
+ if (activeRecording === null) {
+ return;
+ }
+
+ const { mediaRecorder, mediaStream } = activeRecording;
+ const recordingSession = recordingSessionRef.current;
+
+ if (recordingSession) {
+ recordingSession.wasCancelled = true;
+ }
+ recordingSessionRef.current = null;
+
+ setActiveRecording(null);
+
+ if (mediaRecorder.state !== 'inactive') {
+ mediaRecorder.stop();
+ }
+
+ for (const track of mediaStream.getTracks()) {
+ track.stop();
+ }
+
+ setCaptureResult({ status: 'cancelled' });
+ };
+
+ return (
+ <div
+ data-testid={MEDIA_NOTES_TEST_IDS.root}
+ style={{ fontFamily: 'sans-serif', padding: '16px' }}
+ >
+ <h3 style={{ fontSize: '15px', margin: '0 0 12px' }}>Media notes</h3>
+ <p style={{ color: '#555', fontSize: '13px', margin: '0 0 16px' }}>
+ Record a note with your microphone or camera. The recording is stored in
+ Twenty and attached to a Media note record.
+ </p>
+
+ {activeRecording === null ? (
+ <div style={{ marginBottom: '16px' }}>
+ <button
+ data-testid={MEDIA_NOTES_TEST_IDS.recordAudioButton}
+ style={buttonStyle}
+ disabled={
+ recordingFieldMetadataId === null || isAttaching || isStopping
+ }
+ onClick={() => handleStartRecording('audio')}
+ >
+ Record a voice note
+ </button>
+ <button
+ data-testid={MEDIA_NOTES_TEST_IDS.recordVideoButton}
+ style={buttonStyle}
+ disabled={
+ recordingFieldMetadataId === null || isAttaching || isStopping
+ }
+ onClick={() => handleStartRecording('video')}
+ >
+ Record a video note
+ </button>
+ </div>
+ ) : (
+ <div
+ style={{
+ alignItems: 'center',
+ display: 'flex',
+ gap: '8px',
+ marginBottom: '16px',
+ }}
+ >
+ <span
+ style={{
+ backgroundColor: '#b42318',
+ borderRadius: '50%',
+ display: 'inline-block',
+ height: '10px',
+ width: '10px',
+ }}
+ />
+ <span
+ data-testid={MEDIA_NOTES_TEST_IDS.recordingTimer}
+ style={{ fontSize: '13px', fontVariantNumeric: 'tabular-nums' }}
+ >
+ Recording {activeRecording.mediaType}… {elapsedSeconds}s
+ </span>
+ <button
+ data-testid={MEDIA_NOTES_TEST_IDS.stopRecordingButton}
+ style={buttonStyle}
+ onClick={handleStopRecording}
+ >
+ Stop and save
+ </button>
+ <button
+ data-testid={MEDIA_NOTES_TEST_IDS.cancelRecordingButton}
+ style={{ ...buttonStyle, backgroundColor: '#666' }}
+ onClick={handleCancelRecording}
+ >
+ Cancel
+ </button>
+ </div>
+ )}
+
+ {captureResult !== null && (
+ <div
+ data-testid={MEDIA_NOTES_TEST_IDS.captureStatus}
+ style={{ color: '#333', fontSize: '13px', marginBottom: '12px' }}
+ >
+ {describeResult(captureResult)}
+ </div>
+ )}
+
+ {captureResult?.status === 'captured' && (
+ <div
+ data-testid={MEDIA_NOTES_TEST_IDS.capturedFile}
+ style={{
+ border: '1px solid #ddd',
+ borderRadius: '6px',
+ fontSize: '12px',
+ padding: '12px',
+ }}
+ >
+ <p style={{ margin: '0 0 8px' }}>
+ {captureResult.file.mimeType} · {captureResult.file.durationSeconds}
+ s · {Math.round(captureResult.file.size / 1024)} KB
+ </p>
+ {captureResult.file.mimeType.startsWith('audio/') ? (
+ <audio
+ data-testid={MEDIA_NOTES_TEST_IDS.capturedAudio}
+ controls
+ src={captureResult.file.url}
+ style={{ width: '100%' }}
+ />
+ ) : (
+ <video
+ data-testid={MEDIA_NOTES_TEST_IDS.capturedVideo}
+ controls
+ src={captureResult.file.url}
+ style={{ maxHeight: '200px', width: '100%' }}
+ />
+ )}
+ {savedRecordId !== null && (
+ <p
+ data-testid={MEDIA_NOTES_TEST_IDS.savedRecord}
+ style={{ color: '#555', margin: '8px 0 0' }}
+ >
+ Attached to media note {savedRecordId}
+ </p>
+ )}
+ {failedAttach !== null && (
+ <div
+ data-testid={MEDIA_NOTES_TEST_IDS.attachError}
+ style={{ color: '#b42318', margin: '8px 0 0' }}
+ >
+ <p style={{ margin: '0 0 8px' }}>
+ The recording was uploaded but could not be attached to a media
+ note. It stays temporary until it is attached.
+ </p>
+ <button
+ data-testid={MEDIA_NOTES_TEST_IDS.retryAttachButton}
+ style={buttonStyle}
+ disabled={isAttaching}
+ onClick={() => attachToNewMediaNote(failedAttach)}
+ >
+ Retry attaching
+ </button>
+ </div>
+ )}
+ </div>
+ )}
+ </div>
+ );
+};
+
+export default defineFrontComponent({
+ universalIdentifier: MEDIA_NOTES_FRONT_COMPONENT_UNIVERSAL_IDENTIFIER,
+ name: 'media-notes-component',
+ description: 'Record audio or video notes with standard web media APIs',
+ component: MediaNotes,
+});
diff --git a/packages/twenty-apps/examples/media-notes/src/objects/media-note.object.ts b/packages/twenty-apps/examples/media-notes/src/objects/media-note.object.ts
new file mode 100644
index 00000000000..e3aea77bec4
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/src/objects/media-note.object.ts
@@ -0,0 +1,41 @@
+import { FieldType, defineObject } from 'twenty-sdk/define';
+
+export const MEDIA_NOTE_UNIVERSAL_IDENTIFIER =
+ '68d3bcc0-1a6c-4e7d-9fc3-c7bc57b8241d';
+
+export const TITLE_FIELD_UNIVERSAL_IDENTIFIER =
+ 'c1d4e7a2-3b56-4c89-9e12-4f6a8b0d3e75';
+
+export const RECORDING_FIELD_UNIVERSAL_IDENTIFIER =
+ 'd2e5f8b3-4c67-4d9a-8f23-5a7b9c1e4f86';
+
+export default defineObject({
+ universalIdentifier: MEDIA_NOTE_UNIVERSAL_IDENTIFIER,
+ nameSingular: 'mediaNote',
+ namePlural: 'mediaNotes',
+ labelSingular: 'Media note',
+ labelPlural: 'Media notes',
+ description: 'A note captured with the microphone or camera',
+ icon: 'IconMicrophone',
+ labelIdentifierFieldMetadataUniversalIdentifier:
+ TITLE_FIELD_UNIVERSAL_IDENTIFIER,
+ fields: [
+ {
+ universalIdentifier: TITLE_FIELD_UNIVERSAL_IDENTIFIER,
+ type: FieldType.TEXT,
+ label: 'Title',
+ description: "The media note's title",
+ icon: 'IconAbc',
+ name: 'title',
+ },
+ {
+ universalIdentifier: RECORDING_FIELD_UNIVERSAL_IDENTIFIER,
+ type: FieldType.FILES,
+ label: 'Recording',
+ description: 'The captured audio or video note',
+ icon: 'IconMicrophone',
+ name: 'recording',
+ universalSettings: { maxNumberOfValues: 10 },
+ },
+ ],
+});
diff --git a/packages/twenty-apps/examples/media-notes/src/roles/default.role.ts b/packages/twenty-apps/examples/media-notes/src/roles/default.role.ts
new file mode 100644
index 00000000000..b551bd747c1
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/src/roles/default.role.ts
@@ -0,0 +1,28 @@
+import { defineRole } from 'twenty-sdk/define';
+import { MEDIA_NOTE_UNIVERSAL_IDENTIFIER } from '../objects/media-note.object';
+
+export const DEFAULT_ROLE_UNIVERSAL_IDENTIFIER =
+ '04981304-eca8-4757-9e9a-254390eafc79';
+
+export default defineRole({
+ universalIdentifier: DEFAULT_ROLE_UNIVERSAL_IDENTIFIER,
+ label: 'Media notes default role',
+ description: 'Default role for the media notes example app',
+ canReadAllObjectRecords: false,
+ canUpdateAllObjectRecords: false,
+ canSoftDeleteAllObjectRecords: false,
+ canDestroyAllObjectRecords: false,
+ canUpdateAllSettings: false,
+ canBeAssignedToAgents: false,
+ canBeAssignedToUsers: false,
+ canBeAssignedToApiKeys: false,
+ objectPermissions: [
+ {
+ objectUniversalIdentifier: MEDIA_NOTE_UNIVERSAL_IDENTIFIER,
+ canReadObjectRecords: true,
+ canUpdateObjectRecords: true,
+ canSoftDeleteObjectRecords: false,
+ canDestroyObjectRecords: false,
+ },
+ ],
+});
diff --git a/packages/twenty-apps/examples/media-notes/tsconfig.json b/packages/twenty-apps/examples/media-notes/tsconfig.json
new file mode 100644
index 00000000000..04e4bc11815
--- /dev/null
+++ b/packages/twenty-apps/examples/media-notes/tsconfig.json
@@ -0,0 +1,39 @@
+{
+ "compileOnSave": false,
+ "compilerOptions": {
+ "sourceMap": true,
+ "declaration": true,
+ "outDir": "./dist",
+ "rootDir": ".",
+ "jsx": "react-jsx",
+ "moduleResolution": "bundler",
+ "allowSyntheticDefaultImports": true,
+ "emitDecoratorMetadata": true,
+ "experimentalDecorators": true,
+ "importHelpers": true,
+ "allowUnreachableCode": false,
+ "strict": true,
+ "alwaysStrict": true,
+ "noImplicitAny": true,
+ "strictBindCallApply": false,
+ "target": "es2018",
+ "module": "esnext",
+ "lib": ["es2020", "dom"],
+ "skipLibCheck": true,
+ "skipDefaultLibCheck": true,
+ "resolveJsonModule": true,
+ "paths": {
+ "src/*": ["./src/*"],
+ "~/*": ["./*"]
+ }
+ },
+ "exclude": [
+ "node_modules",
+ "dist",
+ "e2e",
+ "playwright.config.ts",
+ "**/*.test.ts",
+ "**/*.spec.ts",
+ "**/*.integration-test.ts"
+ ]
+}
diff --git a/packages/twenty-docs/developers/extend/apps/layout/front-components.mdx b/packages/twenty-docs/developers/extend/apps/layout/front-components.mdx
index 77f210361b7..c70cde94226 100644
--- a/packages/twenty-docs/developers/extend/apps/layout/front-components.mdx
+++ b/packages/twenty-docs/developers/extend/apps/layout/front-components.mdx
@@ -475,6 +475,7 @@ Front components can trigger navigation, modals, and notifications using functio
| `enqueueSnackbar(params)` | Show a toast notification |
| `unmountFrontComponent()` | Unmount the component |
| `updateProgress(progress)` | Update a progress indicator |
+| `uploadFile(file, { fieldMetadataId, fileName? })` | Upload a `Blob` into a FILES field; resolves to `{ status: 'uploaded', file: { fileId, path, url, size, mimeType } }` or `{ status: 'failed', reason: 'invalid-params' \| 'upload-failed' }` |
Here is an example that uses the host API to show a snackbar and close the side panel after an action completes:
@@ -833,9 +834,13 @@ Component CSS, whether from `import './styles.css'`, CSS-in-JS or a `style` elem
- Other origins leave the sandbox with `Origin: null`, so a third-party API answers only if it sends `Access-Control-Allow-Origin: *`. Call it from a logic function instead.
- `fetch('/rest/people')` is never matched to the Twenty API, because the sandbox has no page URL to resolve a relative path against.
+### Media capture
+
+`navigator.mediaDevices.getUserMedia()` and `MediaRecorder` work inside front components through sandbox polyfills, so standard recording code runs unchanged and `MediaRecorder.isTypeSupported` answers for common container/codec combinations. Detailed `getUserMedia` constraint objects are accepted but not forwarded — the host captures with its defaults for the requested kinds — and only one capture can be live at a time across applications. Store a recorded `Blob` with the `uploadFile` host function.
+
### Other gaps
-- **File contents.** An `input` of type `file` gives your handler file metadata only, not the bytes, so `FileReader` and uploads are not possible yet.
+- **File contents.** An `input` of type `file` gives your handler file metadata only, not the bytes, so `FileReader` is not available. To upload a `Blob` your code already holds — e.g. one produced by `MediaRecorder` — use the `uploadFile` host function.
- **Drag-and-drop payloads.** Drag events fire, but `event.dataTransfer` is `undefined`.
- **Node built-ins.** `fs`, `path` and `node:crypto` fail the build, so move that work into a [logic function](/developers/extend/apps/logic/logic-functions). Web Crypto, `fetch`, `TextEncoder` and `URL` are available.
- **`iframe`** is always re-sandboxed without `allow-same-origin`, so an embed relying on its own session renders logged out. It has no `onLoad` either.
diff --git a/packages/twenty-front-component-renderer/scripts/front-component-stories/build-source-examples.ts b/packages/twenty-front-component-renderer/scripts/front-component-stories/build-source-examples.ts
index 34d640841f9..acca14327fd 100644
--- a/packages/twenty-front-component-renderer/scripts/front-component-stories/build-source-examples.ts
+++ b/packages/twenty-front-component-renderer/scripts/front-component-stories/build-source-examples.ts
@@ -19,6 +19,7 @@ const exampleSourcesBuiltPreactDir = path.resolve(
const SOURCE_SCAN_ROOTS = [
'html-tag',
'host-api',
+ 'media',
'showcase',
'twenty-ui-gallery',
];
diff --git a/packages/twenty-front-component-renderer/src/__stories__/media/media-recorder-polyfill.front-component.tsx b/packages/twenty-front-component-renderer/src/__stories__/media/media-recorder-polyfill.front-component.tsx
new file mode 100644
index 00000000000..eb02d16bd1f
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/__stories__/media/media-recorder-polyfill.front-component.tsx
@@ -0,0 +1,95 @@
+import { useState } from 'react';
+import { defineFrontComponent } from 'twenty-sdk/define';
+
+import { FrontComponentCard } from '@/__stories__/shared/front-components/front-component-card';
+import {
+ BUTTON_STYLE,
+ STATUS_STYLE,
+} from '@/__stories__/shared/front-components/styles';
+
+// Deliberately uses only the standard web APIs — getUserMedia, MediaStream,
+// MediaRecorder — the way a third-party recording library would. It only
+// works if the media polyfills reached the real sandbox worker realm.
+const MediaRecorderPolyfillFrontComponent = () => {
+ const [status, setStatus] = useState('idle');
+
+ const handleClick = async () => {
+ try {
+ if (typeof navigator.mediaDevices?.getUserMedia !== 'function') {
+ setStatus('media:missing-get-user-media');
+
+ return;
+ }
+
+ if (typeof MediaRecorder !== 'function') {
+ setStatus('media:missing-media-recorder');
+
+ return;
+ }
+
+ if (!MediaRecorder.isTypeSupported('audio/webm')) {
+ setStatus('media:missing-capability-snapshot');
+
+ return;
+ }
+
+ const mediaStream = await navigator.mediaDevices.getUserMedia({
+ audio: true,
+ });
+
+ const mediaRecorder = new MediaRecorder(mediaStream);
+ const recordedChunks: Blob[] = [];
+
+ mediaRecorder.ondataavailable = (event) => {
+ const dataEvent = event as Event & { data: Blob };
+
+ recordedChunks.push(dataEvent.data);
+ };
+
+ const recordingStopped = new Promise<void>((resolve) => {
+ mediaRecorder.onstop = () => resolve();
+ });
+
+ mediaRecorder.onstart = () => mediaRecorder.stop();
+ mediaRecorder.start();
+
+ await recordingStopped;
+
+ for (const track of mediaStream.getTracks()) {
+ track.stop();
+ }
+
+ const recordedBlob = new Blob(recordedChunks);
+
+ setStatus(`media:captured:${recordedBlob.size}`);
+ } catch (error) {
+ setStatus(
+ `media:error:${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
+ };
+
+ return (
+ <FrontComponentCard title="media:media-recorder-polyfill">
+ <button
+ data-testid="subject"
+ type="button"
+ onClick={handleClick}
+ style={BUTTON_STYLE}
+ >
+ Record
+ </button>
+ <span data-testid="api-status" style={STATUS_STYLE}>
+ {status}
+ </span>
+ </FrontComponentCard>
+ );
+};
+
+export default defineFrontComponent({
+ universalIdentifier: 'fc-media-recorder-00000000-0000-0000-0000-000000000022',
+ name: 'media-recorder-polyfill-front-component',
+ description:
+ 'Front component covering the getUserMedia and MediaRecorder polyfills',
+ component: MediaRecorderPolyfillFrontComponent,
+});
diff --git a/packages/twenty-front-component-renderer/src/__stories__/media/media-recorder-polyfill.stories.tsx b/packages/twenty-front-component-renderer/src/__stories__/media/media-recorder-polyfill.stories.tsx
new file mode 100644
index 00000000000..55c8dbb5e27
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/__stories__/media/media-recorder-polyfill.stories.tsx
@@ -0,0 +1,132 @@
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import { expect, fn, userEvent, waitFor, within } from 'storybook/test';
+
+import { FrontComponentRenderer } from '@/host/components/FrontComponentRenderer';
+import {
+ FRONT_COMPONENT_STORY_DEFAULT_ARGS,
+ resetFrontComponentStoryMocks,
+} from '@/__stories__/shared/test-utils/createFrontComponentStoryMeta';
+import { expectFrontComponentMounted } from '@/__stories__/shared/test-utils/matchers/expectFrontComponentMounted';
+import { runFrontComponentStory } from '@/__stories__/shared/test-utils/runFrontComponentStory';
+import {
+ HOST_API_TIMEOUT,
+ INTERACTION_TIMEOUT,
+} from '@/__stories__/shared/test-utils/timeouts';
+import { type MediaSessionEventTransport } from '@/host/media/types/FrontComponentMediaSessionHost';
+
+const TEST_STREAM_ID = 'stream-under-test';
+const TEST_TRACK_ID = 'track-under-test';
+const TEST_RECORDER_ID = 'recorder-under-test';
+const FAKE_RECORDED_BYTES = 'fake-audio-bytes';
+
+// A scripted session host: the worker side stays fully real (polyfills,
+// bridge, thread), while the host side answers with fabricated devices so
+// the story is deterministic without real hardware.
+const createMockMediaSessionHost = () => {
+ let transport: MediaSessionEventTransport | null = null;
+
+ return {
+ mediaStartStream: fn(async () => ({
+ status: 'started' as const,
+ streamId: TEST_STREAM_ID,
+ tracks: [{ trackId: TEST_TRACK_ID, kind: 'audio' as const }],
+ })),
+ mediaStopStreamTrack: fn(async () => {}),
+ mediaSetTrackEnabled: fn(async () => {}),
+ mediaStartRecorder: fn(async () => ({
+ status: 'started' as const,
+ recorderId: TEST_RECORDER_ID,
+ mimeType: 'audio/webm',
+ })),
+ mediaStopRecorder: fn(async () => {
+ await transport?.pushMediaSessionEvents({
+ events: [
+ {
+ type: 'recorder-data',
+ recorderId: TEST_RECORDER_ID,
+ data: new Blob([FAKE_RECORDED_BYTES]),
+ },
+ { type: 'recorder-stop', recorderId: TEST_RECORDER_ID },
+ ],
+ });
+ }),
+ mediaPauseRecorder: fn(async () => {}),
+ mediaResumeRecorder: fn(async () => {}),
+ mediaRequestRecorderData: fn(async () => {}),
+ connectEventTransport: (nextTransport: MediaSessionEventTransport) => {
+ transport = nextTransport;
+ },
+ disconnectEventTransport: () => {
+ transport = null;
+ },
+ stopAllSessions: fn(),
+ getRecorderCapabilities: () => ({ supportedMimeTypes: ['audio/webm'] }),
+ };
+};
+
+const mockMediaSessionHost = createMockMediaSessionHost();
+
+const meta: Meta<typeof FrontComponentRenderer> = {
+ title: 'FrontComponent/Media/MediaRecorderPolyfill',
+ component: FrontComponentRenderer,
+ parameters: { layout: 'centered' },
+ args: FRONT_COMPONENT_STORY_DEFAULT_ARGS,
+ beforeEach: resetFrontComponentStoryMocks,
+};
+
+export default meta;
+
+type Story = StoryObj<typeof FrontComponentRenderer>;
+
+const baseStory = runFrontComponentStory({
+ frontComponentBundleName: 'media-recorder-polyfill',
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+
+ await expectFrontComponentMounted(canvas);
+
+ const subject = await canvas.findByTestId('subject');
+
+ await userEvent.click(subject);
+
+ // The full loop ran: getUserMedia and MediaRecorder in the worker,
+ // capture RPCs to the host, and the recorded Blob pushed back across
+ // the bridge into standard dataavailable/stop events.
+ expect(
+ await canvas.findByText(
+ `media:captured:${FAKE_RECORDED_BYTES.length}`,
+ {},
+ { timeout: INTERACTION_TIMEOUT },
+ ),
+ ).toBeVisible();
+
+ await waitFor(
+ () => {
+ expect(mockMediaSessionHost.mediaStartStream).toHaveBeenCalledWith(
+ expect.objectContaining({ audio: true, video: false }),
+ );
+ expect(mockMediaSessionHost.mediaStartRecorder).toHaveBeenCalledWith(
+ expect.objectContaining({ streamId: TEST_STREAM_ID }),
+ );
+ expect(mockMediaSessionHost.mediaStopRecorder).toHaveBeenCalledWith(
+ expect.objectContaining({ recorderId: TEST_RECORDER_ID }),
+ );
+ expect(mockMediaSessionHost.mediaStopStreamTrack).toHaveBeenCalledWith(
+ expect.objectContaining({
+ streamId: TEST_STREAM_ID,
+ trackId: TEST_TRACK_ID,
+ }),
+ );
+ },
+ { timeout: HOST_API_TIMEOUT },
+ );
+ },
+});
+
+export const MediaRecorderPolyfill: Story = {
+ ...baseStory,
+ args: {
+ ...baseStory.args,
+ mediaSessionHost: mockMediaSessionHost,
+ },
+};
diff --git a/packages/twenty-front-component-renderer/src/__stories__/shared/test-utils/createFrontComponentStoryMeta.ts b/packages/twenty-front-component-renderer/src/__stories__/shared/test-utils/createFrontComponentStoryMeta.ts
index 1d50233b6ed..dc69a01269c 100644
--- a/packages/twenty-front-component-renderer/src/__stories__/shared/test-utils/createFrontComponentStoryMeta.ts
+++ b/packages/twenty-front-component-renderer/src/__stories__/shared/test-utils/createFrontComponentStoryMeta.ts
@@ -15,6 +15,10 @@ export const hostApiMocks = {
requestAccessTokenRefresh: fn().mockResolvedValue('refreshed-token'),
openCommandConfirmationModal: fn().mockResolvedValue(undefined),
copyToClipboard: fn().mockResolvedValue(undefined),
+ uploadFile: fn().mockResolvedValue({
+ status: 'failed',
+ reason: 'upload-failed',
+ }),
storageSet: fn().mockResolvedValue(undefined),
storageDelete: fn().mockResolvedValue(undefined),
storageClear: fn().mockResolvedValue(undefined),
@@ -47,6 +51,7 @@ export const resetFrontComponentStoryMocks = () => {
hostApiMocks.requestAccessTokenRefresh.mockClear();
hostApiMocks.openCommandConfirmationModal.mockClear();
hostApiMocks.copyToClipboard.mockClear();
+ hostApiMocks.uploadFile.mockClear();
hostApiMocks.storageSet.mockClear();
hostApiMocks.storageDelete.mockClear();
hostApiMocks.storageClear.mockClear();
diff --git a/packages/twenty-front-component-renderer/src/host/components/FrontComponentRenderer.tsx b/packages/twenty-front-component-renderer/src/host/components/FrontComponentRenderer.tsx
index 9db3222bf24..4933cd348ed 100644
--- a/packages/twenty-front-component-renderer/src/host/components/FrontComponentRenderer.tsx
+++ b/packages/twenty-front-component-renderer/src/host/components/FrontComponentRenderer.tsx
@@ -5,8 +5,10 @@ import { FrontComponentConfirmationModalResultEffect } from '@/host/effect-compo
import { FrontComponentErrorEffect } from '@/host/effect-components/FrontComponentErrorEffect';
import { FrontComponentGeometryTrackerEffect } from '@/host/effect-components/FrontComponentGeometryTrackerEffect';
import { FrontComponentInitializeHostCommunicationApiEffect } from '@/host/effect-components/FrontComponentInitializeHostCommunicationApiEffect';
+import { FrontComponentMediaSessionEffect } from '@/host/effect-components/FrontComponentMediaSessionEffect';
import { FrontComponentUpdateContextEffect } from '@/host/effect-components/FrontComponentUpdateContextEffect';
import { FrontComponentUpdateHostCommunicationApiEffect } from '@/host/effect-components/FrontComponentUpdateHostCommunicationApiEffect';
+import { type FrontComponentMediaSessionHost } from '@/host/media/types/FrontComponentMediaSessionHost';
import { type FrontComponentHostCommunicationApi } from '@/types/FrontComponentHostCommunicationApi';
import { type FrontComponentThread } from '@/types/FrontComponentThread';
import { type SdkClientUrls } from '@/types/SdkClientUrls';
@@ -39,6 +41,7 @@ type FrontComponentRendererProps = {
storageNamespace?: string;
executionContext: FrontComponentExecutionContext;
frontComponentHostCommunicationApi: FrontComponentHostCommunicationApi;
+ mediaSessionHost?: FrontComponentMediaSessionHost;
onError: (error?: Error) => void;
colorScheme: 'light' | 'dark';
loadingFallback?: ReactNode;
@@ -55,6 +58,7 @@ export const FrontComponentRenderer = ({
storageNamespace,
executionContext,
frontComponentHostCommunicationApi,
+ mediaSessionHost,
onError,
colorScheme,
loadingFallback,
@@ -81,6 +85,7 @@ export const FrontComponentRenderer = ({
applicationVariables={applicationVariables}
storageNamespace={storageNamespace}
geometryTracker={geometryTracker}
+ mediaSessionHost={mediaSessionHost}
setReceiver={setReceiver}
setThread={setThread}
setError={setError}
@@ -108,6 +113,12 @@ export const FrontComponentRenderer = ({
thread={thread}
geometryTracker={geometryTracker}
/>
+ {isDefined(mediaSessionHost) && (
+ <FrontComponentMediaSessionEffect
+ thread={thread}
+ mediaSessionHost={mediaSessionHost}
+ />
+ )}
<FrontComponentUpdateContextEffect
thread={thread}
executionContext={executionContext}
diff --git a/packages/twenty-front-component-renderer/src/host/effect-components/FrontComponentMediaSessionEffect.tsx b/packages/twenty-front-component-renderer/src/host/effect-components/FrontComponentMediaSessionEffect.tsx
new file mode 100644
index 00000000000..0024a3671e8
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/host/effect-components/FrontComponentMediaSessionEffect.tsx
@@ -0,0 +1,27 @@
+import { useEffect } from 'react';
+
+import { type FrontComponentMediaSessionHost } from '@/host/media/types/FrontComponentMediaSessionHost';
+import { type FrontComponentThread } from '@/types/FrontComponentThread';
+
+type FrontComponentMediaSessionEffectProps = {
+ thread: FrontComponentThread;
+ mediaSessionHost: FrontComponentMediaSessionHost;
+};
+
+export const FrontComponentMediaSessionEffect = ({
+ thread,
+ mediaSessionHost,
+}: FrontComponentMediaSessionEffectProps) => {
+ useEffect(() => {
+ mediaSessionHost.connectEventTransport(thread.imports);
+
+ return () => {
+ mediaSessionHost.disconnectEventTransport();
+ // The worker that could stop these devices is going away with the
+ // renderer, so nothing may keep capturing past unmount.
+ mediaSessionHost.stopAllSessions();
+ };
+ }, [thread, mediaSessionHost]);
+
+ return null;
+};
diff --git a/packages/twenty-front-component-renderer/src/host/effect-components/FrontComponentWorkerEffect.tsx b/packages/twenty-front-component-renderer/src/host/effect-components/FrontComponentWorkerEffect.tsx
index e16767f882c..5bc0c33d0dd 100644
--- a/packages/twenty-front-component-renderer/src/host/effect-components/FrontComponentWorkerEffect.tsx
+++ b/packages/twenty-front-component-renderer/src/host/effect-components/FrontComponentWorkerEffect.tsx
@@ -7,6 +7,7 @@ import { buildHostFetchPolicyFromFrontComponentUrls } from '@/host/fetch/utils/b
import { createFrontComponentHostThread } from '@/host/thread/utils/createFrontComponentHostThread';
import { createHostFetchEnforcingPolicy } from '@/host/fetch/utils/createHostFetchEnforcingPolicy';
import { type GeometryTracker } from '@/host/geometry/types/GeometryTracker';
+import { type FrontComponentMediaSessionHost } from '@/host/media/types/FrontComponentMediaSessionHost';
import { fetchComponentSource } from '@/host/component-source/utils/fetchComponentSource';
import { fetchSdkClientSources } from '@/host/component-source/utils/fetchSdkClientSources';
import { buildFrontComponentStorageSnapshots } from '@/host/storage/utils/buildFrontComponentStorageSnapshots';
@@ -29,6 +30,7 @@ type FrontComponentWorkerEffectProps = {
applicationVariables?: Record<string, string>;
storageNamespace?: string;
geometryTracker: GeometryTracker;
+ mediaSessionHost?: FrontComponentMediaSessionHost;
setReceiver: React.Dispatch<React.SetStateAction<RemoteReceiver | null>>;
setThread: React.Dispatch<React.SetStateAction<FrontComponentThread | null>>;
setError: React.Dispatch<React.SetStateAction<Error | null>>;
@@ -44,6 +46,7 @@ export const FrontComponentWorkerEffect = ({
applicationVariables,
storageNamespace,
geometryTracker,
+ mediaSessionHost,
setReceiver,
setThread,
setError,
@@ -78,6 +81,7 @@ export const FrontComponentWorkerEffect = ({
hostMessagePort: channel.port1,
hostFetch,
geometryTracker,
+ mediaSessionHost,
});
const handleSandboxMessage = createFrontComponentSandboxMessageHandler({
@@ -144,6 +148,8 @@ export const FrontComponentWorkerEffect = ({
applicationVariables,
initialViewportGeometry: geometryTracker.getViewportGeometry(),
storageSnapshots,
+ mediaRecorderCapabilities:
+ mediaSessionHost?.getRecorderCapabilities(),
});
} catch (error) {
if (!isCancelled) {
@@ -175,6 +181,7 @@ export const FrontComponentWorkerEffect = ({
applicationVariables,
storageNamespace,
geometryTracker,
+ mediaSessionHost,
setError,
setReceiver,
setThread,
diff --git a/packages/twenty-front-component-renderer/src/host/media/constants/MediaRecorderCandidateMimeTypes.ts b/packages/twenty-front-component-renderer/src/host/media/constants/MediaRecorderCandidateMimeTypes.ts
new file mode 100644
index 00000000000..69e8747a192
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/host/media/constants/MediaRecorderCandidateMimeTypes.ts
@@ -0,0 +1,20 @@
+// isTypeSupported answers in the worker come from a snapshot, so the host
+// probes a list of the container/codec combinations applications actually
+// ask for. Anything outside the list reports as unsupported.
+export const MEDIA_RECORDER_CANDIDATE_MIME_TYPES = [
+ 'audio/webm',
+ 'audio/webm;codecs=opus',
+ 'audio/mp4',
+ 'audio/mp4;codecs=mp4a.40.2',
+ 'audio/ogg',
+ 'audio/ogg;codecs=opus',
+ 'video/webm',
+ 'video/webm;codecs=vp8',
+ 'video/webm;codecs=vp9',
+ 'video/webm;codecs=vp8,opus',
+ 'video/webm;codecs=vp9,opus',
+ 'video/webm;codecs=h264',
+ 'video/mp4',
+ 'video/mp4;codecs=h264',
+ 'video/x-matroska;codecs=avc1',
+];
diff --git a/packages/twenty-front-component-renderer/src/host/media/types/FrontComponentMediaSessionHost.ts b/packages/twenty-front-component-renderer/src/host/media/types/FrontComponentMediaSessionHost.ts
new file mode 100644
index 00000000000..e26e4f68d91
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/host/media/types/FrontComponentMediaSessionHost.ts
@@ -0,0 +1,41 @@
+import {
+ type MediaRecorderCapabilities,
+ type MediaSessionEventBatch,
+ type MediaSessionHostFunctions,
+ type MediaSessionMediaType,
+} from '@/types/MediaSession';
+
+export type FrontComponentActiveMediaSession = {
+ streamId: string;
+ mediaType: MediaSessionMediaType;
+ startedAt: number;
+ getLiveMediaStream: () => MediaStream | null;
+};
+
+export type MediaSessionEventTransport = {
+ pushMediaSessionEvents: (batch: MediaSessionEventBatch) => Promise<void>;
+};
+
+export type MediaSessionStartVeto = {
+ errorName: string;
+ errorMessage: string;
+};
+
+export type CreateFrontComponentMediaSessionHostInput = {
+ // Extra host page policy, called before any device is touched: return a
+ // veto (as a DOMException name + message) to refuse the capture. The
+ // one-capture-at-a-time slot is enforced internally, before this hook.
+ beforeStartStream?: (
+ mediaType: MediaSessionMediaType,
+ ) => MediaSessionStartVeto | null;
+ onActiveSessionsChange?: (
+ activeSessions: FrontComponentActiveMediaSession[],
+ ) => void;
+};
+
+export type FrontComponentMediaSessionHost = MediaSessionHostFunctions & {
+ connectEventTransport: (transport: MediaSessionEventTransport) => void;
+ disconnectEventTransport: () => void;
+ stopAllSessions: () => void;
+ getRecorderCapabilities: () => MediaRecorderCapabilities;
+};
diff --git a/packages/twenty-front-component-renderer/src/host/media/utils/__tests__/createFrontComponentMediaSessionHost.test.ts b/packages/twenty-front-component-renderer/src/host/media/utils/__tests__/createFrontComponentMediaSessionHost.test.ts
new file mode 100644
index 00000000000..e68ed365f9d
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/host/media/utils/__tests__/createFrontComponentMediaSessionHost.test.ts
@@ -0,0 +1,132 @@
+import { createFrontComponentMediaSessionHost } from '../createFrontComponentMediaSessionHost';
+
+type DeferredStream = {
+ resolve: (stream: unknown) => void;
+ reject: (error: Error) => void;
+};
+
+const createFakeTrack = () => ({
+ id: `track-${Math.random().toString(36).slice(2)}`,
+ kind: 'audio',
+ readyState: 'live',
+ stop: jest.fn(),
+ addEventListener: jest.fn(),
+});
+
+const createFakeStream = () => {
+ const track = createFakeTrack();
+
+ track.stop.mockImplementation(() => {
+ track.readyState = 'ended';
+ });
+
+ return { getTracks: () => [track] };
+};
+
+const installGetUserMediaMock = () => {
+ const pendingStarts: DeferredStream[] = [];
+
+ const getUserMedia = jest.fn(
+ () =>
+ new Promise((resolve, reject) => {
+ pendingStarts.push({ resolve, reject });
+ }),
+ );
+
+ Object.defineProperty(globalThis.navigator, 'mediaDevices', {
+ configurable: true,
+ value: { getUserMedia },
+ });
+
+ return { pendingStarts };
+};
+
+describe('createFrontComponentMediaSessionHost capture slot', () => {
+ it('should hold one capture slot across hosts, through pending starts, live sessions, and failures', async () => {
+ const { pendingStarts } = installGetUserMediaMock();
+
+ const firstHost = createFrontComponentMediaSessionHost();
+ const secondHost = createFrontComponentMediaSessionHost();
+
+ // A start still waiting on the permission prompt already holds the slot.
+ const firstStartPromise = firstHost.mediaStartStream({
+ audio: true,
+ video: false,
+ });
+
+ const rejectedWhilePending = await secondHost.mediaStartStream({
+ audio: true,
+ video: false,
+ });
+
+ expect(rejectedWhilePending.status).toBe('failed');
+ expect(
+ rejectedWhilePending.status === 'failed' &&
+ rejectedWhilePending.errorName,
+ ).toBe('NotReadableError');
+
+ pendingStarts[0].resolve(createFakeStream());
+
+ const firstStartResult = await firstStartPromise;
+
+ expect(firstStartResult.status).toBe('started');
+
+ // A live session keeps the slot held.
+ const rejectedWhileLive = await secondHost.mediaStartStream({
+ audio: true,
+ video: false,
+ });
+
+ expect(rejectedWhileLive.status).toBe('failed');
+
+ // Ending the session frees the slot for another host.
+ firstHost.stopAllSessions();
+
+ const secondStartPromise = secondHost.mediaStartStream({
+ audio: true,
+ video: false,
+ });
+
+ pendingStarts[1].resolve(createFakeStream());
+
+ const secondStartResult = await secondStartPromise;
+
+ expect(secondStartResult.status).toBe('started');
+
+ secondHost.stopAllSessions();
+ });
+
+ it('should free the slot when a start fails', async () => {
+ const { pendingStarts } = installGetUserMediaMock();
+
+ const host = createFrontComponentMediaSessionHost();
+
+ const failingStartPromise = host.mediaStartStream({
+ audio: true,
+ video: false,
+ });
+
+ pendingStarts[0].reject(
+ Object.assign(new Error('Permission denied'), {
+ name: 'NotAllowedError',
+ }),
+ );
+
+ const failedResult = await failingStartPromise;
+
+ expect(failedResult.status).toBe('failed');
+ expect(failedResult.status === 'failed' && failedResult.errorName).toBe(
+ 'NotAllowedError',
+ );
+
+ const retryPromise = host.mediaStartStream({ audio: true, video: false });
+
+ pendingStarts[1].resolve(createFakeStream());
+
+ const retryResult = await retryPromise;
+
+ expect(retryResult.status).toBe('started');
+
+ host.stopAllSessions();
+ });
+});
diff --git a/packages/twenty-front-component-renderer/src/host/media/utils/createFrontComponentMediaSessionHost.ts b/packages/twenty-front-component-renderer/src/host/media/utils/createFrontComponentMediaSessionHost.ts
new file mode 100644
index 00000000000..16fa712f16f
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/host/media/utils/createFrontComponentMediaSessionHost.ts
@@ -0,0 +1,465 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { MEDIA_RECORDER_CANDIDATE_MIME_TYPES } from '@/host/media/constants/MediaRecorderCandidateMimeTypes';
+import { generateRandomId } from '@/utils/generateRandomId';
+import {
+ type CreateFrontComponentMediaSessionHostInput,
+ type FrontComponentActiveMediaSession,
+ type FrontComponentMediaSessionHost,
+ type MediaSessionEventTransport,
+} from '@/host/media/types/FrontComponentMediaSessionHost';
+import {
+ type MediaSessionEvent,
+ type MediaSessionMediaType,
+ type StartMediaRecorderResult,
+ type StartMediaStreamResult,
+} from '@/types/MediaSession';
+
+type HostStreamSession = {
+ streamId: string;
+ mediaType: MediaSessionMediaType;
+ startedAt: number;
+ mediaStream: MediaStream;
+};
+
+type HostRecorderSession = {
+ recorderId: string;
+ streamId: string;
+ mediaRecorder: MediaRecorder;
+};
+
+// One capture at a time across every session host on the page. The
+// reservation is scoped to the mediaStartStream call itself (released in
+// its finally), so no callback ordering can leak or double-release it.
+let isCaptureSlotReserved = false;
+let liveCaptureSessionCount = 0;
+
+const CAPTURE_SLOT_BUSY_FAILURE = {
+ status: 'failed',
+ errorName: 'NotReadableError',
+ errorMessage: 'A recording is already in progress',
+} as const;
+
+const toFailure = (
+ error: unknown,
+): { status: 'failed'; errorName: string; errorMessage: string } => ({
+ status: 'failed',
+ errorName: error instanceof Error ? error.name : 'UnknownError',
+ errorMessage: error instanceof Error ? error.message : String(error),
+});
+
+export const createFrontComponentMediaSessionHost = ({
+ beforeStartStream,
+ onActiveSessionsChange,
+}: CreateFrontComponentMediaSessionHostInput = {}): FrontComponentMediaSessionHost => {
+ const streamSessions = new Map<string, HostStreamSession>();
+ const recorderSessions = new Map<string, HostRecorderSession>();
+
+ let transport: MediaSessionEventTransport | null = null;
+ // Bumped by stopAllSessions so a getUserMedia still pending at teardown
+ // cannot register a stream nobody owns anymore.
+ let teardownGeneration = 0;
+ // Recorder chunks must survive the window before the worker transport is
+ // connected, so events buffer instead of dropping.
+ let bufferedEvents: MediaSessionEvent[] = [];
+
+ const pushEvents = (events: MediaSessionEvent[]): void => {
+ if (events.length === 0) {
+ return;
+ }
+
+ if (!isDefined(transport)) {
+ bufferedEvents.push(...events);
+ return;
+ }
+
+ transport.pushMediaSessionEvents({ events }).catch(() => {
+ console.warn(
+ 'A media session event could not reach the front component worker',
+ );
+ });
+ };
+
+ const notifyActiveSessionsChange = (): void => {
+ if (!isDefined(onActiveSessionsChange)) {
+ return;
+ }
+
+ const activeSessions: FrontComponentActiveMediaSession[] = [
+ ...streamSessions.values(),
+ ]
+ .filter((session) =>
+ session.mediaStream
+ .getTracks()
+ .some((track) => track.readyState === 'live'),
+ )
+ .sort(
+ (leftSession, rightSession) =>
+ leftSession.startedAt - rightSession.startedAt,
+ )
+ .map((session) => ({
+ streamId: session.streamId,
+ mediaType: session.mediaType,
+ startedAt: session.startedAt,
+ getLiveMediaStream: () =>
+ streamSessions.has(session.streamId) ? session.mediaStream : null,
+ }));
+
+ onActiveSessionsChange(activeSessions);
+ };
+
+ const cleanUpStreamSessionIfEnded = (streamId: string): void => {
+ const session = streamSessions.get(streamId);
+
+ if (!isDefined(session)) {
+ return;
+ }
+
+ const hasLiveTrack = session.mediaStream
+ .getTracks()
+ .some((track) => track.readyState === 'live');
+
+ if (hasLiveTrack) {
+ return;
+ }
+
+ streamSessions.delete(streamId);
+ liveCaptureSessionCount = Math.max(0, liveCaptureSessionCount - 1);
+ notifyActiveSessionsChange();
+ };
+
+ const mediaStartStream = async ({
+ audio,
+ video,
+ }: {
+ audio: boolean;
+ video: boolean;
+ }): Promise<StartMediaStreamResult> => {
+ if (isCaptureSlotReserved || liveCaptureSessionCount > 0) {
+ return CAPTURE_SLOT_BUSY_FAILURE;
+ }
+
+ isCaptureSlotReserved = true;
+
+ try {
+ return await startStreamWithReservedSlot({ audio, video });
+ } finally {
+ isCaptureSlotReserved = false;
+ }
+ };
+
+ const startStreamWithReservedSlot = async ({
+ audio,
+ video,
+ }: {
+ audio: boolean;
+ video: boolean;
+ }): Promise<StartMediaStreamResult> => {
+ const mediaType: MediaSessionMediaType = video ? 'video' : 'audio';
+ const veto = beforeStartStream?.(mediaType) ?? null;
+
+ if (isDefined(veto)) {
+ return { status: 'failed', ...veto };
+ }
+
+ if (typeof navigator === 'undefined' || !navigator.mediaDevices) {
+ return {
+ status: 'failed',
+ errorName: 'NotSupportedError',
+ errorMessage: 'Media capture is not available in this environment',
+ };
+ }
+
+ const startGeneration = teardownGeneration;
+
+ let mediaStream: MediaStream;
+
+ try {
+ mediaStream = await navigator.mediaDevices.getUserMedia({
+ audio,
+ video,
+ });
+ } catch (error) {
+ return toFailure(error);
+ }
+
+ if (startGeneration !== teardownGeneration) {
+ for (const track of mediaStream.getTracks()) {
+ track.stop();
+ }
+
+ return {
+ status: 'failed',
+ errorName: 'AbortError',
+ errorMessage: 'The capture was interrupted',
+ };
+ }
+
+ const streamId = generateRandomId();
+
+ streamSessions.set(streamId, {
+ streamId,
+ mediaType,
+ startedAt: Date.now(),
+ mediaStream,
+ });
+ liveCaptureSessionCount += 1;
+
+ for (const track of mediaStream.getTracks()) {
+ // Fires for external endings (device unplugged, permission revoked),
+ // not for stop() calls: those are reported to the worker explicitly.
+ track.addEventListener('ended', () => {
+ pushEvents([{ type: 'track-ended', streamId, trackId: track.id }]);
+ cleanUpStreamSessionIfEnded(streamId);
+ });
+ }
+
+ notifyActiveSessionsChange();
+
+ return {
+ status: 'started',
+ streamId,
+ tracks: mediaStream.getTracks().map((track) => ({
+ trackId: track.id,
+ kind: track.kind === 'video' ? 'video' : 'audio',
+ })),
+ };
+ };
+
+ const mediaStopStreamTrack = async ({
+ streamId,
+ trackId,
+ }: {
+ streamId: string;
+ trackId: string;
+ }): Promise<void> => {
+ const session = streamSessions.get(streamId);
+
+ if (!isDefined(session)) {
+ return;
+ }
+
+ const track = session.mediaStream
+ .getTracks()
+ .find((sessionTrack) => sessionTrack.id === trackId);
+
+ track?.stop();
+ cleanUpStreamSessionIfEnded(streamId);
+ };
+
+ const mediaSetTrackEnabled = async ({
+ streamId,
+ trackId,
+ enabled,
+ }: {
+ streamId: string;
+ trackId: string;
+ enabled: boolean;
+ }): Promise<void> => {
+ const session = streamSessions.get(streamId);
+
+ if (!isDefined(session)) {
+ return;
+ }
+
+ const track = session.mediaStream
+ .getTracks()
+ .find((sessionTrack) => sessionTrack.id === trackId);
+
+ if (isDefined(track)) {
+ track.enabled = enabled;
+ }
+ };
+
+ const mediaStartRecorder = async ({
+ streamId,
+ mimeType,
+ timesliceMs,
+ }: {
+ streamId: string;
+ mimeType?: string;
+ timesliceMs?: number;
+ }): Promise<StartMediaRecorderResult> => {
+ const session = streamSessions.get(streamId);
+
+ if (!isDefined(session)) {
+ return {
+ status: 'failed',
+ errorName: 'NotFoundError',
+ errorMessage: 'No live capture stream matches this recording request',
+ };
+ }
+
+ let mediaRecorder: MediaRecorder;
+
+ try {
+ mediaRecorder = new MediaRecorder(
+ session.mediaStream,
+ isDefined(mimeType) ? { mimeType } : undefined,
+ );
+ } catch (error) {
+ return toFailure(error);
+ }
+
+ const recorderId = generateRandomId();
+
+ mediaRecorder.addEventListener('dataavailable', (event) => {
+ pushEvents([{ type: 'recorder-data', recorderId, data: event.data }]);
+ });
+
+ mediaRecorder.addEventListener('stop', () => {
+ recorderSessions.delete(recorderId);
+ pushEvents([{ type: 'recorder-stop', recorderId }]);
+ });
+
+ mediaRecorder.addEventListener('error', (event) => {
+ const errorEvent = event as Event & { error?: Error };
+
+ pushEvents([
+ {
+ type: 'recorder-error',
+ recorderId,
+ errorMessage:
+ errorEvent.error?.message ?? 'The media recorder failed',
+ },
+ ]);
+ });
+
+ try {
+ mediaRecorder.start(timesliceMs);
+ } catch (error) {
+ return toFailure(error);
+ }
+
+ recorderSessions.set(recorderId, { recorderId, streamId, mediaRecorder });
+
+ return {
+ status: 'started',
+ recorderId,
+ mimeType: mediaRecorder.mimeType,
+ };
+ };
+
+ const mediaStopRecorder = async ({
+ recorderId,
+ }: {
+ recorderId: string;
+ }): Promise<void> => {
+ const session = recorderSessions.get(recorderId);
+
+ if (isDefined(session) && session.mediaRecorder.state !== 'inactive') {
+ session.mediaRecorder.stop();
+ }
+ };
+
+ const mediaPauseRecorder = async ({
+ recorderId,
+ }: {
+ recorderId: string;
+ }): Promise<void> => {
+ const session = recorderSessions.get(recorderId);
+
+ if (isDefined(session) && session.mediaRecorder.state === 'recording') {
+ session.mediaRecorder.pause();
+ }
+ };
+
+ const mediaResumeRecorder = async ({
+ recorderId,
+ }: {
+ recorderId: string;
+ }): Promise<void> => {
+ const session = recorderSessions.get(recorderId);
+
+ if (isDefined(session) && session.mediaRecorder.state === 'paused') {
+ session.mediaRecorder.resume();
+ }
+ };
+
+ const mediaRequestRecorderData = async ({
+ recorderId,
+ }: {
+ recorderId: string;
+ }): Promise<void> => {
+ const session = recorderSessions.get(recorderId);
+
+ if (isDefined(session) && session.mediaRecorder.state !== 'inactive') {
+ session.mediaRecorder.requestData();
+ }
+ };
+
+ const stopAllSessions = (): void => {
+ teardownGeneration += 1;
+
+ for (const recorderSession of recorderSessions.values()) {
+ if (recorderSession.mediaRecorder.state !== 'inactive') {
+ recorderSession.mediaRecorder.stop();
+ }
+ }
+
+ const endedTrackEvents: MediaSessionEvent[] = [];
+
+ for (const streamSession of streamSessions.values()) {
+ for (const track of streamSession.mediaStream.getTracks()) {
+ if (track.readyState !== 'live') {
+ continue;
+ }
+
+ track.stop();
+ // stop() fires no native ended event, so the worker is told
+ // explicitly that its tracks are gone.
+ endedTrackEvents.push({
+ type: 'track-ended',
+ streamId: streamSession.streamId,
+ trackId: track.id,
+ });
+ }
+ }
+
+ pushEvents(endedTrackEvents);
+ liveCaptureSessionCount = Math.max(
+ 0,
+ liveCaptureSessionCount - streamSessions.size,
+ );
+ streamSessions.clear();
+ notifyActiveSessionsChange();
+ };
+
+ const connectEventTransport = (
+ nextTransport: MediaSessionEventTransport,
+ ): void => {
+ transport = nextTransport;
+
+ const eventsToFlush = bufferedEvents;
+ bufferedEvents = [];
+ pushEvents(eventsToFlush);
+ };
+
+ const disconnectEventTransport = (): void => {
+ transport = null;
+ };
+
+ const getRecorderCapabilities = () => ({
+ supportedMimeTypes:
+ typeof MediaRecorder === 'undefined'
+ ? []
+ : MEDIA_RECORDER_CANDIDATE_MIME_TYPES.filter((candidateMimeType) =>
+ MediaRecorder.isTypeSupported(candidateMimeType),
+ ),
+ });
+
+ return {
+ mediaStartStream,
+ mediaStopStreamTrack,
+ mediaSetTrackEnabled,
+ mediaStartRecorder,
+ mediaStopRecorder,
+ mediaPauseRecorder,
+ mediaResumeRecorder,
+ mediaRequestRecorderData,
+ connectEventTransport,
+ disconnectEventTransport,
+ stopAllSessions,
+ getRecorderCapabilities,
+ };
+};
diff --git a/packages/twenty-front-component-renderer/src/host/thread/constants/FrontComponentHostCommunicationApiNoop.ts b/packages/twenty-front-component-renderer/src/host/thread/constants/FrontComponentHostCommunicationApiNoop.ts
index 7d8740877ba..0b492e28056 100644
--- a/packages/twenty-front-component-renderer/src/host/thread/constants/FrontComponentHostCommunicationApiNoop.ts
+++ b/packages/twenty-front-component-renderer/src/host/thread/constants/FrontComponentHostCommunicationApiNoop.ts
@@ -11,6 +11,7 @@ export const FRONT_COMPONENT_HOST_COMMUNICATION_API_NOOP: FrontComponentHostComm
closeSidePanel: async () => {},
updateProgress: async () => {},
copyToClipboard: async () => {},
+ uploadFile: async () => ({ status: 'failed', reason: 'upload-failed' }),
storageSet: async () => {},
storageDelete: async () => {},
storageClear: async () => {},
diff --git a/packages/twenty-front-component-renderer/src/host/thread/utils/createFrontComponentHostThread.ts b/packages/twenty-front-component-renderer/src/host/thread/utils/createFrontComponentHostThread.ts
index 7567c8060d0..c59f0848983 100644
--- a/packages/twenty-front-component-renderer/src/host/thread/utils/createFrontComponentHostThread.ts
+++ b/packages/twenty-front-component-renderer/src/host/thread/utils/createFrontComponentHostThread.ts
@@ -1,10 +1,13 @@
import { ThreadMessagePort } from '@quilted/threads';
+import { isDefined } from 'twenty-shared/utils';
import { FRONT_COMPONENT_HOST_COMMUNICATION_API_NOOP } from '@/host/thread/constants/FrontComponentHostCommunicationApiNoop';
import { type GeometryTracker } from '@/host/geometry/types/GeometryTracker';
+import { type FrontComponentMediaSessionHost } from '@/host/media/types/FrontComponentMediaSessionHost';
import { type FrontComponentHostThreadExports } from '@/types/FrontComponentHostThreadExports';
import { type FrontComponentThread } from '@/types/FrontComponentThread';
import { type HostFetchFunction } from '@/types/HostFetchFunction';
+import { type MediaSessionHostFunctions } from '@/types/MediaSession';
import { type WorkerExports } from '@/types/WorkerExports';
import { createClonableErrorThreadSerialization } from '@/utils/clonable-error/createClonableErrorThreadSerialization';
@@ -12,12 +15,48 @@ type CreateFrontComponentHostThreadInput = {
hostMessagePort: MessagePort;
hostFetch: HostFetchFunction;
geometryTracker: GeometryTracker;
+ mediaSessionHost?: FrontComponentMediaSessionHost;
+};
+
+const MEDIA_SESSION_UNAVAILABLE_FAILURE = {
+ status: 'failed',
+ errorName: 'NotSupportedError',
+ errorMessage: 'Media capture is not available in this surface',
+} as const;
+
+const buildMediaSessionThreadExports = (
+ mediaSessionHost?: FrontComponentMediaSessionHost,
+): MediaSessionHostFunctions => {
+ if (isDefined(mediaSessionHost)) {
+ return {
+ mediaStartStream: mediaSessionHost.mediaStartStream,
+ mediaStopStreamTrack: mediaSessionHost.mediaStopStreamTrack,
+ mediaSetTrackEnabled: mediaSessionHost.mediaSetTrackEnabled,
+ mediaStartRecorder: mediaSessionHost.mediaStartRecorder,
+ mediaStopRecorder: mediaSessionHost.mediaStopRecorder,
+ mediaPauseRecorder: mediaSessionHost.mediaPauseRecorder,
+ mediaResumeRecorder: mediaSessionHost.mediaResumeRecorder,
+ mediaRequestRecorderData: mediaSessionHost.mediaRequestRecorderData,
+ };
+ }
+
+ return {
+ mediaStartStream: async () => MEDIA_SESSION_UNAVAILABLE_FAILURE,
+ mediaStopStreamTrack: async () => {},
+ mediaSetTrackEnabled: async () => {},
+ mediaStartRecorder: async () => MEDIA_SESSION_UNAVAILABLE_FAILURE,
+ mediaStopRecorder: async () => {},
+ mediaPauseRecorder: async () => {},
+ mediaResumeRecorder: async () => {},
+ mediaRequestRecorderData: async () => {},
+ };
};
export const createFrontComponentHostThread = ({
hostMessagePort,
hostFetch,
geometryTracker,
+ mediaSessionHost,
}: CreateFrontComponentHostThreadInput): FrontComponentThread => {
const thread = new ThreadMessagePort<
WorkerExports,
@@ -25,6 +64,7 @@ export const createFrontComponentHostThread = ({
>(hostMessagePort, {
exports: {
...FRONT_COMPONENT_HOST_COMMUNICATION_API_NOOP,
+ ...buildMediaSessionThreadExports(mediaSessionHost),
hostFetch,
observeElementGeometry: async (remoteElementIds) => {
geometryTracker.observe(remoteElementIds);
diff --git a/packages/twenty-front-component-renderer/src/index.ts b/packages/twenty-front-component-renderer/src/index.ts
index c2fc5832d4c..203cb98a8fc 100644
--- a/packages/twenty-front-component-renderer/src/index.ts
+++ b/packages/twenty-front-component-renderer/src/index.ts
@@ -8,6 +8,22 @@ export { FrontComponentInitializeHostCommunicationApiEffect } from '@/host/effec
export { FrontComponentUpdateContextEffect } from '@/host/effect-components/FrontComponentUpdateContextEffect';
export { FrontComponentUpdateHostCommunicationApiEffect } from '@/host/effect-components/FrontComponentUpdateHostCommunicationApiEffect';
export { FrontComponentWorkerEffect } from '@/host/effect-components/FrontComponentWorkerEffect';
+export { FrontComponentMediaSessionEffect } from '@/host/effect-components/FrontComponentMediaSessionEffect';
+export { createFrontComponentMediaSessionHost } from '@/host/media/utils/createFrontComponentMediaSessionHost';
+export type {
+ CreateFrontComponentMediaSessionHostInput,
+ FrontComponentActiveMediaSession,
+ FrontComponentMediaSessionHost,
+ MediaSessionEventTransport,
+ MediaSessionStartVeto,
+} from '@/host/media/types/FrontComponentMediaSessionHost';
+export type {
+ MediaRecorderCapabilities,
+ MediaSessionEvent,
+ MediaSessionEventBatch,
+ MediaSessionHostFunctions,
+ MediaSessionMediaType,
+} from '@/types/MediaSession';
export {
HtmlA,
HtmlArticle,
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/constants/MediaTransportFailureWarning.ts b/packages/twenty-front-component-renderer/src/polyfills/media/constants/MediaTransportFailureWarning.ts
new file mode 100644
index 00000000000..4a31c75c2d1
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/constants/MediaTransportFailureWarning.ts
@@ -0,0 +1,2 @@
+export const MEDIA_TRANSPORT_FAILURE_WARNING =
+ 'A media capture call could not reach the host. The recording may be incomplete.';
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/states/workerMediaBridge.ts b/packages/twenty-front-component-renderer/src/polyfills/media/states/workerMediaBridge.ts
new file mode 100644
index 00000000000..a4368846677
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/states/workerMediaBridge.ts
@@ -0,0 +1,3 @@
+import { createWorkerMediaBridge } from '@/polyfills/media/utils/createWorkerMediaBridge';
+
+export const workerMediaBridge = createWorkerMediaBridge();
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/types/WorkerMediaBridge.ts b/packages/twenty-front-component-renderer/src/polyfills/media/types/WorkerMediaBridge.ts
new file mode 100644
index 00000000000..01e5034f535
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/types/WorkerMediaBridge.ts
@@ -0,0 +1,49 @@
+import {
+ type MediaRecorderCapabilities,
+ type MediaSessionEventBatch,
+ type MediaSessionHostFunctions,
+ type StartMediaRecorderResult,
+ type StartMediaStreamResult,
+} from '@/types/MediaSession';
+
+export type WorkerMediaTrackEventHandlers = {
+ onEnded: () => void;
+};
+
+export type WorkerMediaRecorderEventHandlers = {
+ onData: (data: Blob) => void;
+ onStop: () => void;
+ onError: (errorMessage: string) => void;
+};
+
+export type WorkerMediaBridge = {
+ connectTransport: (transport: MediaSessionHostFunctions) => void;
+ seedRecorderCapabilities: (capabilities: MediaRecorderCapabilities) => void;
+ isRecorderMimeTypeSupported: (mimeType: string) => boolean;
+ startStream: (params: {
+ audio: boolean;
+ video: boolean;
+ }) => Promise<StartMediaStreamResult>;
+ registerTrackEventHandlers: (params: {
+ streamId: string;
+ trackId: string;
+ handlers: WorkerMediaTrackEventHandlers;
+ }) => void;
+ stopStreamTrack: (params: { streamId: string; trackId: string }) => void;
+ setStreamTrackEnabled: (params: {
+ streamId: string;
+ trackId: string;
+ enabled: boolean;
+ }) => void;
+ startRecorder: (params: {
+ streamId: string;
+ mimeType?: string;
+ timesliceMs?: number;
+ handlers: WorkerMediaRecorderEventHandlers;
+ }) => Promise<StartMediaRecorderResult>;
+ stopRecorder: (recorderId: string) => void;
+ pauseRecorder: (recorderId: string) => void;
+ resumeRecorder: (recorderId: string) => void;
+ requestRecorderData: (recorderId: string) => void;
+ dispatchEvents: (batch: MediaSessionEventBatch) => void;
+};
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/utils/__tests__/createWorkerMediaBridge.test.ts b/packages/twenty-front-component-renderer/src/polyfills/media/utils/__tests__/createWorkerMediaBridge.test.ts
new file mode 100644
index 00000000000..e9a11de2322
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/utils/__tests__/createWorkerMediaBridge.test.ts
@@ -0,0 +1,168 @@
+import { type MediaSessionHostFunctions } from '@/types/MediaSession';
+import { createWorkerMediaBridge } from '../createWorkerMediaBridge';
+
+const createTransportStub = (): MediaSessionHostFunctions => ({
+ mediaStartStream: jest.fn(async () => ({
+ status: 'started' as const,
+ streamId: 'stream-1',
+ tracks: [{ trackId: 'track-1', kind: 'audio' as const }],
+ })),
+ mediaStopStreamTrack: jest.fn(async () => {}),
+ mediaSetTrackEnabled: jest.fn(async () => {}),
+ mediaStartRecorder: jest.fn(async () => ({
+ status: 'started' as const,
+ recorderId: 'recorder-1',
+ mimeType: 'audio/webm',
+ })),
+ mediaStopRecorder: jest.fn(async () => {}),
+ mediaPauseRecorder: jest.fn(async () => {}),
+ mediaResumeRecorder: jest.fn(async () => {}),
+ mediaRequestRecorderData: jest.fn(async () => {}),
+});
+
+describe('createWorkerMediaBridge', () => {
+ it('should fail stream and recorder starts when no transport is connected', async () => {
+ const bridge = createWorkerMediaBridge();
+
+ const streamResult = await bridge.startStream({
+ audio: true,
+ video: false,
+ });
+ const recorderResult = await bridge.startRecorder({
+ streamId: 'stream-1',
+ handlers: { onData: jest.fn(), onStop: jest.fn(), onError: jest.fn() },
+ });
+
+ expect(streamResult.status).toBe('failed');
+ expect(recorderResult.status).toBe('failed');
+ });
+
+ it('should forward stream starts to the transport', async () => {
+ const bridge = createWorkerMediaBridge();
+ const transport = createTransportStub();
+
+ bridge.connectTransport(transport);
+
+ const result = await bridge.startStream({ audio: true, video: true });
+
+ expect(transport.mediaStartStream).toHaveBeenCalledWith({
+ audio: true,
+ video: true,
+ });
+ expect(result.status).toBe('started');
+ });
+
+ it('should dispatch recorder events to the handlers registered at start', async () => {
+ const bridge = createWorkerMediaBridge();
+ bridge.connectTransport(createTransportStub());
+
+ const onData = jest.fn();
+ const onStop = jest.fn();
+
+ await bridge.startRecorder({
+ streamId: 'stream-1',
+ handlers: { onData, onStop, onError: jest.fn() },
+ });
+
+ const recordedBlob = new Blob(['bytes']);
+
+ bridge.dispatchEvents({
+ events: [
+ { type: 'recorder-data', recorderId: 'recorder-1', data: recordedBlob },
+ { type: 'recorder-stop', recorderId: 'recorder-1' },
+ ],
+ });
+
+ expect(onData).toHaveBeenCalledWith(recordedBlob);
+ expect(onStop).toHaveBeenCalledTimes(1);
+ });
+
+ it('should stop dispatching to a recorder after its stop event', async () => {
+ const bridge = createWorkerMediaBridge();
+ bridge.connectTransport(createTransportStub());
+
+ const onData = jest.fn();
+
+ await bridge.startRecorder({
+ streamId: 'stream-1',
+ handlers: { onData, onStop: jest.fn(), onError: jest.fn() },
+ });
+
+ bridge.dispatchEvents({
+ events: [{ type: 'recorder-stop', recorderId: 'recorder-1' }],
+ });
+ bridge.dispatchEvents({
+ events: [
+ { type: 'recorder-data', recorderId: 'recorder-1', data: new Blob() },
+ ],
+ });
+
+ expect(onData).not.toHaveBeenCalled();
+ });
+
+ it('should dispatch track endings once and drop the registration', () => {
+ const bridge = createWorkerMediaBridge();
+ const onEnded = jest.fn();
+
+ bridge.registerTrackEventHandlers({
+ streamId: 'stream-1',
+ trackId: 'track-1',
+ handlers: { onEnded },
+ });
+
+ const trackEndedBatch = {
+ events: [
+ {
+ type: 'track-ended' as const,
+ streamId: 'stream-1',
+ trackId: 'track-1',
+ },
+ ],
+ };
+
+ bridge.dispatchEvents(trackEndedBatch);
+ bridge.dispatchEvents(trackEndedBatch);
+
+ expect(onEnded).toHaveBeenCalledTimes(1);
+ });
+
+ it('should not dispatch track endings after a self-initiated stop', () => {
+ const bridge = createWorkerMediaBridge();
+ bridge.connectTransport(createTransportStub());
+
+ const onEnded = jest.fn();
+
+ bridge.registerTrackEventHandlers({
+ streamId: 'stream-1',
+ trackId: 'track-1',
+ handlers: { onEnded },
+ });
+
+ bridge.stopStreamTrack({ streamId: 'stream-1', trackId: 'track-1' });
+
+ bridge.dispatchEvents({
+ events: [
+ { type: 'track-ended', streamId: 'stream-1', trackId: 'track-1' },
+ ],
+ });
+
+ expect(onEnded).not.toHaveBeenCalled();
+ });
+
+ it('should answer isTypeSupported from the seeded snapshot', () => {
+ const bridge = createWorkerMediaBridge();
+
+ bridge.seedRecorderCapabilities({
+ supportedMimeTypes: ['audio/webm;codecs=opus'],
+ });
+
+ expect(bridge.isRecorderMimeTypeSupported('')).toBe(true);
+ expect(bridge.isRecorderMimeTypeSupported('audio/webm;codecs=opus')).toBe(
+ true,
+ );
+ expect(bridge.isRecorderMimeTypeSupported('AUDIO/WEBM;CODECS=OPUS')).toBe(
+ true,
+ );
+ expect(bridge.isRecorderMimeTypeSupported('video/mp4')).toBe(false);
+ });
+});
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/utils/__tests__/installMediaCapturePolyfills.test.ts b/packages/twenty-front-component-renderer/src/polyfills/media/utils/__tests__/installMediaCapturePolyfills.test.ts
new file mode 100644
index 00000000000..2b29a83d031
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/utils/__tests__/installMediaCapturePolyfills.test.ts
@@ -0,0 +1,340 @@
+import { type MediaSessionHostFunctions } from '@/types/MediaSession';
+import { createWorkerMediaBridge } from '../createWorkerMediaBridge';
+import { installMediaCapturePolyfills } from '../installMediaCapturePolyfills';
+
+type MediaGlobals = {
+ navigator: {
+ mediaDevices: {
+ getUserMedia: (constraints?: unknown) => Promise<MediaStreamLike>;
+ };
+ };
+ MediaStream: new (streamOrTracks?: unknown) => MediaStreamLike;
+ MediaStreamTrack: new () => unknown;
+ MediaRecorder: (new (
+ stream: MediaStreamLike,
+ options?: { mimeType?: string },
+ ) => MediaRecorderLike) & { isTypeSupported: (mimeType: string) => boolean };
+};
+
+type MediaStreamTrackLike = {
+ id: string;
+ kind: string;
+ readyState: string;
+ enabled: boolean;
+ onended: (() => void) | null;
+ stop: () => void;
+};
+
+type MediaStreamLike = {
+ id: string;
+ active: boolean;
+ getTracks: () => MediaStreamTrackLike[];
+};
+
+type MediaRecorderLike = {
+ state: string;
+ mimeType: string;
+ start: (timesliceMs?: number) => void;
+ stop: () => void;
+ ondataavailable: ((event: { data: Blob }) => void) | null;
+ onstart: (() => void) | null;
+ onstop: (() => void) | null;
+ onerror: ((event: { error: Error }) => void) | null;
+};
+
+const createTransportStub = (): MediaSessionHostFunctions => ({
+ mediaStartStream: jest.fn(async () => ({
+ status: 'started' as const,
+ streamId: 'stream-1',
+ tracks: [{ trackId: 'track-1', kind: 'audio' as const }],
+ })),
+ mediaStopStreamTrack: jest.fn(async () => {}),
+ mediaSetTrackEnabled: jest.fn(async () => {}),
+ mediaStartRecorder: jest.fn(async () => ({
+ status: 'started' as const,
+ recorderId: 'recorder-1',
+ mimeType: 'audio/webm;codecs=opus',
+ })),
+ mediaStopRecorder: jest.fn(async () => {}),
+ mediaPauseRecorder: jest.fn(async () => {}),
+ mediaResumeRecorder: jest.fn(async () => {}),
+ mediaRequestRecorderData: jest.fn(async () => {}),
+});
+
+const installOnFreshScope = (transport: MediaSessionHostFunctions) => {
+ const bridge = createWorkerMediaBridge();
+ bridge.connectTransport(transport);
+ bridge.seedRecorderCapabilities({ supportedMimeTypes: ['audio/webm'] });
+
+ const globalScope: Record<string, unknown> = {};
+ installMediaCapturePolyfills({ globalScope, bridge });
+
+ return { bridge, mediaGlobals: globalScope as unknown as MediaGlobals };
+};
+
+describe('installMediaCapturePolyfills', () => {
+ it('should install the media globals and navigator.mediaDevices', () => {
+ const { mediaGlobals } = installOnFreshScope(createTransportStub());
+
+ expect(typeof mediaGlobals.MediaStream).toBe('function');
+ expect(typeof mediaGlobals.MediaStreamTrack).toBe('function');
+ expect(typeof mediaGlobals.MediaRecorder).toBe('function');
+ expect(typeof mediaGlobals.navigator.mediaDevices.getUserMedia).toBe(
+ 'function',
+ );
+ });
+
+ it('should reject getUserMedia without audio or video', async () => {
+ const { mediaGlobals } = installOnFreshScope(createTransportStub());
+
+ await expect(
+ mediaGlobals.navigator.mediaDevices.getUserMedia({}),
+ ).rejects.toThrow(TypeError);
+ });
+
+ it('should reject getUserMedia with the host failure as a named error', async () => {
+ const transport = createTransportStub();
+ transport.mediaStartStream = jest.fn(async () => ({
+ status: 'failed' as const,
+ errorName: 'NotAllowedError',
+ errorMessage: 'Permission denied',
+ }));
+
+ const { mediaGlobals } = installOnFreshScope(transport);
+
+ await expect(
+ mediaGlobals.navigator.mediaDevices.getUserMedia({ audio: true }),
+ ).rejects.toMatchObject({ name: 'NotAllowedError' });
+ });
+
+ it('should build a live stream whose tracks stop through the transport', async () => {
+ const transport = createTransportStub();
+ const { mediaGlobals } = installOnFreshScope(transport);
+
+ const mediaStream = await mediaGlobals.navigator.mediaDevices.getUserMedia({
+ audio: true,
+ });
+
+ expect(mediaStream.active).toBe(true);
+
+ const [track] = mediaStream.getTracks();
+
+ expect(track.kind).toBe('audio');
+ expect(track.readyState).toBe('live');
+
+ const endedHandler = jest.fn();
+ track.onended = endedHandler;
+ track.stop();
+
+ expect(track.readyState).toBe('ended');
+ expect(mediaStream.active).toBe(false);
+ // Self-initiated stops fire no ended event, matching the native API.
+ expect(endedHandler).not.toHaveBeenCalled();
+ expect(transport.mediaStopStreamTrack).toHaveBeenCalledWith({
+ streamId: 'stream-1',
+ trackId: 'track-1',
+ });
+ });
+
+ it('should fire ended on tracks the host reports as ended', async () => {
+ const transport = createTransportStub();
+ const { bridge, mediaGlobals } = installOnFreshScope(transport);
+
+ const mediaStream = await mediaGlobals.navigator.mediaDevices.getUserMedia({
+ audio: true,
+ });
+ const [track] = mediaStream.getTracks();
+
+ const endedHandler = jest.fn();
+ track.onended = endedHandler;
+
+ bridge.dispatchEvents({
+ events: [
+ { type: 'track-ended', streamId: 'stream-1', trackId: 'track-1' },
+ ],
+ });
+
+ expect(endedHandler).toHaveBeenCalledTimes(1);
+ expect(track.readyState).toBe('ended');
+ });
+
+ it('should record through the transport and replay pushed chunks as events', async () => {
+ const transport = createTransportStub();
+ const { bridge, mediaGlobals } = installOnFreshScope(transport);
+
+ const mediaStream = await mediaGlobals.navigator.mediaDevices.getUserMedia({
+ audio: true,
+ });
+
+ const mediaRecorder = new mediaGlobals.MediaRecorder(mediaStream);
+
+ expect(mediaRecorder.state).toBe('inactive');
+
+ const receivedChunks: Blob[] = [];
+ const startHandler = jest.fn();
+ const stopHandler = jest.fn();
+
+ mediaRecorder.ondataavailable = (event) => receivedChunks.push(event.data);
+ mediaRecorder.onstart = startHandler;
+ mediaRecorder.onstop = stopHandler;
+
+ mediaRecorder.start();
+
+ expect(mediaRecorder.state).toBe('recording');
+
+ // The start acknowledgement crosses several await boundaries; a
+ // macrotask drains them all.
+ await new Promise((resolve) => setTimeout(resolve, 0));
+
+ expect(startHandler).toHaveBeenCalledTimes(1);
+ expect(mediaRecorder.mimeType).toBe('audio/webm;codecs=opus');
+ expect(transport.mediaStartRecorder).toHaveBeenCalledWith(
+ expect.objectContaining({ streamId: 'stream-1' }),
+ );
+
+ mediaRecorder.stop();
+
+ expect(mediaRecorder.state).toBe('inactive');
+ expect(transport.mediaStopRecorder).toHaveBeenCalledWith({
+ recorderId: 'recorder-1',
+ });
+
+ const recordedBlob = new Blob(['bytes']);
+
+ bridge.dispatchEvents({
+ events: [
+ { type: 'recorder-data', recorderId: 'recorder-1', data: recordedBlob },
+ { type: 'recorder-stop', recorderId: 'recorder-1' },
+ ],
+ });
+
+ expect(receivedChunks).toEqual([recordedBlob]);
+ expect(stopHandler).toHaveBeenCalledTimes(1);
+ });
+
+ it('should ignore stale host events after the recorder restarts', async () => {
+ const transport = createTransportStub();
+ let recorderCounter = 0;
+ transport.mediaStartRecorder = jest.fn(async () => ({
+ status: 'started' as const,
+ recorderId: `recorder-${recorderCounter++}`,
+ mimeType: 'audio/webm',
+ }));
+
+ const { bridge, mediaGlobals } = installOnFreshScope(transport);
+
+ const mediaStream = await mediaGlobals.navigator.mediaDevices.getUserMedia({
+ audio: true,
+ });
+
+ const mediaRecorder = new mediaGlobals.MediaRecorder(mediaStream);
+ const stopHandler = jest.fn();
+ mediaRecorder.onstop = stopHandler;
+
+ mediaRecorder.start();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+
+ mediaRecorder.stop();
+ // Restart before the previous host stop event has arrived.
+ mediaRecorder.start();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+
+ expect(mediaRecorder.state).toBe('recording');
+
+ // The late stop event of the previous recording must not end the new
+ // one.
+ bridge.dispatchEvents({
+ events: [{ type: 'recorder-stop', recorderId: 'recorder-0' }],
+ });
+
+ expect(mediaRecorder.state).toBe('recording');
+
+ bridge.dispatchEvents({
+ events: [{ type: 'recorder-stop', recorderId: 'recorder-1' }],
+ });
+
+ expect(mediaRecorder.state).toBe('inactive');
+ expect(stopHandler).toHaveBeenCalledTimes(1);
+ });
+
+ it('should fire error and then stop when the host cannot start the recorder', async () => {
+ const transport = createTransportStub();
+ transport.mediaStartRecorder = jest.fn(async () => ({
+ status: 'failed' as const,
+ errorName: 'NotReadableError',
+ errorMessage: 'The recorder could not start',
+ }));
+
+ const { mediaGlobals } = installOnFreshScope(transport);
+
+ const mediaStream = await mediaGlobals.navigator.mediaDevices.getUserMedia({
+ audio: true,
+ });
+
+ const mediaRecorder = new mediaGlobals.MediaRecorder(mediaStream);
+
+ const errorHandler = jest.fn();
+ const stopHandler = jest.fn();
+
+ mediaRecorder.onerror = errorHandler;
+ mediaRecorder.onstop = stopHandler;
+
+ mediaRecorder.start();
+
+ await new Promise((resolve) => setTimeout(resolve, 0));
+
+ expect(errorHandler).toHaveBeenCalledTimes(1);
+ expect(errorHandler.mock.calls[0][0].error.name).toBe('NotReadableError');
+ // The stop event follows the error, like a native recorder, so callers
+ // waiting on it never hang.
+ expect(stopHandler).toHaveBeenCalledTimes(1);
+ expect(mediaRecorder.state).toBe('inactive');
+ });
+
+ it('should throw for recorder construction on foreign values and unsupported types', async () => {
+ const { mediaGlobals } = installOnFreshScope(createTransportStub());
+
+ expect(
+ () =>
+ new mediaGlobals.MediaRecorder({
+ id: 'not-a-stream',
+ } as unknown as MediaStreamLike),
+ ).toThrow(TypeError);
+
+ const mediaStream = await mediaGlobals.navigator.mediaDevices.getUserMedia({
+ audio: true,
+ });
+
+ expect(
+ () =>
+ new mediaGlobals.MediaRecorder(mediaStream, {
+ mimeType: 'video/unsupported',
+ }),
+ ).toThrow(expect.objectContaining({ name: 'NotSupportedError' }));
+ });
+
+ it('should refuse to start a recorder on an inactive stream', async () => {
+ const { mediaGlobals } = installOnFreshScope(createTransportStub());
+
+ const mediaStream = await mediaGlobals.navigator.mediaDevices.getUserMedia({
+ audio: true,
+ });
+
+ const mediaRecorder = new mediaGlobals.MediaRecorder(mediaStream);
+
+ for (const track of mediaStream.getTracks()) {
+ track.stop();
+ }
+
+ expect(() => mediaRecorder.start()).toThrow(
+ expect.objectContaining({ name: 'InvalidStateError' }),
+ );
+ });
+
+ it('should answer isTypeSupported from the seeded snapshot', () => {
+ const { mediaGlobals } = installOnFreshScope(createTransportStub());
+
+ expect(mediaGlobals.MediaRecorder.isTypeSupported('audio/webm')).toBe(true);
+ expect(mediaGlobals.MediaRecorder.isTypeSupported('video/mp4')).toBe(false);
+ });
+});
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/utils/createDomException.ts b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createDomException.ts
new file mode 100644
index 00000000000..44be5ddcf27
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createDomException.ts
@@ -0,0 +1,12 @@
+// Media polyfill failures mirror native error shapes so application code can
+// branch on error.name the way it would against the real APIs.
+export const createDomException = (message: string, name: string): Error => {
+ if (typeof DOMException === 'function') {
+ return new DOMException(message, name);
+ }
+
+ const fallbackError = new Error(message);
+ fallbackError.name = name;
+
+ return fallbackError;
+};
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaRecorderClass.ts b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaRecorderClass.ts
new file mode 100644
index 00000000000..19677140bea
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaRecorderClass.ts
@@ -0,0 +1,371 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type WorkerMediaBridge } from '@/polyfills/media/types/WorkerMediaBridge';
+import { createDomException } from '@/polyfills/media/utils/createDomException';
+import { type WorkerMediaStreamInstance } from '@/polyfills/media/utils/createMediaStreamClass';
+
+export type WorkerMediaRecorderState = 'inactive' | 'recording' | 'paused';
+
+export type WorkerMediaRecorderDataEvent = Event & { data: Blob };
+
+export type WorkerMediaRecorderErrorEvent = Event & { error: Error };
+
+type CreateMediaRecorderClassInput = {
+ bridge: WorkerMediaBridge;
+ isMediaStreamInstance: (
+ stream: unknown,
+ ) => stream is WorkerMediaStreamInstance;
+ resolveCapturedStreamId: (stream: object) => string | null;
+};
+
+type WorkerMediaRecorderOptions = {
+ mimeType?: string;
+};
+
+const createRecorderDataEvent = (data: Blob): WorkerMediaRecorderDataEvent => {
+ const dataEvent = new Event('dataavailable') as WorkerMediaRecorderDataEvent;
+
+ Object.defineProperty(dataEvent, 'data', { value: data });
+
+ return dataEvent;
+};
+
+const createRecorderErrorEvent = (
+ error: Error,
+): WorkerMediaRecorderErrorEvent => {
+ const errorEvent = new Event('error') as WorkerMediaRecorderErrorEvent;
+
+ Object.defineProperty(errorEvent, 'error', { value: error });
+
+ return errorEvent;
+};
+
+export const createMediaRecorderClass = ({
+ bridge,
+ isMediaStreamInstance,
+ resolveCapturedStreamId,
+}: CreateMediaRecorderClassInput) => {
+ class MediaRecorderImplementation extends EventTarget {
+ readonly stream: WorkerMediaStreamInstance;
+
+ #state: WorkerMediaRecorderState = 'inactive';
+ #mimeType: string;
+ #recorderId: string | null = null;
+ // Restarting before the previous host stop event arrives must not let
+ // stale acknowledgements or events touch the new recording.
+ #startGeneration = 0;
+ // A stop, pause, or data request can land while the start round trip is
+ // still in flight; they are applied once the host recorder id is known.
+ #hasPendingStopRequest = false;
+ #hasPendingPauseRequest = false;
+ #hasPendingDataRequest = false;
+
+ #eventHandlers = new Map<string, EventListener>();
+
+ static isTypeSupported(mimeType: string): boolean {
+ return bridge.isRecorderMimeTypeSupported(String(mimeType));
+ }
+
+ constructor(
+ stream: WorkerMediaStreamInstance,
+ options?: WorkerMediaRecorderOptions,
+ ) {
+ super();
+
+ if (!isMediaStreamInstance(stream)) {
+ throw new TypeError(
+ "Failed to construct 'MediaRecorder': parameter 1 is not of type 'MediaStream'",
+ );
+ }
+
+ const requestedMimeType = options?.mimeType ?? '';
+
+ if (
+ requestedMimeType !== '' &&
+ !bridge.isRecorderMimeTypeSupported(requestedMimeType)
+ ) {
+ throw createDomException(
+ `Failed to construct 'MediaRecorder': Failed to initialize native MediaRecorder the type provided (${requestedMimeType}) is not supported`,
+ 'NotSupportedError',
+ );
+ }
+
+ this.stream = stream;
+ this.#mimeType = requestedMimeType;
+ }
+
+ get state(): WorkerMediaRecorderState {
+ return this.#state;
+ }
+
+ get mimeType(): string {
+ return this.#mimeType;
+ }
+
+ get ondataavailable(): EventListener | null {
+ return this.#getEventHandler('dataavailable');
+ }
+
+ set ondataavailable(handler: EventListener | null) {
+ this.#setEventHandler('dataavailable', handler);
+ }
+
+ get onstart(): EventListener | null {
+ return this.#getEventHandler('start');
+ }
+
+ set onstart(handler: EventListener | null) {
+ this.#setEventHandler('start', handler);
+ }
+
+ get onstop(): EventListener | null {
+ return this.#getEventHandler('stop');
+ }
+
+ set onstop(handler: EventListener | null) {
+ this.#setEventHandler('stop', handler);
+ }
+
+ get onerror(): EventListener | null {
+ return this.#getEventHandler('error');
+ }
+
+ set onerror(handler: EventListener | null) {
+ this.#setEventHandler('error', handler);
+ }
+
+ get onpause(): EventListener | null {
+ return this.#getEventHandler('pause');
+ }
+
+ set onpause(handler: EventListener | null) {
+ this.#setEventHandler('pause', handler);
+ }
+
+ get onresume(): EventListener | null {
+ return this.#getEventHandler('resume');
+ }
+
+ set onresume(handler: EventListener | null) {
+ this.#setEventHandler('resume', handler);
+ }
+
+ start(timesliceMs?: number): void {
+ if (this.#state !== 'inactive') {
+ throw createDomException(
+ "Failed to execute 'start' on 'MediaRecorder': The MediaRecorder's state is not 'inactive'",
+ 'InvalidStateError',
+ );
+ }
+
+ if (!this.stream.active) {
+ throw createDomException(
+ "Failed to execute 'start' on 'MediaRecorder': The MediaStream is inactive",
+ 'InvalidStateError',
+ );
+ }
+
+ const capturedStreamId = resolveCapturedStreamId(this.stream);
+
+ if (!isDefined(capturedStreamId)) {
+ throw createDomException(
+ "Failed to execute 'start' on 'MediaRecorder': Only streams returned by getUserMedia can be recorded in this environment",
+ 'NotSupportedError',
+ );
+ }
+
+ this.#state = 'recording';
+ this.#recorderId = null;
+ this.#hasPendingStopRequest = false;
+ this.#hasPendingPauseRequest = false;
+ this.#hasPendingDataRequest = false;
+
+ const startGeneration = ++this.#startGeneration;
+ const isCurrentGeneration = () =>
+ startGeneration === this.#startGeneration;
+
+ bridge
+ .startRecorder({
+ streamId: capturedStreamId,
+ mimeType: this.#mimeType === '' ? undefined : this.#mimeType,
+ timesliceMs,
+ handlers: {
+ onData: (data) => {
+ if (isCurrentGeneration()) {
+ this.dispatchEvent(createRecorderDataEvent(data));
+ }
+ },
+ onStop: () => {
+ if (isCurrentGeneration()) {
+ this.#handleHostStop();
+ }
+ },
+ onError: (errorMessage) => {
+ if (isCurrentGeneration()) {
+ this.dispatchEvent(
+ createRecorderErrorEvent(
+ createDomException(errorMessage, 'UnknownError'),
+ ),
+ );
+ }
+ },
+ },
+ })
+ .then((result) => {
+ if (!isCurrentGeneration()) {
+ // A newer start owns this recorder object now; do not leak the
+ // host recorder this stale acknowledgement created.
+ if (result.status === 'started') {
+ bridge.stopRecorder(result.recorderId);
+ }
+ return;
+ }
+
+ if (result.status === 'failed') {
+ // Native recorders that fail to start fire error and then stop,
+ // so waiting on the stop event never hangs.
+ this.#state = 'inactive';
+ this.#hasPendingStopRequest = false;
+ this.dispatchEvent(
+ createRecorderErrorEvent(
+ createDomException(result.errorMessage, result.errorName),
+ ),
+ );
+ this.dispatchEvent(new Event('stop'));
+ return;
+ }
+
+ this.#recorderId = result.recorderId;
+ this.#mimeType = result.mimeType;
+ this.dispatchEvent(new Event('start'));
+
+ if (this.#hasPendingStopRequest) {
+ this.#hasPendingStopRequest = false;
+ this.#recorderId = null;
+ bridge.stopRecorder(result.recorderId);
+ return;
+ }
+
+ if (this.#hasPendingPauseRequest) {
+ this.#hasPendingPauseRequest = false;
+ // The pause event was deferred so it follows the start event in
+ // native order.
+ this.dispatchEvent(new Event('pause'));
+ bridge.pauseRecorder(result.recorderId);
+ }
+
+ if (this.#hasPendingDataRequest) {
+ this.#hasPendingDataRequest = false;
+ bridge.requestRecorderData(result.recorderId);
+ }
+ });
+ }
+
+ stop(): void {
+ if (this.#state === 'inactive') {
+ throw createDomException(
+ "Failed to execute 'stop' on 'MediaRecorder': The MediaRecorder's state is 'inactive'",
+ 'InvalidStateError',
+ );
+ }
+
+ this.#state = 'inactive';
+
+ if (isDefined(this.#recorderId)) {
+ bridge.stopRecorder(this.#recorderId);
+ this.#recorderId = null;
+ return;
+ }
+
+ this.#hasPendingStopRequest = true;
+ }
+
+ pause(): void {
+ if (this.#state === 'inactive') {
+ throw createDomException(
+ "Failed to execute 'pause' on 'MediaRecorder': The MediaRecorder's state is 'inactive'",
+ 'InvalidStateError',
+ );
+ }
+
+ if (this.#state === 'paused') {
+ return;
+ }
+
+ this.#state = 'paused';
+
+ if (isDefined(this.#recorderId)) {
+ this.dispatchEvent(new Event('pause'));
+ bridge.pauseRecorder(this.#recorderId);
+ return;
+ }
+
+ // Event deferred until the start acknowledgement so pause never fires
+ // before start.
+ this.#hasPendingPauseRequest = true;
+ }
+
+ resume(): void {
+ if (this.#state === 'inactive') {
+ throw createDomException(
+ "Failed to execute 'resume' on 'MediaRecorder': The MediaRecorder's state is 'inactive'",
+ 'InvalidStateError',
+ );
+ }
+
+ if (this.#state === 'recording') {
+ return;
+ }
+
+ this.#state = 'recording';
+ this.#hasPendingPauseRequest = false;
+ this.dispatchEvent(new Event('resume'));
+
+ if (isDefined(this.#recorderId)) {
+ bridge.resumeRecorder(this.#recorderId);
+ }
+ }
+
+ requestData(): void {
+ if (this.#state === 'inactive') {
+ throw createDomException(
+ "Failed to execute 'requestData' on 'MediaRecorder': The MediaRecorder's state is 'inactive'",
+ 'InvalidStateError',
+ );
+ }
+
+ if (isDefined(this.#recorderId)) {
+ bridge.requestRecorderData(this.#recorderId);
+ return;
+ }
+
+ this.#hasPendingDataRequest = true;
+ }
+
+ #handleHostStop(): void {
+ this.#recorderId = null;
+ this.#state = 'inactive';
+ this.dispatchEvent(new Event('stop'));
+ }
+
+ #getEventHandler(eventType: string): EventListener | null {
+ return this.#eventHandlers.get(eventType) ?? null;
+ }
+
+ #setEventHandler(eventType: string, handler: EventListener | null): void {
+ const previousHandler = this.#eventHandlers.get(eventType);
+
+ if (isDefined(previousHandler)) {
+ this.removeEventListener(eventType, previousHandler);
+ this.#eventHandlers.delete(eventType);
+ }
+
+ if (handler !== null) {
+ this.#eventHandlers.set(eventType, handler);
+ this.addEventListener(eventType, handler);
+ }
+ }
+ }
+
+ return { MediaRecorderImplementation };
+};
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaStreamClass.ts b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaStreamClass.ts
new file mode 100644
index 00000000000..ee74dabb410
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaStreamClass.ts
@@ -0,0 +1,131 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type WorkerMediaStreamTrackInstance } from '@/polyfills/media/utils/createMediaStreamTrackClass';
+import { generateRandomId } from '@/utils/generateRandomId';
+
+export type WorkerMediaStreamInstance = EventTarget & {
+ readonly id: string;
+ readonly active: boolean;
+ getTracks: () => WorkerMediaStreamTrackInstance[];
+ getAudioTracks: () => WorkerMediaStreamTrackInstance[];
+ getVideoTracks: () => WorkerMediaStreamTrackInstance[];
+ getTrackById: (trackId: string) => WorkerMediaStreamTrackInstance | null;
+ addTrack: (track: WorkerMediaStreamTrackInstance) => void;
+ removeTrack: (track: WorkerMediaStreamTrackInstance) => void;
+};
+
+const generateLocalMediaStreamId = (): string =>
+ `local-media-stream-${generateRandomId()}`;
+
+const createTrackEvent = (
+ eventType: string,
+ track: WorkerMediaStreamTrackInstance,
+): Event => {
+ const trackEvent = new Event(eventType);
+
+ Object.defineProperty(trackEvent, 'track', { value: track });
+
+ return trackEvent;
+};
+
+export const createMediaStreamClass = () => {
+ const capturedStreamIds = new WeakMap<object, string>();
+
+ class MediaStreamImplementation
+ extends EventTarget
+ implements WorkerMediaStreamInstance
+ {
+ readonly id: string;
+
+ #tracks: WorkerMediaStreamTrackInstance[] = [];
+
+ constructor(
+ streamOrTracks?:
+ | MediaStreamImplementation
+ | WorkerMediaStreamTrackInstance[],
+ ) {
+ super();
+
+ if (streamOrTracks instanceof MediaStreamImplementation) {
+ this.#tracks = [...streamOrTracks.getTracks()];
+
+ // A clone of a captured stream stays recordable, like the native
+ // constructor which keeps the same source tracks.
+ const sourceCapturedStreamId = capturedStreamIds.get(streamOrTracks);
+
+ if (isDefined(sourceCapturedStreamId)) {
+ capturedStreamIds.set(this, sourceCapturedStreamId);
+ }
+ } else if (Array.isArray(streamOrTracks)) {
+ this.#tracks = [...streamOrTracks];
+ }
+
+ this.id = generateLocalMediaStreamId();
+ }
+
+ get active(): boolean {
+ return this.#tracks.some((track) => track.readyState === 'live');
+ }
+
+ getTracks(): WorkerMediaStreamTrackInstance[] {
+ return [...this.#tracks];
+ }
+
+ getAudioTracks(): WorkerMediaStreamTrackInstance[] {
+ return this.#tracks.filter((track) => track.kind === 'audio');
+ }
+
+ getVideoTracks(): WorkerMediaStreamTrackInstance[] {
+ return this.#tracks.filter((track) => track.kind === 'video');
+ }
+
+ getTrackById(trackId: string): WorkerMediaStreamTrackInstance | null {
+ return this.#tracks.find((track) => track.id === trackId) ?? null;
+ }
+
+ addTrack(track: WorkerMediaStreamTrackInstance): void {
+ if (this.#tracks.includes(track)) {
+ return;
+ }
+
+ this.#tracks.push(track);
+ this.dispatchEvent(createTrackEvent('addtrack', track));
+ }
+
+ removeTrack(track: WorkerMediaStreamTrackInstance): void {
+ if (!this.#tracks.includes(track)) {
+ return;
+ }
+
+ this.#tracks = this.#tracks.filter(
+ (existingTrack) => existingTrack !== track,
+ );
+ this.dispatchEvent(createTrackEvent('removetrack', track));
+ }
+ }
+
+ // Captured streams keep their host streamId out of band: the public id
+ // stays a local value while the recorder resolves the host session id.
+ const instantiateCapturedMediaStream = ({
+ streamId,
+ tracks,
+ }: {
+ streamId: string;
+ tracks: WorkerMediaStreamTrackInstance[];
+ }): WorkerMediaStreamInstance => {
+ const capturedStream = new MediaStreamImplementation(tracks);
+
+ capturedStreamIds.set(capturedStream, streamId);
+
+ return capturedStream;
+ };
+
+ const resolveCapturedStreamId = (stream: object): string | null =>
+ capturedStreamIds.get(stream) ?? null;
+
+ return {
+ MediaStreamImplementation,
+ instantiateCapturedMediaStream,
+ resolveCapturedStreamId,
+ };
+};
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaStreamTrackClass.ts b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaStreamTrackClass.ts
new file mode 100644
index 00000000000..8b7f8cf4916
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createMediaStreamTrackClass.ts
@@ -0,0 +1,134 @@
+import { type WorkerMediaBridge } from '@/polyfills/media/types/WorkerMediaBridge';
+import { type MediaSessionMediaType } from '@/types/MediaSession';
+
+type CreateMediaStreamTrackClassInput = {
+ bridge: WorkerMediaBridge;
+};
+
+export type WorkerMediaStreamTrackInit = {
+ streamId: string;
+ trackId: string;
+ kind: MediaSessionMediaType;
+};
+
+export type WorkerMediaStreamTrackInstance = EventTarget & {
+ readonly id: string;
+ readonly kind: MediaSessionMediaType;
+ readonly label: string;
+ readonly readyState: 'live' | 'ended';
+ enabled: boolean;
+ onended: ((event: Event) => void) | null;
+ stop: () => void;
+};
+
+export const createMediaStreamTrackClass = ({
+ bridge,
+}: CreateMediaStreamTrackClassInput) => {
+ const constructionKey = Symbol('workerMediaStreamTrackConstruction');
+
+ class MediaStreamTrackImplementation
+ extends EventTarget
+ implements WorkerMediaStreamTrackInstance
+ {
+ readonly id: string;
+ readonly kind: MediaSessionMediaType;
+ readonly label = '';
+
+ #streamId: string;
+ #readyState: 'live' | 'ended' = 'live';
+ #enabled = true;
+ #onended: ((event: Event) => void) | null = null;
+
+ constructor(key: symbol, init: WorkerMediaStreamTrackInit) {
+ super();
+
+ // Real tracks only come from capture calls; direct construction is not
+ // meaningful, which is also what native implementations enforce.
+ if (key !== constructionKey) {
+ throw new TypeError('Illegal constructor');
+ }
+
+ this.id = init.trackId;
+ this.kind = init.kind;
+ this.#streamId = init.streamId;
+
+ bridge.registerTrackEventHandlers({
+ streamId: init.streamId,
+ trackId: init.trackId,
+ handlers: {
+ onEnded: () => this.#handleHostEnded(),
+ },
+ });
+ }
+
+ get readyState(): 'live' | 'ended' {
+ return this.#readyState;
+ }
+
+ get enabled(): boolean {
+ return this.#enabled;
+ }
+
+ set enabled(enabled: boolean) {
+ const nextEnabled = Boolean(enabled);
+
+ if (this.#enabled === nextEnabled) {
+ return;
+ }
+
+ this.#enabled = nextEnabled;
+
+ if (this.#readyState === 'live') {
+ bridge.setStreamTrackEnabled({
+ streamId: this.#streamId,
+ trackId: this.id,
+ enabled: nextEnabled,
+ });
+ }
+ }
+
+ get onended(): ((event: Event) => void) | null {
+ return this.#onended;
+ }
+
+ set onended(handler: ((event: Event) => void) | null) {
+ if (this.#onended !== null) {
+ this.removeEventListener('ended', this.#onended);
+ }
+
+ this.#onended = handler;
+
+ if (handler !== null) {
+ this.addEventListener('ended', handler);
+ }
+ }
+
+ stop(): void {
+ if (this.#readyState === 'ended') {
+ return;
+ }
+
+ // Native stop() flips readyState synchronously and fires no ended
+ // event: that event is reserved for externally caused endings.
+ this.#readyState = 'ended';
+
+ bridge.stopStreamTrack({ streamId: this.#streamId, trackId: this.id });
+ }
+
+ #handleHostEnded(): void {
+ if (this.#readyState === 'ended') {
+ return;
+ }
+
+ this.#readyState = 'ended';
+ this.dispatchEvent(new Event('ended'));
+ }
+ }
+
+ const instantiateMediaStreamTrack = (
+ init: WorkerMediaStreamTrackInit,
+ ): WorkerMediaStreamTrackInstance =>
+ new MediaStreamTrackImplementation(constructionKey, init);
+
+ return { MediaStreamTrackImplementation, instantiateMediaStreamTrack };
+};
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/utils/createWorkerMediaBridge.ts b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createWorkerMediaBridge.ts
new file mode 100644
index 00000000000..5ccda421283
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/utils/createWorkerMediaBridge.ts
@@ -0,0 +1,264 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { MEDIA_TRANSPORT_FAILURE_WARNING } from '@/polyfills/media/constants/MediaTransportFailureWarning';
+import {
+ type WorkerMediaBridge,
+ type WorkerMediaRecorderEventHandlers,
+ type WorkerMediaTrackEventHandlers,
+} from '@/polyfills/media/types/WorkerMediaBridge';
+import {
+ type MediaRecorderCapabilities,
+ type MediaSessionHostFunctions,
+ type StartMediaRecorderResult,
+ type StartMediaStreamResult,
+} from '@/types/MediaSession';
+
+const buildTrackKey = (streamId: string, trackId: string): string =>
+ `${streamId}:${trackId}`;
+
+const MEDIA_TRANSPORT_UNAVAILABLE_FAILURE = {
+ status: 'failed',
+ errorName: 'NotSupportedError',
+ errorMessage: 'Media capture is not available in this environment',
+} as const;
+
+export const createWorkerMediaBridge = (): WorkerMediaBridge => {
+ const trackEventHandlersByTrackKey = new Map<
+ string,
+ WorkerMediaTrackEventHandlers
+ >();
+ const recorderEventHandlersByRecorderId = new Map<
+ string,
+ WorkerMediaRecorderEventHandlers
+ >();
+ const supportedRecorderMimeTypes = new Set<string>();
+
+ let transport: MediaSessionHostFunctions | null = null;
+ let hasWarnedAboutTransportFailure = false;
+
+ const warnAboutTransportFailure = (): void => {
+ if (hasWarnedAboutTransportFailure) {
+ return;
+ }
+
+ hasWarnedAboutTransportFailure = true;
+ console.warn(MEDIA_TRANSPORT_FAILURE_WARNING);
+ };
+
+ const connectTransport = (nextTransport: MediaSessionHostFunctions): void => {
+ transport = nextTransport;
+ };
+
+ const seedRecorderCapabilities = (
+ capabilities: MediaRecorderCapabilities,
+ ): void => {
+ for (const mimeType of capabilities.supportedMimeTypes) {
+ supportedRecorderMimeTypes.add(mimeType.trim().toLowerCase());
+ }
+ };
+
+ const isRecorderMimeTypeSupported = (mimeType: string): boolean => {
+ // The spec treats the empty string as "no preference", which is always
+ // supported.
+ if (mimeType === '') {
+ return true;
+ }
+
+ return supportedRecorderMimeTypes.has(mimeType.trim().toLowerCase());
+ };
+
+ const startStream = async (params: {
+ audio: boolean;
+ video: boolean;
+ }): Promise<StartMediaStreamResult> => {
+ if (!isDefined(transport)) {
+ return MEDIA_TRANSPORT_UNAVAILABLE_FAILURE;
+ }
+
+ try {
+ return await transport.mediaStartStream(params);
+ } catch {
+ return MEDIA_TRANSPORT_UNAVAILABLE_FAILURE;
+ }
+ };
+
+ const registerTrackEventHandlers = ({
+ streamId,
+ trackId,
+ handlers,
+ }: {
+ streamId: string;
+ trackId: string;
+ handlers: WorkerMediaTrackEventHandlers;
+ }): void => {
+ trackEventHandlersByTrackKey.set(
+ buildTrackKey(streamId, trackId),
+ handlers,
+ );
+ };
+
+ const stopStreamTrack = ({
+ streamId,
+ trackId,
+ }: {
+ streamId: string;
+ trackId: string;
+ }): void => {
+ // A self-initiated stop expects no further events for the track.
+ trackEventHandlersByTrackKey.delete(buildTrackKey(streamId, trackId));
+
+ if (!isDefined(transport)) {
+ warnAboutTransportFailure();
+ return;
+ }
+
+ transport
+ .mediaStopStreamTrack({ streamId, trackId })
+ .catch(warnAboutTransportFailure);
+ };
+
+ const setStreamTrackEnabled = ({
+ streamId,
+ trackId,
+ enabled,
+ }: {
+ streamId: string;
+ trackId: string;
+ enabled: boolean;
+ }): void => {
+ if (!isDefined(transport)) {
+ warnAboutTransportFailure();
+ return;
+ }
+
+ transport
+ .mediaSetTrackEnabled({ streamId, trackId, enabled })
+ .catch(warnAboutTransportFailure);
+ };
+
+ const startRecorder = async ({
+ streamId,
+ mimeType,
+ timesliceMs,
+ handlers,
+ }: {
+ streamId: string;
+ mimeType?: string;
+ timesliceMs?: number;
+ handlers: WorkerMediaRecorderEventHandlers;
+ }): Promise<StartMediaRecorderResult> => {
+ if (!isDefined(transport)) {
+ return MEDIA_TRANSPORT_UNAVAILABLE_FAILURE;
+ }
+
+ let result: StartMediaRecorderResult;
+
+ try {
+ result = await transport.mediaStartRecorder({
+ streamId,
+ mimeType,
+ timesliceMs,
+ });
+ } catch {
+ return MEDIA_TRANSPORT_UNAVAILABLE_FAILURE;
+ }
+
+ // Events for this recorder can only arrive after this response: both
+ // travel the same message port, so registering now cannot miss any.
+ if (result.status === 'started') {
+ recorderEventHandlersByRecorderId.set(result.recorderId, handlers);
+ }
+
+ return result;
+ };
+
+ const callRecorderTransport = (
+ recorderId: string,
+ callTransport: (
+ connectedTransport: MediaSessionHostFunctions,
+ ) => Promise<void>,
+ ): void => {
+ if (!isDefined(transport)) {
+ warnAboutTransportFailure();
+ return;
+ }
+
+ callTransport(transport).catch(warnAboutTransportFailure);
+ };
+
+ const stopRecorder = (recorderId: string): void => {
+ callRecorderTransport(recorderId, (connectedTransport) =>
+ connectedTransport.mediaStopRecorder({ recorderId }),
+ );
+ };
+
+ const pauseRecorder = (recorderId: string): void => {
+ callRecorderTransport(recorderId, (connectedTransport) =>
+ connectedTransport.mediaPauseRecorder({ recorderId }),
+ );
+ };
+
+ const resumeRecorder = (recorderId: string): void => {
+ callRecorderTransport(recorderId, (connectedTransport) =>
+ connectedTransport.mediaResumeRecorder({ recorderId }),
+ );
+ };
+
+ const requestRecorderData = (recorderId: string): void => {
+ callRecorderTransport(recorderId, (connectedTransport) =>
+ connectedTransport.mediaRequestRecorderData({ recorderId }),
+ );
+ };
+
+ const dispatchEvents: WorkerMediaBridge['dispatchEvents'] = (batch) => {
+ for (const event of batch.events) {
+ switch (event.type) {
+ case 'track-ended': {
+ const trackKey = buildTrackKey(event.streamId, event.trackId);
+ const handlers = trackEventHandlersByTrackKey.get(trackKey);
+
+ trackEventHandlersByTrackKey.delete(trackKey);
+ handlers?.onEnded();
+ break;
+ }
+ case 'recorder-data': {
+ recorderEventHandlersByRecorderId
+ .get(event.recorderId)
+ ?.onData(event.data);
+ break;
+ }
+ case 'recorder-stop': {
+ const handlers = recorderEventHandlersByRecorderId.get(
+ event.recorderId,
+ );
+
+ recorderEventHandlersByRecorderId.delete(event.recorderId);
+ handlers?.onStop();
+ break;
+ }
+ case 'recorder-error': {
+ recorderEventHandlersByRecorderId
+ .get(event.recorderId)
+ ?.onError(event.errorMessage);
+ break;
+ }
+ }
+ }
+ };
+
+ return {
+ connectTransport,
+ seedRecorderCapabilities,
+ isRecorderMimeTypeSupported,
+ startStream,
+ registerTrackEventHandlers,
+ stopStreamTrack,
+ setStreamTrackEnabled,
+ startRecorder,
+ stopRecorder,
+ pauseRecorder,
+ resumeRecorder,
+ requestRecorderData,
+ dispatchEvents,
+ };
+};
diff --git a/packages/twenty-front-component-renderer/src/polyfills/media/utils/installMediaCapturePolyfills.ts b/packages/twenty-front-component-renderer/src/polyfills/media/utils/installMediaCapturePolyfills.ts
new file mode 100644
index 00000000000..3fe77b8da69
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/polyfills/media/utils/installMediaCapturePolyfills.ts
@@ -0,0 +1,114 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type WorkerMediaBridge } from '@/polyfills/media/types/WorkerMediaBridge';
+import { createDomException } from '@/polyfills/media/utils/createDomException';
+import { createMediaRecorderClass } from '@/polyfills/media/utils/createMediaRecorderClass';
+import {
+ createMediaStreamClass,
+ type WorkerMediaStreamInstance,
+} from '@/polyfills/media/utils/createMediaStreamClass';
+import { createMediaStreamTrackClass } from '@/polyfills/media/utils/createMediaStreamTrackClass';
+import { resolveGlobalScopeInstallTargets } from '@/polyfills/utils/resolveGlobalScopeInstallTargets';
+
+type InstallMediaCapturePolyfillsInput = {
+ globalScope: Record<string, unknown>;
+ bridge: WorkerMediaBridge;
+};
+
+type UserMediaConstraints = {
+ audio?: unknown;
+ video?: unknown;
+};
+
+type NavigatorWithMediaDevices = {
+ mediaDevices?: unknown;
+};
+
+export const installMediaCapturePolyfills = ({
+ globalScope,
+ bridge,
+}: InstallMediaCapturePolyfillsInput): void => {
+ const { MediaStreamTrackImplementation, instantiateMediaStreamTrack } =
+ createMediaStreamTrackClass({ bridge });
+
+ const {
+ MediaStreamImplementation,
+ instantiateCapturedMediaStream,
+ resolveCapturedStreamId,
+ } = createMediaStreamClass();
+
+ const { MediaRecorderImplementation } = createMediaRecorderClass({
+ bridge,
+ isMediaStreamInstance: (stream): stream is WorkerMediaStreamInstance =>
+ stream instanceof MediaStreamImplementation,
+ resolveCapturedStreamId,
+ });
+
+ const getUserMedia = async (
+ constraints?: UserMediaConstraints,
+ ): Promise<WorkerMediaStreamInstance> => {
+ const isAudioRequested = Boolean(constraints?.audio);
+ const isVideoRequested = Boolean(constraints?.video);
+
+ if (!isAudioRequested && !isVideoRequested) {
+ throw new TypeError(
+ "Failed to execute 'getUserMedia' on 'MediaDevices': At least one of audio and video must be requested",
+ );
+ }
+
+ // Detailed constraint objects (deviceId, resolution, ...) are accepted
+ // but not forwarded: the host captures with its defaults for the
+ // requested kinds.
+ const startStreamResult = await bridge.startStream({
+ audio: isAudioRequested,
+ video: isVideoRequested,
+ });
+
+ if (startStreamResult.status === 'failed') {
+ throw createDomException(
+ startStreamResult.errorMessage,
+ startStreamResult.errorName,
+ );
+ }
+
+ const tracks = startStreamResult.tracks.map((trackDescriptor) =>
+ instantiateMediaStreamTrack({
+ streamId: startStreamResult.streamId,
+ trackId: trackDescriptor.trackId,
+ kind: trackDescriptor.kind,
+ }),
+ );
+
+ return instantiateCapturedMediaStream({
+ streamId: startStreamResult.streamId,
+ tracks,
+ });
+ };
+
+ const mediaDevices = { getUserMedia };
+
+ for (const installTarget of resolveGlobalScopeInstallTargets(globalScope)) {
+ installTarget.MediaStream = MediaStreamImplementation;
+ installTarget.MediaStreamTrack = MediaStreamTrackImplementation;
+ installTarget.MediaRecorder = MediaRecorderImplementation;
+
+ const targetNavigator = (installTarget.navigator ??
+ globalScope.navigator) as NavigatorWithMediaDevices | undefined;
+
+ if (!isDefined(targetNavigator)) {
+ installTarget.navigator = { mediaDevices };
+ continue;
+ }
+
+ installTarget.navigator ??= targetNavigator;
+
+ // The worker's own navigator is a platform object whose mediaDevices
+ // does not exist; define it without clobbering anything already there.
+ if (!isDefined(targetNavigator.mediaDevices)) {
+ Object.defineProperty(targetNavigator, 'mediaDevices', {
+ configurable: true,
+ value: mediaDevices,
+ });
+ }
+ }
+};
diff --git a/packages/twenty-front-component-renderer/src/remote/worker/remote-worker.ts b/packages/twenty-front-component-renderer/src/remote/worker/remote-worker.ts
index 6332f07135d..0c09f160259 100644
--- a/packages/twenty-front-component-renderer/src/remote/worker/remote-worker.ts
+++ b/packages/twenty-front-component-renderer/src/remote/worker/remote-worker.ts
@@ -18,6 +18,8 @@ import { installMutationObserver } from '@/polyfills/dom/utils/installMutationOb
import { workerGeometryStore } from '@/polyfills/geometry/states/workerGeometryStore';
import { installElementGeometryPolyfill } from '@/polyfills/geometry/utils/installElementGeometryPolyfill';
import { installWindowGeometryPolyfill } from '@/polyfills/geometry/utils/installWindowGeometryPolyfill';
+import { workerMediaBridge } from '@/polyfills/media/states/workerMediaBridge';
+import { installMediaCapturePolyfills } from '@/polyfills/media/utils/installMediaCapturePolyfills';
import { frontComponentStorageBridges } from '@/polyfills/storage/states/frontComponentStorageBridges';
import { toGlobalScopeRecord } from '@/polyfills/utils/toGlobalScopeRecord';
import { installStorageBridge } from '@/polyfills/storage/utils/installStorageBridge';
@@ -81,6 +83,11 @@ installClipboardPolyfill({
},
});
+installMediaCapturePolyfills({
+ globalScope: toGlobalScopeRecord(globalThis),
+ bridge: workerMediaBridge,
+});
+
exposeGlobals({
__HTML_TAG_TO_CUSTOM_ELEMENT_TAG__: HTML_TAG_TO_CUSTOM_ELEMENT_TAG,
});
@@ -122,6 +129,9 @@ const workerExports: WorkerExports = {
pushGeometryUpdates: async (batch) => {
workerGeometryStore.applyGeometryBatch(batch);
},
+ pushMediaSessionEvents: async (batch) => {
+ workerMediaBridge.dispatchEvents(batch);
+ },
};
self.addEventListener('message', (event) => {
@@ -141,6 +151,7 @@ self.addEventListener('message', (event) => {
hostThread = nextHostThread;
workerGeometryStore.connectTransport(nextHostThread.imports);
+ workerMediaBridge.connectTransport(nextHostThread.imports);
transferredPort.start();
});
diff --git a/packages/twenty-front-component-renderer/src/remote/worker/rendering/utils/renderFrontComponent.ts b/packages/twenty-front-component-renderer/src/remote/worker/rendering/utils/renderFrontComponent.ts
index dfba3a2bedb..af2e4bd68fc 100644
--- a/packages/twenty-front-component-renderer/src/remote/worker/rendering/utils/renderFrontComponent.ts
+++ b/packages/twenty-front-component-renderer/src/remote/worker/rendering/utils/renderFrontComponent.ts
@@ -2,6 +2,7 @@ import { type RemoteConnection } from '@remote-dom/core/elements';
import { CustomError, isDefined } from 'twenty-shared/utils';
import { workerGeometryStore } from '@/polyfills/geometry/states/workerGeometryStore';
+import { workerMediaBridge } from '@/polyfills/media/states/workerMediaBridge';
import { frontComponentStorageBridges } from '@/polyfills/storage/states/frontComponentStorageBridges';
import { attachRemoteRenderRootToWorkerDocument } from '@/remote/worker/rendering/utils/attachRemoteRenderRootToWorkerDocument';
import { installHostFetchProxy } from '@/remote/worker/fetch-proxy/utils/installHostFetchProxy';
@@ -40,6 +41,12 @@ export const renderFrontComponent = async ({
});
}
+ if (isDefined(renderContext.mediaRecorderCapabilities)) {
+ workerMediaBridge.seedRecorderCapabilities(
+ renderContext.mediaRecorderCapabilities,
+ );
+ }
+
const storageSnapshots = renderContext.storageSnapshots;
if (isDefined(storageSnapshots)) {
diff --git a/packages/twenty-front-component-renderer/src/remote/worker/thread/utils/__tests__/buildFrontComponentHostCommunicationApiFromThreadImports.test.ts b/packages/twenty-front-component-renderer/src/remote/worker/thread/utils/__tests__/buildFrontComponentHostCommunicationApiFromThreadImports.test.ts
index c1e93a106f6..79a907aac94 100644
--- a/packages/twenty-front-component-renderer/src/remote/worker/thread/utils/__tests__/buildFrontComponentHostCommunicationApiFromThreadImports.test.ts
+++ b/packages/twenty-front-component-renderer/src/remote/worker/thread/utils/__tests__/buildFrontComponentHostCommunicationApiFromThreadImports.test.ts
@@ -13,6 +13,7 @@ const createHostThreadImportsStub = () =>
closeSidePanel: jest.fn(),
updateProgress: jest.fn(),
copyToClipboard: jest.fn(),
+ uploadFile: jest.fn(),
storageSet: jest.fn(),
storageDelete: jest.fn(),
storageClear: jest.fn(),
@@ -45,6 +46,7 @@ describe('buildFrontComponentHostCommunicationApiFromThreadImports', () => {
'storageSet',
'unmountFrontComponent',
'updateProgress',
+ 'uploadFile',
]);
expect(hostCommunicationApi.navigate).toBe(hostThreadImports.navigate);
expect(hostCommunicationApi.requestAccessTokenRefresh).toBe(
@@ -56,6 +58,7 @@ describe('buildFrontComponentHostCommunicationApiFromThreadImports', () => {
expect(hostCommunicationApi.unmountFrontComponent).toBe(
hostThreadImports.unmountFrontComponent,
);
+ expect(hostCommunicationApi.uploadFile).toBe(hostThreadImports.uploadFile);
expect(hostCommunicationApi.enqueueSnackbar).toBe(
hostThreadImports.enqueueSnackbar,
);
diff --git a/packages/twenty-front-component-renderer/src/remote/worker/thread/utils/buildFrontComponentHostCommunicationApiFromThreadImports.ts b/packages/twenty-front-component-renderer/src/remote/worker/thread/utils/buildFrontComponentHostCommunicationApiFromThreadImports.ts
index a2caac60334..88598bf2113 100644
--- a/packages/twenty-front-component-renderer/src/remote/worker/thread/utils/buildFrontComponentHostCommunicationApiFromThreadImports.ts
+++ b/packages/twenty-front-component-renderer/src/remote/worker/thread/utils/buildFrontComponentHostCommunicationApiFromThreadImports.ts
@@ -15,6 +15,7 @@ export const buildFrontComponentHostCommunicationApiFromThreadImports = (
closeSidePanel: hostThreadImports.closeSidePanel,
updateProgress: hostThreadImports.updateProgress,
copyToClipboard: hostThreadImports.copyToClipboard,
+ uploadFile: hostThreadImports.uploadFile,
storageSet: hostThreadImports.storageSet,
storageDelete: hostThreadImports.storageDelete,
storageClear: hostThreadImports.storageClear,
diff --git a/packages/twenty-front-component-renderer/src/types/FrontComponentHostCommunicationApi.ts b/packages/twenty-front-component-renderer/src/types/FrontComponentHostCommunicationApi.ts
index 2213f1b052c..389aec022d5 100644
--- a/packages/twenty-front-component-renderer/src/types/FrontComponentHostCommunicationApi.ts
+++ b/packages/twenty-front-component-renderer/src/types/FrontComponentHostCommunicationApi.ts
@@ -11,6 +11,7 @@ import {
type StorageSetFunction,
type UnmountFrontComponentFunction,
type UpdateProgressFunction,
+ type UploadFileFunction,
} from 'twenty-sdk/front-component';
export type FrontComponentHostCommunicationApi = {
@@ -23,6 +24,7 @@ export type FrontComponentHostCommunicationApi = {
closeSidePanel: CloseSidePanelFunction;
updateProgress: UpdateProgressFunction;
copyToClipboard: CopyToClipboardFunction;
+ uploadFile: UploadFileFunction;
storageSet: StorageSetFunction;
storageDelete: StorageDeleteFunction;
storageClear: StorageClearFunction;
diff --git a/packages/twenty-front-component-renderer/src/types/FrontComponentHostThreadExports.ts b/packages/twenty-front-component-renderer/src/types/FrontComponentHostThreadExports.ts
index b87ee7715ce..f9d6be9a655 100644
--- a/packages/twenty-front-component-renderer/src/types/FrontComponentHostThreadExports.ts
+++ b/packages/twenty-front-component-renderer/src/types/FrontComponentHostThreadExports.ts
@@ -1,9 +1,11 @@
import { type FrontComponentHostCommunicationApi } from '@/types/FrontComponentHostCommunicationApi';
import { type HostFetchFunction } from '@/types/HostFetchFunction';
+import { type MediaSessionHostFunctions } from '@/types/MediaSession';
export type FrontComponentHostThreadExports =
- FrontComponentHostCommunicationApi & {
- hostFetch: HostFetchFunction;
- observeElementGeometry: (remoteElementIds: string[]) => Promise<void>;
- unobserveElementGeometry: (remoteElementIds: string[]) => Promise<void>;
- };
+ FrontComponentHostCommunicationApi &
+ MediaSessionHostFunctions & {
+ hostFetch: HostFetchFunction;
+ observeElementGeometry: (remoteElementIds: string[]) => Promise<void>;
+ unobserveElementGeometry: (remoteElementIds: string[]) => Promise<void>;
+ };
diff --git a/packages/twenty-front-component-renderer/src/types/HostToWorkerRenderContext.ts b/packages/twenty-front-component-renderer/src/types/HostToWorkerRenderContext.ts
index cd3bf3dad43..384bc5917b1 100644
--- a/packages/twenty-front-component-renderer/src/types/HostToWorkerRenderContext.ts
+++ b/packages/twenty-front-component-renderer/src/types/HostToWorkerRenderContext.ts
@@ -1,4 +1,5 @@
import { type FrontComponentStorageSnapshots } from '@/types/FrontComponentStorageSnapshots';
+import { type MediaRecorderCapabilities } from '@/types/MediaSession';
import { type SdkClientSources } from '@/types/SdkClientSources';
import { type ViewportGeometrySnapshot } from '@/types/ViewportGeometrySnapshot';
@@ -14,4 +15,7 @@ export type HostToWorkerRenderContext = {
applicationVariables?: Record<string, string>;
initialViewportGeometry?: ViewportGeometrySnapshot;
storageSnapshots?: FrontComponentStorageSnapshots;
+ // MediaRecorder.isTypeSupported is synchronous, so the worker polyfill
+ // answers from this snapshot instead of a round trip to the host.
+ mediaRecorderCapabilities?: MediaRecorderCapabilities;
};
diff --git a/packages/twenty-front-component-renderer/src/types/MediaSession.ts b/packages/twenty-front-component-renderer/src/types/MediaSession.ts
new file mode 100644
index 00000000000..a68e74c4c75
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/types/MediaSession.ts
@@ -0,0 +1,67 @@
+export type MediaSessionMediaType = 'audio' | 'video';
+
+export type MediaSessionTrackDescriptor = {
+ trackId: string;
+ kind: MediaSessionMediaType;
+};
+
+// Failures carry the standard DOMException name (NotAllowedError,
+// NotFoundError, NotReadableError, ...) so the worker polyfill can reject
+// with the same error shape native getUserMedia produces.
+export type StartMediaStreamResult =
+ | {
+ status: 'started';
+ streamId: string;
+ tracks: MediaSessionTrackDescriptor[];
+ }
+ | { status: 'failed'; errorName: string; errorMessage: string };
+
+export type StartMediaRecorderResult =
+ | { status: 'started'; recorderId: string; mimeType: string }
+ | { status: 'failed'; errorName: string; errorMessage: string };
+
+export type MediaSessionEvent =
+ | { type: 'track-ended'; streamId: string; trackId: string }
+ | { type: 'recorder-data'; recorderId: string; data: Blob }
+ | { type: 'recorder-stop'; recorderId: string }
+ | { type: 'recorder-error'; recorderId: string; errorMessage: string };
+
+export type MediaSessionEventBatch = {
+ events: MediaSessionEvent[];
+};
+
+export type MediaRecorderCapabilities = {
+ supportedMimeTypes: string[];
+};
+
+// Media capture crosses the sandbox boundary like hostFetch and geometry:
+// as dedicated thread functions owned by the renderer, not through the
+// application-facing host communication api.
+export type MediaSessionHostFunctions = {
+ // The requested kinds are forwarded as booleans so video-only capture
+ // does not silently open the microphone.
+ mediaStartStream: (params: {
+ audio: boolean;
+ video: boolean;
+ }) => Promise<StartMediaStreamResult>;
+ mediaStopStreamTrack: (params: {
+ streamId: string;
+ trackId: string;
+ }) => Promise<void>;
+ // track.enabled must reach the real track: a worker-local flag would let
+ // an application believe it muted a device that is still being captured.
+ mediaSetTrackEnabled: (params: {
+ streamId: string;
+ trackId: string;
+ enabled: boolean;
+ }) => Promise<void>;
+ mediaStartRecorder: (params: {
+ streamId: string;
+ mimeType?: string;
+ timesliceMs?: number;
+ }) => Promise<StartMediaRecorderResult>;
+ mediaStopRecorder: (params: { recorderId: string }) => Promise<void>;
+ mediaPauseRecorder: (params: { recorderId: string }) => Promise<void>;
+ mediaResumeRecorder: (params: { recorderId: string }) => Promise<void>;
+ mediaRequestRecorderData: (params: { recorderId: string }) => Promise<void>;
+};
diff --git a/packages/twenty-front-component-renderer/src/types/WorkerExports.ts b/packages/twenty-front-component-renderer/src/types/WorkerExports.ts
index 1c0fe398b32..d364c3c361c 100644
--- a/packages/twenty-front-component-renderer/src/types/WorkerExports.ts
+++ b/packages/twenty-front-component-renderer/src/types/WorkerExports.ts
@@ -3,6 +3,7 @@ import { type CommandConfirmationModalResult } from 'twenty-sdk/front-component'
import { type FrontComponentExecutionContext } from '@/types/FrontComponentExecutionContext';
import { type GeometryUpdateBatch } from '@/types/GeometryUpdateBatch';
import { type HostToWorkerRenderContext } from '@/types/HostToWorkerRenderContext';
+import { type MediaSessionEventBatch } from '@/types/MediaSession';
export type WorkerExports = {
render: (
@@ -15,4 +16,5 @@ export type WorkerExports = {
result: CommandConfirmationModalResult,
) => Promise<void>;
pushGeometryUpdates: (batch: GeometryUpdateBatch) => Promise<void>;
+ pushMediaSessionEvents: (batch: MediaSessionEventBatch) => Promise<void>;
};
diff --git a/packages/twenty-front-component-renderer/src/types/WorkerFrontComponentHostCommunicationApi.ts b/packages/twenty-front-component-renderer/src/types/WorkerFrontComponentHostCommunicationApi.ts
index bd5be8a2950..0ff842aedfa 100644
--- a/packages/twenty-front-component-renderer/src/types/WorkerFrontComponentHostCommunicationApi.ts
+++ b/packages/twenty-front-component-renderer/src/types/WorkerFrontComponentHostCommunicationApi.ts
@@ -11,6 +11,7 @@ import {
type StorageSetFunction,
type UnmountFrontComponentFunction,
type UpdateProgressFunction,
+ type UploadFileFunction,
} from 'twenty-sdk/front-component';
export type WorkerFrontComponentHostCommunicationApi = {
@@ -23,6 +24,7 @@ export type WorkerFrontComponentHostCommunicationApi = {
closeSidePanel?: CloseSidePanelFunction;
updateProgress?: UpdateProgressFunction;
copyToClipboard?: CopyToClipboardFunction;
+ uploadFile?: UploadFileFunction;
storageSet?: StorageSetFunction;
storageDelete?: StorageDeleteFunction;
storageClear?: StorageClearFunction;
diff --git a/packages/twenty-front-component-renderer/src/utils/generateRandomId.ts b/packages/twenty-front-component-renderer/src/utils/generateRandomId.ts
new file mode 100644
index 00000000000..069080998b9
--- /dev/null
+++ b/packages/twenty-front-component-renderer/src/utils/generateRandomId.ts
@@ -0,0 +1,14 @@
+// The sandboxed worker runs in an opaque origin, which is not a secure
+// context: crypto.randomUUID does not exist there, but getRandomValues does.
+export const generateRandomId = (): string => {
+ if (typeof crypto.randomUUID === 'function') {
+ return crypto.randomUUID();
+ }
+
+ const randomBytes = new Uint8Array(16);
+ crypto.getRandomValues(randomBytes);
+
+ return [...randomBytes]
+ .map((randomByte) => randomByte.toString(16).padStart(2, '0'))
+ .join('');
+};
diff --git a/packages/twenty-front/src/modules/front-components/components/FrontComponentRenderer.tsx b/packages/twenty-front/src/modules/front-components/components/FrontComponentRenderer.tsx
index f1638ea8691..0ed15672b7c 100644
--- a/packages/twenty-front/src/modules/front-components/components/FrontComponentRenderer.tsx
+++ b/packages/twenty-front/src/modules/front-components/components/FrontComponentRenderer.tsx
@@ -2,6 +2,7 @@ import { FrontComponentApplicationTokenPairEffect } from '@/front-components/com
import { FrontComponentLoadErrorSnackBarEffect } from '@/front-components/components/FrontComponentLoadErrorSnackBarEffect';
import { FrontComponentRendererProvider } from '@/front-components/components/FrontComponentRendererProvider';
import { useFrontComponentExecutionContext } from '@/front-components/hooks/useFrontComponentExecutionContext';
+import { useFrontComponentMediaSession } from '@/front-components/media-session/hooks/useFrontComponentMediaSession';
import { useOnApplicationSdkClientChecksumsUpdated } from '@/front-components/hooks/useOnApplicationSdkClientChecksumsUpdated';
import { useOnFrontComponentUpdated } from '@/front-components/hooks/useOnFrontComponentUpdated';
import { getFingerprintedRestUrl } from '@/front-components/utils/getFingerprintedRestUrl';
@@ -100,6 +101,8 @@ const FrontComponentRendererContent = ({
colorScheme,
});
+ const { mediaSessionHost } = useFrontComponentMediaSession();
+
const handleError = useCallback(
(error?: Error) => {
if (!isDefined(error)) {
@@ -174,6 +177,7 @@ const FrontComponentRendererContent = ({
frontComponentHostCommunicationApi={
frontComponentHostCommunicationApi
}
+ mediaSessionHost={mediaSessionHost}
applicationVariables={applicationVariables}
storageNamespace={storageNamespace}
onError={handleError}
diff --git a/packages/twenty-front/src/modules/front-components/hooks/__tests__/useFrontComponentExecutionContext.test.tsx b/packages/twenty-front/src/modules/front-components/hooks/__tests__/useFrontComponentExecutionContext.test.tsx
index 3d703a0c3a7..5b0617b1282 100644
--- a/packages/twenty-front/src/modules/front-components/hooks/__tests__/useFrontComponentExecutionContext.test.tsx
+++ b/packages/twenty-front/src/modules/front-components/hooks/__tests__/useFrontComponentExecutionContext.test.tsx
@@ -18,6 +18,17 @@ jest.mock('@/object-metadata/hooks/useObjectMetadataItems', () => ({
}),
}));
+jest.mock('@/object-metadata/utils/getFieldMetadataItemById', () => ({
+ getFieldMetadataItemById: (parameters: { fieldMetadataId: string }) => ({
+ fieldMetadataItem:
+ parameters.fieldMetadataId === 'files-field-id'
+ ? { id: 'files-field-id', type: 'FILES' }
+ : parameters.fieldMetadataId === 'text-field-id'
+ ? { id: 'text-field-id', type: 'TEXT' }
+ : undefined,
+ }),
+}));
+
const mockNavigateApp = jest.fn();
const mockRequestAccessTokenRefresh = jest.fn();
const mockOpenConfirmationModal = jest.fn();
@@ -36,6 +47,7 @@ const mockEnqueueWarningSnackBar = jest.fn();
const mockCloseSidePanelMenu = jest.fn();
const mockSetCommandMenuItemProgress = jest.fn();
const mockCopyToClipboard = jest.fn();
+const mockDirectUploadFile = jest.fn();
const mockSetRecordPageActiveTabId = jest.fn();
const mockStorageSet = jest.fn();
const mockStorageDelete = jest.fn();
@@ -143,6 +155,12 @@ jest.mock('~/hooks/useCopyToClipboard', () => ({
}),
}));
+jest.mock('@/file/hooks/useDirectFileUpload', () => ({
+ useDirectFileUpload: () => ({
+ uploadFile: mockDirectUploadFile,
+ }),
+}));
+
jest.mock('twenty-front-component-renderer', () => ({
buildFrontComponentStorageNamespace: ({
applicationId,
@@ -809,6 +827,156 @@ describe('useFrontComponentExecutionContext', () => {
});
});
+ describe('uploadFile', () => {
+ const buildRecordedBlob = () =>
+ new Blob(['recorded-bytes'], { type: 'audio/webm' });
+
+ beforeEach(() => {
+ // clearAllMocks keeps implementations; drop resolved/rejected values
+ // so these tests stay order-independent.
+ mockDirectUploadFile.mockReset();
+ });
+
+ it('should upload a blob into a FILES field and return the stored file', async () => {
+ mockDirectUploadFile.mockResolvedValue({
+ id: 'file-1',
+ path: 'files-field/file-1.webm',
+ url: 'https://example.com/files/file-1.webm',
+ size: 14,
+ });
+
+ const { result } = renderUseFrontComponentExecutionContext({
+ frontComponentId: FRONT_COMPONENT_ID,
+ });
+
+ const uploadResult =
+ await result.current.frontComponentHostCommunicationApi.uploadFile(
+ buildRecordedBlob(),
+ { fieldMetadataId: 'files-field-id', fileName: 'note.webm' },
+ );
+
+ expect(mockDirectUploadFile).toHaveBeenCalledWith(
+ expect.any(File),
+ expect.objectContaining({ fieldMetadataId: 'files-field-id' }),
+ );
+ expect(uploadResult).toEqual({
+ status: 'uploaded',
+ file: {
+ fileId: 'file-1',
+ path: 'files-field/file-1.webm',
+ url: 'https://example.com/files/file-1.webm',
+ size: 14,
+ mimeType: 'audio/webm',
+ },
+ });
+
+ const [uploadedFile] = mockDirectUploadFile.mock.calls[0];
+ expect(uploadedFile.name).toBe('note.webm');
+ });
+
+ it('should reject non-FILES fields without uploading', async () => {
+ const { result } = renderUseFrontComponentExecutionContext({
+ frontComponentId: FRONT_COMPONENT_ID,
+ });
+
+ const uploadResult =
+ await result.current.frontComponentHostCommunicationApi.uploadFile(
+ buildRecordedBlob(),
+ { fieldMetadataId: 'text-field-id' },
+ );
+
+ expect(uploadResult).toEqual({
+ status: 'failed',
+ reason: 'invalid-params',
+ });
+ expect(mockDirectUploadFile).not.toHaveBeenCalled();
+ });
+
+ it('should reject malformed arguments without uploading', async () => {
+ const { result } = renderUseFrontComponentExecutionContext({
+ frontComponentId: FRONT_COMPONENT_ID,
+ });
+
+ const emptyBlobResult =
+ await result.current.frontComponentHostCommunicationApi.uploadFile(
+ new Blob([], { type: 'audio/webm' }),
+ { fieldMetadataId: 'files-field-id' },
+ );
+ const missingFieldResult =
+ await result.current.frontComponentHostCommunicationApi.uploadFile(
+ buildRecordedBlob(),
+ { fieldMetadataId: '' },
+ );
+ const nonBlobResult =
+ await result.current.frontComponentHostCommunicationApi.uploadFile(
+ 'not-a-blob' as unknown as Blob,
+ { fieldMetadataId: 'files-field-id' },
+ );
+
+ expect(emptyBlobResult).toEqual({
+ status: 'failed',
+ reason: 'invalid-params',
+ });
+ expect(missingFieldResult).toEqual({
+ status: 'failed',
+ reason: 'invalid-params',
+ });
+ expect(nonBlobResult).toEqual({
+ status: 'failed',
+ reason: 'invalid-params',
+ });
+ expect(mockDirectUploadFile).not.toHaveBeenCalled();
+ });
+
+ it('should strip path separators from the file name and fall back when empty', async () => {
+ mockDirectUploadFile.mockResolvedValue({
+ id: 'file-2',
+ path: 'files-field/file-2.webm',
+ url: 'https://example.com/files/file-2.webm',
+ size: 14,
+ });
+
+ const { result } = renderUseFrontComponentExecutionContext({
+ frontComponentId: FRONT_COMPONENT_ID,
+ });
+
+ await result.current.frontComponentHostCommunicationApi.uploadFile(
+ buildRecordedBlob(),
+ { fieldMetadataId: 'files-field-id', fileName: '../../etc/passwd' },
+ );
+
+ const [traversalFile] = mockDirectUploadFile.mock.calls[0];
+ expect(traversalFile.name).toBe('....etcpasswd');
+
+ await result.current.frontComponentHostCommunicationApi.uploadFile(
+ buildRecordedBlob(),
+ { fieldMetadataId: 'files-field-id', fileName: '///' },
+ );
+
+ const [fallbackFile] = mockDirectUploadFile.mock.calls[1];
+ expect(fallbackFile.name).toMatch(/^upload-.*\.webm$/);
+ });
+
+ it('should report upload failures as upload-failed', async () => {
+ mockDirectUploadFile.mockRejectedValue(new Error('network down'));
+
+ const { result } = renderUseFrontComponentExecutionContext({
+ frontComponentId: FRONT_COMPONENT_ID,
+ });
+
+ const uploadResult =
+ await result.current.frontComponentHostCommunicationApi.uploadFile(
+ buildRecordedBlob(),
+ { fieldMetadataId: 'files-field-id' },
+ );
+
+ expect(uploadResult).toEqual({
+ status: 'failed',
+ reason: 'upload-failed',
+ });
+ });
+ });
+
describe('copyToClipboard', () => {
it('should call useCopyToClipboard with the provided text and a preview message', async () => {
const { result } = renderUseFrontComponentExecutionContext({
diff --git a/packages/twenty-front/src/modules/front-components/hooks/useFrontComponentExecutionContext.ts b/packages/twenty-front/src/modules/front-components/hooks/useFrontComponentExecutionContext.ts
index 62bf63e9d09..4cee52a7586 100644
--- a/packages/twenty-front/src/modules/front-components/hooks/useFrontComponentExecutionContext.ts
+++ b/packages/twenty-front/src/modules/front-components/hooks/useFrontComponentExecutionContext.ts
@@ -1,4 +1,5 @@
import { useObjectMetadataItems } from '@/object-metadata/hooks/useObjectMetadataItems';
+import { getFieldMetadataItemById } from '@/object-metadata/utils/getFieldMetadataItemById';
import { resolveOpenRecordIn } from '@/object-record/record-index/utils/resolveOpenRecordIn';
import { useSetAtomState } from '@/ui/utilities/state/jotai/hooks/useSetAtomState';
import { isNonEmptyString } from '@sniptt/guards';
@@ -14,6 +15,7 @@ import {
} from 'twenty-front-component-renderer';
import {
AppPath,
+ FieldMetadataType,
ObjectOpenRecordIn,
OpenRecordIn,
SidePanelPages,
@@ -28,6 +30,8 @@ import { useUnmountCommand } from '@/command-menu-item/engine-command/hooks/useU
import { commandMenuItemProgressFamilyState } from '@/command-menu-item/states/commandMenuItemProgressFamilyState';
import { MAIN_CONTEXT_STORE_INSTANCE_ID } from '@/context-store/constants/MainContextStoreInstanceId';
import { contextStoreRecordShowParentViewComponentState } from '@/context-store/states/contextStoreRecordShowParentViewComponentState';
+import { useDirectFileUpload } from '@/file/hooks/useDirectFileUpload';
+import { getMediaFileExtension } from '@/front-components/media-session/utils/getMediaFileExtension';
import { useRequestApplicationTokenRefresh } from '@/front-components/hooks/useRequestApplicationTokenRefresh';
import { useNavigateSidePanel } from '@/side-panel/hooks/useNavigateSidePanel';
import { useOpenComposeEmailInSidePanel } from '@/side-panel/hooks/useOpenComposeEmailInSidePanel';
@@ -46,11 +50,29 @@ import { useIcons } from 'twenty-ui/icon';
import { useIsMobile } from 'twenty-ui/utilities';
import { useCopyToClipboard } from '~/hooks/useCopyToClipboard';
import { useNavigateApp } from '~/hooks/useNavigateApp';
+import { FileFolder } from '~/generated-metadata/graphql';
const FRONT_COMPONENT_CLIPBOARD_MAX_LENGTH = 64 * 1024;
const FRONT_COMPONENT_CLIPBOARD_RATE_LIMIT_MS = 1000;
const FRONT_COMPONENT_CLIPBOARD_PREVIEW_LENGTH = 30;
+const FRONT_COMPONENT_UPLOAD_FILE_NAME_MAX_LENGTH = 200;
+
+const sanitizeUploadFileName = (fileName: string, mimeType: string): string => {
+ const withoutSeparators = fileName.replace(/[/\\\u0000-\u001f]/g, '').trim();
+
+ if (withoutSeparators === '') {
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+
+ return `upload-${timestamp}.${getMediaFileExtension(mimeType)}`;
+ }
+
+ return withoutSeparators.slice(
+ 0,
+ FRONT_COMPONENT_UPLOAD_FILE_NAME_MAX_LENGTH,
+ );
+};
+
export const useFrontComponentExecutionContext = ({
frontComponentId,
applicationId,
@@ -95,6 +117,7 @@ export const useFrontComponentExecutionContext = ({
} = useSnackBar();
const { closeSidePanelMenu } = useSidePanelMenu();
const { copyToClipboard: copyToClipboardWithSnackbar } = useCopyToClipboard();
+ const { uploadFile: uploadFileToFilesField } = useDirectFileUpload();
const { t, i18n } = useLingui();
// oxlint-disable-next-line twenty/no-state-useref
const lastCopyToClipboardCallAtRef = useRef<number>(Number.NEGATIVE_INFINITY);
@@ -368,6 +391,60 @@ export const useFrontComponentExecutionContext = ({
);
};
+ const hostUploadFile: FrontComponentHostCommunicationApi['uploadFile'] =
+ async (file, params) => {
+ // Arguments come from sandboxed application code: reject malformed
+ // shapes here. fieldMetadataId is mandatory — a file uploaded outside
+ // a FILES field could never be attached to a record and would leak.
+ if (
+ !(file instanceof Blob) ||
+ file.size === 0 ||
+ !isDefined(params) ||
+ !isNonEmptyString(params.fieldMetadataId)
+ ) {
+ return { status: 'failed', reason: 'invalid-params' };
+ }
+
+ // A non-FILES target would upload fine and then fail at attach time,
+ // stranding the file; reject it before uploading anything.
+ const { fieldMetadataItem } = getFieldMetadataItemById({
+ fieldMetadataId: params.fieldMetadataId,
+ objectMetadataItems,
+ });
+
+ if (fieldMetadataItem?.type !== FieldMetadataType.FILES) {
+ return { status: 'failed', reason: 'invalid-params' };
+ }
+
+ const fileName = sanitizeUploadFileName(
+ isNonEmptyString(params.fileName) ? params.fileName : '',
+ file.type,
+ );
+
+ try {
+ const uploadedFile = await uploadFileToFilesField(
+ new File([file], fileName, { type: file.type }),
+ {
+ fileFolder: FileFolder.FilesField,
+ fieldMetadataId: params.fieldMetadataId,
+ },
+ );
+
+ return {
+ status: 'uploaded',
+ file: {
+ fileId: uploadedFile.id,
+ path: uploadedFile.path,
+ url: uploadedFile.url,
+ size: uploadedFile.size,
+ mimeType: file.type.split(';')[0],
+ },
+ };
+ } catch {
+ return { status: 'failed', reason: 'upload-failed' };
+ }
+ };
+
const currentUserId = currentUser?.id;
const storageNamespace = isDefined(currentUserId)
@@ -429,6 +506,7 @@ export const useFrontComponentExecutionContext = ({
closeSidePanel,
updateProgress,
copyToClipboard,
+ uploadFile: hostUploadFile,
storageSet,
storageDelete,
storageClear,
diff --git a/packages/twenty-front/src/modules/front-components/media-session/hooks/useFrontComponentMediaSession.ts b/packages/twenty-front/src/modules/front-components/media-session/hooks/useFrontComponentMediaSession.ts
new file mode 100644
index 00000000000..4e4f2264cdd
--- /dev/null
+++ b/packages/twenty-front/src/modules/front-components/media-session/hooks/useFrontComponentMediaSession.ts
@@ -0,0 +1,17 @@
+import { useState } from 'react';
+import {
+ createFrontComponentMediaSessionHost,
+ type FrontComponentMediaSessionHost,
+} from 'twenty-front-component-renderer';
+
+// One capture session host per rendered front component. The host itself
+// enforces the one-live-capture-at-a-time policy across the page.
+export const useFrontComponentMediaSession = (): {
+ mediaSessionHost: FrontComponentMediaSessionHost;
+} => {
+ const [mediaSessionHost] = useState(() =>
+ createFrontComponentMediaSessionHost(),
+ );
+
+ return { mediaSessionHost };
+};
diff --git a/packages/twenty-front/src/modules/front-components/media-session/utils/getMediaFileExtension.ts b/packages/twenty-front/src/modules/front-components/media-session/utils/getMediaFileExtension.ts
new file mode 100644
index 00000000000..03f1293cfcd
--- /dev/null
+++ b/packages/twenty-front/src/modules/front-components/media-session/utils/getMediaFileExtension.ts
@@ -0,0 +1,15 @@
+const MIME_TYPE_TO_FILE_EXTENSION: Record<string, string> = {
+ 'audio/webm': 'webm',
+ 'video/webm': 'webm',
+ 'audio/mp4': 'm4a',
+ 'video/mp4': 'mp4',
+ 'audio/ogg': 'ogg',
+ 'video/ogg': 'ogg',
+ 'audio/opus': 'ogg',
+};
+
+export const getMediaFileExtension = (mimeType: string): string => {
+ const baseMimeType = mimeType.split(';')[0].trim().toLowerCase();
+
+ return MIME_TYPE_TO_FILE_EXTENSION[baseMimeType] ?? 'webm';
+};
diff --git a/packages/twenty-sdk/src/sdk/front-component/functions/uploadFile.ts b/packages/twenty-sdk/src/sdk/front-component/functions/uploadFile.ts
new file mode 100644
index 00000000000..00ddae1a002
--- /dev/null
+++ b/packages/twenty-sdk/src/sdk/front-component/functions/uploadFile.ts
@@ -0,0 +1,16 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import {
+ type UploadFileFunction,
+ frontComponentHostCommunicationApi,
+} from '../globals/frontComponentHostCommunicationApi';
+
+export const uploadFile: UploadFileFunction = (file, params) => {
+ const uploadFileFunction = frontComponentHostCommunicationApi.uploadFile;
+
+ if (!isDefined(uploadFileFunction)) {
+ throw new Error('uploadFileFunction is not set');
+ }
+
+ return uploadFileFunction(file, params);
+};
diff --git a/packages/twenty-sdk/src/sdk/front-component/globals/frontComponentHostCommunicationApi.ts b/packages/twenty-sdk/src/sdk/front-component/globals/frontComponentHostCommunicationApi.ts
index 6716aa0fec4..1abdcc5f4d1 100644
--- a/packages/twenty-sdk/src/sdk/front-component/globals/frontComponentHostCommunicationApi.ts
+++ b/packages/twenty-sdk/src/sdk/front-component/globals/frontComponentHostCommunicationApi.ts
@@ -102,6 +102,32 @@ export type RequestAccessTokenRefreshFunction = () => Promise<string>;
export type CopyToClipboardFunction = (text: string) => Promise<void>;
+export type UploadedFrontComponentFile = {
+ fileId: string;
+ path: string;
+ url: string;
+ size: number;
+ mimeType: string;
+};
+
+export type UploadFileParams = {
+ // Uploads target a FILES field so the file can be attached to a record;
+ // anything else would strand it as a temporary orphan.
+ fieldMetadataId: string;
+ fileName?: string;
+};
+
+export type UploadFileFailureReason = 'invalid-params' | 'upload-failed';
+
+export type UploadFileResult =
+ | { status: 'uploaded'; file: UploadedFrontComponentFile }
+ | { status: 'failed'; reason: UploadFileFailureReason };
+
+export type UploadFileFunction = (
+ file: Blob,
+ params: UploadFileParams,
+) => Promise<UploadFileResult>;
+
export type OpenCommandConfirmationModalHostFunction = (
params: Parameters<OpenCommandConfirmationModalFunction>[0],
) => Promise<void>;
@@ -131,6 +157,7 @@ export type FrontComponentHostCommunicationApiStore = {
closeSidePanel?: CloseSidePanelFunction;
updateProgress?: UpdateProgressFunction;
copyToClipboard?: CopyToClipboardFunction;
+ uploadFile?: UploadFileFunction;
storageSet?: StorageSetFunction;
storageDelete?: StorageDeleteFunction;
storageClear?: StorageClearFunction;
diff --git a/packages/twenty-sdk/src/sdk/front-component/index.ts b/packages/twenty-sdk/src/sdk/front-component/index.ts
index b50371b145b..c51bb159e77 100644
--- a/packages/twenty-sdk/src/sdk/front-component/index.ts
+++ b/packages/twenty-sdk/src/sdk/front-component/index.ts
@@ -9,6 +9,7 @@ export { openCommandConfirmationModal } from './functions/openCommandConfirmatio
export { openSidePanelPage } from './functions/openSidePanelPage';
export { unmountFrontComponent } from './functions/unmountFrontComponent';
export { updateProgress } from './functions/updateProgress';
+export { uploadFile } from './functions/uploadFile';
export { useColorScheme } from './hooks/useColorScheme';
export { useFrontComponentExecutionContext } from './hooks/useFrontComponentExecutionContext';
export { useFrontComponentId } from './hooks/useFrontComponentId';
@@ -44,6 +45,11 @@ export type {
StorageSetFunction,
UnmountFrontComponentFunction,
UpdateProgressFunction,
+ UploadedFrontComponentFile,
+ UploadFileFailureReason,
+ UploadFileFunction,
+ UploadFileParams,
+ UploadFileResult,
} from './globals/frontComponentHostCommunicationApi';
export {