There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(native): media capture (#9992)
Brooooooklyn5dbffba08dd6
diff --git a/Cargo.lock b/Cargo.lockindex 37a83a82e3..e440e0ebd2 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -28,6 +28,25 @@ dependencies = ["sha3",]+[[package]]+name = "affine_media_capture"+version = "0.0.0"+dependencies = [+ "block2",+ "core-foundation",+ "coreaudio-rs",+ "dispatch2",+ "napi",+ "napi-build",+ "napi-derive",+ "objc2",+ "objc2-foundation",+ "rubato",+ "screencapturekit",+ "thiserror 2.0.11",+ "uuid",+]+[[package]]name = "affine_mobile_native"version = "0.0.0"@@ -50,6 +69,7 @@ name = "affine_native"version = "0.0.0"dependencies = ["affine_common",+ "affine_media_capture","affine_nbstore","affine_sqlite_v1","napi",@@ -325,6 +345,24 @@ dependencies = ["serde",]+[[package]]+name = "bindgen"+version = "0.70.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f"+dependencies = [+ "bitflags 2.8.0",+ "cexpr",+ "clang-sys",+ "itertools",+ "proc-macro2",+ "quote",+ "regex",+ "rustc-hash 1.1.0",+ "shlex",+ "syn",+]+[[package]]name = "bit-set"version = "0.5.3"@@ -340,6 +378,12 @@ version = "0.6.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"+[[package]]+name = "bitflags"+version = "1.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"+[[package]]name = "bitflags"version = "2.8.0"@@ -470,6 +514,15 @@ dependencies = ["shlex",]+[[package]]+name = "cexpr"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"+dependencies = [+ "nom",+]+[[package]]name = "cfg-if"version = "1.0.0"@@ -482,6 +535,15 @@ version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"+[[package]]+name = "cgl"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff"+dependencies = [+ "libc",+]+[[package]]name = "chrono"version = "0.4.39"@@ -523,6 +585,17 @@ dependencies = ["half",]+[[package]]+name = "clang-sys"+version = "1.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"+dependencies = [+ "glob",+ "libc",+ "libloading",+]+[[package]]name = "clap"version = "4.5.28"@@ -594,12 +667,112 @@ dependencies = ["unicode-segmentation",]+[[package]]+name = "core-audio-types-rs"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "02f7359c779907f80443d2b2d1b5a61182abb6d8ffd43b6fcb87a27c327d845f"+dependencies = [+ "core-foundation",+]++[[package]]+name = "core-foundation"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63"+dependencies = [+ "core-foundation-sys",+ "libc",+ "uuid",+]+[[package]]name = "core-foundation-sys"version = "0.8.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"+[[package]]+name = "core-graphics"+version = "0.24.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1"+dependencies = [+ "bitflags 2.8.0",+ "core-foundation",+ "core-graphics-types",+ "foreign-types",+ "libc",+]++[[package]]+name = "core-graphics-types"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb"+dependencies = [+ "bitflags 2.8.0",+ "core-foundation",+ "libc",+]++[[package]]+name = "core-media-rs"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3e33a8804301de5fc0f705ea1cdea692233c08bdcee8e42abb258f5de3b9a5e7"+dependencies = [+ "core-audio-types-rs",+ "core-foundation",+ "core-utils-rs",+ "core-video-rs",+ "thiserror 2.0.11",+]++[[package]]+name = "core-utils-rs"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "068ec1aa07335261033bf610b2868ea9db05353468b72b0045ae469e00d26121"+dependencies = [+ "core-foundation",+ "four-char-code",+]++[[package]]+name = "core-video-rs"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7559e93f816c05607068cb0b741d997a47709f4ba70b36a02d335f7136b30c46"+dependencies = [+ "core-foundation",+ "core-graphics",+ "core-utils-rs",+ "io-surface",+ "thiserror 2.0.11",+]++[[package]]+name = "coreaudio-rs"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "34ca07354f6d0640333ef95f48d460a4bcf34812a7e7967f9b44c728a8f37c28"+dependencies = [+ "bitflags 1.3.2",+ "core-foundation-sys",+ "coreaudio-sys",+]++[[package]]+name = "coreaudio-sys"+version = "0.2.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2ce857aa0b77d77287acc1ac3e37a05a8c95a2af3647d23b15f263bdaeb7562b"+dependencies = [+ "bindgen",+]+[[package]]name = "cpufeatures"version = "0.2.17"@@ -745,6 +918,24 @@ dependencies = ["subtle",]+[[package]]+name = "dispatch"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b"++[[package]]+name = "dispatch2"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1a0d569e003ff27784e0e14e4a594048698e0c0f0b66cabcb51511be55a7caa0"+dependencies = [+ "bitflags 2.8.0",+ "block2",+ "libc",+ "objc2",+]+[[package]]name = "displaydoc"version = "0.2.5"@@ -860,6 +1051,33 @@ version = "0.1.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"+[[package]]+name = "foreign-types"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"+dependencies = [+ "foreign-types-macros",+ "foreign-types-shared",+]++[[package]]+name = "foreign-types-macros"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "foreign-types-shared"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"+[[package]]name = "form_urlencoded"version = "1.2.1"@@ -869,6 +1087,12 @@ dependencies = ["percent-encoding",]+[[package]]+name = "four-char-code"+version = "2.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c661315fd366b2a1f970df7b7cb1a28d2678d49ef4872f7dcc19b4a83150f20b"+[[package]]name = "fs-err"version = "2.11.0"@@ -1288,12 +1512,34 @@ dependencies = ["hashbrown 0.15.2",]+[[package]]+name = "io-surface"+version = "0.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8283575d5f0b2e7447ec0840363879d71c0fa325d4c699d5b45208ea4a51f45e"+dependencies = [+ "cgl",+ "core-foundation",+ "core-foundation-sys",+ "leaky-cow",+ "libc",+]+[[package]]name = "is_terminal_polyfill"version = "1.70.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"+[[package]]+name = "itertools"+version = "0.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "1.0.14"@@ -1338,6 +1584,21 @@ dependencies = ["spin",]+[[package]]+name = "leak"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bd100e01f1154f2908dfa7d02219aeab25d0b9c7fa955164192e3245255a0c73"++[[package]]+name = "leaky-cow"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "40a8225d44241fd324a8af2806ba635fc7c8a7e9a7de4d5cf3ef54e71f5926fc"+dependencies = [+ "leak",+]+[[package]]name = "libc"version = "0.2.169"@@ -1424,6 +1685,15 @@ dependencies = ["tracing-subscriber",]+[[package]]+name = "malloc_buf"+version = "0.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"+dependencies = [+ "libc",+]+[[package]]name = "matchers"version = "0.1.0"@@ -1516,7 +1786,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0dd957e2cc4bd62b730b10ff1f35775f8a81dac84a3bfac273b0ec4336f53ab8"dependencies = ["anyhow",- "bitflags",+ "bitflags 2.8.0","chrono","ctor","napi-build",@@ -1572,7 +1842,7 @@ version = "0.29.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"dependencies = [- "bitflags",+ "bitflags 2.8.0","cfg-if","cfg_aliases","libc",@@ -1615,6 +1885,15 @@ dependencies = ["zeroize",]+[[package]]+name = "num-complex"+version = "0.4.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"+dependencies = [+ "num-traits",+]+[[package]]name = "num-integer"version = "0.1.46"@@ -1645,6 +1924,16 @@ dependencies = ["libm",]+[[package]]+name = "objc"+version = "0.2.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1"+dependencies = [+ "malloc_buf",+ "objc_exception",+]+[[package]]name = "objc2"version = "0.6.0"@@ -1660,7 +1949,7 @@ version = "0.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "daeaf60f25471d26948a1c2f840e3f7d86f4109e3af4e8e4b5cd70c39690d925"dependencies = [- "bitflags",+ "bitflags 2.8.0","objc2",]@@ -1676,13 +1965,22 @@ version = "0.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3a21c6c9014b82c39515db5b396f91645182611c97d24637cf56ac01e5f8d998"dependencies = [- "bitflags",+ "bitflags 2.8.0","block2","libc","objc2","objc2-core-foundation",]+[[package]]+name = "objc_exception"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4"+dependencies = [+ "cc",+]+[[package]]name = "object"version = "0.36.7"@@ -1833,6 +2131,15 @@ dependencies = ["zerocopy 0.7.35",]+[[package]]+name = "primal-check"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"+dependencies = [+ "num-integer",+]+[[package]]name = "proc-macro2"version = "1.0.93"@@ -1948,13 +2255,22 @@ dependencies = ["crossbeam-utils",]+[[package]]+name = "realfft"+version = "3.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "390252372b7f2aac8360fc5e72eba10136b166d6faeed97e6d0c8324eb99b2b1"+dependencies = [+ "rustfft",+]+[[package]]name = "redox_syscall"version = "0.5.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"dependencies = [- "bitflags",+ "bitflags 2.8.0",][[package]]@@ -2075,6 +2391,18 @@ dependencies = ["zeroize",]+[[package]]+name = "rubato"+version = "0.16.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cdd96992d7e24b3d7f35fdfe02af037a356ac90d41b466945cf3333525a86eea"+dependencies = [+ "num-complex",+ "num-integer",+ "num-traits",+ "realfft",+]+[[package]]name = "rustc-demangle"version = "0.1.24"@@ -2093,13 +2421,28 @@ version = "2.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"+[[package]]+name = "rustfft"+version = "6.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86"+dependencies = [+ "num-complex",+ "num-integer",+ "num-traits",+ "primal-check",+ "strength_reduce",+ "transpose",+ "version_check",+]+[[package]]name = "rustix"version = "0.38.44"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"dependencies = [- "bitflags",+ "bitflags 2.8.0","errno","libc","linux-raw-sys",@@ -2179,6 +2522,22 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"+[[package]]+name = "screencapturekit"+version = "0.3.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd797e2f07759d67fc00c3085c55dae07c37e73dba29dc1159484cbb2946bd6f"+dependencies = [+ "block2",+ "core-foundation",+ "core-graphics",+ "core-media-rs",+ "core-utils-rs",+ "core-video-rs",+ "dispatch",+ "objc",+]+[[package]]name = "scroll"version = "0.12.0"@@ -2484,7 +2843,7 @@ checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233"dependencies = ["atoi","base64 0.22.1",- "bitflags",+ "bitflags 2.8.0","byteorder","bytes","chrono",@@ -2527,7 +2886,7 @@ checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613"dependencies = ["atoi","base64 0.22.1",- "bitflags",+ "bitflags 2.8.0","byteorder","chrono","crc",@@ -2593,6 +2952,12 @@ version = "1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"+[[package]]+name = "strength_reduce"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"+[[package]]name = "stringprep"version = "0.1.5"@@ -2869,6 +3234,16 @@ dependencies = ["tracing-log",]+[[package]]+name = "transpose"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"+dependencies = [+ "num-integer",+ "strength_reduce",+]+[[package]]name = "typenum"version = "1.17.0"@@ -3063,6 +3438,15 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"+[[package]]+name = "uuid"+version = "1.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0"+dependencies = [+ "getrandom 0.3.1",+]+[[package]]name = "v_htmlescape"version = "0.15.8"@@ -3527,7 +3911,7 @@ version = "0.33.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"dependencies = [- "bitflags",+ "bitflags 2.8.0",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex 66762ae2c0..0c888d98ab 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -15,8 +15,12 @@ affine_common = { path = "./packages/common/native" }affine_nbstore = { path = "./packages/frontend/native/nbstore" }anyhow = "1"base64-simd = "0.8"+block2 = "0.6"chrono = "0.4"+core-foundation = "0.10"+coreaudio-rs = "0.12"criterion2 = { version = "2", default-features = false }+dispatch2 = "0.2"dotenvy = "0.15"file-format = { version = "0.26", features = ["reader"] }homedir = "0.3"@@ -31,6 +35,8 @@ once_cell = "1"parking_lot = "0.12"rand = "0.9"rayon = "1.10"+rubato = "0.16"+screencapturekit = "0.3"serde = "1"serde_json = "1"sha3 = "0.10"diff --git a/packages/frontend/media-capture-playground/.gitignore b/packages/frontend/media-capture-playground/.gitignorenew file mode 100644index 0000000000..4314969635--- /dev/null+++ b/packages/frontend/media-capture-playground/.gitignore@@ -0,0 +1,2 @@+recordings+.env\ No newline at end of filediff --git a/packages/frontend/media-capture-playground/package.json b/packages/frontend/media-capture-playground/package.jsonnew file mode 100644index 0000000000..6b6b331be2--- /dev/null+++ b/packages/frontend/media-capture-playground/package.json@@ -0,0 +1,43 @@+{+ "name": "@affine/media-capture-playground",+ "private": true,+ "type": "module",+ "version": "0.0.0",+ "scripts": {+ "dev:web": "vite",+ "dev:server": "tsx --env-file=.env --watch server/main.ts"+ },+ "dependencies": {+ "@affine/native": "workspace:*",+ "@google/generative-ai": "^0.21.0",+ "@tailwindcss/vite": "^4.0.6",+ "@types/express": "^4",+ "@types/multer": "^1",+ "@types/react": "^19.0.8",+ "@types/react-dom": "^19.0.3",+ "@types/socket.io": "^3.0.2",+ "@types/socket.io-client": "^3.0.0",+ "@vitejs/plugin-react": "^4.3.4",+ "chokidar": "^4.0.3",+ "express": "^4.21.2",+ "express-rate-limit": "^7.1.5",+ "fs-extra": "^11.3.0",+ "multer": "^1.4.5-lts.1",+ "openai": "^4.85.1",+ "react": "^19.0.0",+ "react-dom": "^19.0.0",+ "react-markdown": "^9.0.3",+ "rxjs": "^7.8.1",+ "socket.io": "^4.7.4",+ "socket.io-client": "^4.7.4",+ "swr": "^2.3.2",+ "tailwindcss": "^4.0.6",+ "tsx": "^4.19.2",+ "vite": "^6.1.0"+ },+ "devDependencies": {+ "@types/fs-extra": "^11",+ "@types/react": "^19.0.1",+ "@types/react-dom": "^19.0.2"+ }+}diff --git a/packages/frontend/media-capture-playground/server/gemini.ts b/packages/frontend/media-capture-playground/server/gemini.tsnew file mode 100644index 0000000000..723a5e742c--- /dev/null+++ b/packages/frontend/media-capture-playground/server/gemini.ts@@ -0,0 +1,200 @@+import { GoogleGenerativeAI } from '@google/generative-ai';+import {+ GoogleAIFileManager,+ type UploadFileResponse,+} from '@google/generative-ai/server';++const DEFAULT_MODEL = 'gemini-2.0-flash';++export interface TranscriptionResult {+ title: string;+ summary: string;+ segments: {+ speaker: string;+ start_time: string;+ end_time: string;+ transcription: string;+ }[];+}++const PROMPT_TRANSCRIPTION = `+Generate audio transcription and diarization for the recording.+The recording source is most likely from a video call with multiple speakers.+Output in JSON format with the following structure:++{+ "segments": [+ {+ "speaker": "Speaker A",+ "start_time": "MM:SS",+ "end_time": "MM:SS",+ "transcription": "..."+ },+ ...+ ],+}+- Use consistent speaker labels throughout+- Accurate timestamps in MM:SS format+- Clean transcription with proper punctuation+- Identify speakers by name if possible, otherwise use "Speaker A/B/C"+`;++const PROMPT_SUMMARY = `+Generate a short title and summary of the conversation. The input is in the following JSON format:+{+ "segments": [+ {+ "speaker": "Speaker A",+ "start_time": "MM:SS",+ "end_time": "MM:SS",+ "transcription": "..."+ },+ ...+ ],+}++Output in JSON format with the following structure:++{+ "title": "Title of the recording",+ "summary": "Summary of the conversation in markdown format"+}++1. Summary Structure:+- The sumary should be inferred from the speakers' language and context+- All insights should be derived directly from speakers' language and context+- Use hierarchical organization for clear information structure+- Use markdown format for the summary. Use bullet points, lists and other markdown styles when appropriate++2. Title:+- Come up with a title for the recording.+- The title should be a short description of the recording.+- The title should be a single sentence or a few words.+`;++export async function gemini(+ audioFilePath: string,+ options?: {+ model?: 'gemini-2.0-flash' | 'gemini-1.5-flash';+ mode?: 'transcript' | 'summary';+ }+) {+ if (!process.env.GOOGLE_GEMINI_API_KEY) {+ console.error('Missing GOOGLE_GEMINI_API_KEY environment variable');+ throw new Error('GOOGLE_GEMINI_API_KEY is not set');+ }++ // Initialize GoogleGenerativeAI and FileManager with your API_KEY+ const genAI = new GoogleGenerativeAI(process.env.GOOGLE_GEMINI_API_KEY);+ const fileManager = new GoogleAIFileManager(+ process.env.GOOGLE_GEMINI_API_KEY+ );++ async function transcribe(+ audioFilePath: string+ ): Promise<TranscriptionResult | null> {+ let uploadResult: UploadFileResponse | null = null;++ try {+ // Upload the audio file+ uploadResult = await fileManager.uploadFile(audioFilePath, {+ mimeType: 'audio/wav',+ displayName: 'audio_transcription.wav',+ });+ console.log('File uploaded:', uploadResult.file.uri);++ // Initialize a Gemini model appropriate for your use case.+ const model = genAI.getGenerativeModel({+ model: options?.model || DEFAULT_MODEL,+ generationConfig: {+ responseMimeType: 'application/json',+ },+ });++ // Generate content using a prompt and the uploaded file+ const result = await model.generateContent([+ {+ fileData: {+ fileUri: uploadResult.file.uri,+ mimeType: uploadResult.file.mimeType,+ },+ },+ {+ text: PROMPT_TRANSCRIPTION,+ },+ ]);++ const text = result.response.text();++ try {+ const parsed = JSON.parse(text);+ return parsed;+ } catch (e) {+ console.error('Failed to parse transcription JSON:', e);+ console.error('Raw text that failed to parse:', text);+ return null;+ }+ } catch (e) {+ console.error('Error during transcription:', e);+ return null;+ } finally {+ if (uploadResult) {+ await fileManager.deleteFile(uploadResult.file.name);+ }+ }+ }++ async function summarize(transcription: TranscriptionResult) {+ try {+ const model = genAI.getGenerativeModel({+ model: options?.model || DEFAULT_MODEL,+ generationConfig: {+ responseMimeType: 'application/json',+ },+ });++ const result = await model.generateContent([+ {+ text: PROMPT_SUMMARY + '\n\n' + JSON.stringify(transcription),+ },+ ]);++ const text = result.response.text();++ try {+ const parsed = JSON.parse(text);+ return parsed;+ } catch (e) {+ console.error('Failed to parse summary JSON:', e);+ console.error('Raw text that failed to parse:', text);+ return null;+ }+ } catch (e) {+ console.error('Error during summarization:', e);+ return null;+ }+ }++ const transcription = await transcribe(audioFilePath);+ if (!transcription) {+ console.error('Transcription failed');+ return null;+ }++ const summary = await summarize(transcription);+ if (!summary) {+ console.error('Summary generation failed');+ return transcription;+ }++ const result = {+ ...transcription,+ ...summary,+ };+ console.log('Processing completed:', {+ title: result.title,+ segmentsCount: result.segments?.length,+ });++ return result;+}diff --git a/packages/frontend/media-capture-playground/server/main.ts b/packages/frontend/media-capture-playground/server/main.tsnew file mode 100644index 0000000000..e132447d63--- /dev/null+++ b/packages/frontend/media-capture-playground/server/main.ts@@ -0,0 +1,759 @@+/* eslint-disable @typescript-eslint/no-misused-promises */+import { exec } from 'node:child_process';+import { createServer } from 'node:http';+import { promisify } from 'node:util';++import {+ type Application,+ type AudioTapStream,+ ShareableContent,+} from '@affine/native';+import type { FSWatcher } from 'chokidar';+import chokidar from 'chokidar';+import express from 'express';+import rateLimit from 'express-rate-limit';+import fs from 'fs-extra';+import { Server } from 'socket.io';++import { gemini, type TranscriptionResult } from './gemini';+import { WavWriter } from './wav-writer';++// Constants+const RECORDING_DIR = './recordings';+const PORT = process.env.PORT || 6544;++// Ensure recordings directory exists+fs.ensureDirSync(RECORDING_DIR);+console.log(`📁 Ensuring recordings directory exists at ${RECORDING_DIR}`);++// Types+interface Recording {+ app: Application;+ appGroup: Application | null;+ buffers: Float32Array[];+ stream: AudioTapStream;+ startTime: number;+ isWriting: boolean;+}++interface RecordingStatus {+ processId: number;+ bundleIdentifier: string;+ name: string;+ startTime: number;+ duration: number;+}++interface RecordingMetadata {+ appName: string;+ bundleIdentifier: string;+ processId: number;+ recordingStartTime: number;+ recordingEndTime: number;+ recordingDuration: number;+ sampleRate: number;+ totalSamples: number;+}++interface AppInfo {+ app: Application;+ processId: number;+ processGroupId: number | null;+ bundleIdentifier: string;+ name: string;+ running: boolean;+}++interface TranscriptionMetadata {+ transcriptionStartTime: number;+ transcriptionEndTime: number;+ transcriptionStatus: 'not_started' | 'pending' | 'completed' | 'error';+ transcription?: TranscriptionResult;+ error?: string;+}++// State+const recordingMap = new Map<number, Recording>();+let appsSubscriber = () => {};+let fsWatcher: FSWatcher | null = null;++// Server setup+const app = express();+const httpServer = createServer(app);+const io = new Server(httpServer, {+ cors: { origin: '*' },+});++app.use(express.json());++// Update the static file serving to handle the new folder structure+app.use(+ '/recordings',+ (req, res, next) => {+ // Extract the folder name from the path+ const parts = req.path.split('/');+ if (parts.length < 2) {+ return res.status(400).json({ error: 'Invalid request path' });+ }++ const folderName = parts[1];+ if (!validateAndSanitizeFolderName(folderName)) {+ return res.status(400).json({ error: 'Invalid folder name format' });+ }++ if (req.path.endsWith('.wav')) {+ res.setHeader('Content-Type', 'audio/wav');+ } else if (req.path.endsWith('.png')) {+ res.setHeader('Content-Type', 'image/png');+ }+ next();+ },+ express.static(RECORDING_DIR)+);++// Recording management+async function saveRecording(recording: Recording): Promise<string | null> {+ try {+ recording.isWriting = true;+ const app = recording.appGroup || recording.app;++ const totalSamples = recording.buffers.reduce(+ (acc, buf) => acc + buf.length,+ 0+ );++ const recordingEndTime = Date.now();+ const recordingDuration = (recordingEndTime - recording.startTime) / 1000;+ const expectedSamples = recordingDuration * 44100;++ console.log(`💾 Saving recording for ${app.name}:`);+ console.log(`- Process ID: ${app.processId}`);+ console.log(`- Bundle ID: ${app.bundleIdentifier}`);+ console.log(`- Actual duration: ${recordingDuration.toFixed(2)}s`);+ console.log(`- Expected samples: ${Math.floor(expectedSamples)}`);+ console.log(`- Actual samples: ${totalSamples}`);+ console.log(+ `- Sample ratio: ${(totalSamples / expectedSamples).toFixed(2)}`+ );++ // Create a buffer for the mono audio+ const buffer = new Float32Array(totalSamples);+ let offset = 0;+ recording.buffers.forEach(buf => {+ buffer.set(buf, offset);+ offset += buf.length;+ });++ await fs.ensureDir(RECORDING_DIR);++ const timestamp = Date.now();+ const baseFilename = `${recording.app.bundleIdentifier}-${recording.app.processId}-${timestamp}`;+ const recordingDir = `${RECORDING_DIR}/${baseFilename}`;+ await fs.ensureDir(recordingDir);++ const wavFilename = `${recordingDir}/recording.wav`;+ const transcriptionWavFilename = `${recordingDir}/transcription.wav`;+ const metadataFilename = `${recordingDir}/metadata.json`;+ const iconFilename = `${recordingDir}/icon.png`;++ // Save high-quality WAV file for playback (44.1kHz)+ console.log(`📝 Writing high-quality WAV file to ${wavFilename}`);+ const writer = new WavWriter(wavFilename, { targetSampleRate: 44100 });+ writer.write(buffer);+ await writer.end();+ console.log('✅ High-quality WAV file written successfully');++ // Save low-quality WAV file for transcription (8kHz)+ console.log(+ `📝 Writing transcription WAV file to ${transcriptionWavFilename}`+ );+ const transcriptionWriter = new WavWriter(transcriptionWavFilename, {+ targetSampleRate: 8000,+ });+ transcriptionWriter.write(buffer);+ await transcriptionWriter.end();+ console.log('✅ Transcription WAV file written successfully');++ // Save app icon if available+ if (app.icon) {+ console.log(`📝 Writing app icon to ${iconFilename}`);+ await fs.writeFile(iconFilename, app.icon);+ console.log('✅ App icon written successfully');+ }++ console.log(`📝 Writing metadata to ${metadataFilename}`);+ // Save metadata (without icon)+ const metadata: RecordingMetadata = {+ appName: app.name,+ bundleIdentifier: app.bundleIdentifier,+ processId: app.processId,+ recordingStartTime: recording.startTime,+ recordingEndTime,+ recordingDuration,+ sampleRate: 44100,+ totalSamples,+ };++ await fs.writeJson(metadataFilename, metadata, { spaces: 2 });+ console.log('✅ Metadata file written successfully');++ return baseFilename;+ } catch (error) {+ console.error('❌ Error saving recording:', error);+ return null;+ }+}++function getRecordingStatus(): RecordingStatus[] {+ return Array.from(recordingMap.entries()).map(([processId, recording]) => ({+ processId,+ bundleIdentifier: recording.app.bundleIdentifier,+ name: recording.app.name,+ startTime: recording.startTime,+ duration: Date.now() - recording.startTime,+ }));+}++function emitRecordingStatus() {+ io.emit('apps:recording', { recordings: getRecordingStatus() });+}++async function startRecording(app: Application) {+ if (recordingMap.has(app.processId)) {+ console.log(+ `⚠️ Recording already in progress for ${app.name} (PID: ${app.processId})`+ );+ return;+ }++ // Find the root app of the process group+ const processGroupId = await getProcessGroupId(app.processId);+ const rootApp = processGroupId+ ? (shareableContent+ .applications()+ .find(a => a.processId === processGroupId) ?? app)+ : app;++ console.log(+ `🎙️ Starting recording for ${rootApp.name} (PID: ${rootApp.processId})`+ );++ const buffers: Float32Array[] = [];+ const stream = app.tapAudio((err, samples) => {+ if (err) {+ console.error(`❌ Audio stream error for ${rootApp.name}:`, err);+ return;+ }+ const recording = recordingMap.get(app.processId);+ if (recording && !recording.isWriting) {+ buffers.push(new Float32Array(samples));+ }+ });++ recordingMap.set(app.processId, {+ app,+ appGroup: rootApp,+ buffers,+ stream,+ startTime: Date.now(),+ isWriting: false,+ });++ console.log(`✅ Recording started successfully for ${rootApp.name}`);+ emitRecordingStatus();+}++async function stopRecording(processId: number) {+ const recording = recordingMap.get(processId);+ if (!recording) {+ console.log(`ℹ️ No active recording found for process ID ${processId}`);+ return;+ }++ const app = recording.appGroup || recording.app;++ console.log(`⏹️ Stopping recording for ${app.name} (PID: ${app.processId})`);+ console.log(+ `⏱️ Recording duration: ${((Date.now() - recording.startTime) / 1000).toFixed(2)}s`+ );++ recording.stream.stop();+ const filename = await saveRecording(recording);+ recordingMap.delete(processId);++ if (filename) {+ console.log(`✅ Recording saved successfully to ${filename}`);+ } else {+ console.error(`❌ Failed to save recording for ${app.name}`);+ }++ emitRecordingStatus();+ return filename;+}++// File management+async function getRecordings(): Promise<+ {+ wav: string;+ metadata?: RecordingMetadata;+ transcription?: TranscriptionMetadata;+ }[]+> {+ try {+ const allItems = await fs.readdir(RECORDING_DIR);++ // First filter out non-directories+ const dirs = (+ await Promise.all(+ allItems.map(async item => {+ const fullPath = `${RECORDING_DIR}/${item}`;+ try {+ const stat = await fs.stat(fullPath);+ return stat.isDirectory() ? item : null;+ } catch {+ return null;+ }+ })+ )+ ).filter((d): d is string => d !== null);++ const recordings = await Promise.all(+ dirs.map(async dir => {+ try {+ const recordingPath = `${RECORDING_DIR}/${dir}`;+ const metadataPath = `${recordingPath}/metadata.json`;+ const transcriptionPath = `${recordingPath}/transcription.json`;++ let metadata: RecordingMetadata | undefined;+ try {+ metadata = await fs.readJson(metadataPath);+ } catch {+ // Metadata might not exist+ }++ let transcription: TranscriptionMetadata | undefined;+ try {+ // Check if transcription file exists+ const transcriptionExists = await fs.pathExists(transcriptionPath);+ if (transcriptionExists) {+ transcription = await fs.readJson(transcriptionPath);+ } else {+ // If transcription.wav exists but no transcription.json, it means transcription is available but not started+ transcription = {+ transcriptionStartTime: 0,+ transcriptionEndTime: 0,+ transcriptionStatus: 'not_started',+ };+ }+ } catch (error) {+ console.error(`Error reading transcription for ${dir}:`, error);+ }++ return {+ wav: dir,+ metadata,+ transcription,+ };+ } catch (error) {+ console.error(`Error processing directory ${dir}:`, error);+ return null;+ }+ })+ );++ // Filter out nulls and sort by recording start time+ return recordings+ .filter((r): r is NonNullable<typeof r> => r !== null)+ .sort(+ (a, b) =>+ (b.metadata?.recordingStartTime ?? 0) -+ (a.metadata?.recordingStartTime ?? 0)+ );+ } catch (error) {+ console.error('Error reading recordings directory:', error);+ return [];+ }+}++async function setupRecordingsWatcher() {+ if (fsWatcher) {+ console.log('🔄 Closing existing recordings watcher');+ await fsWatcher.close();+ }++ try {+ console.log('👀 Setting up recordings watcher...');+ const files = await getRecordings();+ console.log(`📊 Found ${files.length} existing recordings`);+ io.emit('apps:saved', { recordings: files });++ fsWatcher = chokidar.watch(RECORDING_DIR, {+ ignored: /(^|[/\\])\../, // ignore dotfiles+ persistent: true,+ ignoreInitial: true,+ awaitWriteFinish: {+ stabilityThreshold: 500,+ pollInterval: 100,+ },+ });++ // Handle file events+ fsWatcher+ .on('add', async path => {+ if (path.endsWith('.wav') || path.endsWith('.json')) {+ console.log(`📝 File added: ${path}`);+ const files = await getRecordings();+ io.emit('apps:saved', { recordings: files });+ }+ })+ .on('change', async path => {+ if (path.endsWith('.wav') || path.endsWith('.json')) {+ console.log(`📝 File changed: ${path}`);+ const files = await getRecordings();+ io.emit('apps:saved', { recordings: files });+ }+ })+ .on('unlink', async path => {+ if (path.endsWith('.wav') || path.endsWith('.json')) {+ console.log(`🗑️ File removed: ${path}`);+ const files = await getRecordings();+ io.emit('apps:saved', { recordings: files });+ }+ })+ .on('error', error => {+ console.error('❌ Error watching recordings directory:', error);+ })+ .on('ready', () => {+ console.log('✅ Recordings watcher setup complete');+ });+ } catch (error) {+ console.error('❌ Error setting up recordings watcher:', error);+ }+}++// Process management+async function getProcessGroupId(pid: number): Promise<number | null> {+ try {+ const execAsync = promisify(exec);+ const { stdout } = await execAsync(`ps -o pgid -p ${pid}`);+ const lines = stdout.trim().split('\n');+ if (lines.length < 2) return null;++ const pgid = parseInt(lines[1].trim(), 10);+ return isNaN(pgid) ? null : pgid;+ } catch {+ return null;+ }+}++// Application management+const shareableContent = new ShareableContent();++async function getAllApps(): Promise<AppInfo[]> {+ const apps = await Promise.all(+ shareableContent.applications().map(async app => {+ try {+ return {+ app,+ processId: app.processId,+ processGroupId: await getProcessGroupId(app.processId),+ bundleIdentifier: app.bundleIdentifier,+ name: app.name,+ running: app.isRunning,+ };+ } catch (error) {+ console.error(error);+ return null;+ }+ })+ );++ const filteredApps = apps.filter(+ (v): v is AppInfo =>+ v !== null && !v.bundleIdentifier.startsWith('com.apple')+ );++ // Stop recording if app is not listed+ await Promise.all(+ filteredApps.map(async ({ app }) => {+ if (!filteredApps.some(a => a.processId === app.processId)) {+ await stopRecording(app.processId);+ }+ })+ );++ return filteredApps;+}++function listenToAppStateChanges(apps: AppInfo[]) {+ const subscribers = apps.map(({ app }) => {+ return ShareableContent.onAppStateChanged(app, () => {+ setTimeout(() => {+ console.log(+ `🔄 Application state changed: ${app.name} (PID: ${app.processId}) is now ${+ app.isRunning ? '▶️ running' : '⏹️ stopped'+ }`+ );+ io.emit('apps:state-changed', {+ processId: app.processId,+ running: app.isRunning,+ });+ if (!app.isRunning) {+ stopRecording(app.processId).catch(error => {+ console.error('❌ Error stopping recording:', error);+ });+ }+ }, 50);+ });+ });++ appsSubscriber();+ appsSubscriber = () => {+ subscribers.forEach(subscriber => subscriber.unsubscribe());+ };+}++// Socket.IO setup+io.on('connection', async socket => {+ console.log('🔌 New client connected');+ const initialApps = await getAllApps();+ console.log(`📤 Sending ${initialApps.length} applications to new client`);+ socket.emit('apps:all', { apps: initialApps });+ socket.emit('apps:recording', { recordings: getRecordingStatus() });++ const files = await getRecordings();+ console.log(`📤 Sending ${files.length} saved recordings to new client`);+ socket.emit('apps:saved', { recordings: files });++ listenToAppStateChanges(initialApps);++ socket.on('disconnect', () => {+ console.log('🔌 Client disconnected');+ });+});++// Application list change listener+ShareableContent.onApplicationListChanged(() => {+ (async () => {+ try {+ console.log('🔄 Application list changed, updating clients...');+ const apps = await getAllApps();+ console.log(`📢 Broadcasting ${apps.length} applications to all clients`);+ io.emit('apps:all', { apps });+ } catch (error) {+ console.error('❌ Error handling application list change:', error);+ }+ })().catch(error => {+ console.error('❌ Error in application list change handler:', error);+ });+});++// API Routes+const rateLimiter = rateLimit({+ windowMs: 1000,+ max: 200,+ message: { error: 'Too many requests, please try again later.' },+});++app.get('/permissions', (req, res) => {+ const permission = shareableContent.checkRecordingPermissions();+ res.json({ permission });+});++app.get('/apps', async (_req, res) => {+ const apps = await getAllApps();+ listenToAppStateChanges(apps);+ res.json({ apps });+});++app.get('/apps/saved', rateLimiter, async (_req, res) => {+ const files = await getRecordings();+ res.json({ recordings: files });+});++// Utility function to validate and sanitize folder name+function validateAndSanitizeFolderName(folderName: string): string | null {+ // Allow alphanumeric characters, hyphens, dots (for bundle IDs)+ // Format: bundleId-processId-timestamp+ if (!/^[\w.-]+-\d+-\d+$/.test(folderName)) {+ return null;+ }++ // Remove any path traversal attempts+ const sanitized = folderName.replace(/^\.+|\.+$/g, '').replace(/[/\\]/g, '');+ return sanitized;+}++app.delete('/recordings/:foldername', rateLimiter, async (req, res) => {+ const foldername = validateAndSanitizeFolderName(req.params.foldername);+ if (!foldername) {+ console.error('❌ Invalid folder name format:', req.params.foldername);+ return res.status(400).json({ error: 'Invalid folder name format' });+ }++ const recordingDir = `${RECORDING_DIR}/${foldername}`;++ try {+ // Ensure the resolved path is within RECORDING_DIR+ const resolvedPath = await fs.realpath(recordingDir);+ const recordingDirPath = await fs.realpath(RECORDING_DIR);++ if (!resolvedPath.startsWith(recordingDirPath)) {+ console.error('❌ Path traversal attempt detected:', {+ resolvedPath,+ recordingDirPath,+ requestedFile: foldername,+ });+ return res.status(403).json({ error: 'Access denied' });+ }++ console.log(`🗑️ Deleting recording folder: ${foldername}`);+ await fs.remove(recordingDir);+ console.log('✅ Recording folder deleted successfully');+ res.status(200).json({ success: true });+ } catch (error) {+ const typedError = error as NodeJS.ErrnoException;+ if (typedError.code === 'ENOENT') {+ console.error('❌ Folder not found:', recordingDir);+ res.status(404).json({ error: 'Folder not found' });+ } else {+ console.error('❌ Error deleting folder:', {+ error: typedError,+ code: typedError.code,+ message: typedError.message,+ path: recordingDir,+ });+ res.status(500).json({+ error: `Failed to delete folder: ${typedError.message || 'Unknown error'}`,+ });+ }+ }+});++app.get('/apps/:process_id/icon', (req, res) => {+ const processId = parseInt(req.params.process_id);+ try {+ const app = shareableContent.applicationWithProcessId(processId);+ const icon = app.icon;+ res.set('Content-Type', 'image/png');+ res.send(icon);+ } catch {+ res.status(404).json({ error: 'App icon not found' });+ }+});++app.post('/apps/:process_id/record', async (req, res) => {+ const processId = parseInt(req.params.process_id);+ const app = shareableContent.applicationWithProcessId(processId);+ await startRecording(app);+ res.json({ success: true });+});++app.post('/apps/:process_id/stop', async (req, res) => {+ const processId = parseInt(req.params.process_id);+ await stopRecording(processId);+ res.json({ success: true });+});++// Update transcription endpoint to use folder validation+app.post(+ '/recordings/:foldername/transcribe',+ rateLimiter,+ async (req, res) => {+ const foldername = validateAndSanitizeFolderName(req.params.foldername);+ if (!foldername) {+ console.error('❌ Invalid folder name format:', req.params.foldername);+ return res.status(400).json({ error: 'Invalid folder name format' });+ }++ const recordingDir = `${RECORDING_DIR}/${foldername}`;++ try {+ // Check if directory exists+ await fs.access(recordingDir);++ const transcriptionWavPath = `${recordingDir}/transcription.wav`;+ const transcriptionMetadataPath = `${recordingDir}/transcription.json`;++ // Check if transcription file exists+ await fs.access(transcriptionWavPath);++ // Create initial transcription metadata+ const initialMetadata: TranscriptionMetadata = {+ transcriptionStartTime: Date.now(),+ transcriptionEndTime: 0,+ transcriptionStatus: 'pending',+ };+ await fs.writeJson(transcriptionMetadataPath, initialMetadata);++ // Notify clients that transcription has started+ io.emit('apps:recording-transcription-start', { filename: foldername });++ const transcription = await gemini(transcriptionWavPath, {+ mode: 'transcript',+ });++ // Update transcription metadata with results+ const metadata: TranscriptionMetadata = {+ transcriptionStartTime: initialMetadata.transcriptionStartTime,+ transcriptionEndTime: Date.now(),+ transcriptionStatus: 'completed',+ transcription: transcription ?? undefined,+ };++ await fs.writeJson(transcriptionMetadataPath, metadata);++ // Notify clients that transcription is complete+ io.emit('apps:recording-transcription-end', {+ filename: foldername,+ success: true,+ transcription,+ });++ res.json({ success: true });+ } catch (error) {+ console.error('❌ Error during transcription:', error);++ // Update transcription metadata with error+ const metadata: TranscriptionMetadata = {+ transcriptionStartTime: Date.now(),+ transcriptionEndTime: Date.now(),+ transcriptionStatus: 'error',+ error: error instanceof Error ? error.message : 'Unknown error',+ };++ await fs+ .writeJson(`${recordingDir}/transcription.json`, metadata)+ .catch(err => {+ console.error('❌ Error saving transcription metadata:', err);+ });++ // Notify clients of transcription error+ io.emit('apps:recording-transcription-end', {+ filename: foldername,+ success: false,+ error: error instanceof Error ? error.message : 'Unknown error',+ });++ res.status(500).json({+ error: error instanceof Error ? error.message : 'Unknown error',+ });+ }+ }+);++// Start server+httpServer.listen(PORT, () => {+ console.log(`+🎙️ Media Capture Server started successfully:+- Port: ${PORT}+- Recordings directory: ${RECORDING_DIR}+- Sample rate: 44.1kHz+- Channels: Mono+`);+});++// Initialize file watcher+setupRecordingsWatcher().catch(error => {+ console.error('Failed to setup recordings watcher:', error);+});diff --git a/packages/frontend/media-capture-playground/server/types.d.ts b/packages/frontend/media-capture-playground/server/types.d.tsnew file mode 100644index 0000000000..85b042e1ef--- /dev/null+++ b/packages/frontend/media-capture-playground/server/types.d.ts@@ -0,0 +1,4 @@+declare module '*.txt' {+ const content: string;+ export default content;+}diff --git a/packages/frontend/media-capture-playground/server/wav-writer.ts b/packages/frontend/media-capture-playground/server/wav-writer.tsnew file mode 100644index 0000000000..78f3765ddc--- /dev/null+++ b/packages/frontend/media-capture-playground/server/wav-writer.ts@@ -0,0 +1,125 @@+import fs from 'fs-extra';++interface WavWriterConfig {+ targetSampleRate?: number;+}++export class WavWriter {+ private readonly file: fs.WriteStream;+ private readonly originalSampleRate: number = 44100;+ private readonly targetSampleRate: number;+ private readonly numChannels = 1; // The audio is mono+ private samplesWritten = 0;+ private readonly tempFilePath: string;+ private readonly finalFilePath: string;++ constructor(finalPath: string, config: WavWriterConfig = {}) {+ this.finalFilePath = finalPath;+ this.tempFilePath = finalPath + '.tmp';+ this.targetSampleRate = config.targetSampleRate ?? this.originalSampleRate;+ this.file = fs.createWriteStream(this.tempFilePath);+ this.writeHeader(); // Always write header immediately+ }++ private writeHeader() {+ const buffer = Buffer.alloc(44); // WAV header is 44 bytes++ // RIFF chunk descriptor+ buffer.write('RIFF', 0);+ buffer.writeUInt32LE(36, 4); // Initial file size - 8 (will be updated later)+ buffer.write('WAVE', 8);++ // fmt sub-chunk+ buffer.write('fmt ', 12);+ buffer.writeUInt32LE(16, 16); // Subchunk1Size (16 for PCM)+ buffer.writeUInt16LE(3, 20); // AudioFormat (3 for IEEE float)+ buffer.writeUInt16LE(this.numChannels, 22); // NumChannels+ buffer.writeUInt32LE(this.targetSampleRate, 24); // SampleRate+ buffer.writeUInt32LE(this.targetSampleRate * this.numChannels * 4, 28); // ByteRate+ buffer.writeUInt16LE(this.numChannels * 4, 32); // BlockAlign+ buffer.writeUInt16LE(32, 34); // BitsPerSample (32 for float)++ // data sub-chunk+ buffer.write('data', 36);+ buffer.writeUInt32LE(0, 40); // Initial data size (will be updated later)++ this.file.write(buffer);+ }++ private resample(samples: Float32Array): Float32Array {+ const ratio = this.originalSampleRate / this.targetSampleRate;+ const newLength = Math.floor(samples.length / ratio);+ const result = new Float32Array(newLength);++ for (let i = 0; i < newLength; i++) {+ const position = i * ratio;+ const index = Math.floor(position);+ const fraction = position - index;++ // Linear interpolation between adjacent samples+ if (index + 1 < samples.length) {+ result[i] =+ samples[index] * (1 - fraction) + samples[index + 1] * fraction;+ } else {+ result[i] = samples[index];+ }+ }++ return result;+ }++ write(samples: Float32Array) {+ // Resample the input samples+ const resampledData = this.resample(samples);++ // Create a buffer with the correct size (4 bytes per float)+ const buffer = Buffer.alloc(resampledData.length * 4);++ // Write each float value properly+ for (let i = 0; i < resampledData.length; i++) {+ buffer.writeFloatLE(resampledData[i], i * 4);+ }++ this.file.write(buffer);+ this.samplesWritten += resampledData.length;+ }++ async end(): Promise<void> {+ return new Promise<void>((resolve, reject) => {+ this.file.end(() => {+ void this.updateHeaderAndCleanup().then(resolve).catch(reject);+ });+ });+ }++ private async updateHeaderAndCleanup(): Promise<void> {+ // Read the entire temporary file+ const data = await fs.promises.readFile(this.tempFilePath);++ // Update the header with correct sizes+ const dataSize = this.samplesWritten * 4;+ const fileSize = dataSize + 36;++ data.writeUInt32LE(fileSize, 4); // Update RIFF chunk size+ data.writeUInt32LE(dataSize, 40); // Update data chunk size++ // Write the updated file+ await fs.promises.writeFile(this.finalFilePath, data);++ // Clean up temp file+ await fs.promises.unlink(this.tempFilePath);+ }+}++/**+ * Creates a Buffer from Float32Array audio data+ * @param float32Array - The Float32Array containing audio samples+ * @returns FileData - The audio data as a Buffer+ */+export function FileData(float32Array: Float32Array): Buffer {+ const buffer = Buffer.alloc(float32Array.length * 4); // 4 bytes per float+ for (let i = 0; i < float32Array.length; i++) {+ buffer.writeFloatLE(float32Array[i], i * 4);+ }+ return buffer;+}diff --git a/packages/frontend/media-capture-playground/tsconfg.node.json b/packages/frontend/media-capture-playground/tsconfg.node.jsonnew file mode 100644index 0000000000..ed58d9ef2d--- /dev/null+++ b/packages/frontend/media-capture-playground/tsconfg.node.json@@ -0,0 +1,7 @@+{+ "extends": "../../../tsconfig.node.json",+ "compilerOptions": {+ "rootDir": "./server"+ },+ "include": ["./server"]+}diff --git a/packages/frontend/media-capture-playground/tsconfig.json b/packages/frontend/media-capture-playground/tsconfig.jsonnew file mode 100644index 0000000000..41a46a0d40--- /dev/null+++ b/packages/frontend/media-capture-playground/tsconfig.json@@ -0,0 +1,10 @@+{+ "extends": "../../../tsconfig.web.json",+ "compilerOptions": {+ "rootDir": "./web",+ "outDir": "./dist",+ "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo"+ },+ "include": ["./web", "server/types.d.ts"],+ "references": [{ "path": "../native" }]+}diff --git a/packages/frontend/media-capture-playground/vite.config.ts b/packages/frontend/media-capture-playground/vite.config.tsnew file mode 100644index 0000000000..c7ab16ad12--- /dev/null+++ b/packages/frontend/media-capture-playground/vite.config.ts@@ -0,0 +1,18 @@+import tailwindcss from '@tailwindcss/vite';+import react from '@vitejs/plugin-react';+import { defineConfig } from 'vite';++// https://vite.dev/config/+export default defineConfig({+ plugins: [react(), tailwindcss()],+ root: './web',+ server: {+ proxy: {+ '/api': {+ target: 'http://localhost:6544',+ changeOrigin: true,+ rewrite: path => path.replace(/^\/api/, ''),+ },+ },+ },+});diff --git a/packages/frontend/media-capture-playground/web/app.tsx b/packages/frontend/media-capture-playground/web/app.tsxnew file mode 100644index 0000000000..a3aaa7ff4d--- /dev/null+++ b/packages/frontend/media-capture-playground/web/app.tsx@@ -0,0 +1,33 @@+import { AppList } from './components/app-list';+import { SavedRecordings } from './components/saved-recordings';++export function App() {+ return (+ <div className="h-screen bg-gray-50 overflow-hidden">+ <div className="h-full p-4 flex gap-4 max-w-[1800px] mx-auto">+ <div className="flex-1 flex flex-col min-h-0">+ <h1 className="text-xl font-bold text-gray-900 mb-1">+ Running Applications+ </h1>+ <p className="text-sm text-gray-500 mb-2">+ Select an application to start recording its audio+ </p>+ <div className="flex-1 bg-white shadow-lg rounded-lg border border-gray-100 overflow-auto">+ <AppList />+ </div>+ </div>+ <div className="w-[1024px] flex flex-col min-h-0">+ <h1 className="text-xl font-bold text-gray-900 mb-1">+ Saved Recordings+ </h1>+ <p className="text-sm text-gray-500 mb-2">+ Listen to and manage your recorded audio files+ </p>+ <div className="flex-1 bg-white shadow-lg rounded-lg border border-gray-100 p-4 overflow-auto">+ <SavedRecordings />+ </div>+ </div>+ </div>+ </div>+ );+}diff --git a/packages/frontend/media-capture-playground/web/components/app-item.tsx b/packages/frontend/media-capture-playground/web/components/app-item.tsxnew file mode 100644index 0000000000..cde56c9778--- /dev/null+++ b/packages/frontend/media-capture-playground/web/components/app-item.tsx@@ -0,0 +1,122 @@+import React from 'react';++import type { AppGroup, RecordingStatus } from '../types';+import { formatDuration } from '../utils';++interface AppItemProps {+ app: AppGroup;+ recordings?: RecordingStatus[];+}++export function AppItem({ app, recordings }: AppItemProps) {+ const [imgError, setImgError] = React.useState(false);+ const [isRecording, setIsRecording] = React.useState(false);++ const appName = app.rootApp.name || '';+ const bundleId = app.rootApp.bundleIdentifier || '';+ const firstLetter = appName.charAt(0).toUpperCase();+ const isRunning = app.apps.some(a => a.running);++ const recording = recordings?.find((r: RecordingStatus) =>+ app.apps.some(a => a.processId === r.processId)+ );++ const handleRecordClick = React.useCallback(() => {+ const recordingApp = app.apps.find(a => a.running);+ if (!recordingApp) {+ return;+ }+ if (isRecording) {+ void fetch(`/api/apps/${recordingApp.processId}/stop`, {+ method: 'POST',+ })+ .then(() => setIsRecording(false))+ .catch(error => console.error('Failed to stop recording:', error));+ } else {+ void fetch(`/api/apps/${recordingApp.processId}/record`, {+ method: 'POST',+ })+ .then(() => setIsRecording(true))+ .catch(error => console.error('Failed to start recording:', error));+ }+ }, [app.apps, isRecording]);++ React.useEffect(() => {+ setIsRecording(!!recording);+ }, [recording]);++ const [duration, setDuration] = React.useState(0);++ React.useEffect(() => {+ if (recording) {+ const interval = setInterval(() => {+ setDuration(Date.now() - recording.startTime);+ }, 1000);+ return () => clearInterval(interval);+ } else {+ setDuration(0);+ }+ return () => {};+ }, [recording]);++ return (+ <div className="flex items-center h-16 space-x-2 p-3 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100">+ {imgError ? (+ <div className="w-8 h-8 rounded-lg bg-gray-50 border border-gray-100 flex items-center justify-center text-gray-600 font-semibold text-base">+ {firstLetter}+ </div>+ ) : (+ <img+ src={`/api/apps/${app.rootApp.processId}/icon`}+ loading="lazy"+ alt={appName}+ className="w-8 h-8 object-contain rounded-lg bg-gray-50 border border-gray-100"+ onError={() => setImgError(true)}+ />+ )}+ <div className="flex-1 min-w-0">+ <div className="flex items-center space-x-1 mb-1">+ {appName ? (+ <span className="text-gray-900 font-medium text-sm truncate">+ {appName}+ </span>+ ) : (+ <span className="text-gray-400 italic font-medium text-sm">+ Unnamed Application+ </span>+ )}+ <span className="text-xs px-1 bg-gray-50 text-gray-500 rounded border border-gray-100">+ PID: {app.rootApp.processId}+ </span>+ <span+ className={`text-xs px-2 py-0.5 rounded-full font-medium border ${recording ? 'bg-red-50 text-red-600 border-red-100 opacity-100' : 'opacity-0'}`}+ >+ {recording ? formatDuration(duration) : '00:00:00'}+ </span>+ </div>+ <div className="text-xs text-gray-500 font-mono truncate opacity-80">+ {bundleId}+ </div>+ </div>+ {(isRunning || isRecording) && (+ <button+ onClick={handleRecordClick}+ className={`h-8 min-w-[80px] flex items-center justify-center rounded-lg text-sm font-medium transition-all duration-200 ${+ isRecording+ ? 'bg-red-50 text-red-600 hover:bg-red-100 border border-red-200'+ : 'bg-blue-50 text-blue-600 hover:bg-blue-100 border border-blue-200'+ }`}+ >+ {isRecording ? (+ <>+ <div className="w-1.5 h-1.5 rounded-full bg-red-500 animate-pulse mr-2" />+ <span>Stop</span>+ </>+ ) : (+ <span>Record</span>+ )}+ </button>+ )}+ </div>+ );+}diff --git a/packages/frontend/media-capture-playground/web/components/app-list.tsx b/packages/frontend/media-capture-playground/web/components/app-list.tsxnew file mode 100644index 0000000000..d5d3e91162--- /dev/null+++ b/packages/frontend/media-capture-playground/web/components/app-list.tsx@@ -0,0 +1,144 @@+import React from 'react';+import useSWRSubscription from 'swr/subscription';++import type { App, AppGroup, RecordingStatus } from '../types';+import { socket } from '../utils';+import { AppItem } from './app-item';++export function AppList() {+ const { data: apps = [] } = useSWRSubscription('apps', (_key, { next }) => {+ let apps: App[] = [];+ // Initial apps fetch+ fetch('/api/apps')+ .then(res => res.json())+ .then(data => {+ apps = data.apps;+ next(null, apps);+ })+ .catch(err => next(err));++ // Subscribe to app updates+ socket.on('apps:all', data => {+ next(null, data.apps);+ apps = data.apps;+ });+ socket.on('apps:state-changed', data => {+ const index = apps.findIndex(a => a.processId === data.processId);+ if (index !== -1) {+ next(+ null,+ apps.toSpliced(index, 1, {+ ...apps[index],+ running: data.running,+ })+ );+ }+ });+ socket.on('connect', () => {+ // Refetch on reconnect+ fetch('/api/apps')+ .then(res => res.json())+ .then(data => next(null, data.apps))+ .catch(err => next(err));+ });++ return () => {+ socket.off('apps:all');+ socket.off('apps:state-changed');+ socket.off('connect');+ };+ });++ const { data: recordings = [] } = useSWRSubscription<RecordingStatus[]>(+ 'recordings',+ (+ _key: string,+ { next }: { next: (err: Error | null, data?: RecordingStatus[]) => void }+ ) => {+ // Subscribe to recording updates+ socket.on('apps:recording', (data: { recordings: RecordingStatus[] }) => {+ next(null, data.recordings);+ });++ return () => {+ socket.off('apps:recording');+ };+ }+ );++ const appGroups: AppGroup[] = React.useMemo(() => {+ const mapping = apps.reduce((acc: Record<number, AppGroup>, app: App) => {+ if (!acc[app.processGroupId]) {+ acc[app.processGroupId] = {+ processGroupId: app.processGroupId,+ apps: [],+ rootApp:+ apps.find((a: App) => a.processId === app.processGroupId) || app,+ };+ }+ acc[app.processGroupId].apps.push(app);+ return acc;+ }, {});+ return Object.values(mapping);+ }, [apps]);++ const runningApps = (appGroups || []).filter(app =>+ app.apps.some(a => a.running)+ );+ const notRunningApps = (appGroups || []).filter(+ app => !app.apps.some(a => a.running)+ );++ return (+ <div className="h-full flex flex-col divide-y divide-gray-100">+ <div className="p-4 relative">+ <div className="flex items-center justify-between sticky top-0 bg-white z-10 mb-2">+ <h2 className="text-sm font-semibold text-gray-900">+ Active Applications+ </h2>+ <span className="text-xs px-2 py-1 bg-blue-50 rounded-full text-blue-600 font-medium">+ {runningApps.length} listening+ </span>+ </div>+ <div className="space-y-2">+ {runningApps.map(app => (+ <AppItem+ key={app.processGroupId}+ app={app}+ recordings={recordings}+ />+ ))}+ {runningApps.length === 0 && (+ <div className="text-sm text-gray-500 italic bg-gray-50 rounded-xl p-4 text-center">+ No applications are currently listening+ </div>+ )}+ </div>+ </div>+ <div className="p-4 flex-1 relative">+ <div className="flex items-center justify-between sticky top-0 bg-white z-10 mb-2">+ <h2 className="text-sm font-semibold text-gray-900">+ Other Applications+ </h2>+ <span className="text-xs px-2 py-1 bg-gray-50 rounded-full text-gray-600 font-medium">+ {notRunningApps.length} available+ </span>+ </div>+ <div className="space-y-2">+ {notRunningApps.map(app => (+ <AppItem+ key={app.processGroupId}+ app={app}+ recordings={recordings}+ />+ ))}+ {notRunningApps.length === 0 && (+ <div className="text-sm text-gray-500 italic bg-gray-50 rounded-xl p-4 text-center">+ No other applications found+ </div>+ )}+ </div>+ </div>+ </div>+ );+}diff --git a/packages/frontend/media-capture-playground/web/components/icons.tsx b/packages/frontend/media-capture-playground/web/components/icons.tsxnew file mode 100644index 0000000000..f7d359b581--- /dev/null+++ b/packages/frontend/media-capture-playground/web/components/icons.tsx@@ -0,0 +1,163 @@+import type { ReactElement } from 'react';++export function PlayIcon(): ReactElement {+ return (+ <svg+ className="w-6 h-6 text-gray-900"+ viewBox="0 0 24 24"+ fill="none"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ fillRule="evenodd"+ clipRule="evenodd"+ d="M4.5 5.653c0-1.426 1.529-2.33 2.779-1.643l11.54 6.348c1.295.712 1.295 2.573 0 3.285L7.28 19.991c-1.25.687-2.779-.217-2.779-1.643V5.653z"+ fill="currentColor"+ />+ </svg>+ );+}++export function PauseIcon(): ReactElement {+ return (+ <svg+ className="w-6 h-6 text-gray-900"+ viewBox="0 0 24 24"+ fill="none"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ fillRule="evenodd"+ clipRule="evenodd"+ d="M6.75 5.25a.75.75 0 01.75-.75H9a.75.75 0 01.75.75v13.5a.75.75 0 01-.75.75H7.5a.75.75 0 01-.75-.75V5.25zm7 0a.75.75 0 01.75-.75h1.5a.75.75 0 01.75.75v13.5a.75.75 0 01-.75.75h-1.5a.75.75 0 01-.75-.75V5.25z"+ fill="currentColor"+ />+ </svg>+ );+}++export function RewindIcon(): ReactElement {+ return (+ <svg+ className="w-5 h-5 text-gray-600"+ viewBox="0 0 24 24"+ fill="none"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ d="M12.066 11.2a1 1 0 000 1.6l5.334 4A1 1 0 0019 16V8a1 1 0 00-1.6-.8l-5.334 4zM11 8a1 1 0 00-1.6-.8l-5.334 4a1 1 0 000 1.6l5.334 4A1 1 0 0011 16V8z"+ fill="currentColor"+ />+ </svg>+ );+}++export function ForwardIcon(): ReactElement {+ return (+ <svg+ className="w-5 h-5 text-gray-600"+ viewBox="0 0 24 24"+ fill="none"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ d="M5 8a1 1 0 011.6-.8l5.334 4a1 1 0 010 1.6L6.6 16.8A1 1 0 015 16V8zm7.066-.8a1 1 0 00-1.6.8v8a1 1 0 001.6.8l5.334-4a1 1 0 000-1.6l-5.334-4z"+ fill="currentColor"+ />+ </svg>+ );+}++export function DeleteIcon(): ReactElement {+ return (+ <svg+ className="w-5 h-5"+ viewBox="0 0 24 24"+ fill="none"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"+ stroke="currentColor"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ </svg>+ );+}++export function LoadingSpinner(): ReactElement {+ return (+ <svg className="animate-spin h-4 w-4" viewBox="0 0 24 24">+ <circle+ className="opacity-25"+ cx="12"+ cy="12"+ r="10"+ stroke="currentColor"+ strokeWidth="4"+ fill="none"+ />+ <path+ className="opacity-75"+ fill="currentColor"+ d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"+ />+ </svg>+ );+}++export function ErrorIcon(): ReactElement {+ return (+ <svg+ className="w-4 h-4 mr-1.5 flex-shrink-0"+ fill="currentColor"+ viewBox="0 0 20 20"+ >+ <path+ fillRule="evenodd"+ d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z"+ clipRule="evenodd"+ />+ </svg>+ );+}++export function MicrophoneIcon(): ReactElement {+ return (+ <svg+ className="w-4 h-4 mr-1.5 text-blue-500"+ viewBox="0 0 20 20"+ fill="currentColor"+ >+ <path d="M7 4a3 3 0 016 0v4a3 3 0 11-6 0V4zm4 10.93A7.001 7.001 0 0017 8a1 1 0 10-2 0A5 5 0 015 8a1 1 0 00-2 0 7.001 7.001 0 006 6.93V17H6a1 1 0 100 2h8a1 1 0 100-2h-3v-2.07z" />+ </svg>+ );+}++export function WarningIcon(): ReactElement {+ return (+ <svg className="w-4 h-4 mr-1.5" viewBox="0 0 20 20" fill="currentColor">+ <path+ fillRule="evenodd"+ d="M8.257 3.099c.765-1.36 2.722-1.36 3.486 0l5.58 9.92c.75 1.334-.213 2.98-1.742 2.98H4.42c-1.53 0-2.493-1.646-1.743-2.98l5.58-9.92zM11 13a1 1 0 11-2 0 1 1 0 012 0zm-1-8a1 1 0 00-1 1v3a1 1 0 002 0V6a1 1 0 00-1-1z"+ clipRule="evenodd"+ />+ </svg>+ );+}++export function DefaultAppIcon(): ReactElement {+ return (+ <svg+ xmlns="http://www.w3.org/2000/svg"+ className="h-6 w-6"+ viewBox="0 0 20 20"+ fill="currentColor"+ >+ <path d="M10 2a3 3 0 00-3 3v4a3 3 0 006 0V5a3 3 0 00-3-3zm0 2a1 1 0 011 1v4a1 1 0 11-2 0V5a1 1 0 011-1z" />+ <path d="M3 10a7 7 0 1014 0h-2a5 5 0 11-10 0H3z" />+ </svg>+ );+}diff --git a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsxnew file mode 100644index 0000000000..41e52dda17--- /dev/null+++ b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx@@ -0,0 +1,872 @@+import type { ReactElement } from 'react';+import React from 'react';+import ReactMarkdown from 'react-markdown';++import type { SavedRecording, TranscriptionMetadata } from '../types';+import { formatDuration, socket } from '../utils';+import {+ DefaultAppIcon,+ DeleteIcon,+ ErrorIcon,+ ForwardIcon,+ LoadingSpinner,+ MicrophoneIcon,+ PauseIcon,+ PlayIcon,+ RewindIcon,+ WarningIcon,+} from './icons';++interface SavedRecordingItemProps {+ recording: SavedRecording;+}++// Audio player controls component+function AudioControls({+ audioRef,+ playbackRate,+ onPlaybackRateChange,+ onSeek,+ onPlayPause,+}: {+ audioRef: React.RefObject<HTMLAudioElement | null>;+ playbackRate: number;+ onPlaybackRateChange: () => void;+ onSeek: (seconds: number) => void;+ onPlayPause: () => void;+}): ReactElement {+ const [currentTime, setCurrentTime] = React.useState('00:00');+ const [duration, setDuration] = React.useState('00:00');++ React.useEffect(() => {+ const audio = audioRef.current;+ if (!audio) return;++ const formatTime = (time: number) => {+ const minutes = Math.floor(time / 60);+ const seconds = Math.floor(time % 60);+ return `${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`;+ };++ const updateTime = () => {+ setCurrentTime(formatTime(audio.currentTime));+ setDuration(formatTime(audio.duration));+ };++ audio.addEventListener('timeupdate', updateTime);+ audio.addEventListener('loadedmetadata', updateTime);++ return () => {+ audio.removeEventListener('timeupdate', updateTime);+ audio.removeEventListener('loadedmetadata', updateTime);+ };+ }, [audioRef]);++ return (+ <div className="flex items-center justify-between">+ <div className="flex items-center space-x-2">+ <button+ onClick={() => onSeek(-15)}+ className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"+ title="Back 15 seconds"+ >+ <RewindIcon />+ </button>+ <button+ onClick={onPlayPause}+ className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"+ >+ {audioRef.current?.paused ? <PlayIcon /> : <PauseIcon />}+ </button>+ <button+ onClick={() => onSeek(30)}+ className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"+ title="Forward 30 seconds"+ >+ <ForwardIcon />+ </button>+ <div className="text-sm font-mono text-gray-500 ml-2">+ {currentTime} <span className="text-gray-400">/</span> {duration}+ </div>+ </div>+ <button+ onClick={onPlaybackRateChange}+ className="px-3 py-1.5 text-sm font-medium text-gray-600 bg-gray-50 hover:bg-gray-100 rounded-lg transition-all duration-200 border border-gray-100 hover:shadow-sm"+ >+ {playbackRate}x+ </button>+ </div>+ );+}++// Waveform visualization component+function WaveformVisualizer({+ containerRef,+ waveformData,+ currentTime,+ fileName,+}: {+ containerRef: React.RefObject<HTMLDivElement | null>;+ waveformData: number[];+ currentTime: number;+ fileName: string;+}): ReactElement {+ return (+ <div+ className="relative h-14 bg-gray-50 overflow-hidden rounded-lg border border-gray-100"+ ref={containerRef}+ >+ <div className="absolute inset-0 flex items-end">+ {waveformData.map((amplitude, i) => (+ <div+ key={`${fileName}-bar-${i}`}+ className="flex-1 bg-red-400 transition-all duration-200"+ style={{+ height: `${Math.max(amplitude * 100, 3)}%`,+ opacity:+ i < Math.floor(currentTime * waveformData.length) ? 1 : 0.3,+ margin: '0 0.5px',+ }}+ />+ ))}+ </div>+ </div>+ );+}++// Update TranscriptionMessage component+function TranscriptionMessage({+ item,+ isNewSpeaker,+ isCurrentMessage,+}: {+ item: {+ speaker: string;+ start_time: string;+ transcription: string;+ };+ isNewSpeaker: boolean;+ isCurrentMessage: boolean;+}): ReactElement {+ return (+ <div className="flex items-start gap-3 group transition-all duration-300 w-full">+ <div className="w-[120px] flex-shrink-0">+ <div className="flex flex-col items-start gap-1">+ {isNewSpeaker && (+ <div+ className={`px-2.5 py-1 rounded-lg text-xs font-medium border transition-colors duration-300 ${+ isCurrentMessage+ ? 'bg-blue-100 text-blue-700 border-blue-200'+ : 'bg-blue-50 text-blue-600 border-blue-100'+ }`}+ >+ {item.speaker}+ </div>+ )}+ <div+ className={`text-[11px] font-mono ml-2 transition-colors duration-300 ${+ isCurrentMessage ? 'text-blue-500' : 'text-gray-400'+ }`}+ >+ {item.start_time}+ </div>+ </div>+ </div>+ <div className="flex-1 min-w-0 w-full">+ <div+ className={`text-sm leading-relaxed rounded-xl px-4 py-2 border transition-all inline-flex duration-300 ${+ isCurrentMessage+ ? 'bg-blue-50/50 text-blue-900 border-blue-200 shadow-md'+ : 'bg-white text-gray-600 border-gray-100 shadow-sm hover:shadow-md'+ }`}+ >+ {item.transcription}+ </div>+ </div>+ </div>+ );+}++// Add new Summary component+function TranscriptionSummary({ summary }: { summary: string }): ReactElement {+ return (+ <div className="mb-6 bg-blue-50/50 rounded-xl p-4 border border-blue-100">+ <div className="text-xs font-medium text-blue-600 mb-2 uppercase tracking-wider">+ Summary+ </div>+ <div className="text-sm text-gray-700 leading-relaxed prose prose-sm max-w-none prose-headings:text-gray-900 prose-a:text-blue-600 whitespace-pre-wrap">+ <ReactMarkdown>{summary}</ReactMarkdown>+ </div>+ </div>+ );+}++// Update TranscriptionContent component+function TranscriptionContent({+ transcriptionData,+ currentAudioTime,+}: {+ transcriptionData: {+ segments: Array<{+ speaker: string;+ start_time: string;+ transcription: string;+ }>;+ summary: string;+ title: string;+ };+ currentAudioTime: number;+}): ReactElement {+ const parseTimestamp = (timestamp: string) => {+ // Handle "MM:SS" format (without hours)+ const [minutes, seconds] = timestamp.split(':');+ return parseInt(minutes, 10) * 60 + parseInt(seconds, 10);+ };++ return (+ <div className="space-y-2 py-2 max-h-[400px] overflow-y-auto pr-2 scrollbar-thin scrollbar-thumb-gray-300 scrollbar-track-transparent hover:scrollbar-thumb-gray-400 w-full">+ <TranscriptionSummary summary={transcriptionData.summary} />+ {transcriptionData.segments.map((item, index) => {+ const isNewSpeaker =+ index === 0 ||+ transcriptionData.segments[index - 1].speaker !== item.speaker;++ const startTime = parseTimestamp(item.start_time);+ // Use next segment's start time as end time, or add 3 seconds for the last segment+ const endTime =+ index < transcriptionData.segments.length - 1+ ? parseTimestamp(transcriptionData.segments[index + 1].start_time)+ : startTime + 3;++ const isCurrentMessage =+ currentAudioTime >= startTime && currentAudioTime < endTime;++ return (+ <TranscriptionMessage+ key={`${item.speaker}-${item.start_time}-${index}`}+ item={item}+ isNewSpeaker={isNewSpeaker}+ isCurrentMessage={isCurrentMessage}+ />+ );+ })}+ </div>+ );+}++// Update TranscriptionStatus component+function TranscriptionStatus({+ transcription,+ transcriptionError,+ currentAudioTime,+}: {+ transcription?: TranscriptionMetadata;+ transcriptionError: string | null;+ currentAudioTime: number;+}): ReactElement | null {+ if (!transcription && !transcriptionError) {+ return null;+ }++ if (transcription?.transcriptionStatus === 'pending') {+ return (+ <div className="my-2">+ <div className="text-sm text-gray-600 bg-gray-50/50 p-4 border border-gray-100 w-full">+ <div className="font-medium text-gray-900 mb-4 flex items-center sticky top-0 bg-gray-50/50 backdrop-blur-sm z-10 py-2">+ <MicrophoneIcon />+ <span>Processing Audio</span>+ </div>+ <div className="flex items-center justify-center py-8">+ <div className="flex flex-col items-center gap-3">+ <LoadingSpinner />+ <div className="text-sm text-gray-600">+ <span className="font-medium">Starting transcription</span>+ <span className="text-gray-400 animate-pulse">...</span>+ </div>+ <div className="text-xs text-gray-400 max-w-sm text-center">+ This may take a few moments depending on the length of the+ recording+ </div>+ </div>+ </div>+ </div>+ </div>+ );+ }++ if (transcriptionError) {+ return (+ <div className="text-xs text-red-500 m-2 flex items-center bg-red-50 rounded-lg p-2 border border-red-100">+ <ErrorIcon />+ {transcriptionError}+ </div>+ );+ }++ if (+ transcription?.transcriptionStatus === 'completed' &&+ transcription.transcription+ ) {+ try {+ const transcriptionData = transcription.transcription;+ if (+ !transcriptionData.segments ||+ !Array.isArray(transcriptionData.segments)+ ) {+ throw new Error('Invalid transcription data format');+ }++ return (+ <div className="my-2">+ <div className="text-sm text-gray-600 bg-gray-50/50 p-4 border border-gray-100 w-full">+ <div className="font-medium text-gray-900 mb-4 flex items-center sticky top-0 bg-gray-50/50 backdrop-blur-sm z-10 py-2">+ <MicrophoneIcon />+ <span>Conversation Transcript</span>+ </div>+ {transcriptionData.title && (+ <div className="mb-4 bg-blue-50/50 rounded-lg p-3 border border-blue-100">+ <div className="text-xs font-medium text-blue-600 uppercase tracking-wider mb-1">+ Title+ </div>+ <div className="text-base font-medium text-gray-900">+ {transcriptionData.title}+ </div>+ </div>+ )}+ <TranscriptionContent+ transcriptionData={transcriptionData}+ currentAudioTime={currentAudioTime}+ />+ </div>+ </div>+ );+ } catch (error) {+ return (+ <div className="text-sm text-red-500 bg-red-50 rounded-lg p-2 border border-red-100 m-2">+ {error instanceof Error+ ? error.message+ : 'Failed to parse transcription data'}+ </div>+ );+ }+ }++ return null;+}++// Add new RecordingHeader component+function RecordingHeader({+ metadata,+ fileName,+ recordingDate,+ duration,+ error,+ isDeleting,+ showDeleteConfirm,+ setShowDeleteConfirm,+ handleDeleteClick,+}: {+ metadata: SavedRecording['metadata'];+ fileName: string;+ recordingDate: string;+ duration: string;+ error: string | null;+ isDeleting: boolean;+ showDeleteConfirm: boolean;+ setShowDeleteConfirm: (show: boolean) => void;+ handleDeleteClick: () => void;+ transcriptionError: string | null;+}): ReactElement {+ const [imgError, setImgError] = React.useState(false);++ return (+ <div className="flex items-start space-x-4 p-4 bg-gray-50/30">+ <div className="relative w-12 h-12 flex-shrink-0">+ {!imgError ? (+ <img+ src={`/api/recordings/${fileName}/icon.png`}+ alt={metadata?.appName || 'Unknown Application'}+ className="w-12 h-12 object-contain rounded-lg bg-gray-50 border border-gray-100 shadow-sm transition-transform duration-200 hover:scale-105"+ onError={() => setImgError(true)}+ />+ ) : (+ <div className="w-12 h-12 rounded-xl flex items-center justify-center text-gray-500 bg-gray-50 border border-gray-100 shadow-sm">+ <DefaultAppIcon />+ </div>+ )}+ </div>+ <div className="flex-1 min-w-0">+ <div className="flex items-center justify-between">+ <div className="flex items-center space-x-2">+ <span className="text-gray-900 font-semibold text-base truncate">+ {metadata?.appName || 'Unknown Application'}+ </span>+ <span className="text-xs px-2 py-0.5 bg-blue-50 rounded-full text-blue-600 font-medium border border-blue-100">+ {duration}+ </span>+ </div>+ <div className="flex items-center">+ {showDeleteConfirm ? (+ <div className="flex items-center space-x-2">+ <button+ onClick={() => setShowDeleteConfirm(false)}+ className="h-8 px-3 text-sm font-medium text-gray-600 hover:bg-gray-50 rounded-lg transition-colors border border-gray-100"+ disabled={isDeleting}+ >+ Cancel+ </button>+ <button+ onClick={handleDeleteClick}+ className="h-8 px-3 text-sm font-medium text-red-600 hover:bg-red-50 rounded-lg transition-colors border border-red-100 disabled:opacity-50 disabled:cursor-not-allowed"+ disabled={isDeleting}+ >+ {isDeleting ? (+ <div className="flex items-center space-x-2">+ <LoadingSpinner />+ <span>Deleting...</span>+ </div>+ ) : (+ 'Confirm'+ )}+ </button>+ </div>+ ) : (+ <button+ onClick={() => setShowDeleteConfirm(true)}+ className="h-8 w-8 flex items-center justify-center text-gray-400 hover:text-red-500 hover:bg-red-50 rounded-lg transition-colors"+ title="Delete recording"+ >+ <DeleteIcon />+ </button>+ )}+ </div>+ </div>+ <div className="text-sm text-gray-600 mt-1">{recordingDate}</div>+ <div className="text-xs text-gray-400 font-mono mt-0.5 truncate">+ {metadata?.bundleIdentifier || fileName}+ </div>+ {error && (+ <div className="text-xs text-red-500 mt-2 flex items-center bg-red-50 rounded-lg p-2 border border-red-100">+ <ErrorIcon />+ {error}+ </div>+ )}+ </div>+ </div>+ );+}++// Add new AudioPlayer component+function AudioPlayer({+ isLoading,+ error,+ audioRef,+ playbackRate,+ handlePlaybackRateChange,+ handleSeek,+ handlePlayPause,+ containerRef,+ waveformData,+ currentTime,+ fileName,+}: {+ isLoading: boolean;+ error: string | null;+ audioRef: React.RefObject<HTMLAudioElement>;+ playbackRate: number;+ handlePlaybackRateChange: () => void;+ handleSeek: (seconds: number) => void;+ handlePlayPause: () => void;+ containerRef: React.RefObject<HTMLDivElement>;+ waveformData: number[];+ currentTime: number;+ fileName: string;+}): ReactElement {+ return (+ <div className="px-4 pb-4">+ {isLoading && !error ? (+ <div className="h-14 bg-gray-50 rounded-lg flex items-center justify-center border border-gray-100">+ <LoadingSpinner />+ <span className="ml-2 text-sm text-gray-600 font-medium">+ Loading audio...+ </span>+ </div>+ ) : (+ <div className="flex flex-col space-y-3">+ <AudioControls+ audioRef={audioRef}+ playbackRate={playbackRate}+ onPlaybackRateChange={handlePlaybackRateChange}+ onSeek={handleSeek}+ onPlayPause={handlePlayPause}+ />+ <WaveformVisualizer+ containerRef={containerRef}+ waveformData={waveformData}+ currentTime={currentTime}+ fileName={fileName}+ />+ </div>+ )}+ </div>+ );+}++// Add new TranscribeButton component+function TranscribeButton({+ transcriptionStatus,+ onTranscribe,+}: {+ transcriptionStatus?: TranscriptionMetadata['transcriptionStatus'];+ onTranscribe: () => void;+}): ReactElement {+ return (+ <div className="px-4 pb-4">+ <div className="flex justify-end">+ <button+ onClick={onTranscribe}+ disabled={transcriptionStatus === 'pending'}+ className={`h-8 px-3 text-sm font-medium rounded-lg transition-colors border flex items-center space-x-2+ ${+ transcriptionStatus === 'pending'+ ? 'bg-blue-50 text-blue-600 border-blue-200 cursor-not-allowed'+ : transcriptionStatus === 'completed'+ ? 'text-blue-600 hover:bg-blue-50 border-blue-100'+ : transcriptionStatus === 'error'+ ? 'text-red-600 hover:bg-red-50 border-red-100'+ : 'text-blue-600 hover:bg-blue-50 border-blue-100'+ }`}+ >+ {transcriptionStatus === 'pending' ? (+ <>+ <LoadingSpinner />+ <span>Transcribing...</span>+ </>+ ) : transcriptionStatus === 'completed' ? (+ <>+ <MicrophoneIcon />+ <span>Transcribe Again</span>+ </>+ ) : transcriptionStatus === 'error' ? (+ <>+ <WarningIcon />+ <span>Retry Transcription</span>+ </>+ ) : (+ <>+ <MicrophoneIcon />+ <span>Transcribe</span>+ </>+ )}+ </button>+ </div>+ </div>+ );+}++// Main SavedRecordingItem component (simplified)+export function SavedRecordingItem({+ recording,+}: SavedRecordingItemProps): ReactElement {+ const [error, setError] = React.useState<string | null>(null);+ const [isLoading, setIsLoading] = React.useState(true);+ const [isDeleting, setIsDeleting] = React.useState(false);+ const [showDeleteConfirm, setShowDeleteConfirm] = React.useState(false);+ const [playbackRate, setPlaybackRate] = React.useState(1);+ const [waveformData, setWaveformData] = React.useState<number[]>([]);+ const [currentTime, setCurrentTime] = React.useState(0);+ const audioRef = React.useRef<HTMLAudioElement | null>(null);+ const containerRef = React.useRef<HTMLDivElement | null>(null);+ const [segments, setSegments] = React.useState(40);+ const [currentAudioTime, setCurrentAudioTime] = React.useState(0);+ const [transcriptionError, setTranscriptionError] = React.useState<+ string | null+ >(null);++ const metadata = recording.metadata;+ const fileName = recording.wav;+ const recordingDate = metadata+ ? new Date(metadata.recordingStartTime).toLocaleString()+ : 'Unknown date';+ const duration = metadata+ ? formatDuration(metadata.recordingDuration * 1000)+ : 'Unknown duration';++ // Update current audio time+ React.useEffect(() => {+ const audio = audioRef.current;+ if (audio) {+ const handleTimeUpdate = () => {+ setCurrentAudioTime(audio.currentTime);+ };+ audio.addEventListener('timeupdate', handleTimeUpdate);+ return () => audio.removeEventListener('timeupdate', handleTimeUpdate);+ }+ return () => {};+ }, []);++ // Calculate number of segments based on container width+ React.useEffect(() => {+ const updateSegments = () => {+ if (containerRef.current) {+ // Each bar should be at least 2px wide (1px bar + 1px gap)+ const width = containerRef.current.offsetWidth;+ setSegments(Math.floor(width / 2));+ }+ };++ updateSegments();+ const resizeObserver = new ResizeObserver(updateSegments);+ if (containerRef.current) {+ resizeObserver.observe(containerRef.current);+ }++ return () => resizeObserver.disconnect();+ }, []);++ const processAudioData = React.useCallback(async () => {+ try {+ const response = await fetch(`/api/recordings/${fileName}/recording.wav`);+ if (!response.ok) {+ throw new Error(+ `Failed to fetch audio file (${response.status}): ${response.statusText}`+ );+ }++ const audioContext = new AudioContext();+ const arrayBuffer = await response.arrayBuffer();++ // Ensure we have data to process+ if (!arrayBuffer || arrayBuffer.byteLength === 0) {+ throw new Error('No audio data received');+ }++ const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);+ const channelData = audioBuffer.getChannelData(0);++ // Process the audio data in chunks to create the waveform+ const numberOfSamples = channelData.length;+ const samplesPerSegment = Math.floor(numberOfSamples / segments);++ const waveform: number[] = [];+ for (let i = 0; i < segments; i++) {+ const start = i * samplesPerSegment;+ const end = start + samplesPerSegment;+ const segmentData = channelData.slice(start, end);++ // Calculate RMS (root mean square) for better amplitude representation+ const rms = Math.sqrt(+ segmentData.reduce((sum, sample) => sum + sample * sample, 0) /+ segmentData.length+ );++ waveform.push(rms);+ }++ // Normalize the waveform data to a 0-1 range+ const maxAmplitude = Math.max(...waveform);+ const normalizedWaveform = waveform.map(amp => amp / maxAmplitude);++ setWaveformData(normalizedWaveform);+ setIsLoading(false);+ } catch (err) {+ console.error('Error processing audio:', err);+ setError(+ err instanceof Error ? err.message : 'Failed to process audio data'+ );+ setIsLoading(false);+ }+ }, [fileName, segments]);++ React.useEffect(() => {+ const audio = audioRef.current;+ if (audio) {+ const handleError = (e: ErrorEvent) => {+ console.error('Audio error:', e);+ setError('Failed to load audio');+ setIsLoading(false);+ };++ const handleLoadedMetadata = () => {+ void processAudioData().catch(err => {+ console.error('Error processing audio data:', err);+ setError('Failed to process audio data');+ setIsLoading(false);+ });+ };++ const handleTimeUpdate = () => {+ setCurrentTime(audio.currentTime / audio.duration);+ };++ audio.addEventListener('error', handleError as EventListener);+ audio.addEventListener('loadedmetadata', handleLoadedMetadata);+ audio.addEventListener('timeupdate', handleTimeUpdate);++ return () => {+ audio.removeEventListener('error', handleError as EventListener);+ audio.removeEventListener('loadedmetadata', handleLoadedMetadata);+ audio.removeEventListener('timeupdate', handleTimeUpdate);+ };+ }+ return () => {};+ }, [processAudioData]);++ const handlePlayPause = React.useCallback(() => {+ if (audioRef.current) {+ if (audioRef.current.paused) {+ void audioRef.current.play();+ } else {+ audioRef.current.pause();+ }+ }+ }, []);++ const handleSeek = React.useCallback((seconds: number) => {+ if (audioRef.current) {+ audioRef.current.currentTime += seconds;+ }+ }, []);++ const handlePlaybackRateChange = React.useCallback(() => {+ if (audioRef.current) {+ const newRate = playbackRate === 1 ? 1.5 : 1;+ audioRef.current.playbackRate = newRate;+ setPlaybackRate(newRate);+ }+ }, [playbackRate]);++ const handleDelete = React.useCallback(async () => {+ setIsDeleting(true);+ setError(null); // Clear any previous errors++ try {+ const response = await fetch(`/api/recordings/${recording.wav}`, {+ method: 'DELETE',+ });++ if (!response.ok) {+ let errorMessage: string;+ try {+ const errorData = await response.json();+ errorMessage = errorData.error;+ } catch {+ errorMessage = `Server error (${response.status}): ${response.statusText}`;+ }+ throw new Error(errorMessage);+ }++ setShowDeleteConfirm(false);+ } catch (err) {+ console.error('Error deleting recording:', err);+ setError(+ err instanceof Error ? err.message : 'An unexpected error occurred'+ );+ } finally {+ setIsDeleting(false);+ }+ }, [recording.wav]);++ const handleDeleteClick = React.useCallback(() => {+ void handleDelete().catch(err => {+ console.error('Unexpected error during deletion:', err);+ setError('An unexpected error occurred');+ });+ }, [handleDelete]);++ React.useEffect(() => {+ // Listen for transcription events+ socket.on(+ 'apps:recording-transcription-start',+ (data: { filename: string }) => {+ if (data.filename === recording.wav) {+ setTranscriptionError(null);+ }+ }+ );++ socket.on(+ 'apps:recording-transcription-end',+ (data: {+ filename: string;+ success: boolean;+ transcription?: string;+ error?: string;+ }) => {+ if (data.filename === recording.wav && !data.success) {+ setTranscriptionError(data.error || 'Transcription failed');+ }+ }+ );++ return () => {+ socket.off('apps:recording-transcription-start');+ socket.off('apps:recording-transcription-end');+ };+ }, [recording.wav]);++ const handleTranscribe = React.useCallback(async () => {+ try {+ const response = await fetch(+ `/api/recordings/${recording.wav}/transcribe`,+ {+ method: 'POST',+ }+ );++ if (!response.ok) {+ const error = await response.json();+ throw new Error(error.error || 'Failed to start transcription');+ }+ } catch (err) {+ setTranscriptionError(+ err instanceof Error ? err.message : 'Failed to start transcription'+ );+ }+ }, [recording.wav]);++ return (+ <div className="bg-white rounded-lg shadow-sm hover:shadow-md transition-all duration-300 overflow-hidden mb-3 border border-gray-100 hover:border-gray-200">+ <RecordingHeader+ metadata={metadata}+ fileName={fileName}+ recordingDate={recordingDate}+ duration={duration}+ error={error}+ isDeleting={isDeleting}+ showDeleteConfirm={showDeleteConfirm}+ setShowDeleteConfirm={setShowDeleteConfirm}+ handleDeleteClick={handleDeleteClick}+ transcriptionError={transcriptionError}+ />+ <AudioPlayer+ isLoading={isLoading}+ error={error}+ audioRef={audioRef as React.RefObject<HTMLAudioElement>}+ playbackRate={playbackRate}+ handlePlaybackRateChange={handlePlaybackRateChange}+ handleSeek={handleSeek}+ handlePlayPause={handlePlayPause}+ containerRef={containerRef as React.RefObject<HTMLDivElement>}+ waveformData={waveformData}+ currentTime={currentTime}+ fileName={fileName}+ />+ <audio+ ref={audioRef}+ src={`/api/recordings/${fileName}/recording.wav`}+ preload="metadata"+ className="hidden"+ />+ <TranscriptionStatus+ transcription={recording.transcription}+ transcriptionError={transcriptionError}+ currentAudioTime={currentAudioTime}+ />+ <TranscribeButton+ transcriptionStatus={recording.transcription?.transcriptionStatus}+ onTranscribe={() => void handleTranscribe()}+ />+ </div>+ );+}diff --git a/packages/frontend/media-capture-playground/web/components/saved-recordings.tsx b/packages/frontend/media-capture-playground/web/components/saved-recordings.tsxnew file mode 100644index 0000000000..2d6d25f184--- /dev/null+++ b/packages/frontend/media-capture-playground/web/components/saved-recordings.tsx@@ -0,0 +1,41 @@+import useSWRSubscription from 'swr/subscription';++import type { SavedRecording } from '../types';+import { socket } from '../utils';+import { SavedRecordingItem } from './saved-recording-item';++export function SavedRecordings(): React.ReactElement {+ const { data: recordings = [] } = useSWRSubscription<SavedRecording[]>(+ 'saved-recordings',+ (+ _key: string,+ { next }: { next: (err: Error | null, data?: SavedRecording[]) => void }+ ) => {+ // Subscribe to saved recordings updates+ socket.on('apps:saved', (data: { recordings: SavedRecording[] }) => {+ next(null, data.recordings);+ });++ fetch('/api/apps/saved')+ .then(res => res.json())+ .then(data => next(null, data.recordings))+ .catch(err => next(err));++ return () => {+ socket.off('apps:saved');+ };+ }+ );++ if (recordings.length === 0) {+ return <p className="text-gray-500 italic text-sm">No saved recordings</p>;+ }++ return (+ <div className="space-y-1">+ {recordings.map(recording => (+ <SavedRecordingItem key={recording.wav} recording={recording} />+ ))}+ </div>+ );+}diff --git a/packages/frontend/media-capture-playground/web/index.html b/packages/frontend/media-capture-playground/web/index.htmlnew file mode 100644index 0000000000..b65f7dfec4--- /dev/null+++ b/packages/frontend/media-capture-playground/web/index.html@@ -0,0 +1,13 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="UTF-8" />+ <link rel="icon" type="image/svg+xml" href="/vite.svg" />+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />+ <title>Media Capture Playground</title>+ </head>+ <body>+ <div id="root"></div>+ <script type="module" src="/main.tsx"></script>+ </body>+</html>diff --git a/packages/frontend/media-capture-playground/web/main.css b/packages/frontend/media-capture-playground/web/main.cssnew file mode 100644index 0000000000..d4b5078586--- /dev/null+++ b/packages/frontend/media-capture-playground/web/main.css@@ -0,0 +1 @@+@import 'tailwindcss';diff --git a/packages/frontend/media-capture-playground/web/main.tsx b/packages/frontend/media-capture-playground/web/main.tsxnew file mode 100644index 0000000000..df8a85c6a4--- /dev/null+++ b/packages/frontend/media-capture-playground/web/main.tsx@@ -0,0 +1,11 @@+import './main.css';++import { createRoot } from 'react-dom/client';++import { App } from './app';++const rootElement = document.getElementById('root');+if (!rootElement) {+ throw new Error('Failed to find the root element');+}+createRoot(rootElement).render(<App />);diff --git a/packages/frontend/media-capture-playground/web/types.ts b/packages/frontend/media-capture-playground/web/types.tsnew file mode 100644index 0000000000..e1ec56a4b0--- /dev/null+++ b/packages/frontend/media-capture-playground/web/types.ts@@ -0,0 +1,55 @@+export interface App {+ processId: number;+ processGroupId: number;+ bundleIdentifier: string;+ name: string;+ running: boolean;+}++export interface AppGroup {+ processGroupId: number;+ rootApp: App;+ apps: App[];+}++export interface RecordingStatus {+ processId: number;+ bundleIdentifier: string;+ name: string;+ startTime: number;+}++export interface RecordingMetadata {+ appName: string;+ bundleIdentifier: string;+ processId: number;+ recordingStartTime: number;+ recordingEndTime: number;+ recordingDuration: number;+ sampleRate: number;+ totalSamples: number;+ icon?: Uint8Array;+}++export interface TranscriptionMetadata {+ transcriptionStartTime: number;+ transcriptionEndTime: number;+ transcriptionStatus: 'not_started' | 'pending' | 'completed' | 'error';+ transcription?: {+ title: string;+ segments: Array<{+ speaker: string;+ start_time: string;+ end_time: string;+ transcription: string;+ }>;+ summary: string;+ };+ error?: string;+}++export interface SavedRecording {+ wav: string;+ metadata?: RecordingMetadata;+ transcription?: TranscriptionMetadata;+}diff --git a/packages/frontend/media-capture-playground/web/utils.ts b/packages/frontend/media-capture-playground/web/utils.tsnew file mode 100644index 0000000000..61a924248f--- /dev/null+++ b/packages/frontend/media-capture-playground/web/utils.ts@@ -0,0 +1,19 @@+import { io } from 'socket.io-client';++// Create a singleton socket instance+export const socket = io('http://localhost:6544');++export function formatDuration(ms: number): string {+ const seconds = Math.floor(ms / 1000);+ const minutes = Math.floor(seconds / 60);+ const hours = Math.floor(minutes / 60);+ return `${hours.toString().padStart(2, '0')}:${(minutes % 60)+ .toString()+ .padStart(2, '0')}:${(seconds % 60).toString().padStart(2, '0')}`;+}++// Helper function to convert timestamp (MM:SS.mmm) to seconds+export function timestampToSeconds(timestamp: string): number {+ const [minutes, seconds] = timestamp.split(':').map(parseFloat);+ return minutes * 60 + seconds;+}diff --git a/packages/frontend/native/.gitignore b/packages/frontend/native/.gitignoreindex 7f927850c0..682bfaaae3 100644--- a/packages/frontend/native/.gitignore+++ b/packages/frontend/native/.gitignore@@ -1,2 +1,3 @@*.fixturelib+*.bindiff --git a/packages/frontend/native/Cargo.toml b/packages/frontend/native/Cargo.tomlindex f22a5e105a..1c5011a911 100644--- a/packages/frontend/native/Cargo.toml+++ b/packages/frontend/native/Cargo.toml@@ -7,14 +7,15 @@ version = "0.0.0"crate-type = ["cdylib", "rlib"][dependencies]-affine_common = { workspace = true }-affine_nbstore = { path = "./nbstore" }-affine_sqlite_v1 = { path = "./sqlite_v1" }-napi = { workspace = true }-napi-derive = { workspace = true }-once_cell = { workspace = true }-sqlx = { workspace = true, default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] }-tokio = { workspace = true, features = ["full"] }+affine_common = { workspace = true }+affine_media_capture = { path = "./media_capture" }+affine_nbstore = { path = "./nbstore" }+affine_sqlite_v1 = { path = "./sqlite_v1" }+napi = { workspace = true }+napi-derive = { workspace = true }+once_cell = { workspace = true }+sqlx = { workspace = true, default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] }+tokio = { workspace = true, features = ["full"] }[build-dependencies]napi-build = { workspace = true }diff --git a/packages/frontend/native/media-capture-exapmle.ts b/packages/frontend/native/media-capture-exapmle.tsnew file mode 100644index 0000000000..712d62bd3a--- /dev/null+++ b/packages/frontend/native/media-capture-exapmle.ts@@ -0,0 +1,149 @@+import { join } from 'node:path';+import { fileURLToPath } from 'node:url';++import {+ Whisper,+ WhisperFullParams,+ WhisperSamplingStrategy,+} from '@napi-rs/whisper';+import { BehaviorSubject, EMPTY, Observable } from 'rxjs';+import {+ distinctUntilChanged,+ exhaustMap,+ groupBy,+ mergeMap,+ switchMap,+ tap,+} from 'rxjs/operators';++import { type Application, ShareableContent } from './index.js';++const rootDir = join(fileURLToPath(import.meta.url), '..');++const shareableContent = new ShareableContent();++const appList = new Set([+ 'com.tinyspeck.slackmacgap.helper',+ 'us.zoom.xos',+ 'org.mozilla.firefoxdeveloperedition',+]);++console.info(shareableContent.applications().map(app => app.bundleIdentifier));++const GGLM_LARGE = join(rootDir, 'ggml-large-v3-turbo.bin');++const whisper = new Whisper(GGLM_LARGE, {+ useGpu: true,+ gpuDevice: 1,+});++const whisperParams = new WhisperFullParams(WhisperSamplingStrategy.Greedy);++const SAMPLE_WINDOW_MS = 3000; // 3 seconds, similar to stream.cpp's step_ms+const SAMPLES_PER_WINDOW = (SAMPLE_WINDOW_MS / 1000) * 16000; // 16kHz sample rate++// eslint-disable-next-line rxjs/finnish+const runningApplications = new BehaviorSubject(+ shareableContent.applications()+);++const applicationListChangedSubscriber =+ ShareableContent.onApplicationListChanged(() => {+ runningApplications.next(shareableContent.applications());+ });++runningApplications+ .pipe(+ mergeMap(apps => apps.filter(app => appList.has(app.bundleIdentifier))),+ groupBy(app => app.bundleIdentifier),+ mergeMap(app$ =>+ app$.pipe(+ exhaustMap(app =>+ new Observable<[Application, boolean]>(subscriber => {+ const stateSubscriber = ShareableContent.onAppStateChanged(+ app,+ err => {+ if (err) {+ subscriber.error(err);+ return;+ }+ subscriber.next([app, app.isRunning]);+ }+ );+ return () => {+ stateSubscriber.unsubscribe();+ };+ }).pipe(+ distinctUntilChanged(+ ([_, isRunningA], [__, isRunningB]) => isRunningA === isRunningB+ ),+ switchMap(([app]) =>+ !app.isRunning+ ? EMPTY+ : new Observable(observer => {+ const buffers: Float32Array[] = [];+ const audioStream = app.tapAudio((err, samples) => {+ if (err) {+ observer.error(err);+ return;+ }++ if (samples) {+ buffers.push(samples);+ observer.next(samples);++ // Calculate total samples in buffer+ const totalSamples = buffers.reduce(+ (acc, buf) => acc + buf.length,+ 0+ );++ // Process when we have enough samples for our window+ if (totalSamples >= SAMPLES_PER_WINDOW) {+ // Concatenate all buffers+ const concatenated = new Float32Array(totalSamples);+ let offset = 0;+ buffers.forEach(buf => {+ concatenated.set(buf, offset);+ offset += buf.length;+ });++ // Transcribe the audio+ const result = whisper.full(+ whisperParams,+ concatenated+ );++ // Print results+ console.info(result);++ // Keep any remaining samples for next window+ const remainingSamples =+ totalSamples - SAMPLES_PER_WINDOW;+ if (remainingSamples > 0) {+ const lastBuffer = buffers[buffers.length - 1];+ buffers.length = 0;+ buffers.push(lastBuffer.slice(-remainingSamples));+ } else {+ buffers.length = 0;+ }+ }+ }+ });++ return () => {+ audioStream.stop();+ };+ })+ )+ )+ )+ )+ ),+ tap({+ finalize: () => {+ applicationListChangedSubscriber.unsubscribe();+ },+ })+ )+ .subscribe();diff --git a/packages/frontend/native/media_capture/Cargo.toml b/packages/frontend/native/media_capture/Cargo.tomlnew file mode 100644index 0000000000..222cd65882--- /dev/null+++ b/packages/frontend/native/media_capture/Cargo.toml@@ -0,0 +1,26 @@+[package]+edition = "2021"+name = "affine_media_capture"+version = "0.0.0"++[lib]+crate-type = ["cdylib", "rlib"]++[dependencies]+napi = { workspace = true, features = ["napi4"] }+napi-derive = { workspace = true, features = ["type-def"] }+rubato = { workspace = true }++[target.'cfg(target_os = "macos")'.dependencies]+block2 = { workspace = true }+core-foundation = { workspace = true, features = ["with-uuid"] }+coreaudio-rs = { workspace = true }+dispatch2 = { workspace = true }+objc2 = { workspace = true }+objc2-foundation = { workspace = true }+screencapturekit = { workspace = true }+thiserror = { workspace = true }+uuid = { workspace = true, features = ["v4"] }++[build-dependencies]+napi-build = { workspace = true }diff --git a/packages/frontend/native/media_capture/build.rs b/packages/frontend/native/media_capture/build.rsnew file mode 100644index 0000000000..bbfc9e4b9e--- /dev/null+++ b/packages/frontend/native/media_capture/build.rs@@ -0,0 +1,3 @@+fn main() {+ napi_build::setup();+}diff --git a/packages/frontend/native/media_capture/src/lib.rs b/packages/frontend/native/media_capture/src/lib.rsnew file mode 100644index 0000000000..a429d055b6--- /dev/null+++ b/packages/frontend/native/media_capture/src/lib.rs@@ -0,0 +1,4 @@+#[cfg(target_os = "macos")]+pub mod macos;+#[cfg(target_os = "macos")]+pub(crate) use macos::*;diff --git a/packages/frontend/native/media_capture/src/macos/audio_stream_basic_desc.rs b/packages/frontend/native/media_capture/src/macos/audio_stream_basic_desc.rsnew file mode 100644index 0000000000..2688154a63--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/audio_stream_basic_desc.rs@@ -0,0 +1,282 @@+use std::{fmt::Display, mem, ptr};++use coreaudio::sys::{+ kAudioHardwareNoError, kAudioObjectPropertyElementMain, kAudioObjectPropertyScopeGlobal,+ kAudioTapPropertyFormat, AudioObjectGetPropertyData, AudioObjectID, AudioObjectPropertyAddress,+};+use objc2::{Encode, Encoding, RefEncode};++use crate::error::CoreAudioError;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+#[repr(u32)]+pub enum AudioFormatID {+ LinearPcm = 0x6C70636D, // 'lpcm'+ Ac3 = 0x61632D33, // 'ac-3'+ Ac360958 = 0x63616333, // 'cac3'+ AppleIma4 = 0x696D6134, // 'ima4'+ Mpeg4Aac = 0x61616320, // 'aac '+ Mpeg4Celp = 0x63656C70, // 'celp'+ Mpeg4Hvxc = 0x68767863, // 'hvxc'+ Mpeg4TwinVq = 0x74777671, // 'twvq'+ Mace3 = 0x4D414333, // 'MAC3'+ Mace6 = 0x4D414336, // 'MAC6'+ ULaw = 0x756C6177, // 'ulaw'+ ALaw = 0x616C6177, // 'alaw'+ QDesign = 0x51444D43, // 'QDMC'+ QDesign2 = 0x51444D32, // 'QDM2'+ Qualcomm = 0x51636C70, // 'Qclp'+ MpegLayer1 = 0x2E6D7031, // '.mp1'+ MpegLayer2 = 0x2E6D7032, // '.mp2'+ MpegLayer3 = 0x2E6D7033, // '.mp3'+ TimeCode = 0x74696D65, // 'time'+ MidiStream = 0x6D696469, // 'midi'+ ParameterValueStream = 0x61707673, // 'apvs'+ AppleLossless = 0x616C6163, // 'alac'+ Mpeg4AacHe = 0x61616368, // 'aach'+ Mpeg4AacLd = 0x6161636C, // 'aacl'+ Mpeg4AacEld = 0x61616365, // 'aace'+ Mpeg4AacEldSbr = 0x61616366, // 'aacf'+ Mpeg4AacEldV2 = 0x61616367, // 'aacg'+ Mpeg4AacHeV2 = 0x61616370, // 'aacp'+ Mpeg4AacSpatial = 0x61616373, // 'aacs'+ MpegdUsac = 0x75736163, // 'usac'+ Amr = 0x73616D72, // 'samr'+ AmrWb = 0x73617762, // 'sawb'+ Audible = 0x41554442, // 'AUDB'+ ILbc = 0x696C6263, // 'ilbc'+ DviIntelIma = 0x6D730011,+ MicrosoftGsm = 0x6D730031,+ Aes3 = 0x61657333, // 'aes3'+ EnhancedAc3 = 0x65632D33, // 'ec-3'+ Flac = 0x666C6163, // 'flac'+ Opus = 0x6F707573, // 'opus'+ Apac = 0x61706163, // 'apac'+ Unknown = 0x00000000,+}++impl From<u32> for AudioFormatID {+ fn from(value: u32) -> Self {+ match value {+ 0x6C70636D => Self::LinearPcm,+ 0x61632D33 => Self::Ac3,+ 0x63616333 => Self::Ac360958,+ 0x696D6134 => Self::AppleIma4,+ 0x61616320 => Self::Mpeg4Aac,+ 0x63656C70 => Self::Mpeg4Celp,+ 0x68767863 => Self::Mpeg4Hvxc,+ 0x74777671 => Self::Mpeg4TwinVq,+ 0x4D414333 => Self::Mace3,+ 0x4D414336 => Self::Mace6,+ 0x756C6177 => Self::ULaw,+ 0x616C6177 => Self::ALaw,+ 0x51444D43 => Self::QDesign,+ 0x51444D32 => Self::QDesign2,+ 0x51636C70 => Self::Qualcomm,+ 0x2E6D7031 => Self::MpegLayer1,+ 0x2E6D7032 => Self::MpegLayer2,+ 0x2E6D7033 => Self::MpegLayer3,+ 0x74696D65 => Self::TimeCode,+ 0x6D696469 => Self::MidiStream,+ 0x61707673 => Self::ParameterValueStream,+ 0x616C6163 => Self::AppleLossless,+ 0x61616368 => Self::Mpeg4AacHe,+ 0x6161636C => Self::Mpeg4AacLd,+ 0x61616365 => Self::Mpeg4AacEld,+ 0x61616366 => Self::Mpeg4AacEldSbr,+ 0x61616367 => Self::Mpeg4AacEldV2,+ 0x61616370 => Self::Mpeg4AacHeV2,+ 0x61616373 => Self::Mpeg4AacSpatial,+ 0x75736163 => Self::MpegdUsac,+ 0x73616D72 => Self::Amr,+ 0x73617762 => Self::AmrWb,+ 0x41554442 => Self::Audible,+ 0x696C6263 => Self::ILbc,+ 0x6D730011 => Self::DviIntelIma,+ 0x6D730031 => Self::MicrosoftGsm,+ 0x61657333 => Self::Aes3,+ 0x65632D33 => Self::EnhancedAc3,+ 0x666C6163 => Self::Flac,+ 0x6F707573 => Self::Opus,+ 0x61706163 => Self::Apac,+ _ => Self::Unknown,+ }+ }+}++#[derive(Clone, Copy, PartialEq, Eq)]+pub struct AudioFormatFlags(pub u32);++#[allow(unused)]+impl AudioFormatFlags {+ pub const IS_FLOAT: u32 = 1 << 0;+ pub const IS_BIG_ENDIAN: u32 = 1 << 1;+ pub const IS_SIGNED_INTEGER: u32 = 1 << 2;+ pub const IS_PACKED: u32 = 1 << 3;+ pub const IS_ALIGNED_HIGH: u32 = 1 << 4;+ pub const IS_NON_INTERLEAVED: u32 = 1 << 5;+ pub const IS_NON_MIXABLE: u32 = 1 << 6;+ pub const ARE_ALL_CLEAR: u32 = 0x80000000;++ pub const LINEAR_PCM_IS_FLOAT: u32 = Self::IS_FLOAT;+ pub const LINEAR_PCM_IS_BIG_ENDIAN: u32 = Self::IS_BIG_ENDIAN;+ pub const LINEAR_PCM_IS_SIGNED_INTEGER: u32 = Self::IS_SIGNED_INTEGER;+ pub const LINEAR_PCM_IS_PACKED: u32 = Self::IS_PACKED;+ pub const LINEAR_PCM_IS_ALIGNED_HIGH: u32 = Self::IS_ALIGNED_HIGH;+ pub const LINEAR_PCM_IS_NON_INTERLEAVED: u32 = Self::IS_NON_INTERLEAVED;+ pub const LINEAR_PCM_IS_NON_MIXABLE: u32 = Self::IS_NON_MIXABLE;+ pub const LINEAR_PCM_SAMPLE_FRACTION_SHIFT: u32 = 7;+ pub const LINEAR_PCM_SAMPLE_FRACTION_MASK: u32 = 0x3F << Self::LINEAR_PCM_SAMPLE_FRACTION_SHIFT;+ pub const LINEAR_PCM_ARE_ALL_CLEAR: u32 = Self::ARE_ALL_CLEAR;++ pub const APPLE_LOSSLESS_FORMAT_FLAG_16_BIT_SOURCE_DATA: u32 = 1;+ pub const APPLE_LOSSLESS_FORMAT_FLAG_20_BIT_SOURCE_DATA: u32 = 2;+ pub const APPLE_LOSSLESS_FORMAT_FLAG_24_BIT_SOURCE_DATA: u32 = 3;+ pub const APPLE_LOSSLESS_FORMAT_FLAG_32_BIT_SOURCE_DATA: u32 = 4;+}++impl std::fmt::Display for AudioFormatFlags {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ let mut flags = Vec::new();++ if self.0 & Self::IS_FLOAT != 0 {+ flags.push("FLOAT");+ }+ if self.0 & Self::IS_BIG_ENDIAN != 0 {+ flags.push("BIG_ENDIAN");+ }+ if self.0 & Self::IS_SIGNED_INTEGER != 0 {+ flags.push("SIGNED_INTEGER");+ }+ if self.0 & Self::IS_PACKED != 0 {+ flags.push("PACKED");+ }+ if self.0 & Self::IS_ALIGNED_HIGH != 0 {+ flags.push("ALIGNED_HIGH");+ }+ if self.0 & Self::IS_NON_INTERLEAVED != 0 {+ flags.push("NON_INTERLEAVED");+ }+ if self.0 & Self::IS_NON_MIXABLE != 0 {+ flags.push("NON_MIXABLE");+ }+ if self.0 & Self::ARE_ALL_CLEAR != 0 {+ flags.push("ALL_CLEAR");+ }++ if flags.is_empty() {+ write!(f, "NONE")+ } else {+ write!(f, "{}", flags.join(" | "))+ }+ }+}++impl std::fmt::Debug for AudioFormatFlags {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "AudioFormatFlags({})", self)+ }+}++impl From<u32> for AudioFormatFlags {+ fn from(value: u32) -> Self {+ Self(value)+ }+}++/// [Apple's documentation](https://developer.apple.com/documentation/coreaudiotypes/audiostreambasicdescription?language=objc)+#[repr(C)]+#[derive(Clone, Copy, Debug, PartialEq)]+#[allow(non_snake_case)]+pub struct AudioStreamBasicDescription {+ pub mSampleRate: f64,+ pub mFormatID: u32,+ pub mFormatFlags: u32,+ pub mBytesPerPacket: u32,+ pub mFramesPerPacket: u32,+ pub mBytesPerFrame: u32,+ pub mChannelsPerFrame: u32,+ pub mBitsPerChannel: u32,+ pub mReserved: u32,+}++unsafe impl Encode for AudioStreamBasicDescription {+ const ENCODING: Encoding = Encoding::Struct(+ "AudioStreamBasicDescription",+ &[+ <f64>::ENCODING,+ <u32>::ENCODING,+ <u32>::ENCODING,+ <u32>::ENCODING,+ <u32>::ENCODING,+ <u32>::ENCODING,+ <u32>::ENCODING,+ <u32>::ENCODING,+ <u32>::ENCODING,+ ],+ );+}++unsafe impl RefEncode for AudioStreamBasicDescription {+ const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);+}++#[derive(Debug, Clone, Copy)]+#[repr(transparent)]+pub struct AudioStreamDescription(pub(crate) AudioStreamBasicDescription);++impl Display for AudioStreamDescription {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(+ f,+ "AudioStreamBasicDescription {{ mSampleRate: {}, mFormatID: {:?}, mFormatFlags: {}, \+ mBytesPerPacket: {}, mFramesPerPacket: {}, mBytesPerFrame: {}, mChannelsPerFrame: {}, \+ mBitsPerChannel: {}, mReserved: {} }}",+ self.0.mSampleRate,+ AudioFormatID::from(self.0.mFormatID),+ AudioFormatFlags(self.0.mFormatFlags),+ self.0.mBytesPerPacket,+ self.0.mFramesPerPacket,+ self.0.mBytesPerFrame,+ self.0.mChannelsPerFrame,+ self.0.mBitsPerChannel,+ self.0.mReserved+ )+ }+}++pub fn read_audio_stream_basic_description(+ tap_id: AudioObjectID,+) -> std::result::Result<AudioStreamDescription, CoreAudioError> {+ let mut data_size = mem::size_of::<AudioStreamBasicDescription>();+ let address = AudioObjectPropertyAddress {+ mSelector: kAudioTapPropertyFormat,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };+ let mut data = AudioStreamBasicDescription {+ mSampleRate: 0.0,+ mFormatID: 0,+ mFormatFlags: 0,+ mBytesPerPacket: 0,+ mFramesPerPacket: 0,+ mBytesPerFrame: 0,+ mChannelsPerFrame: 0,+ mBitsPerChannel: 0,+ mReserved: 0,+ };+ let status = unsafe {+ AudioObjectGetPropertyData(+ tap_id,+ &address,+ 0,+ ptr::null_mut(),+ (&mut data_size as *mut usize).cast(),+ (&mut data as *mut AudioStreamBasicDescription).cast(),+ )+ };+ if status != kAudioHardwareNoError as i32 {+ return Err(CoreAudioError::GetAudioStreamBasicDescriptionFailed(status));+ }+ Ok(AudioStreamDescription(data))+}diff --git a/packages/frontend/native/media_capture/src/macos/av_audio_file.rs b/packages/frontend/native/media_capture/src/macos/av_audio_file.rsnew file mode 100644index 0000000000..3f6bdc0502--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/av_audio_file.rs@@ -0,0 +1,71 @@+use std::ptr;++use objc2::{+ msg_send,+ runtime::{AnyClass, AnyObject},+ AllocAnyThread,+};+use objc2_foundation::{NSDictionary, NSError, NSNumber, NSString, NSUInteger, NSURL};++use crate::{+ av_audio_format::AVAudioFormat, av_audio_pcm_buffer::AVAudioPCMBuffer, error::CoreAudioError,+};++#[allow(unused)]+pub(crate) struct AVAudioFile {+ inner: *mut AnyObject,+}++#[allow(unused)]+impl AVAudioFile {+ pub(crate) fn new(url: &str, format: &AVAudioFormat) -> Result<Self, CoreAudioError> {+ let cls = AnyClass::get(c"AVAudioFile").ok_or(CoreAudioError::AVAudioFileClassNotFound)?;+ let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };+ if obj.is_null() {+ return Err(CoreAudioError::AllocAVAudioFileFailed);+ }+ let url: &NSURL = &*unsafe { NSURL::fileURLWithPath(&NSString::from_str(url)) };+ let settings = &*NSDictionary::from_retained_objects(+ &[+ &*NSString::from_str("AVFormatIDKey"),+ &*NSString::from_str("AVSampleRateKey"),+ &*NSString::from_str("AVNumberOfChannelsKey"),+ ],+ &[+ NSNumber::initWithUnsignedInt(+ NSNumber::alloc(),+ format.audio_stream_basic_description.0.mFormatID,+ ),+ NSNumber::initWithDouble(NSNumber::alloc(), format.get_sample_rate()),+ NSNumber::initWithUnsignedInt(NSNumber::alloc(), format.get_channel_count()),+ ],+ );+ let is_interleaved = format.is_interleaved();+ let mut error: *mut NSError = ptr::null_mut();+ let common_format: NSUInteger = 1;+ let obj: *mut AnyObject = unsafe {+ msg_send![+ obj,+ initForWriting: url,+ settings: settings,+ commonFormat: common_format,+ interleaved: is_interleaved,+ error: &mut error+ ]+ };+ if obj.is_null() {+ return Err(CoreAudioError::InitAVAudioFileFailed);+ }+ Ok(Self { inner: obj })+ }++ pub(crate) fn write(&self, buffer: AVAudioPCMBuffer) -> Result<(), CoreAudioError> {+ let mut error: *mut NSError = ptr::null_mut();+ let success: bool =+ unsafe { msg_send![self.inner, writeFromBuffer: buffer.inner, error: &mut error] };+ if !success {+ return Err(CoreAudioError::WriteAVAudioFileFailed);+ }+ Ok(())+ }+}diff --git a/packages/frontend/native/media_capture/src/macos/av_audio_format.rs b/packages/frontend/native/media_capture/src/macos/av_audio_format.rsnew file mode 100644index 0000000000..1a126fff05--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/av_audio_format.rs@@ -0,0 +1,95 @@+use objc2::{+ msg_send,+ runtime::{AnyClass, AnyObject},+ Encode, Encoding, RefEncode,+};++use crate::{audio_stream_basic_desc::AudioStreamDescription, error::CoreAudioError};++#[derive(Debug)]+#[allow(unused)]+pub(crate) struct AVAudioFormat {+ pub(crate) inner: AVAudioFormatRef,+ pub(crate) audio_stream_basic_description: AudioStreamDescription,+}++#[repr(transparent)]+#[derive(Debug, Clone, Copy)]+pub(crate) struct AVAudioFormatRef(pub(crate) *mut AnyObject);++unsafe impl Encode for AVAudioFormatRef {+ const ENCODING: Encoding = Encoding::Struct(+ "AVAudioFormat",+ &[+ Encoding::Double,+ Encoding::UInt,+ Encoding::Pointer(&Encoding::Struct(+ "AVAudioChannelLayout",+ &[+ Encoding::UInt,+ Encoding::UInt,+ Encoding::Pointer(&Encoding::Struct(+ "AudioChannelLayout",+ &[+ Encoding::UInt,+ Encoding::UInt,+ Encoding::Array(+ 1,+ &Encoding::Struct(+ "AudioChannelDescription",+ &[+ Encoding::UInt,+ Encoding::UInt,+ Encoding::Array(3, &Encoding::Float),+ ],+ ),+ ),+ Encoding::UInt,+ Encoding::UInt,+ ],+ )),+ Encoding::UInt,+ ],+ )),+ Encoding::Pointer(&Encoding::Object),+ ],+ );+}++unsafe impl RefEncode for AVAudioFormatRef {+ const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);+}++#[allow(unused)]+impl AVAudioFormat {+ pub fn new(+ audio_stream_basic_description: AudioStreamDescription,+ ) -> Result<Self, CoreAudioError> {+ let cls = AnyClass::get(c"AVAudioFormat").ok_or(CoreAudioError::AVAudioFormatClassNotFound)?;+ let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };+ if obj.is_null() {+ return Err(CoreAudioError::AllocAVAudioFormatFailed);+ }+ let obj: *mut AnyObject =+ unsafe { msg_send![obj, initWithStreamDescription: &audio_stream_basic_description.0] };+ if obj.is_null() {+ return Err(CoreAudioError::InitAVAudioFormatFailed);+ }+ Ok(Self {+ inner: AVAudioFormatRef(obj),+ audio_stream_basic_description,+ })+ }++ pub(crate) fn get_sample_rate(&self) -> f64 {+ unsafe { msg_send![self.inner.0, sampleRate] }+ }++ pub(crate) fn get_channel_count(&self) -> u32 {+ unsafe { msg_send![self.inner.0, channelCount] }+ }++ pub(crate) fn is_interleaved(&self) -> bool {+ unsafe { msg_send![self.inner.0, isInterleaved] }+ }+}diff --git a/packages/frontend/native/media_capture/src/macos/av_audio_pcm_buffer.rs b/packages/frontend/native/media_capture/src/macos/av_audio_pcm_buffer.rsnew file mode 100644index 0000000000..9d36117bb4--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/av_audio_pcm_buffer.rs@@ -0,0 +1,35 @@+use block2::RcBlock;+use objc2::{+ msg_send,+ runtime::{AnyClass, AnyObject},+};++use crate::{av_audio_format::AVAudioFormat, error::CoreAudioError, tap_audio::AudioBufferList};++#[allow(unused)]+pub(crate) struct AVAudioPCMBuffer {+ pub(crate) inner: *mut AnyObject,+}++#[allow(unused)]+impl AVAudioPCMBuffer {+ pub(crate) fn new(+ audio_format: &AVAudioFormat,+ buffer_list: *const AudioBufferList,+ ) -> Result<Self, CoreAudioError> {+ let cls =+ AnyClass::get(c"AVAudioPCMBuffer").ok_or(CoreAudioError::AVAudioPCMBufferClassNotFound)?;+ let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };+ if obj.is_null() {+ return Err(CoreAudioError::AllocAVAudioPCMBufferFailed);+ }+ let deallocator = RcBlock::new(|_buffer_list: *const AudioBufferList| {});+ let obj: *mut AnyObject = unsafe {+ msg_send![obj, initWithPCMFormat: audio_format.inner.0, bufferListNoCopy: buffer_list, deallocator: &*deallocator]+ };+ if obj.is_null() {+ return Err(CoreAudioError::InitAVAudioPCMBufferFailed);+ }+ Ok(Self { inner: obj })+ }+}diff --git a/packages/frontend/native/media_capture/src/macos/ca_tap_description.rs b/packages/frontend/native/media_capture/src/macos/ca_tap_description.rsnew file mode 100644index 0000000000..38b28bd425--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/ca_tap_description.rs@@ -0,0 +1,84 @@+use core_foundation::{+ base::{FromVoid, ItemRef},+ string::CFString,+};+use coreaudio::sys::AudioObjectID;+use objc2::{+ msg_send,+ runtime::{AnyClass, AnyObject},+ AllocAnyThread,+};+use objc2_foundation::{NSArray, NSNumber, NSString, NSUUID};++use crate::error::CoreAudioError;++pub(crate) struct CATapDescription {+ pub(crate) inner: *mut AnyObject,+}++impl CATapDescription {+ pub fn init_stereo_mixdown_of_processes(+ process: AudioObjectID,+ ) -> std::result::Result<Self, CoreAudioError> {+ let cls =+ AnyClass::get(c"CATapDescription").ok_or(CoreAudioError::CATapDescriptionClassNotFound)?;+ let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };+ if obj.is_null() {+ return Err(CoreAudioError::AllocCATapDescriptionFailed);+ }+ let processes_array =+ NSArray::from_retained_slice(&[NSNumber::initWithUnsignedInt(NSNumber::alloc(), process)]);+ let obj: *mut AnyObject =+ unsafe { msg_send![obj, initStereoMixdownOfProcesses: &*processes_array] };+ if obj.is_null() {+ return Err(CoreAudioError::InitStereoMixdownOfProcessesFailed);+ }++ Ok(Self { inner: obj })+ }++ pub fn init_stereo_global_tap_but_exclude_processes(+ processes: &[AudioObjectID],+ ) -> std::result::Result<Self, CoreAudioError> {+ let cls =+ AnyClass::get(c"CATapDescription").ok_or(CoreAudioError::CATapDescriptionClassNotFound)?;+ let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };+ if obj.is_null() {+ return Err(CoreAudioError::AllocCATapDescriptionFailed);+ }+ let processes_array = NSArray::from_retained_slice(+ processes+ .iter()+ .map(|p| NSNumber::initWithUnsignedInt(NSNumber::alloc(), *p))+ .collect::<Vec<_>>()+ .as_slice(),+ );+ let obj: *mut AnyObject =+ unsafe { msg_send![obj, initStereoMixdownOfProcesses: &*processes_array] };+ if obj.is_null() {+ return Err(CoreAudioError::InitStereoMixdownOfProcessesFailed);+ }++ Ok(Self { inner: obj })+ }++ pub fn get_uuid(&self) -> std::result::Result<ItemRef<CFString>, CoreAudioError> {+ let uuid: *mut NSUUID = unsafe { msg_send![self.inner, UUID] };+ if uuid.is_null() {+ return Err(CoreAudioError::GetCATapDescriptionUUIDFailed);+ }+ let uuid_string: *mut NSString = unsafe { msg_send![uuid, UUIDString] };+ if uuid_string.is_null() {+ return Err(CoreAudioError::ConvertUUIDToCFStringFailed);+ }+ Ok(unsafe { CFString::from_void(uuid_string.cast()) })+ }+}++impl Drop for CATapDescription {+ fn drop(&mut self) {+ unsafe {+ let _: () = msg_send![self.inner, release];+ }+ }+}diff --git a/packages/frontend/native/media_capture/src/macos/device.rs b/packages/frontend/native/media_capture/src/macos/device.rsnew file mode 100644index 0000000000..b5af983934--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/device.rs@@ -0,0 +1,66 @@+use std::{mem, ptr};++use core_foundation::{base::TCFType, string::CFString};+use coreaudio::sys::{+ kAudioDevicePropertyDeviceUID, kAudioHardwareNoError, kAudioObjectPropertyElementMain,+ kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, AudioDeviceID,+ AudioObjectGetPropertyData, AudioObjectID, AudioObjectPropertyAddress, CFStringRef,+};++use crate::error::CoreAudioError;++pub(crate) fn get_device_uid(+ device_id: AudioDeviceID,+) -> std::result::Result<CFString, CoreAudioError> {+ let system_output_id = get_device_audio_id(device_id)?;+ let address = AudioObjectPropertyAddress {+ mSelector: kAudioDevicePropertyDeviceUID,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };++ let mut output_uid: CFStringRef = ptr::null_mut();+ let mut data_size = mem::size_of::<CFStringRef>();+ let status = unsafe {+ AudioObjectGetPropertyData(+ system_output_id,+ &address,+ 0,+ ptr::null_mut(),+ (&mut data_size as *mut usize).cast(),+ (&mut output_uid as *mut CFStringRef).cast(),+ )+ };++ if status != 0 {+ return Err(CoreAudioError::GetDeviceUidFailed(status));+ }+ Ok(unsafe { CFString::wrap_under_create_rule(output_uid.cast()) })+}++pub(crate) fn get_device_audio_id(+ device_id: AudioDeviceID,+) -> std::result::Result<AudioObjectID, CoreAudioError> {+ let mut system_output_id: AudioObjectID = 0;+ let mut data_size = mem::size_of::<AudioObjectID>();++ let address = AudioObjectPropertyAddress {+ mSelector: device_id,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };+ let status = unsafe {+ AudioObjectGetPropertyData(+ kAudioObjectSystemObject,+ &address,+ 0,+ ptr::null_mut(),+ (&mut data_size as *mut usize).cast(),+ (&mut system_output_id as *mut AudioObjectID).cast(),+ )+ };+ if status != kAudioHardwareNoError as i32 {+ return Err(CoreAudioError::GetDefaultDeviceFailed(status));+ }+ Ok(system_output_id)+}diff --git a/packages/frontend/native/media_capture/src/macos/error.rs b/packages/frontend/native/media_capture/src/macos/error.rsnew file mode 100644index 0000000000..314e4ab11f--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/error.rs@@ -0,0 +1,81 @@+use thiserror::Error;++#[derive(Error, Debug)]+pub enum CoreAudioError {+ #[error("Map pid {0} to AudioObjectID failed")]+ PidNotFound(i32),+ #[error("Create process tap failed, status: {0}")]+ CreateProcessTapFailed(i32),+ #[error("Get default device failed, status: {0}")]+ GetDefaultDeviceFailed(i32),+ #[error("Get device uid failed, status: {0}")]+ GetDeviceUidFailed(i32),+ #[error("Create aggregate device failed, status: {0}")]+ CreateAggregateDeviceFailed(i32),+ #[error("Get process object list size failed, status: {0}")]+ GetProcessObjectListSizeFailed(i32),+ #[error("Get process object list failed, status: {0}")]+ GetProcessObjectListFailed(i32),+ #[error("AudioObjectGetPropertyDataSize failed, status: {0}")]+ AudioObjectGetPropertyDataSizeFailed(i32),+ #[error("CATapDescription class not found")]+ CATapDescriptionClassNotFound,+ #[error("Alloc CATapDescription failed")]+ AllocCATapDescriptionFailed,+ #[error("Call initStereoMixdownOfProcesses on CATapDescription failed")]+ InitStereoMixdownOfProcessesFailed,+ #[error("Get UUID on CATapDescription failed")]+ GetCATapDescriptionUUIDFailed,+ #[error("Get mute behavior on CATapDescription failed")]+ GetMuteBehaviorFailed,+ #[error("Convert UUID to CFString failed")]+ ConvertUUIDToCFStringFailed,+ #[error("Get AudioStreamBasicDescription failed, status: {0}")]+ GetAudioStreamBasicDescriptionFailed(i32),+ #[error("AVAudioFormat class not found")]+ AVAudioFormatClassNotFound,+ #[error("Alloc AVAudioFormat failed")]+ AllocAVAudioFormatFailed,+ #[error("Init AVAudioFormat failed")]+ InitAVAudioFormatFailed,+ #[error("Create IOProcIDWithBlock failed, status: {0}")]+ CreateIOProcIDWithBlockFailed(i32),+ #[error("Get hardware devices failed, status: {0}")]+ GetHardwareDevicesFailed(i32),+ #[error("AudioDeviceStart failed, status: {0}")]+ AudioDeviceStartFailed(i32),+ #[error("AudioDeviceStop failed, status: {0}")]+ AudioDeviceStopFailed(i32),+ #[error("AudioDeviceDestroyIOProcID failed, status: {0}")]+ AudioDeviceDestroyIOProcIDFailed(i32),+ #[error("AudioHardwareDestroyAggregateDevice failed, status: {0}")]+ AudioHardwareDestroyAggregateDeviceFailed(i32),+ #[error("AudioHardwareDestroyProcessTap failed, status: {0}")]+ AudioHardwareDestroyProcessTapFailed(i32),+ #[error("Get aggregate device property full sub device list failed, status: {0}")]+ GetAggregateDevicePropertyFullSubDeviceListFailed(i32),+ #[error("Add property listener block failed, status: {0}")]+ AddPropertyListenerBlockFailed(i32),+ #[error("AudioObjectGetPropertyData failed, status: {0}")]+ AudioObjectGetPropertyDataFailed(i32),+ #[error("AVAudioFile class not found")]+ AVAudioFileClassNotFound,+ #[error("Alloc AVAudioFile failed")]+ AllocAVAudioFileFailed,+ #[error("Init AVAudioFile failed")]+ InitAVAudioFileFailed,+ #[error("AVAudioPCMBuffer class not found")]+ AVAudioPCMBufferClassNotFound,+ #[error("Alloc AVAudioPCMBuffer failed")]+ AllocAVAudioPCMBufferFailed,+ #[error("Init AVAudioPCMBuffer failed")]+ InitAVAudioPCMBufferFailed,+ #[error("Write AVAudioFile failed")]+ WriteAVAudioFileFailed,+}++impl From<CoreAudioError> for napi::Error {+ fn from(value: CoreAudioError) -> Self {+ napi::Error::new(napi::Status::GenericFailure, value.to_string())+ }+}diff --git a/packages/frontend/native/media_capture/src/macos/mod.rs b/packages/frontend/native/media_capture/src/macos/mod.rsnew file mode 100644index 0000000000..89f25680af--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/mod.rs@@ -0,0 +1,11 @@+pub mod audio_stream_basic_desc;+pub mod av_audio_file;+pub mod av_audio_format;+pub mod av_audio_pcm_buffer;+pub mod ca_tap_description;+pub mod device;+pub(crate) mod error;+pub mod pid;+pub mod queue;+pub mod screen_capture_kit;+pub mod tap_audio;diff --git a/packages/frontend/native/media_capture/src/macos/pid.rs b/packages/frontend/native/media_capture/src/macos/pid.rsnew file mode 100644index 0000000000..caa5cb901d--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/pid.rs@@ -0,0 +1,98 @@+use std::{mem::MaybeUninit, ptr};++use coreaudio::sys::{+ kAudioHardwareNoError, kAudioHardwarePropertyProcessObjectList, kAudioObjectPropertyElementMain,+ kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, AudioObjectGetPropertyData,+ AudioObjectGetPropertyDataSize, AudioObjectID, AudioObjectPropertyAddress,+ AudioObjectPropertySelector,+};++use crate::error::CoreAudioError;++pub fn audio_process_list() -> Result<Vec<AudioObjectID>, CoreAudioError> {+ let address = AudioObjectPropertyAddress {+ mSelector: kAudioHardwarePropertyProcessObjectList,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };++ let mut data_size = 0u32;+ let status = unsafe {+ AudioObjectGetPropertyDataSize(+ kAudioObjectSystemObject,+ &address,+ 0,+ ptr::null_mut(),+ &mut data_size,+ )+ };++ if status != kAudioHardwareNoError as i32 {+ return Err(CoreAudioError::GetProcessObjectListSizeFailed(status));+ }++ let mut process_list: Vec<AudioObjectID> = vec![0; data_size as usize];++ let status = unsafe {+ AudioObjectGetPropertyData(+ kAudioObjectSystemObject,+ &address,+ 0,+ ptr::null_mut(),+ (&mut data_size as *mut u32).cast(),+ process_list.as_mut_ptr().cast(),+ )+ };++ if status != kAudioHardwareNoError as i32 {+ return Err(CoreAudioError::GetProcessObjectListFailed(status));+ }++ Ok(process_list)+}++pub fn get_process_property<T: Sized>(+ object: &AudioObjectID,+ selector: AudioObjectPropertySelector,+) -> Result<T, CoreAudioError> {+ let object_id = *object;+ let address = AudioObjectPropertyAddress {+ mSelector: selector,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };++ let mut data_size = 0u32;+ let status = unsafe {+ AudioObjectGetPropertyDataSize(object_id, &address, 0, ptr::null_mut(), &mut data_size)+ };++ if status != kAudioHardwareNoError as i32 {+ return Err(CoreAudioError::AudioObjectGetPropertyDataSizeFailed(status));+ }+ get_property_data(object_id, &address, &mut data_size)+}++pub fn get_property_data<T: Sized>(+ object_id: AudioObjectID,+ address: &AudioObjectPropertyAddress,+ data_size: &mut u32,+) -> Result<T, CoreAudioError> {+ let mut property = MaybeUninit::<T>::uninit();+ let status = unsafe {+ AudioObjectGetPropertyData(+ object_id,+ address,+ 0,+ ptr::null_mut(),+ (data_size as *mut u32).cast(),+ property.as_mut_ptr().cast(),+ )+ };++ if status != kAudioHardwareNoError as i32 {+ return Err(CoreAudioError::AudioObjectGetPropertyDataFailed(status));+ }++ Ok(unsafe { property.assume_init() })+}diff --git a/packages/frontend/native/media_capture/src/macos/queue.rs b/packages/frontend/native/media_capture/src/macos/queue.rsnew file mode 100644index 0000000000..9c68249703--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/queue.rs@@ -0,0 +1,12 @@+pub(crate) fn create_audio_tap_queue() -> *mut dispatch2::ffi::dispatch_queue_s {+ let queue_attr = unsafe {+ dispatch2::ffi::dispatch_queue_attr_make_with_qos_class(+ dispatch2::ffi::DISPATCH_QUEUE_SERIAL,+ dispatch2::ffi::dispatch_qos_class_t::QOS_CLASS_USER_INITIATED,+ 0,+ )+ };+ unsafe {+ dispatch2::ffi::dispatch_queue_create(c"ProcessTapRecorder".as_ptr().cast(), queue_attr)+ }+}diff --git a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rsnew file mode 100644index 0000000000..45b7201041--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs@@ -0,0 +1,623 @@+use std::{+ collections::HashMap,+ ffi::c_void,+ ptr,+ sync::{+ atomic::{AtomicPtr, Ordering},+ Arc, LazyLock, RwLock,+ },+};++use block2::{Block, RcBlock};+use core_foundation::{+ base::TCFType,+ string::{CFString, CFStringRef},+};+use coreaudio::sys::{+ kAudioHardwarePropertyProcessObjectList, kAudioObjectPropertyElementMain,+ kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, kAudioProcessPropertyBundleID,+ kAudioProcessPropertyIsRunning, kAudioProcessPropertyIsRunningInput, kAudioProcessPropertyPID,+ AudioObjectAddPropertyListenerBlock, AudioObjectID, AudioObjectPropertyAddress,+ AudioObjectRemovePropertyListenerBlock,+};+use napi::{+ bindgen_prelude::{Buffer, Error, Float32Array, Result, Status},+ threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},+};+use napi_derive::napi;+use objc2::{+ msg_send,+ rc::Retained,+ runtime::{AnyClass, AnyObject},+ Encode, Encoding,+};+use objc2_foundation::NSString;+use screencapturekit::shareable_content::SCShareableContent;+use uuid::Uuid;++use crate::{+ error::CoreAudioError,+ pid::{audio_process_list, get_process_property},+ tap_audio::{AggregateDevice, AudioTapStream},+};++#[repr(C)]+#[derive(Debug, Copy, Clone)]+struct NSSize {+ width: f64,+ height: f64,+}++#[repr(C)]+#[derive(Debug, Copy, Clone)]+struct NSPoint {+ x: f64,+ y: f64,+}++#[repr(C)]+#[derive(Debug, Copy, Clone)]+struct NSRect {+ origin: NSPoint,+ size: NSSize,+}++unsafe impl Encode for NSSize {+ const ENCODING: Encoding = Encoding::Struct("NSSize", &[f64::ENCODING, f64::ENCODING]);+}++unsafe impl Encode for NSPoint {+ const ENCODING: Encoding = Encoding::Struct("NSPoint", &[f64::ENCODING, f64::ENCODING]);+}++unsafe impl Encode for NSRect {+ const ENCODING: Encoding = Encoding::Struct("NSRect", &[<NSPoint>::ENCODING, <NSSize>::ENCODING]);+}++static RUNNING_APPLICATIONS: LazyLock<RwLock<Vec<AudioObjectID>>> =+ LazyLock::new(|| RwLock::new(audio_process_list().expect("Failed to get running applications")));++static APPLICATION_STATE_CHANGED_SUBSCRIBERS: LazyLock<+ RwLock<HashMap<AudioObjectID, HashMap<Uuid, Arc<ThreadsafeFunction<(), ()>>>>>,+> = LazyLock::new(|| RwLock::new(HashMap::new()));++static APPLICATION_STATE_CHANGED_LISTENER_BLOCKS: LazyLock<+ RwLock<HashMap<AudioObjectID, AtomicPtr<c_void>>>,+> = LazyLock::new(|| RwLock::new(HashMap::new()));++static NSRUNNING_APPLICATION_CLASS: LazyLock<Option<&'static AnyClass>> =+ LazyLock::new(|| AnyClass::get(c"NSRunningApplication"));++static AVCAPTUREDEVICE_CLASS: LazyLock<Option<&'static AnyClass>> =+ LazyLock::new(|| AnyClass::get(c"AVCaptureDevice"));++static SCSTREAM_CLASS: LazyLock<Option<&'static AnyClass>> =+ LazyLock::new(|| AnyClass::get(c"SCStream"));++struct TappableApplication {+ object_id: AudioObjectID,+}++impl TappableApplication {+ fn new(object_id: AudioObjectID) -> Self {+ Self { object_id }+ }++ fn process_id(&self) -> std::result::Result<i32, CoreAudioError> {+ get_process_property(&self.object_id, kAudioProcessPropertyPID)+ }++ fn bundle_identifier(&self) -> Result<String> {+ let bundle_id: CFStringRef =+ get_process_property(&self.object_id, kAudioProcessPropertyBundleID)?;+ Ok(unsafe { CFString::wrap_under_get_rule(bundle_id) }.to_string())+ }++ fn name(&self) -> Result<String> {+ let pid = self.process_id()?;++ // Get NSRunningApplication class+ let running_app_class = NSRUNNING_APPLICATION_CLASS.as_ref().ok_or_else(|| {+ Error::new(+ Status::GenericFailure,+ "NSRunningApplication class not found",+ )+ })?;++ // Get running application with PID+ let running_app: *mut AnyObject =+ unsafe { msg_send![*running_app_class, runningApplicationWithProcessIdentifier: pid] };+ if running_app.is_null() {+ return Ok(String::new());+ }++ // Get localized name+ let name: *mut NSString = unsafe { msg_send![running_app, localizedName] };+ if name.is_null() {+ return Ok(String::new());+ }++ // Create a safe wrapper and convert to string+ let name = unsafe {+ Retained::from_raw(name).ok_or_else(|| {+ Error::new(+ Status::GenericFailure,+ "Failed to create safe wrapper for localizedName",+ )+ })?+ };+ Ok(name.to_string())+ }++ fn icon(&self) -> Result<Vec<u8>> {+ let pid = self.process_id()?;++ // Get NSRunningApplication class+ let running_app_class = NSRUNNING_APPLICATION_CLASS.as_ref().ok_or_else(|| {+ Error::new(+ Status::GenericFailure,+ "NSRunningApplication class not found",+ )+ })?;++ // Get running application with PID+ let running_app: *mut AnyObject =+ unsafe { msg_send![*running_app_class, runningApplicationWithProcessIdentifier: pid] };+ if running_app.is_null() {+ return Ok(Vec::new());+ }++ unsafe {+ // Get original icon+ let icon: *mut AnyObject = msg_send![running_app, icon];+ if icon.is_null() {+ return Ok(Vec::new());+ }++ // Create a new NSImage with 64x64 size+ let nsimage_class = AnyClass::get(c"NSImage")+ .ok_or_else(|| Error::new(Status::GenericFailure, "NSImage class not found"))?;+ let resized_image: *mut AnyObject = msg_send![nsimage_class, alloc];+ let resized_image: *mut AnyObject =+ msg_send![resized_image, initWithSize: NSSize { width: 64.0, height: 64.0 }];+ let _: () = msg_send![resized_image, lockFocus];++ // Define drawing rectangle for 64x64 image+ let draw_rect = NSRect {+ origin: NSPoint { x: 0.0, y: 0.0 },+ size: NSSize {+ width: 64.0,+ height: 64.0,+ },+ };++ // Draw the original icon into draw_rect (using NSCompositingOperationCopy = 2)+ let _: () = msg_send![icon, drawInRect: draw_rect, fromRect: NSRect { origin: NSPoint { x: 0.0, y: 0.0 }, size: NSSize { width: 0.0, height: 0.0 } }, operation: 2, fraction: 1.0];+ let _: () = msg_send![resized_image, unlockFocus];++ // Get TIFF representation from the downsized image+ let tiff_data: *mut AnyObject = msg_send![resized_image, TIFFRepresentation];+ if tiff_data.is_null() {+ return Ok(Vec::new());+ }++ // Create bitmap image rep from TIFF+ let bitmap_class = AnyClass::get(c"NSBitmapImageRep")+ .ok_or_else(|| Error::new(Status::GenericFailure, "NSBitmapImageRep class not found"))?;+ let bitmap: *mut AnyObject = msg_send![bitmap_class, imageRepWithData: tiff_data];+ if bitmap.is_null() {+ return Ok(Vec::new());+ }++ // Create properties dictionary with compression factor+ let dict_class = AnyClass::get(c"NSMutableDictionary").ok_or_else(|| {+ Error::new(+ Status::GenericFailure,+ "NSMutableDictionary class not found",+ )+ })?;+ let properties: *mut AnyObject = msg_send![dict_class, dictionary];++ // Add compression properties+ let compression_key = NSString::from_str("NSImageCompressionFactor");+ let number_class = AnyClass::get(c"NSNumber")+ .ok_or_else(|| Error::new(Status::GenericFailure, "NSNumber class not found"))?;+ let compression_value: *mut AnyObject = msg_send![number_class, numberWithDouble: 0.8];+ let _: () = msg_send![properties, setObject: compression_value, forKey: &*compression_key];++ // Get PNG data with properties+ let png_data: *mut AnyObject =+ msg_send![bitmap, representationUsingType: 4, properties: properties]; // 4 = PNG++ if png_data.is_null() {+ return Ok(Vec::new());+ }++ // Get bytes from NSData+ let bytes: *const u8 = msg_send![png_data, bytes];+ let length: usize = msg_send![png_data, length];++ if bytes.is_null() {+ return Ok(Vec::new());+ }++ // Copy bytes into a Vec<u8>+ let data = std::slice::from_raw_parts(bytes, length).to_vec();+ Ok(data)+ }+ }+}++#[napi]+pub struct Application {+ inner: TappableApplication,+ pub(crate) object_id: AudioObjectID,+ pub(crate) process_id: i32,+ pub(crate) bundle_identifier: String,+ pub(crate) name: String,+}++#[napi]+impl Application {+ fn new(app: TappableApplication) -> Result<Self> {+ let object_id = app.object_id;+ let bundle_identifier = app.bundle_identifier()?;+ let name = app.name()?;+ let process_id = app.process_id()?;++ Ok(Self {+ inner: app,+ object_id,+ process_id,+ bundle_identifier,+ name,+ })+ }++ #[napi]+ pub fn tap_global_audio(+ excluded_processes: Option<Vec<&Application>>,+ audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,+ ) -> Result<AudioTapStream> {+ let mut device = AggregateDevice::create_global_tap_but_exclude_processes(+ &excluded_processes+ .unwrap_or_default()+ .iter()+ .map(|app| app.object_id)+ .collect::<Vec<_>>(),+ )?;+ device.start(audio_stream_callback)+ }++ #[napi(getter)]+ pub fn process_id(&self) -> i32 {+ self.process_id+ }++ #[napi(getter)]+ pub fn bundle_identifier(&self) -> String {+ self.bundle_identifier.clone()+ }++ #[napi(getter)]+ pub fn name(&self) -> String {+ self.name.clone()+ }++ #[napi(getter)]+ pub fn icon(&self) -> Result<Buffer> {+ let icon = self.inner.icon()?;+ Ok(Buffer::from(icon))+ }++ #[napi(getter)]+ pub fn get_is_running(&self) -> Result<bool> {+ Ok(get_process_property(+ &self.object_id,+ kAudioProcessPropertyIsRunningInput,+ )?)+ }++ #[napi]+ pub fn tap_audio(+ &self,+ audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,+ ) -> Result<AudioTapStream> {+ let mut device = AggregateDevice::new(self)?;+ device.start(audio_stream_callback)+ }+}++#[napi]+pub struct ApplicationListChangedSubscriber {+ listener_block: *const Block<dyn Fn(u32, *mut c_void)>,+}++#[napi]+impl ApplicationListChangedSubscriber {+ #[napi]+ pub fn unsubscribe(&self) -> Result<()> {+ let status = unsafe {+ AudioObjectRemovePropertyListenerBlock(+ kAudioObjectSystemObject,+ &AudioObjectPropertyAddress {+ mSelector: kAudioHardwarePropertyProcessObjectList,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ },+ ptr::null_mut(),+ self.listener_block.cast_mut().cast(),+ )+ };+ if status != 0 {+ return Err(Error::new(+ Status::GenericFailure,+ "Failed to remove property listener",+ ));+ }+ Ok(())+ }+}++#[napi]+pub struct ApplicationStateChangedSubscriber {+ id: Uuid,+ object_id: AudioObjectID,+}++#[napi]+impl ApplicationStateChangedSubscriber {+ #[napi]+ pub fn unsubscribe(&self) {+ if let Ok(mut lock) = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write() {+ if let Some(subscribers) = lock.get_mut(&self.object_id) {+ subscribers.remove(&self.id);+ if subscribers.is_empty() {+ lock.remove(&self.object_id);+ if let Some(listener_block) = APPLICATION_STATE_CHANGED_LISTENER_BLOCKS+ .write()+ .ok()+ .as_mut()+ .and_then(|map| map.remove(&self.object_id))+ {+ unsafe {+ AudioObjectRemovePropertyListenerBlock(+ self.object_id,+ &AudioObjectPropertyAddress {+ mSelector: kAudioProcessPropertyIsRunning,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ },+ ptr::null_mut(),+ listener_block.load(Ordering::Relaxed),+ );+ }+ }+ }+ }+ }+ }+}++#[napi]+pub struct ShareableContent {+ _inner: SCShareableContent,+}++#[napi]+#[derive(Default)]+pub struct RecordingPermissions {+ pub audio: bool,+ pub screen: bool,+}++#[napi]+impl ShareableContent {+ #[napi]+ pub fn on_application_list_changed(+ callback: Arc<ThreadsafeFunction<(), ()>>,+ ) -> Result<ApplicationListChangedSubscriber> {+ let callback_block: RcBlock<dyn Fn(u32, *mut c_void)> =+ RcBlock::new(move |_in_number_addresses, _in_addresses: *mut c_void| {+ if let Err(err) = RUNNING_APPLICATIONS+ .write()+ .map_err(|_| {+ Error::new(+ Status::GenericFailure,+ "Poisoned RwLock while writing RunningApplications",+ )+ })+ .and_then(|mut running_applications| {+ audio_process_list().map_err(From::from).map(|apps| {+ *running_applications = apps;+ })+ })+ {+ callback.call(Err(err), ThreadsafeFunctionCallMode::NonBlocking);+ } else {+ callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking);+ }+ });+ let listener_block = &*callback_block as *const Block<dyn Fn(u32, *mut c_void)>;+ let status = unsafe {+ AudioObjectAddPropertyListenerBlock(+ kAudioObjectSystemObject,+ &AudioObjectPropertyAddress {+ mSelector: kAudioHardwarePropertyProcessObjectList,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ },+ ptr::null_mut(),+ listener_block.cast_mut().cast(),+ )+ };+ if status != 0 {+ return Err(Error::new(+ Status::GenericFailure,+ "Failed to add property listener",+ ));+ }+ Ok(ApplicationListChangedSubscriber { listener_block })+ }++ #[napi]+ pub fn on_app_state_changed(+ app: &Application,+ callback: Arc<ThreadsafeFunction<(), ()>>,+ ) -> Result<ApplicationStateChangedSubscriber> {+ let id = Uuid::new_v4();+ let mut lock = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write().map_err(|_| {+ Error::new(+ Status::GenericFailure,+ "Poisoned RwLock while writing ApplicationStateChangedSubscribers",+ )+ })?;+ if let Some(subscribers) = lock.get_mut(&app.object_id) {+ subscribers.insert(id, callback);+ } else {+ let object_id = app.object_id;+ let list_change: RcBlock<dyn Fn(u32, *mut c_void)> =+ RcBlock::new(move |in_number_addresses, in_addresses: *mut c_void| {+ let addresses = unsafe {+ std::slice::from_raw_parts(+ in_addresses as *mut AudioObjectPropertyAddress,+ in_number_addresses as usize,+ )+ };+ for address in addresses {+ if address.mSelector == kAudioProcessPropertyIsRunning {+ if let Some(subscribers) = APPLICATION_STATE_CHANGED_SUBSCRIBERS+ .read()+ .ok()+ .as_ref()+ .and_then(|map| map.get(&object_id))+ {+ for callback in subscribers.values() {+ callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking);+ }+ }+ }+ }+ });+ let address = AudioObjectPropertyAddress {+ mSelector: kAudioProcessPropertyIsRunning,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };+ let listener_block = &*list_change as *const Block<dyn Fn(u32, *mut c_void)>;+ let status = unsafe {+ AudioObjectAddPropertyListenerBlock(+ app.object_id,+ &address,+ ptr::null_mut(),+ listener_block.cast_mut().cast(),+ )+ };+ if status != 0 {+ return Err(Error::new(+ Status::GenericFailure,+ "Failed to add property listener",+ ));+ }+ let subscribers = {+ let mut map = HashMap::new();+ map.insert(id, callback);+ map+ };+ lock.insert(app.object_id, subscribers);+ }+ Ok(ApplicationStateChangedSubscriber {+ id,+ object_id: app.object_id,+ })+ }++ #[napi(constructor)]+ pub fn new() -> Result<Self> {+ Ok(Self {+ _inner: SCShareableContent::get().map_err(|err| Error::new(Status::GenericFailure, err))?,+ })+ }++ #[napi]+ pub fn applications(&self) -> Result<Vec<Application>> {+ RUNNING_APPLICATIONS+ .read()+ .map_err(|_| {+ Error::new(+ Status::GenericFailure,+ "Poisoned RwLock while reading RunningApplications",+ )+ })?+ .iter()+ .filter_map(|id| {+ let app = TappableApplication::new(*id);+ if !app.bundle_identifier().ok()?.is_empty() {+ Some(Application::new(app))+ } else {+ None+ }+ })+ .collect()+ }++ #[napi]+ pub fn application_with_process_id(&self, process_id: u32) -> Result<Application> {+ // Find the AudioObjectID for the given process ID+ let audio_object_id = {+ let running_apps = RUNNING_APPLICATIONS.read().map_err(|_| {+ Error::new(+ Status::GenericFailure,+ "Poisoned RwLock while reading RunningApplications",+ )+ })?;++ *running_apps+ .iter()+ .find(|&&id| {+ let app = TappableApplication::new(id);+ app+ .process_id()+ .map(|pid| pid as u32 == process_id)+ .unwrap_or(false)+ })+ .ok_or_else(|| {+ Error::new(+ Status::GenericFailure,+ format!("No application found with process ID {}", process_id),+ )+ })?+ };++ let app = TappableApplication::new(audio_object_id);+ Application::new(app)+ }++ #[napi]+ pub fn check_recording_permissions(&self) -> Result<RecordingPermissions> {+ let av_capture_class = AVCAPTUREDEVICE_CLASS+ .as_ref()+ .ok_or_else(|| Error::new(Status::GenericFailure, "AVCaptureDevice class not found"))?;++ let sc_stream_class = SCSTREAM_CLASS+ .as_ref()+ .ok_or_else(|| Error::new(Status::GenericFailure, "SCStream class not found"))?;++ let media_type = NSString::from_str("com.apple.avfoundation.avcapturedevice.built-in_audio");++ let audio_status: i32 = unsafe {+ msg_send![+ *av_capture_class,+ authorizationStatusForMediaType: &*media_type+ ]+ };++ let screen_status: bool = unsafe { msg_send![*sc_stream_class, isScreenCaptureAuthorized] };++ Ok(RecordingPermissions {+ // AVAuthorizationStatusAuthorized = 3+ audio: audio_status == 3,+ screen: screen_status,+ })+ }+}diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rsnew file mode 100644index 0000000000..093346ac35--- /dev/null+++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs@@ -0,0 +1,360 @@+use std::{ffi::c_void, sync::Arc};++use block2::{Block, RcBlock};+use core_foundation::{+ array::CFArray,+ base::{CFType, ItemRef, TCFType},+ boolean::CFBoolean,+ dictionary::CFDictionary,+ string::CFString,+ uuid::CFUUID,+};+use coreaudio::sys::{+ kAudioAggregateDeviceIsPrivateKey, kAudioAggregateDeviceIsStackedKey,+ kAudioAggregateDeviceMainSubDeviceKey, kAudioAggregateDeviceNameKey,+ kAudioAggregateDeviceSubDeviceListKey, kAudioAggregateDeviceTapAutoStartKey,+ kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, kAudioHardwareNoError,+ kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultSystemOutputDevice,+ kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey, kAudioSubTapUIDKey,+ AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID,+ AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice,+ AudioHardwareDestroyAggregateDevice, AudioObjectID, AudioTimeStamp, OSStatus,+};+use napi::{+ bindgen_prelude::Float32Array,+ threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},+ Result,+};+use napi_derive::napi;+use objc2::{runtime::AnyObject, Encode, Encoding, RefEncode};++use crate::{+ ca_tap_description::CATapDescription, device::get_device_uid, error::CoreAudioError,+ queue::create_audio_tap_queue, screen_capture_kit::Application,+};++extern "C" {+ fn AudioHardwareCreateProcessTap(+ inDescription: *mut AnyObject,+ outTapID: *mut AudioObjectID,+ ) -> OSStatus;++ fn AudioHardwareDestroyProcessTap(tapID: AudioObjectID) -> OSStatus;+}++/// [Apple's documentation](https://developer.apple.com/documentation/coreaudiotypes/audiobuffer?language=objc)+#[repr(C)]+#[derive(Clone, Copy, Debug, PartialEq)]+#[allow(non_snake_case)]+pub struct AudioBuffer {+ pub mNumberChannels: u32,+ pub mDataByteSize: u32,+ pub mData: *mut c_void,+}++unsafe impl Encode for AudioBuffer {+ const ENCODING: Encoding = Encoding::Struct(+ "AudioBuffer",+ &[<u32>::ENCODING, <u32>::ENCODING, <*mut c_void>::ENCODING],+ );+}++unsafe impl RefEncode for AudioBuffer {+ const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);+}++#[repr(C)]+#[derive(Clone, Copy, Debug, PartialEq)]+#[allow(non_snake_case)]+pub struct AudioBufferList {+ pub mNumberBuffers: u32,+ pub mBuffers: [AudioBuffer; 1],+}++unsafe impl Encode for AudioBufferList {+ const ENCODING: Encoding = Encoding::Struct(+ "AudioBufferList",+ &[<u32>::ENCODING, <[AudioBuffer; 1]>::ENCODING],+ );+}++unsafe impl RefEncode for AudioBufferList {+ const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);+}++pub struct AggregateDevice {+ pub tap_id: AudioObjectID,+ pub id: AudioObjectID,+}++impl AggregateDevice {+ pub fn new(app: &Application) -> Result<Self> {+ let mut tap_id: AudioObjectID = 0;++ let tap_description = CATapDescription::init_stereo_mixdown_of_processes(app.object_id)?;+ let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) };++ if status != 0 {+ return Err(CoreAudioError::CreateProcessTapFailed(status).into());+ }++ let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?;++ let mut aggregate_device_id: AudioObjectID = 0;++ let status = unsafe {+ AudioHardwareCreateAggregateDevice(+ description_dict.as_concrete_TypeRef().cast(),+ &mut aggregate_device_id,+ )+ };++ // Check the status and return the appropriate result+ if status != 0 {+ return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into());+ }++ Ok(Self {+ tap_id,+ id: aggregate_device_id,+ })+ }++ pub fn create_global_tap_but_exclude_processes(processes: &[AudioObjectID]) -> Result<Self> {+ let mut tap_id: AudioObjectID = 0;+ let tap_description =+ CATapDescription::init_stereo_global_tap_but_exclude_processes(processes)?;+ let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) };++ if status != 0 {+ return Err(CoreAudioError::CreateProcessTapFailed(status).into());+ }++ let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?;++ let mut aggregate_device_id: AudioObjectID = 0;++ let status = unsafe {+ AudioHardwareCreateAggregateDevice(+ description_dict.as_concrete_TypeRef().cast(),+ &mut aggregate_device_id,+ )+ };++ // Check the status and return the appropriate result+ if status != 0 {+ return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into());+ }++ Ok(Self {+ tap_id,+ id: aggregate_device_id,+ })+ }++ pub fn start(+ &mut self,+ audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,+ ) -> Result<AudioTapStream> {+ let queue = create_audio_tap_queue();+ let mut in_proc_id: AudioDeviceIOProcID = None;++ let in_io_block: RcBlock<+ dyn Fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void) -> i32,+ > = RcBlock::new(+ move |_in_now: *mut c_void,+ in_input_data: *mut c_void,+ in_input_time: *mut c_void,+ _out_output_data: *mut c_void,+ _in_output_time: *mut c_void| {+ let AudioTimeStamp { mSampleTime, .. } = unsafe { &*in_input_time.cast() };++ // ignore pre-roll+ if *mSampleTime < 0.0 {+ return kAudioHardwareNoError as i32;+ }+ let AudioBufferList { mBuffers, .. } =+ unsafe { &mut *in_input_data.cast::<AudioBufferList>() };+ let [AudioBuffer {+ mData,+ mNumberChannels,+ mDataByteSize,+ }] = mBuffers;+ // Only create slice if we have valid data+ if !mData.is_null() && *mDataByteSize > 0 {+ // Calculate total number of samples (accounting for interleaved stereo)+ let total_samples = *mDataByteSize as usize / 4; // 4 bytes per f32++ // Create a slice of all samples+ let samples: &[f32] =+ unsafe { std::slice::from_raw_parts(mData.cast::<f32>(), total_samples) };++ // Convert to mono if needed+ let mono_samples: Vec<f32> = if *mNumberChannels > 1 {+ samples+ .chunks(*mNumberChannels as usize)+ .map(|chunk| chunk.iter().sum::<f32>() / *mNumberChannels as f32)+ .collect()+ } else {+ samples.to_vec()+ };++ audio_stream_callback.call(+ Ok(mono_samples.into()),+ ThreadsafeFunctionCallMode::NonBlocking,+ );+ }++ kAudioHardwareNoError as i32+ },+ );++ let status = unsafe {+ AudioDeviceCreateIOProcIDWithBlock(+ &mut in_proc_id,+ self.id,+ queue.cast(),+ (&*in_io_block+ as *const Block<+ dyn Fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void) -> i32,+ >)+ .cast_mut()+ .cast(),+ )+ };+ if status != 0 {+ return Err(CoreAudioError::CreateIOProcIDWithBlockFailed(status).into());+ }+ let status = unsafe { AudioDeviceStart(self.id, in_proc_id) };+ if status != 0 {+ return Err(CoreAudioError::AudioDeviceStartFailed(status).into());+ }++ Ok(AudioTapStream {+ device_id: self.id,+ in_proc_id,+ stop_called: false,+ })+ }++ fn create_aggregate_description(+ tap_id: AudioObjectID,+ tap_uuid_string: ItemRef<CFString>,+ ) -> Result<CFDictionary<CFType, CFType>> {+ let system_output_uid = get_device_uid(kAudioHardwarePropertyDefaultSystemOutputDevice)?;+ let default_input_uid = get_device_uid(kAudioHardwarePropertyDefaultInputDevice)?;++ let aggregate_device_name = CFString::new(&format!("Tap-{}", tap_id));+ let aggregate_device_uid: uuid::Uuid = CFUUID::new().into();+ let aggregate_device_uid_string = aggregate_device_uid.to_string();++ // Sub-device UID key and dictionary+ let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[(+ cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),+ system_output_uid.as_CFType(),+ )]);++ let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[(+ cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),+ default_input_uid.as_CFType(),+ )]);++ let tap_device_dict = CFDictionary::from_CFType_pairs(&[+ (+ cfstring_from_bytes_with_nul(kAudioSubTapDriftCompensationKey).as_CFType(),+ CFBoolean::false_value().as_CFType(),+ ),+ (+ cfstring_from_bytes_with_nul(kAudioSubTapUIDKey).as_CFType(),+ tap_uuid_string.as_CFType(),+ ),+ ]);++ let capture_device_list = vec![sub_device_input_dict, sub_device_output_dict];++ // Sub-device list+ let sub_device_list = CFArray::from_CFTypes(&capture_device_list);++ let tap_list = CFArray::from_CFTypes(&[tap_device_dict]);++ // Create the aggregate device description dictionary+ let description_dict = CFDictionary::from_CFType_pairs(&[+ (+ cfstring_from_bytes_with_nul(kAudioAggregateDeviceNameKey).as_CFType(),+ aggregate_device_name.as_CFType(),+ ),+ (+ cfstring_from_bytes_with_nul(kAudioAggregateDeviceUIDKey).as_CFType(),+ CFString::new(aggregate_device_uid_string.as_str()).as_CFType(),+ ),+ (+ cfstring_from_bytes_with_nul(kAudioAggregateDeviceMainSubDeviceKey).as_CFType(),+ system_output_uid.as_CFType(),+ ),+ (+ cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsPrivateKey).as_CFType(),+ CFBoolean::true_value().as_CFType(),+ ),+ (+ cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsStackedKey).as_CFType(),+ CFBoolean::false_value().as_CFType(),+ ),+ (+ cfstring_from_bytes_with_nul(kAudioAggregateDeviceTapAutoStartKey).as_CFType(),+ CFBoolean::true_value().as_CFType(),+ ),+ (+ cfstring_from_bytes_with_nul(kAudioAggregateDeviceSubDeviceListKey).as_CFType(),+ sub_device_list.as_CFType(),+ ),+ (+ cfstring_from_bytes_with_nul(kAudioAggregateDeviceTapListKey).as_CFType(),+ tap_list.as_CFType(),+ ),+ ]);+ Ok(description_dict)+ }+}++#[napi]+pub struct AudioTapStream {+ device_id: AudioObjectID,+ in_proc_id: AudioDeviceIOProcID,+ stop_called: bool,+}++#[napi]+impl AudioTapStream {+ #[napi]+ pub fn stop(&mut self) -> Result<()> {+ if self.stop_called {+ return Ok(());+ }+ self.stop_called = true;+ let status = unsafe { AudioDeviceStop(self.device_id, self.in_proc_id) };+ if status != 0 {+ return Err(CoreAudioError::AudioDeviceStopFailed(status).into());+ }+ let status = unsafe { AudioDeviceDestroyIOProcID(self.device_id, self.in_proc_id) };+ if status != 0 {+ return Err(CoreAudioError::AudioDeviceDestroyIOProcIDFailed(status).into());+ }+ let status = unsafe { AudioHardwareDestroyAggregateDevice(self.device_id) };+ if status != 0 {+ return Err(CoreAudioError::AudioHardwareDestroyAggregateDeviceFailed(status).into());+ }+ let status = unsafe { AudioHardwareDestroyProcessTap(self.device_id) };+ if status != 0 {+ return Err(CoreAudioError::AudioHardwareDestroyProcessTapFailed(status).into());+ }+ Ok(())+ }+}++fn cfstring_from_bytes_with_nul(bytes: &'static [u8]) -> CFString {+ CFString::new(+ unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(bytes) }+ .to_string_lossy()+ .as_ref(),+ )+}diff --git a/packages/frontend/native/package.json b/packages/frontend/native/package.jsonindex e83789f3fb..beca6f1b45 100644--- a/packages/frontend/native/package.json+++ b/packages/frontend/native/package.json@@ -26,8 +26,10 @@},"devDependencies": {"@napi-rs/cli": "3.0.0-alpha.70",+ "@napi-rs/whisper": "^0.0.4","@types/node": "^22.0.0","ava": "^6.2.0",+ "rxjs": "^7.8.1","ts-node": "^10.9.2","typescript": "^5.7.2"},diff --git a/packages/frontend/native/src/lib.rs b/packages/frontend/native/src/lib.rsindex f9b3fdb30e..b17678a551 100644--- a/packages/frontend/native/src/lib.rs+++ b/packages/frontend/native/src/lib.rs@@ -1,4 +1,6 @@pub mod hashcash;+#[allow(unused_imports)]+pub use affine_media_capture::*;pub use affine_nbstore::*;pub use affine_sqlite_v1::*;diff --git a/packages/frontend/native/tsconfig.json b/packages/frontend/native/tsconfig.jsonindex 32fe80bd50..ee4e505640 100644--- a/packages/frontend/native/tsconfig.json+++ b/packages/frontend/native/tsconfig.json@@ -3,6 +3,6 @@"compilerOptions": {"outDir": "./dist"},- "include": ["index.d.ts"],+ "include": ["index.d.ts", "media-capture-example.ts"],"references": []}diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.tsindex 5c30988aaa..e1177e13cd 100644--- a/tools/utils/src/workspace.gen.ts+++ b/tools/utils/src/workspace.gen.ts@@ -715,6 +715,11 @@ export const PackageList = ['tools/utils',],},+ {+ location: 'packages/frontend/media-capture-playground',+ name: '@affine/media-capture-playground',+ workspaceDependencies: ['packages/frontend/native'],+ },{location: 'packages/frontend/native',name: '@affine/native',@@ -866,6 +871,7 @@ export type PackageName =| '@affine/electron-api'| '@affine/graphql'| '@affine/i18n'+ | '@affine/media-capture-playground'| '@affine/native'| '@affine/templates'| '@affine/track'diff --git a/tsconfig.json b/tsconfig.jsonindex cd8227ea92..50ed943e9f 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -107,6 +107,7 @@{ "path": "./packages/frontend/electron-api" },{ "path": "./packages/frontend/graphql" },{ "path": "./packages/frontend/i18n" },+ { "path": "./packages/frontend/media-capture-playground" },{ "path": "./packages/frontend/native" },{ "path": "./packages/frontend/track" },{ "path": "./tests/affine-cloud" },diff --git a/yarn.lock b/yarn.lockindex 13b67b06e9..e839d9c038 100644--- a/yarn.lock+++ b/yarn.lock@@ -639,6 +639,40 @@ __metadata:languageName: unknownlinkType: soft+"@affine/media-capture-playground@workspace:packages/frontend/media-capture-playground":+ version: 0.0.0-use.local+ resolution: "@affine/media-capture-playground@workspace:packages/frontend/media-capture-playground"+ dependencies:+ "@affine/native": "workspace:*"+ "@google/generative-ai": "npm:^0.21.0"+ "@tailwindcss/vite": "npm:^4.0.6"+ "@types/express": "npm:^4"+ "@types/fs-extra": "npm:^11"+ "@types/multer": "npm:^1"+ "@types/react": "npm:^19.0.1"+ "@types/react-dom": "npm:^19.0.2"+ "@types/socket.io": "npm:^3.0.2"+ "@types/socket.io-client": "npm:^3.0.0"+ "@vitejs/plugin-react": "npm:^4.3.4"+ chokidar: "npm:^4.0.3"+ express: "npm:^4.21.2"+ express-rate-limit: "npm:^7.1.5"+ fs-extra: "npm:^11.3.0"+ multer: "npm:^1.4.5-lts.1"+ openai: "npm:^4.85.1"+ react: "npm:^19.0.0"+ react-dom: "npm:^19.0.0"+ react-markdown: "npm:^9.0.3"+ rxjs: "npm:^7.8.1"+ socket.io: "npm:^4.7.4"+ socket.io-client: "npm:^4.7.4"+ swr: "npm:^2.3.2"+ tailwindcss: "npm:^4.0.6"+ tsx: "npm:^4.19.2"+ vite: "npm:^6.1.0"+ languageName: unknown+ linkType: soft+"@affine/mobile@workspace:packages/frontend/apps/mobile":version: 0.0.0-use.localresolution: "@affine/mobile@workspace:packages/frontend/apps/mobile"@@ -715,8 +749,10 @@ __metadata:resolution: "@affine/native@workspace:packages/frontend/native"dependencies:"@napi-rs/cli": "npm:3.0.0-alpha.70"+ "@napi-rs/whisper": "npm:^0.0.4""@types/node": "npm:^22.0.0"ava: "npm:^6.2.0"+ rxjs: "npm:^7.8.1"ts-node: "npm:^10.9.2"typescript: "npm:^5.7.2"languageName: unknown@@ -1890,7 +1926,7 @@ __metadata:languageName: nodelinkType: hard-"@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.10.4, @babel/code-frame@npm:^7.22.13, @babel/code-frame@npm:^7.24.2, @babel/code-frame@npm:^7.25.9, @babel/code-frame@npm:^7.26.0, @babel/code-frame@npm:^7.26.2":+"@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.10.4, @babel/code-frame@npm:^7.22.13, @babel/code-frame@npm:^7.24.2, @babel/code-frame@npm:^7.26.0, @babel/code-frame@npm:^7.26.2":version: 7.26.2resolution: "@babel/code-frame@npm:7.26.2"dependencies:@@ -1954,26 +1990,27 @@ __metadata:languageName: nodelinkType: hard-"@babel/core@npm:^7.12.3, @babel/core@npm:^7.18.5, @babel/core@npm:^7.18.9, @babel/core@npm:^7.22.1, @babel/core@npm:^7.22.9, @babel/core@npm:^7.23.9":- version: 7.26.7- resolution: "@babel/core@npm:7.26.7"+"@babel/core@npm:^7.12.3, @babel/core@npm:^7.18.5, @babel/core@npm:^7.18.9, @babel/core@npm:^7.22.1, @babel/core@npm:^7.22.9, @babel/core@npm:^7.23.9, @babel/core@npm:^7.26.0":+ version: 7.26.8+ resolution: "@babel/core@npm:7.26.8"dependencies:"@ampproject/remapping": "npm:^2.2.0""@babel/code-frame": "npm:^7.26.2"- "@babel/generator": "npm:^7.26.5"+ "@babel/generator": "npm:^7.26.8""@babel/helper-compilation-targets": "npm:^7.26.5""@babel/helper-module-transforms": "npm:^7.26.0""@babel/helpers": "npm:^7.26.7"- "@babel/parser": "npm:^7.26.7"- "@babel/template": "npm:^7.25.9"- "@babel/traverse": "npm:^7.26.7"- "@babel/types": "npm:^7.26.7"+ "@babel/parser": "npm:^7.26.8"+ "@babel/template": "npm:^7.26.8"+ "@babel/traverse": "npm:^7.26.8"+ "@babel/types": "npm:^7.26.8"+ "@types/gensync": "npm:^1.0.0"convert-source-map: "npm:^2.0.0"debug: "npm:^4.1.0"gensync: "npm:^1.0.0-beta.2"json5: "npm:^2.2.3"semver: "npm:^6.3.1"- checksum: 10/1ca1c9b1366a1ee77ade9c72302f288b2b148e4190e0f36bc032d09c686b2c7973d3309e4eec2c57243508c16cf907c17dec4e34ba95e7a18badd57c61bbcb7c+ checksum: 10/a70d903dfd2c97e044b27fb480584fab6111954ced6987c6628ee4d37071ca446eca7830d72763a8d16a0da64eb83e02e3073d16c09e9eefa4a4ab38162636b4languageName: nodelinkType: hard@@ -1991,16 +2028,16 @@ __metadata:languageName: nodelinkType: hard-"@babel/generator@npm:^7.14.0, @babel/generator@npm:^7.18.13, @babel/generator@npm:^7.24.5, @babel/generator@npm:^7.26.0, @babel/generator@npm:^7.26.5":- version: 7.26.5- resolution: "@babel/generator@npm:7.26.5"+"@babel/generator@npm:^7.14.0, @babel/generator@npm:^7.18.13, @babel/generator@npm:^7.24.5, @babel/generator@npm:^7.26.0, @babel/generator@npm:^7.26.8":+ version: 7.26.8+ resolution: "@babel/generator@npm:7.26.8"dependencies:- "@babel/parser": "npm:^7.26.5"- "@babel/types": "npm:^7.26.5"+ "@babel/parser": "npm:^7.26.8"+ "@babel/types": "npm:^7.26.8""@jridgewell/gen-mapping": "npm:^0.3.5""@jridgewell/trace-mapping": "npm:^0.3.25"jsesc: "npm:^3.0.2"- checksum: 10/aa5f176155431d1fb541ca11a7deddec0fc021f20992ced17dc2f688a0a9584e4ff4280f92e8a39302627345cd325762f70f032764806c579c6fd69432542bcb+ checksum: 10/8c5af0f74aad2e575f2f833af0a9a38dda5abe0574752b5e0812677c78e5dc713b6b0c9ac3b30799ba6ef883614f9f0ef79d3aa10ba8f0e54f7f0284381b0059languageName: nodelinkType: hard@@ -2207,14 +2244,14 @@ __metadata:languageName: nodelinkType: hard-"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.0, @babel/parser@npm:^7.16.8, @babel/parser@npm:^7.20.7, @babel/parser@npm:^7.22.6, @babel/parser@npm:^7.23.9, @babel/parser@npm:^7.24.5, @babel/parser@npm:^7.25.4, @babel/parser@npm:^7.25.9, @babel/parser@npm:^7.26.0, @babel/parser@npm:^7.26.5, @babel/parser@npm:^7.26.7":- version: 7.26.7- resolution: "@babel/parser@npm:7.26.7"+"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.0, @babel/parser@npm:^7.16.8, @babel/parser@npm:^7.20.7, @babel/parser@npm:^7.22.6, @babel/parser@npm:^7.23.9, @babel/parser@npm:^7.24.5, @babel/parser@npm:^7.25.4, @babel/parser@npm:^7.26.0, @babel/parser@npm:^7.26.8":+ version: 7.26.8+ resolution: "@babel/parser@npm:7.26.8"dependencies:- "@babel/types": "npm:^7.26.7"+ "@babel/types": "npm:^7.26.8"bin:parser: ./bin/babel-parser.js- checksum: 10/3ccc384366ca9a9b49c54f5b24c9d8cff9a505f2fbdd1cfc04941c8e1897084cc32f100e77900c12bc14a176cf88daa3c155faad680d9a23491b997fd2a59ffc+ checksum: 10/0dd9d6b2022806b696b7a9ffb50b147f13525c497663d758a95adcc3ca0fa1d1bbb605fcc0604acc1cade60c3dbf2c1e0dd22b7aed17f8ad1c58c954208ffe7alanguageName: nodelinkType: hard@@ -2920,6 +2957,28 @@ __metadata:languageName: nodelinkType: hard+"@babel/plugin-transform-react-jsx-self@npm:^7.25.9":+ version: 7.25.9+ resolution: "@babel/plugin-transform-react-jsx-self@npm:7.25.9"+ dependencies:+ "@babel/helper-plugin-utils": "npm:^7.25.9"+ peerDependencies:+ "@babel/core": ^7.0.0-0+ checksum: 10/41c833cd7f91b1432710f91b1325706e57979b2e8da44e83d86312c78bbe96cd9ef778b4e79e4e17ab25fa32c72b909f2be7f28e876779ede28e27506c41f4ae+ languageName: node+ linkType: hard++"@babel/plugin-transform-react-jsx-source@npm:^7.25.9":+ version: 7.25.9+ resolution: "@babel/plugin-transform-react-jsx-source@npm:7.25.9"+ dependencies:+ "@babel/helper-plugin-utils": "npm:^7.25.9"+ peerDependencies:+ "@babel/core": ^7.0.0-0+ checksum: 10/a3e0e5672e344e9d01fb20b504fe29a84918eaa70cec512c4d4b1b035f72803261257343d8e93673365b72c371f35cf34bb0d129720bf178a4c87812c8b9c662+ languageName: node+ linkType: hard+"@babel/plugin-transform-react-jsx@npm:^7.0.0, @babel/plugin-transform-react-jsx@npm:^7.25.9":version: 7.25.9resolution: "@babel/plugin-transform-react-jsx@npm:7.25.9"@@ -3230,39 +3289,39 @@ __metadata:languageName: nodelinkType: hard-"@babel/template@npm:^7.18.10, @babel/template@npm:^7.20.7, @babel/template@npm:^7.24.0, @babel/template@npm:^7.25.9":- version: 7.25.9- resolution: "@babel/template@npm:7.25.9"+"@babel/template@npm:^7.18.10, @babel/template@npm:^7.20.7, @babel/template@npm:^7.24.0, @babel/template@npm:^7.25.9, @babel/template@npm:^7.26.8":+ version: 7.26.8+ resolution: "@babel/template@npm:7.26.8"dependencies:- "@babel/code-frame": "npm:^7.25.9"- "@babel/parser": "npm:^7.25.9"- "@babel/types": "npm:^7.25.9"- checksum: 10/e861180881507210150c1335ad94aff80fd9e9be6202e1efa752059c93224e2d5310186ddcdd4c0f0b0fc658ce48cb47823f15142b5c00c8456dde54f5de80b2+ "@babel/code-frame": "npm:^7.26.2"+ "@babel/parser": "npm:^7.26.8"+ "@babel/types": "npm:^7.26.8"+ checksum: 10/bc45db0fd4e92d35813c2a8e8fa80b8a887c275b323537b8ebd9c64228c1614e81c74236d08f744017a6562987e48b10501688f7a8be5d6a53fb6acb61aa01c8languageName: nodelinkType: hard-"@babel/traverse@npm:^7.16.8, @babel/traverse@npm:^7.18.9, @babel/traverse@npm:^7.24.5, @babel/traverse@npm:^7.25.9, @babel/traverse@npm:^7.26.5, @babel/traverse@npm:^7.26.7":- version: 7.26.7- resolution: "@babel/traverse@npm:7.26.7"+"@babel/traverse@npm:^7.16.8, @babel/traverse@npm:^7.18.9, @babel/traverse@npm:^7.24.5, @babel/traverse@npm:^7.25.9, @babel/traverse@npm:^7.26.5, @babel/traverse@npm:^7.26.8":+ version: 7.26.8+ resolution: "@babel/traverse@npm:7.26.8"dependencies:"@babel/code-frame": "npm:^7.26.2"- "@babel/generator": "npm:^7.26.5"- "@babel/parser": "npm:^7.26.7"- "@babel/template": "npm:^7.25.9"- "@babel/types": "npm:^7.26.7"+ "@babel/generator": "npm:^7.26.8"+ "@babel/parser": "npm:^7.26.8"+ "@babel/template": "npm:^7.26.8"+ "@babel/types": "npm:^7.26.8"debug: "npm:^4.3.1"globals: "npm:^11.1.0"- checksum: 10/c821c9682fe0b9edf7f7cbe9cc3e0787ffee3f73b52c13b21b463f8979950a6433f5e7e482a74348d22c0b7a05180e6f72b23eb6732328b49c59fc6388ebf6e5+ checksum: 10/2785718e54d7a243a4c1b92fe9c2cec0d3b8725b095061b8fdb9812bbcf1b94b743b39d96312644efa05692f9c2646772a8154c89625f428aa6b568cebf4ecf9languageName: nodelinkType: hard-"@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.8, @babel/types@npm:^7.18.13, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.7, @babel/types@npm:^7.24.5, @babel/types@npm:^7.25.4, @babel/types@npm:^7.25.9, @babel/types@npm:^7.26.0, @babel/types@npm:^7.26.5, @babel/types@npm:^7.26.7, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3":- version: 7.26.7- resolution: "@babel/types@npm:7.26.7"+"@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.8, @babel/types@npm:^7.18.13, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.7, @babel/types@npm:^7.24.5, @babel/types@npm:^7.25.4, @babel/types@npm:^7.25.9, @babel/types@npm:^7.26.0, @babel/types@npm:^7.26.7, @babel/types@npm:^7.26.8, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3":+ version: 7.26.8+ resolution: "@babel/types@npm:7.26.8"dependencies:"@babel/helper-string-parser": "npm:^7.25.9""@babel/helper-validator-identifier": "npm:^7.25.9"- checksum: 10/2264efd02cc261ca5d1c5bc94497c8995238f28afd2b7483b24ea64dd694cf46b00d51815bf0c87f0d0061ea221569c77893aeecb0d4b4bb254e9c2f938d7669+ checksum: 10/e6889246889706ee5e605cbfe62657c829427e0ddef0e4d18679a0d989bdb23e700b5a851d84821c2bdce3ded9ae5b9285fe1028562201b28f816e3ade6c3d0dlanguageName: nodelinkType: hard@@ -5463,6 +5522,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/aix-ppc64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/aix-ppc64@npm:0.23.1"+ conditions: os=aix & cpu=ppc64+ languageName: node+ linkType: hard+"@esbuild/aix-ppc64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/aix-ppc64@npm:0.24.2"@@ -5484,6 +5550,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/android-arm64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/android-arm64@npm:0.23.1"+ conditions: os=android & cpu=arm64+ languageName: node+ linkType: hard+"@esbuild/android-arm64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/android-arm64@npm:0.24.2"@@ -5505,6 +5578,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/android-arm@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/android-arm@npm:0.23.1"+ conditions: os=android & cpu=arm+ languageName: node+ linkType: hard+"@esbuild/android-arm@npm:0.24.2":version: 0.24.2resolution: "@esbuild/android-arm@npm:0.24.2"@@ -5526,6 +5606,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/android-x64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/android-x64@npm:0.23.1"+ conditions: os=android & cpu=x64+ languageName: node+ linkType: hard+"@esbuild/android-x64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/android-x64@npm:0.24.2"@@ -5547,6 +5634,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/darwin-arm64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/darwin-arm64@npm:0.23.1"+ conditions: os=darwin & cpu=arm64+ languageName: node+ linkType: hard+"@esbuild/darwin-arm64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/darwin-arm64@npm:0.24.2"@@ -5568,6 +5662,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/darwin-x64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/darwin-x64@npm:0.23.1"+ conditions: os=darwin & cpu=x64+ languageName: node+ linkType: hard+"@esbuild/darwin-x64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/darwin-x64@npm:0.24.2"@@ -5589,6 +5690,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/freebsd-arm64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/freebsd-arm64@npm:0.23.1"+ conditions: os=freebsd & cpu=arm64+ languageName: node+ linkType: hard+"@esbuild/freebsd-arm64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/freebsd-arm64@npm:0.24.2"@@ -5610,6 +5718,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/freebsd-x64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/freebsd-x64@npm:0.23.1"+ conditions: os=freebsd & cpu=x64+ languageName: node+ linkType: hard+"@esbuild/freebsd-x64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/freebsd-x64@npm:0.24.2"@@ -5631,6 +5746,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-arm64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-arm64@npm:0.23.1"+ conditions: os=linux & cpu=arm64+ languageName: node+ linkType: hard+"@esbuild/linux-arm64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-arm64@npm:0.24.2"@@ -5652,6 +5774,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-arm@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-arm@npm:0.23.1"+ conditions: os=linux & cpu=arm+ languageName: node+ linkType: hard+"@esbuild/linux-arm@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-arm@npm:0.24.2"@@ -5673,6 +5802,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-ia32@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-ia32@npm:0.23.1"+ conditions: os=linux & cpu=ia32+ languageName: node+ linkType: hard+"@esbuild/linux-ia32@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-ia32@npm:0.24.2"@@ -5694,6 +5830,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-loong64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-loong64@npm:0.23.1"+ conditions: os=linux & cpu=loong64+ languageName: node+ linkType: hard+"@esbuild/linux-loong64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-loong64@npm:0.24.2"@@ -5715,6 +5858,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-mips64el@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-mips64el@npm:0.23.1"+ conditions: os=linux & cpu=mips64el+ languageName: node+ linkType: hard+"@esbuild/linux-mips64el@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-mips64el@npm:0.24.2"@@ -5736,6 +5886,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-ppc64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-ppc64@npm:0.23.1"+ conditions: os=linux & cpu=ppc64+ languageName: node+ linkType: hard+"@esbuild/linux-ppc64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-ppc64@npm:0.24.2"@@ -5757,6 +5914,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-riscv64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-riscv64@npm:0.23.1"+ conditions: os=linux & cpu=riscv64+ languageName: node+ linkType: hard+"@esbuild/linux-riscv64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-riscv64@npm:0.24.2"@@ -5778,6 +5942,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-s390x@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-s390x@npm:0.23.1"+ conditions: os=linux & cpu=s390x+ languageName: node+ linkType: hard+"@esbuild/linux-s390x@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-s390x@npm:0.24.2"@@ -5799,6 +5970,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/linux-x64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/linux-x64@npm:0.23.1"+ conditions: os=linux & cpu=x64+ languageName: node+ linkType: hard+"@esbuild/linux-x64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/linux-x64@npm:0.24.2"@@ -5834,6 +6012,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/netbsd-x64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/netbsd-x64@npm:0.23.1"+ conditions: os=netbsd & cpu=x64+ languageName: node+ linkType: hard+"@esbuild/netbsd-x64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/netbsd-x64@npm:0.24.2"@@ -5855,6 +6040,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/openbsd-arm64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/openbsd-arm64@npm:0.23.1"+ conditions: os=openbsd & cpu=arm64+ languageName: node+ linkType: hard+"@esbuild/openbsd-arm64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/openbsd-arm64@npm:0.24.2"@@ -5876,6 +6068,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/openbsd-x64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/openbsd-x64@npm:0.23.1"+ conditions: os=openbsd & cpu=x64+ languageName: node+ linkType: hard+"@esbuild/openbsd-x64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/openbsd-x64@npm:0.24.2"@@ -5897,6 +6096,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/sunos-x64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/sunos-x64@npm:0.23.1"+ conditions: os=sunos & cpu=x64+ languageName: node+ linkType: hard+"@esbuild/sunos-x64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/sunos-x64@npm:0.24.2"@@ -5918,6 +6124,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/win32-arm64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/win32-arm64@npm:0.23.1"+ conditions: os=win32 & cpu=arm64+ languageName: node+ linkType: hard+"@esbuild/win32-arm64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/win32-arm64@npm:0.24.2"@@ -5939,6 +6152,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/win32-ia32@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/win32-ia32@npm:0.23.1"+ conditions: os=win32 & cpu=ia32+ languageName: node+ linkType: hard+"@esbuild/win32-ia32@npm:0.24.2":version: 0.24.2resolution: "@esbuild/win32-ia32@npm:0.24.2"@@ -5960,6 +6180,13 @@ __metadata:languageName: nodelinkType: hard+"@esbuild/win32-x64@npm:0.23.1":+ version: 0.23.1+ resolution: "@esbuild/win32-x64@npm:0.23.1"+ conditions: os=win32 & cpu=x64+ languageName: node+ linkType: hard+"@esbuild/win32-x64@npm:0.24.2":version: 0.24.2resolution: "@esbuild/win32-x64@npm:0.24.2"@@ -6217,6 +6444,13 @@ __metadata:languageName: nodelinkType: hard+"@google/generative-ai@npm:^0.21.0":+ version: 0.21.0+ resolution: "@google/generative-ai@npm:0.21.0"+ checksum: 10/68c4163b7ac0c449cdbdf83b85dd77b8d96ee2b883b0190bcb1c7e76d98b56968d3e554ff62c555c3a3814855bac3bfc1cf2035f2b7ddf71f313b5c846b60f31+ languageName: node+ linkType: hard+"@googleapis/androidpublisher@npm:^22.0.0":version: 22.0.0resolution: "@googleapis/androidpublisher@npm:22.0.0"@@ -8822,6 +9056,47 @@ __metadata:languageName: nodelinkType: hard+"@napi-rs/whisper-darwin-arm64@npm:0.0.4":+ version: 0.0.4+ resolution: "@napi-rs/whisper-darwin-arm64@npm:0.0.4"+ conditions: os=darwin & cpu=arm64+ languageName: node+ linkType: hard++"@napi-rs/whisper-darwin-x64@npm:0.0.4":+ version: 0.0.4+ resolution: "@napi-rs/whisper-darwin-x64@npm:0.0.4"+ conditions: os=darwin & cpu=x64+ languageName: node+ linkType: hard++"@napi-rs/whisper-linux-x64-gnu@npm:0.0.4":+ version: 0.0.4+ resolution: "@napi-rs/whisper-linux-x64-gnu@npm:0.0.4"+ conditions: os=linux & cpu=x64 & libc=glibc+ languageName: node+ linkType: hard++"@napi-rs/whisper@npm:^0.0.4":+ version: 0.0.4+ resolution: "@napi-rs/whisper@npm:0.0.4"+ dependencies:+ "@napi-rs/whisper-darwin-arm64": "npm:0.0.4"+ "@napi-rs/whisper-darwin-x64": "npm:0.0.4"+ "@napi-rs/whisper-linux-x64-gnu": "npm:0.0.4"+ dependenciesMeta:+ "@napi-rs/whisper-darwin-arm64":+ optional: true+ "@napi-rs/whisper-darwin-x64":+ optional: true+ "@napi-rs/whisper-linux-x64-gnu":+ optional: true+ bin:+ download-whisper-model: scripts/download-ggml-model.mjs+ checksum: 10/8c181730daf0f0970bf1d6705bf181c0b83a32622d848526a50ce35e847fd3439c8a8c5abf765def814d6a9e549712897e7856a07dbc827ab37ed12551efad33+ languageName: node+ linkType: hard+"@napi-rs/xattr-android-arm-eabi@npm:1.0.1":version: 1.0.1resolution: "@napi-rs/xattr-android-arm-eabi@npm:1.0.1"@@ -14501,6 +14776,20 @@ __metadata:languageName: nodelinkType: hard+"@tailwindcss/vite@npm:^4.0.6":+ version: 4.0.6+ resolution: "@tailwindcss/vite@npm:4.0.6"+ dependencies:+ "@tailwindcss/node": "npm:^4.0.6"+ "@tailwindcss/oxide": "npm:^4.0.6"+ lightningcss: "npm:^1.29.1"+ tailwindcss: "npm:4.0.6"+ peerDependencies:+ vite: ^5.2.0 || ^6+ checksum: 10/b01050554d1b8b0234ad71fdca4272a6f6f084541b298114a3e14f91ca081083f656f67acc779ecb5dc573317b736c06fab9358905395f587ff51dad5eb8ccff+ languageName: node+ linkType: hard+"@tanstack/react-table@npm:^8.20.5":version: 8.21.2resolution: "@tanstack/react-table@npm:8.21.2"@@ -14806,7 +15095,7 @@ __metadata:languageName: nodelinkType: hard-"@types/babel__core@npm:^7.18.0":+"@types/babel__core@npm:^7.18.0, @types/babel__core@npm:^7.20.5":version: 7.20.5resolution: "@types/babel__core@npm:7.20.5"dependencies:@@ -15038,6 +15327,15 @@ __metadata:languageName: nodelinkType: hard+"@types/estree-jsx@npm:^1.0.0":+ version: 1.0.5+ resolution: "@types/estree-jsx@npm:1.0.5"+ dependencies:+ "@types/estree": "npm:*"+ checksum: 10/a028ab0cd7b2950168a05c6a86026eb3a36a54a4adfae57f13911d7b49dffe573d9c2b28421b2d029b49b3d02fcd686611be2622dc3dad6d9791166c083f6008+ languageName: node+ linkType: hard+"@types/estree@npm:*, @types/estree@npm:1.0.6, @types/estree@npm:^1.0.0, @types/estree@npm:^1.0.6":version: 1.0.6resolution: "@types/estree@npm:1.0.6"@@ -15081,7 +15379,7 @@ __metadata:languageName: nodelinkType: hard-"@types/express@npm:^4.17.13, @types/express@npm:^4.17.21":+"@types/express@npm:^4, @types/express@npm:^4.17.13, @types/express@npm:^4.17.21":version: 4.17.21resolution: "@types/express@npm:4.17.21"dependencies:@@ -15102,7 +15400,7 @@ __metadata:languageName: nodelinkType: hard-"@types/fs-extra@npm:^11.0.4":+"@types/fs-extra@npm:^11, @types/fs-extra@npm:^11.0.4":version: 11.0.4resolution: "@types/fs-extra@npm:11.0.4"dependencies:@@ -15121,6 +15419,13 @@ __metadata:languageName: nodelinkType: hard+"@types/gensync@npm:^1.0.0":+ version: 1.0.4+ resolution: "@types/gensync@npm:1.0.4"+ checksum: 10/99c3aa0d3f1198973c7e51bea5947b815f3338ce89ce09a39ac8abb41cd844c5b95189da254ea45e50a395fe25fd215664d8ca76c5438814963597afb01f686e+ languageName: node+ linkType: hard+"@types/graphql-upload@npm:^17.0.0":version: 17.0.0resolution: "@types/graphql-upload@npm:17.0.0"@@ -15409,6 +15714,15 @@ __metadata:languageName: nodelinkType: hard+"@types/multer@npm:^1":+ version: 1.4.12+ resolution: "@types/multer@npm:1.4.12"+ dependencies:+ "@types/express": "npm:*"+ checksum: 10/3d2b32da58ddd67f972d4ef1021492f78d65f33f936b6fb25dd461bb6cc7b03bfd1de1a11562c4310680dac8054e4398038db51767a0ffbf1fe62457b3706e95+ languageName: node+ linkType: hard+"@types/mustache@npm:^4.2.5":version: 4.2.5resolution: "@types/mustache@npm:4.2.5"@@ -15686,6 +16000,24 @@ __metadata:languageName: nodelinkType: hard+"@types/socket.io-client@npm:^3.0.0":+ version: 3.0.0+ resolution: "@types/socket.io-client@npm:3.0.0"+ dependencies:+ socket.io-client: "npm:*"+ checksum: 10/6eef7529afc1d732fb4091864aa6396a2f676ad25eda0035dd1f2c228cbcaf8520b15236ee0f2c714e061307eeab2b6c8edad637bc83a317c326ae70729d88b0+ languageName: node+ linkType: hard++"@types/socket.io@npm:^3.0.2":+ version: 3.0.2+ resolution: "@types/socket.io@npm:3.0.2"+ dependencies:+ socket.io: "npm:*"+ checksum: 10/168cf77e48b466582bd69d1c25930bb657225e742288d658a03a1e8af254350daa61f9441ec2be0d8354f1dcc747966e4342000bce875549fe54ec03e2fafec3+ languageName: node+ linkType: hard+"@types/sockjs@npm:^0.3.36":version: 0.3.36resolution: "@types/sockjs@npm:0.3.36"@@ -15761,6 +16093,13 @@ __metadata:languageName: nodelinkType: hard+"@types/unist@npm:^2.0.0":+ version: 2.0.11+ resolution: "@types/unist@npm:2.0.11"+ checksum: 10/6d436e832bc35c6dde9f056ac515ebf2b3384a1d7f63679d12358766f9b313368077402e9c1126a14d827f10370a5485e628bf61aa91117cf4fc882423191a4e+ languageName: node+ linkType: hard+"@types/uuid@npm:^10.0.0":version: 10.0.0resolution: "@types/uuid@npm:10.0.0"@@ -16053,6 +16392,21 @@ __metadata:languageName: nodelinkType: hard+"@vitejs/plugin-react@npm:^4.3.4":+ version: 4.3.4+ resolution: "@vitejs/plugin-react@npm:4.3.4"+ dependencies:+ "@babel/core": "npm:^7.26.0"+ "@babel/plugin-transform-react-jsx-self": "npm:^7.25.9"+ "@babel/plugin-transform-react-jsx-source": "npm:^7.25.9"+ "@types/babel__core": "npm:^7.20.5"+ react-refresh: "npm:^0.14.2"+ peerDependencies:+ vite: ^4.2.0 || ^5.0.0 || ^6.0.0+ checksum: 10/3b220908ed9b7b96a380a9c53e82fb428ca1f76b798ab59d1c63765bdff24de61b4778dd3655952b7d3d922645aea2d97644503b879aba6e3fcf467605b9913d+ languageName: node+ linkType: hard+"@vitest/browser@npm:3.0.6":version: 3.0.6resolution: "@vitest/browser@npm:3.0.6"@@ -18239,6 +18593,13 @@ __metadata:languageName: nodelinkType: hard+"character-reference-invalid@npm:^2.0.0":+ version: 2.0.1+ resolution: "character-reference-invalid@npm:2.0.1"+ checksum: 10/98d3b1a52ae510b7329e6ee7f6210df14f1e318c5415975d4c9e7ee0ef4c07875d47c6e74230c64551f12f556b4a8ccc24d9f3691a2aa197019e72a95e9297ee+ languageName: node+ linkType: hard+"chardet@npm:^0.7.0":version: 0.7.0resolution: "chardet@npm:0.7.0"@@ -18301,7 +18662,7 @@ __metadata:languageName: nodelinkType: hard-"chokidar@npm:4.0.3, chokidar@npm:^4.0.1":+"chokidar@npm:4.0.3, chokidar@npm:^4.0.1, chokidar@npm:^4.0.3":version: 4.0.3resolution: "chokidar@npm:4.0.3"dependencies:@@ -20978,6 +21339,89 @@ __metadata:languageName: nodelinkType: hard+"esbuild@npm:~0.23.0":+ version: 0.23.1+ resolution: "esbuild@npm:0.23.1"+ dependencies:+ "@esbuild/aix-ppc64": "npm:0.23.1"+ "@esbuild/android-arm": "npm:0.23.1"+ "@esbuild/android-arm64": "npm:0.23.1"+ "@esbuild/android-x64": "npm:0.23.1"+ "@esbuild/darwin-arm64": "npm:0.23.1"+ "@esbuild/darwin-x64": "npm:0.23.1"+ "@esbuild/freebsd-arm64": "npm:0.23.1"+ "@esbuild/freebsd-x64": "npm:0.23.1"+ "@esbuild/linux-arm": "npm:0.23.1"+ "@esbuild/linux-arm64": "npm:0.23.1"+ "@esbuild/linux-ia32": "npm:0.23.1"+ "@esbuild/linux-loong64": "npm:0.23.1"+ "@esbuild/linux-mips64el": "npm:0.23.1"+ "@esbuild/linux-ppc64": "npm:0.23.1"+ "@esbuild/linux-riscv64": "npm:0.23.1"+ "@esbuild/linux-s390x": "npm:0.23.1"+ "@esbuild/linux-x64": "npm:0.23.1"+ "@esbuild/netbsd-x64": "npm:0.23.1"+ "@esbuild/openbsd-arm64": "npm:0.23.1"+ "@esbuild/openbsd-x64": "npm:0.23.1"+ "@esbuild/sunos-x64": "npm:0.23.1"+ "@esbuild/win32-arm64": "npm:0.23.1"+ "@esbuild/win32-ia32": "npm:0.23.1"+ "@esbuild/win32-x64": "npm:0.23.1"+ dependenciesMeta:+ "@esbuild/aix-ppc64":+ optional: true+ "@esbuild/android-arm":+ optional: true+ "@esbuild/android-arm64":+ optional: true+ "@esbuild/android-x64":+ optional: true+ "@esbuild/darwin-arm64":+ optional: true+ "@esbuild/darwin-x64":+ optional: true+ "@esbuild/freebsd-arm64":+ optional: true+ "@esbuild/freebsd-x64":+ optional: true+ "@esbuild/linux-arm":+ optional: true+ "@esbuild/linux-arm64":+ optional: true+ "@esbuild/linux-ia32":+ optional: true+ "@esbuild/linux-loong64":+ optional: true+ "@esbuild/linux-mips64el":+ optional: true+ "@esbuild/linux-ppc64":+ optional: true+ "@esbuild/linux-riscv64":+ optional: true+ "@esbuild/linux-s390x":+ optional: true+ "@esbuild/linux-x64":+ optional: true+ "@esbuild/netbsd-x64":+ optional: true+ "@esbuild/openbsd-arm64":+ optional: true+ "@esbuild/openbsd-x64":+ optional: true+ "@esbuild/sunos-x64":+ optional: true+ "@esbuild/win32-arm64":+ optional: true+ "@esbuild/win32-ia32":+ optional: true+ "@esbuild/win32-x64":+ optional: true+ bin:+ esbuild: bin/esbuild+ checksum: 10/f55fbd0bfb0f86ce67a6d2c6f6780729d536c330999ecb9f5a38d578fb9fda820acbbc67d6d1d377eed8fed50fc38f14ff9cb014f86dafab94269a7fb2177018+ languageName: node+ linkType: hard+"escalade@npm:^3.1.1, escalade@npm:^3.2.0":version: 3.2.0resolution: "escalade@npm:3.2.0"@@ -21330,6 +21774,13 @@ __metadata:languageName: nodelinkType: hard+"estree-util-is-identifier-name@npm:^3.0.0":+ version: 3.0.0+ resolution: "estree-util-is-identifier-name@npm:3.0.0"+ checksum: 10/cdc9187614fdb269d714eddfdf72c270a79daa9ed51e259bb78527983be6dcc68da6a914ccc41175b662194c67fbd2a1cd262f85fac1eef7111cfddfaf6f77f8+ languageName: node+ linkType: hard+"estree-walker@npm:2.0.2, estree-walker@npm:^2.0.2":version: 2.0.2resolution: "estree-walker@npm:2.0.2"@@ -21542,6 +21993,15 @@ __metadata:languageName: nodelinkType: hard+"express-rate-limit@npm:^7.1.5":+ version: 7.5.0+ resolution: "express-rate-limit@npm:7.5.0"+ peerDependencies:+ express: ^4.11 || 5 || ^5.0.0-beta.1+ checksum: 10/eff34c83bf586789933a332a339b66649e2cca95c8e977d193aa8bead577d3182ac9f0e9c26f39389287539b8038890ff023f910b54ebb506a26a2ce135b92ca+ languageName: node+ linkType: hard+"express@npm:4.21.2, express@npm:^4.21.1, express@npm:^4.21.2":version: 4.21.2resolution: "express@npm:4.21.2"@@ -22220,7 +22680,7 @@ __metadata:languageName: nodelinkType: hard-"fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0":+"fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0, fs-extra@npm:^11.3.0":version: 11.3.0resolution: "fs-extra@npm:11.3.0"dependencies:@@ -23145,6 +23605,29 @@ __metadata:languageName: nodelinkType: hard+"hast-util-to-jsx-runtime@npm:^2.0.0":+ version: 2.3.2+ resolution: "hast-util-to-jsx-runtime@npm:2.3.2"+ dependencies:+ "@types/estree": "npm:^1.0.0"+ "@types/hast": "npm:^3.0.0"+ "@types/unist": "npm:^3.0.0"+ comma-separated-tokens: "npm:^2.0.0"+ devlop: "npm:^1.0.0"+ estree-util-is-identifier-name: "npm:^3.0.0"+ hast-util-whitespace: "npm:^3.0.0"+ mdast-util-mdx-expression: "npm:^2.0.0"+ mdast-util-mdx-jsx: "npm:^3.0.0"+ mdast-util-mdxjs-esm: "npm:^2.0.0"+ property-information: "npm:^6.0.0"+ space-separated-tokens: "npm:^2.0.0"+ style-to-object: "npm:^1.0.0"+ unist-util-position: "npm:^5.0.0"+ vfile-message: "npm:^4.0.0"+ checksum: 10/3d72f83e2d8c29adc6576d2c6b41479902fd51fac8cfb2b67c35fd68fcb9c25c274699442e4dee901a7ab926a0ff6851713ed5d92448ac09ae0f10daf293476c+ languageName: node+ linkType: hard+"hast-util-whitespace@npm:^3.0.0":version: 3.0.0resolution: "hast-util-whitespace@npm:3.0.0"@@ -23326,6 +23809,13 @@ __metadata:languageName: nodelinkType: hard+"html-url-attributes@npm:^3.0.0":+ version: 3.0.1+ resolution: "html-url-attributes@npm:3.0.1"+ checksum: 10/494074c2f730c5c0e517aa1b10111fb36732534a2d2b70427582c4a615472b47da472cf3a17562cc653826d378d20960f2783e0400f4f7cf0c3c2d91c6188d13+ languageName: node+ linkType: hard+"html-validate@npm:^9.0.0":version: 9.3.0resolution: "html-validate@npm:9.3.0"@@ -23894,6 +24384,13 @@ __metadata:languageName: nodelinkType: hard+"inline-style-parser@npm:0.2.4":+ version: 0.2.4+ resolution: "inline-style-parser@npm:0.2.4"+ checksum: 10/80814479d1f3c9cbd102f9de4cd6558cf43cc2e48640e81c4371c3634f1e8b6dfeb2f21063cfa31d46cc83e834c20cd59ed9eeed9bfd45ef5bc02187ad941faf+ languageName: node+ linkType: hard+"input-otp@npm:^1.4.1":version: 1.4.2resolution: "input-otp@npm:1.4.2"@@ -24021,6 +24518,23 @@ __metadata:languageName: nodelinkType: hard+"is-alphabetical@npm:^2.0.0":+ version: 2.0.1+ resolution: "is-alphabetical@npm:2.0.1"+ checksum: 10/56207db8d9de0850f0cd30f4966bf731eb82cedfe496cbc2e97e7c3bacaf66fc54a972d2d08c0d93bb679cb84976a05d24c5ad63de56fabbfc60aadae312edaa+ languageName: node+ linkType: hard++"is-alphanumerical@npm:^2.0.0":+ version: 2.0.1+ resolution: "is-alphanumerical@npm:2.0.1"+ dependencies:+ is-alphabetical: "npm:^2.0.0"+ is-decimal: "npm:^2.0.0"+ checksum: 10/87acc068008d4c9c4e9f5bd5e251041d42e7a50995c77b1499cf6ed248f971aadeddb11f239cabf09f7975ee58cac7a48ffc170b7890076d8d227b24a68663c9+ languageName: node+ linkType: hard+"is-arguments@npm:@nolyfill/is-arguments@^1":version: 1.0.29resolution: "@nolyfill/is-arguments@npm:1.0.29"@@ -24080,6 +24594,13 @@ __metadata:languageName: nodelinkType: hard+"is-decimal@npm:^2.0.0":+ version: 2.0.1+ resolution: "is-decimal@npm:2.0.1"+ checksum: 10/97132de7acdce77caa7b797632970a2ecd649a88e715db0e4dbc00ab0708b5e7574ba5903962c860cd4894a14fd12b100c0c4ac8aed445cf6f55c6cf747a4158+ languageName: node+ linkType: hard+"is-docker@npm:^2.0.0, is-docker@npm:^2.1.1":version: 2.2.1resolution: "is-docker@npm:2.2.1"@@ -24151,6 +24672,13 @@ __metadata:languageName: nodelinkType: hard+"is-hexadecimal@npm:^2.0.0":+ version: 2.0.1+ resolution: "is-hexadecimal@npm:2.0.1"+ checksum: 10/66a2ea85994c622858f063f23eda506db29d92b52580709eb6f4c19550552d4dcf3fb81952e52f7cf972097237959e00adc7bb8c9400cd12886e15bf06145321+ languageName: node+ linkType: hard+"is-inside-container@npm:^1.0.0":version: 1.0.0resolution: "is-inside-container@npm:1.0.0"@@ -26165,6 +26693,54 @@ __metadata:languageName: nodelinkType: hard+"mdast-util-mdx-expression@npm:^2.0.0":+ version: 2.0.1+ resolution: "mdast-util-mdx-expression@npm:2.0.1"+ dependencies:+ "@types/estree-jsx": "npm:^1.0.0"+ "@types/hast": "npm:^3.0.0"+ "@types/mdast": "npm:^4.0.0"+ devlop: "npm:^1.0.0"+ mdast-util-from-markdown: "npm:^2.0.0"+ mdast-util-to-markdown: "npm:^2.0.0"+ checksum: 10/70e860f8ee22c4f478449942750055d649d4380bf43b235d0710af510189d285fb057e401d20b59596d9789f4e270fce08ca892dc849676f9e3383b991d52485+ languageName: node+ linkType: hard++"mdast-util-mdx-jsx@npm:^3.0.0":+ version: 3.2.0+ resolution: "mdast-util-mdx-jsx@npm:3.2.0"+ dependencies:+ "@types/estree-jsx": "npm:^1.0.0"+ "@types/hast": "npm:^3.0.0"+ "@types/mdast": "npm:^4.0.0"+ "@types/unist": "npm:^3.0.0"+ ccount: "npm:^2.0.0"+ devlop: "npm:^1.1.0"+ mdast-util-from-markdown: "npm:^2.0.0"+ mdast-util-to-markdown: "npm:^2.0.0"+ parse-entities: "npm:^4.0.0"+ stringify-entities: "npm:^4.0.0"+ unist-util-stringify-position: "npm:^4.0.0"+ vfile-message: "npm:^4.0.0"+ checksum: 10/62cd650a522e5d72ea6afd6d4a557fc86525b802d097a29a2fbe17d22e7b97c502a580611873e4d685777fe77c6ff8d39fb6e37d026b3acbc86c3b24927f4ad9+ languageName: node+ linkType: hard++"mdast-util-mdxjs-esm@npm:^2.0.0":+ version: 2.0.1+ resolution: "mdast-util-mdxjs-esm@npm:2.0.1"+ dependencies:+ "@types/estree-jsx": "npm:^1.0.0"+ "@types/hast": "npm:^3.0.0"+ "@types/mdast": "npm:^4.0.0"+ devlop: "npm:^1.0.0"+ mdast-util-from-markdown: "npm:^2.0.0"+ mdast-util-to-markdown: "npm:^2.0.0"+ checksum: 10/05474226e163a3f407fccb5780b0d8585a95e548e5da4a85227df43f281b940c7941a9a9d4af1be4f885fe554731647addb057a728e87aa1f503ff9cc72c9163+ languageName: node+ linkType: hard+"mdast-util-phrasing@npm:^4.0.0":version: 4.1.0resolution: "mdast-util-phrasing@npm:4.1.0"@@ -27223,6 +27799,21 @@ __metadata:languageName: nodelinkType: hard+"multer@npm:^1.4.5-lts.1":+ version: 1.4.5-lts.1+ resolution: "multer@npm:1.4.5-lts.1"+ dependencies:+ append-field: "npm:^1.0.0"+ busboy: "npm:^1.0.0"+ concat-stream: "npm:^1.5.2"+ mkdirp: "npm:^0.5.4"+ object-assign: "npm:^4.1.1"+ type-is: "npm:^1.6.4"+ xtend: "npm:^4.0.0"+ checksum: 10/957c09956f3b7f79d8586cac5e2a50e9a5c3011eb841667b5e4590c5f31d9464f5b46aecd399c83e183a15b88b019cccf0e4fa5620db40bf16b9e3af7fab3ac6+ languageName: node+ linkType: hard+"multicast-dns@npm:^7.2.5":version: 7.2.5resolution: "multicast-dns@npm:7.2.5"@@ -28026,7 +28617,7 @@ __metadata:languageName: nodelinkType: hard-"openai@npm:^4.83.0":+"openai@npm:^4.83.0, openai@npm:^4.85.1":version: 4.85.1resolution: "openai@npm:4.85.1"dependencies:@@ -28387,6 +28978,21 @@ __metadata:languageName: nodelinkType: hard+"parse-entities@npm:^4.0.0":+ version: 4.0.2+ resolution: "parse-entities@npm:4.0.2"+ dependencies:+ "@types/unist": "npm:^2.0.0"+ character-entities-legacy: "npm:^3.0.0"+ character-reference-invalid: "npm:^2.0.0"+ decode-named-character-reference: "npm:^1.0.0"+ is-alphanumerical: "npm:^2.0.0"+ is-decimal: "npm:^2.0.0"+ is-hexadecimal: "npm:^2.0.0"+ checksum: 10/b0ce693d0b3d7ed1cea6fe814e6e077c71532695f01178e846269e9a2bc2f7ff34ca4bb8db80b48af0451100f25bb010df6591c9bb6306e4680ccb423d1e4038+ languageName: node+ linkType: hard+"parse-filepath@npm:^1.0.2":version: 1.0.2resolution: "parse-filepath@npm:1.0.2"@@ -29982,6 +30588,27 @@ __metadata:languageName: nodelinkType: hard+"react-markdown@npm:^9.0.3":+ version: 9.0.3+ resolution: "react-markdown@npm:9.0.3"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ devlop: "npm:^1.0.0"+ hast-util-to-jsx-runtime: "npm:^2.0.0"+ html-url-attributes: "npm:^3.0.0"+ mdast-util-to-hast: "npm:^13.0.0"+ remark-parse: "npm:^11.0.0"+ remark-rehype: "npm:^11.0.0"+ unified: "npm:^11.0.0"+ unist-util-visit: "npm:^5.0.0"+ vfile: "npm:^6.0.0"+ peerDependencies:+ "@types/react": ">=18"+ react: ">=18"+ checksum: 10/b97eb9a61762762043263286ece030bd878acabe24bbf433767a9518cb18e434e26a75b1b810a7cb966e304ddb4e16bd4a15edcc808113b11b4fb85a68d99e8d+ languageName: node+ linkType: hard+"react-paginate@npm:^8.2.0":version: 8.3.0resolution: "react-paginate@npm:8.3.0"@@ -30002,6 +30629,13 @@ __metadata:languageName: nodelinkType: hard+"react-refresh@npm:^0.14.2":+ version: 0.14.2+ resolution: "react-refresh@npm:0.14.2"+ checksum: 10/512abf97271ab8623486061be04b608c39d932e3709f9af1720b41573415fa4993d0009fa5138b6705b60a98f4102f744d4e26c952b14f41a0e455521c6be4cc+ languageName: node+ linkType: hard+"react-refresh@npm:^0.16.0":version: 0.16.0resolution: "react-refresh@npm:0.16.0"@@ -30502,6 +31136,19 @@ __metadata:languageName: nodelinkType: hard+"remark-rehype@npm:^11.0.0":+ version: 11.1.1+ resolution: "remark-rehype@npm:11.1.1"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ "@types/mdast": "npm:^4.0.0"+ mdast-util-to-hast: "npm:^13.0.0"+ unified: "npm:^11.0.0"+ vfile: "npm:^6.0.0"+ checksum: 10/39404bd19c57b2b69660be7e3d587ddb2240495845d42fad3bcc506c9c132d07abacb0a20182b73c530857b2da0c463ad5658382b448243ce432152ab49af08d+ languageName: node+ linkType: hard+"remark-stringify@npm:^11.0.0":version: 11.0.0resolution: "remark-stringify@npm:11.0.0"@@ -31713,7 +32360,7 @@ __metadata:languageName: nodelinkType: hard-"socket.io-client@npm:^4.8.1":+"socket.io-client@npm:*, socket.io-client@npm:^4.7.4, socket.io-client@npm:^4.8.1":version: 4.8.1resolution: "socket.io-client@npm:4.8.1"dependencies:@@ -31735,7 +32382,7 @@ __metadata:languageName: nodelinkType: hard-"socket.io@npm:4.8.1, socket.io@npm:^4.8.1":+"socket.io@npm:*, socket.io@npm:4.8.1, socket.io@npm:^4.7.4, socket.io@npm:^4.8.1":version: 4.8.1resolution: "socket.io@npm:4.8.1"dependencies:@@ -32370,6 +33017,15 @@ __metadata:languageName: nodelinkType: hard+"style-to-object@npm:^1.0.0":+ version: 1.0.8+ resolution: "style-to-object@npm:1.0.8"+ dependencies:+ inline-style-parser: "npm:0.2.4"+ checksum: 10/530b067325e3119bfaf75bdbe25cc86b02b559db00d881a74b98a2d5bb10ac953d1b455ed90c825963cf3b4bdaa1bda45f406d78d987391434b8d8ab3835df4e+ languageName: node+ linkType: hard+"styled-jsx@npm:5.1.6":version: 5.1.6resolution: "styled-jsx@npm:5.1.6"@@ -32547,7 +33203,7 @@ __metadata:languageName: nodelinkType: hard-"swr@npm:2.3.2, swr@npm:^2.2.5":+"swr@npm:2.3.2, swr@npm:^2.2.5, swr@npm:^2.3.2":version: 2.3.2resolution: "swr@npm:2.3.2"dependencies:@@ -32616,7 +33272,7 @@ __metadata:languageName: nodelinkType: hard-"tailwindcss@npm:4.0.6, tailwindcss@npm:^4.0.0":+"tailwindcss@npm:4.0.6, tailwindcss@npm:^4.0.0, tailwindcss@npm:^4.0.6":version: 4.0.6resolution: "tailwindcss@npm:4.0.6"checksum: 10/b1141d98730b89164a38dab2c6fe0150a0910ab4ab45865d345b1fbf9eb9c0893f8cbe1392b7bcf59e52eb9f2e9750970b1d912f2ac4458e4b9e4cabe4518798@@ -33223,6 +33879,22 @@ __metadata:languageName: nodelinkType: hard+"tsx@npm:^4.19.2":+ version: 4.19.2+ resolution: "tsx@npm:4.19.2"+ dependencies:+ esbuild: "npm:~0.23.0"+ fsevents: "npm:~2.3.3"+ get-tsconfig: "npm:^4.7.5"+ dependenciesMeta:+ fsevents:+ optional: true+ bin:+ tsx: dist/cli.mjs+ checksum: 10/4c5610ed1fb2f80d766681f8ac7827e1e8118dfe354c18f74800691f3ef1e9ed676a29842ab818806bcf8613cdc97c6af84b5645e768ddb7f4b0527b9100deda+ languageName: node+ linkType: hard+"tunnel-agent@npm:^0.6.0":version: 0.6.0resolution: "tunnel-agent@npm:0.6.0"