Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat: add exact audio and screen overlay tracks

kashorinee2c8b5f5a2f
diff --git a/README.md b/README.md
index 82686ac1..24aebd62 100644
--- a/README.md
+++ b/README.md
@@ -213,6 +213,7 @@ The domain-neutral Run/Runtime foundation, developer inspection path and replace
Endpoint environments are implemented. Current work is split between preparing trusted-developer
distribution and completing the video-domain packages needed by real deliveries. Fresh generation,
historical reuse and previews remain ordinary explicit `.svrun` choices rather than a second
-workflow layer. Text, Caption and B-roll already execute as vertical slices; their full visual
-breadth, Ranking and the final video compatibility freeze remain deferred. See the
+workflow layer. Caption, exact Audio and self-contained Screen Overlay execute; Text and Media
+remain vertical slices, while their full visual breadth, Deck, Ranking and the final video
+compatibility freeze remain active work. See the
[implementation status](docs/implementation-status.md) and [roadmap](docs/roadmap.md).
diff --git a/examples/talking-film-graph-check/svml.packages.lock b/examples/talking-film-graph-check/svml.packages.lock
index d559f33b..9b217405 100644
--- a/examples/talking-film-graph-check/svml.packages.lock
+++ b/examples/talking-film-graph-check/svml.packages.lock
@@ -9,7 +9,7 @@
{
"name": "@narratage/broll",
"version": "0.0.0-dev",
- "digest": "sha256:f058e3161f5ab570e5f180d548ef31e8c70945e8f37d7530caced1d9db31e368"
+ "digest": "sha256:3b29285798f91a041a02b19abea52c09e7d954541186a51ea752368d0f5241a2"
},
{
"name": "@narratage/caption-fine",
@@ -24,7 +24,7 @@
{
"name": "@narratage/caption",
"version": "0.0.0-dev",
- "digest": "sha256:6c33baaaee4cb439cd2a0592504bc29a1f7cfc3fa3c663820637fb45e6523991"
+ "digest": "sha256:5853264aff66206c602d427ab31bfce5f44814f061e4fb46875d4176afd0ce7d"
},
{
"name": "@narratage/component-kit",
@@ -34,7 +34,7 @@
{
"name": "@narratage/composition",
"version": "0.0.0-dev",
- "digest": "sha256:7184d720c545663536476115bfc2004f92625eacdeeeac5958f0c087e5fa7d5a"
+ "digest": "sha256:b02b20d71e8ee4006056cb05d567cb4087971aab2d7f7823357c413b1319325e"
},
{
"name": "@narratage/core",
@@ -69,12 +69,12 @@
{
"name": "@narratage/hyperframes",
"version": "0.0.0-dev",
- "digest": "sha256:48f6b2bb2f90ea9f5d11fb8b1d94b5e7ccdb552dd5eb4f08cdbb5631c69de8c0"
+ "digest": "sha256:bb074d98e7f4a43adce3ea2860486798f9a5be6a30c5e412adec8be6cf5fd031"
},
{
"name": "@narratage/media-pipeline",
"version": "0.0.0-dev",
- "digest": "sha256:4ceb5c9ef785e456f3807809b0384cbd40d463c4e5f1878375dcc8b017799922"
+ "digest": "sha256:7da4f1b8f84fa6c7ef3a463ade22eabd50b37857fa8d9d3b7813dd34b95de714"
},
{
"name": "@narratage/media",
@@ -94,7 +94,7 @@
{
"name": "@narratage/program-space",
"version": "0.0.0-dev",
- "digest": "sha256:3a321f064e7f763ae056662ed07bbe4b0fdf1b1628502891c4c73beb5a20edd6"
+ "digest": "sha256:18ff5d1109b91ac83733c69ca5d0d5162a07a6f1e99d1473128c12382d116eaf"
},
{
"name": "@narratage/protocol",
@@ -139,17 +139,17 @@
{
"name": "@narratage/spatial",
"version": "0.0.0-dev",
- "digest": "sha256:6ca15f348ffb032d9862aa4f476ff7910e26e23017f459d7502ddd71bfd1a940"
+ "digest": "sha256:f0bc551c5ccf6897181e05aa89e2265dbe98f8a71e02cd2b10cb5f946a7037a3"
},
{
"name": "@narratage/speech-alignment",
"version": "0.0.0-dev",
- "digest": "sha256:9f54aefc38b06344902d0a9f1bb0e76f03cd9e9e29d1bd8d79263979c73da6d2"
+ "digest": "sha256:98d4d047d49998e6a7370f9e44bb8c3fea7576225068a5a8c0144bfa616547d8"
},
{
"name": "@narratage/speech-basis",
"version": "0.0.0-dev",
- "digest": "sha256:9d4ed7feb7b5b3acf6007047a4aa347b458f30fa1763ef680670ea9f7d7dc7f4"
+ "digest": "sha256:7df71fda858bf23f32a563e7ca308bf98a8293a63146b8ba341e769fe9de62d6"
},
{
"name": "@narratage/speech-evidence",
@@ -159,12 +159,12 @@
{
"name": "@narratage/speech-spine",
"version": "0.0.0-dev",
- "digest": "sha256:d7f8c6b8c274a78b7c62c67e2dcf4b5010d4fb74f04690b17649df0183ec6efb"
+ "digest": "sha256:5b7fde204276f70d5b6f5b141055f4b77de6b13c5cd204b41050910aaeee4b0b"
},
{
"name": "@narratage/speech",
"version": "0.0.0-dev",
- "digest": "sha256:1296f5241ab87621cbed2019ffd09e51a2ecd79a49848715ade628a9d722d8c1"
+ "digest": "sha256:0a6efec25a5f16e1f33dd4683955ccd26fc56f0471018ce87245dbb97a4a4a16"
},
{
"name": "@narratage/svs",
@@ -174,7 +174,7 @@
{
"name": "@narratage/temporal",
"version": "0.0.0-dev",
- "digest": "sha256:86f27c2e4dde016f871c404b603a41adb0b551913163eb7ff00ad969e7296886"
+ "digest": "sha256:6bae9e338f618a2e40973f8ebbd37e0b3815b2604276f06a77ba7f5fa590d6c3"
},
{
"name": "@narratage/text-track",
@@ -199,7 +199,7 @@
{
"name": "@narratage/whisperx",
"version": "0.0.0-dev",
- "digest": "sha256:df11edd20f992955c1d0851828e1df94dcfefeecf8ece037a0f5a0d740fe72ee"
+ "digest": "sha256:a72627ed3b15fa02239ba7670b03cd581dfa05ef92451fbd04dff7a5e3b5bb56"
}
],
"packages": [
@@ -218,8 +218,8 @@
"name": "@narratage/broll",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:29d590f5b5b125db139773046ce48eba938c675f39ed2b1cdda7e24bff1ea75a",
- "closureDigest": "sha256:0e66298d657de9705e2fd4ff6807f9f840a133c182698c4428b9071b32375a67"
+ "facetsDigest": "sha256:61f962f6ce0478a71baafc4e043ca9e977d746914126a2363047ca14ffa7db4c",
+ "closureDigest": "sha256:9a91f192c5bd38044d51ed58e1c23d5ca742c098baaf049c52db583f9f1e38f6"
},
{
"specifier": "@narratage/caption",
@@ -227,8 +227,8 @@
"name": "@narratage/caption",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:453e1a9e707850357a1d076387b11e4e70e416ebbed1e220817519dec3467344",
- "closureDigest": "sha256:4e6159cbb484bc3ff42aded8655b6f48680ccfd35cbc4a0b5a7703f90ad31187"
+ "facetsDigest": "sha256:83e25dd21523d0986c23ec090b17bef9b1b8e01414b966caaebde4f238399648",
+ "closureDigest": "sha256:3a29f5d83eea4f3f145efc4a0ed71632b40b047a55369ec9e268994ab1cbd7c9"
},
{
"specifier": "@narratage/caption-fine",
@@ -236,8 +236,8 @@
"name": "@narratage/caption-fine",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:d88c93d7dd955581dc4137a65e32a4ae1cf22b7c4ee2cbbb496b8b9f71325f4b",
- "closureDigest": "sha256:3f2310ac164d73aae783b6f1e2d41483c1417bf75cb44c35e1d1c6fe64da4a17"
+ "facetsDigest": "sha256:1364a1a82036a5506c658a4482ae6351a3f3a9b7f20e53baf6a717a7a03281cd",
+ "closureDigest": "sha256:996ea794bc4df1de51020e33514d5a1aa5736c5a7b1d873eae9f30f35f9b035a"
},
{
"specifier": "@narratage/caption-gemini",
@@ -245,8 +245,8 @@
"name": "@narratage/caption-gemini",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:9d363283f203fcac1403ab88927c0232a1000dc1db330faee9e40ce63bd250fb",
- "closureDigest": "sha256:39f07b712f705487e17af373879c949d4c795ea29b437be49d5bf0d2038bf55e"
+ "facetsDigest": "sha256:e3a0c9a52ab4ad3012b4c35983a6a35bd6c44b81276d2b025aff505bff302d2c",
+ "closureDigest": "sha256:3c3f55d03c860687ff472dbb499b6b9bf64e0381bf9fc01d1be9b81e79de0b12"
},
{
"specifier": "@narratage/composition",
@@ -254,8 +254,8 @@
"name": "@narratage/composition",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:235c69438be252eae6cc91b8d8cdaeadf5e074322c211c923c7e5a1a97d047f8",
- "closureDigest": "sha256:76678cf3a82fe486488b4f5b993ef96f2488b5223238a22c57e1e450fc09d336"
+ "facetsDigest": "sha256:63033490f1be1fb90998152809079acf6ae1a20e359146ef1d1bbbfc8f8e608f",
+ "closureDigest": "sha256:feaeb83ebb2c6150eebe666ee7bd15f234ab8ceba9f083b27b37c5266acb8f7b"
},
{
"specifier": "@narratage/estimate",
@@ -263,8 +263,8 @@
"name": "@narratage/estimate",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:195e4840e8238866ccd6ae9362e990e124bf69de41fba71758436f1959609fbb",
- "closureDigest": "sha256:323a82a2db80e6ba21178dd278e4a391552bcb0cb54d3bb54eb699e5cf1adfd9"
+ "facetsDigest": "sha256:2bae43e32fb4e857294c13363b7d834d81bbfdb2992534c8418b7ca1de5e9cb4",
+ "closureDigest": "sha256:ddf4ea9e7db521f3a786f5bb45818df8a42643328956510d66cceeda4237fe17"
},
{
"specifier": "@narratage/film",
@@ -272,8 +272,8 @@
"name": "@narratage/film",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:efee0fdfbfe306cac9a4d79122c69b594bb3e56e040ec618481ea638da97013c",
- "closureDigest": "sha256:b61b788768b556160cddb481c35e1535d91f9d5e9337deb0506dbfa6afe422dd"
+ "facetsDigest": "sha256:c05f05c0f91cc84e9393749ef03cab2fd261d326e4f03d54747f8c132aecd36f",
+ "closureDigest": "sha256:6a25be2681ce047899cfa06e7cf6af7544e60f43477d99d93a4818f1a4a2fb30"
},
{
"specifier": "@narratage/generation",
@@ -290,8 +290,8 @@
"name": "@narratage/hyperframes",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:a556fa46f3b8466f96041e630a7b54a07028ee2a5a49983af6148c917516a226",
- "closureDigest": "sha256:56cb214ee86a115c456a937a606f7b4cbbadbef70c32915b9453d99855becb6c"
+ "facetsDigest": "sha256:be96ac7d0e7b980426fa5596272b326453a819c99ceb581d9280d76f0d58c009",
+ "closureDigest": "sha256:9614017ee055d0f166a8a73e0e75938a4ccebec4c40f83c33ac39317b7f94d39"
},
{
"specifier": "@narratage/media",
@@ -308,8 +308,8 @@
"name": "@narratage/media-pipeline",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:8db585fd108e36178a39ff2f44d77897cfd18cb6bc1870b793d85f3b18e30792",
- "closureDigest": "sha256:26cd5ffacebc0e5af04992c28223ad2b275aeea425ea28a95d06612024bf1103"
+ "facetsDigest": "sha256:295304e9d68a8e02bf8ef51b0ebb2a4828337b8ddb665e9dee360c9c965c8ba4",
+ "closureDigest": "sha256:1aecf7ff98e966329cd1fa8bef018739b6ea0de30a53b3ef2e66e70ce062571a"
},
{
"specifier": "@narratage/narrative",
@@ -327,7 +327,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:5fa9b120e8d37ca2edd478e3388c3dc4e398113e74c6589c6d25ef604a1d348a",
- "closureDigest": "sha256:802f9ab8cf9bd284b8df0ef8ab78fc780339b175996d3626e5ba30f18f5349f1"
+ "closureDigest": "sha256:86cca852dc5774c67f521da3249b8fea664b947fce1bb81c9b665d6fba7ea73c"
},
{
"specifier": "@narratage/render-hyperframes",
@@ -335,8 +335,8 @@
"name": "@narratage/render-hyperframes",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:d9b87a3622ce9153dc3a23d5601458caec1c72a8e568c821a166b945ff758868",
- "closureDigest": "sha256:f406b5fded56ce2f31d2cab2e0cdb06b0c714f1a23bdf81845913cfaee1eda25"
+ "facetsDigest": "sha256:0ab12fa11602bb87353e5d6ad8e21dc0801a9b79e1dd6a45f7f1812047d3ba84",
+ "closureDigest": "sha256:34d96f4abbfbcd795765578140162f3409a0d55f98e9b23c05ad65eb404c23f2"
},
{
"specifier": "@narratage/run-text",
@@ -362,8 +362,8 @@
"name": "@narratage/seedance",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:c3b8b4b87fa07ab54de7629380f3f3aba50c7f78413f193e55d5056bba32fd4f",
- "closureDigest": "sha256:e0c54d6e9ab97fa03029a92e8160bd03ac12287993b3063c045a351e329bd270"
+ "facetsDigest": "sha256:c0a306fdfdf2dfa9489b6bdbd00e48275e2989ce1cbd6d32bc467618c7ae13cd",
+ "closureDigest": "sha256:4bff3f4e973dc3be6317ae6d3e01e9306d4b696829ad553070a6fe9c78e6db39"
},
{
"specifier": "@narratage/semantic-map",
@@ -371,8 +371,8 @@
"name": "@narratage/semantic-map",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:0935b3fde2040ca370ba47b09da4ac7285067709cbf77885b01b74a02ed1765f",
- "closureDigest": "sha256:c5ec936affa7d20bdeb1e3f57f691c10f60a885bda1b1335325c7b6bdb98b818"
+ "facetsDigest": "sha256:3598b75d25da49ae97aae0a74e0e97d2632902c88445994264671eb4d5d59edd",
+ "closureDigest": "sha256:b5728f9b4f90b6692d18bed8aae3f52f9dc232928f7137f070af8195a9fd72d2"
},
{
"specifier": "@narratage/spatial",
@@ -381,7 +381,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:1a3004524c7cdb09be7c84dfdfac577a7e133b7f7b6e6c2301298aaad91053aa",
- "closureDigest": "sha256:beff3382fe5d2d7a237644a524b4dcca409ccf8da04100e64f0340f84506f395"
+ "closureDigest": "sha256:fe56db252c01c4ec6df7b2521e19e2405f072283711724cd82d9708460d06b9b"
},
{
"specifier": "@narratage/speech",
@@ -389,8 +389,8 @@
"name": "@narratage/speech",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:1b1a0539eba2204f733bcca51c503f1c947986444db02eb77ced761e80109d1d",
- "closureDigest": "sha256:1b24fc651047d6e940aff90a9a7976956d83276832b481764b31921431df06a3"
+ "facetsDigest": "sha256:3d05fd0f8e2238518bf0e26a8ca0b89af9ec213c8a2569f318a484bd692b18d7",
+ "closureDigest": "sha256:92017fe3ea14c52c33c7e1ef3e30946bdcdd6441512c55fd762b9fa18d025a7c"
},
{
"specifier": "@narratage/speech-alignment",
@@ -398,8 +398,8 @@
"name": "@narratage/speech-alignment",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:c9b967337f259992350626efbab9c2b56c7687d5404a82b058c79076b57e3029",
- "closureDigest": "sha256:3d2a3fff4f529755ff2da02cf68e3bce56105b999c1ee92113498f01204466f7"
+ "facetsDigest": "sha256:068f7e40d6c82e122d28e105830e8e746e7a383e9bfe8e975004d56173d47f16",
+ "closureDigest": "sha256:9a29d77058bd3bc507f1d69548888827c4c0d1b98ade5ab767a6d00e64179ec6"
},
{
"specifier": "@narratage/speech-basis",
@@ -407,8 +407,8 @@
"name": "@narratage/speech-basis",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:e072ae8af0da0900055e8bbe3e8d3dc0d321a7d95e728bd676d025c4000492cc",
- "closureDigest": "sha256:e197c04f00404b1f61d87cbe3820209ac9303ec10c3606e1223302c06b7d3c2a"
+ "facetsDigest": "sha256:a05cf6c4ed4839bd7c82aec72d414c0a969127c7f03c23d7b82860a7a6c4f831",
+ "closureDigest": "sha256:46700e5257dcb675c816a73f40674e237a76168d8ef4228e334ea36d36199200"
},
{
"specifier": "@narratage/speech-evidence",
@@ -416,8 +416,8 @@
"name": "@narratage/speech-evidence",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:c3441e4dca55aded30e49188889422969a59c9515512f8098c3c780e8337fafb",
- "closureDigest": "sha256:15fad31dfd08d44577168cce1009cef5817a31c5be55f29063dfe43d427c893f"
+ "facetsDigest": "sha256:8f6bae5441065ef2c99bbb09bcf2b9b2bf70a7b1ca38fae2211baaecff3e6539",
+ "closureDigest": "sha256:e22ab9b6ef69591df0432dd915b62f8009a158718ca9875a770c8657b83faa81"
},
{
"specifier": "@narratage/speech-spine",
@@ -425,8 +425,8 @@
"name": "@narratage/speech-spine",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:dabf22bbcf3ab13504b71209d3643b9175880f859f38971a119505214c0d58fa",
- "closureDigest": "sha256:4be19980d2f68dda18a3b10984a6c3ac1cdbe157aa2ef2a34a463bb6bff5d510"
+ "facetsDigest": "sha256:780a944f7151a67ab92fd3830c153a867544e733f100106b4b1a133408808221",
+ "closureDigest": "sha256:0c3a6e7d936edd49005e241765de4446009912e72da5ea9041bbb650d00e1cd1"
},
{
"specifier": "@narratage/svs",
@@ -443,8 +443,8 @@
"name": "@narratage/text-track",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:54dd6c09b7e39090ac844d681228b2043018851684be40fbcde8ee7c6e378484",
- "closureDigest": "sha256:517db51a395163ab01aa23642940154f1d109a1f7af56f1bb53fbe01cfb26dfb"
+ "facetsDigest": "sha256:c62ee46204b26647aea09b0110ac59724119763fcae5b7a04549b6a1e3eaa66f",
+ "closureDigest": "sha256:b7b8a63ece30b1c22853801d8ae5853a015a6173253bfb08d354c96f58e66c20"
},
{
"specifier": "@narratage/visual-ir",
@@ -461,9 +461,9 @@
"name": "@narratage/whisperx",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:a0d0b282085f5d6b0994660d39009f0d859780fe4597622f8edb6e743ffc2bae",
- "closureDigest": "sha256:747dda39abd4c81eba23f4db6be19698dfe653517bdd1fc7b9896a3839bbda3f"
+ "facetsDigest": "sha256:58bd17f5beb706ba6d4ee8c9221a85fa81fb5a65772a17747c480dd881b21cde",
+ "closureDigest": "sha256:dae2140029b7054846648d38c991857adf2160ed1afb8a3720a71f6f741d5ae2"
}
],
- "digest": "sha256:800eeda337d08eaa7cdd3199c1f83a63c7e28ae00dbaf2e35770b20b6be01dc1"
+ "digest": "sha256:b43c4e571daf47f059f942956a443159112c587f42f1655866f7f4e0d170f113"
}
diff --git a/package.json b/package.json
index c2512aed..baab1950 100644
--- a/package.json
+++ b/package.json
@@ -22,6 +22,7 @@
},
"devDependencies": {
"@narratage/artifact": "workspace:*",
+ "@narratage/audio-track": "workspace:*",
"@narratage/broll": "workspace:*",
"@narratage/caption": "workspace:*",
"@narratage/caption-fine": "workspace:*",
@@ -35,6 +36,7 @@
"@narratage/image-transform": "workspace:*",
"@narratage/local": "workspace:*",
"@narratage/media": "workspace:*",
+ "@narratage/media-track": "workspace:*",
"@narratage/media-pipeline": "workspace:*",
"@narratage/narrative": "workspace:*",
"@narratage/program-space": "workspace:*",
@@ -50,6 +52,7 @@
"@narratage/runtime-adapter": "workspace:*",
"@narratage/runtime-adapter-node": "workspace:*",
"@narratage/script": "workspace:*",
+ "@narratage/screen-overlay": "workspace:*",
"@narratage/seedance": "workspace:*",
"@narratage/seedance-speaker": "workspace:*",
"@narratage/semantic-map": "workspace:*",
diff --git a/packages/audio-track/README.md b/packages/audio-track/README.md
new file mode 100644
index 00000000..fa425408
--- /dev/null
+++ b/packages/audio-track/README.md
@@ -0,0 +1,11 @@
+# `@narratage/audio-track`
+
+Official provider-free Audio Track authoring package.
+
+It consumes explicitly selected and normalized `SynchronizedMedia`, resolves Program, Selection or
+Moment windows through `@narratage/temporal`, applies exact source trim and one-shot, loop or bounded
+pitch-preserving stretch occupancy, and lowers independent items to one ordinary peer `AudioTrack`.
+
+The public audio waist uses one exact 48 kHz sample mapping. It contains no Provider, queue, path,
+Narrative identity, privileged speech lane, automatic extraction, loudness normalization or hidden
+mixing behavior. Local and remote media Endpoints execute the same `AudioProgramPlan`.
diff --git a/packages/audio-track/package.json b/packages/audio-track/package.json
new file mode 100644
index 00000000..7f2fb20b
--- /dev/null
+++ b/packages/audio-track/package.json
@@ -0,0 +1,24 @@
+{
+ "name": "@narratage/audio-track",
+ "version": "0.0.0-dev",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "svml": {
+ "activation": "./src/activation.ts"
+ },
+ "dependencies": {
+ "@narratage/component-kit": "workspace:*",
+ "@narratage/composition": "workspace:*",
+ "@narratage/elaborator": "workspace:*",
+ "@narratage/media": "workspace:*",
+ "@narratage/narrative": "workspace:*",
+ "@narratage/program-space": "workspace:*",
+ "@narratage/protocol": "workspace:*",
+ "@narratage/semantic-map": "workspace:*",
+ "@narratage/temporal": "workspace:*",
+ "@narratage/text": "workspace:*"
+ }
+}
diff --git a/packages/audio-track/src/activation.ts b/packages/audio-track/src/activation.ts
new file mode 100644
index 00000000..ed023dc9
--- /dev/null
+++ b/packages/audio-track/src/activation.ts
@@ -0,0 +1,23 @@
+import { createTextSurfaceHostFacet } from "@narratage/text";
+import {
+ audioTrackComponent,
+ audioTrackManifest,
+ audioTrackModuleRef,
+ audioTrackSurfaceImplementationDigest,
+ decodeAudioTrackSurface,
+} from "./index.js";
+
+export const svmlPackage = {
+ format: "svml.node-package@1" as const,
+ name: "@narratage/audio-track",
+ modules: [{ manifest: audioTrackManifest, specifiers: [audioTrackModuleRef.name, `${audioTrackModuleRef.name}@1`] }],
+ components: [audioTrackComponent],
+ hostFacets: [createTextSurfaceHostFacet({
+ module: audioTrackModuleRef,
+ surface: "track",
+ mode: "structured",
+ implementationDigest: audioTrackSurfaceImplementationDigest,
+ handler: decodeAudioTrackSurface,
+ })],
+};
+export default svmlPackage;
diff --git a/packages/audio-track/src/component.ts b/packages/audio-track/src/component.ts
new file mode 100644
index 00000000..b1f52da2
--- /dev/null
+++ b/packages/audio-track/src/component.ts
@@ -0,0 +1,64 @@
+import type { ComponentPackage } from "@narratage/component-kit";
+import type { SynchronizedMedia } from "@narratage/media";
+import type { NarrativeMomentRef, NarrativeSelectionRef } from "@narratage/narrative";
+import type { ProgramSpace } from "@narratage/program-space";
+import { canonicalize } from "@narratage/protocol";
+import type { StoredValue } from "@narratage/protocol";
+import type { CompleteSemanticMap } from "@narratage/semantic-map";
+
+import { audioTrackProducers, audioTrackTypes } from "./manifest.js";
+import {
+ appendMomentAudioItem,
+ appendProgramAudioItem,
+ appendSelectionAudioItem,
+ assertAudioTrackProgram,
+ audioTrackImplementationDigests,
+ audioTrackValidatorDigests,
+ createAudioTrackSet,
+ finalizeAudioTrack,
+ renderAudioTrack,
+} from "./program.js";
+import type { AudioClipSpec, AudioTrackHeader, AudioTrackProgram, AudioTrackSet } from "./types.js";
+
+function inline<T>(value: StoredValue | undefined, label: string): T {
+ if (value?.kind !== "inline") throw new Error(`${label} must be inline.`);
+ return value.value as unknown as T;
+}
+const output = (value: unknown) => ({ kind: "inline" as const, value: canonicalize(value) });
+
+const base = (inputs: Parameters<typeof appendProgramAudioItem>) => output(appendProgramAudioItem(...inputs));
+
+export const audioTrackComponent = {
+ name: "@narratage/audio-track",
+ producers: [
+ { producer: audioTrackProducers.createSet, implementationDigest: audioTrackImplementationDigests.createSet, handler: () => ({ outputs: { set: output(createAudioTrackSet()) }, needs: {} }) },
+ { producer: audioTrackProducers.appendProgram, implementationDigest: audioTrackImplementationDigests.appendProgram, handler: ({ inputs }) => ({ outputs: { set: base([
+ inline<AudioTrackSet>(inputs.set?.value, "AudioTrackSet"), inline<AudioTrackHeader>(inputs.header?.value, "AudioTrackHeader"),
+ inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"), inline<SynchronizedMedia>(inputs.media?.value, "SynchronizedMedia"),
+ inline<AudioClipSpec>(inputs.spec?.value, "AudioClipSpec"),
+ ]) }, needs: {} }) },
+ { producer: audioTrackProducers.appendSelection, implementationDigest: audioTrackImplementationDigests.appendSelection, handler: ({ inputs }) => ({ outputs: { set: output(appendSelectionAudioItem(
+ inline<AudioTrackSet>(inputs.set?.value, "AudioTrackSet"), inline<AudioTrackHeader>(inputs.header?.value, "AudioTrackHeader"),
+ inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"), inline<SynchronizedMedia>(inputs.media?.value, "SynchronizedMedia"),
+ inline<CompleteSemanticMap>(inputs.map?.value, "CompleteSemanticMap"), inline<NarrativeSelectionRef>(inputs.selection?.value, "NarrativeSelection"),
+ inline<AudioClipSpec>(inputs.spec?.value, "AudioClipSpec"),
+ )) }, needs: {} }) },
+ { producer: audioTrackProducers.appendMoment, implementationDigest: audioTrackImplementationDigests.appendMoment, handler: ({ inputs }) => ({ outputs: { set: output(appendMomentAudioItem(
+ inline<AudioTrackSet>(inputs.set?.value, "AudioTrackSet"), inline<AudioTrackHeader>(inputs.header?.value, "AudioTrackHeader"),
+ inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"), inline<SynchronizedMedia>(inputs.media?.value, "SynchronizedMedia"),
+ inline<CompleteSemanticMap>(inputs.map?.value, "CompleteSemanticMap"), inline<NarrativeMomentRef>(inputs.moment?.value, "NarrativeMoment"),
+ inline<AudioClipSpec>(inputs.spec?.value, "AudioClipSpec"),
+ )) }, needs: {} }) },
+ { producer: audioTrackProducers.finalize, implementationDigest: audioTrackImplementationDigests.finalize, handler: ({ inputs }) => ({ outputs: { program: output(finalizeAudioTrack(
+ inline<AudioTrackSet>(inputs.set?.value, "AudioTrackSet"), inline<AudioTrackHeader>(inputs.header?.value, "AudioTrackHeader"),
+ )) }, needs: {} }) },
+ { producer: audioTrackProducers.render, implementationDigest: audioTrackImplementationDigests.render, handler: ({ inputs }) => ({ outputs: { track: output(renderAudioTrack(
+ inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"), inline<AudioTrackProgram>(inputs.program?.value, "AudioTrackProgram"),
+ )) }, needs: {} }) },
+ ],
+ validators: [{
+ type: audioTrackTypes.program,
+ implementationDigest: audioTrackValidatorDigests.program,
+ handler: ({ value }) => assertAudioTrackProgram(inline<AudioTrackProgram>(value, "AudioTrackProgram")),
+ }],
+} satisfies ComponentPackage;
diff --git a/packages/audio-track/src/fragment.ts b/packages/audio-track/src/fragment.ts
new file mode 100644
index 00000000..cbb2dd5a
--- /dev/null
+++ b/packages/audio-track/src/fragment.ts
@@ -0,0 +1,75 @@
+import { compositionTypes } from "@narratage/composition";
+import { sealGraphFragment } from "@narratage/elaborator";
+import type { FragmentOperation } from "@narratage/elaborator";
+import { mediaTypes } from "@narratage/media";
+import { narrativeTypes } from "@narratage/narrative";
+import { programSpaceTypes } from "@narratage/program-space";
+import { semanticMapTypes } from "@narratage/semantic-map";
+
+import { audioTrackProducers, audioTrackTypes } from "./manifest.js";
+
+export type AudioTrackFragmentItem =
+ | { readonly kind: "program"; readonly mediaName: string; readonly specName: string }
+ | { readonly kind: "selection"; readonly mediaName: string; readonly specName: string; readonly mapName: string; readonly sourceName: string }
+ | { readonly kind: "moment"; readonly mediaName: string; readonly specName: string; readonly mapName: string; readonly sourceName: string };
+
+const input = (name: string) => ({ kind: "fragment-input" as const, name });
+const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
+
+export function createAudioTrackFragment(items: readonly AudioTrackFragmentItem[], name: string) {
+ if (items.length === 0) throw new Error("Audio Track Fragment requires at least one Item.");
+ const inputTypes = new Map<string, (typeof audioTrackTypes.clipSpec | typeof mediaTypes.synchronized | typeof semanticMapTypes.complete | typeof narrativeTypes.selection | typeof narrativeTypes.moment)>();
+ const operations: FragmentOperation[] = [
+ { id: "audio:set:empty", producer: audioTrackProducers.createSet, inputs: {}, result: { kind: "output", name: "set" } },
+ ];
+ let current = "audio:set:empty";
+ items.forEach((item, index) => {
+ inputTypes.set(item.mediaName, mediaTypes.synchronized);
+ inputTypes.set(item.specName, audioTrackTypes.clipSpec);
+ if (item.kind !== "program") {
+ inputTypes.set(item.mapName, semanticMapTypes.complete);
+ inputTypes.set(item.sourceName, item.kind === "selection" ? narrativeTypes.selection : narrativeTypes.moment);
+ }
+ const id = `audio:set:append:${String(index + 1).padStart(4, "0")}`;
+ operations.push({
+ id,
+ producer: item.kind === "program"
+ ? audioTrackProducers.appendProgram
+ : item.kind === "selection"
+ ? audioTrackProducers.appendSelection
+ : audioTrackProducers.appendMoment,
+ inputs: {
+ set: operation(current), header: input("header"), space: input("space"),
+ media: input(item.mediaName), spec: input(item.specName),
+ ...(item.kind === "program" ? {} : {
+ map: input(item.mapName),
+ [item.kind]: input(item.sourceName),
+ }),
+ },
+ result: { kind: "output", name: "set" },
+ });
+ current = id;
+ });
+ operations.push(
+ { id: "audio:program", producer: audioTrackProducers.finalize, inputs: { set: operation(current), header: input("header") }, result: { kind: "output", name: "program" } },
+ { id: "audio:track", producer: audioTrackProducers.render, inputs: { space: input("space"), program: operation("audio:program") }, result: { kind: "output", name: "track" } },
+ );
+ const semanticInputs = ["header", "space", ...inputTypes.keys()];
+ return sealGraphFragment({
+ name,
+ inputs: [
+ { name: "header", type: audioTrackTypes.header },
+ { name: "space", type: programSpaceTypes.programSpace },
+ ...[...inputTypes].map(([inputName, type]) => ({ name: inputName, type })),
+ ],
+ operations,
+ exports: [
+ { name: "program", type: audioTrackTypes.program, root: operation("audio:program"), semanticInputs, fidelity: "exact" },
+ { name: "track", type: compositionTypes.audioTrack, root: operation("audio:track"), semanticInputs, fidelity: "exact" },
+ ],
+ });
+}
+
+export const programAudioTrackFragment = createAudioTrackFragment([
+ { kind: "program", mediaName: "media", specName: "spec" },
+], "@narratage/audio-track/one-program-item@1");
diff --git a/packages/audio-track/src/index.ts b/packages/audio-track/src/index.ts
new file mode 100644
index 00000000..6de66c55
--- /dev/null
+++ b/packages/audio-track/src/index.ts
@@ -0,0 +1,35 @@
+export { audioTrackComponent } from "./component.js";
+export { createAudioTrackFragment, programAudioTrackFragment } from "./fragment.js";
+export type { AudioTrackFragmentItem } from "./fragment.js";
+export {
+ audioClipSpecSchema,
+ audioTrackDependency,
+ audioTrackHeaderSchema,
+ audioTrackManifest,
+ audioTrackManifestDigest,
+ audioTrackModuleRef,
+ audioTrackProducers,
+ audioTrackProgramSchema,
+ audioTrackSetSchema,
+ audioTrackSurfaceImplementationDigest,
+ audioTrackTypes,
+} from "./manifest.js";
+export { decodeAudioTrackSurface } from "./surface.js";
+export {
+ appendMomentAudioItem,
+ appendProgramAudioItem,
+ appendSelectionAudioItem,
+ assertAudioClipSpec,
+ assertAudioTrackHeader,
+ assertAudioTrackProgram,
+ assertAudioTrackSet,
+ audioTrackImplementationDigests,
+ audioTrackValidatorDigests,
+ createAudioTrackSet,
+ finalizeAudioTrack,
+ renderAudioTrack,
+ sealAudioClipSpec,
+ sealAudioTrackHeader,
+ sealAudioTrackProgram,
+} from "./program.js";
+export type * from "./types.js";
diff --git a/packages/audio-track/src/manifest.ts b/packages/audio-track/src/manifest.ts
new file mode 100644
index 00000000..80cc929c
--- /dev/null
+++ b/packages/audio-track/src/manifest.ts
@@ -0,0 +1,128 @@
+import { compositionDependency, compositionTypes } from "@narratage/composition";
+import { mediaArtifactSchema, mediaDependency, mediaTypes } from "@narratage/media";
+import { narrativeDependency, narrativeTypes } from "@narratage/narrative";
+import { programSpaceDependency, programSpaceTypes } from "@narratage/program-space";
+import { digestOf } from "@narratage/protocol";
+import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@narratage/protocol";
+import { semanticMapDependency, semanticMapTypes } from "@narratage/semantic-map";
+import { temporalDependency } from "@narratage/temporal";
+
+import { audioTrackImplementationDigests, audioTrackValidatorDigests } from "./program.js";
+
+export const audioTrackModuleRef = { name: "@narratage/audio-track", version: "0.0.0-dev" } as const;
+export const audioTrackTypes = {
+ header: { module: audioTrackModuleRef, name: "AudioTrackHeader" },
+ clipSpec: { module: audioTrackModuleRef, name: "AudioClipSpec" },
+ set: { module: audioTrackModuleRef, name: "AudioTrackSet" },
+ program: { module: audioTrackModuleRef, name: "AudioTrackProgram" },
+} satisfies Record<string, TypeRef>;
+export const audioTrackProducers = {
+ createSet: { module: audioTrackModuleRef, name: "create-audio-track-set" },
+ appendProgram: { module: audioTrackModuleRef, name: "append-program-audio-item" },
+ appendSelection: { module: audioTrackModuleRef, name: "append-selection-audio-item" },
+ appendMoment: { module: audioTrackModuleRef, name: "append-moment-audio-item" },
+ finalize: { module: audioTrackModuleRef, name: "finalize-audio-track" },
+ render: { module: audioTrackModuleRef, name: "render-audio-track" },
+} satisfies Record<string, ProducerRef>;
+
+const string = { kind: "string", minLength: 1 } as const;
+const number = { kind: "number", minimum: 0 } as const;
+const integer = { kind: "number", integer: true, minimum: 0 } as const;
+const signedInteger = { kind: "number", integer: true } as const;
+const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>): ValueSchema => ({ kind: "object", fields });
+const duration: ValueSchema = { kind: "oneOf", variants: [
+ object({ unit: { schema: { kind: "literal", value: "frames" } }, value: { schema: integer } }),
+ object({ unit: { schema: { kind: "literal", value: "milliseconds" } }, value: { schema: integer } }),
+ object({ unit: { schema: { kind: "literal", value: "seconds" } }, numerator: { schema: integer }, denominator: { schema: { kind: "number", integer: true, minimum: 1 } } }),
+] };
+const signedDuration: ValueSchema = { kind: "oneOf", variants: [
+ object({ unit: { schema: { kind: "literal", value: "frames" } }, value: { schema: signedInteger } }),
+ object({ unit: { schema: { kind: "literal", value: "milliseconds" } }, value: { schema: signedInteger } }),
+ object({ unit: { schema: { kind: "literal", value: "seconds" } }, numerator: { schema: signedInteger }, denominator: { schema: { kind: "number", integer: true, minimum: 1 } } }),
+] };
+const point: ValueSchema = { kind: "oneOf", variants: [
+ ...["program.start", "program.end", "selection.start", "selection.end", "moment.cue"].map((ref) => object({
+ ref: { schema: { kind: "literal", value: ref } }, offset: { schema: signedDuration, optional: true },
+ })),
+ object({ ref: { schema: { kind: "literal", value: "absolute" } }, at: { schema: duration } }),
+] };
+const projection = object({ start: { schema: point }, end: { schema: point } });
+const occupancy: ValueSchema = { kind: "oneOf", variants: [
+ object({ mode: { schema: { kind: "literal", value: "once" } }, align: { schema: { kind: "string", enum: ["start", "end"] } } }),
+ object({ mode: { schema: { kind: "literal", value: "loop" } }, align: { schema: { kind: "string", enum: ["start", "end"] } } }),
+ object({ mode: { schema: { kind: "literal", value: "stretch" } }, minRate: { schema: { kind: "number", minimum: 0.000001, maximum: 100 } }, maxRate: { schema: { kind: "number", minimum: 0.000001, maximum: 100 } }, pitch: { schema: { kind: "literal", value: "preserve" } } }),
+] };
+const clipSpec = object({
+ contract: { schema: { kind: "literal", value: "svml.audio-clip-spec@1" } },
+ id: { schema: string },
+ projection: { schema: projection },
+ expansion: { schema: object({ kind: { schema: { kind: "string", enum: ["one", "each"] } } }) },
+ trim: { schema: object({ start: { schema: duration, optional: true }, end: { schema: duration, optional: true } }) },
+ occupancy: { schema: occupancy },
+ mix: { schema: object({ gain: { schema: number }, fadeIn: { schema: duration }, fadeOut: { schema: duration } }) },
+});
+const frameSpan = object({ startFrame: { schema: integer }, endFrameExclusive: { schema: integer } });
+const resolvedOccupancy = occupancy;
+const item = object({
+ id: { schema: string }, sourceOccurrenceId: { schema: string }, window: { schema: frameSpan },
+ source: { schema: object({ artifact: { schema: mediaArtifactSchema }, sampleFrames: { schema: { kind: "number", integer: true, minimum: 1 } } }) },
+ trim: { schema: object({ startSample: { schema: integer }, endSampleExclusive: { schema: { kind: "number", integer: true, minimum: 1 } } }) },
+ occupancy: { schema: resolvedOccupancy },
+ mix: { schema: object({ gain: { schema: number }, fadeInSamples: { schema: integer }, fadeOutSamples: { schema: integer } }) },
+});
+export const audioTrackHeaderSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.audio-track-header@1" } }, id: { schema: string },
+});
+export const audioClipSpecSchema: ValueSchema = clipSpec;
+export const audioTrackSetSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.audio-track-set@1" } }, items: { schema: { kind: "array", items: item } },
+});
+export const audioTrackProgramSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.audio-track-program@1" } }, id: { schema: string },
+ items: { schema: { kind: "array", minItems: 1, items: item } },
+});
+
+export const audioTrackSurfaceImplementationDigest = digestOf("@narratage/audio-track/track-surface@1");
+
+const validator = (locator: string, digest: ReturnType<typeof digestOf>) => ({
+ abi: "svml.type-validator@1" as const,
+ implementation: { kind: "registered" as const, locator, digest },
+});
+const baseInputs = [
+ { name: "set", type: audioTrackTypes.set }, { name: "header", type: audioTrackTypes.header },
+ { name: "space", type: programSpaceTypes.programSpace }, { name: "media", type: mediaTypes.synchronized },
+ { name: "spec", type: audioTrackTypes.clipSpec },
+] as const;
+
+export const audioTrackManifest: ModuleManifest = {
+ format: "svml.module@1",
+ name: audioTrackModuleRef.name,
+ version: audioTrackModuleRef.version,
+ dependencies: [mediaDependency, narrativeDependency, semanticMapDependency, programSpaceDependency, temporalDependency, compositionDependency],
+ types: [
+ { name: audioTrackTypes.header.name, schema: audioTrackHeaderSchema },
+ { name: audioTrackTypes.clipSpec.name, schema: audioClipSpecSchema },
+ { name: audioTrackTypes.set.name, schema: audioTrackSetSchema },
+ { name: audioTrackTypes.program.name, schema: audioTrackProgramSchema, validator: validator("@narratage/audio-track/validate-program", audioTrackValidatorDigests.program) },
+ ],
+ capabilities: [],
+ surfaces: [{
+ name: "track", tag: "Track", mode: "structured",
+ outputs: [audioTrackTypes.header, audioTrackTypes.clipSpec, audioTrackTypes.program, compositionTypes.audioTrack],
+ implementation: {
+ kind: "trusted-frontend-surface",
+ locator: "@narratage/audio-track/track-surface",
+ digest: audioTrackSurfaceImplementationDigest,
+ },
+ }],
+ producers: [
+ { name: audioTrackProducers.createSet.name, inputs: [], outputs: [{ name: "set", type: audioTrackTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/audio-track/create-set", digest: audioTrackImplementationDigests.createSet } },
+ { name: audioTrackProducers.appendProgram.name, inputs: [...baseInputs], outputs: [{ name: "set", type: audioTrackTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/audio-track/append-program", digest: audioTrackImplementationDigests.appendProgram } },
+ { name: audioTrackProducers.appendSelection.name, inputs: [...baseInputs, { name: "map", type: semanticMapTypes.complete }, { name: "selection", type: narrativeTypes.selection }], outputs: [{ name: "set", type: audioTrackTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/audio-track/append-selection", digest: audioTrackImplementationDigests.appendSelection } },
+ { name: audioTrackProducers.appendMoment.name, inputs: [...baseInputs, { name: "map", type: semanticMapTypes.complete }, { name: "moment", type: narrativeTypes.moment }], outputs: [{ name: "set", type: audioTrackTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/audio-track/append-moment", digest: audioTrackImplementationDigests.appendMoment } },
+ { name: audioTrackProducers.finalize.name, inputs: [{ name: "set", type: audioTrackTypes.set }, { name: "header", type: audioTrackTypes.header }], outputs: [{ name: "program", type: audioTrackTypes.program }], needs: [], implementation: { kind: "registered", locator: "@narratage/audio-track/finalize", digest: audioTrackImplementationDigests.finalize } },
+ { name: audioTrackProducers.render.name, inputs: [{ name: "space", type: programSpaceTypes.programSpace }, { name: "program", type: audioTrackTypes.program }], outputs: [{ name: "track", type: compositionTypes.audioTrack }], needs: [], implementation: { kind: "registered", locator: "@narratage/audio-track/render", digest: audioTrackImplementationDigests.render } },
+ ],
+};
+export const audioTrackManifestDigest = digestOf(audioTrackManifest);
+export const audioTrackDependency = { module: audioTrackModuleRef, digest: audioTrackManifestDigest } as const;
diff --git a/packages/audio-track/src/program.ts b/packages/audio-track/src/program.ts
new file mode 100644
index 00000000..85e63664
--- /dev/null
+++ b/packages/audio-track/src/program.ts
@@ -0,0 +1,317 @@
+import { assertAudioTrackIdentity, sealAudioTrack } from "@narratage/composition";
+import type { AudioClip, AudioTrack } from "@narratage/composition";
+import { verifySynchronizedMedia } from "@narratage/media";
+import type { SynchronizedMedia } from "@narratage/media";
+import type { NarrativeMomentRef, NarrativeSelectionRef } from "@narratage/narrative";
+import {
+ assertProgramSpaceIdentity,
+ programFrameSampleBoundary,
+ programSpaceSampleFrames,
+} from "@narratage/program-space";
+import type { ProgramSpace } from "@narratage/program-space";
+import { canonicalize, digestOf, isDigest } from "@narratage/protocol";
+import type { CompleteSemanticMap } from "@narratage/semantic-map";
+import {
+ projectMomentWindows,
+ projectProgramWindow,
+ projectSelectionWindows,
+ temporalDurationInSamples,
+} from "@narratage/temporal";
+import type { ProjectedOccurrence, TemporalDuration } from "@narratage/temporal";
+
+import type {
+ AudioClipSpec,
+ AudioItemProgram,
+ AudioOccupancy,
+ AudioTrackHeader,
+ AudioTrackProgram,
+ AudioTrackSet,
+} from "./types.js";
+
+export const audioTrackImplementationDigests = {
+ createSet: digestOf("@narratage/audio-track/create-set@1"),
+ appendProgram: digestOf("@narratage/audio-track/append-program@1"),
+ appendSelection: digestOf("@narratage/audio-track/append-selection@1"),
+ appendMoment: digestOf("@narratage/audio-track/append-moment@1"),
+ finalize: digestOf("@narratage/audio-track/finalize@1"),
+ render: digestOf("@narratage/audio-track/render@1"),
+} as const;
+
+export const audioTrackValidatorDigests = {
+ program: digestOf("@narratage/audio-track/validate-program@1"),
+} as const;
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function assertIdentity(value: string, label: string): void {
+ assert(/^[A-Za-z][A-Za-z0-9_.:#-]{0,191}$/u.test(value), `${label} is invalid.`);
+}
+
+function assertDuration(value: TemporalDuration, label: string, signed = false): void {
+ if (value.unit === "seconds") {
+ assert(Number.isSafeInteger(value.numerator) && (signed || value.numerator >= 0)
+ && Number.isSafeInteger(value.denominator) && value.denominator > 0, `${label} is invalid.`);
+ } else {
+ assert(Number.isSafeInteger(value.value) && (signed || value.value >= 0), `${label} is invalid.`);
+ }
+}
+
+function assertOccupancy(value: AudioOccupancy, label: string): void {
+ if (value.mode === "once" || value.mode === "loop") {
+ assert(value.align === "start" || value.align === "end", `${label} alignment is invalid.`);
+ return;
+ }
+ assert(value.mode === "stretch" && value.pitch === "preserve"
+ && Number.isFinite(value.minRate) && value.minRate > 0
+ && Number.isFinite(value.maxRate) && value.maxRate >= value.minRate && value.maxRate <= 100,
+ `${label} stretch bounds are invalid.`);
+}
+
+export function sealAudioTrackHeader(value: AudioTrackHeader): AudioTrackHeader {
+ assertAudioTrackHeader(value);
+ return canonicalize(value) as unknown as AudioTrackHeader;
+}
+
+export function assertAudioTrackHeader(value: AudioTrackHeader): void {
+ assert(value.contract === "svml.audio-track-header@1", "Unsupported AudioTrackHeader contract.");
+ assertIdentity(value.id, "AudioTrackHeader.id");
+}
+
+export function sealAudioClipSpec(value: AudioClipSpec): AudioClipSpec {
+ assertAudioClipSpec(value);
+ return canonicalize(value) as unknown as AudioClipSpec;
+}
+
+export function assertAudioClipSpec(value: AudioClipSpec): void {
+ assert(value.contract === "svml.audio-clip-spec@1", "Unsupported AudioClipSpec contract.");
+ assertIdentity(value.id, "AudioClipSpec.id");
+ assert(value.expansion.kind === "one" || value.expansion.kind === "each", "AudioClipSpec expansion is invalid.");
+ for (const point of [value.projection.start, value.projection.end]) {
+ assert(["program.start", "program.end", "selection.start", "selection.end", "moment.cue", "absolute"].includes(point.ref),
+ "AudioClipSpec projection point is invalid.");
+ if (point.ref === "absolute") assertDuration(point.at, "AudioClipSpec absolute point");
+ else if (point.offset !== undefined) assertDuration(point.offset, "AudioClipSpec point offset", true);
+ }
+ if (value.trim.start !== undefined) assertDuration(value.trim.start, "AudioClipSpec trim start");
+ if (value.trim.end !== undefined) assertDuration(value.trim.end, "AudioClipSpec trim end");
+ assertOccupancy(value.occupancy, "AudioClipSpec occupancy");
+ assert(Number.isFinite(value.mix.gain) && value.mix.gain >= 0 && value.mix.gain <= 64,
+ "AudioClipSpec gain is invalid.");
+ assertDuration(value.mix.fadeIn, "AudioClipSpec fade in");
+ assertDuration(value.mix.fadeOut, "AudioClipSpec fade out");
+}
+
+export function createAudioTrackSet(): AudioTrackSet {
+ return { contract: "svml.audio-track-set@1", items: [] };
+}
+
+export function assertAudioTrackSet(value: AudioTrackSet): void {
+ assert(value.contract === "svml.audio-track-set@1" && Array.isArray(value.items), "AudioTrackSet is invalid.");
+}
+
+function sourceFacts(media: SynchronizedMedia): AudioItemProgram["source"] {
+ verifySynchronizedMedia(media);
+ assert(media.audio !== undefined, "Audio Track source has no explicitly normalized audio member.");
+ const { artifact, sampleFrames } = media.audio;
+ return {
+ artifact: structuredClone(artifact),
+ sampleFrames,
+ };
+}
+
+function realizedItems(
+ set: AudioTrackSet,
+ header: AudioTrackHeader,
+ space: ProgramSpace,
+ media: SynchronizedMedia,
+ spec: AudioClipSpec,
+ occurrences: readonly ProjectedOccurrence[],
+): AudioTrackSet {
+ assertAudioTrackSet(set);
+ assertAudioTrackHeader(header);
+ assertProgramSpaceIdentity(space);
+ assertAudioClipSpec(spec);
+ const source = sourceFacts(media);
+ const trimStart = spec.trim.start === undefined ? 0 : temporalDurationInSamples(spec.trim.start, space);
+ const trimEnd = spec.trim.end === undefined ? source.sampleFrames : temporalDurationInSamples(spec.trim.end, space);
+ assert(trimStart >= 0 && trimStart < source.sampleFrames, `Audio Clip ${spec.id} trim start is outside its source.`);
+ assert(trimEnd > trimStart && trimEnd <= source.sampleFrames, `Audio Clip ${spec.id} trim end is outside its source.`);
+ const fadeInSamples = temporalDurationInSamples(spec.mix.fadeIn, space);
+ const fadeOutSamples = temporalDurationInSamples(spec.mix.fadeOut, space);
+ const additions = occurrences.map((occurrence) => ({
+ id: occurrence.id,
+ sourceOccurrenceId: occurrence.sourceOccurrenceId,
+ window: { ...occurrence.span },
+ source: structuredClone(source),
+ trim: { startSample: trimStart, endSampleExclusive: trimEnd },
+ occupancy: structuredClone(spec.occupancy),
+ mix: { gain: spec.mix.gain, fadeInSamples, fadeOutSamples },
+ } satisfies AudioItemProgram));
+ const ids = new Set(set.items.map((item) => item.id));
+ for (const item of additions) {
+ assert(!ids.has(item.id), `Audio Track ${header.id} already contains Item ${item.id}.`);
+ ids.add(item.id);
+ }
+ return { contract: "svml.audio-track-set@1", items: [...set.items, ...additions] };
+}
+
+export function appendProgramAudioItem(
+ set: AudioTrackSet,
+ header: AudioTrackHeader,
+ space: ProgramSpace,
+ media: SynchronizedMedia,
+ spec: AudioClipSpec,
+): AudioTrackSet {
+ assert(spec.expansion.kind === "one", `Program Audio Clip ${spec.id} must use one occurrence.`);
+ return realizedItems(set, header, space, media, spec, [projectProgramWindow({
+ itemId: spec.id,
+ space,
+ projection: spec.projection,
+ })]);
+}
+
+export function appendSelectionAudioItem(
+ set: AudioTrackSet,
+ header: AudioTrackHeader,
+ space: ProgramSpace,
+ media: SynchronizedMedia,
+ map: CompleteSemanticMap,
+ selection: NarrativeSelectionRef,
+ spec: AudioClipSpec,
+): AudioTrackSet {
+ return realizedItems(set, header, space, media, spec, projectSelectionWindows({
+ itemId: spec.id, map, selection, space, expansion: spec.expansion, projection: spec.projection,
+ }));
+}
+
+export function appendMomentAudioItem(
+ set: AudioTrackSet,
+ header: AudioTrackHeader,
+ space: ProgramSpace,
+ media: SynchronizedMedia,
+ map: CompleteSemanticMap,
+ moment: NarrativeMomentRef,
+ spec: AudioClipSpec,
+): AudioTrackSet {
+ return realizedItems(set, header, space, media, spec, projectMomentWindows({
+ itemId: spec.id, map, moment, space, expansion: spec.expansion, projection: spec.projection,
+ }));
+}
+
+function normalizeProgram(value: AudioTrackProgram): AudioTrackProgram {
+ return {
+ contract: "svml.audio-track-program@1",
+ id: value.id,
+ items: [...value.items].map((item) => structuredClone(item)).sort((left, right) => left.id.localeCompare(right.id)),
+ };
+}
+
+export function sealAudioTrackProgram(value: AudioTrackProgram): AudioTrackProgram {
+ const normalized = normalizeProgram(value);
+ assertAudioTrackProgram(normalized);
+ return canonicalize(normalized) as unknown as AudioTrackProgram;
+}
+
+export function finalizeAudioTrack(set: AudioTrackSet, header: AudioTrackHeader): AudioTrackProgram {
+ assertAudioTrackSet(set);
+ assertAudioTrackHeader(header);
+ assert(set.items.length > 0, "Audio Track requires at least one Item.");
+ return sealAudioTrackProgram({ contract: "svml.audio-track-program@1", id: header.id, items: set.items });
+}
+
+export function assertAudioTrackProgram(value: AudioTrackProgram): void {
+ assert(value.contract === "svml.audio-track-program@1", "Unsupported AudioTrackProgram contract.");
+ assertIdentity(value.id, "AudioTrackProgram.id");
+ assert(value.items.length > 0, "AudioTrackProgram requires at least one Item.");
+ const ids = new Set<string>();
+ for (const item of value.items) {
+ assertIdentity(item.id, "AudioItemProgram.id");
+ assert(!ids.has(item.id), `AudioTrackProgram contains duplicate Item ${item.id}.`);
+ ids.add(item.id);
+ assert(item.sourceOccurrenceId.length > 0, `Audio Item ${item.id} has no occurrence identity.`);
+ assert(Number.isSafeInteger(item.window.startFrame) && item.window.startFrame >= 0
+ && Number.isSafeInteger(item.window.endFrameExclusive)
+ && item.window.endFrameExclusive > item.window.startFrame, `Audio Item ${item.id} window is invalid.`);
+ assert(isDigest(item.source.artifact.digest) && item.source.artifact.mediaType === "audio/wav"
+ && Number.isSafeInteger(item.source.sampleFrames) && item.source.sampleFrames > 0,
+ `Audio Item ${item.id} source is invalid.`);
+ assert(item.trim.startSample >= 0 && item.trim.endSampleExclusive > item.trim.startSample
+ && item.trim.endSampleExclusive <= item.source.sampleFrames, `Audio Item ${item.id} trim is invalid.`);
+ assertOccupancy(item.occupancy, `Audio Item ${item.id} occupancy`);
+ assert(Number.isFinite(item.mix.gain) && item.mix.gain >= 0 && item.mix.gain <= 64
+ && Number.isSafeInteger(item.mix.fadeInSamples) && item.mix.fadeInSamples >= 0
+ && Number.isSafeInteger(item.mix.fadeOutSamples) && item.mix.fadeOutSamples >= 0,
+ `Audio Item ${item.id} mix is invalid.`);
+ }
+}
+
+function terminalClip(item: AudioItemProgram, space: ProgramSpace): AudioClip {
+ const windowStart = programFrameSampleBoundary(space, item.window.startFrame, 48_000);
+ const windowEnd = programFrameSampleBoundary(space, item.window.endFrameExclusive, 48_000);
+ const windowLength = windowEnd - windowStart;
+ const effectiveLength = item.trim.endSampleExclusive - item.trim.startSample;
+ let targetStart = windowStart;
+ let targetEnd = windowEnd;
+ let sourceStart = item.trim.startSample;
+ let sourceEnd = item.trim.endSampleExclusive;
+ let loop = false;
+ let phaseSample = 0;
+ let playbackRate = 1;
+ if (item.occupancy.mode === "once") {
+ const audibleLength = Math.min(windowLength, effectiveLength);
+ if (item.occupancy.align === "start") {
+ targetEnd = targetStart + audibleLength;
+ sourceEnd = sourceStart + audibleLength;
+ } else {
+ targetStart = targetEnd - audibleLength;
+ sourceStart = sourceEnd - audibleLength;
+ }
+ } else if (item.occupancy.mode === "loop") {
+ loop = true;
+ phaseSample = item.occupancy.align === "start"
+ ? 0
+ : (effectiveLength - (windowLength % effectiveLength)) % effectiveLength;
+ } else {
+ playbackRate = effectiveLength / windowLength;
+ assert(playbackRate >= item.occupancy.minRate && playbackRate <= item.occupancy.maxRate,
+ `Audio Item ${item.id} requires playback rate ${playbackRate}, outside authored bounds.`);
+ }
+ const audibleLength = targetEnd - targetStart;
+ assert(item.mix.fadeInSamples <= audibleLength && item.mix.fadeOutSamples <= audibleLength,
+ `Audio Item ${item.id} fade exceeds its audible interval.`);
+ return {
+ id: item.id,
+ artifact: structuredClone(item.source.artifact),
+ target: { startSample: targetStart, endSampleExclusive: targetEnd },
+ source: {
+ sampleFrames: item.source.sampleFrames,
+ startSample: sourceStart,
+ endSampleExclusive: sourceEnd,
+ loop,
+ phaseSample,
+ },
+ playbackRate,
+ pitch: "preserve",
+ gain: item.mix.gain,
+ fadeInSamples: item.mix.fadeInSamples,
+ fadeOutSamples: item.mix.fadeOutSamples,
+ };
+}
+
+export function renderAudioTrack(space: ProgramSpace, program: AudioTrackProgram): AudioTrack {
+ assertProgramSpaceIdentity(space);
+ assertAudioTrackProgram(program);
+ const totalSamples = programSpaceSampleFrames(space, 48_000);
+ const track = sealAudioTrack({
+ contract: "svml.audio-track@1",
+ id: program.id,
+ clips: program.items.map((item) => terminalClip(item, space)),
+ });
+ for (const clip of track.clips) {
+ assert(clip.target.endSampleExclusive <= totalSamples, `Audio Clip ${clip.id} is outside ProgramSpace.`);
+ }
+ assertAudioTrackIdentity(track, space);
+ return track;
+}
diff --git a/packages/audio-track/src/surface.ts b/packages/audio-track/src/surface.ts
new file mode 100644
index 00000000..a885983a
--- /dev/null
+++ b/packages/audio-track/src/surface.ts
@@ -0,0 +1,269 @@
+import { mediaTypes } from "@narratage/media";
+import { narrativeTypes } from "@narratage/narrative";
+import { programSpaceTypes } from "@narratage/program-space";
+import { semanticMapTypes } from "@narratage/semantic-map";
+import type {
+ StructuredElement,
+ StructuredSurfaceHandler,
+ SurfaceRecordDraft,
+ SurfaceResolvedReference,
+ TextAttributeValue,
+} from "@narratage/text";
+import type { TemporalDuration, TemporalPointExpression } from "@narratage/temporal";
+
+import { createAudioTrackFragment } from "./fragment.js";
+import { audioTrackTypes } from "./manifest.js";
+import { sealAudioClipSpec, sealAudioTrackHeader } from "./program.js";
+import type { AudioClipSpec, AudioOccupancy } from "./types.js";
+
+function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
+ return left.module.name === right.module.name && left.module.version === right.module.version && left.name === right.name;
+}
+
+function allowed(element: StructuredElement, names: readonly string[]): void {
+ const permit = new Set(names);
+ const unexpected = Object.keys(element.attributes).filter((name) => !permit.has(name));
+ if (unexpected.length > 0) throw new Error(`${element.name} has unsupported attributes ${unexpected.join(", ")}.`);
+}
+
+function text(element: StructuredElement, name: string, fallback?: string): string {
+ const value = element.attributes[name];
+ if (value === undefined && fallback !== undefined) return fallback;
+ if (typeof value !== "string" || !value.trim()) throw new Error(`${element.name}.${name} must be text.`);
+ return value.trim();
+}
+
+function optionalText(element: StructuredElement, name: string): string | undefined {
+ const value = element.attributes[name];
+ if (value === undefined) return undefined;
+ if (typeof value !== "string" || !value.trim()) throw new Error(`${element.name}.${name} must be text.`);
+ return value.trim();
+}
+
+function resolved(
+ raw: TextAttributeValue | undefined,
+ label: string,
+ expected: SurfaceResolvedReference["type"],
+ resolve: (path: string) => SurfaceResolvedReference | undefined,
+): SurfaceResolvedReference {
+ if (typeof raw !== "object" || raw.kind !== "reference") throw new Error(`${label} must be a reference.`);
+ const value = resolve(raw.path);
+ if (value === undefined || !sameType(value.type, expected)) throw new Error(`${label} has the wrong Type.`);
+ return value;
+}
+
+function divisor(left: number, right: number): number {
+ let a = Math.abs(left);
+ let b = Math.abs(right);
+ while (b !== 0) [a, b] = [b, a % b];
+ return a;
+}
+
+function duration(value: string, label: string): TemporalDuration {
+ const match = /^(\d+)(?:\.(\d+))?(f|ms|s)$/u.exec(value.trim());
+ if (!match) throw new Error(`${label} must be an exact duration such as 12f, 250ms or 1.5s.`);
+ const whole = Number(match[1]);
+ const fraction = match[2] ?? "";
+ const unit = match[3];
+ if (!Number.isSafeInteger(whole)) throw new Error(`${label} is outside safe arithmetic.`);
+ if (unit === "f" || unit === "ms") {
+ if (fraction.length > 0) throw new Error(`${label} ${unit} duration must be an integer.`);
+ return { unit: unit === "f" ? "frames" : "milliseconds", value: whole };
+ }
+ const scale = 10 ** fraction.length;
+ const numerator = whole * scale + (fraction.length === 0 ? 0 : Number(fraction));
+ if (!Number.isSafeInteger(numerator) || !Number.isSafeInteger(scale)) throw new Error(`${label} is outside safe arithmetic.`);
+ const gcd = divisor(numerator, scale);
+ return { unit: "seconds", numerator: numerator / gcd, denominator: scale / gcd };
+}
+
+function negate(value: TemporalDuration): TemporalDuration {
+ return value.unit === "seconds"
+ ? { ...value, numerator: -value.numerator }
+ : { ...value, value: -value.value };
+}
+
+function point(value: string, label: string): TemporalPointExpression {
+ const trimmed = value.trim();
+ const refs = ["program.start", "program.end", "selection.start", "selection.end", "moment.cue"] as const;
+ for (const ref of refs) {
+ if (trimmed === ref) return { ref };
+ const escaped = ref.replace(".", "\\.");
+ const match = new RegExp(`^${escaped}\\s*([+-])\\s*(.+)$`, "u").exec(trimmed);
+ if (match !== null) {
+ const offset = duration(match[2]!, `${label} offset`);
+ return { ref, offset: match[1] === "-" ? negate(offset) : offset };
+ }
+ }
+ return { ref: "absolute", at: duration(trimmed, label) };
+}
+
+function numeric(element: StructuredElement, name: string, fallback?: number): number {
+ const raw = optionalText(element, name);
+ if (raw === undefined && fallback !== undefined) return fallback;
+ const value = Number(raw);
+ if (!Number.isFinite(value)) throw new Error(`${element.name}.${name} must be a finite number.`);
+ return value;
+}
+
+function occupancy(element: StructuredElement): AudioOccupancy {
+ switch (text(element, "playback", "once")) {
+ case "once":
+ case "once-start": return { mode: "once", align: "start" };
+ case "once-end": return { mode: "once", align: "end" };
+ case "loop":
+ case "loop-start": return { mode: "loop", align: "start" };
+ case "loop-end": return { mode: "loop", align: "end" };
+ case "stretch": return {
+ mode: "stretch",
+ minRate: numeric(element, "min-rate"),
+ maxRate: numeric(element, "max-rate"),
+ pitch: "preserve",
+ };
+ default: throw new Error(`${element.name}.playback is unsupported.`);
+ }
+}
+
+type TemporalBinding = {
+ readonly kind: "program" | "selection" | "moment";
+ readonly projection: AudioClipSpec["projection"];
+ readonly source?: SurfaceResolvedReference;
+ readonly map?: SurfaceResolvedReference;
+};
+
+function temporalBinding(
+ element: StructuredElement,
+ resolve: (path: string) => SurfaceResolvedReference | undefined,
+): TemporalBinding {
+ const during = element.attributes.during;
+ const at = element.attributes.at;
+ const explicitStart = optionalText(element, "start");
+ const explicitEnd = optionalText(element, "end");
+ const selection = element.attributes.selection;
+ const moment = element.attributes.moment;
+ const mapRaw = element.attributes.map;
+ const forms = Number(during !== undefined) + Number(at !== undefined) + Number(explicitStart !== undefined || explicitEnd !== undefined);
+ if (forms !== 1) throw new Error(`${element.name} requires exactly one of during, at/for, or start/end.`);
+ if (during !== undefined) {
+ if (typeof during === "string") {
+ if (during.trim() !== "program") throw new Error(`${element.name}.during text must be program.`);
+ return { kind: "program", projection: { start: { ref: "program.start" }, end: { ref: "program.end" } } };
+ }
+ const source = resolved(during, `${element.name}.during`, narrativeTypes.selection, resolve);
+ const map = resolved(mapRaw, `${element.name}.map`, semanticMapTypes.complete, resolve);
+ return { kind: "selection", source, map, projection: { start: { ref: "selection.start" }, end: { ref: "selection.end" } } };
+ }
+ if (at !== undefined) {
+ const source = resolved(at, `${element.name}.at`, narrativeTypes.moment, resolve);
+ const map = resolved(mapRaw, `${element.name}.map`, semanticMapTypes.complete, resolve);
+ const forDuration = duration(text(element, "for"), `${element.name}.for`);
+ return { kind: "moment", source, map, projection: {
+ start: { ref: "moment.cue" }, end: { ref: "moment.cue", offset: forDuration },
+ } };
+ }
+ if (explicitStart === undefined || explicitEnd === undefined) throw new Error(`${element.name} explicit timing requires start and end.`);
+ if (selection !== undefined && moment !== undefined) throw new Error(`${element.name} cannot bind Selection and Moment together.`);
+ if (selection !== undefined) return {
+ kind: "selection",
+ source: resolved(selection, `${element.name}.selection`, narrativeTypes.selection, resolve),
+ map: resolved(mapRaw, `${element.name}.map`, semanticMapTypes.complete, resolve),
+ projection: { start: point(explicitStart, `${element.name}.start`), end: point(explicitEnd, `${element.name}.end`) },
+ };
+ if (moment !== undefined) return {
+ kind: "moment",
+ source: resolved(moment, `${element.name}.moment`, narrativeTypes.moment, resolve),
+ map: resolved(mapRaw, `${element.name}.map`, semanticMapTypes.complete, resolve),
+ projection: { start: point(explicitStart, `${element.name}.start`), end: point(explicitEnd, `${element.name}.end`) },
+ };
+ if (mapRaw !== undefined) throw new Error(`${element.name}.map requires Selection or Moment.`);
+ return {
+ kind: "program",
+ projection: { start: point(explicitStart, `${element.name}.start`), end: point(explicitEnd, `${element.name}.end`) },
+ };
+}
+
+export const decodeAudioTrackSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ allowed(element, ["id", "space"]);
+ const id = text(element, "id");
+ const space = resolved(element.attributes.space, `${element.name}.space`, programSpaceTypes.programSpace, resolveReference);
+ const headerId = `${id}.header`;
+ const records: SurfaceRecordDraft[] = [{
+ id: headerId,
+ type: audioTrackTypes.header,
+ value: { kind: "inline", value: sealAudioTrackHeader({ contract: "svml.audio-track-header@1", id }) },
+ range: element.range,
+ }];
+ const fragmentItems: Parameters<typeof createAudioTrackFragment>[0][number][] = [];
+ const inputs: Record<string, { kind: "record"; id: string } | { kind: "component-output"; component: string; output: string }> = {
+ header: { kind: "record", id: headerId },
+ space: space.ref,
+ };
+ let itemIndex = 0;
+ for (const child of element.children) {
+ if (child.kind === "text") {
+ if (child.value.trim()) throw new Error(`${element.name} accepts only Clip children.`);
+ continue;
+ }
+ if (!child.name.endsWith(":Clip") && child.name !== "Clip") throw new Error(`${element.name} accepts only Clip children.`);
+ if (child.children.some((node) => node.kind === "element" || node.value.trim())) throw new Error(`${child.name} must be empty.`);
+ allowed(child, [
+ "id", "source", "during", "at", "for", "start", "end", "selection", "moment", "map", "occurrences",
+ "trim-start", "trim-end", "playback", "min-rate", "max-rate", "gain", "fade-in", "fade-out",
+ ]);
+ itemIndex += 1;
+ const suffix = String(itemIndex).padStart(4, "0");
+ const clipId = optionalText(child, "id") ?? `${id}.clip.${suffix}`;
+ const source = resolved(child.attributes.source, `${child.name}.source`, mediaTypes.synchronized, resolveReference);
+ const binding = temporalBinding(child, resolveReference);
+ const playback = occupancy(child);
+ if (playback.mode !== "stretch" && (child.attributes["min-rate"] !== undefined || child.attributes["max-rate"] !== undefined)) {
+ throw new Error(`${child.name} rate bounds require stretch playback.`);
+ }
+ const expansion = text(child, "occurrences", "one");
+ if (expansion !== "one" && expansion !== "each") throw new Error(`${child.name}.occurrences must be one or each.`);
+ const clipSpec = sealAudioClipSpec({
+ contract: "svml.audio-clip-spec@1",
+ id: clipId,
+ projection: binding.projection,
+ expansion: { kind: expansion },
+ trim: {
+ ...(optionalText(child, "trim-start") === undefined ? {} : { start: duration(optionalText(child, "trim-start")!, `${child.name}.trim-start`) }),
+ ...(optionalText(child, "trim-end") === undefined ? {} : { end: duration(optionalText(child, "trim-end")!, `${child.name}.trim-end`) }),
+ },
+ occupancy: playback,
+ mix: {
+ gain: numeric(child, "gain", 1),
+ fadeIn: duration(text(child, "fade-in", "0f"), `${child.name}.fade-in`),
+ fadeOut: duration(text(child, "fade-out", "0f"), `${child.name}.fade-out`),
+ },
+ });
+ const mediaName = `item-${suffix}-media`;
+ const specName = `item-${suffix}-spec`;
+ const specId = `${id}.clip.${suffix}.spec`;
+ records.push({ id: specId, type: audioTrackTypes.clipSpec, value: { kind: "inline", value: clipSpec }, range: child.range });
+ inputs[mediaName] = source.ref;
+ inputs[specName] = { kind: "record", id: specId };
+ if (binding.kind === "program") {
+ fragmentItems.push({ kind: "program", mediaName, specName });
+ } else {
+ const mapName = `item-${suffix}-map`;
+ const sourceName = `item-${suffix}-${binding.kind}`;
+ inputs[mapName] = binding.map!.ref;
+ inputs[sourceName] = binding.source!.ref;
+ fragmentItems.push({ kind: binding.kind, mediaName, specName, mapName, sourceName });
+ }
+ }
+ if (fragmentItems.length === 0) throw new Error(`${element.name} requires at least one Clip.`);
+ const fragment = createAudioTrackFragment(fragmentItems, `@narratage/audio-track/surface/${id}@1`);
+ return {
+ records,
+ components: [{
+ id,
+ fragment: fragment.id,
+ inputs,
+ outputs: { program: `${id}.program`, track: `${id}.track` },
+ range: element.range,
+ }],
+ fragments: [fragment],
+ };
+};
diff --git a/packages/audio-track/src/types.ts b/packages/audio-track/src/types.ts
new file mode 100644
index 00000000..ebfa18bb
--- /dev/null
+++ b/packages/audio-track/src/types.ts
@@ -0,0 +1,74 @@
+import type { AudioTrack, FrameSpan } from "@narratage/composition";
+import type { BlobRef } from "@narratage/protocol";
+import type {
+ OccurrenceExpansion,
+ TemporalDuration,
+ TemporalWindowProjection,
+} from "@narratage/temporal";
+
+export type AudioOccupancy =
+ | { readonly mode: "once"; readonly align: "start" | "end" }
+ | { readonly mode: "loop"; readonly align: "start" | "end" }
+ | { readonly mode: "stretch"; readonly minRate: number; readonly maxRate: number; readonly pitch: "preserve" };
+
+export type AudioSourceTrim = {
+ readonly start?: TemporalDuration;
+ readonly end?: TemporalDuration;
+};
+
+export type AudioItemMix = {
+ readonly gain: number;
+ readonly fadeIn: TemporalDuration;
+ readonly fadeOut: TemporalDuration;
+};
+
+export type AudioClipSpec = {
+ readonly contract: "svml.audio-clip-spec@1";
+ readonly id: string;
+ readonly projection: TemporalWindowProjection;
+ readonly expansion: OccurrenceExpansion;
+ readonly trim: AudioSourceTrim;
+ readonly occupancy: AudioOccupancy;
+ readonly mix: AudioItemMix;
+};
+
+export type AudioTrackHeader = {
+ readonly contract: "svml.audio-track-header@1";
+ readonly id: string;
+};
+
+export type AudioItemProgram = {
+ readonly id: string;
+ readonly sourceOccurrenceId: string;
+ readonly window: FrameSpan;
+ readonly source: {
+ readonly artifact: BlobRef;
+ readonly sampleFrames: number;
+ };
+ readonly trim: {
+ readonly startSample: number;
+ readonly endSampleExclusive: number;
+ };
+ readonly occupancy: AudioOccupancy;
+ readonly mix: {
+ readonly gain: number;
+ readonly fadeInSamples: number;
+ readonly fadeOutSamples: number;
+ };
+};
+
+export type AudioTrackProgram = {
+ readonly contract: "svml.audio-track-program@1";
+ readonly id: string;
+ readonly items: readonly AudioItemProgram[];
+};
+
+export type AudioTrackSet = {
+ readonly contract: "svml.audio-track-set@1";
+ readonly items: readonly AudioItemProgram[];
+};
+
+export type AudioTrackProduct = {
+ readonly contract: "svml.audio-track-product@1";
+ readonly track: AudioTrack;
+};
diff --git a/packages/audio-track/test/audio-track.test.ts b/packages/audio-track/test/audio-track.test.ts
new file mode 100644
index 00000000..892874a3
--- /dev/null
+++ b/packages/audio-track/test/audio-track.test.ts
@@ -0,0 +1,343 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ audioTrackManifest,
+ audioTrackModuleRef,
+ audioTrackProducers,
+ audioTrackSurfaceImplementationDigest,
+ appendMomentAudioItem,
+ appendProgramAudioItem,
+ appendSelectionAudioItem,
+ createAudioTrackSet,
+ createAudioTrackFragment,
+ finalizeAudioTrack,
+ renderAudioTrack,
+ sealAudioClipSpec,
+ sealAudioTrackHeader,
+ decodeAudioTrackSurface,
+} from "@narratage/audio-track";
+import type { AudioClipSpec, AudioTrackSet } from "@narratage/audio-track";
+import { artifactManifest } from "@narratage/artifact";
+import { registerTypeValidatorFacets } from "@narratage/component-kit";
+import { sealComposition, compositionManifest, compositionTypes } from "@narratage/composition";
+import { createResolvedClosure, sealBuildRequest, start } from "@narratage/core";
+import { AuthorFrontendRegistry, compileSourceClosure, resolveCompiledSourceExport } from "@narratage/elaborator";
+import type { SynchronizedMedia } from "@narratage/media";
+import { mediaComponent, mediaDependency, mediaManifest, mediaTypes } from "@narratage/media";
+import type { NarrativeMomentRef, NarrativeSelectionRef } from "@narratage/narrative";
+import { narrativeManifest } from "@narratage/narrative";
+import { compileAudioProgramPlan } from "@narratage/media-pipeline";
+import { programSpaceDependency, programSpaceManifest, programSpaceTypes, sealProgramSpace } from "@narratage/program-space";
+import { digestOf } from "@narratage/protocol";
+import type { ModuleManifest } from "@narratage/protocol";
+import type { CompleteSemanticMap } from "@narratage/semantic-map";
+import { semanticMapManifest } from "@narratage/semantic-map";
+import { speechEvidenceManifest } from "@narratage/speech-evidence";
+import { speechManifest } from "@narratage/speech";
+import { temporalManifest } from "@narratage/temporal";
+import { TextSurfaceRegistry, createTextAuthorFrontend } from "@narratage/text";
+import { createRecordAdmitter, TypeValidatorRegistry } from "@narratage/validation";
+import { visualIrManifest } from "@narratage/visual-ir";
+
+const space = sealProgramSpace({
+ contract: "svml.program-space@1",
+ durationSec: 10,
+ frameRate: { numerator: 30, denominator: 1 },
+});
+const header = sealAudioTrackHeader({ contract: "svml.audio-track-header@1", id: "sound" });
+const zero = { unit: "frames" as const, value: 0 };
+
+function media(id: string, sampleFrames: number): SynchronizedMedia {
+ return {
+ contract: "svml.synchronized-media@1",
+ timeline: {
+ spanAuthority: "audio",
+ frameRate: { numerator: 30, denominator: 1 },
+ frameCount: Math.max(1, Math.round(sampleFrames / 1_600)),
+ sampleRate: 48_000,
+ sampleFrames,
+ },
+ sourceMap: {
+ sourceOriginPts: { ticks: "0", timeBase: { numerator: 1, denominator: 48_000 } },
+ sourceEndPts: { ticks: String(sampleFrames), timeBase: { numerator: 1, denominator: 48_000 } },
+ audioTrimStartSamples: 0,
+ audioTrimEndSamples: 0,
+ audioHeadSamples: 0,
+ audioContentSamples: sampleFrames,
+ audioTailSamples: 0,
+ },
+ audio: {
+ artifact: { kind: "blob", digest: digestOf(`audio:${id}`), size: sampleFrames * 4, mediaType: "audio/wav" },
+ sourceStreamIndex: 0,
+ codec: "pcm_s16le",
+ sampleRate: 48_000,
+ channels: 2,
+ sampleFrames,
+ loudness: "preserved",
+ },
+ };
+}
+
+function spec(overrides: Partial<AudioClipSpec> = {}): AudioClipSpec {
+ return sealAudioClipSpec({
+ contract: "svml.audio-clip-spec@1",
+ id: "clip",
+ projection: {
+ start: { ref: "program.start" },
+ end: { ref: "absolute", at: { unit: "seconds", numerator: 2, denominator: 1 } },
+ },
+ expansion: { kind: "one" },
+ trim: {},
+ occupancy: { mode: "once", align: "start" },
+ mix: { gain: 1, fadeIn: zero, fadeOut: zero },
+ ...overrides,
+ });
+}
+
+function programTrack(source: SynchronizedMedia, clipSpec: AudioClipSpec) {
+ const set = appendProgramAudioItem(createAudioTrackSet(), header, space, source, clipSpec);
+ return renderAudioTrack(space, finalizeAudioTrack(set, header));
+}
+
+test("once start/end preserve exact audible silence and source head/tail semantics", () => {
+ const short = media("short", 48_000);
+ const start = programTrack(short, spec());
+ assert.deepEqual(start.clips[0]?.target, { startSample: 0, endSampleExclusive: 48_000 });
+ assert.deepEqual(start.clips[0]?.source, {
+ sampleFrames: 48_000, startSample: 0, endSampleExclusive: 48_000, loop: false, phaseSample: 0,
+ });
+
+ const end = programTrack(short, spec({ occupancy: { mode: "once", align: "end" } }));
+ assert.deepEqual(end.clips[0]?.target, { startSample: 48_000, endSampleExclusive: 96_000 });
+
+ const long = programTrack(media("long", 144_000), spec({ occupancy: { mode: "once", align: "end" } }));
+ assert.deepEqual(long.clips[0]?.target, { startSample: 0, endSampleExclusive: 96_000 });
+ assert.deepEqual(long.clips[0]?.source, {
+ sampleFrames: 144_000, startSample: 48_000, endSampleExclusive: 144_000, loop: false, phaseSample: 0,
+ });
+});
+
+test("loop alignment uses one source interval and an exact phase instead of duplicating clips", () => {
+ const projection = {
+ start: { ref: "program.start" as const },
+ end: { ref: "absolute" as const, at: { unit: "seconds" as const, numerator: 5, denominator: 2 } },
+ };
+ const start = programTrack(media("loop", 48_000), spec({ projection, occupancy: { mode: "loop", align: "start" } }));
+ assert.equal(start.clips[0]?.source.loop, true);
+ assert.equal(start.clips[0]?.source.phaseSample, 0);
+ assert.deepEqual(start.clips[0]?.target, { startSample: 0, endSampleExclusive: 120_000 });
+ const end = programTrack(media("loop", 48_000), spec({ projection, occupancy: { mode: "loop", align: "end" } }));
+ assert.equal(end.clips[0]?.source.phaseSample, 24_000);
+});
+
+test("bounded pitch-preserving stretch succeeds exactly or fails without truncation", () => {
+ const stretched = programTrack(media("stretch", 96_000), spec({
+ projection: { start: { ref: "program.start" }, end: { ref: "absolute", at: { unit: "seconds", numerator: 4, denominator: 1 } } },
+ occupancy: { mode: "stretch", minRate: 0.4, maxRate: 0.6, pitch: "preserve" },
+ }));
+ assert.equal(stretched.clips[0]?.playbackRate, 0.5);
+ assert.equal(stretched.clips[0]?.pitch, "preserve");
+ assert.throws(() => programTrack(media("stretch-fail", 96_000), spec({
+ projection: { start: { ref: "program.start" }, end: { ref: "absolute", at: { unit: "seconds", numerator: 4, denominator: 1 } } },
+ occupancy: { mode: "stretch", minRate: 0.8, maxRate: 1.2, pitch: "preserve" },
+ })), /outside authored bounds/u);
+});
+
+test("trim and fades quantize once into the same sample domain", () => {
+ const track = programTrack(media("trim", 144_000), spec({
+ trim: {
+ start: { unit: "milliseconds", value: 250 },
+ end: { unit: "seconds", numerator: 9, denominator: 4 },
+ },
+ mix: {
+ gain: 0.25,
+ fadeIn: { unit: "milliseconds", value: 100 },
+ fadeOut: { unit: "milliseconds", value: 200 },
+ },
+ }));
+ assert.equal(track.clips[0]?.source.startSample, 12_000);
+ assert.equal(track.clips[0]?.source.endSampleExclusive, 108_000);
+ assert.equal(track.clips[0]?.gain, 0.25);
+ assert.equal(track.clips[0]?.fadeInSamples, 4_800);
+ assert.equal(track.clips[0]?.fadeOutSamples, 9_600);
+ assert.throws(() => programTrack(media("fade", 24_000), spec({
+ mix: { gain: 1, fadeIn: { unit: "seconds", numerator: 1, denominator: 1 }, fadeOut: zero },
+ })), /fade exceeds/u);
+});
+
+const map: CompleteSemanticMap = {
+ contract: "svml.complete-semantic-map@1",
+ tokens: [],
+ anchors: [
+ { identity: "a", timeSec: 1, frame: 30 },
+ { identity: "b", timeSec: 2, frame: 60 },
+ { identity: "c", timeSec: 3, frame: 90 },
+ { identity: "d", timeSec: 4, frame: 120 },
+ ],
+};
+
+test("Selection and Moment each expansion creates independent overlapping items", () => {
+ const selection: NarrativeSelectionRef = {
+ contract: "svml.narrative-selection@1",
+ id: "mentions",
+ occurrences: [
+ { occurrence: 0, startAnchorId: "a", endAnchorId: "b" },
+ { occurrence: 1, startAnchorId: "c", endAnchorId: "d" },
+ ],
+ };
+ const moment: NarrativeMomentRef = {
+ contract: "svml.narrative-moment@1",
+ id: "hits",
+ occurrences: [{ occurrence: 0, anchorId: "a" }, { occurrence: 1, anchorId: "c" }],
+ };
+ let set: AudioTrackSet = createAudioTrackSet();
+ set = appendSelectionAudioItem(set, header, space, media("selection", 48_000), map, selection, spec({
+ id: "selected",
+ projection: { start: { ref: "selection.start" }, end: { ref: "selection.end" } },
+ expansion: { kind: "each" },
+ }));
+ set = appendMomentAudioItem(set, header, space, media("moment", 48_000), map, moment, spec({
+ id: "hit",
+ projection: { start: { ref: "moment.cue" }, end: { ref: "moment.cue", offset: { unit: "seconds", numerator: 1, denominator: 1 } } },
+ expansion: { kind: "each" },
+ }));
+ const track = renderAudioTrack(space, finalizeAudioTrack(set, header));
+ assert.equal(track.clips.length, 4);
+ assert.deepEqual(track.clips.map((clip) => clip.target.startSample), [48_000, 48_000, 144_000, 144_000]);
+});
+
+test("one Track with overlaps and two peer Tracks compile to the same deterministic mix facts", () => {
+ const sourceA = media("a", 48_000);
+ const sourceB = media("b", 48_000);
+ const first = { ...programTrack(sourceA, spec({ id: "a" })), id: "first" };
+ const second = { ...programTrack(sourceB, spec({ id: "b" })), id: "second" };
+ const combined = {
+ contract: "svml.audio-track@1" as const,
+ id: "combined",
+ clips: [
+ { ...first.clips[0]!, id: "a" },
+ { ...second.clips[0]!, id: "b" },
+ ],
+ };
+ const peerPlan = compileAudioProgramPlan(sealComposition({
+ contract: "svml.composition@1", id: "peer", canvas: { width: 1, height: 1, clearColor: "#000000" },
+ tracks: [first, second],
+ }), space);
+ const combinedPlan = compileAudioProgramPlan(sealComposition({
+ contract: "svml.composition@1", id: "combined", canvas: { width: 1, height: 1, clearColor: "#000000" },
+ tracks: [combined],
+ }), space);
+ assert.deepEqual(
+ peerPlan.clips.map(({ id: _id, ...clip }) => clip),
+ combinedPlan.clips.map(({ id: _id, ...clip }) => clip),
+ );
+ const localReceived = structuredClone(peerPlan);
+ const remoteReceived = JSON.parse(JSON.stringify(peerPlan)) as typeof peerPlan;
+ assert.deepEqual(localReceived, remoteReceived);
+});
+
+test("dynamic Fragment keeps every material and temporal dependency as an explicit input", () => {
+ const fragment = createAudioTrackFragment([
+ { kind: "program", mediaName: "music", specName: "music-spec" },
+ { kind: "selection", mediaName: "voice", specName: "voice-spec", mapName: "map", sourceName: "selection" },
+ { kind: "moment", mediaName: "impact", specName: "impact-spec", mapName: "map", sourceName: "moment" },
+ ], "example.audio@1");
+ assert.deepEqual(fragment.inputs.map((input) => input.name), [
+ "header", "impact", "impact-spec", "map", "moment", "music", "music-spec", "selection", "space", "voice", "voice-spec",
+ ]);
+ assert.equal(fragment.exports[1]?.name, "track");
+});
+
+test("the self-described Audio Surface parses into the same finite Producer graph", async () => {
+ const fixtureModule = { name: "example.audio-inputs", version: "1" } as const;
+ const fixtureSurfaceDigest = digestOf("example.audio-inputs/surface@1");
+ const fixtureManifest: ModuleManifest = {
+ format: "svml.module@1",
+ name: fixtureModule.name,
+ version: fixtureModule.version,
+ dependencies: [mediaDependency, programSpaceDependency],
+ types: [],
+ capabilities: [],
+ surfaces: [{
+ name: "inputs",
+ tag: "Inputs",
+ mode: "structured",
+ outputs: [mediaTypes.synchronized, programSpaceTypes.programSpace],
+ implementation: {
+ kind: "trusted-frontend-surface",
+ locator: "example.audio-inputs/surface",
+ digest: fixtureSurfaceDigest,
+ },
+ }],
+ producers: [],
+ };
+ const closure = createResolvedClosure([
+ artifactManifest,
+ mediaManifest,
+ narrativeManifest,
+ programSpaceManifest,
+ speechManifest,
+ speechEvidenceManifest,
+ semanticMapManifest,
+ temporalManifest,
+ visualIrManifest,
+ compositionManifest,
+ audioTrackManifest,
+ fixtureManifest,
+ ]);
+ const registry = new TextSurfaceRegistry();
+ registry.registerStructured(fixtureModule, "inputs", fixtureSurfaceDigest, ({ element }) => ({
+ records: [
+ { id: "source", type: mediaTypes.synchronized, value: { kind: "inline", value: media("surface", 48_000) }, range: element.range },
+ { id: "space", type: programSpaceTypes.programSpace, value: { kind: "inline", value: space }, range: element.range },
+ ],
+ components: [],
+ fragments: [],
+ }));
+ registry.registerStructured(
+ audioTrackModuleRef,
+ "track",
+ audioTrackSurfaceImplementationDigest,
+ decodeAudioTrackSurface,
+ );
+ const frontends = new AuthorFrontendRegistry();
+ frontends.register(createTextAuthorFrontend({
+ registry,
+ resolveModule: (request) => request.from === "example.audio-inputs@1" ? fixtureModule : audioTrackModuleRef,
+ }));
+ const validators = new TypeValidatorRegistry();
+ registerTypeValidatorFacets(validators, mediaComponent.validators ?? []);
+ const compiled = await compileSourceClosure({
+ entry: {
+ id: "/project/audio.svml",
+ name: "audio.svml",
+ text: `<?svml using="@narratage/text@1"?>
+ <svml>
+ <import as="fixture" from="example.audio-inputs@1"/>
+ <import as="audio" from="@narratage/audio-track@1"/>
+ <fixture:Inputs/>
+ <audio:Track id="sound" space={space}>
+ <audio:Clip source={source} during="program" playback="loop-end" gain="0.5" fade-in="2f" fade-out="3f"/>
+ </audio:Track>
+ </svml>`,
+ },
+ closure,
+ frontends,
+ admitRecord: createRecordAdmitter(validators),
+ resolveSource() { throw new Error("Audio fixture has no source imports."); },
+ });
+ const trackExport = resolveCompiledSourceExport(compiled, "sound.track", compositionTypes.audioTrack);
+ assert.equal(trackExport.ref.kind, "logical-output");
+ const build = start(compiled.program, compiled.elaboration.graph, sealBuildRequest({
+ graph: compiled.elaboration.graph.id,
+ targets: [{ output: trackExport.ref.kind === "logical-output" ? trackExport.ref.id : "", accepts: "exact" }],
+ satisfactions: [],
+ }));
+ assert.deepEqual(build.plan.steps.map((step) => step.producer.name).sort(), [
+ audioTrackProducers.createSet.name,
+ audioTrackProducers.appendProgram.name,
+ audioTrackProducers.finalize.name,
+ audioTrackProducers.render.name,
+ ].sort());
+});
diff --git a/packages/broll/src/manifest.ts b/packages/broll/src/manifest.ts
index 617aa4f4..07f119a0 100644
--- a/packages/broll/src/manifest.ts
+++ b/packages/broll/src/manifest.ts
@@ -47,6 +47,12 @@ const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema;
kind: "object",
fields,
});
+const audioBlobRef = object({
+ kind: { schema: { kind: "literal", value: "blob" } },
+ digest: { schema: { kind: "string", minLength: 71, maxLength: 71 } },
+ size: { schema: unsignedInteger },
+ mediaType: { schema: { kind: "literal", value: "audio/wav" } },
+});
const frameSpan = object({
startFrame: { schema: unsignedInteger },
@@ -73,8 +79,11 @@ const item = object({
fit: { schema: { kind: "string", enum: ["contain", "cover"] } },
playback: { schema: { kind: "string", enum: ["freeze", "native", "loop", "stretch"] } },
mediaStartSec: { schema: number, optional: true },
- includeAudio: { schema: { kind: "boolean" }, optional: true },
- audioGain: { schema: number, optional: true },
+ audio: { schema: object({
+ artifact: { schema: audioBlobRef },
+ sampleFrames: { schema: unsignedInteger },
+ gain: { schema: number },
+ }), optional: true },
enter: { schema: motion, optional: true },
exit: { schema: motion, optional: true },
backgroundColor: { schema: string, optional: true },
@@ -99,8 +108,9 @@ const transition = object({
operator: { schema: { kind: "string", enum: ["push", "page-turn"] } },
direction: { schema: { kind: "string", enum: ["left", "right", "up", "down"] } },
sfx: { schema: object({
- artifact: { schema: mediaArtifactSchema },
- gain: { schema: number, optional: true },
+ artifact: { schema: audioBlobRef },
+ sampleFrames: { schema: unsignedInteger },
+ gain: { schema: number },
}), optional: true },
});
diff --git a/packages/broll/src/program.ts b/packages/broll/src/program.ts
index 193c2a35..24b4ddcf 100644
--- a/packages/broll/src/program.ts
+++ b/packages/broll/src/program.ts
@@ -1,5 +1,10 @@
import type { MediaArtifactRef } from "@narratage/media";
-import { assertProgramSpaceIdentity, programSpaceFrameCount } from "@narratage/program-space";
+import type { BlobRef } from "@narratage/protocol";
+import {
+ assertProgramSpaceIdentity,
+ programFrameSampleBoundary,
+ programSpaceFrameCount,
+} from "@narratage/program-space";
import type { ProgramSpace } from "@narratage/program-space";
import { assertAudioTrackIdentity, assertVisualTrackIdentity, sealAudioTrack, sealVisualTrack } from "@narratage/composition";
import type { AudioClip, AudioTrack, VisualAnimation, VisualElement, VisualStyleDeclaration } from "@narratage/composition";
@@ -55,6 +60,18 @@ function assertArtifact(artifact: MediaArtifactRef, label: string, kinds: readon
}
}
+function assertAudioArtifact(artifact: BlobRef, label: string): void {
+ if (
+ artifact.kind !== "blob"
+ || !isDigest(artifact.digest)
+ || !Number.isSafeInteger(artifact.size)
+ || artifact.size < 0
+ || artifact.mediaType !== "audio/wav"
+ ) {
+ throw new Error(`${label} must be a canonical WAV BlobRef.`);
+ }
+}
+
function assertSpan(item: { readonly startFrame: number; readonly endFrameExclusive: number }, totalFrames: number, label: string): void {
if (
!Number.isSafeInteger(item.startFrame)
@@ -95,18 +112,19 @@ function assertItem(item: BrollItem, totalFrames: number): void {
if (item.mediaStartSec !== undefined && (!Number.isFinite(item.mediaStartSec) || item.mediaStartSec < 0)) {
throw new Error(`B-roll item ${item.id} mediaStartSec is invalid.`);
}
- if (item.audioGain !== undefined && (!Number.isFinite(item.audioGain) || item.audioGain < 0)) {
- throw new Error(`B-roll item ${item.id} audioGain is invalid.`);
- }
if (item.backgroundColor !== undefined && !/^#[0-9a-f]{6}(?:[0-9a-f]{2})?$/iu.test(item.backgroundColor)) {
throw new Error(`B-roll item ${item.id} background color is invalid.`);
}
if (item.borderRadiusPx !== undefined && (!Number.isFinite(item.borderRadiusPx) || item.borderRadiusPx < 0)) {
throw new Error(`B-roll item ${item.id} border radius is invalid.`);
}
- if (item.includeAudio && isImage) throw new Error(`B-roll image ${item.id} cannot contribute source audio.`);
- if (item.includeAudio && item.playback === "loop") {
- throw new Error(`B-roll item ${item.id} cannot loop source audio in the v1 AudioTrack contract.`);
+ if (item.audio !== undefined) {
+ assertAudioArtifact(item.audio.artifact, `B-roll item ${item.id} source audio`);
+ if (isImage
+ || !Number.isSafeInteger(item.audio.sampleFrames) || item.audio.sampleFrames <= 0
+ || !Number.isFinite(item.audio.gain) || item.audio.gain < 0) {
+ throw new Error(`B-roll item ${item.id} source audio is invalid.`);
+ }
}
const duration = item.span.endFrameExclusive - item.span.startFrame;
assertMotion(item.enter, duration, `B-roll item ${item.id} enter motion`);
@@ -151,8 +169,9 @@ function transitionMaps(program: BrollProgram, totalFrames: number) {
if (!["push", "page-turn"].includes(transition.operator)) throw new Error(`B-roll transition ${transition.id} operator is unsupported.`);
if (!["left", "right", "up", "down"].includes(transition.direction)) throw new Error(`B-roll transition ${transition.id} direction is unsupported.`);
if (transition.sfx) {
- assertArtifact(transition.sfx.artifact, `B-roll transition ${transition.id} SFX`, ["audio"]);
- if (transition.sfx.gain !== undefined && (!Number.isFinite(transition.sfx.gain) || transition.sfx.gain < 0)) {
+ assertAudioArtifact(transition.sfx.artifact, `B-roll transition ${transition.id} SFX`);
+ if (!Number.isSafeInteger(transition.sfx.sampleFrames) || transition.sfx.sampleFrames <= 0
+ || !Number.isFinite(transition.sfx.gain) || transition.sfx.gain < 0) {
throw new Error(`B-roll transition ${transition.id} SFX gain is invalid.`);
}
}
@@ -321,46 +340,60 @@ function itemElements(
...(item.mediaStartSec === undefined ? {} : { mediaStartSec: item.mediaStartSec }),
...(playbackRate === undefined ? {} : { playbackRate }),
...(!isImage && item.playback === "loop" ? { loop: true } : {}),
- ...(!isImage ? { muted: item.includeAudio !== true } : {}),
+ ...(!isImage ? { muted: true } : {}),
},
];
}
function audioClips(program: BrollProgram, programSpace: ProgramSpace): AudioClip[] {
const clips: AudioClip[] = program.items.flatMap((item) => {
- if (!item.includeAudio) return [];
- const presentSeconds = (item.span.endFrameExclusive - item.span.startFrame)
- * programSpace.frameRate.denominator / programSpace.frameRate.numerator;
- const playbackRate = item.playback === "stretch" && item.artifact.durationSec > 0
- ? item.artifact.durationSec / presentSeconds
- : undefined;
+ if (item.audio === undefined) return [];
+ const targetStart = programFrameSampleBoundary(programSpace, item.span.startFrame, 48_000);
+ const targetEnd = programFrameSampleBoundary(programSpace, item.span.endFrameExclusive, 48_000);
+ const targetLength = targetEnd - targetStart;
+ const playbackRate = item.playback === "stretch"
+ ? item.audio.sampleFrames / targetLength
+ : 1;
return [{
id: `source:${item.id}`,
- span: { ...item.span },
- artifact: item.artifact,
- ...(item.mediaStartSec === undefined ? {} : { mediaStartSec: item.mediaStartSec }),
- ...(playbackRate === undefined ? {} : { playbackRate }),
- ...(item.audioGain === undefined ? {} : { gain: item.audioGain }),
- bus: "source" as const,
+ artifact: item.audio.artifact,
+ target: { startSample: targetStart, endSampleExclusive: targetEnd },
+ source: {
+ sampleFrames: item.audio.sampleFrames,
+ startSample: 0,
+ endSampleExclusive: item.audio.sampleFrames,
+ loop: item.playback === "loop",
+ phaseSample: 0,
+ },
+ playbackRate,
+ pitch: "preserve" as const,
+ gain: item.audio.gain,
+ fadeInSamples: 0,
+ fadeOutSamples: 0,
}];
});
const totalFrames = programSpaceFrameCount(programSpace);
for (const transition of program.transitions) {
if (!transition.sfx) continue;
- const durationFrames = Math.max(1, Math.round(
- transition.sfx.artifact.durationSec
- * programSpace.frameRate.numerator
- / programSpace.frameRate.denominator,
- ));
+ const startSample = programFrameSampleBoundary(programSpace, transition.span.startFrame, 48_000);
+ const programSamples = programFrameSampleBoundary(programSpace, totalFrames, 48_000);
+ const endSample = Math.min(programSamples, startSample + transition.sfx.sampleFrames);
clips.push({
id: `transition:${transition.id}`,
- span: {
- startFrame: transition.span.startFrame,
- endFrameExclusive: Math.min(totalFrames, transition.span.startFrame + durationFrames),
- },
artifact: transition.sfx.artifact,
- ...(transition.sfx.gain === undefined ? {} : { gain: transition.sfx.gain }),
- bus: "sfx",
+ target: { startSample, endSampleExclusive: endSample },
+ source: {
+ sampleFrames: transition.sfx.sampleFrames,
+ startSample: 0,
+ endSampleExclusive: endSample - startSample,
+ loop: false,
+ phaseSample: 0,
+ },
+ playbackRate: 1,
+ pitch: "preserve",
+ gain: transition.sfx.gain,
+ fadeInSamples: 0,
+ fadeOutSamples: 0,
});
}
return clips;
diff --git a/packages/broll/src/types.ts b/packages/broll/src/types.ts
index e25fb5c5..650c620f 100644
--- a/packages/broll/src/types.ts
+++ b/packages/broll/src/types.ts
@@ -1,4 +1,5 @@
import type { MediaArtifactRef } from "@narratage/media";
+import type { BlobRef } from "@narratage/protocol";
import type { AudioTrack, FrameSpan, VisualTrack } from "@narratage/composition";
export type BrollBox = {
@@ -27,8 +28,11 @@ export type BrollItem = {
readonly fit: "contain" | "cover";
readonly playback: "freeze" | "native" | "loop" | "stretch";
readonly mediaStartSec?: number;
- readonly includeAudio?: boolean;
- readonly audioGain?: number;
+ readonly audio?: {
+ readonly artifact: BlobRef;
+ readonly sampleFrames: number;
+ readonly gain: number;
+ };
readonly enter?: BrollMotion;
readonly exit?: BrollMotion;
readonly backgroundColor?: string;
@@ -43,8 +47,9 @@ export type BrollPairTransition = {
readonly operator: "push" | "page-turn";
readonly direction: "left" | "right" | "up" | "down";
readonly sfx?: {
- readonly artifact: MediaArtifactRef;
- readonly gain?: number;
+ readonly artifact: BlobRef;
+ readonly sampleFrames: number;
+ readonly gain: number;
};
};
diff --git a/packages/broll/test/broll.test.ts b/packages/broll/test/broll.test.ts
index 4ea6ee5e..72435170 100644
--- a/packages/broll/test/broll.test.ts
+++ b/packages/broll/test/broll.test.ts
@@ -17,6 +17,7 @@ import test from "node:test";
import type { BrollItem, BrollPairTransition } from "@narratage/broll";
import { compileHyperframesDocument } from "@narratage/hyperframes";
import { digestOf } from "@narratage/protocol";
+import type { BlobRef } from "@narratage/protocol";
const programSpace = sealProgramSpace({
contract: "svml.program-space@1",
@@ -28,6 +29,10 @@ function artifact(id: string, mediaType = "video/mp4", durationSec = 4): MediaAr
return { digest: digestOf(`broll:${id}`), size: 100, mediaType, durationSec };
}
+function audioArtifact(id: string): BlobRef {
+ return { kind: "blob", digest: digestOf(`broll:${id}`), size: 100, mediaType: "audio/wav" };
+}
+
function item(overrides: Partial<BrollItem> & Pick<BrollItem, "id" | "span" | "z">): BrollItem {
return {
artifact: artifact(overrides.id),
@@ -65,8 +70,7 @@ test("B-roll owns local motion while every item remains an independently stacked
z: 80,
enter: { operator: "slide-down", durationFrames: 12 },
exit: { operator: "slide-up", durationFrames: 12 },
- includeAudio: true,
- audioGain: 0.7,
+ audio: { artifact: audioArtifact("cutaway-audio"), sampleFrames: 192_000, gain: 0.7 },
}),
]);
assert.doesNotThrow(() => assertBrollProgramIdentity(authored, programSpace));
@@ -125,7 +129,7 @@ test("push is lowered into complementary animations inside one B-roll-owned hand
});
test("page-turn and transition SFX remain B-roll-owned and lower to generic visual/audio facts", () => {
- const sfx = artifact("page-sfx", "audio/wav", 0.2);
+ const sfx = audioArtifact("page-sfx");
const authored = program([
item({ id: "page-a", span: { startFrame: 0, endFrameExclusive: 90 }, z: 40 }),
item({ id: "page-b", span: { startFrame: 75, endFrameExclusive: 180 }, z: 41 }),
@@ -136,11 +140,14 @@ test("page-turn and transition SFX remain B-roll-owned and lower to generic visu
span: { startFrame: 75, endFrameExclusive: 90 },
operator: "page-turn",
direction: "left",
- sfx: { artifact: sfx, gain: 0.8 },
+ sfx: { artifact: sfx, sampleFrames: 9_600, gain: 0.8 },
}]);
const product = compileBrollProduct(programSpace, authored);
assert.equal(product.audioTrack.clips[0]?.id, "transition:turn");
- assert.deepEqual(product.audioTrack.clips[0]?.span, { startFrame: 75, endFrameExclusive: 81 });
+ assert.deepEqual(product.audioTrack.clips[0]?.target, {
+ startSample: 120_000,
+ endSampleExclusive: 129_600,
+ });
assert.equal(product.audioTrack.clips[0]?.artifact.digest, sfx.digest);
const document = compileHyperframesDocument(sealComposition({
contract: "svml.composition@1",
diff --git a/packages/caption/test/caption.test.ts b/packages/caption/test/caption.test.ts
index 360b5698..eae0c60b 100644
--- a/packages/caption/test/caption.test.ts
+++ b/packages/caption/test/caption.test.ts
@@ -35,7 +35,7 @@ function locate(narrative: Narrative, durationSec: number, segments: readonly Al
durationSec,
frameRate: { numerator: 30, denominator: 1 },
});
- const audio = { digest: digestOf("caption:test-audio"), size: 1, mediaType: "audio/wav", durationSec };
+ const audio = { kind: "blob" as const, digest: digestOf("caption:test-audio"), size: 1, mediaType: "audio/wav" };
const basisSegments = narrative.segments.map((segment, index) => ({
segmentId: segment.id,
startSec: segments[index]!.startSec,
diff --git a/packages/composition/src/schema.ts b/packages/composition/src/schema.ts
index 36bb863a..f7455e25 100644
--- a/packages/composition/src/schema.ts
+++ b/packages/composition/src/schema.ts
@@ -6,6 +6,12 @@ export { mediaDependency, programSpaceDependency };
const string = { kind: "string", minLength: 1 } as const; const number = { kind: "number", minimum: 0 } as const;
const integer = { kind: "number", integer: true, minimum: 0 } as const; const signedInteger = { kind: "number", integer: true } as const;
const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>): ValueSchema => ({ kind: "object", fields });
+const audioBlobRef = object({
+ kind: { schema: { kind: "literal", value: "blob" } },
+ digest: { schema: { kind: "string", minLength: 71, maxLength: 71 } },
+ size: { schema: integer },
+ mediaType: { schema: { kind: "literal", value: "audio/wav" } },
+});
const styleDeclaration: ValueSchema = { kind: "oneOf", variants: VISUAL_STYLE_NAMES_V1.map((name) => object({
name: { schema: { kind: "literal", value: name } }, value: { schema: Object.hasOwn(VISUAL_STYLE_ENUM_VALUES_V1, name)
? { kind: "string", enum: VISUAL_STYLE_ENUM_VALUES_V1[name as keyof typeof VISUAL_STYLE_ENUM_VALUES_V1] }
@@ -23,6 +29,23 @@ const element: ValueSchema = { kind: "oneOf", variants: [box, text, media("image
const span = object({ startFrame: { schema: integer }, endFrameExclusive: { schema: integer } });
const present = object({ id: { schema: string }, span: { schema: span }, stacking: { schema: object({ order: { schema: signedInteger }, tieBreak: { schema: string } }) }, elements: { schema: { kind: "array", minItems: 1, items: element } } });
export const visualTrackSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.visual-track@1" } }, visualIr: { schema: { kind: "literal", value: VISUAL_IR_V1 } }, id: { schema: string }, presents: { schema: { kind: "array", items: present } } });
-const audioClip = object({ id: { schema: string }, span: { schema: span }, artifact: { schema: mediaArtifactSchema }, mediaStartSec: { schema: number, optional: true }, playbackRate: { schema: number, optional: true }, gain: { schema: number, optional: true }, fadeInSec: { schema: number, optional: true }, fadeOutSec: { schema: number, optional: true }, bus: { schema: { kind: "string", enum: ["speech", "music", "sfx", "source"] }, optional: true } });
+const audioSampleSpan = object({ startSample: { schema: integer }, endSampleExclusive: { schema: { kind: "number", integer: true, minimum: 1 } } });
+const audioClip = object({
+ id: { schema: string },
+ artifact: { schema: audioBlobRef },
+ target: { schema: audioSampleSpan },
+ source: { schema: object({
+ sampleFrames: { schema: { kind: "number", integer: true, minimum: 1 } },
+ startSample: { schema: integer },
+ endSampleExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },
+ loop: { schema: { kind: "boolean" } },
+ phaseSample: { schema: integer },
+ }) },
+ playbackRate: { schema: { kind: "number", minimum: 0.000001, maximum: 100 } },
+ pitch: { schema: { kind: "literal", value: "preserve" } },
+ gain: { schema: { kind: "number", minimum: 0, maximum: 64 } },
+ fadeInSamples: { schema: integer },
+ fadeOutSamples: { schema: integer },
+});
export const audioTrackSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.audio-track@1" } }, id: { schema: string }, clips: { schema: { kind: "array", items: audioClip } } });
export const compositionSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.composition@1" } }, id: { schema: string }, canvas: { schema: object({ width: { schema: integer }, height: { schema: integer }, clearColor: { schema: string } }) }, tracks: { schema: { kind: "array", items: { kind: "oneOf", variants: [visualTrackSchema, audioTrackSchema] } } } });
diff --git a/packages/composition/src/track.ts b/packages/composition/src/track.ts
index 15de87d0..a843479f 100644
--- a/packages/composition/src/track.ts
+++ b/packages/composition/src/track.ts
@@ -1,10 +1,15 @@
import { digestOf, isDigest } from "@narratage/protocol";
+import type { BlobRef } from "@narratage/protocol";
import {
assertVisualStyleV1,
VISUAL_IR_V1,
} from "@narratage/visual-ir";
-import { assertProgramSpaceIdentity, programSpaceFrameCount } from "@narratage/program-space";
+import {
+ assertProgramSpaceIdentity,
+ programSpaceFrameCount,
+ programSpaceSampleFrames,
+} from "@narratage/program-space";
import type { ProgramSpace } from "@narratage/program-space";
import { assertCompositableSurfaceRef, assertFontArtifactRef } from "@narratage/media";
import type { CompositableSurfaceRef, FontArtifactRef, MediaArtifactRef } from "@narratage/media";
@@ -107,14 +112,27 @@ export type VisualTrack = {
export type AudioClip = {
readonly id: string;
- readonly span: FrameSpan;
- readonly artifact: MediaArtifactRef;
- readonly mediaStartSec?: number;
- readonly playbackRate?: number;
- readonly gain?: number;
- readonly fadeInSec?: number;
- readonly fadeOutSec?: number;
- readonly bus?: "speech" | "music" | "sfx" | "source";
+ /** Bytes only. Exact duration belongs to source.sampleFrames, not duplicated floating metadata. */
+ readonly artifact: BlobRef;
+ /** Exact audible placement in the canonical 48 kHz ProgramSpace sample domain. */
+ readonly target: {
+ readonly startSample: number;
+ readonly endSampleExclusive: number;
+ };
+ /** Exact sampling interval in one canonical 48 kHz stereo WAV Artifact. */
+ readonly source: {
+ readonly sampleFrames: number;
+ readonly startSample: number;
+ readonly endSampleExclusive: number;
+ readonly loop: boolean;
+ /** Offset inside the effective source interval used only when looping. */
+ readonly phaseSample: number;
+ };
+ readonly playbackRate: number;
+ readonly pitch: "preserve";
+ readonly gain: number;
+ readonly fadeInSamples: number;
+ readonly fadeOutSamples: number;
};
export type AudioTrack = {
@@ -160,7 +178,7 @@ function assertFrameSpan(span: FrameSpan, totalFrames: number, label: string): v
}
}
-function assertArtifact(artifact: MediaArtifactRef, label: string): void {
+function assertMediaArtifact(artifact: MediaArtifactRef, label: string): void {
if (
!isDigest(artifact.digest)
|| !Number.isSafeInteger(artifact.size)
@@ -173,6 +191,18 @@ function assertArtifact(artifact: MediaArtifactRef, label: string): void {
}
}
+function assertAudioArtifact(artifact: BlobRef, label: string): void {
+ if (
+ artifact.kind !== "blob"
+ || !isDigest(artifact.digest)
+ || !Number.isSafeInteger(artifact.size)
+ || artifact.size < 0
+ || artifact.mediaType !== "audio/wav"
+ ) {
+ throw new Error(`${label} must be a canonical WAV BlobRef.`);
+ }
+}
+
function assertStyle(style: readonly VisualStyleDeclaration[], label: string): void {
const names = new Set<string>();
for (const declaration of style) {
@@ -259,7 +289,7 @@ function assertPresent(present: VisualPresent, programSpace: ProgramSpace | unde
`${trackId}.${present.id}.${element.id}`,
);
if (element.kind === "image" || element.kind === "video") {
- assertArtifact(element.artifact, `${trackId}.${present.id}.${element.id}.artifact`);
+ assertMediaArtifact(element.artifact, `${trackId}.${present.id}.${element.id}.artifact`);
if (!element.artifact.mediaType.startsWith(`${element.kind}/`)) {
throw new Error(`${trackId}.${present.id}.${element.id} media kind does not match its Artifact.`);
}
@@ -418,16 +448,16 @@ function audioTrackContent(value: AudioTrack): AudioTrack {
clips: [...value.clips]
.map((clip) => ({
id: clip.id,
- span: { ...clip.span },
artifact: { ...clip.artifact },
- ...(clip.mediaStartSec === undefined ? {} : { mediaStartSec: clip.mediaStartSec }),
- ...(clip.playbackRate === undefined ? {} : { playbackRate: clip.playbackRate }),
- ...(clip.gain === undefined ? {} : { gain: clip.gain }),
- ...(clip.fadeInSec === undefined ? {} : { fadeInSec: clip.fadeInSec }),
- ...(clip.fadeOutSec === undefined ? {} : { fadeOutSec: clip.fadeOutSec }),
- ...(clip.bus === undefined ? {} : { bus: clip.bus }),
+ target: { ...clip.target },
+ source: { ...clip.source },
+ playbackRate: clip.playbackRate,
+ pitch: clip.pitch,
+ gain: clip.gain,
+ fadeInSamples: clip.fadeInSamples,
+ fadeOutSamples: clip.fadeOutSamples,
}))
- .sort((a, b) => a.span.startFrame - b.span.startFrame || a.id.localeCompare(b.id)),
+ .sort((a, b) => a.target.startSample - b.target.startSample || a.id.localeCompare(b.id)),
};
}
@@ -456,30 +486,47 @@ export function assertAudioTrackIdentity(track: AudioTrack, programSpace?: Progr
if (programSpace !== undefined) assertProgramSpaceIdentity(programSpace);
if (track.contract !== "svml.audio-track@1") throw new Error("Unsupported AudioTrack contract.");
assertNonEmpty(track.id, "AudioTrack id");
- const totalFrames = programSpace === undefined ? Number.MAX_SAFE_INTEGER : programSpaceFrameCount(programSpace);
+ const totalSamples = programSpace === undefined
+ ? Number.MAX_SAFE_INTEGER
+ : programSpaceSampleFrames(programSpace, 48_000);
const clipIds = new Set<string>();
for (const clip of track.clips) {
if (clipIds.has(clip.id)) throw new Error(`${track.id} has duplicate clip ${clip.id}.`);
clipIds.add(clip.id);
assertNonEmpty(clip.id, `${track.id} clip id`);
- assertFrameSpan(clip.span, totalFrames, `${track.id}.${clip.id}.span`);
- assertArtifact(clip.artifact, `${track.id}.${clip.id}.artifact`);
- if (!clip.artifact.mediaType.startsWith("audio/") && !clip.artifact.mediaType.startsWith("video/")) {
- throw new Error(`${track.id}.${clip.id} is not an audio-capable Artifact.`);
+ assertAudioArtifact(clip.artifact, `${track.id}.${clip.id}.artifact`);
+ if (!Number.isSafeInteger(clip.target.startSample) || clip.target.startSample < 0
+ || !Number.isSafeInteger(clip.target.endSampleExclusive)
+ || clip.target.endSampleExclusive <= clip.target.startSample
+ || clip.target.endSampleExclusive > totalSamples) {
+ throw new Error(`${track.id}.${clip.id} target sample interval is outside ProgramSpace.`);
}
- for (const [name, value] of Object.entries({
- mediaStartSec: clip.mediaStartSec,
- gain: clip.gain,
- fadeInSec: clip.fadeInSec,
- fadeOutSec: clip.fadeOutSec,
- })) {
- if (value !== undefined && (!Number.isFinite(value) || value < 0)) {
- throw new Error(`${track.id}.${clip.id} has invalid ${name}.`);
- }
+ if (!Number.isSafeInteger(clip.source.sampleFrames) || clip.source.sampleFrames <= 0
+ || !Number.isSafeInteger(clip.source.startSample) || clip.source.startSample < 0
+ || !Number.isSafeInteger(clip.source.endSampleExclusive)
+ || clip.source.endSampleExclusive <= clip.source.startSample
+ || clip.source.endSampleExclusive > clip.source.sampleFrames) {
+ throw new Error(`${track.id}.${clip.id} source sample interval is invalid.`);
+ }
+ const sourceLength = clip.source.endSampleExclusive - clip.source.startSample;
+ if (!Number.isSafeInteger(clip.source.phaseSample) || clip.source.phaseSample < 0
+ || clip.source.phaseSample >= sourceLength
+ || (!clip.source.loop && clip.source.phaseSample !== 0)) {
+ throw new Error(`${track.id}.${clip.id} source loop phase is invalid.`);
}
- if (clip.playbackRate !== undefined && (!Number.isFinite(clip.playbackRate) || clip.playbackRate <= 0)) {
+ if (!Number.isFinite(clip.playbackRate) || clip.playbackRate <= 0 || clip.playbackRate > 100) {
throw new Error(`${track.id}.${clip.id} has invalid playbackRate.`);
}
+ if (clip.pitch !== "preserve") throw new Error(`${track.id}.${clip.id} pitch policy is invalid.`);
+ if (!Number.isFinite(clip.gain) || clip.gain < 0 || clip.gain > 64) {
+ throw new Error(`${track.id}.${clip.id} gain is invalid.`);
+ }
+ const targetLength = clip.target.endSampleExclusive - clip.target.startSample;
+ if (!Number.isSafeInteger(clip.fadeInSamples) || clip.fadeInSamples < 0 || clip.fadeInSamples > targetLength
+ || !Number.isSafeInteger(clip.fadeOutSamples) || clip.fadeOutSamples < 0
+ || clip.fadeOutSamples > targetLength) {
+ throw new Error(`${track.id}.${clip.id} fades are invalid.`);
+ }
}
}
diff --git a/packages/composition/test/track.test.ts b/packages/composition/test/track.test.ts
index 64b9a7c1..adb9bf1c 100644
--- a/packages/composition/test/track.test.ts
+++ b/packages/composition/test/track.test.ts
@@ -4,6 +4,7 @@ import test from "node:test";
import { registerTypeValidatorFacets } from "@narratage/component-kit";
import { createResolvedClosure } from "@narratage/core";
import { canonicalize, digestOf } from "@narratage/protocol";
+import type { BlobRef } from "@narratage/protocol";
import { TypeValidatorRegistry, validateValue } from "@narratage/validation";
import { artifactManifest } from "@narratage/artifact";
import type { MediaArtifactRef } from "@narratage/media";
@@ -36,11 +37,11 @@ const image: MediaArtifactRef = {
mediaType: "image/png",
durationSec: 0,
};
-const audio: MediaArtifactRef = {
+const audio: BlobRef = {
+ kind: "blob",
digest: digestOf("audio"),
size: 24,
mediaType: "audio/wav",
- durationSec: 4,
};
function fixture() {
@@ -66,7 +67,17 @@ function fixture() {
const sound = sealAudioTrack({
contract: "svml.audio-track@1",
id: "speech",
- clips: [{ id: "speech", span: { startFrame: 0, endFrameExclusive: 120 }, artifact: audio, bus: "speech" }],
+ clips: [{
+ id: "speech",
+ artifact: audio,
+ target: { startSample: 0, endSampleExclusive: 192_000 },
+ source: { sampleFrames: 192_000, startSample: 0, endSampleExclusive: 192_000, loop: false, phaseSample: 0 },
+ playbackRate: 1,
+ pitch: "preserve",
+ gain: 1,
+ fadeInSamples: 0,
+ fadeOutSamples: 0,
+ }],
});
return { programSpace, visual, sound };
}
diff --git a/packages/hyperframes/test/browser-visual.test.ts b/packages/hyperframes/test/browser-visual.test.ts
index 96582ed5..bc3aeeb0 100644
--- a/packages/hyperframes/test/browser-visual.test.ts
+++ b/packages/hyperframes/test/browser-visual.test.ts
@@ -20,6 +20,16 @@ import {
} from "@narratage/hyperframes";
import type { Digest } from "@narratage/protocol";
import { captionDisplaySequence, parseScript } from "@narratage/script";
+import {
+ appendProgramScreenOverlay,
+ createScreenOverlaySet,
+ finalizeScreenOverlay,
+ renderScreenOverlay,
+ sealScreenOverlayHeader,
+ sealScreenOverlayItemSpec,
+} from "@narratage/screen-overlay";
+import type { ScreenOverlayComponent, ScreenOverlaySet } from "@narratage/screen-overlay";
+import { sealCanvasSpace } from "@narratage/spatial";
import type { SvsRecipe } from "@narratage/svs";
const enabled = process.env.SVML_BROWSER_TESTS === "1";
@@ -648,3 +658,88 @@ test("installed open fonts render CJK, emoji and independent stroke, shadow and
await rm(temp, { recursive: true, force: true });
}
});
+
+test("every official Screen Overlay survives real sequential and parallel browser frames identically", {
+ skip: !enabled,
+ timeout: 120_000,
+}, async () => {
+ const temp = await mkdtemp(path.join(os.tmpdir(), "svml-screen-overlay-visual-"));
+ try {
+ const width = 128;
+ const height = 96;
+ const canvas = sealCanvasSpace({
+ contract: "svml.canvas-space@1",
+ widthPx: width,
+ heightPx: height,
+ origin: "top-left",
+ xDirection: "right",
+ yDirection: "down",
+ pixelAspect: "square",
+ });
+ const space = sealProgramSpace({
+ contract: "svml.program-space@1",
+ durationSec: 1,
+ frameRate: { numerator: 12, denominator: 1 },
+ });
+ const header = sealScreenOverlayHeader({ contract: "svml.screen-overlay-header@1", id: "browser-overlays" });
+ const components: readonly ScreenOverlayComponent[] = [
+ { kind: "flash", color: "#ffffff", intensity: 0.2, attackFrames: 2, holdFrames: 2, decayFrames: 4 },
+ { kind: "color-wash", color: "#2030ff", opacity: 0.08 },
+ { kind: "vignette", center: { x: 0.5, y: 0.5 }, radius: { x: 0.8, y: 0.7 }, softness: 0.4, color: "#000000", opacity: 0.25 },
+ { kind: "scan-lines", spacingPx: 7, thicknessPx: 1, angleDeg: 3, opacity: 0.1, travelPx: 8 },
+ { kind: "directional-matte", angleDeg: 12, coverage: 0.4, feather: 0.3, color: "#ffcc88", opacity: 0.08, progress: { from: -0.5, to: 1.5 } },
+ { kind: "whip-veil", direction: "right", widthPx: 36, softnessPx: 8, travelPx: 180, opacity: 0.12 },
+ { kind: "glitch-veil", bars: 5, colors: ["#ff0055", "#00ddff"], opacity: 0.12, travelPx: 16, seed: 7 },
+ { kind: "grain", amount: 0.12, grainSizePx: 2, chroma: "monochrome", motionRatePxPerFrame: 0.25, seed: 11 },
+ { kind: "light-leak", colors: ["#ff3300", "#ffd000"], angleDeg: 25, softness: 0.25, travelPx: 24, intensity: 0.1, seed: 13 },
+ { kind: "bokeh", amount: 0.12, sizeMinPx: 4, sizeMaxPx: 12, color: "#ffe2aa", warmth: 0.3, driftPx: 12, seed: 17 },
+ { kind: "tv-static", amount: 0.05, noiseSizePx: 4, scanLineOpacity: 0.08, motionRatePxPerFrame: 0.2, seed: 19 },
+ ];
+ let set: ScreenOverlaySet = createScreenOverlaySet();
+ components.forEach((content, index) => {
+ set = appendProgramScreenOverlay(set, header, space, sealScreenOverlayItemSpec({
+ contract: "svml.screen-overlay-item-spec@1",
+ id: `${content.kind}-${index + 1}`,
+ content,
+ projection: { start: { ref: "program.start" }, end: { ref: "program.end" } },
+ expansion: { kind: "one" },
+ stackingOrder: index + 1,
+ }));
+ });
+ const track = renderScreenOverlay(canvas, space, finalizeScreenOverlay(set, header));
+ const document = compileHyperframesDocument(sealComposition({
+ contract: "svml.composition@1",
+ id: "screen-overlay-browser-proof",
+ canvas: { width, height, clearColor: "#000000" },
+ tracks: [track],
+ }), space);
+ await writeFile(path.join(temp, "index.html"), materializeHyperframesHtml(document, (artifact) => {
+ throw new Error(`Screen Overlay unexpectedly requested Artifact ${artifact.digest}`);
+ }));
+ const render = async (workers: number, name: string): Promise<string[]> => {
+ const output = path.join(temp, name);
+ await mkdir(output);
+ const result = spawnSync(process.execPath, [
+ hyperframesCli, "render", temp,
+ "--format", "png-sequence", "--output", output, "--fps", "12", "--workers", String(workers),
+ "--no-browser-gpu", "--no-best-effort", "--quiet",
+ ], { encoding: "utf8", timeout: 110_000 });
+ assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
+ return collectPngs(output);
+ };
+ const sequential = await render(1, "sequential");
+ const parallel = await render(3, "parallel");
+ assert.equal(sequential.length, 12);
+ assert.equal(parallel.length, 12);
+ const sequentialPixels = sequential.map((frame) => decodedRgba(frame, width, height));
+ const parallelPixels = parallel.map((frame) => decodedRgba(frame, width, height));
+ for (let frame = 0; frame < 12; frame += 1) {
+ assert.deepEqual(parallelPixels[frame], sequentialPixels[frame], `frame ${frame} changed under partitioning`);
+ }
+ assert.notDeepEqual(sequentialPixels[0], sequentialPixels[6], "authored Overlay envelopes painted no temporal change");
+ assert.ok(sequentialPixels.some((pixels) => pixels.some((channel, index) => index % 4 !== 3 && channel > 0)),
+ "the combined official Overlay set painted no visible pixels");
+ } finally {
+ await rm(temp, { recursive: true, force: true });
+ }
+});
diff --git a/packages/hyperframes/test/document.test.ts b/packages/hyperframes/test/document.test.ts
index 9dbbf15c..99f2b57e 100644
--- a/packages/hyperframes/test/document.test.ts
+++ b/packages/hyperframes/test/document.test.ts
@@ -29,10 +29,10 @@ function fixture() {
durationSec: 0,
};
const sound = {
+ kind: "blob" as const,
digest: digestOf("hyperframes:sound"),
size: 20,
mediaType: "audio/wav",
- durationSec: programSpace.durationSec,
};
const lower = sealVisualTrack({
contract: "svml.visual-track@1",
@@ -68,7 +68,17 @@ function fixture() {
const audio = sealAudioTrack({
contract: "svml.audio-track@1",
id: "sound",
- clips: [{ id: "main", span: { startFrame: 0, endFrameExclusive: 30 }, artifact: sound, bus: "speech" }],
+ clips: [{
+ id: "main",
+ artifact: sound,
+ target: { startSample: 0, endSampleExclusive: 48_000 },
+ source: { sampleFrames: 48_000, startSample: 0, endSampleExclusive: 48_000, loop: false, phaseSample: 0 },
+ playbackRate: 1,
+ pitch: "preserve",
+ gain: 1,
+ fadeInSamples: 0,
+ fadeOutSamples: 0,
+ }],
});
const composition = sealComposition({
contract: "svml.composition@1",
diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.ts
index d6e1d1b8..9bd63ebe 100644
--- a/packages/media-execution/src/execute.ts
+++ b/packages/media-execution/src/execute.ts
@@ -420,9 +420,15 @@ function atempo(rate: number): string[] {
function audioClipFilter(clip: AudioProgramClip, inputIndex: number, outputIndex: number): string {
const length = clip.targetEndSampleExclusive - clip.targetStartSample;
+ const sourceLength = clip.sourceEndSampleExclusive - clip.sourceStartSample;
const filters = [
- `atrim=start_sample=${clip.sourceStartSample}`,
+ `atrim=start_sample=${clip.sourceStartSample}:end_sample=${clip.sourceEndSampleExclusive}`,
"asetpts=PTS-STARTPTS",
+ ...(clip.sourceLoop ? [`aloop=loop=-1:size=${sourceLength}:start=0`] : []),
+ ...(clip.sourcePhaseSample === 0 ? [] : [
+ `atrim=start_sample=${clip.sourcePhaseSample}`,
+ "asetpts=PTS-STARTPTS",
+ ]),
...atempo(clip.playbackRate),
`atrim=start_sample=0:end_sample=${length}`,
`apad=whole_len=${length}`,
@@ -706,9 +712,14 @@ export async function executeRenderTimelineAudio(
const plan: AudioProgramPlan = need.plan;
const work = await mkdtemp(join(tmpdir(), "svml-media-audio-"));
try {
- const artifacts = new Map<string, { source: BlobRef; path: string; inputIndex: number }>();
+ const artifacts = new Map<string, { source: BlobRef; path: string; inputIndex: number; sampleFrames: number }>();
for (const clip of plan.clips) {
- if (artifacts.has(clip.artifact.digest)) continue;
+ const existing = artifacts.get(clip.artifact.digest);
+ if (existing !== undefined) {
+ assert(existing.sampleFrames === clip.sourceSampleFrames,
+ `Audio input ${clip.artifact.digest} has conflicting sample counts in one plan`);
+ continue;
+ }
const inputIndex = artifacts.size;
const path = join(work, `input-${inputIndex}.wav`);
await stageArtifact(env, clip.artifact, path);
@@ -720,8 +731,14 @@ export async function executeRenderTimelineAudio(
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
- assert(audio.decodedSampleFrames > 0, `Audio input ${clip.artifact.digest} is empty`);
- artifacts.set(clip.artifact.digest, { source: clip.artifact, path, inputIndex });
+ assert(audio.decodedSampleFrames === clip.sourceSampleFrames,
+ `Audio input ${clip.artifact.digest} sample count differs from its plan`);
+ artifacts.set(clip.artifact.digest, {
+ source: clip.artifact,
+ path,
+ inputIndex,
+ sampleFrames: audio.decodedSampleFrames,
+ });
}
const output = join(work, "program.wav");
diff --git a/packages/media-pipeline/src/audio-plan.ts b/packages/media-pipeline/src/audio-plan.ts
index db5bfe7a..46a00822 100644
--- a/packages/media-pipeline/src/audio-plan.ts
+++ b/packages/media-pipeline/src/audio-plan.ts
@@ -1,4 +1,4 @@
-import { programSpaceFrameCount } from "@narratage/program-space";
+import { programFrameSampleBoundary, programSpaceFrameCount } from "@narratage/program-space";
import type { ProgramSpace } from "@narratage/program-space";
import { assertCompositionIdentity } from "@narratage/composition";
import type { Composition } from "@narratage/composition";
@@ -10,21 +10,6 @@ function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
-function sampleBoundary(frame: number, numerator: number, denominator: number): number {
- const scaled = BigInt(frame) * 48_000n * BigInt(denominator);
- const rate = BigInt(numerator);
- const rounded = (scaled * 2n + rate) / (rate * 2n);
- assert(rounded >= 0n && rounded <= BigInt(Number.MAX_SAFE_INTEGER), "Audio sample boundary is unsafe");
- return Number(rounded);
-}
-
-function samplesFromSeconds(seconds: number, subject: string): number {
- assert(Number.isFinite(seconds) && seconds >= 0, `${subject} is invalid`);
- const result = Math.round(seconds * 48_000);
- assert(Number.isSafeInteger(result), `${subject} exceeds safe sample arithmetic`);
- return result;
-}
-
export function sealAudioProgramPlan(value: AudioProgramPlan): AudioProgramPlan {
return canonicalize(value) as unknown as AudioProgramPlan;
}
@@ -38,7 +23,12 @@ export function verifyAudioProgramPlan(value: unknown): asserts value is AudioPr
"AudioProgramPlan frame rate is invalid");
assert(Number.isSafeInteger(item.frameCount) && item.frameCount > 0, "AudioProgramPlan frame count is invalid");
assert(item.sampleRate === 48_000, "AudioProgramPlan sample rate must be 48000");
- assert(item.sampleFrames === sampleBoundary(item.frameCount, item.frameRate.numerator, item.frameRate.denominator),
+ const planSpace = {
+ contract: "svml.program-space@1" as const,
+ durationSec: item.frameCount * item.frameRate.denominator / item.frameRate.numerator,
+ frameRate: item.frameRate,
+ };
+ assert(item.sampleFrames === programFrameSampleBoundary(planSpace, item.frameCount, 48_000),
"AudioProgramPlan sample count differs from its frame domain");
assert(Array.isArray(item.clips), "AudioProgramPlan clips are invalid");
const ids = new Set<string>();
@@ -57,16 +47,26 @@ export function verifyAudioProgramPlan(value: unknown): asserts value is AudioPr
`AudioProgramPlan clip ${clip.id} target interval is invalid`);
assert(Number.isSafeInteger(clip.sourceStartSample) && clip.sourceStartSample >= 0,
`AudioProgramPlan clip ${clip.id} source start is invalid`);
+ assert(Number.isSafeInteger(clip.sourceSampleFrames) && clip.sourceSampleFrames > 0
+ && Number.isSafeInteger(clip.sourceEndSampleExclusive)
+ && clip.sourceEndSampleExclusive > clip.sourceStartSample
+ && clip.sourceEndSampleExclusive <= clip.sourceSampleFrames,
+ `AudioProgramPlan clip ${clip.id} source interval is invalid`);
+ const sourceLength = clip.sourceEndSampleExclusive - clip.sourceStartSample;
+ assert(typeof clip.sourceLoop === "boolean"
+ && Number.isSafeInteger(clip.sourcePhaseSample) && clip.sourcePhaseSample >= 0
+ && clip.sourcePhaseSample < sourceLength
+ && (clip.sourceLoop || clip.sourcePhaseSample === 0),
+ `AudioProgramPlan clip ${clip.id} loop phase is invalid`);
assert(Number.isFinite(clip.playbackRate) && clip.playbackRate > 0 && clip.playbackRate <= 100,
`AudioProgramPlan clip ${clip.id} playback rate is invalid`);
+ assert(clip.pitch === "preserve", `AudioProgramPlan clip ${clip.id} pitch policy is invalid`);
assert(Number.isFinite(clip.gain) && clip.gain >= 0 && clip.gain <= 64,
`AudioProgramPlan clip ${clip.id} gain is invalid`);
const length = clip.targetEndSampleExclusive - clip.targetStartSample;
assert(Number.isSafeInteger(clip.fadeInSamples) && clip.fadeInSamples >= 0 && clip.fadeInSamples <= length
&& Number.isSafeInteger(clip.fadeOutSamples) && clip.fadeOutSamples >= 0 && clip.fadeOutSamples <= length,
`AudioProgramPlan clip ${clip.id} fade is invalid`);
- assert(clip.bus === "speech" || clip.bus === "music" || clip.bus === "sfx" || clip.bus === "source",
- `AudioProgramPlan clip ${clip.id} bus is invalid`);
}
assert(item.mix?.normalize === false && item.mix?.limiter === "none",
"AudioProgramPlan cannot hide normalization or limiting");
@@ -75,15 +75,12 @@ export function verifyAudioProgramPlan(value: unknown): asserts value is AudioPr
export function compileAudioProgramPlan(composition: Composition, programSpace: ProgramSpace): AudioProgramPlan {
assertCompositionIdentity(composition, programSpace);
const frameCount = programSpaceFrameCount(programSpace);
- const { numerator, denominator } = programSpace.frameRate;
const clips = composition.tracks
.filter((track) => track.contract === "svml.audio-track@1")
.flatMap((track) => track.clips.map((clip) => {
if (clip.artifact.mediaType !== "audio/wav") {
throw new Error(`Audio clip ${track.id}.${clip.id} must be normalized to canonical WAV before mixing`);
}
- const targetStartSample = sampleBoundary(clip.span.startFrame, numerator, denominator);
- const targetEndSampleExclusive = sampleBoundary(clip.span.endFrameExclusive, numerator, denominator);
return {
id: `${track.id}:${clip.id}`,
artifact: {
@@ -92,14 +89,18 @@ export function compileAudioProgramPlan(composition: Composition, programSpace:
size: clip.artifact.size,
mediaType: clip.artifact.mediaType,
},
- targetStartSample,
- targetEndSampleExclusive,
- sourceStartSample: samplesFromSeconds(clip.mediaStartSec ?? 0, `${track.id}.${clip.id}.mediaStartSec`),
- playbackRate: clip.playbackRate ?? 1,
- gain: clip.gain ?? 1,
- fadeInSamples: samplesFromSeconds(clip.fadeInSec ?? 0, `${track.id}.${clip.id}.fadeInSec`),
- fadeOutSamples: samplesFromSeconds(clip.fadeOutSec ?? 0, `${track.id}.${clip.id}.fadeOutSec`),
- bus: clip.bus ?? "source" as const,
+ targetStartSample: clip.target.startSample,
+ targetEndSampleExclusive: clip.target.endSampleExclusive,
+ sourceSampleFrames: clip.source.sampleFrames,
+ sourceStartSample: clip.source.startSample,
+ sourceEndSampleExclusive: clip.source.endSampleExclusive,
+ sourceLoop: clip.source.loop,
+ sourcePhaseSample: clip.source.phaseSample,
+ playbackRate: clip.playbackRate,
+ pitch: clip.pitch,
+ gain: clip.gain,
+ fadeInSamples: clip.fadeInSamples,
+ fadeOutSamples: clip.fadeOutSamples,
};
}));
return sealAudioProgramPlan({
@@ -107,7 +108,7 @@ export function compileAudioProgramPlan(composition: Composition, programSpace:
frameRate: { ...programSpace.frameRate },
frameCount,
sampleRate: 48_000,
- sampleFrames: sampleBoundary(frameCount, numerator, denominator),
+ sampleFrames: programFrameSampleBoundary(programSpace, frameCount, 48_000),
clips,
mix: { normalize: false, limiter: "none" },
});
diff --git a/packages/media-pipeline/src/manifest.ts b/packages/media-pipeline/src/manifest.ts
index 2be0d155..098803f6 100644
--- a/packages/media-pipeline/src/manifest.ts
+++ b/packages/media-pipeline/src/manifest.ts
@@ -108,12 +108,16 @@ export const audioProgramPlanSchema: ValueSchema = {
artifact: { schema: blobRefSchema },
targetStartSample: { schema: integer },
targetEndSampleExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },
+ sourceSampleFrames: { schema: { kind: "number", integer: true, minimum: 1 } },
sourceStartSample: { schema: integer },
+ sourceEndSampleExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },
+ sourceLoop: { schema: { kind: "boolean" } },
+ sourcePhaseSample: { schema: integer },
playbackRate: { schema: { kind: "number", minimum: 0.000001, maximum: 100 } },
+ pitch: { schema: { kind: "literal", value: "preserve" } },
gain: { schema: { kind: "number", minimum: 0, maximum: 64 } },
fadeInSamples: { schema: integer },
fadeOutSamples: { schema: integer },
- bus: { schema: { kind: "string", enum: ["speech", "music", "sfx", "source"] } },
},
} } },
mix: { schema: {
diff --git a/packages/media-pipeline/src/types.ts b/packages/media-pipeline/src/types.ts
index 4ff80244..186a74ec 100644
--- a/packages/media-pipeline/src/types.ts
+++ b/packages/media-pipeline/src/types.ts
@@ -55,12 +55,16 @@ export type AudioProgramClip = {
readonly artifact: BlobRef;
readonly targetStartSample: number;
readonly targetEndSampleExclusive: number;
+ readonly sourceSampleFrames: number;
readonly sourceStartSample: number;
+ readonly sourceEndSampleExclusive: number;
+ readonly sourceLoop: boolean;
+ readonly sourcePhaseSample: number;
readonly playbackRate: number;
+ readonly pitch: "preserve";
readonly gain: number;
readonly fadeInSamples: number;
readonly fadeOutSamples: number;
- readonly bus: "speech" | "music" | "sfx" | "source";
};
/** Pure, content-addressed plan. Executing it is always a Provider Need. */
diff --git a/packages/media-track/README.md b/packages/media-track/README.md
new file mode 100644
index 00000000..50cd67f7
--- /dev/null
+++ b/packages/media-track/README.md
@@ -0,0 +1,12 @@
+# `@narratage/media-track`
+
+Official provider-free Media Item and Sequence authoring package.
+
+The current executable slice implements one independently timed still-image Item. Its Artifact,
+intrinsic extent, Placement Frame, Content Fit and ProgramSpace are separate graph inputs. The
+resolved package-owned Program lowers to an ordinary peer `VisualTrack`; it does not add Media,
+B-roll or Provider meaning to Core, Film, Composition or HyperFrames.
+
+Timed visual sources, frame Paint, motion, Sequence handoffs, explicit audio projection and the
+author Surface remain governed by `spec/media-track.md` and are intentionally not claimed by this
+first witness.
diff --git a/packages/media-track/package.json b/packages/media-track/package.json
new file mode 100644
index 00000000..9b73fa4c
--- /dev/null
+++ b/packages/media-track/package.json
@@ -0,0 +1,23 @@
+{
+ "name": "@narratage/media-track",
+ "version": "0.0.0-dev",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "svml": {
+ "activation": "./src/activation.ts"
+ },
+ "dependencies": {
+ "@narratage/artifact": "workspace:*",
+ "@narratage/component-kit": "workspace:*",
+ "@narratage/composition": "workspace:*",
+ "@narratage/elaborator": "workspace:*",
+ "@narratage/media": "workspace:*",
+ "@narratage/program-space": "workspace:*",
+ "@narratage/protocol": "workspace:*",
+ "@narratage/spatial": "workspace:*",
+ "@narratage/temporal": "workspace:*"
+ }
+}
diff --git a/packages/media-track/src/activation.ts b/packages/media-track/src/activation.ts
new file mode 100644
index 00000000..1a330b84
--- /dev/null
+++ b/packages/media-track/src/activation.ts
@@ -0,0 +1,9 @@
+import { mediaTrackComponent, mediaTrackManifest, mediaTrackModuleRef } from "./index.js";
+
+export const svmlPackage = {
+ format: "svml.node-package@1" as const,
+ name: "@narratage/media-track",
+ modules: [{ manifest: mediaTrackManifest, specifiers: [mediaTrackModuleRef.name, `${mediaTrackModuleRef.name}@1`] }],
+ components: [mediaTrackComponent],
+};
+export default svmlPackage;
diff --git a/packages/media-track/src/component.ts b/packages/media-track/src/component.ts
new file mode 100644
index 00000000..253f27c7
--- /dev/null
+++ b/packages/media-track/src/component.ts
@@ -0,0 +1,57 @@
+import type { ComponentPackage } from "@narratage/component-kit";
+import type { ProgramSpace } from "@narratage/program-space";
+import { canonicalize } from "@narratage/protocol";
+import type { BlobRef, StoredValue } from "@narratage/protocol";
+import type { ContentFit, IntrinsicExtent, SpatialFrame } from "@narratage/spatial";
+
+import { mediaTrackProducers, mediaTrackTypes } from "./manifest.js";
+import {
+ appendFullStillMediaItem,
+ assertMediaTrackProgram,
+ createMediaTrackSet,
+ finalizeMediaTrack,
+ mediaTrackImplementationDigests,
+ mediaTrackValidatorDigests,
+ renderMediaTrack,
+} from "./program.js";
+import type { MediaStillItemSpec, MediaTrackHeader, MediaTrackProgram, MediaTrackSet } from "./types.js";
+
+function inline<T>(value: StoredValue | undefined, label: string): T {
+ if (value?.kind !== "inline") throw new Error(`${label} must be inline.`);
+ return value.value as unknown as T;
+}
+function blob(value: StoredValue | undefined): BlobRef {
+ if (value?.kind !== "blob") throw new Error("Media source must be a BlobArtifact.");
+ return value;
+}
+const output = (value: unknown) => ({ kind: "inline" as const, value: canonicalize(value) });
+
+export const mediaTrackComponent = {
+ name: "@narratage/media-track",
+ producers: [
+ { producer: mediaTrackProducers.createSet, implementationDigest: mediaTrackImplementationDigests.createSet, handler: () => ({ outputs: { set: output(createMediaTrackSet()) }, needs: {} }) },
+ { producer: mediaTrackProducers.appendFullStill, implementationDigest: mediaTrackImplementationDigests.appendFullStill, handler: ({ inputs }) => ({ outputs: { set: output(appendFullStillMediaItem(
+ inline<MediaTrackSet>(inputs.set?.value, "MediaTrackSet"),
+ inline<MediaTrackHeader>(inputs.header?.value, "MediaTrackHeader"),
+ inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"),
+ blob(inputs.source?.value),
+ inline<IntrinsicExtent>(inputs.extent?.value, "IntrinsicExtent"),
+ inline<SpatialFrame>(inputs.frame?.value, "SpatialFrame"),
+ inline<ContentFit>(inputs.fit?.value, "ContentFit"),
+ inline<MediaStillItemSpec>(inputs.spec?.value, "MediaStillItemSpec"),
+ )) }, needs: {} }) },
+ { producer: mediaTrackProducers.finalize, implementationDigest: mediaTrackImplementationDigests.finalize, handler: ({ inputs }) => ({ outputs: { program: output(finalizeMediaTrack(
+ inline<MediaTrackSet>(inputs.set?.value, "MediaTrackSet"),
+ inline<MediaTrackHeader>(inputs.header?.value, "MediaTrackHeader"),
+ )) }, needs: {} }) },
+ { producer: mediaTrackProducers.render, implementationDigest: mediaTrackImplementationDigests.render, handler: ({ inputs }) => ({ outputs: { track: output(renderMediaTrack(
+ inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"),
+ inline<MediaTrackProgram>(inputs.program?.value, "MediaTrackProgram"),
+ )) }, needs: {} }) },
+ ],
+ validators: [{
+ type: mediaTrackTypes.program,
+ implementationDigest: mediaTrackValidatorDigests.program,
+ handler: ({ value }) => assertMediaTrackProgram(inline<MediaTrackProgram>(value, "MediaTrackProgram")),
+ }],
+} satisfies ComponentPackage;
diff --git a/packages/media-track/src/fragment.ts b/packages/media-track/src/fragment.ts
new file mode 100644
index 00000000..c7761405
--- /dev/null
+++ b/packages/media-track/src/fragment.ts
@@ -0,0 +1,43 @@
+import { artifactTypes } from "@narratage/artifact";
+import { compositionTypes } from "@narratage/composition";
+import { sealGraphFragment } from "@narratage/elaborator";
+import { programSpaceTypes } from "@narratage/program-space";
+import { spatialTypes } from "@narratage/spatial";
+
+import { mediaTrackProducers, mediaTrackTypes } from "./manifest.js";
+
+const input = (name: string) => ({ kind: "fragment-input" as const, name });
+const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
+
+export const stillMediaTrackFragment = sealGraphFragment({
+ name: "@narratage/media-track/one-full-still@1",
+ inputs: [
+ { name: "header", type: mediaTrackTypes.header },
+ { name: "space", type: programSpaceTypes.programSpace },
+ { name: "source", type: artifactTypes.blob },
+ { name: "extent", type: spatialTypes.extent },
+ { name: "frame", type: spatialTypes.frame },
+ { name: "fit", type: spatialTypes.fit },
+ { name: "spec", type: mediaTrackTypes.stillItemSpec },
+ ],
+ operations: [
+ { id: "set", producer: mediaTrackProducers.createSet, inputs: {}, result: { kind: "output", name: "set" } },
+ { id: "append", producer: mediaTrackProducers.appendFullStill, inputs: {
+ set: operation("set"), header: input("header"), space: input("space"), source: input("source"),
+ extent: input("extent"), frame: input("frame"), fit: input("fit"), spec: input("spec"),
+ }, result: { kind: "output", name: "set" } },
+ { id: "finalize", producer: mediaTrackProducers.finalize, inputs: { set: operation("append"), header: input("header") }, result: { kind: "output", name: "program" } },
+ { id: "render", producer: mediaTrackProducers.render, inputs: { space: input("space"), program: operation("finalize") }, result: { kind: "output", name: "track" } },
+ ],
+ exports: [
+ { name: "program", type: mediaTrackTypes.program, root: operation("finalize"), semanticInputs: ["header", "space", "source", "extent", "frame", "fit", "spec"], fidelity: "exact" },
+ { name: "track", type: compositionTypes.visualTrack, root: operation("render"), semanticInputs: ["header", "space", "source", "extent", "frame", "fit", "spec"], fidelity: "exact" },
+ ],
+});
+
+export const renderMediaTrackFragment = sealGraphFragment({
+ name: "@narratage/media-track/render@1",
+ inputs: [{ name: "space", type: programSpaceTypes.programSpace }, { name: "program", type: mediaTrackTypes.program }],
+ operations: [{ id: "render", producer: mediaTrackProducers.render, inputs: { space: input("space"), program: input("program") }, result: { kind: "output", name: "track" } }],
+ exports: [{ name: "track", type: compositionTypes.visualTrack, root: operation("render"), semanticInputs: ["space", "program"], fidelity: "exact" }],
+});
diff --git a/packages/media-track/src/index.ts b/packages/media-track/src/index.ts
new file mode 100644
index 00000000..7eee9f42
--- /dev/null
+++ b/packages/media-track/src/index.ts
@@ -0,0 +1,31 @@
+export { mediaTrackComponent } from "./component.js";
+export { renderMediaTrackFragment, stillMediaTrackFragment } from "./fragment.js";
+export {
+ mediaStillItemSpecSchema,
+ mediaTrackDependency,
+ mediaTrackHeaderSchema,
+ mediaTrackManifest,
+ mediaTrackManifestDigest,
+ mediaTrackModuleRef,
+ mediaTrackProducers,
+ mediaTrackProgramSchema,
+ mediaTrackSetSchema,
+ mediaTrackTypes,
+} from "./manifest.js";
+export {
+ appendFullStillMediaItem,
+ assertMediaStillItemSpec,
+ assertMediaTrackHeader,
+ assertMediaTrackProgram,
+ assertMediaTrackProgramIdentity,
+ assertMediaTrackSet,
+ createMediaTrackSet,
+ finalizeMediaTrack,
+ mediaTrackImplementationDigests,
+ mediaTrackValidatorDigests,
+ renderMediaTrack,
+ sealMediaStillItemSpec,
+ sealMediaTrackHeader,
+ sealMediaTrackProgram,
+} from "./program.js";
+export type * from "./types.js";
diff --git a/packages/media-track/src/manifest.ts b/packages/media-track/src/manifest.ts
new file mode 100644
index 00000000..0e845990
--- /dev/null
+++ b/packages/media-track/src/manifest.ts
@@ -0,0 +1,100 @@
+import { artifactDependency, artifactTypes } from "@narratage/artifact";
+import { compositionDependency, compositionTypes } from "@narratage/composition";
+import { mediaArtifactSchema, mediaDependency } from "@narratage/media";
+import { programSpaceDependency, programSpaceTypes } from "@narratage/program-space";
+import { digestOf } from "@narratage/protocol";
+import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@narratage/protocol";
+import {
+ contentFitSchema,
+ intrinsicExtentSchema,
+ spatialDependency,
+ spatialFrameSchema,
+ spatialTypes,
+} from "@narratage/spatial";
+
+import { mediaTrackImplementationDigests, mediaTrackValidatorDigests } from "./program.js";
+
+export const mediaTrackModuleRef = { name: "@narratage/media-track", version: "0.0.0-dev" } as const;
+export const mediaTrackTypes = {
+ program: { module: mediaTrackModuleRef, name: "MediaTrackProgram" },
+ header: { module: mediaTrackModuleRef, name: "MediaTrackHeader" },
+ stillItemSpec: { module: mediaTrackModuleRef, name: "MediaStillItemSpec" },
+ set: { module: mediaTrackModuleRef, name: "MediaTrackSet" },
+} satisfies Record<string, TypeRef>;
+export const mediaTrackProducers = {
+ createSet: { module: mediaTrackModuleRef, name: "create-media-track-set" },
+ appendFullStill: { module: mediaTrackModuleRef, name: "append-full-still-media-item" },
+ finalize: { module: mediaTrackModuleRef, name: "finalize-media-track" },
+ render: { module: mediaTrackModuleRef, name: "render-media-track" },
+} satisfies Record<string, ProducerRef>;
+
+const string = { kind: "string", minLength: 1 } as const;
+const integer = { kind: "number", integer: true } as const;
+const unsignedInteger = { kind: "number", integer: true, minimum: 0 } as const;
+const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>): ValueSchema => ({ kind: "object", fields });
+const frameSpan = object({ startFrame: { schema: unsignedInteger }, endFrameExclusive: { schema: unsignedInteger } });
+const presentation = object({
+ clip: { schema: { kind: "literal", value: "frame" } },
+ fill: { schema: object({ kind: { schema: { kind: "literal", value: "transparent" } } }) },
+});
+const stillLayer = object({
+ id: { schema: string },
+ kind: { schema: { kind: "literal", value: "still" } },
+ artifact: { schema: mediaArtifactSchema },
+ extent: { schema: intrinsicExtentSchema },
+ fit: { schema: contentFitSchema },
+});
+const item = object({
+ id: { schema: string }, span: { schema: frameSpan }, frame: { schema: spatialFrameSchema },
+ presentation: { schema: presentation },
+ layers: { schema: { kind: "array", minItems: 1, items: stillLayer } },
+ stacking: { schema: object({ order: { schema: integer }, tieBreak: { schema: string } }) },
+});
+export const mediaTrackProgramSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.media-track-program@1" } },
+ id: { schema: string }, items: { schema: { kind: "array", minItems: 1, items: item } },
+});
+export const mediaTrackHeaderSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.media-track-header@1" } }, id: { schema: string },
+});
+export const mediaStillItemSpecSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.media-still-item-spec@1" } },
+ id: { schema: string }, stackingOrder: { schema: integer }, presentation: { schema: presentation },
+});
+export const mediaTrackSetSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.media-track-set@1" } },
+ items: { schema: { kind: "array", items: item } },
+});
+
+const validator = (locator: string, digest: ReturnType<typeof digestOf>) => ({
+ abi: "svml.type-validator@1" as const,
+ implementation: { kind: "registered" as const, locator, digest },
+});
+
+export const mediaTrackManifest: ModuleManifest = {
+ format: "svml.module@1",
+ name: mediaTrackModuleRef.name,
+ version: mediaTrackModuleRef.version,
+ dependencies: [artifactDependency, mediaDependency, programSpaceDependency, spatialDependency, compositionDependency],
+ types: [
+ { name: mediaTrackTypes.program.name, schema: mediaTrackProgramSchema, validator: validator("@narratage/media-track/validate-program", mediaTrackValidatorDigests.program) },
+ { name: mediaTrackTypes.header.name, schema: mediaTrackHeaderSchema },
+ { name: mediaTrackTypes.stillItemSpec.name, schema: mediaStillItemSpecSchema },
+ { name: mediaTrackTypes.set.name, schema: mediaTrackSetSchema },
+ ],
+ capabilities: [],
+ surfaces: [],
+ producers: [
+ { name: mediaTrackProducers.createSet.name, inputs: [], outputs: [{ name: "set", type: mediaTrackTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/media-track/create-set", digest: mediaTrackImplementationDigests.createSet } },
+ { name: mediaTrackProducers.appendFullStill.name, inputs: [
+ { name: "set", type: mediaTrackTypes.set }, { name: "header", type: mediaTrackTypes.header },
+ { name: "space", type: programSpaceTypes.programSpace }, { name: "source", type: artifactTypes.blob },
+ { name: "extent", type: spatialTypes.extent }, { name: "frame", type: spatialTypes.frame },
+ { name: "fit", type: spatialTypes.fit }, { name: "spec", type: mediaTrackTypes.stillItemSpec },
+ ], outputs: [{ name: "set", type: mediaTrackTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/media-track/append-full-still", digest: mediaTrackImplementationDigests.appendFullStill } },
+ { name: mediaTrackProducers.finalize.name, inputs: [{ name: "set", type: mediaTrackTypes.set }, { name: "header", type: mediaTrackTypes.header }], outputs: [{ name: "program", type: mediaTrackTypes.program }], needs: [], implementation: { kind: "registered", locator: "@narratage/media-track/finalize", digest: mediaTrackImplementationDigests.finalize } },
+ { name: mediaTrackProducers.render.name, inputs: [{ name: "space", type: programSpaceTypes.programSpace }, { name: "program", type: mediaTrackTypes.program }], outputs: [{ name: "track", type: compositionTypes.visualTrack }], needs: [], implementation: { kind: "registered", locator: "@narratage/media-track/render", digest: mediaTrackImplementationDigests.render } },
+ ],
+};
+export const mediaTrackManifestDigest = digestOf(mediaTrackManifest);
+export const mediaTrackDependency = { module: mediaTrackModuleRef, digest: mediaTrackManifestDigest } as const;
diff --git a/packages/media-track/src/program.ts b/packages/media-track/src/program.ts
new file mode 100644
index 00000000..5976a2d4
--- /dev/null
+++ b/packages/media-track/src/program.ts
@@ -0,0 +1,234 @@
+import { assertProgramSpaceIdentity, programSpaceFrameCount } from "@narratage/program-space";
+import type { ProgramSpace } from "@narratage/program-space";
+import { sealVisualTrack } from "@narratage/composition";
+import type { VisualElement, VisualStyleDeclaration, VisualTrack } from "@narratage/composition";
+import { canonicalize, digestOf, isDigest } from "@narratage/protocol";
+import type { BlobRef } from "@narratage/protocol";
+import {
+ assertContentFit,
+ assertIntrinsicExtent,
+ assertSpatialFrame,
+ fitContent,
+} from "@narratage/spatial";
+import type { ContentFit, IntrinsicExtent, SpatialFrame } from "@narratage/spatial";
+import { projectProgramWindow } from "@narratage/temporal";
+
+import type {
+ MediaItemProgram,
+ MediaStillItemSpec,
+ MediaTrackHeader,
+ MediaTrackProgram,
+ MediaTrackSet,
+} from "./types.js";
+
+export const mediaTrackImplementationDigests = {
+ createSet: digestOf("@narratage/media-track/create-set@1"),
+ appendFullStill: digestOf("@narratage/media-track/append-full-still@1"),
+ finalize: digestOf("@narratage/media-track/finalize@1"),
+ render: digestOf("@narratage/media-track/render@1"),
+} as const;
+
+export const mediaTrackValidatorDigests = {
+ program: digestOf("@narratage/media-track/validate-program@1"),
+} as const;
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function assertIdentity(value: string, label: string): void {
+ assert(/^[A-Za-z][A-Za-z0-9_.:-]{0,191}$/u.test(value), `${label} is invalid.`);
+}
+
+function assertPresentation(value: MediaStillItemSpec["presentation"], label: string): void {
+ assert(value.clip === "frame" && value.fill.kind === "transparent", `${label} is unsupported.`);
+}
+
+function assertArtifact(value: MediaItemProgram["layers"][number]["artifact"], label: string): void {
+ assert(isDigest(value.digest) && Number.isSafeInteger(value.size) && value.size >= 0
+ && value.mediaType.startsWith("image/") && value.durationSec === 0, `${label} is not a still-image Artifact.`);
+}
+
+export function sealMediaTrackHeader(value: MediaTrackHeader): MediaTrackHeader {
+ assertMediaTrackHeader(value);
+ return canonicalize(value) as unknown as MediaTrackHeader;
+}
+
+export function assertMediaTrackHeader(value: MediaTrackHeader): void {
+ assert(value.contract === "svml.media-track-header@1", "Unsupported MediaTrackHeader contract.");
+ assertIdentity(value.id, "MediaTrackHeader.id");
+}
+
+export function sealMediaStillItemSpec(value: MediaStillItemSpec): MediaStillItemSpec {
+ assertMediaStillItemSpec(value);
+ return canonicalize(value) as unknown as MediaStillItemSpec;
+}
+
+export function assertMediaStillItemSpec(value: MediaStillItemSpec): void {
+ assert(value.contract === "svml.media-still-item-spec@1", "Unsupported MediaStillItemSpec contract.");
+ assertIdentity(value.id, "MediaStillItemSpec.id");
+ assert(Number.isSafeInteger(value.stackingOrder), "MediaStillItemSpec.stackingOrder must be an integer.");
+ assertPresentation(value.presentation, "MediaStillItemSpec.presentation");
+}
+
+export function createMediaTrackSet(): MediaTrackSet {
+ return { contract: "svml.media-track-set@1", items: [] };
+}
+
+export function assertMediaTrackSet(value: MediaTrackSet): void {
+ assert(value.contract === "svml.media-track-set@1" && Array.isArray(value.items), "MediaTrackSet is invalid.");
+}
+
+export function appendFullStillMediaItem(
+ set: MediaTrackSet,
+ header: MediaTrackHeader,
+ space: ProgramSpace,
+ source: BlobRef,
+ extent: IntrinsicExtent,
+ frame: SpatialFrame,
+ fit: ContentFit,
+ spec: MediaStillItemSpec,
+): MediaTrackSet {
+ assertMediaTrackSet(set);
+ assertMediaTrackHeader(header);
+ assertProgramSpaceIdentity(space);
+ assert(source.kind === "blob" && source.mediaType.startsWith("image/"), "Media still source must be an image BlobArtifact.");
+ assertIntrinsicExtent(extent);
+ assertSpatialFrame(frame);
+ assertContentFit(fit);
+ assertMediaStillItemSpec(spec);
+ assert(!set.items.some((item) => item.id === spec.id), `Media Track already contains Item ${spec.id}.`);
+ const projected = projectProgramWindow({
+ itemId: spec.id,
+ space,
+ projection: { start: { ref: "program.start" }, end: { ref: "program.end" } },
+ });
+ const item: MediaItemProgram = {
+ id: spec.id,
+ span: projected.span,
+ frame: { ...frame },
+ presentation: structuredClone(spec.presentation),
+ layers: [{
+ id: `${spec.id}:content`,
+ kind: "still",
+ artifact: {
+ digest: source.digest,
+ size: source.size,
+ mediaType: source.mediaType,
+ durationSec: 0,
+ },
+ extent: { ...extent },
+ fit: structuredClone(fit),
+ }],
+ stacking: { order: spec.stackingOrder, tieBreak: `${header.id}:${spec.id}` },
+ };
+ return { contract: "svml.media-track-set@1", items: [...set.items, item] };
+}
+
+function normalizeProgram(value: MediaTrackProgram): MediaTrackProgram {
+ return {
+ contract: "svml.media-track-program@1",
+ id: value.id,
+ items: [...value.items]
+ .map((item) => structuredClone(item))
+ .sort((left, right) => left.id.localeCompare(right.id)),
+ };
+}
+
+export function sealMediaTrackProgram(value: MediaTrackProgram): MediaTrackProgram {
+ const normalized = normalizeProgram(value);
+ assertMediaTrackProgram(normalized);
+ return canonicalize(normalized) as unknown as MediaTrackProgram;
+}
+
+export function finalizeMediaTrack(set: MediaTrackSet, header: MediaTrackHeader): MediaTrackProgram {
+ assertMediaTrackSet(set);
+ assertMediaTrackHeader(header);
+ assert(set.items.length > 0, "Media Track requires at least one Item.");
+ return sealMediaTrackProgram({ contract: "svml.media-track-program@1", id: header.id, items: set.items });
+}
+
+export function assertMediaTrackProgram(value: MediaTrackProgram): void {
+ assert(value.contract === "svml.media-track-program@1", "Unsupported MediaTrackProgram contract.");
+ assertIdentity(value.id, "MediaTrackProgram.id");
+ assert(value.items.length > 0, "MediaTrackProgram requires at least one Item.");
+ const ids = new Set<string>();
+ for (const item of value.items) {
+ assertIdentity(item.id, "MediaItemProgram.id");
+ assert(!ids.has(item.id), `MediaTrackProgram contains duplicate Item ${item.id}.`);
+ ids.add(item.id);
+ assert(Number.isSafeInteger(item.span.startFrame) && Number.isSafeInteger(item.span.endFrameExclusive)
+ && item.span.startFrame >= 0 && item.span.endFrameExclusive > item.span.startFrame,
+ `Media Item ${item.id} span is invalid.`);
+ assertSpatialFrame(item.frame);
+ assertPresentation(item.presentation, `Media Item ${item.id} presentation`);
+ assert(Number.isSafeInteger(item.stacking.order) && item.stacking.tieBreak.length > 0,
+ `Media Item ${item.id} stacking is invalid.`);
+ assert(item.layers.length > 0, `Media Item ${item.id} requires at least one layer.`);
+ const layerIds = new Set<string>();
+ for (const layer of item.layers) {
+ assertIdentity(layer.id, `Media Item ${item.id} layer id`);
+ assert(!layerIds.has(layer.id), `Media Item ${item.id} contains duplicate layer ${layer.id}.`);
+ layerIds.add(layer.id);
+ assert(layer.kind === "still", `Media Item ${item.id} contains an unsupported layer.`);
+ assertArtifact(layer.artifact, `Media Item ${item.id} layer ${layer.id}`);
+ assertIntrinsicExtent(layer.extent);
+ assertContentFit(layer.fit);
+ }
+ }
+}
+
+export function assertMediaTrackProgramIdentity(value: MediaTrackProgram, space: ProgramSpace): void {
+ assertMediaTrackProgram(value);
+ assertProgramSpaceIdentity(space);
+ const totalFrames = programSpaceFrameCount(space);
+ for (const item of value.items) {
+ assert(item.span.endFrameExclusive <= totalFrames, `Media Item ${item.id} is outside ProgramSpace.`);
+ }
+}
+
+function px(value: number): string { return `${value}px`; }
+
+function itemElements(item: MediaItemProgram): readonly VisualElement[] {
+ const rootStyle: VisualStyleDeclaration[] = [
+ { name: "height", value: px(item.frame.heightPx) },
+ { name: "left", value: px(item.frame.xPx) },
+ { name: "overflow", value: "hidden" },
+ { name: "position", value: "absolute" },
+ { name: "top", value: px(item.frame.yPx) },
+ { name: "width", value: px(item.frame.widthPx) },
+ ];
+ const media = item.layers.map((layer, index) => {
+ const fitted = fitContent(item.frame, layer.extent, layer.fit).contentFrame;
+ return {
+ id: layer.id,
+ parent: "root",
+ order: index + 1,
+ kind: "image" as const,
+ artifact: layer.artifact,
+ style: [
+ { name: "height", value: px(fitted.heightPx) },
+ { name: "left", value: px(fitted.xPx - item.frame.xPx) },
+ { name: "position", value: "absolute" },
+ { name: "top", value: px(fitted.yPx - item.frame.yPx) },
+ { name: "width", value: px(fitted.widthPx) },
+ ],
+ } satisfies VisualElement;
+ });
+ return [{ id: "root", order: 0, kind: "box", style: rootStyle }, ...media];
+}
+
+export function renderMediaTrack(space: ProgramSpace, program: MediaTrackProgram): VisualTrack {
+ assertMediaTrackProgramIdentity(program, space);
+ return sealVisualTrack({
+ contract: "svml.visual-track@1",
+ visualIr: "svml.visual-ir@1",
+ id: program.id,
+ presents: program.items.map((item) => ({
+ id: item.id,
+ span: { ...item.span },
+ stacking: { ...item.stacking },
+ elements: itemElements(item),
+ })),
+ });
+}
diff --git a/packages/media-track/src/types.ts b/packages/media-track/src/types.ts
new file mode 100644
index 00000000..099671cf
--- /dev/null
+++ b/packages/media-track/src/types.ts
@@ -0,0 +1,51 @@
+import type { FrameSpan } from "@narratage/composition";
+import type { MediaArtifactRef } from "@narratage/media";
+import type { ContentFit, IntrinsicExtent, SpatialFrame } from "@narratage/spatial";
+
+export type MediaFramePresentation = {
+ readonly clip: "frame";
+ readonly fill: { readonly kind: "transparent" };
+};
+
+export type MediaStillLayerProgram = {
+ readonly id: string;
+ readonly kind: "still";
+ readonly artifact: MediaArtifactRef;
+ readonly extent: IntrinsicExtent;
+ readonly fit: ContentFit;
+};
+
+export type MediaItemProgram = {
+ readonly id: string;
+ readonly span: FrameSpan;
+ readonly frame: SpatialFrame;
+ readonly presentation: MediaFramePresentation;
+ readonly layers: readonly MediaStillLayerProgram[];
+ readonly stacking: {
+ readonly order: number;
+ readonly tieBreak: string;
+ };
+};
+
+export type MediaTrackProgram = {
+ readonly contract: "svml.media-track-program@1";
+ readonly id: string;
+ readonly items: readonly MediaItemProgram[];
+};
+
+export type MediaTrackHeader = {
+ readonly contract: "svml.media-track-header@1";
+ readonly id: string;
+};
+
+export type MediaStillItemSpec = {
+ readonly contract: "svml.media-still-item-spec@1";
+ readonly id: string;
+ readonly stackingOrder: number;
+ readonly presentation: MediaFramePresentation;
+};
+
+export type MediaTrackSet = {
+ readonly contract: "svml.media-track-set@1";
+ readonly items: readonly MediaItemProgram[];
+};
diff --git a/packages/media-track/test/media-track.test.ts b/packages/media-track/test/media-track.test.ts
new file mode 100644
index 00000000..6f56672a
--- /dev/null
+++ b/packages/media-track/test/media-track.test.ts
@@ -0,0 +1,95 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { sealProgramSpace } from "@narratage/program-space";
+import { digestOf } from "@narratage/protocol";
+import type { BlobRef } from "@narratage/protocol";
+import {
+ appendFullStillMediaItem,
+ createMediaTrackSet,
+ finalizeMediaTrack,
+ renderMediaTrack,
+ sealMediaStillItemSpec,
+ sealMediaTrackHeader,
+ stillMediaTrackFragment,
+} from "@narratage/media-track";
+
+const space = sealProgramSpace({
+ contract: "svml.program-space@1",
+ durationSec: 4,
+ frameRate: { numerator: 30, denominator: 1 },
+});
+const source: BlobRef = {
+ kind: "blob",
+ digest: digestOf("media-track:still"),
+ size: 4_096,
+ mediaType: "image/png",
+};
+const extent = { contract: "svml.intrinsic-extent@1" as const, widthPx: 800, heightPx: 800 };
+const frame = { contract: "svml.spatial-frame@1" as const, xPx: 100, yPx: 200, widthPx: 400, heightPx: 300 };
+const fit = {
+ contract: "svml.content-fit@1" as const,
+ sizing: "contain" as const,
+ framePoint: { x: 0.5, y: 0.5 },
+ contentPoint: { x: 0.5, y: 0.5 },
+ offsetPx: { x: 0, y: 0 },
+ constraint: "bounded" as const,
+};
+
+test("one still Media Item consumes explicit time, source, extent, Frame and Fit edges", () => {
+ const header = sealMediaTrackHeader({ contract: "svml.media-track-header@1", id: "proof" });
+ const spec = sealMediaStillItemSpec({
+ contract: "svml.media-still-item-spec@1",
+ id: "product",
+ stackingOrder: 40,
+ presentation: { clip: "frame", fill: { kind: "transparent" } },
+ });
+ const program = finalizeMediaTrack(appendFullStillMediaItem(
+ createMediaTrackSet(), header, space, source, extent, frame, fit, spec,
+ ), header);
+ assert.deepEqual(program.items[0]?.span, { startFrame: 0, endFrameExclusive: 120 });
+ assert.equal(program.items[0]?.layers[0]?.artifact.digest, source.digest);
+ const track = renderMediaTrack(space, program);
+ assert.equal(track.presents.length, 1);
+ assert.deepEqual(track.presents[0]?.stacking, { order: 40, tieBreak: "proof:product" });
+ const root = track.presents[0]?.elements[0];
+ const image = track.presents[0]?.elements[1];
+ assert.equal(root?.kind, "box");
+ assert.deepEqual(root?.style, [
+ { name: "height", value: "300px" },
+ { name: "left", value: "100px" },
+ { name: "overflow", value: "hidden" },
+ { name: "position", value: "absolute" },
+ { name: "top", value: "200px" },
+ { name: "width", value: "400px" },
+ ]);
+ assert.equal(image?.kind, "image");
+ assert.deepEqual(image?.style, [
+ { name: "height", value: "300px" },
+ { name: "left", value: "50px" },
+ { name: "position", value: "absolute" },
+ { name: "top", value: "0px" },
+ { name: "width", value: "300px" },
+ ]);
+});
+
+test("the first Media Fragment exposes every external dependency as a semantic input", () => {
+ assert.deepEqual(stillMediaTrackFragment.inputs.map((input) => input.name), [
+ "extent", "fit", "frame", "header", "source", "space", "spec",
+ ]);
+ assert.deepEqual(stillMediaTrackFragment.exports[0]?.semanticInputs, [
+ "extent", "fit", "frame", "header", "source", "space", "spec",
+ ]);
+});
+
+test("a still Item rejects non-image bytes instead of inferring from a filename or Provider", () => {
+ const header = sealMediaTrackHeader({ contract: "svml.media-track-header@1", id: "proof" });
+ const spec = sealMediaStillItemSpec({
+ contract: "svml.media-still-item-spec@1", id: "bad", stackingOrder: 0,
+ presentation: { clip: "frame", fill: { kind: "transparent" } },
+ });
+ assert.throws(() => appendFullStillMediaItem(
+ createMediaTrackSet(), header, space,
+ { ...source, mediaType: "video/mp4" }, extent, frame, fit, spec,
+ ), /image BlobArtifact/u);
+});
diff --git a/packages/program-space/src/index.ts b/packages/program-space/src/index.ts
index c0423d4a..c89cf89c 100644
--- a/packages/program-space/src/index.ts
+++ b/packages/program-space/src/index.ts
@@ -42,11 +42,22 @@ export function programSpaceFrameCount(programSpace: ProgramSpace): number {
}
export function programSpaceSampleFrames(programSpace: ProgramSpace, sampleRate: number): number {
const frames = programSpaceFrameCount(programSpace);
+ return programFrameSampleBoundary(programSpace, frames, sampleRate);
+}
+export function programFrameSampleBoundary(
+ programSpace: ProgramSpace,
+ frame: number,
+ sampleRate: number,
+): number {
+ const frames = programSpaceFrameCount(programSpace);
+ if (!Number.isSafeInteger(frame) || frame < 0 || frame > frames) {
+ throw new Error("ProgramSpace frame boundary is invalid.");
+ }
if (!Number.isSafeInteger(sampleRate) || sampleRate <= 0) throw new Error("ProgramSpace sample rate is invalid.");
- const numerator = BigInt(frames) * BigInt(sampleRate) * BigInt(programSpace.frameRate.denominator);
+ const numerator = BigInt(frame) * BigInt(sampleRate) * BigInt(programSpace.frameRate.denominator);
const denominator = BigInt(programSpace.frameRate.numerator);
const value = (numerator * 2n + denominator) / (denominator * 2n);
- if (value < 1n || value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("ProgramSpace sample domain exceeds safe arithmetic.");
+ if (value < 0n || value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("ProgramSpace sample domain exceeds safe arithmetic.");
return Number(value);
}
export function assertProgramSpaceIdentity(programSpace: ProgramSpace): void {
diff --git a/packages/provider-media-aws-lambda/test/provider.test.ts b/packages/provider-media-aws-lambda/test/provider.test.ts
index 6ca68993..a0f6f2ca 100644
--- a/packages/provider-media-aws-lambda/test/provider.test.ts
+++ b/packages/provider-media-aws-lambda/test/provider.test.ts
@@ -32,7 +32,7 @@ const inspectNeed = {
} as unknown as CanonicalValue;
/** Installs the Provider into a registrar that keeps the handlers, as the Runtime does. */
-async function endpointFor(invoker: JsonInvoker, options: { bucket?: string } = {}) {
+async function endpointFor(invoker: JsonInvoker, options: { bucket?: string; capability?: string } = {}) {
const provider = createAwsLambdaMediaProvider({
functionArn: ARN,
bucket: options.bucket ?? "team-artifacts",
@@ -53,8 +53,9 @@ async function endpointFor(invoker: JsonInvoker, options: { bucket?: string } =
throw new Error("the AWS media Provider registers no recoverable endpoint");
},
} as never);
- const handler = handlers.get("inspect-media");
- assert.ok(handler, "the Provider offers media inspection");
+ const capability = options.capability ?? "inspect-media";
+ const handler = handlers.get(capability);
+ assert.ok(handler, `the Provider offers ${capability}`);
return { handler };
}
@@ -97,6 +98,59 @@ test("the Provider sends the Need verbatim with the bucket the function should u
assert.equal((outcome as { conformance: string }).conformance, "exact");
});
+test("the remote Provider receives the exact AudioProgramPlan compiled for local execution", async () => {
+ const store = new MemoryArtifactStore();
+ const output = await store.put(new TextEncoder().encode("timeline-audio"), "audio/wav");
+ const plan = canonicalize({
+ contract: "svml.audio-program-plan@1",
+ frameRate: { numerator: 30_000, denominator: 1_001 },
+ frameCount: 30,
+ sampleRate: 48_000,
+ sampleFrames: 48_048,
+ clips: [{
+ id: "end-loop",
+ artifact: { kind: "blob", digest: digestOf("audio-source"), size: 400, mediaType: "audio/wav" },
+ targetStartSample: 0,
+ targetEndSampleExclusive: 48_048,
+ sourceSampleFrames: 20_000,
+ sourceStartSample: 1_000,
+ sourceEndSampleExclusive: 19_000,
+ sourceLoop: true,
+ sourcePhaseSample: 12_048,
+ playbackRate: 1,
+ pitch: "preserve",
+ gain: 0.5,
+ fadeInSamples: 480,
+ fadeOutSamples: 960,
+ }],
+ mix: { normalize: false, limiter: "none" },
+ } as unknown as CanonicalValue);
+ const constraints = canonicalize({
+ contract: "svml.render-audio-request@1",
+ plan,
+ } as unknown as CanonicalValue);
+ const { invoker, seen } = recordingInvoker(() => canonicalize({
+ contract: "svml.media-lambda-response@1",
+ operation: "render-audio",
+ ok: true,
+ value: {
+ contract: "svml.timeline-audio@1",
+ artifact: output,
+ codec: "pcm_s16le",
+ sampleRate: 48_000,
+ channels: 2,
+ sampleFrames: 48_048,
+ loudness: "planned",
+ },
+ metadata: { provider: "media.aws-lambda" },
+ } as unknown as CanonicalValue));
+ const capability = await endpointFor(invoker, { capability: "render-timeline-audio" });
+ await capability.handler({ need: { constraints }, artifacts: store, credentials: {} } as never);
+ const request = parseMediaLambdaRequest(seen[0]);
+ assert.equal(request.operation, "render-audio");
+ assert.deepEqual(request.constraints, constraints);
+});
+
test("a Provider aimed at another bucket fails naming the Artifact, not later with an unreadable Record", async () => {
const store = new MemoryArtifactStore();
const stranger = digestOf("an artifact this Build's store never received");
diff --git a/packages/provider-media-local/test/provider.test.ts b/packages/provider-media-local/test/provider.test.ts
index 8484095d..96d51fed 100644
--- a/packages/provider-media-local/test/provider.test.ts
+++ b/packages/provider-media-local/test/provider.test.ts
@@ -38,6 +38,21 @@ async function run(executable: string, args: readonly string[]): Promise<void> {
});
}
+function rampWav(sampleFrames: number): Buffer {
+ const dataBytes = sampleFrames * 2 * 2;
+ const bytes = Buffer.alloc(44 + dataBytes);
+ bytes.write("RIFF", 0); bytes.writeUInt32LE(36 + dataBytes, 4); bytes.write("WAVE", 8);
+ bytes.write("fmt ", 12); bytes.writeUInt32LE(16, 16); bytes.writeUInt16LE(1, 20);
+ bytes.writeUInt16LE(2, 22); bytes.writeUInt32LE(48_000, 24); bytes.writeUInt32LE(48_000 * 4, 28);
+ bytes.writeUInt16LE(4, 32); bytes.writeUInt16LE(16, 34); bytes.write("data", 36); bytes.writeUInt32LE(dataBytes, 40);
+ for (let frame = 0; frame < sampleFrames; frame += 1) {
+ const value = frame + 1;
+ bytes.writeInt16LE(value, 44 + frame * 4);
+ bytes.writeInt16LE(value, 46 + frame * 4);
+ }
+ return bytes;
+}
+
function presentationSampleFrames(stream: MediaAudioStream): number {
assert(stream.startPts !== undefined && stream.endPts !== undefined);
const startNumerator = BigInt(stream.startPts.ticks) * BigInt(stream.startPts.timeBase.numerator);
@@ -435,24 +450,32 @@ test("local media Provider renders one frame-domain audio plan and muxes exactly
artifact: first,
targetStartSample: 0,
targetEndSampleExclusive: 24_000,
+ sourceSampleFrames: 24_000,
sourceStartSample: 0,
+ sourceEndSampleExclusive: 24_000,
+ sourceLoop: false,
+ sourcePhaseSample: 0,
playbackRate: 1,
+ pitch: "preserve",
gain: 1,
fadeInSamples: 0,
fadeOutSamples: 0,
- bus: "speech",
},
{
id: "speech:second",
artifact: second,
targetStartSample: 24_000,
targetEndSampleExclusive: 48_000,
+ sourceSampleFrames: 24_000,
sourceStartSample: 0,
+ sourceEndSampleExclusive: 24_000,
+ sourceLoop: false,
+ sourcePhaseSample: 0,
playbackRate: 1,
+ pitch: "preserve",
gain: 1,
fadeInSamples: 0,
fadeOutSamples: 0,
- bus: "speech",
},
],
mix: { normalize: false, limiter: "none" },
@@ -506,3 +529,60 @@ test("local media Provider renders one frame-domain audio plan and muxes exactly
await rm(root, { recursive: true, force: true });
}
});
+
+test("local media Provider executes an end-aligned loop from the exact authored sample phase", {
+ skip: !hasMediaBinaries,
+}, async () => {
+ const root = await mkdtemp(join(tmpdir(), "svml-provider-media-loop-"));
+ try {
+ const artifacts = new MemoryArtifactStore();
+ const source = await artifacts.put(rampWav(100), "audio/wav");
+ const plan = sealAudioProgramPlan({
+ contract: "svml.audio-program-plan@1",
+ frameRate: { numerator: 30, denominator: 1 },
+ frameCount: 30,
+ sampleRate: 48_000,
+ sampleFrames: 48_000,
+ clips: [{
+ id: "loop:end",
+ artifact: source,
+ targetStartSample: 0,
+ targetEndSampleExclusive: 250,
+ sourceSampleFrames: 100,
+ sourceStartSample: 0,
+ sourceEndSampleExclusive: 100,
+ sourceLoop: true,
+ sourcePhaseSample: 50,
+ playbackRate: 1,
+ pitch: "preserve",
+ gain: 1,
+ fadeInSamples: 0,
+ fadeOutSamples: 0,
+ }],
+ mix: { normalize: false, limiter: "none" },
+ });
+ const value = await fulfillInline(artifacts, need(
+ "need:render-loop-audio",
+ mediaPipelineCapabilities.renderAudio,
+ mediaTypes.timelineAudio,
+ canonicalize({ contract: "svml.render-audio-request@1", plan }),
+ ));
+ verifyTimelineAudio(value);
+ const audio = value as unknown as TimelineAudio;
+ assert.equal(audio.sampleFrames, 48_000);
+ const wav = await artifacts.get(audio.artifact.digest);
+ assert(wav);
+ const wavPath = join(root, "loop.wav");
+ const rawPath = join(root, "loop.raw");
+ await writeFile(wavPath, wav);
+ await run("ffmpeg", ["-v", "error", "-y", "-i", wavPath, "-f", "s16le", "-acodec", "pcm_s16le", rawPath]);
+ const raw = await readFile(rawPath);
+ assert.equal(raw.readInt16LE(0), 51, "end alignment must begin at source phase 50");
+ assert.equal(raw.readInt16LE(49 * 4), 100);
+ assert.equal(raw.readInt16LE(50 * 4), 1, "the complete authored source interval must loop");
+ assert.equal(raw.readInt16LE(249 * 4), 100);
+ assert.equal(raw.readInt16LE(250 * 4), 0, "silence is absence after the audible target interval");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
diff --git a/packages/screen-overlay/README.md b/packages/screen-overlay/README.md
new file mode 100644
index 00000000..46f6f181
--- /dev/null
+++ b/packages/screen-overlay/README.md
@@ -0,0 +1,11 @@
+# `@narratage/screen-overlay`
+
+Official provider-free self-contained screen-paint package.
+
+Every item consumes only its typed parameters, explicit Temporal source, CanvasSpace, ProgramSpace,
+seed and local frame. Flash, ColorWash, Vignette, ScanLines, DirectionalMatte, WhipVeil,
+GlitchVeil, Grain, LightLeak, Bokeh and TVStatic lower to ordinary absolute-stack `VisualTrack`
+Presents. No item can read lower pixels, sibling Tracks, Film state or hidden audio.
+
+Gaussian blur, color adjustment, zoom blur and cross-Track blend modes intentionally do not exist in
+this package. Their honest form is an explicit-input media/surface transformation.
diff --git a/packages/screen-overlay/package.json b/packages/screen-overlay/package.json
new file mode 100644
index 00000000..042d93ab
--- /dev/null
+++ b/packages/screen-overlay/package.json
@@ -0,0 +1,20 @@
+{
+ "name": "@narratage/screen-overlay",
+ "version": "0.0.0-dev",
+ "private": true,
+ "type": "module",
+ "exports": { ".": "./src/index.ts" },
+ "svml": { "activation": "./src/activation.ts" },
+ "dependencies": {
+ "@narratage/component-kit": "workspace:*",
+ "@narratage/composition": "workspace:*",
+ "@narratage/elaborator": "workspace:*",
+ "@narratage/narrative": "workspace:*",
+ "@narratage/program-space": "workspace:*",
+ "@narratage/protocol": "workspace:*",
+ "@narratage/semantic-map": "workspace:*",
+ "@narratage/spatial": "workspace:*",
+ "@narratage/temporal": "workspace:*",
+ "@narratage/text": "workspace:*"
+ }
+}
diff --git a/packages/screen-overlay/src/activation.ts b/packages/screen-overlay/src/activation.ts
new file mode 100644
index 00000000..8a3db42d
--- /dev/null
+++ b/packages/screen-overlay/src/activation.ts
@@ -0,0 +1,15 @@
+import { createTextSurfaceHostFacet } from "@narratage/text";
+import {
+ decodeScreenOverlaySurface, screenOverlayComponent, screenOverlayManifest, screenOverlayModuleRef,
+ screenOverlaySurfaceImplementationDigest,
+} from "./index.js";
+export const svmlPackage = {
+ format: "svml.node-package@1" as const, name: "@narratage/screen-overlay",
+ modules: [{ manifest: screenOverlayManifest, specifiers: [screenOverlayModuleRef.name, `${screenOverlayModuleRef.name}@1`] }],
+ components: [screenOverlayComponent],
+ hostFacets: [createTextSurfaceHostFacet({
+ module: screenOverlayModuleRef, surface: "track", mode: "structured",
+ implementationDigest: screenOverlaySurfaceImplementationDigest, handler: decodeScreenOverlaySurface,
+ })],
+};
+export default svmlPackage;
diff --git a/packages/screen-overlay/src/component.ts b/packages/screen-overlay/src/component.ts
new file mode 100644
index 00000000..cdfeb8c3
--- /dev/null
+++ b/packages/screen-overlay/src/component.ts
@@ -0,0 +1,28 @@
+import type { ComponentPackage } from "@narratage/component-kit";
+import type { NarrativeMomentRef, NarrativeSelectionRef } from "@narratage/narrative";
+import type { ProgramSpace } from "@narratage/program-space";
+import { canonicalize } from "@narratage/protocol";
+import type { StoredValue } from "@narratage/protocol";
+import type { CompleteSemanticMap } from "@narratage/semantic-map";
+import type { CanvasSpace } from "@narratage/spatial";
+import { screenOverlayProducers, screenOverlayTypes } from "./manifest.js";
+import {
+ appendMomentScreenOverlay, appendProgramScreenOverlay, appendSelectionScreenOverlay,
+ assertScreenOverlayProgram, createScreenOverlaySet, finalizeScreenOverlay, renderScreenOverlay,
+ screenOverlayImplementationDigests, screenOverlayValidatorDigests,
+} from "./program.js";
+import type { ScreenOverlayHeader, ScreenOverlayItemSpec, ScreenOverlayProgram, ScreenOverlaySet } from "./types.js";
+function inline<T>(value: StoredValue | undefined, label: string): T { if (value?.kind !== "inline") throw new Error(`${label} must be inline.`); return value.value as unknown as T; }
+const output = (value: unknown) => ({ kind: "inline" as const, value: canonicalize(value) });
+export const screenOverlayComponent = {
+ name: "@narratage/screen-overlay",
+ producers: [
+ { producer: screenOverlayProducers.createSet, implementationDigest: screenOverlayImplementationDigests.createSet, handler: () => ({ outputs: { set: output(createScreenOverlaySet()) }, needs: {} }) },
+ { producer: screenOverlayProducers.appendProgram, implementationDigest: screenOverlayImplementationDigests.appendProgram, handler: ({ inputs }) => ({ outputs: { set: output(appendProgramScreenOverlay(inline<ScreenOverlaySet>(inputs.set?.value, "ScreenOverlaySet"), inline<ScreenOverlayHeader>(inputs.header?.value, "ScreenOverlayHeader"), inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"), inline<ScreenOverlayItemSpec>(inputs.spec?.value, "ScreenOverlayItemSpec"))) }, needs: {} }) },
+ { producer: screenOverlayProducers.appendSelection, implementationDigest: screenOverlayImplementationDigests.appendSelection, handler: ({ inputs }) => ({ outputs: { set: output(appendSelectionScreenOverlay(inline<ScreenOverlaySet>(inputs.set?.value, "ScreenOverlaySet"), inline<ScreenOverlayHeader>(inputs.header?.value, "ScreenOverlayHeader"), inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"), inline<CompleteSemanticMap>(inputs.map?.value, "CompleteSemanticMap"), inline<NarrativeSelectionRef>(inputs.selection?.value, "NarrativeSelection"), inline<ScreenOverlayItemSpec>(inputs.spec?.value, "ScreenOverlayItemSpec"))) }, needs: {} }) },
+ { producer: screenOverlayProducers.appendMoment, implementationDigest: screenOverlayImplementationDigests.appendMoment, handler: ({ inputs }) => ({ outputs: { set: output(appendMomentScreenOverlay(inline<ScreenOverlaySet>(inputs.set?.value, "ScreenOverlaySet"), inline<ScreenOverlayHeader>(inputs.header?.value, "ScreenOverlayHeader"), inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"), inline<CompleteSemanticMap>(inputs.map?.value, "CompleteSemanticMap"), inline<NarrativeMomentRef>(inputs.moment?.value, "NarrativeMoment"), inline<ScreenOverlayItemSpec>(inputs.spec?.value, "ScreenOverlayItemSpec"))) }, needs: {} }) },
+ { producer: screenOverlayProducers.finalize, implementationDigest: screenOverlayImplementationDigests.finalize, handler: ({ inputs }) => ({ outputs: { program: output(finalizeScreenOverlay(inline<ScreenOverlaySet>(inputs.set?.value, "ScreenOverlaySet"), inline<ScreenOverlayHeader>(inputs.header?.value, "ScreenOverlayHeader"))) }, needs: {} }) },
+ { producer: screenOverlayProducers.render, implementationDigest: screenOverlayImplementationDigests.render, handler: ({ inputs }) => ({ outputs: { track: output(renderScreenOverlay(inline<CanvasSpace>(inputs.canvas?.value, "CanvasSpace"), inline<ProgramSpace>(inputs.space?.value, "ProgramSpace"), inline<ScreenOverlayProgram>(inputs.program?.value, "ScreenOverlayProgram"))) }, needs: {} }) },
+ ],
+ validators: [{ type: screenOverlayTypes.program, implementationDigest: screenOverlayValidatorDigests.program, handler: ({ value }) => assertScreenOverlayProgram(inline<ScreenOverlayProgram>(value, "ScreenOverlayProgram")) }],
+} satisfies ComponentPackage;
diff --git a/packages/screen-overlay/src/fragment.ts b/packages/screen-overlay/src/fragment.ts
new file mode 100644
index 00000000..f1fd5a59
--- /dev/null
+++ b/packages/screen-overlay/src/fragment.ts
@@ -0,0 +1,46 @@
+import { compositionTypes } from "@narratage/composition";
+import { sealGraphFragment } from "@narratage/elaborator";
+import type { FragmentOperation } from "@narratage/elaborator";
+import { narrativeTypes } from "@narratage/narrative";
+import { programSpaceTypes } from "@narratage/program-space";
+import { semanticMapTypes } from "@narratage/semantic-map";
+import { spatialTypes } from "@narratage/spatial";
+import { screenOverlayProducers, screenOverlayTypes } from "./manifest.js";
+
+export type ScreenOverlayFragmentItem =
+ | { readonly kind: "program"; readonly specName: string }
+ | { readonly kind: "selection"; readonly specName: string; readonly mapName: string; readonly sourceName: string }
+ | { readonly kind: "moment"; readonly specName: string; readonly mapName: string; readonly sourceName: string };
+const input = (name: string) => ({ kind: "fragment-input" as const, name });
+const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
+export function createScreenOverlayFragment(items: readonly ScreenOverlayFragmentItem[], name: string) {
+ if (items.length === 0) throw new Error("Screen Overlay Fragment requires at least one Item.");
+ const types = new Map<string, typeof screenOverlayTypes.itemSpec | typeof semanticMapTypes.complete | typeof narrativeTypes.selection | typeof narrativeTypes.moment>();
+ const operations: FragmentOperation[] = [{ id: "overlay:set:empty", producer: screenOverlayProducers.createSet, inputs: {}, result: { kind: "output", name: "set" } }];
+ let current = "overlay:set:empty";
+ items.forEach((item, index) => {
+ types.set(item.specName, screenOverlayTypes.itemSpec);
+ if (item.kind !== "program") {
+ types.set(item.mapName, semanticMapTypes.complete);
+ types.set(item.sourceName, item.kind === "selection" ? narrativeTypes.selection : narrativeTypes.moment);
+ }
+ const id = `overlay:set:append:${String(index + 1).padStart(4, "0")}`;
+ operations.push({ id, producer: item.kind === "program" ? screenOverlayProducers.appendProgram : item.kind === "selection" ? screenOverlayProducers.appendSelection : screenOverlayProducers.appendMoment,
+ inputs: { set: operation(current), header: input("header"), space: input("space"), spec: input(item.specName),
+ ...(item.kind === "program" ? {} : { map: input(item.mapName), [item.kind]: input(item.sourceName) }) }, result: { kind: "output", name: "set" } });
+ current = id;
+ });
+ operations.push(
+ { id: "overlay:program", producer: screenOverlayProducers.finalize, inputs: { set: operation(current), header: input("header") }, result: { kind: "output", name: "program" } },
+ { id: "overlay:track", producer: screenOverlayProducers.render, inputs: { canvas: input("canvas"), space: input("space"), program: operation("overlay:program") }, result: { kind: "output", name: "track" } },
+ );
+ const semanticInputs = ["canvas", "header", "space", ...types.keys()];
+ return sealGraphFragment({ name, inputs: [
+ { name: "canvas", type: spatialTypes.canvas }, { name: "header", type: screenOverlayTypes.header },
+ { name: "space", type: programSpaceTypes.programSpace }, ...[...types].map(([inputName, type]) => ({ name: inputName, type })),
+ ], operations, exports: [
+ { name: "program", type: screenOverlayTypes.program, root: operation("overlay:program"), semanticInputs, fidelity: "exact" },
+ { name: "track", type: compositionTypes.visualTrack, root: operation("overlay:track"), semanticInputs, fidelity: "exact" },
+ ] });
+}
+export const programScreenOverlayFragment = createScreenOverlayFragment([{ kind: "program", specName: "spec" }], "@narratage/screen-overlay/one-program-item@1");
diff --git a/packages/screen-overlay/src/index.ts b/packages/screen-overlay/src/index.ts
new file mode 100644
index 00000000..e7a8a70e
--- /dev/null
+++ b/packages/screen-overlay/src/index.ts
@@ -0,0 +1,17 @@
+export { screenOverlayComponent } from "./component.js";
+export { createScreenOverlayFragment, programScreenOverlayFragment } from "./fragment.js";
+export type { ScreenOverlayFragmentItem } from "./fragment.js";
+export {
+ screenOverlayDependency, screenOverlayHeaderSchema, screenOverlayItemSpecSchema, screenOverlayManifest,
+ screenOverlayManifestDigest, screenOverlayModuleRef, screenOverlayProducers, screenOverlayProgramSchema,
+ screenOverlaySetSchema, screenOverlaySurfaceImplementationDigest, screenOverlayTypes,
+} from "./manifest.js";
+export {
+ appendMomentScreenOverlay, appendProgramScreenOverlay, appendSelectionScreenOverlay,
+ assertScreenOverlayComponent, assertScreenOverlayHeader, assertScreenOverlayItemSpec,
+ assertScreenOverlayProgram, assertScreenOverlaySet, createScreenOverlaySet, finalizeScreenOverlay,
+ renderScreenOverlay, screenOverlayImplementationDigests, screenOverlayValidatorDigests,
+ sealScreenOverlayHeader, sealScreenOverlayItemSpec, sealScreenOverlayProgram,
+} from "./program.js";
+export { decodeScreenOverlaySurface } from "./surface.js";
+export type * from "./types.js";
diff --git a/packages/screen-overlay/src/manifest.ts b/packages/screen-overlay/src/manifest.ts
new file mode 100644
index 00000000..67915299
--- /dev/null
+++ b/packages/screen-overlay/src/manifest.ts
@@ -0,0 +1,98 @@
+import { compositionDependency, compositionTypes } from "@narratage/composition";
+import { narrativeDependency, narrativeTypes } from "@narratage/narrative";
+import { programSpaceDependency, programSpaceTypes } from "@narratage/program-space";
+import { digestOf } from "@narratage/protocol";
+import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@narratage/protocol";
+import { semanticMapDependency, semanticMapTypes } from "@narratage/semantic-map";
+import { spatialDependency, spatialTypes } from "@narratage/spatial";
+import { temporalDependency } from "@narratage/temporal";
+
+import { screenOverlayImplementationDigests, screenOverlayValidatorDigests } from "./program.js";
+
+export const screenOverlayModuleRef = { name: "@narratage/screen-overlay", version: "0.0.0-dev" } as const;
+export const screenOverlayTypes = {
+ header: { module: screenOverlayModuleRef, name: "ScreenOverlayHeader" },
+ itemSpec: { module: screenOverlayModuleRef, name: "ScreenOverlayItemSpec" },
+ set: { module: screenOverlayModuleRef, name: "ScreenOverlaySet" },
+ program: { module: screenOverlayModuleRef, name: "ScreenOverlayProgram" },
+} satisfies Record<string, TypeRef>;
+export const screenOverlayProducers = {
+ createSet: { module: screenOverlayModuleRef, name: "create-screen-overlay-set" },
+ appendProgram: { module: screenOverlayModuleRef, name: "append-program-screen-overlay" },
+ appendSelection: { module: screenOverlayModuleRef, name: "append-selection-screen-overlay" },
+ appendMoment: { module: screenOverlayModuleRef, name: "append-moment-screen-overlay" },
+ finalize: { module: screenOverlayModuleRef, name: "finalize-screen-overlay" },
+ render: { module: screenOverlayModuleRef, name: "render-screen-overlay" },
+} satisfies Record<string, ProducerRef>;
+
+const string = { kind: "string", minLength: 1 } as const;
+const number = { kind: "number" } as const;
+const nonNegative = { kind: "number", minimum: 0 } as const;
+const integer = { kind: "number", integer: true } as const;
+const unsignedInteger = { kind: "number", integer: true, minimum: 0 } as const;
+const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>): ValueSchema => ({ kind: "object", fields });
+const pointDuration: ValueSchema = { kind: "oneOf", variants: [
+ object({ unit: { schema: { kind: "literal", value: "frames" } }, value: { schema: integer } }),
+ object({ unit: { schema: { kind: "literal", value: "milliseconds" } }, value: { schema: integer } }),
+ object({ unit: { schema: { kind: "literal", value: "seconds" } }, numerator: { schema: integer }, denominator: { schema: { kind: "number", integer: true, minimum: 1 } } }),
+] };
+const point: ValueSchema = { kind: "oneOf", variants: [
+ ...["program.start", "program.end", "selection.start", "selection.end", "moment.cue"].map((ref) => object({ ref: { schema: { kind: "literal", value: ref } }, offset: { schema: pointDuration, optional: true } })),
+ object({ ref: { schema: { kind: "literal", value: "absolute" } }, at: { schema: pointDuration } }),
+] };
+const projection = object({ start: { schema: point }, end: { schema: point } });
+const normalizedPoint = object({ x: { schema: number }, y: { schema: number } });
+const colorArray: ValueSchema = { kind: "array", minItems: 1, items: string };
+const component: ValueSchema = { kind: "oneOf", variants: [
+ object({ kind: { schema: { kind: "literal", value: "flash" } }, color: { schema: string }, intensity: { schema: nonNegative }, attackFrames: { schema: unsignedInteger }, holdFrames: { schema: unsignedInteger }, decayFrames: { schema: unsignedInteger } }),
+ object({ kind: { schema: { kind: "literal", value: "color-wash" } }, color: { schema: string }, opacity: { schema: nonNegative } }),
+ object({ kind: { schema: { kind: "literal", value: "vignette" } }, center: { schema: normalizedPoint }, radius: { schema: normalizedPoint }, softness: { schema: nonNegative }, color: { schema: string }, opacity: { schema: nonNegative } }),
+ object({ kind: { schema: { kind: "literal", value: "scan-lines" } }, spacingPx: { schema: nonNegative }, thicknessPx: { schema: nonNegative }, angleDeg: { schema: number }, opacity: { schema: nonNegative }, travelPx: { schema: number } }),
+ object({ kind: { schema: { kind: "literal", value: "directional-matte" } }, angleDeg: { schema: number }, coverage: { schema: nonNegative }, feather: { schema: nonNegative }, color: { schema: string }, opacity: { schema: nonNegative }, progress: { schema: object({ from: { schema: number }, to: { schema: number } }) } }),
+ object({ kind: { schema: { kind: "literal", value: "whip-veil" } }, direction: { schema: { kind: "string", enum: ["left", "right", "up", "down"] } }, widthPx: { schema: nonNegative }, softnessPx: { schema: nonNegative }, travelPx: { schema: nonNegative }, opacity: { schema: nonNegative } }),
+ object({ kind: { schema: { kind: "literal", value: "glitch-veil" } }, bars: { schema: unsignedInteger }, colors: { schema: colorArray }, opacity: { schema: nonNegative }, travelPx: { schema: number }, seed: { schema: unsignedInteger } }),
+ object({ kind: { schema: { kind: "literal", value: "grain" } }, amount: { schema: nonNegative }, grainSizePx: { schema: nonNegative }, chroma: { schema: { kind: "string", enum: ["monochrome", "color"] } }, motionRatePxPerFrame: { schema: number }, seed: { schema: unsignedInteger } }),
+ object({ kind: { schema: { kind: "literal", value: "light-leak" } }, colors: { schema: colorArray }, angleDeg: { schema: number }, softness: { schema: nonNegative }, travelPx: { schema: number }, intensity: { schema: nonNegative }, seed: { schema: unsignedInteger } }),
+ object({ kind: { schema: { kind: "literal", value: "bokeh" } }, amount: { schema: nonNegative }, sizeMinPx: { schema: nonNegative }, sizeMaxPx: { schema: nonNegative }, color: { schema: string }, warmth: { schema: number }, driftPx: { schema: number }, seed: { schema: unsignedInteger } }),
+ object({ kind: { schema: { kind: "literal", value: "tv-static" } }, amount: { schema: nonNegative }, noiseSizePx: { schema: nonNegative }, scanLineOpacity: { schema: nonNegative }, motionRatePxPerFrame: { schema: number }, seed: { schema: unsignedInteger } }),
+] };
+const itemSpec = object({
+ contract: { schema: { kind: "literal", value: "svml.screen-overlay-item-spec@1" } }, id: { schema: string },
+ content: { schema: component }, projection: { schema: projection },
+ expansion: { schema: object({ kind: { schema: { kind: "string", enum: ["one", "each"] } } }) },
+ stackingOrder: { schema: integer },
+});
+const frameSpan = object({ startFrame: { schema: unsignedInteger }, endFrameExclusive: { schema: unsignedInteger } });
+const item = object({
+ id: { schema: string }, sourceOccurrenceId: { schema: string }, span: { schema: frameSpan }, content: { schema: component },
+ stacking: { schema: object({ order: { schema: integer }, tieBreak: { schema: string } }) },
+});
+export const screenOverlayHeaderSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.screen-overlay-header@1" } }, id: { schema: string } });
+export const screenOverlayItemSpecSchema: ValueSchema = itemSpec;
+export const screenOverlaySetSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.screen-overlay-set@1" } }, items: { schema: { kind: "array", items: item } } });
+export const screenOverlayProgramSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.screen-overlay-program@1" } }, id: { schema: string }, items: { schema: { kind: "array", minItems: 1, items: item } } });
+export const screenOverlaySurfaceImplementationDigest = digestOf("@narratage/screen-overlay/track-surface@1");
+const validator = (locator: string, digest: ReturnType<typeof digestOf>) => ({ abi: "svml.type-validator@1" as const, implementation: { kind: "registered" as const, locator, digest } });
+const appendInputs = [{ name: "set", type: screenOverlayTypes.set }, { name: "header", type: screenOverlayTypes.header }, { name: "space", type: programSpaceTypes.programSpace }, { name: "spec", type: screenOverlayTypes.itemSpec }] as const;
+
+export const screenOverlayManifest: ModuleManifest = {
+ format: "svml.module@1", name: screenOverlayModuleRef.name, version: screenOverlayModuleRef.version,
+ dependencies: [narrativeDependency, semanticMapDependency, programSpaceDependency, spatialDependency, temporalDependency, compositionDependency],
+ types: [
+ { name: screenOverlayTypes.header.name, schema: screenOverlayHeaderSchema },
+ { name: screenOverlayTypes.itemSpec.name, schema: screenOverlayItemSpecSchema },
+ { name: screenOverlayTypes.set.name, schema: screenOverlaySetSchema },
+ { name: screenOverlayTypes.program.name, schema: screenOverlayProgramSchema, validator: validator("@narratage/screen-overlay/validate-program", screenOverlayValidatorDigests.program) },
+ ], capabilities: [],
+ surfaces: [{ name: "track", tag: "Track", mode: "structured", outputs: [screenOverlayTypes.header, screenOverlayTypes.itemSpec, screenOverlayTypes.program, compositionTypes.visualTrack], implementation: { kind: "trusted-frontend-surface", locator: "@narratage/screen-overlay/track-surface", digest: screenOverlaySurfaceImplementationDigest } }],
+ producers: [
+ { name: screenOverlayProducers.createSet.name, inputs: [], outputs: [{ name: "set", type: screenOverlayTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/screen-overlay/create-set", digest: screenOverlayImplementationDigests.createSet } },
+ { name: screenOverlayProducers.appendProgram.name, inputs: [...appendInputs], outputs: [{ name: "set", type: screenOverlayTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/screen-overlay/append-program", digest: screenOverlayImplementationDigests.appendProgram } },
+ { name: screenOverlayProducers.appendSelection.name, inputs: [...appendInputs, { name: "map", type: semanticMapTypes.complete }, { name: "selection", type: narrativeTypes.selection }], outputs: [{ name: "set", type: screenOverlayTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/screen-overlay/append-selection", digest: screenOverlayImplementationDigests.appendSelection } },
+ { name: screenOverlayProducers.appendMoment.name, inputs: [...appendInputs, { name: "map", type: semanticMapTypes.complete }, { name: "moment", type: narrativeTypes.moment }], outputs: [{ name: "set", type: screenOverlayTypes.set }], needs: [], implementation: { kind: "registered", locator: "@narratage/screen-overlay/append-moment", digest: screenOverlayImplementationDigests.appendMoment } },
+ { name: screenOverlayProducers.finalize.name, inputs: [{ name: "set", type: screenOverlayTypes.set }, { name: "header", type: screenOverlayTypes.header }], outputs: [{ name: "program", type: screenOverlayTypes.program }], needs: [], implementation: { kind: "registered", locator: "@narratage/screen-overlay/finalize", digest: screenOverlayImplementationDigests.finalize } },
+ { name: screenOverlayProducers.render.name, inputs: [{ name: "canvas", type: spatialTypes.canvas }, { name: "space", type: programSpaceTypes.programSpace }, { name: "program", type: screenOverlayTypes.program }], outputs: [{ name: "track", type: compositionTypes.visualTrack }], needs: [], implementation: { kind: "registered", locator: "@narratage/screen-overlay/render", digest: screenOverlayImplementationDigests.render } },
+ ],
+};
+export const screenOverlayManifestDigest = digestOf(screenOverlayManifest);
+export const screenOverlayDependency = { module: screenOverlayModuleRef, digest: screenOverlayManifestDigest } as const;
diff --git a/packages/screen-overlay/src/program.ts b/packages/screen-overlay/src/program.ts
new file mode 100644
index 00000000..ce8179d8
--- /dev/null
+++ b/packages/screen-overlay/src/program.ts
@@ -0,0 +1,334 @@
+import { assertVisualTrackIdentity, sealVisualTrack } from "@narratage/composition";
+import type { VisualAnimation, VisualElement, VisualStyleDeclaration, VisualTrack } from "@narratage/composition";
+import type { NarrativeMomentRef, NarrativeSelectionRef } from "@narratage/narrative";
+import { assertProgramSpaceIdentity } from "@narratage/program-space";
+import type { ProgramSpace } from "@narratage/program-space";
+import { canonicalize, digestOf } from "@narratage/protocol";
+import type { CompleteSemanticMap } from "@narratage/semantic-map";
+import { assertCanvasSpace } from "@narratage/spatial";
+import type { CanvasSpace } from "@narratage/spatial";
+import { projectMomentWindows, projectProgramWindow, projectSelectionWindows } from "@narratage/temporal";
+import type { ProjectedOccurrence } from "@narratage/temporal";
+
+import type {
+ ScreenOverlayComponent,
+ ScreenOverlayHeader,
+ ScreenOverlayItemProgram,
+ ScreenOverlayItemSpec,
+ ScreenOverlayProgram,
+ ScreenOverlaySet,
+} from "./types.js";
+
+export const screenOverlayImplementationDigests = {
+ createSet: digestOf("@narratage/screen-overlay/create-set@1"),
+ appendProgram: digestOf("@narratage/screen-overlay/append-program@1"),
+ appendSelection: digestOf("@narratage/screen-overlay/append-selection@1"),
+ appendMoment: digestOf("@narratage/screen-overlay/append-moment@1"),
+ finalize: digestOf("@narratage/screen-overlay/finalize@1"),
+ render: digestOf("@narratage/screen-overlay/render@1"),
+} as const;
+export const screenOverlayValidatorDigests = { program: digestOf("@narratage/screen-overlay/validate-program@1") } as const;
+
+function assert(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(message); }
+function identity(value: string, label: string): void {
+ assert(/^[A-Za-z][A-Za-z0-9_.:#-]{0,191}$/u.test(value), `${label} is invalid.`);
+}
+function finite(value: number, label: string): void { assert(Number.isFinite(value), `${label} is invalid.`); }
+function range(value: number, minimum: number, maximum: number, label: string): void {
+ finite(value, label); assert(value >= minimum && value <= maximum, `${label} is outside ${minimum}..${maximum}.`);
+}
+function positive(value: number, label: string): void { finite(value, label); assert(value > 0, `${label} must be positive.`); }
+function nonNegativeInteger(value: number, label: string): void {
+ assert(Number.isSafeInteger(value) && value >= 0, `${label} must be a non-negative integer.`);
+}
+function color(value: string, label: string): void {
+ assert(/^#[0-9a-f]{6}(?:[0-9a-f]{2})?$/iu.test(value), `${label} must be a hexadecimal color.`);
+}
+function colors(values: readonly string[], label: string): void {
+ assert(values.length > 0, `${label} requires at least one color.`);
+ values.forEach((value, index) => color(value, `${label}.${index + 1}`));
+}
+
+export function assertScreenOverlayComponent(value: ScreenOverlayComponent): void {
+ switch (value.kind) {
+ case "flash":
+ color(value.color, "Flash color"); range(value.intensity, 0, 1, "Flash intensity");
+ nonNegativeInteger(value.attackFrames, "Flash attack"); nonNegativeInteger(value.holdFrames, "Flash hold");
+ nonNegativeInteger(value.decayFrames, "Flash decay"); return;
+ case "color-wash": color(value.color, "ColorWash color"); range(value.opacity, 0, 1, "ColorWash opacity"); return;
+ case "vignette":
+ range(value.center.x, 0, 1, "Vignette center.x"); range(value.center.y, 0, 1, "Vignette center.y");
+ positive(value.radius.x, "Vignette radius.x"); positive(value.radius.y, "Vignette radius.y");
+ range(value.softness, 0, 1, "Vignette softness"); color(value.color, "Vignette color");
+ range(value.opacity, 0, 1, "Vignette opacity"); return;
+ case "scan-lines":
+ positive(value.spacingPx, "ScanLines spacing"); positive(value.thicknessPx, "ScanLines thickness");
+ assert(value.thicknessPx <= value.spacingPx, "ScanLines thickness exceeds spacing.");
+ finite(value.angleDeg, "ScanLines angle"); range(value.opacity, 0, 1, "ScanLines opacity");
+ finite(value.travelPx, "ScanLines travel"); return;
+ case "directional-matte":
+ finite(value.angleDeg, "DirectionalMatte angle"); range(value.coverage, 0, 1, "DirectionalMatte coverage");
+ range(value.feather, 0, 1, "DirectionalMatte feather"); color(value.color, "DirectionalMatte color");
+ range(value.opacity, 0, 1, "DirectionalMatte opacity"); range(value.progress.from, -1, 2, "DirectionalMatte progress.from");
+ range(value.progress.to, -1, 2, "DirectionalMatte progress.to"); return;
+ case "whip-veil":
+ assert(["left", "right", "up", "down"].includes(value.direction), "WhipVeil direction is invalid.");
+ positive(value.widthPx, "WhipVeil width"); assert(Number.isFinite(value.softnessPx) && value.softnessPx >= 0, "WhipVeil softness is invalid.");
+ positive(value.travelPx, "WhipVeil travel"); range(value.opacity, 0, 1, "WhipVeil opacity"); return;
+ case "glitch-veil":
+ assert(Number.isSafeInteger(value.bars) && value.bars > 0 && value.bars <= 256, "GlitchVeil bars are invalid.");
+ colors(value.colors, "GlitchVeil colors"); range(value.opacity, 0, 1, "GlitchVeil opacity");
+ finite(value.travelPx, "GlitchVeil travel"); nonNegativeInteger(value.seed, "GlitchVeil seed"); return;
+ case "grain":
+ range(value.amount, 0, 1, "Grain amount"); positive(value.grainSizePx, "Grain size");
+ assert(value.chroma === "monochrome" || value.chroma === "color", "Grain chroma is invalid.");
+ finite(value.motionRatePxPerFrame, "Grain motion rate"); nonNegativeInteger(value.seed, "Grain seed"); return;
+ case "light-leak":
+ colors(value.colors, "LightLeak colors"); finite(value.angleDeg, "LightLeak angle");
+ range(value.softness, 0, 1, "LightLeak softness"); finite(value.travelPx, "LightLeak travel");
+ range(value.intensity, 0, 1, "LightLeak intensity"); nonNegativeInteger(value.seed, "LightLeak seed"); return;
+ case "bokeh":
+ range(value.amount, 0, 1, "Bokeh amount"); positive(value.sizeMinPx, "Bokeh minimum size");
+ assert(Number.isFinite(value.sizeMaxPx) && value.sizeMaxPx >= value.sizeMinPx, "Bokeh size range is invalid.");
+ color(value.color, "Bokeh color"); range(value.warmth, -1, 1, "Bokeh warmth"); finite(value.driftPx, "Bokeh drift");
+ nonNegativeInteger(value.seed, "Bokeh seed"); return;
+ case "tv-static":
+ range(value.amount, 0, 1, "TVStatic amount"); positive(value.noiseSizePx, "TVStatic noise size");
+ range(value.scanLineOpacity, 0, 1, "TVStatic scan-line opacity");
+ finite(value.motionRatePxPerFrame, "TVStatic motion rate"); nonNegativeInteger(value.seed, "TVStatic seed"); return;
+ }
+ throw new Error("Screen Overlay component kind is unsupported.");
+}
+
+export function sealScreenOverlayHeader(value: ScreenOverlayHeader): ScreenOverlayHeader {
+ assertScreenOverlayHeader(value); return canonicalize(value) as unknown as ScreenOverlayHeader;
+}
+export function assertScreenOverlayHeader(value: ScreenOverlayHeader): void {
+ assert(value.contract === "svml.screen-overlay-header@1", "Unsupported ScreenOverlayHeader contract."); identity(value.id, "ScreenOverlayHeader.id");
+}
+export function sealScreenOverlayItemSpec(value: ScreenOverlayItemSpec): ScreenOverlayItemSpec {
+ assertScreenOverlayItemSpec(value); return canonicalize(value) as unknown as ScreenOverlayItemSpec;
+}
+export function assertScreenOverlayItemSpec(value: ScreenOverlayItemSpec): void {
+ assert(value.contract === "svml.screen-overlay-item-spec@1", "Unsupported ScreenOverlayItemSpec contract.");
+ identity(value.id, "ScreenOverlayItemSpec.id"); assertScreenOverlayComponent(value.content);
+ assert(value.expansion.kind === "one" || value.expansion.kind === "each", "ScreenOverlay expansion is invalid.");
+ assert(Number.isSafeInteger(value.stackingOrder), "ScreenOverlay stacking order must be an integer.");
+}
+export function createScreenOverlaySet(): ScreenOverlaySet { return { contract: "svml.screen-overlay-set@1", items: [] }; }
+export function assertScreenOverlaySet(value: ScreenOverlaySet): void {
+ assert(value.contract === "svml.screen-overlay-set@1" && Array.isArray(value.items), "ScreenOverlaySet is invalid.");
+}
+
+function realized(
+ set: ScreenOverlaySet, header: ScreenOverlayHeader, spec: ScreenOverlayItemSpec,
+ occurrences: readonly ProjectedOccurrence[],
+): ScreenOverlaySet {
+ assertScreenOverlaySet(set); assertScreenOverlayHeader(header); assertScreenOverlayItemSpec(spec);
+ const additions = occurrences.map((occurrence, index) => ({
+ id: occurrence.id,
+ sourceOccurrenceId: occurrence.sourceOccurrenceId,
+ span: { ...occurrence.span },
+ content: structuredClone(spec.content),
+ stacking: { order: spec.stackingOrder, tieBreak: `${header.id}:${spec.id}:${index + 1}` },
+ } satisfies ScreenOverlayItemProgram));
+ const ids = new Set(set.items.map((item) => item.id));
+ additions.forEach((item) => { assert(!ids.has(item.id), `Screen Overlay already contains Item ${item.id}.`); ids.add(item.id); });
+ return { contract: "svml.screen-overlay-set@1", items: [...set.items, ...additions] };
+}
+export function appendProgramScreenOverlay(set: ScreenOverlaySet, header: ScreenOverlayHeader, space: ProgramSpace, spec: ScreenOverlayItemSpec): ScreenOverlaySet {
+ assert(spec.expansion.kind === "one", `Program Overlay ${spec.id} must use one occurrence.`);
+ return realized(set, header, spec, [projectProgramWindow({ itemId: spec.id, space, projection: spec.projection })]);
+}
+export function appendSelectionScreenOverlay(set: ScreenOverlaySet, header: ScreenOverlayHeader, space: ProgramSpace, map: CompleteSemanticMap, selection: NarrativeSelectionRef, spec: ScreenOverlayItemSpec): ScreenOverlaySet {
+ return realized(set, header, spec, projectSelectionWindows({ itemId: spec.id, map, selection, space, expansion: spec.expansion, projection: spec.projection }));
+}
+export function appendMomentScreenOverlay(set: ScreenOverlaySet, header: ScreenOverlayHeader, space: ProgramSpace, map: CompleteSemanticMap, moment: NarrativeMomentRef, spec: ScreenOverlayItemSpec): ScreenOverlaySet {
+ return realized(set, header, spec, projectMomentWindows({ itemId: spec.id, map, moment, space, expansion: spec.expansion, projection: spec.projection }));
+}
+export function sealScreenOverlayProgram(value: ScreenOverlayProgram): ScreenOverlayProgram {
+ const normalized = { contract: "svml.screen-overlay-program@1" as const, id: value.id,
+ items: [...value.items].map((item) => structuredClone(item)).sort((a, b) => a.id.localeCompare(b.id)) };
+ assertScreenOverlayProgram(normalized); return canonicalize(normalized) as unknown as ScreenOverlayProgram;
+}
+export function finalizeScreenOverlay(set: ScreenOverlaySet, header: ScreenOverlayHeader): ScreenOverlayProgram {
+ assertScreenOverlaySet(set); assertScreenOverlayHeader(header); assert(set.items.length > 0, "Screen Overlay requires at least one Item.");
+ return sealScreenOverlayProgram({ contract: "svml.screen-overlay-program@1", id: header.id, items: set.items });
+}
+export function assertScreenOverlayProgram(value: ScreenOverlayProgram): void {
+ assert(value.contract === "svml.screen-overlay-program@1", "Unsupported ScreenOverlayProgram contract.");
+ identity(value.id, "ScreenOverlayProgram.id"); assert(value.items.length > 0, "ScreenOverlayProgram requires Items.");
+ const ids = new Set<string>();
+ for (const item of value.items) {
+ identity(item.id, "ScreenOverlayItemProgram.id"); assert(!ids.has(item.id), `Duplicate Screen Overlay Item ${item.id}.`); ids.add(item.id);
+ assert(item.sourceOccurrenceId.length > 0 && item.span.startFrame >= 0 && item.span.endFrameExclusive > item.span.startFrame,
+ `Screen Overlay Item ${item.id} timing is invalid.`);
+ assertScreenOverlayComponent(item.content); assert(Number.isSafeInteger(item.stacking.order) && item.stacking.tieBreak.length > 0,
+ `Screen Overlay Item ${item.id} stacking is invalid.`);
+ }
+}
+
+type Random = () => number;
+function random(seed: number): Random {
+ let state = seed >>> 0;
+ return () => {
+ state += 0x6d2b79f5;
+ let value = state;
+ value = Math.imul(value ^ value >>> 15, value | 1);
+ value ^= value + Math.imul(value ^ value >>> 7, value | 61);
+ return ((value ^ value >>> 14) >>> 0) / 4_294_967_296;
+ };
+}
+function px(value: number): string { return `${Number(value.toFixed(6))}px`; }
+function percent(value: number): string { return `${Number((value * 100).toFixed(6))}%`; }
+function rootStyle(): VisualStyleDeclaration[] { return [
+ { name: "inset", value: "0" }, { name: "overflow", value: "hidden" }, { name: "position", value: "absolute" },
+]; }
+function fullChild(
+ id: string,
+ style: readonly VisualStyleDeclaration[],
+ animation?: VisualAnimation,
+ order = 1,
+): VisualElement {
+ const names = new Set(style.map((item) => item.name));
+ return { id, parent: "root", order, kind: "box", style: [
+ ...(names.has("inset") ? [] : [{ name: "inset", value: "0" }]),
+ ...(names.has("position") ? [] : [{ name: "position", value: "absolute" }]),
+ ...style,
+ ], ...(animation === undefined ? {} : { animation }) };
+}
+function animation(duration: number, start: readonly VisualStyleDeclaration[], end: readonly VisualStyleDeclaration[]): VisualAnimation {
+ return { keyframes: [{ atFrame: 0, style: start }, { atFrame: duration, style: end }] };
+}
+function flashElements(content: Extract<ScreenOverlayComponent, { kind: "flash" }>, duration: number): readonly VisualElement[] {
+ const total = content.attackFrames + content.holdFrames + content.decayFrames;
+ assert(total <= duration, "Flash envelope exceeds its projected window.");
+ if (total === 0) return [{ id: "root", order: 0, kind: "box", style: [...rootStyle(), { name: "background-color", value: content.color }, { name: "opacity", value: content.intensity }] }];
+ const marks = new Map<number, number>();
+ marks.set(0, content.attackFrames === 0 ? content.intensity : 0);
+ marks.set(content.attackFrames, content.intensity);
+ marks.set(content.attackFrames + content.holdFrames, content.intensity);
+ marks.set(total, 0);
+ marks.set(duration, 0);
+ return [{ id: "root", order: 0, kind: "box", style: [...rootStyle(), { name: "background-color", value: content.color }],
+ animation: { keyframes: [...marks].sort(([a], [b]) => a - b).map(([atFrame, opacity]) => ({ atFrame, style: [{ name: "opacity", value: opacity }] })) } }];
+}
+
+function overlayElements(content: ScreenOverlayComponent, canvas: CanvasSpace, duration: number): readonly VisualElement[] {
+ if (content.kind === "flash") return flashElements(content, duration);
+ const root: VisualElement = { id: "root", order: 0, kind: "box", style: rootStyle() };
+ switch (content.kind) {
+ case "color-wash": return [root, fullChild("wash", [{ name: "background-color", value: content.color }, { name: "opacity", value: content.opacity }])];
+ case "vignette": {
+ const inner = Math.max(0, 1 - content.softness) * 100;
+ return [root, fullChild("vignette", [{ name: "background-image", value: `radial-gradient(ellipse ${percent(content.radius.x)} ${percent(content.radius.y)} at ${percent(content.center.x)} ${percent(content.center.y)},transparent ${inner}%,${content.color} 100%)` }, { name: "opacity", value: content.opacity }])];
+ }
+ case "scan-lines": {
+ const travel = `translate3d(${px(content.travelPx * Math.cos(content.angleDeg * Math.PI / 180))},${px(content.travelPx * Math.sin(content.angleDeg * Math.PI / 180))},0)`;
+ return [root, fullChild("scan-lines", [
+ { name: "background-image", value: `repeating-linear-gradient(${content.angleDeg}deg,#ffffff 0 ${px(content.thicknessPx)},transparent ${px(content.thicknessPx)} ${px(content.spacingPx)})` },
+ { name: "inset", value: "-20%" }, { name: "opacity", value: content.opacity },
+ ], animation(duration, [{ name: "transform", value: "translate3d(0px,0px,0)" }], [{ name: "transform", value: travel }]))];
+ }
+ case "directional-matte": {
+ const extent = Math.hypot(canvas.widthPx, canvas.heightPx) * 2;
+ const move = (progress: number) => `rotate(${content.angleDeg}deg) translate3d(${px((progress - 0.5) * extent)},0px,0)`;
+ const edge = Math.max(0.001, content.feather) * 100;
+ return [root, fullChild("matte", [
+ { name: "background-image", value: `linear-gradient(90deg,${content.color} 0 ${100 - edge}%,transparent 100%)` },
+ { name: "height", value: "200%" }, { name: "left", value: "-50%" }, { name: "opacity", value: content.opacity },
+ { name: "top", value: "-50%" }, { name: "width", value: `${200 * content.coverage}%` },
+ ], animation(duration, [{ name: "transform", value: move(content.progress.from) }], [{ name: "transform", value: move(content.progress.to) }]))];
+ }
+ case "whip-veil": {
+ const horizontal = content.direction === "left" || content.direction === "right";
+ const sign = content.direction === "left" || content.direction === "up" ? -1 : 1;
+ const gradient = horizontal ? "90deg" : "180deg";
+ const transform = (distance: number) => horizontal ? `translate3d(${px(distance)},0px,0)` : `translate3d(0px,${px(distance)},0)`;
+ return [root, fullChild("veil", [
+ { name: "background-image", value: `linear-gradient(${gradient},transparent 0,#ffffff ${px(content.softnessPx)},#ffffff calc(100% - ${px(content.softnessPx)}),transparent 100%)` },
+ { name: "height", value: horizontal ? "100%" : px(content.widthPx) }, { name: "opacity", value: content.opacity },
+ { name: "width", value: horizontal ? px(content.widthPx) : "100%" },
+ ], animation(duration, [{ name: "transform", value: transform(-sign * content.travelPx) }], [{ name: "transform", value: transform(sign * content.travelPx) }]))];
+ }
+ case "glitch-veil": {
+ const rng = random(content.seed);
+ return [root, ...Array.from({ length: content.bars }, (_, index): VisualElement => {
+ const height = 1 + rng() * 12;
+ const y = rng() * 100;
+ const x = (rng() - 0.5) * 20;
+ const colorValue = content.colors[index % content.colors.length]!;
+ return { id: `bar-${index + 1}`, parent: "root", order: index + 1, kind: "box", style: [
+ { name: "background-color", value: colorValue }, { name: "height", value: percent(height / 100) },
+ { name: "left", value: percent(x / 100) }, { name: "opacity", value: content.opacity * (0.35 + rng() * 0.65) },
+ { name: "position", value: "absolute" }, { name: "top", value: percent(y / 100) }, { name: "width", value: "120%" },
+ ], animation: animation(duration, [{ name: "transform", value: "translate3d(0px,0px,0)" }], [{ name: "transform", value: `translate3d(${px(content.travelPx * (rng() < 0.5 ? -1 : 1))},0px,0)` }]) };
+ })];
+ }
+ case "grain": {
+ const rng = random(content.seed); const count = Math.max(1, Math.round(content.amount * 160));
+ return [root, ...Array.from({ length: count }, (_, index): VisualElement => {
+ const hue = content.chroma === "color" ? Math.round(rng() * 360) : 0;
+ const shade = Math.round(rng() * 255);
+ const background = content.chroma === "color" ? `hsl(${hue} 85% 60%)` : `rgb(${shade} ${shade} ${shade})`;
+ const drift = content.motionRatePxPerFrame * duration * (rng() < 0.5 ? -1 : 1);
+ return { id: `grain-${index + 1}`, parent: "root", order: index + 1, kind: "box", style: [
+ { name: "background-color", value: background }, { name: "height", value: px(content.grainSizePx) },
+ { name: "left", value: percent(rng()) }, { name: "opacity", value: 0.08 + content.amount * 0.42 },
+ { name: "position", value: "absolute" }, { name: "top", value: percent(rng()) }, { name: "width", value: px(content.grainSizePx) },
+ ], animation: animation(duration, [{ name: "transform", value: "translate3d(0px,0px,0)" }], [{ name: "transform", value: `translate3d(${px(drift)},${px(-drift)},0)` }]) };
+ })];
+ }
+ case "light-leak": {
+ const rng = random(content.seed); const offset = (rng() - 0.5) * 30;
+ const stops = content.colors.map((entry, index) => `${entry} ${Number(index * 100 / Math.max(1, content.colors.length - 1)).toFixed(3)}%`).join(",");
+ return [root, fullChild("light-leak", [
+ { name: "background-image", value: `linear-gradient(${content.angleDeg + offset}deg,transparent 0,${stops},transparent 100%)` },
+ { name: "filter", value: `blur(${px(content.softness * 80)})` }, { name: "inset", value: "-20%" },
+ { name: "opacity", value: content.intensity },
+ ], animation(duration, [{ name: "transform", value: `translate3d(${px(-content.travelPx)},0px,0)` }], [{ name: "transform", value: `translate3d(${px(content.travelPx)},0px,0)` }]))];
+ }
+ case "bokeh": {
+ const rng = random(content.seed); const count = Math.max(1, Math.round(content.amount * 48));
+ return [root, ...Array.from({ length: count }, (_, index): VisualElement => {
+ const size = content.sizeMinPx + rng() * (content.sizeMaxPx - content.sizeMinPx);
+ const warmthFilter = content.warmth === 0 ? "" : `sepia(${Math.abs(content.warmth)}) hue-rotate(${content.warmth < 0 ? 180 : 0}deg) `;
+ return { id: `bokeh-${index + 1}`, parent: "root", order: index + 1, kind: "box", style: [
+ { name: "background-color", value: content.color }, { name: "border-radius", value: "50%" },
+ { name: "filter", value: `${warmthFilter}blur(${px(size * 0.08)})` }, { name: "height", value: px(size) },
+ { name: "left", value: percent(rng()) }, { name: "opacity", value: 0.08 + rng() * 0.32 },
+ { name: "position", value: "absolute" }, { name: "top", value: percent(rng()) }, { name: "width", value: px(size) },
+ ], animation: animation(duration, [{ name: "transform", value: "translate3d(0px,0px,0)" }], [{ name: "transform", value: `translate3d(${px((rng() - 0.5) * content.driftPx)},${px((rng() - 0.5) * content.driftPx)},0)` }]) };
+ })];
+ }
+ case "tv-static": {
+ const rng = random(content.seed); const columns = Math.ceil(canvas.widthPx / content.noiseSizePx);
+ const rows = Math.ceil(canvas.heightPx / content.noiseSizePx); const count = Math.min(512, Math.max(1, Math.round(columns * rows * content.amount)));
+ const cells = Array.from({ length: count }, (_, index): VisualElement => {
+ const shade = Math.round(rng() * 255); const drift = content.motionRatePxPerFrame * duration;
+ return { id: `static-${index + 1}`, parent: "root", order: index + 1, kind: "box", style: [
+ { name: "background-color", value: `rgb(${shade} ${shade} ${shade})` }, { name: "height", value: px(content.noiseSizePx) },
+ { name: "left", value: percent(rng()) }, { name: "opacity", value: 0.2 + content.amount * 0.6 },
+ { name: "position", value: "absolute" }, { name: "top", value: percent(rng()) }, { name: "width", value: px(content.noiseSizePx) },
+ ], animation: animation(duration, [{ name: "transform", value: "translate3d(0px,0px,0)" }], [{ name: "transform", value: `translate3d(0px,${px(drift * (rng() < 0.5 ? -1 : 1))},0)` }]) };
+ });
+ return [root, ...cells, fullChild("static-scan-lines", [
+ { name: "background-image", value: "repeating-linear-gradient(180deg,transparent 0 3px,#000000 3px 4px)" },
+ { name: "opacity", value: content.scanLineOpacity },
+ ], undefined, cells.length + 1)];
+ }
+ }
+}
+
+export function renderScreenOverlay(canvas: CanvasSpace, space: ProgramSpace, program: ScreenOverlayProgram): VisualTrack {
+ assertCanvasSpace(canvas); assertProgramSpaceIdentity(space); assertScreenOverlayProgram(program);
+ const track = sealVisualTrack({
+ contract: "svml.visual-track@1", visualIr: "svml.visual-ir@1", id: program.id,
+ presents: program.items.map((item) => ({
+ id: item.id, span: { ...item.span }, stacking: { ...item.stacking },
+ elements: overlayElements(item.content, canvas, item.span.endFrameExclusive - item.span.startFrame),
+ })),
+ });
+ assertVisualTrackIdentity(track, space);
+ return track;
+}
diff --git a/packages/screen-overlay/src/surface.ts b/packages/screen-overlay/src/surface.ts
new file mode 100644
index 00000000..33030e14
--- /dev/null
+++ b/packages/screen-overlay/src/surface.ts
@@ -0,0 +1,142 @@
+import { narrativeTypes } from "@narratage/narrative";
+import { programSpaceTypes } from "@narratage/program-space";
+import { semanticMapTypes } from "@narratage/semantic-map";
+import { spatialTypes } from "@narratage/spatial";
+import type { StructuredElement, StructuredSurfaceHandler, SurfaceRecordDraft, SurfaceResolvedReference, TextAttributeValue } from "@narratage/text";
+import type { TemporalDuration, TemporalPointExpression } from "@narratage/temporal";
+import { createScreenOverlayFragment } from "./fragment.js";
+import { screenOverlayTypes } from "./manifest.js";
+import { sealScreenOverlayHeader, sealScreenOverlayItemSpec } from "./program.js";
+import type { ScreenOverlayComponent, ScreenOverlayItemSpec } from "./types.js";
+
+const TIMING = ["during", "at", "for", "start", "end", "selection", "moment", "map", "occurrences"] as const;
+function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
+ return left.module.name === right.module.name && left.module.version === right.module.version && left.name === right.name;
+}
+function allowed(element: StructuredElement, names: readonly string[]): void {
+ const permit = new Set(names); const unexpected = Object.keys(element.attributes).filter((name) => !permit.has(name));
+ if (unexpected.length > 0) throw new Error(`${element.name} has unsupported attributes ${unexpected.join(", ")}.`);
+}
+function text(element: StructuredElement, name: string, fallback?: string): string {
+ const value = element.attributes[name]; if (value === undefined && fallback !== undefined) return fallback;
+ if (typeof value !== "string" || !value.trim()) throw new Error(`${element.name}.${name} must be text.`); return value.trim();
+}
+function optionalText(element: StructuredElement, name: string): string | undefined {
+ const value = element.attributes[name]; if (value === undefined) return undefined;
+ if (typeof value !== "string" || !value.trim()) throw new Error(`${element.name}.${name} must be text.`); return value.trim();
+}
+function ref(raw: TextAttributeValue | undefined, label: string, expected: SurfaceResolvedReference["type"], resolve: (path: string) => SurfaceResolvedReference | undefined): SurfaceResolvedReference {
+ if (typeof raw !== "object" || raw.kind !== "reference") throw new Error(`${label} must be a reference.`);
+ const value = resolve(raw.path); if (value === undefined || !sameType(value.type, expected)) throw new Error(`${label} has the wrong Type.`); return value;
+}
+function gcd(left: number, right: number): number { let a = Math.abs(left); let b = Math.abs(right); while (b !== 0) [a, b] = [b, a % b]; return a; }
+function duration(value: string, label: string): TemporalDuration {
+ const match = /^(\d+)(?:\.(\d+))?(f|ms|s)$/u.exec(value.trim());
+ if (!match) throw new Error(`${label} must be an exact duration.`);
+ const whole = Number(match[1]); const fraction = match[2] ?? ""; const unit = match[3];
+ if (unit === "f" || unit === "ms") {
+ if (fraction) throw new Error(`${label} ${unit} duration must be integral.`);
+ return { unit: unit === "f" ? "frames" : "milliseconds", value: whole };
+ }
+ const scale = 10 ** fraction.length; const numerator = whole * scale + (fraction ? Number(fraction) : 0); const divisor = gcd(numerator, scale);
+ return { unit: "seconds", numerator: numerator / divisor, denominator: scale / divisor };
+}
+function negate(value: TemporalDuration): TemporalDuration { return value.unit === "seconds" ? { ...value, numerator: -value.numerator } : { ...value, value: -value.value }; }
+function point(value: string, label: string): TemporalPointExpression {
+ const trimmed = value.trim(); const refs = ["program.start", "program.end", "selection.start", "selection.end", "moment.cue"] as const;
+ for (const reference of refs) {
+ if (trimmed === reference) return { ref: reference };
+ const match = new RegExp(`^${reference.replace(".", "\\.")}\\s*([+-])\\s*(.+)$`, "u").exec(trimmed);
+ if (match) { const offset = duration(match[2]!, `${label} offset`); return { ref: reference, offset: match[1] === "-" ? negate(offset) : offset }; }
+ }
+ return { ref: "absolute", at: duration(trimmed, label) };
+}
+function numeric(element: StructuredElement, name: string, fallback?: number): number {
+ const raw = optionalText(element, name); if (raw === undefined && fallback !== undefined) return fallback;
+ const value = Number(raw); if (!Number.isFinite(value)) throw new Error(`${element.name}.${name} must be numeric.`); return value;
+}
+function integer(element: StructuredElement, name: string, fallback?: number): number {
+ const value = numeric(element, name, fallback); if (!Number.isSafeInteger(value)) throw new Error(`${element.name}.${name} must be an integer.`); return value;
+}
+function colors(element: StructuredElement, name: string): string[] {
+ const values = text(element, name).split(",").map((value) => value.trim()).filter(Boolean);
+ if (values.length === 0) throw new Error(`${element.name}.${name} requires colors.`); return values;
+}
+type Binding = { readonly kind: "program" | "selection" | "moment"; readonly projection: ScreenOverlayItemSpec["projection"]; readonly source?: SurfaceResolvedReference; readonly map?: SurfaceResolvedReference };
+function binding(element: StructuredElement, resolve: (path: string) => SurfaceResolvedReference | undefined): Binding {
+ const during = element.attributes.during; const at = element.attributes.at;
+ const start = optionalText(element, "start"); const end = optionalText(element, "end");
+ const forms = Number(during !== undefined) + Number(at !== undefined) + Number(start !== undefined || end !== undefined);
+ if (forms !== 1) throw new Error(`${element.name} requires exactly one temporal form.`);
+ if (during !== undefined) {
+ if (typeof during === "string") {
+ if (during.trim() !== "program") throw new Error(`${element.name}.during text must be program.`);
+ return { kind: "program", projection: { start: { ref: "program.start" }, end: { ref: "program.end" } } };
+ }
+ return { kind: "selection", source: ref(during, `${element.name}.during`, narrativeTypes.selection, resolve),
+ map: ref(element.attributes.map, `${element.name}.map`, semanticMapTypes.complete, resolve),
+ projection: { start: { ref: "selection.start" }, end: { ref: "selection.end" } } };
+ }
+ if (at !== undefined) return { kind: "moment", source: ref(at, `${element.name}.at`, narrativeTypes.moment, resolve),
+ map: ref(element.attributes.map, `${element.name}.map`, semanticMapTypes.complete, resolve),
+ projection: { start: { ref: "moment.cue" }, end: { ref: "moment.cue", offset: duration(text(element, "for"), `${element.name}.for`) } } };
+ if (start === undefined || end === undefined) throw new Error(`${element.name} explicit timing requires start and end.`);
+ const selection = element.attributes.selection; const moment = element.attributes.moment;
+ if (selection !== undefined && moment !== undefined) throw new Error(`${element.name} cannot bind Selection and Moment together.`);
+ const projection = { start: point(start, `${element.name}.start`), end: point(end, `${element.name}.end`) };
+ if (selection !== undefined) return { kind: "selection", source: ref(selection, `${element.name}.selection`, narrativeTypes.selection, resolve), map: ref(element.attributes.map, `${element.name}.map`, semanticMapTypes.complete, resolve), projection };
+ if (moment !== undefined) return { kind: "moment", source: ref(moment, `${element.name}.moment`, narrativeTypes.moment, resolve), map: ref(element.attributes.map, `${element.name}.map`, semanticMapTypes.complete, resolve), projection };
+ if (element.attributes.map !== undefined) throw new Error(`${element.name}.map requires a temporal source.`);
+ return { kind: "program", projection };
+}
+
+function content(element: StructuredElement): { readonly value: ScreenOverlayComponent; readonly attributes: readonly string[] } {
+ const name = element.name.split(":").at(-1);
+ switch (name) {
+ case "Flash": return { value: { kind: "flash", color: text(element, "color"), intensity: numeric(element, "intensity"), attackFrames: integer(element, "attack"), holdFrames: integer(element, "hold"), decayFrames: integer(element, "decay") }, attributes: ["color", "intensity", "attack", "hold", "decay"] };
+ case "ColorWash": return { value: { kind: "color-wash", color: text(element, "color"), opacity: numeric(element, "opacity") }, attributes: ["color", "opacity"] };
+ case "Vignette": return { value: { kind: "vignette", center: { x: numeric(element, "center-x"), y: numeric(element, "center-y") }, radius: { x: numeric(element, "radius-x"), y: numeric(element, "radius-y") }, softness: numeric(element, "softness"), color: text(element, "color"), opacity: numeric(element, "opacity") }, attributes: ["center-x", "center-y", "radius-x", "radius-y", "softness", "color", "opacity"] };
+ case "ScanLines": return { value: { kind: "scan-lines", spacingPx: numeric(element, "spacing"), thicknessPx: numeric(element, "thickness"), angleDeg: numeric(element, "angle"), opacity: numeric(element, "opacity"), travelPx: numeric(element, "travel") }, attributes: ["spacing", "thickness", "angle", "opacity", "travel"] };
+ case "DirectionalMatte": return { value: { kind: "directional-matte", angleDeg: numeric(element, "angle"), coverage: numeric(element, "coverage"), feather: numeric(element, "feather"), color: text(element, "color"), opacity: numeric(element, "opacity"), progress: { from: numeric(element, "from"), to: numeric(element, "to") } }, attributes: ["angle", "coverage", "feather", "color", "opacity", "from", "to"] };
+ case "WhipVeil": return { value: { kind: "whip-veil", direction: text(element, "direction") as "left", widthPx: numeric(element, "width"), softnessPx: numeric(element, "softness"), travelPx: numeric(element, "travel"), opacity: numeric(element, "opacity") }, attributes: ["direction", "width", "softness", "travel", "opacity"] };
+ case "GlitchVeil": return { value: { kind: "glitch-veil", bars: integer(element, "bars"), colors: colors(element, "colors"), opacity: numeric(element, "opacity"), travelPx: numeric(element, "travel"), seed: integer(element, "seed") }, attributes: ["bars", "colors", "opacity", "travel", "seed"] };
+ case "Grain": return { value: { kind: "grain", amount: numeric(element, "amount"), grainSizePx: numeric(element, "size"), chroma: text(element, "chroma") as "monochrome", motionRatePxPerFrame: numeric(element, "motion-rate"), seed: integer(element, "seed") }, attributes: ["amount", "size", "chroma", "motion-rate", "seed"] };
+ case "LightLeak": return { value: { kind: "light-leak", colors: colors(element, "colors"), angleDeg: numeric(element, "angle"), softness: numeric(element, "softness"), travelPx: numeric(element, "travel"), intensity: numeric(element, "intensity"), seed: integer(element, "seed") }, attributes: ["colors", "angle", "softness", "travel", "intensity", "seed"] };
+ case "Bokeh": return { value: { kind: "bokeh", amount: numeric(element, "amount"), sizeMinPx: numeric(element, "min-size"), sizeMaxPx: numeric(element, "max-size"), color: text(element, "color"), warmth: numeric(element, "warmth"), driftPx: numeric(element, "drift"), seed: integer(element, "seed") }, attributes: ["amount", "min-size", "max-size", "color", "warmth", "drift", "seed"] };
+ case "TVStatic": return { value: { kind: "tv-static", amount: numeric(element, "amount"), noiseSizePx: numeric(element, "size"), scanLineOpacity: numeric(element, "scan-lines"), motionRatePxPerFrame: numeric(element, "motion-rate"), seed: integer(element, "seed") }, attributes: ["amount", "size", "scan-lines", "motion-rate", "seed"] };
+ default: throw new Error(`${element.name} is not an official self-contained Screen Overlay component.`);
+ }
+}
+
+export const decodeScreenOverlaySurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ allowed(element, ["id", "canvas", "space"]); const id = text(element, "id");
+ const canvas = ref(element.attributes.canvas, `${element.name}.canvas`, spatialTypes.canvas, resolveReference);
+ const space = ref(element.attributes.space, `${element.name}.space`, programSpaceTypes.programSpace, resolveReference);
+ const headerId = `${id}.header`; const records: SurfaceRecordDraft[] = [{ id: headerId, type: screenOverlayTypes.header,
+ value: { kind: "inline", value: sealScreenOverlayHeader({ contract: "svml.screen-overlay-header@1", id }) }, range: element.range }];
+ const fragmentItems: Parameters<typeof createScreenOverlayFragment>[0][number][] = [];
+ const inputs: Record<string, typeof canvas.ref> = { canvas: canvas.ref, header: { kind: "record", id: headerId }, space: space.ref };
+ let index = 0;
+ for (const child of element.children) {
+ if (child.kind === "text") { if (child.value.trim()) throw new Error(`${element.name} accepts only component children.`); continue; }
+ if (child.children.some((node) => node.kind === "element" || node.value.trim())) throw new Error(`${child.name} must be empty.`);
+ index += 1; const suffix = String(index).padStart(4, "0"); const decoded = content(child);
+ allowed(child, ["id", "z", ...TIMING, ...decoded.attributes]); const temporal = binding(child, resolveReference);
+ const occurrences = text(child, "occurrences", "one"); if (occurrences !== "one" && occurrences !== "each") throw new Error(`${child.name}.occurrences is invalid.`);
+ const itemSpec = sealScreenOverlayItemSpec({ contract: "svml.screen-overlay-item-spec@1",
+ id: optionalText(child, "id") ?? `${id}.${decoded.value.kind}.${suffix}`, content: decoded.value,
+ projection: temporal.projection, expansion: { kind: occurrences }, stackingOrder: integer(child, "z") });
+ const specId = `${id}.item.${suffix}.spec`; const specName = `item-${suffix}-spec`;
+ records.push({ id: specId, type: screenOverlayTypes.itemSpec, value: { kind: "inline", value: itemSpec }, range: child.range });
+ inputs[specName] = { kind: "record", id: specId };
+ if (temporal.kind === "program") fragmentItems.push({ kind: "program", specName });
+ else {
+ const mapName = `item-${suffix}-map`; const sourceName = `item-${suffix}-${temporal.kind}`;
+ inputs[mapName] = temporal.map!.ref; inputs[sourceName] = temporal.source!.ref;
+ fragmentItems.push({ kind: temporal.kind, specName, mapName, sourceName });
+ }
+ }
+ if (fragmentItems.length === 0) throw new Error(`${element.name} requires at least one component.`);
+ const fragment = createScreenOverlayFragment(fragmentItems, `@narratage/screen-overlay/surface/${id}@1`);
+ return { records, components: [{ id, fragment: fragment.id, inputs, outputs: { program: `${id}.program`, track: `${id}.track` }, range: element.range }], fragments: [fragment] };
+};
diff --git a/packages/screen-overlay/src/types.ts b/packages/screen-overlay/src/types.ts
new file mode 100644
index 00000000..9d8a17f8
--- /dev/null
+++ b/packages/screen-overlay/src/types.ts
@@ -0,0 +1,70 @@
+import type { FrameSpan } from "@narratage/composition";
+import type { OccurrenceExpansion, TemporalWindowProjection } from "@narratage/temporal";
+
+export type FlashOverlay = {
+ readonly kind: "flash"; readonly color: string; readonly intensity: number;
+ readonly attackFrames: number; readonly holdFrames: number; readonly decayFrames: number;
+};
+export type ColorWashOverlay = { readonly kind: "color-wash"; readonly color: string; readonly opacity: number };
+export type VignetteOverlay = {
+ readonly kind: "vignette"; readonly center: { readonly x: number; readonly y: number };
+ readonly radius: { readonly x: number; readonly y: number }; readonly softness: number;
+ readonly color: string; readonly opacity: number;
+};
+export type ScanLinesOverlay = {
+ readonly kind: "scan-lines"; readonly spacingPx: number; readonly thicknessPx: number;
+ readonly angleDeg: number; readonly opacity: number; readonly travelPx: number;
+};
+export type DirectionalMatteOverlay = {
+ readonly kind: "directional-matte"; readonly angleDeg: number; readonly coverage: number;
+ readonly feather: number; readonly color: string; readonly opacity: number;
+ readonly progress: { readonly from: number; readonly to: number };
+};
+export type WhipVeilOverlay = {
+ readonly kind: "whip-veil"; readonly direction: "left" | "right" | "up" | "down";
+ readonly widthPx: number; readonly softnessPx: number; readonly travelPx: number; readonly opacity: number;
+};
+export type GlitchVeilOverlay = {
+ readonly kind: "glitch-veil"; readonly bars: number; readonly colors: readonly string[];
+ readonly opacity: number; readonly travelPx: number; readonly seed: number;
+};
+export type GrainOverlay = {
+ readonly kind: "grain"; readonly amount: number; readonly grainSizePx: number;
+ readonly chroma: "monochrome" | "color"; readonly motionRatePxPerFrame: number; readonly seed: number;
+};
+export type LightLeakOverlay = {
+ readonly kind: "light-leak"; readonly colors: readonly string[]; readonly angleDeg: number;
+ readonly softness: number; readonly travelPx: number; readonly intensity: number; readonly seed: number;
+};
+export type BokehOverlay = {
+ readonly kind: "bokeh"; readonly amount: number; readonly sizeMinPx: number; readonly sizeMaxPx: number;
+ readonly color: string; readonly warmth: number; readonly driftPx: number; readonly seed: number;
+};
+export type TvStaticOverlay = {
+ readonly kind: "tv-static"; readonly amount: number; readonly noiseSizePx: number;
+ readonly scanLineOpacity: number; readonly motionRatePxPerFrame: number; readonly seed: number;
+};
+
+export type ScreenOverlayComponent =
+ | FlashOverlay | ColorWashOverlay | VignetteOverlay | ScanLinesOverlay | DirectionalMatteOverlay
+ | WhipVeilOverlay | GlitchVeilOverlay | GrainOverlay | LightLeakOverlay | BokehOverlay | TvStaticOverlay;
+
+export type ScreenOverlayItemSpec = {
+ readonly contract: "svml.screen-overlay-item-spec@1";
+ readonly id: string;
+ readonly content: ScreenOverlayComponent;
+ readonly projection: TemporalWindowProjection;
+ readonly expansion: OccurrenceExpansion;
+ readonly stackingOrder: number;
+};
+export type ScreenOverlayHeader = { readonly contract: "svml.screen-overlay-header@1"; readonly id: string };
+export type ScreenOverlayItemProgram = {
+ readonly id: string; readonly sourceOccurrenceId: string; readonly span: FrameSpan;
+ readonly content: ScreenOverlayComponent;
+ readonly stacking: { readonly order: number; readonly tieBreak: string };
+};
+export type ScreenOverlaySet = { readonly contract: "svml.screen-overlay-set@1"; readonly items: readonly ScreenOverlayItemProgram[] };
+export type ScreenOverlayProgram = {
+ readonly contract: "svml.screen-overlay-program@1"; readonly id: string;
+ readonly items: readonly ScreenOverlayItemProgram[];
+};
diff --git a/packages/screen-overlay/test/screen-overlay.test.ts b/packages/screen-overlay/test/screen-overlay.test.ts
new file mode 100644
index 00000000..c4bd32fc
--- /dev/null
+++ b/packages/screen-overlay/test/screen-overlay.test.ts
@@ -0,0 +1,221 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { artifactManifest } from "@narratage/artifact";
+import { registerTypeValidatorFacets } from "@narratage/component-kit";
+import { compositionManifest, compositionTypes, sealComposition, sealVisualTrack } from "@narratage/composition";
+import { createResolvedClosure, sealBuildRequest, start } from "@narratage/core";
+import { AuthorFrontendRegistry, compileSourceClosure, resolveCompiledSourceExport } from "@narratage/elaborator";
+import { compileHyperframesDocument } from "@narratage/hyperframes";
+import { mediaManifest } from "@narratage/media";
+import { narrativeManifest } from "@narratage/narrative";
+import { programSpaceDependency, programSpaceManifest, programSpaceTypes, sealProgramSpace } from "@narratage/program-space";
+import { digestOf } from "@narratage/protocol";
+import type { ModuleManifest } from "@narratage/protocol";
+import {
+ appendProgramScreenOverlay,
+ createScreenOverlaySet,
+ createScreenOverlayFragment,
+ decodeScreenOverlaySurface,
+ finalizeScreenOverlay,
+ renderScreenOverlay,
+ screenOverlayManifest,
+ screenOverlayModuleRef,
+ screenOverlayProducers,
+ screenOverlaySurfaceImplementationDigest,
+ sealScreenOverlayHeader,
+ sealScreenOverlayItemSpec,
+} from "@narratage/screen-overlay";
+import type { ScreenOverlayComponent } from "@narratage/screen-overlay";
+import { semanticMapManifest } from "@narratage/semantic-map";
+import { speechEvidenceManifest } from "@narratage/speech-evidence";
+import { speechManifest } from "@narratage/speech";
+import { sealCanvasSpace, spatialComponent, spatialDependency, spatialManifest, spatialTypes } from "@narratage/spatial";
+import { temporalManifest } from "@narratage/temporal";
+import { TextSurfaceRegistry, createTextAuthorFrontend } from "@narratage/text";
+import { createRecordAdmitter, TypeValidatorRegistry } from "@narratage/validation";
+import { visualIrManifest } from "@narratage/visual-ir";
+
+const canvas = sealCanvasSpace({
+ contract: "svml.canvas-space@1", widthPx: 1080, heightPx: 1920,
+ origin: "top-left", xDirection: "right", yDirection: "down", pixelAspect: "square",
+});
+const space = sealProgramSpace({
+ contract: "svml.program-space@1", durationSec: 2, frameRate: { numerator: 30, denominator: 1 },
+});
+const header = sealScreenOverlayHeader({ contract: "svml.screen-overlay-header@1", id: "screen" });
+const components: readonly ScreenOverlayComponent[] = [
+ { kind: "flash", color: "#ffffff", intensity: 0.9, attackFrames: 2, holdFrames: 3, decayFrames: 5 },
+ { kind: "color-wash", color: "#2244ff", opacity: 0.2 },
+ { kind: "vignette", center: { x: 0.5, y: 0.48 }, radius: { x: 0.8, y: 0.65 }, softness: 0.3, color: "#000000", opacity: 0.4 },
+ { kind: "scan-lines", spacingPx: 8, thicknessPx: 1, angleDeg: 4, opacity: 0.18, travelPx: 12 },
+ { kind: "directional-matte", angleDeg: 15, coverage: 0.7, feather: 0.2, color: "#ffffff", opacity: 0.8, progress: { from: -0.4, to: 1.2 } },
+ { kind: "whip-veil", direction: "left", widthPx: 320, softnessPx: 48, travelPx: 1400, opacity: 0.9 },
+ { kind: "glitch-veil", bars: 8, colors: ["#ff0044", "#00ddff"], opacity: 0.6, travelPx: 90, seed: 7 },
+ { kind: "grain", amount: 0.25, grainSizePx: 3, chroma: "monochrome", motionRatePxPerFrame: 0.5, seed: 11 },
+ { kind: "light-leak", colors: ["#ff3300", "#ffd000"], angleDeg: 28, softness: 0.4, travelPx: 260, intensity: 0.55, seed: 13 },
+ { kind: "bokeh", amount: 0.35, sizeMinPx: 12, sizeMaxPx: 70, color: "#ffe2aa", warmth: 0.4, driftPx: 80, seed: 17 },
+ { kind: "tv-static", amount: 0.12, noiseSizePx: 5, scanLineOpacity: 0.16, motionRatePxPerFrame: 0.7, seed: 19 },
+];
+
+function trackFor(content: ScreenOverlayComponent, stackingOrder = 50) {
+ const spec = sealScreenOverlayItemSpec({
+ contract: "svml.screen-overlay-item-spec@1", id: content.kind,
+ content, projection: { start: { ref: "program.start" }, end: { ref: "program.end" } },
+ expansion: { kind: "one" }, stackingOrder,
+ });
+ return renderScreenOverlay(canvas, space, finalizeScreenOverlay(
+ appendProgramScreenOverlay(createScreenOverlaySet(), header, space, spec), header,
+ ));
+}
+
+test("every declared Screen Overlay component lowers to a self-contained deterministic Present", () => {
+ for (const content of components) {
+ const first = trackFor(content);
+ const second = trackFor(structuredClone(content));
+ assert.deepEqual(first, second, `${content.kind} must be deterministic`);
+ assert.equal(first.presents.length, 1);
+ assert.equal(first.presents[0]?.span.endFrameExclusive, 60);
+ assert.ok(first.presents[0]!.elements.length >= 1);
+ for (const element of first.presents[0]!.elements) {
+ assert.equal(element.attributes, undefined, `${content.kind} must not smuggle an external selector or input`);
+ }
+ }
+});
+
+test("Flash owns its exact envelope instead of inheriting a universal peak", () => {
+ const flash = trackFor(components[0]!);
+ assert.deepEqual(flash.presents[0]?.elements[0]?.animation?.keyframes.map((frame) => frame.atFrame), [0, 2, 5, 10, 60]);
+ assert.throws(() => trackFor({ kind: "flash", color: "#ffffff", intensity: 1, attackFrames: 30, holdFrames: 30, decayFrames: 30 }), /envelope exceeds/u);
+});
+
+test("explicit seeds control stochastic component identity", () => {
+ const grain = components.find((item) => item.kind === "grain")! as Extract<ScreenOverlayComponent, { kind: "grain" }>;
+ assert.notDeepEqual(trackFor(grain), trackFor({ ...grain, seed: grain.seed + 1 }));
+ const bokeh = components.find((item) => item.kind === "bokeh")! as Extract<ScreenOverlayComponent, { kind: "bokeh" }>;
+ assert.notDeepEqual(trackFor(bokeh), trackFor({ ...bokeh, seed: bokeh.seed + 1 }));
+});
+
+test("overlay Tracks interleave with peer Tracks only through absolute stacking", () => {
+ const below = trackFor(components[1]!, 20);
+ const above = { ...trackFor(components[0]!, 80), id: "screen-above" };
+ const middle = sealVisualTrack({
+ contract: "svml.visual-track@1", visualIr: "svml.visual-ir@1", id: "middle",
+ presents: [{ id: "middle", span: { startFrame: 0, endFrameExclusive: 60 }, stacking: { order: 50, tieBreak: "middle" },
+ elements: [{ id: "root", order: 0, kind: "box", style: [{ name: "background-color", value: "#112233" }] }] }],
+ });
+ const document = compileHyperframesDocument(sealComposition({
+ contract: "svml.composition@1", id: "stack", canvas: { width: 1080, height: 1920, clearColor: "#000000" },
+ tracks: [above, middle, below],
+ }), space);
+ const belowAt = document.html.indexOf('data-svml-track-id="screen"');
+ const middleAt = document.html.indexOf('data-svml-track-id="middle"');
+ const aboveAt = document.html.indexOf('data-svml-track-id="screen-above"');
+ assert.ok(belowAt < middleAt && middleAt < aboveAt);
+});
+
+test("the package has no lower-composite, sibling Track, backdrop-filter or hidden audio port", () => {
+ const fragment = createScreenOverlayFragment([{ kind: "program", specName: "spec" }], "example.overlay@1");
+ assert.deepEqual(fragment.inputs.map((input) => input.name), ["canvas", "header", "space", "spec"]);
+ assert.equal(fragment.exports.some((output) => output.type.name === "AudioTrack"), false);
+ assert.throws(() => sealScreenOverlayItemSpec({
+ contract: "svml.screen-overlay-item-spec@1", id: "blur",
+ content: { kind: "gaussian-blur" } as unknown as ScreenOverlayComponent,
+ projection: { start: { ref: "program.start" }, end: { ref: "program.end" } },
+ expansion: { kind: "one" }, stackingOrder: 1,
+ }), /unsupported|undefined|kind/iu);
+});
+
+test("the self-described Screen Surface parses into a finite peer-Track graph", async () => {
+ const fixtureModule = { name: "example.screen-inputs", version: "1" } as const;
+ const fixtureSurfaceDigest = digestOf("example.screen-inputs/surface@1");
+ const fixtureManifest: ModuleManifest = {
+ format: "svml.module@1",
+ name: fixtureModule.name,
+ version: fixtureModule.version,
+ dependencies: [spatialDependency, programSpaceDependency],
+ types: [],
+ capabilities: [],
+ surfaces: [{
+ name: "inputs",
+ tag: "Inputs",
+ mode: "structured",
+ outputs: [spatialTypes.canvas, programSpaceTypes.programSpace],
+ implementation: {
+ kind: "trusted-frontend-surface",
+ locator: "example.screen-inputs/surface",
+ digest: fixtureSurfaceDigest,
+ },
+ }],
+ producers: [],
+ };
+ const closure = createResolvedClosure([
+ artifactManifest,
+ mediaManifest,
+ narrativeManifest,
+ programSpaceManifest,
+ speechManifest,
+ speechEvidenceManifest,
+ semanticMapManifest,
+ spatialManifest,
+ temporalManifest,
+ visualIrManifest,
+ compositionManifest,
+ screenOverlayManifest,
+ fixtureManifest,
+ ]);
+ const registry = new TextSurfaceRegistry();
+ registry.registerStructured(fixtureModule, "inputs", fixtureSurfaceDigest, ({ element }) => ({
+ records: [
+ { id: "canvas", type: spatialTypes.canvas, value: { kind: "inline", value: canvas }, range: element.range },
+ { id: "space", type: programSpaceTypes.programSpace, value: { kind: "inline", value: space }, range: element.range },
+ ],
+ components: [],
+ fragments: [],
+ }));
+ registry.registerStructured(
+ screenOverlayModuleRef,
+ "track",
+ screenOverlaySurfaceImplementationDigest,
+ decodeScreenOverlaySurface,
+ );
+ const frontends = new AuthorFrontendRegistry();
+ frontends.register(createTextAuthorFrontend({
+ registry,
+ resolveModule: (request) => request.from === "example.screen-inputs@1" ? fixtureModule : screenOverlayModuleRef,
+ }));
+ const validators = new TypeValidatorRegistry();
+ registerTypeValidatorFacets(validators, spatialComponent.validators ?? []);
+ const compiled = await compileSourceClosure({
+ entry: {
+ id: "/project/screen.svml",
+ name: "screen.svml",
+ text: `<?svml using="@narratage/text@1"?>
+ <svml>
+ <import as="fixture" from="example.screen-inputs@1"/>
+ <import as="screen" from="@narratage/screen-overlay@1"/>
+ <fixture:Inputs/>
+ <screen:Track id="screen-fx" canvas={canvas} space={space}>
+ <screen:Flash during="program" z="70" color="#ffffff" intensity="0.9" attack="2" hold="3" decay="5"/>
+ </screen:Track>
+ </svml>`,
+ },
+ closure,
+ frontends,
+ admitRecord: createRecordAdmitter(validators),
+ resolveSource() { throw new Error("Screen fixture has no source imports."); },
+ });
+ const trackExport = resolveCompiledSourceExport(compiled, "screen-fx.track", compositionTypes.visualTrack);
+ assert.equal(trackExport.ref.kind, "logical-output");
+ const build = start(compiled.program, compiled.elaboration.graph, sealBuildRequest({
+ graph: compiled.elaboration.graph.id,
+ targets: [{ output: trackExport.ref.kind === "logical-output" ? trackExport.ref.id : "", accepts: "exact" }],
+ satisfactions: [],
+ }));
+ assert.deepEqual(build.plan.steps.map((step) => step.producer.name).sort(), [
+ screenOverlayProducers.createSet.name,
+ screenOverlayProducers.appendProgram.name,
+ screenOverlayProducers.finalize.name,
+ screenOverlayProducers.render.name,
+ ].sort());
+});
diff --git a/packages/spatial/src/index.ts b/packages/spatial/src/index.ts
index 18ad5f0f..ac3c90ab 100644
--- a/packages/spatial/src/index.ts
+++ b/packages/spatial/src/index.ts
@@ -16,7 +16,7 @@ export {
spatialSurfaceDigests,
spatialTypes,
} from "./manifest.js";
-export { spatialFrameSchema } from "./schema.js";
+export { contentFitSchema, intrinsicExtentSchema, spatialFrameSchema } from "./schema.js";
export {
decodeAnchoredFrameSurface,
decodeAspectFrameSurface,
diff --git a/packages/speech-alignment/src/locate.ts b/packages/speech-alignment/src/locate.ts
index 3d77f22f..a64cd926 100644
--- a/packages/speech-alignment/src/locate.ts
+++ b/packages/speech-alignment/src/locate.ts
@@ -54,11 +54,13 @@ function validateBasis(narrative: Narrative, basis: SpeechAudioBasis): void {
if (
!Number.isFinite(basis.programSpace.durationSec)
|| basis.programSpace.durationSec <= 0
- || Math.abs(basis.audio.durationSec - basis.programSpace.durationSec) > EPSILON
) {
- fail("SPEECH_BASIS_DURATION", "SpeechAudioBasis audio and ProgramSpace must have the same positive duration.");
+ fail("SPEECH_BASIS_DURATION", "SpeechAudioBasis ProgramSpace must have a positive duration.");
+ }
+ if (basis.audio.kind !== "blob" || !isDigest(basis.audio.digest)
+ || basis.audio.mediaType !== "audio/wav" || !Number.isSafeInteger(basis.audio.size) || basis.audio.size < 0) {
+ fail("SPEECH_AUDIO_DIGEST", "SpeechBasis audio BlobRef is invalid.");
}
- if (!isDigest(basis.audio.digest)) fail("SPEECH_AUDIO_DIGEST", "SpeechBasis audio digest is invalid.");
if (basis.segments.length !== narrative.segments.length) {
fail("SPEECH_BASIS_SEGMENTS", "SpeechAudioBasis must cover every Narrative Segment exactly once.");
}
diff --git a/packages/speech-alignment/test/speech-align.test.ts b/packages/speech-alignment/test/speech-align.test.ts
index 4487d65f..0e5fde25 100644
--- a/packages/speech-alignment/test/speech-align.test.ts
+++ b/packages/speech-alignment/test/speech-align.test.ts
@@ -75,7 +75,7 @@ function speechBasis(
const take = sealSpeechBasis({
contract: "svml.speech-basis@1",
programSpace,
- audio: { digest: audioDigest, size: 1, mediaType: "audio/wav", durationSec },
+ audio: { kind: "blob", digest: audioDigest, size: 1, mediaType: "audio/wav" },
visualTrack: {
clips: segments.map((segment) => ({
segmentId: segment.segmentId,
diff --git a/packages/speech-basis/src/projection.ts b/packages/speech-basis/src/projection.ts
index e8a65fc7..444efaab 100644
--- a/packages/speech-basis/src/projection.ts
+++ b/packages/speech-basis/src/projection.ts
@@ -1,4 +1,4 @@
-import { programSpaceFrameCount } from "@narratage/program-space";
+import { programSpaceFrameCount, programSpaceSampleFrames } from "@narratage/program-space";
import type { ProgramSpace } from "@narratage/program-space";
import { assertSpeechBasisIdentity } from "@narratage/speech";
import type { SpeechAudioBasis, SpeechBasis } from "@narratage/speech";
@@ -81,14 +81,26 @@ export function projectSpeechVisual(basis: SpeechBasis): VisualTrack {
export function projectSpeechAudioTrack(basis: SpeechBasis): AudioTrack {
assertSpeechBasisIdentity(basis);
+ const sampleFrames = programSpaceSampleFrames(basis.programSpace, 48_000);
return sealAudioTrack({
contract: "svml.audio-track@1",
id: `speech-audio:${basis.segments.map((segment) => segment.segmentId).join("+")}`,
clips: [{
id: "speech",
- span: { startFrame: 0, endFrameExclusive: programSpaceFrameCount(basis.programSpace) },
artifact: basis.audio,
- bus: "speech",
+ target: { startSample: 0, endSampleExclusive: sampleFrames },
+ source: {
+ sampleFrames,
+ startSample: 0,
+ endSampleExclusive: sampleFrames,
+ loop: false,
+ phaseSample: 0,
+ },
+ playbackRate: 1,
+ pitch: "preserve",
+ gain: 1,
+ fadeInSamples: 0,
+ fadeOutSamples: 0,
}],
});
}
diff --git a/packages/speech-basis/test/product.test.ts b/packages/speech-basis/test/product.test.ts
index 6e3b7b18..4b216718 100644
--- a/packages/speech-basis/test/product.test.ts
+++ b/packages/speech-basis/test/product.test.ts
@@ -84,10 +84,10 @@ function sampleTake(label = "generated"): SpeechBasis {
frameRate: { numerator: 30, denominator: 1 },
});
const audio = {
+ kind: "blob" as const,
digest: digestOf(`${label}:audio`),
size: 12,
mediaType: "audio/wav",
- durationSec,
};
const visual = {
digest: digestOf(`${label}:visual`),
@@ -276,7 +276,11 @@ test("SpeechBasis projects to peer generic visual and audio Tracks", () => {
assert.equal(visual.contract, "svml.visual-track@1");
assert.equal(audio.contract, "svml.audio-track@1");
assert.deepEqual(programSpace, take.programSpace);
- assert.equal(visual.presents[0]?.span.endFrameExclusive, audio.clips[0]?.span.endFrameExclusive);
+ assert.equal(
+ audio.clips[0]?.target.endSampleExclusive,
+ Math.round(visual.presents[0]!.span.endFrameExclusive * 48_000
+ * take.programSpace.frameRate.denominator / take.programSpace.frameRate.numerator),
+ );
});
test("a substitute visual Candidate does not contaminate an independent exact audio path", () => {
diff --git a/packages/speech-spine/src/program.ts b/packages/speech-spine/src/program.ts
index 07b3736b..d11fe662 100644
--- a/packages/speech-spine/src/program.ts
+++ b/packages/speech-spine/src/program.ts
@@ -150,12 +150,16 @@ export function compileSpeechSpineAudio(program: SpeechSpineProgram, set: Speech
artifact: audio.artifact,
targetStartSample,
targetEndSampleExclusive,
+ sourceSampleFrames: audio.sampleFrames,
sourceStartSample: 0,
+ sourceEndSampleExclusive: audio.sampleFrames,
+ sourceLoop: false,
+ sourcePhaseSample: 0,
playbackRate: 1,
+ pitch: "preserve" as const,
gain: 1,
fadeInSamples: 0,
fadeOutSamples: 0,
- bus: "speech" as const,
};
});
const plan = sealAudioProgramPlan({
@@ -203,12 +207,7 @@ export function assembleSpeechBasis(
return sealSpeechBasis({
contract: "svml.speech-basis@1",
programSpace: space,
- audio: {
- digest: audio.artifact.digest,
- size: audio.artifact.size,
- mediaType: audio.artifact.mediaType,
- durationSec: space.durationSec,
- },
+ audio: structuredClone(audio.artifact),
visualTrack: { clips: visualClips },
segments,
});
diff --git a/packages/speech-spine/test/video-pipeline.test.ts b/packages/speech-spine/test/video-pipeline.test.ts
index fc37eebe..4e976204 100644
--- a/packages/speech-spine/test/video-pipeline.test.ts
+++ b/packages/speech-spine/test/video-pipeline.test.ts
@@ -169,7 +169,7 @@ test("the Speech Spine pipeline resumes without repeating paid calls", async ()
const basis = sealSpeechBasis({
contract: "svml.speech-basis@1",
programSpace,
- audio: { digest: audioDigest, size: 1, mediaType: "audio/wav", durationSec },
+ audio: { kind: "blob", digest: audioDigest, size: 1, mediaType: "audio/wav" },
visualTrack: { clips: [{
segmentId: "line",
artifact: { digest: visualDigest, size: 1, mediaType: "video/mp4", durationSec },
@@ -537,10 +537,10 @@ test("an Existing SpeechBasis cuts generation while a visual substitute cuts the
contract: "svml.speech-basis@1",
programSpace,
audio: {
+ kind: "blob",
digest: digestOf("existing-take:audio"),
size: 1,
mediaType: "audio/wav",
- durationSec: 1,
},
visualTrack: { clips: [{
segmentId: "line",
diff --git a/packages/speech/src/identity.ts b/packages/speech/src/identity.ts
index c0717ba5..e7b38fbc 100644
--- a/packages/speech/src/identity.ts
+++ b/packages/speech/src/identity.ts
@@ -1,6 +1,10 @@
import { isDigest } from "@narratage/protocol";
import { assertProgramSpaceIdentity } from "@narratage/program-space";
import type { SpeechAudioBasis, SpeechBasis, SpeechDuration, SpeechEvidenceAudio } from "./types.js";
+function assertAudioBlob(value: SpeechBasis["audio"], label: string): void {
+ if (value.kind !== "blob" || !isDigest(value.digest) || !Number.isSafeInteger(value.size)
+ || value.size < 0 || value.mediaType !== "audio/wav") throw new Error(`${label} must be a canonical WAV BlobRef.`);
+}
export function sealSpeechDuration(value: SpeechDuration): SpeechDuration { return structuredClone(value); }
export function assertSpeechDurationIdentity(value: SpeechDuration): void {
if (value.contract !== "svml.speech-duration@1" || !Number.isFinite(value.durationSec) || value.durationSec <= 0) throw new Error("SpeechDuration is invalid.");
@@ -28,12 +32,12 @@ export function assertSpeechEvidenceAudioIdentity(value: SpeechEvidenceAudio): v
export function assertSpeechBasisIdentity(basis: SpeechBasis): void {
if (basis.contract !== "svml.speech-basis@1") throw new Error("Unsupported SpeechBasis contract.");
assertProgramSpaceIdentity(basis.programSpace);
- if (basis.audio.durationSec !== basis.programSpace.durationSec || !isDigest(basis.audio.digest)) throw new Error("SpeechBasis audio does not match its ProgramSpace.");
+ assertAudioBlob(basis.audio, "SpeechBasis audio");
}
export function assertSpeechAudioBasisIdentity(basis: SpeechAudioBasis): void {
if (basis.contract !== "svml.speech-audio-basis@1") throw new Error("Unsupported SpeechAudioBasis contract.");
assertProgramSpaceIdentity(basis.programSpace);
- if (basis.audio.durationSec !== basis.programSpace.durationSec || !isDigest(basis.audio.digest)) throw new Error("SpeechAudioBasis audio does not match its ProgramSpace.");
+ assertAudioBlob(basis.audio, "SpeechAudioBasis audio");
if (basis.segments.length === 0) throw new Error("SpeechAudioBasis must contain at least one Segment.");
let previousEnd = 0;
for (const segment of basis.segments) {
diff --git a/packages/speech/src/schema.ts b/packages/speech/src/schema.ts
index 3c1de39d..6b562b4c 100644
--- a/packages/speech/src/schema.ts
+++ b/packages/speech/src/schema.ts
@@ -6,17 +6,23 @@ const number = { kind: "number", minimum: 0 } as const;
const integer = { kind: "number", integer: true, minimum: 0 } as const;
const digest = { kind: "string", minLength: 71, maxLength: 71 } as const;
const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>): ValueSchema => ({ kind: "object", fields });
+const audioBlobRef = object({
+ kind: { schema: { kind: "literal", value: "blob" } },
+ digest: { schema: digest },
+ size: { schema: integer },
+ mediaType: { schema: { kind: "literal", value: "audio/wav" } },
+});
const segment = object({ segmentId: { schema: string }, startSec: { schema: number }, endSec: { schema: number } });
export const speechDurationSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.speech-duration@1" } }, durationSec: { schema: number } });
export const speechBasisSchema: ValueSchema = object({
- contract: { schema: { kind: "literal", value: "svml.speech-basis@1" } }, programSpace: { schema: programSpaceSchema }, audio: { schema: mediaArtifactSchema },
+ contract: { schema: { kind: "literal", value: "svml.speech-basis@1" } }, programSpace: { schema: programSpaceSchema }, audio: { schema: audioBlobRef },
visualTrack: { schema: object({ clips: { schema: { kind: "array", items: object({ segmentId: { schema: string }, artifact: { schema: mediaArtifactSchema }, startSec: { schema: number }, endSec: { schema: number } }) } } }) },
segments: { schema: { kind: "array", minItems: 1, items: segment } },
});
export const speechAudioBasisSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.speech-audio-basis@1" } },
- programSpace: { schema: programSpaceSchema }, audio: { schema: mediaArtifactSchema }, segments: { schema: { kind: "array", minItems: 1, items: segment } } });
+ programSpace: { schema: programSpaceSchema }, audio: { schema: audioBlobRef }, segments: { schema: { kind: "array", minItems: 1, items: segment } } });
export const speechEvidenceAudioSchema: ValueSchema = object({ contract: { schema: { kind: "literal", value: "svml.speech-evidence-audio@1" } },
- artifact: { schema: object({ kind: { schema: { kind: "literal", value: "blob" } }, digest: { schema: digest }, size: { schema: integer }, mediaType: { schema: { kind: "literal", value: "audio/wav" } } }) },
+ artifact: { schema: audioBlobRef },
codec: { schema: { kind: "literal", value: "pcm_s16le" } }, sampleRate: { schema: { kind: "literal", value: 16_000 } }, channels: { schema: { kind: "literal", value: 1 } },
sampleFrames: { schema: { kind: "number", integer: true, minimum: 1 } }, durationSec: { schema: number },
segments: { schema: { kind: "array", minItems: 1, items: segment } }, sampleMap: { schema: object({
diff --git a/packages/speech/src/types.ts b/packages/speech/src/types.ts
index 44c9c7ea..f8ca8e16 100644
--- a/packages/speech/src/types.ts
+++ b/packages/speech/src/types.ts
@@ -4,12 +4,12 @@ import type { BlobRef } from "@narratage/protocol";
export type SpeechDuration = { readonly contract: "svml.speech-duration@1"; readonly durationSec: number };
export type SpeechBasisSegment = { readonly segmentId: string; readonly startSec: number; readonly endSec: number };
export type SpeechBasis = {
- readonly contract: "svml.speech-basis@1"; readonly programSpace: ProgramSpace; readonly audio: MediaArtifactRef;
+ readonly contract: "svml.speech-basis@1"; readonly programSpace: ProgramSpace; readonly audio: BlobRef;
readonly visualTrack: { readonly clips: readonly { readonly segmentId: string; readonly artifact: MediaArtifactRef; readonly startSec: number; readonly endSec: number }[] };
readonly segments: readonly SpeechBasisSegment[];
};
export type SpeechAudioBasis = {
- readonly contract: "svml.speech-audio-basis@1"; readonly programSpace: ProgramSpace; readonly audio: MediaArtifactRef;
+ readonly contract: "svml.speech-audio-basis@1"; readonly programSpace: ProgramSpace; readonly audio: BlobRef;
readonly segments: readonly SpeechBasisSegment[];
};
export type SpeechEvidenceAudio = {
diff --git a/packages/temporal/src/index.ts b/packages/temporal/src/index.ts
index 9ddccd93..aa88a6cc 100644
--- a/packages/temporal/src/index.ts
+++ b/packages/temporal/src/index.ts
@@ -4,6 +4,7 @@ import type { ModuleManifest } from "@narratage/protocol";
export * from "./location.js";
export * from "./projection.js";
export * from "./schedule.js";
+export * from "./sample.js";
export type * from "./types.js";
export const temporalModuleRef = { name: "@narratage/temporal", version: "0.0.0-dev" } as const;
diff --git a/packages/temporal/src/sample.ts b/packages/temporal/src/sample.ts
new file mode 100644
index 00000000..f1d4a318
--- /dev/null
+++ b/packages/temporal/src/sample.ts
@@ -0,0 +1,43 @@
+import type { ProgramSpace } from "@narratage/program-space";
+
+import type { TemporalDuration } from "./types.js";
+
+function integer(value: number, label: string): bigint {
+ if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${label} must be a non-negative safe integer.`);
+ return BigInt(value);
+}
+
+function roundedRatio(numerator: bigint, denominator: bigint, label: string): number {
+ if (denominator <= 0n) throw new Error(`${label} denominator must be positive.`);
+ const value = (numerator * 2n + denominator) / (denominator * 2n);
+ if (value < 0n || value > BigInt(Number.MAX_SAFE_INTEGER)) {
+ throw new Error(`${label} exceeds safe sample arithmetic.`);
+ }
+ return Number(value);
+}
+
+/** Quantize one non-negative authored duration into the canonical sample domain exactly once. */
+export function temporalDurationInSamples(
+ duration: TemporalDuration,
+ space: ProgramSpace,
+ sampleRate = 48_000,
+): number {
+ const rate = integer(sampleRate, "Sample rate");
+ if (rate === 0n) throw new Error("Sample rate must be positive.");
+ switch (duration.unit) {
+ case "frames":
+ return roundedRatio(
+ integer(duration.value, "Frame duration") * rate * integer(space.frameRate.denominator, "Frame-rate denominator"),
+ integer(space.frameRate.numerator, "Frame-rate numerator"),
+ "Frame duration",
+ );
+ case "milliseconds":
+ return roundedRatio(integer(duration.value, "Millisecond duration") * rate, 1_000n, "Millisecond duration");
+ case "seconds":
+ return roundedRatio(
+ integer(duration.numerator, "Second duration numerator") * rate,
+ integer(duration.denominator, "Second duration denominator"),
+ "Second duration",
+ );
+ }
+}
diff --git a/packages/temporal/test/temporal.test.ts b/packages/temporal/test/temporal.test.ts
index 40722eeb..02567d0a 100644
--- a/packages/temporal/test/temporal.test.ts
+++ b/packages/temporal/test/temporal.test.ts
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
import test from "node:test";
import type { NarrativeMomentRef, NarrativeSelectionRef } from "@narratage/narrative";
+import { programFrameSampleBoundary, programSpaceSampleFrames } from "@narratage/program-space";
import type { ProgramSpace } from "@narratage/program-space";
import type { CompleteSemanticMap } from "@narratage/semantic-map";
@@ -12,6 +13,7 @@ import {
projectProgramWindow,
projectSelectionWindows,
resolveTriggeredSchedule,
+ temporalDurationInSamples,
} from "../src/index.js";
const space: ProgramSpace = {
@@ -120,6 +122,26 @@ test("program and absolute projections use exact rational frame-rate arithmetic"
assert.deepEqual(result.span, { startFrame: 15, endFrameExclusive: 30 });
});
+test("frame and authored durations enter one exact sample-boundary rule", () => {
+ const ntsc: ProgramSpace = {
+ contract: "svml.program-space@1",
+ durationSec: 1.001,
+ frameRate: { numerator: 30_000, denominator: 1_001 },
+ };
+ assert.equal(programFrameSampleBoundary(ntsc, 15, 48_000), 24_024);
+ assert.equal(programSpaceSampleFrames(ntsc, 48_000), 48_048);
+ assert.equal(temporalDurationInSamples(frames(15), ntsc), 24_024);
+ assert.equal(temporalDurationInSamples({ unit: "milliseconds", value: 125 }, ntsc), 6_000);
+ assert.equal(temporalDurationInSamples(seconds(1, 3), ntsc), 16_000);
+ const twentyFour: ProgramSpace = {
+ contract: "svml.program-space@1",
+ durationSec: 1,
+ frameRate: { numerator: 24, denominator: 1 },
+ };
+ assert.equal(programFrameSampleBoundary(twentyFour, 1, 44_100), 1_838,
+ "half-sample boundaries round to the later sample deterministically");
+});
+
test("crossed source anchors are allowed until a projection actually consumes the reversal", () => {
const crossed = selection("crossed", [{ occurrence: 0, startAnchorId: "d", endAnchorId: "a" }]);
assert.deepEqual(locateSelectionOccurrences(map, crossed, space)[0], {
diff --git a/packages/whisperx/test/whisperx.test.ts b/packages/whisperx/test/whisperx.test.ts
index cdec6d23..e9edd80e 100644
--- a/packages/whisperx/test/whisperx.test.ts
+++ b/packages/whisperx/test/whisperx.test.ts
@@ -28,7 +28,7 @@ function basis() {
return sealSpeechBasis({
contract: "svml.speech-basis@1",
programSpace,
- audio: { digest: digestOf("whisperx-test:audio"), size: 1, mediaType: "audio/wav", durationSec: 1 },
+ audio: { kind: "blob", digest: digestOf("whisperx-test:audio"), size: 1, mediaType: "audio/wav" },
visualTrack: { clips: [{
segmentId: "line",
artifact: { digest: visual, size: 1, mediaType: "video/mp4", durationSec: 1 },
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 8903bcb6..795d2879 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -11,6 +11,9 @@ importers:
'@narratage/artifact':
specifier: workspace:*
version: link:packages/artifact
+ '@narratage/audio-track':
+ specifier: workspace:*
+ version: link:packages/audio-track
'@narratage/broll':
specifier: workspace:*
version: link:packages/broll
@@ -53,6 +56,9 @@ importers:
'@narratage/media-pipeline':
specifier: workspace:*
version: link:packages/media-pipeline
+ '@narratage/media-track':
+ specifier: workspace:*
+ version: link:packages/media-track
'@narratage/narrative':
specifier: workspace:*
version: link:packages/narrative
@@ -92,6 +98,9 @@ importers:
'@narratage/runtime-adapter-node':
specifier: workspace:*
version: link:packages/runtime-adapter-node
+ '@narratage/screen-overlay':
+ specifier: workspace:*
+ version: link:packages/screen-overlay
'@narratage/script':
specifier: workspace:*
version: link:packages/script
@@ -180,6 +189,39 @@ importers:
specifier: workspace:*
version: link:../runtime-adapter
+ packages/audio-track:
+ dependencies:
+ '@narratage/component-kit':
+ specifier: workspace:*
+ version: link:../component-kit
+ '@narratage/composition':
+ specifier: workspace:*
+ version: link:../composition
+ '@narratage/elaborator':
+ specifier: workspace:*
+ version: link:../elaborator
+ '@narratage/media':
+ specifier: workspace:*
+ version: link:../media
+ '@narratage/narrative':
+ specifier: workspace:*
+ version: link:../narrative
+ '@narratage/program-space':
+ specifier: workspace:*
+ version: link:../program-space
+ '@narratage/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ '@narratage/semantic-map':
+ specifier: workspace:*
+ version: link:../semantic-map
+ '@narratage/temporal':
+ specifier: workspace:*
+ version: link:../temporal
+ '@narratage/text':
+ specifier: workspace:*
+ version: link:../text
+
packages/broll:
dependencies:
'@narratage/artifact':
@@ -1175,6 +1217,36 @@ importers:
specifier: workspace:*
version: link:../validation
+ packages/media-track:
+ dependencies:
+ '@narratage/artifact':
+ specifier: workspace:*
+ version: link:../artifact
+ '@narratage/component-kit':
+ specifier: workspace:*
+ version: link:../component-kit
+ '@narratage/composition':
+ specifier: workspace:*
+ version: link:../composition
+ '@narratage/elaborator':
+ specifier: workspace:*
+ version: link:../elaborator
+ '@narratage/media':
+ specifier: workspace:*
+ version: link:../media
+ '@narratage/program-space':
+ specifier: workspace:*
+ version: link:../program-space
+ '@narratage/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ '@narratage/spatial':
+ specifier: workspace:*
+ version: link:../spatial
+ '@narratage/temporal':
+ specifier: workspace:*
+ version: link:../temporal
+
packages/minimax-h3:
dependencies:
'@narratage/generation':
@@ -1726,6 +1798,39 @@ importers:
specifier: workspace:*
version: link:../runtime-adapter
+ packages/screen-overlay:
+ dependencies:
+ '@narratage/component-kit':
+ specifier: workspace:*
+ version: link:../component-kit
+ '@narratage/composition':
+ specifier: workspace:*
+ version: link:../composition
+ '@narratage/elaborator':
+ specifier: workspace:*
+ version: link:../elaborator
+ '@narratage/narrative':
+ specifier: workspace:*
+ version: link:../narrative
+ '@narratage/program-space':
+ specifier: workspace:*
+ version: link:../program-space
+ '@narratage/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ '@narratage/semantic-map':
+ specifier: workspace:*
+ version: link:../semantic-map
+ '@narratage/spatial':
+ specifier: workspace:*
+ version: link:../spatial
+ '@narratage/temporal':
+ specifier: workspace:*
+ version: link:../temporal
+ '@narratage/text':
+ specifier: workspace:*
+ version: link:../text
+
packages/script:
dependencies:
'@narratage/core':
@@ -2092,9 +2197,15 @@ importers:
'@narratage/semantic-map':
specifier: workspace:*
version: link:../semantic-map
+ '@narratage/spatial':
+ specifier: workspace:*
+ version: link:../spatial
'@narratage/svs':
specifier: workspace:*
version: link:../svs
+ '@narratage/temporal':
+ specifier: workspace:*
+ version: link:../temporal
'@narratage/text':
specifier: workspace:*
version: link:../text
diff --git a/spec/audio-track.md b/spec/audio-track.md
index 1324ec82..514a31d9 100644
--- a/spec/audio-track.md
+++ b/spec/audio-track.md
@@ -1,7 +1,8 @@
# SVML Audio Track Authoring
-Status: design authority for the generic official Audio Track package. It is not implemented by an
-author Surface yet and is not a frozen public ABI.
+Status: implemented pre-release contract for the generic official Audio Track package, including
+its self-described author Surface, exact sample-domain lowering and local/remote execution witness.
+It is not yet a frozen public ABI.
## Purpose
@@ -72,7 +73,7 @@ These are package-level concerns. They are not new fields on Core Operations or
An illustrative author Surface may remain compact:
```xml
-<audio:Track id="mix">
+<audio:Track id="mix" space={speech.space}>
<audio:Clip
source={music.media}
during={program}
@@ -104,7 +105,7 @@ Audio uses the same rules as [`track-authoring.md`](./track-authoring.md):
3. project a directed candidate window;
4. intersect and validate it against ProgramSpace;
5. apply audio source trim and occupancy;
-6. lower the result to frame-exact target spans and exact source samples.
+6. lower both target placement and source sampling into the exact 48 kHz sample domain.
Audio items are independent. Overlapping projected windows mix; they are never clipped according
to authoring order, auto-stitched, deduplicated or treated as competing alternatives. Several Audio
@@ -184,20 +185,18 @@ automatic music-bed behavior. Those are legitimate future audio-processing or mi
they must consume explicit inputs and make their behavior author-visible. A runtime or Provider may
not add them as an environment preference.
-The current terminal `bus: speech | music | sfx | source` field is copied into the audio plan but
-does not affect rendering. The official author package must not attach hidden behavior to this dead
-label. Before the public Track freeze it should either be removed, or be replaced by an explicit
-mix-routing contract that has a real downstream consumer. Merely naming a clip `music` cannot cause
-ducking.
+The former terminal `bus: speech | music | sfx | source` label has been removed. It had no rendering
+effect and therefore carried no honest meaning. A future mix-routing contract must have an explicit
+consumer and graph behavior; merely naming a clip `music` cannot cause ducking.
## 7. Lowering and execution
The deterministic author package lowers every realized item to an ordinary terminal `AudioClip`:
-- target `span` from temporal projection;
+- exact target `{ startSample, endSampleExclusive }` from temporal projection;
- canonical WAV Artifact from the normalized input;
-- exact source offset and playback rate from trim/occupancy;
-- linear gain and fades from presentation.
+- exact source sample interval, loop phase and playback rate from trim/occupancy;
+- pitch-preservation intent, linear gain and exact sample fades from presentation.
Composition validates the Audio Track against an explicitly connected ProgramSpace. The media
pipeline then compiles all peer Audio Tracks into one content-addressed `AudioProgramPlan`. Local
@@ -233,19 +232,20 @@ Retire:
## 9. Implementation and acceptance
-Implementation should wait for the shared `@narratage/temporal` package, then proceed without a
-Core, Composition or Provider change:
+The shared Temporal package and the pre-freeze terminal Audio candidate have now been completed
+without a Core branch, Provider-family branch or privileged Film lane:
-1. add the `@narratage/audio-track` author package and self-described Surface;
-2. consume explicit normalized `SynchronizedMedia` inputs;
-3. implement sample-exact trim and every occupancy law;
-4. lower to the existing terminal `AudioTrack`;
-5. add graph tests for arbitrary item counts and Selection/Moment `one` / `each`;
-6. add sample-level tests for shorter/equal/longer sources, both alignments, loops, stretch bounds
+1. **Implemented:** add the `@narratage/audio-track` author package and self-described Surface;
+2. **Implemented:** consume explicit normalized `SynchronizedMedia` inputs;
+3. **Implemented:** sample-exact trim and every occupancy law;
+4. **Implemented:** lower to the peer terminal `AudioTrack`;
+5. **Implemented:** graph tests for arbitrary item counts and Selection/Moment `one` / `each`;
+6. **Implemented:** sample-level tests for shorter/equal/longer sources, both alignments, loops, stretch bounds
and fades;
-7. prove two overlapping items and two peer Audio Tracks produce the same planned mix facts;
-8. prove a different local/remote media Provider receives the identical `AudioProgramPlan`.
+7. **Implemented:** two overlapping items and two peer Audio Tracks produce the same planned mix facts;
+8. **Implemented:** local FFmpeg and remote Lambda Providers receive the identical
+ content-addressed `AudioProgramPlan`; the real FFmpeg witness additionally checks exact loop
+ phase sample by sample.
The migration is incomplete if adding this package requires a Core branch, a Film audio family, a
new queue, a Provider name in SVML or a hidden Base-audio rule.
-
diff --git a/spec/media-track.md b/spec/media-track.md
index f762db35..4203433a 100644
--- a/spec/media-track.md
+++ b/spec/media-track.md
@@ -1,8 +1,9 @@
# SVML Media Track Authoring
-Status: design authority for the unified official Media Track migration. It supersedes the intended
-author model of the current executable `@narratage/broll` vertical slice, but is not implemented or
-a frozen public ABI.
+Status: design authority for the unified official Media Track migration. Its first explicit
+still-image Item witness is executable; timed material, layers, motion, Sequence, audio projection
+and the author Surface remain unimplemented. It supersedes the intended author model of the current
+`@narratage/broll` vertical slice and is not a frozen public ABI.
## 1. Conclusion
@@ -596,7 +597,8 @@ public compatibility layer. Source assets and generated files remain untouched.
Implement only after the shared Temporal and Spatial slices exist:
-1. introduce `@narratage/media-track` with one still-image Item;
+1. **Implemented:** introduce `@narratage/media-track` with one still-image Item whose BlobArtifact,
+ IntrinsicExtent, SpatialFrame, ContentFit and ProgramSpace are separate semantic inputs;
2. add timed source trim and every occupancy/alignment policy;
3. add ordered Paint/media layers, clipping and the two-frame fit model;
4. add lifecycle and sampling motion with the fixed transform stack;
diff --git a/spec/screen-overlay.md b/spec/screen-overlay.md
index 158e059b..dc8911fb 100644
--- a/spec/screen-overlay.md
+++ b/spec/screen-overlay.md
@@ -1,7 +1,8 @@
# SVML Screen Overlay Authoring
-Status: design authority for the optional official Screen Overlay package. It is not implemented by
-an author Surface yet and is not a frozen public ABI.
+Status: implemented pre-release contract for the optional official Screen Overlay package. Its
+self-described Surface, eleven typed components and sequential/parallel browser evidence execute;
+it is not yet a frozen public ABI.
## Purpose
@@ -58,10 +59,14 @@ never gains a secret “current frame” input.
The official package may expose several readable author components under one namespace:
```xml
-<screen:Track id="screen-paint">
- <screen:Flash at={story.moment.hit} for="240ms" color="#ffffff" intensity={0.9}/>
- <screen:Vignette during={program} opacity={0.28}/>
- <screen:Grain during={program} amount={0.12} seed={23}/>
+<screen:Track id="screen-paint" canvas={vertical} space={speech.space}>
+ <screen:Flash at={story.moment.hit} map={timing.map} for="240ms" z="90"
+ color="#ffffff" intensity="0.9" attack="2" hold="3" decay="5"/>
+ <screen:Vignette during="program" z="20"
+ center-x="0.5" center-y="0.5" radius-x="0.82" radius-y="0.68"
+ softness="0.3" color="#000000" opacity="0.28"/>
+ <screen:Grain during="program" z="70"
+ amount="0.12" size="3" chroma="monochrome" motion-rate="0.5" seed="23"/>
</screen:Track>
```
@@ -82,19 +87,20 @@ effect.
Every overlay item has four independent author concerns:
```ts
-type ScreenOverlayItemProgram = {
+type ScreenOverlayItemSpec = {
readonly id: string;
readonly content: ScreenOverlayComponent;
- readonly temporal: TemporalBinding;
- readonly presentation: ScreenOverlayPresentation;
- readonly stacking: AbsoluteStackingIntent;
+ readonly projection: TemporalWindowProjection;
+ readonly expansion: OccurrenceExpansion;
+ readonly stackingOrder: number;
};
```
- `content` identifies the actual visual component and its typed parameters;
-- `temporal` uses the shared Selection/Moment/Program projection algebra;
-- `presentation` describes the item's within-window envelope or local motion;
-- `stacking` resolves to an ordinary absolute Present stacking key.
+- component-owned envelope and local-motion parameters remain inside that typed component rather
+ than a universal presentation bag;
+- `projection` and `expansion` use the shared Selection/Moment/Program algebra;
+- `stackingOrder` resolves with stable authored identity to an ordinary absolute Present key.
Canvas geometry is not an author-selected Spatial Frame here. A Screen Overlay intentionally owns
the complete connected CanvasSpace. A partial-frame light, image, card or blur is a normal Media,
@@ -136,7 +142,8 @@ These can normally lower to full-canvas boxes, gradients and local keyframes:
### Owned Surface candidates
-These may be easier or more exact as deterministic alpha-bearing `CompositableSurface` values:
+These may be easier or more exact as deterministic alpha-bearing `CompositableSurface` values, but
+the current official implementations fit exactly in the closed code-free Visual IR:
- `Grain` — amount, grain size, monochrome/color, motion rate and required seed;
- `LightLeak` — colors, angle, softness, travel, intensity and seed when stochastic;
@@ -216,19 +223,19 @@ Retire:
## 9. Lowering and acceptance
-Implementation should follow the shared Temporal and existing Visual Track contracts:
-
-1. add the self-described `@narratage/screen-overlay` author package;
-2. implement separate typed author components rather than one unvalidated parameter bag;
-3. lower simple components to `svml.visual-ir@1` and complex ones to typed owned Surfaces;
-4. use the connected CanvasSpace and ProgramSpace, never Film globals;
-5. emit ordinary absolute-stack Presents;
-6. add deterministic frame tests for every component and envelope;
-7. prove identical pixels across local and parallel frame rendering;
-8. prove multiple overlay Tracks interleave with Text/Caption/Media by stacking key;
-9. add negative tests rejecting any lower-composite, sibling-Track, backdrop-filter or hidden audio
+Implementation follows the shared Temporal and existing Visual Track contracts:
+
+1. **Implemented:** the self-described `@narratage/screen-overlay` author package;
+2. **Implemented:** eleven separate typed author components rather than one unvalidated bag;
+3. **Implemented:** exact code-free `svml.visual-ir@1` lowering; the generic typed owned-Surface
+ escape remains available when a future component cannot be represented exactly;
+4. **Implemented:** connected CanvasSpace and ProgramSpace, never Film globals;
+5. **Implemented:** ordinary absolute-stack Presents;
+6. **Implemented:** deterministic structural/frame tests for every component and envelope;
+7. **Implemented:** identical pixels across one-worker and three-worker real Chromium rendering;
+8. **Implemented:** multiple Overlay Tracks interleave with peer Tracks by stacking key;
+9. **Implemented:** negative tests rejecting any lower-composite, sibling-Track, backdrop-filter or hidden audio
dependency.
The design fails if installing a new overlay requires a Core branch, a Composition family, a
HyperFrames component switch or an implicit snapshot of already composed pixels.
-
diff --git a/spec/spatial-layout.md b/spec/spatial-layout.md
index 9921101b..aae1a285 100644
--- a/spec/spatial-layout.md
+++ b/spec/spatial-layout.md
@@ -436,7 +436,7 @@ Retired:
`IntrinsicExtent`, `ContentFit` and pure fit validation;
2. **Implemented:** split Canvas declaration from Film assembly and connect the same Canvas value to Track layout and
Composition;
-3. migrate one simple Text Point/Area and one simple Media image through shared geometry;
+3. **Implemented:** migrate one simple Text Area and one simple Media still image through shared geometry;
4. add exact browser tests for every fit and alignment law;
5. prove ordered transparent/color/self-blur/alternate-source Media layers;
6. migrate Ranking and Comment Sticker geometry;
diff --git a/spec/track-composition.md b/spec/track-composition.md
index db5bf129..f1fab1f7 100644
--- a/spec/track-composition.md
+++ b/spec/track-composition.md
@@ -65,9 +65,10 @@ the accumulated lower composite.
## AudioTrack
-An AudioTrack owns frame-exact clips, typed Artifacts, playback mapping, gain, fades and an optional
-mix bus. It has no visual stacking key and no special relationship to a Speech Track. Speech audio,
-music, source audio and sound effects use the same contract.
+An AudioTrack owns exact 48 kHz sample-domain clips, typed canonical-WAV Artifacts, source interval,
+loop phase, pitch-preserving playback rate, target interval, gain and sample-exact fades. It has no
+visual stacking key, inert bus label or special relationship to a Speech Track. Speech audio, music,
+source audio and sound effects use the same contract.
## Composition
diff --git a/spec/track-expressiveness.md b/spec/track-expressiveness.md
index 3f39f89a..1d9b3966 100644
--- a/spec/track-expressiveness.md
+++ b/spec/track-expressiveness.md
@@ -156,7 +156,8 @@ executable rather than package-family patches:
`packages/hyperframes/test/browser-visual.test.ts` additionally drives real Hyperframes browser
frames: installed content-addressed open fonts paint Latin, CJK and emoji/symbol glyphs through an ordered fallback
stack; Fine Caption proves multiline wrapping, full glyph Paint, every glyph karaoke mode and joined
-trail Pill line geometry; and a
+trail Pill line geometry; all eleven official Screen Overlay components preserve identical pixels
+under sequential and parallel frame rendering; and a
50% straight-alpha PNG composites over a blue Track at the expected pixel values. This opt-in host
test is run with `SVML_BROWSER_TESTS=1`; it proves the current local renderer path, not every future
hosted Runtime.
@@ -168,6 +169,12 @@ candidate-era package migration, and a production Runtime still needs to bind it
implementation and validate that Surface bytes satisfy the declared metadata. Those remaining
facts must stay generic; they must not be patched with Caption-, Text- or B-roll-specific fields.
+E7 and E8 now have their official package witnesses. `@narratage/audio-track` proves the exact
+48 kHz source/target mapping, overlap law, bounded stretch and one shared local/remote
+`AudioProgramPlan`; `@narratage/screen-overlay` proves only self-contained owned pixels, explicit
+seeds, flat stacking and fail-closed rejection of lower-composite effects. E1–E6 remain governed by
+the complete Text/Media/Deck work and final Surface validation below.
+
## Freeze criteria
Track v1 may be called a stable public waist only when all of the following hold:
diff --git a/tsconfig.json b/tsconfig.json
index 5d9570a6..15228956 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -39,6 +39,9 @@
"@narratage/spatial": [
"packages/spatial/src/index.ts"
],
+ "@narratage/media-track": [
+ "packages/media-track/src/index.ts"
+ ],
"@narratage/speech": [
"packages/speech/src/index.ts"
],
@@ -51,6 +54,12 @@
"@narratage/composition": [
"packages/composition/src/index.ts"
],
+ "@narratage/audio-track": [
+ "packages/audio-track/src/index.ts"
+ ],
+ "@narratage/screen-overlay": [
+ "packages/screen-overlay/src/index.ts"
+ ],
"@narratage/visual-ir": [
"packages/visual-ir/src/index.ts"
],