Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

refactor(speech): make timing frame and sample exact

kashorin281d98fbbace
diff --git a/examples/talking-film-graph-check/svml.packages.lock b/examples/talking-film-graph-check/svml.packages.lock
index e8a3be5b..ddc63227 100644
--- a/examples/talking-film-graph-check/svml.packages.lock
+++ b/examples/talking-film-graph-check/svml.packages.lock
@@ -727,27 +727,27 @@
{
"name": "@narratage/speech-alignment",
"version": "0.0.0-dev",
- "digest": "sha256:91622c5ae381b522c097e63864f745dd41e171c673cb547e252b3fc9041baeb9"
+ "digest": "sha256:fda24a3d99eb883471147a2c3b1c5caa5526fb2721e0943a8146e1e8f3dd4ecc"
},
{
"name": "@narratage/speech-basis",
"version": "0.0.0-dev",
- "digest": "sha256:16a0c00b8b2cd5ce0bec02f9de28ee8bdd231fc148de4ae1ecfdf7d989756e22"
+ "digest": "sha256:5ba1ca2e8f0049c198a85e446c0847221113e9c89aafde7c5a8a0c8be86fdd96"
},
{
"name": "@narratage/speech-evidence",
"version": "0.0.0-dev",
- "digest": "sha256:a57d9b08cb1af8b378598cb7a918630e74613fa4c89c20e7515c72281e315a13"
+ "digest": "sha256:991b1dbcb2136186110bdb12885504f35cce1ade335151f4f0b3e5c552950196"
},
{
"name": "@narratage/speech-spine",
"version": "0.0.0-dev",
- "digest": "sha256:32fb8fc01ba28ea05e513b171751eba214af7821dcb8e8b047041a3ac14a3c3b"
+ "digest": "sha256:49ac56c325a71e794f5feb2f73d2051da9ebf2ee846761ca5542b75bdd6ea1d0"
},
{
"name": "@narratage/speech",
"version": "0.0.0-dev",
- "digest": "sha256:226f2ca5cac5767b530234422b887227defed419a1d9da7fb915e75e81a132c2"
+ "digest": "sha256:079a337dfdb293f25dc7bb87a7644b3520524e165df913d8085c6c527ed25158"
},
{
"name": "@narratage/svs",
@@ -777,7 +777,7 @@
{
"name": "@narratage/whisperx",
"version": "0.0.0-dev",
- "digest": "sha256:7424e9d8c90e9f035a920df63b48d3417564d05f269292d450db114e8fca3a37"
+ "digest": "sha256:65555eac7342e1c538083be0bf627af10ca266c9d9a68221d78c2d88778c91c5"
}
],
"packages": [
@@ -806,8 +806,8 @@
"name": "@narratage/caption@1"
}
],
- "facetsDigest": "sha256:256d177fe44bd1fde50e281ba617d8ef6c3837b1c661bb20c4d40756a7846a44",
- "closureDigest": "sha256:3377127ad1e2a1cdcbcfcceddf5c76c71d272c002c7f571004976ce5c87f01dd"
+ "facetsDigest": "sha256:1fbbf9bcdc39382ec612df3f29765b5d6ceb8598c0df2c7b5d6b8515f73367a3",
+ "closureDigest": "sha256:bd47658f06759f0b747685406e7af2e0176db405879dcab30321e7ed07780030"
},
{
"package": {
@@ -820,8 +820,8 @@
"name": "@narratage/caption-fine@1"
}
],
- "facetsDigest": "sha256:15398ca00f85ebc6692884616799e0656f5e2a62bc55126f6546e952f14bb15b",
- "closureDigest": "sha256:1b9a0a20db096122ba95c22110863ff53be480b9d93b5667d131341c1dc8e205"
+ "facetsDigest": "sha256:73e3f528a76fdc553ba2ee87fe5beb5fb2c91b7d4fcdff4ba9e6963d4740fdf6",
+ "closureDigest": "sha256:e91df235af196fcec367de95e84e4113444ae9bc1f3a6ed74f4906e566a3913e"
},
{
"package": {
@@ -834,8 +834,8 @@
"name": "@narratage/caption-gemini@1"
}
],
- "facetsDigest": "sha256:e8301a03017542e15a7dcc6bbfdfd1188d9ea6b1c4defa133069e056adc84416",
- "closureDigest": "sha256:47b73b5a61bd7130a9783efaa81973d1aaa130675a119d2c8af57f684a9525f0"
+ "facetsDigest": "sha256:894d546ffa3c9e221fc6b686d11bc117d15a250399332e3ae1c1d28a547e0f30",
+ "closureDigest": "sha256:51ab8d6ff0af88112c74b82c7b39a02672d1377e03e17e8ce968f27f53d0ce67"
},
{
"package": {
@@ -932,8 +932,8 @@
"name": "@narratage/media-pipeline@1"
}
],
- "facetsDigest": "sha256:1531e4a9513db82ec180df58b47ccd6a40316b6035987712eab2726b61b9bd2a",
- "closureDigest": "sha256:a09586c8ea87a69f113a937087dd5cf5316e40064aea29d8ad5ccd97fa956b27"
+ "facetsDigest": "sha256:a96ae88d7813e660bb496b77e53f66ed7e218b20d4e157632c9a4fd832580a92",
+ "closureDigest": "sha256:b91261dded4ffa419b5c8bcc4cc1f780ab1b5944b66d3b8122f499b6da50ed98"
},
{
"package": {
@@ -946,8 +946,8 @@
"name": "@narratage/media-track@1"
}
],
- "facetsDigest": "sha256:a5ae59fa1ce7d2b84ce6b0a831799e00b75df536f6139aa08d4be78697d2e926",
- "closureDigest": "sha256:466cc45fc00fc7f4a752c2b584c5b747f81411019344bab76fa1b62f13a0de9c"
+ "facetsDigest": "sha256:c9d43bfabee11db7c1ad5f65286ef568ff51c09def5f5f05b4510fca36b0336f",
+ "closureDigest": "sha256:9780098ed41540543dfa7f70c3d85b38ec83fcd36eecbafd2d9b6709b7aa3e08"
},
{
"package": {
@@ -988,8 +988,8 @@
"name": "@narratage/render-hyperframes@1"
}
],
- "facetsDigest": "sha256:147c65c507be2e9fa260f34f9a86cd30a406549f635b7b0c209b154921c9c0eb",
- "closureDigest": "sha256:d61b84120b467e4011965625bdfdd113fe153575bbe03fdccdd2e6302c05a810"
+ "facetsDigest": "sha256:31b63a4bf10385d31feb50114811c5185b78a0caab6b7c1646c86a61c4958128",
+ "closureDigest": "sha256:a652fd059b0a9efd053f9f6bd2294c84ee33b9a9d3c3c395e9a6f682dc181d3e"
},
{
"package": {
@@ -1030,8 +1030,8 @@
"name": "@narratage/seedance@1"
}
],
- "facetsDigest": "sha256:943d0307a0d1141d326bedea224b7916051fedb79368ebe98304596237269e6d",
- "closureDigest": "sha256:4c92223092aa84575561ccf0eb69634232f0f7b50265ac085df3e50e7cbc8163"
+ "facetsDigest": "sha256:41615659f664d3480af5a50c81df188b6fc848628c60bc82d37eb7735071f111",
+ "closureDigest": "sha256:93b6f9ec6494f01f0074fb4bcaa261a5404f46019f941aca03039723ff0bca8b"
},
{
"package": {
@@ -1044,8 +1044,8 @@
"name": "@narratage/semantic-map@1"
}
],
- "facetsDigest": "sha256:798b50ac57d8f37718691422b81700e475e72e3d613a39f129474ae3772c2246",
- "closureDigest": "sha256:c790b3f2d1fe0f9e596a3d135d0cbdae00327959bb144d2c872364f019c975a5"
+ "facetsDigest": "sha256:80cad17557218021f4912919b0fc5bbfda61523fc40242e12dafd92a5ca52d28",
+ "closureDigest": "sha256:162ab0d26ba8577910bea38e44f39909bfd275033175e8c6892a21f8a298f8b0"
},
{
"package": {
@@ -1072,8 +1072,8 @@
"name": "@narratage/speech@1"
}
],
- "facetsDigest": "sha256:8d53b60008fcbda0a22280dc2b2bd5d1f23806c74e3df97dc88f4460fb423dcd",
- "closureDigest": "sha256:7e91611e8eef6bca0f798857b72b417e11159021c238e3e80acd7e31f4fec683"
+ "facetsDigest": "sha256:e86e5c90307ab183d2ea5d75e4ca0daab4f2b65905feb79c387871370d625e3a",
+ "closureDigest": "sha256:c41ea21f64a1d8d7cd3ceaada24c3c9ce2070384f36d1f4ee85e76d3957c09bc"
},
{
"package": {
@@ -1086,8 +1086,8 @@
"name": "@narratage/speech-alignment@1"
}
],
- "facetsDigest": "sha256:10194ffd5b8510c4fa0888dc8c568f98ce8de6c7826c5380b6e4990ad1161d28",
- "closureDigest": "sha256:5ad47f53c5ed538281b085436b0b0d1d5ac1527b5a5352ebcc1e71e90a875ffd"
+ "facetsDigest": "sha256:b55e20b4fdf24cbfaf956447ff62885e81638fdbcbb61f25483392a1c8fac8c9",
+ "closureDigest": "sha256:1b98679bc473bd0cd88ebfe9be0b181b2a7526038b687a14b4b316e55fab3cf0"
},
{
"package": {
@@ -1100,8 +1100,8 @@
"name": "@narratage/speech-basis@1"
}
],
- "facetsDigest": "sha256:19200ba7db7913ac23eeded9cd3fcd902d2f7af300fd8b8cb024c9b65783e151",
- "closureDigest": "sha256:51486c0da4f5691f9d5bfb04b4b814f8a5408206f88205f48ed8b195c8619d41"
+ "facetsDigest": "sha256:3e7413acb9b39071fad0ea2e99777ae9ab7a4b636041a8345ec1d08428859b3f",
+ "closureDigest": "sha256:3f38fd95d5d56cef65cec81d54fb80ae5945c95ca954c0d431c57b6636761b66"
},
{
"package": {
@@ -1114,8 +1114,8 @@
"name": "@narratage/speech-evidence@1"
}
],
- "facetsDigest": "sha256:6fcecb7fe6c7c20a7bdcfb3e0777d272f2756558bc65293cdfb742c7755fb021",
- "closureDigest": "sha256:8bd40d80bd26b9a8b5274db2915a9e62ef0178d5f4354b3564fa760ccbbccdb6"
+ "facetsDigest": "sha256:26f3ab63067b382373a9c4c24569bd672302370f5d0c64a055d61862959fc4d0",
+ "closureDigest": "sha256:0df8b3aa5a3c29300be9779b7f6f53fc44051665f956b20d554551d4100272f6"
},
{
"package": {
@@ -1128,8 +1128,8 @@
"name": "@narratage/speech-spine@1"
}
],
- "facetsDigest": "sha256:14dc318c2d3116235252969c752619710d7d65d84c705202b8bbdaa1724065f2",
- "closureDigest": "sha256:d662c9a496405c9254742c0b9665300b44a45272399b21bc6cd9ab14a6e93efb"
+ "facetsDigest": "sha256:340cf867483022b3896008781f24a04e22aa0abc76873cc44787aefdc0219522",
+ "closureDigest": "sha256:4893913844afa519f405281dd0d44e27f1dcfb8ef1bfc7f111bb00f8df9818d8"
},
{
"package": {
@@ -1161,7 +1161,7 @@
}
],
"facetsDigest": "sha256:f3cf535add0b1746dc5beb7920935bc36cca953ee044961d35c5c89fe6b7a746",
- "closureDigest": "sha256:8026318f56465ec8284b54f23bfbc37e0d87982099856dbc314a908538230758"
+ "closureDigest": "sha256:b761c6395b3867412c3bf52ebefae1b106aa00f2b546cb7aafacce17801ede81"
},
{
"package": {
@@ -1192,8 +1192,8 @@
"name": "@narratage/typography-track@1"
}
],
- "facetsDigest": "sha256:66d1f902879637700fa2faef027348da2944021b228f156dc058ff405abf7e28",
- "closureDigest": "sha256:7c1b15eadf6b2fdd5977b400f0c9d0654c3a9e8216e573c0733521669907c5b4"
+ "facetsDigest": "sha256:063c2641007f401709acfed6eab95ae9c566de1e0987a06c7cdce305abe33aa7",
+ "closureDigest": "sha256:7af21c1fb421fd1f3af42d14750524d608e7ca51b33fe74ca364b89b45331b8c"
},
{
"package": {
@@ -1220,9 +1220,9 @@
"name": "@narratage/whisperx@1"
}
],
- "facetsDigest": "sha256:6b656ef3121f69dc35ebf996ebf853d953233dd8eb61c5d7dcc0122ea5d8ad70",
- "closureDigest": "sha256:0bd7c18ea5edcdedd4b34699354f1bb32e99cdf2f829838552d8a788bfedd469"
+ "facetsDigest": "sha256:64c0e76dea02b19be28bc7425e11938fd9c1dbb9994dde3a6d98c41da819b85e",
+ "closureDigest": "sha256:037b8152a0fc865ccf121e25ad4a92c3a3497af1d340f532b49e6b446071f57d"
}
],
- "digest": "sha256:775eba5aa84aa7327b28cdb055a26fe37f75b21d342c2a37b11cc8b03c06a1bb"
+ "digest": "sha256:89f3081e32c8746f51a071e25da016a12dc73b7a53be611bffe7c21e9ed9cb82"
}
diff --git a/examples/talking-film-live/svml.packages.lock b/examples/talking-film-live/svml.packages.lock
index 7aeb8a63..7ff98bfc 100644
--- a/examples/talking-film-live/svml.packages.lock
+++ b/examples/talking-film-live/svml.packages.lock
@@ -718,27 +718,27 @@
{
"name": "@narratage/speech-alignment",
"version": "0.0.0-dev",
- "digest": "sha256:91622c5ae381b522c097e63864f745dd41e171c673cb547e252b3fc9041baeb9"
+ "digest": "sha256:fda24a3d99eb883471147a2c3b1c5caa5526fb2721e0943a8146e1e8f3dd4ecc"
},
{
"name": "@narratage/speech-basis",
"version": "0.0.0-dev",
- "digest": "sha256:16a0c00b8b2cd5ce0bec02f9de28ee8bdd231fc148de4ae1ecfdf7d989756e22"
+ "digest": "sha256:5ba1ca2e8f0049c198a85e446c0847221113e9c89aafde7c5a8a0c8be86fdd96"
},
{
"name": "@narratage/speech-evidence",
"version": "0.0.0-dev",
- "digest": "sha256:a57d9b08cb1af8b378598cb7a918630e74613fa4c89c20e7515c72281e315a13"
+ "digest": "sha256:991b1dbcb2136186110bdb12885504f35cce1ade335151f4f0b3e5c552950196"
},
{
"name": "@narratage/speech-spine",
"version": "0.0.0-dev",
- "digest": "sha256:32fb8fc01ba28ea05e513b171751eba214af7821dcb8e8b047041a3ac14a3c3b"
+ "digest": "sha256:49ac56c325a71e794f5feb2f73d2051da9ebf2ee846761ca5542b75bdd6ea1d0"
},
{
"name": "@narratage/speech",
"version": "0.0.0-dev",
- "digest": "sha256:226f2ca5cac5767b530234422b887227defed419a1d9da7fb915e75e81a132c2"
+ "digest": "sha256:079a337dfdb293f25dc7bb87a7644b3520524e165df913d8085c6c527ed25158"
},
{
"name": "@narratage/svs",
@@ -768,7 +768,7 @@
{
"name": "@narratage/whisperx",
"version": "0.0.0-dev",
- "digest": "sha256:7424e9d8c90e9f035a920df63b48d3417564d05f269292d450db114e8fca3a37"
+ "digest": "sha256:65555eac7342e1c538083be0bf627af10ca266c9d9a68221d78c2d88778c91c5"
},
{
"name": "cmu-pronouncing-dictionary",
@@ -802,8 +802,8 @@
"name": "@narratage/caption@1"
}
],
- "facetsDigest": "sha256:256d177fe44bd1fde50e281ba617d8ef6c3837b1c661bb20c4d40756a7846a44",
- "closureDigest": "sha256:3377127ad1e2a1cdcbcfcceddf5c76c71d272c002c7f571004976ce5c87f01dd"
+ "facetsDigest": "sha256:1fbbf9bcdc39382ec612df3f29765b5d6ceb8598c0df2c7b5d6b8515f73367a3",
+ "closureDigest": "sha256:bd47658f06759f0b747685406e7af2e0176db405879dcab30321e7ed07780030"
},
{
"package": {
@@ -816,8 +816,8 @@
"name": "@narratage/caption-fine@1"
}
],
- "facetsDigest": "sha256:15398ca00f85ebc6692884616799e0656f5e2a62bc55126f6546e952f14bb15b",
- "closureDigest": "sha256:1b9a0a20db096122ba95c22110863ff53be480b9d93b5667d131341c1dc8e205"
+ "facetsDigest": "sha256:73e3f528a76fdc553ba2ee87fe5beb5fb2c91b7d4fcdff4ba9e6963d4740fdf6",
+ "closureDigest": "sha256:e91df235af196fcec367de95e84e4113444ae9bc1f3a6ed74f4906e566a3913e"
},
{
"package": {
@@ -830,8 +830,8 @@
"name": "@narratage/caption-gemini@1"
}
],
- "facetsDigest": "sha256:e8301a03017542e15a7dcc6bbfdfd1188d9ea6b1c4defa133069e056adc84416",
- "closureDigest": "sha256:47b73b5a61bd7130a9783efaa81973d1aaa130675a119d2c8af57f684a9525f0"
+ "facetsDigest": "sha256:894d546ffa3c9e221fc6b686d11bc117d15a250399332e3ae1c1d28a547e0f30",
+ "closureDigest": "sha256:51ab8d6ff0af88112c74b82c7b39a02672d1377e03e17e8ce968f27f53d0ce67"
},
{
"package": {
@@ -858,8 +858,8 @@
"name": "@narratage/estimate@1"
}
],
- "facetsDigest": "sha256:8faa63f635c9cd6d398f925616cdc979939f39eb72b3a8bbfa996a42c72163ba",
- "closureDigest": "sha256:d855f302d8a981eb5f50dcd9e1941d3748af21d8c174e7a166220f760fe8d8ef"
+ "facetsDigest": "sha256:c22a49bd60d2e8061f9cd57c57869a6bcdcfb1a39886ad68f5f4f62a8ddc663a",
+ "closureDigest": "sha256:fd09d4488aeea804582b7d995f4031e820110a7b7c73ece2184311e233d4832e"
},
{
"package": {
@@ -942,8 +942,8 @@
"name": "@narratage/media-pipeline@1"
}
],
- "facetsDigest": "sha256:1531e4a9513db82ec180df58b47ccd6a40316b6035987712eab2726b61b9bd2a",
- "closureDigest": "sha256:a09586c8ea87a69f113a937087dd5cf5316e40064aea29d8ad5ccd97fa956b27"
+ "facetsDigest": "sha256:a96ae88d7813e660bb496b77e53f66ed7e218b20d4e157632c9a4fd832580a92",
+ "closureDigest": "sha256:b91261dded4ffa419b5c8bcc4cc1f780ab1b5944b66d3b8122f499b6da50ed98"
},
{
"package": {
@@ -984,8 +984,8 @@
"name": "@narratage/render-hyperframes@1"
}
],
- "facetsDigest": "sha256:147c65c507be2e9fa260f34f9a86cd30a406549f635b7b0c209b154921c9c0eb",
- "closureDigest": "sha256:d61b84120b467e4011965625bdfdd113fe153575bbe03fdccdd2e6302c05a810"
+ "facetsDigest": "sha256:31b63a4bf10385d31feb50114811c5185b78a0caab6b7c1646c86a61c4958128",
+ "closureDigest": "sha256:a652fd059b0a9efd053f9f6bd2294c84ee33b9a9d3c3c395e9a6f682dc181d3e"
},
{
"package": {
@@ -1026,8 +1026,8 @@
"name": "@narratage/seedance@1"
}
],
- "facetsDigest": "sha256:943d0307a0d1141d326bedea224b7916051fedb79368ebe98304596237269e6d",
- "closureDigest": "sha256:4c92223092aa84575561ccf0eb69634232f0f7b50265ac085df3e50e7cbc8163"
+ "facetsDigest": "sha256:41615659f664d3480af5a50c81df188b6fc848628c60bc82d37eb7735071f111",
+ "closureDigest": "sha256:93b6f9ec6494f01f0074fb4bcaa261a5404f46019f941aca03039723ff0bca8b"
},
{
"package": {
@@ -1040,8 +1040,8 @@
"name": "@narratage/semantic-map@1"
}
],
- "facetsDigest": "sha256:798b50ac57d8f37718691422b81700e475e72e3d613a39f129474ae3772c2246",
- "closureDigest": "sha256:c790b3f2d1fe0f9e596a3d135d0cbdae00327959bb144d2c872364f019c975a5"
+ "facetsDigest": "sha256:80cad17557218021f4912919b0fc5bbfda61523fc40242e12dafd92a5ca52d28",
+ "closureDigest": "sha256:162ab0d26ba8577910bea38e44f39909bfd275033175e8c6892a21f8a298f8b0"
},
{
"package": {
@@ -1068,8 +1068,8 @@
"name": "@narratage/speech@1"
}
],
- "facetsDigest": "sha256:8d53b60008fcbda0a22280dc2b2bd5d1f23806c74e3df97dc88f4460fb423dcd",
- "closureDigest": "sha256:7e91611e8eef6bca0f798857b72b417e11159021c238e3e80acd7e31f4fec683"
+ "facetsDigest": "sha256:e86e5c90307ab183d2ea5d75e4ca0daab4f2b65905feb79c387871370d625e3a",
+ "closureDigest": "sha256:c41ea21f64a1d8d7cd3ceaada24c3c9ce2070384f36d1f4ee85e76d3957c09bc"
},
{
"package": {
@@ -1082,8 +1082,8 @@
"name": "@narratage/speech-alignment@1"
}
],
- "facetsDigest": "sha256:10194ffd5b8510c4fa0888dc8c568f98ce8de6c7826c5380b6e4990ad1161d28",
- "closureDigest": "sha256:5ad47f53c5ed538281b085436b0b0d1d5ac1527b5a5352ebcc1e71e90a875ffd"
+ "facetsDigest": "sha256:b55e20b4fdf24cbfaf956447ff62885e81638fdbcbb61f25483392a1c8fac8c9",
+ "closureDigest": "sha256:1b98679bc473bd0cd88ebfe9be0b181b2a7526038b687a14b4b316e55fab3cf0"
},
{
"package": {
@@ -1096,8 +1096,8 @@
"name": "@narratage/speech-basis@1"
}
],
- "facetsDigest": "sha256:19200ba7db7913ac23eeded9cd3fcd902d2f7af300fd8b8cb024c9b65783e151",
- "closureDigest": "sha256:51486c0da4f5691f9d5bfb04b4b814f8a5408206f88205f48ed8b195c8619d41"
+ "facetsDigest": "sha256:3e7413acb9b39071fad0ea2e99777ae9ab7a4b636041a8345ec1d08428859b3f",
+ "closureDigest": "sha256:3f38fd95d5d56cef65cec81d54fb80ae5945c95ca954c0d431c57b6636761b66"
},
{
"package": {
@@ -1110,8 +1110,8 @@
"name": "@narratage/speech-evidence@1"
}
],
- "facetsDigest": "sha256:6fcecb7fe6c7c20a7bdcfb3e0777d272f2756558bc65293cdfb742c7755fb021",
- "closureDigest": "sha256:8bd40d80bd26b9a8b5274db2915a9e62ef0178d5f4354b3564fa760ccbbccdb6"
+ "facetsDigest": "sha256:26f3ab63067b382373a9c4c24569bd672302370f5d0c64a055d61862959fc4d0",
+ "closureDigest": "sha256:0df8b3aa5a3c29300be9779b7f6f53fc44051665f956b20d554551d4100272f6"
},
{
"package": {
@@ -1124,8 +1124,8 @@
"name": "@narratage/speech-spine@1"
}
],
- "facetsDigest": "sha256:14dc318c2d3116235252969c752619710d7d65d84c705202b8bbdaa1724065f2",
- "closureDigest": "sha256:d662c9a496405c9254742c0b9665300b44a45272399b21bc6cd9ab14a6e93efb"
+ "facetsDigest": "sha256:340cf867483022b3896008781f24a04e22aa0abc76873cc44787aefdc0219522",
+ "closureDigest": "sha256:4893913844afa519f405281dd0d44e27f1dcfb8ef1bfc7f111bb00f8df9818d8"
},
{
"package": {
@@ -1174,8 +1174,8 @@
"name": "@narratage/typography-track@1"
}
],
- "facetsDigest": "sha256:66d1f902879637700fa2faef027348da2944021b228f156dc058ff405abf7e28",
- "closureDigest": "sha256:7c1b15eadf6b2fdd5977b400f0c9d0654c3a9e8216e573c0733521669907c5b4"
+ "facetsDigest": "sha256:063c2641007f401709acfed6eab95ae9c566de1e0987a06c7cdce305abe33aa7",
+ "closureDigest": "sha256:7af21c1fb421fd1f3af42d14750524d608e7ca51b33fe74ca364b89b45331b8c"
},
{
"package": {
@@ -1202,9 +1202,9 @@
"name": "@narratage/whisperx@1"
}
],
- "facetsDigest": "sha256:6b656ef3121f69dc35ebf996ebf853d953233dd8eb61c5d7dcc0122ea5d8ad70",
- "closureDigest": "sha256:0bd7c18ea5edcdedd4b34699354f1bb32e99cdf2f829838552d8a788bfedd469"
+ "facetsDigest": "sha256:64c0e76dea02b19be28bc7425e11938fd9c1dbb9994dde3a6d98c41da819b85e",
+ "closureDigest": "sha256:037b8152a0fc865ccf121e25ad4a92c3a3497af1d340f532b49e6b446071f57d"
}
],
- "digest": "sha256:0046f27c487a972c94606ef7da09f524785799c7cd68c65dbb295d8c74fdcfc4"
+ "digest": "sha256:34f78a21ca92944cf7796ebe9df84611fcf9a186365b14e60d0c7c0316e67ed6"
}
diff --git a/examples/talking-film-live/svml.runtime-packages.lock b/examples/talking-film-live/svml.runtime-packages.lock
index 5e2324e1..9e1437f6 100644
--- a/examples/talking-film-live/svml.runtime-packages.lock
+++ b/examples/talking-film-live/svml.runtime-packages.lock
@@ -180,7 +180,7 @@
{
"name": "@narratage/provider-whisperx-local",
"version": "0.0.0-dev",
- "digest": "sha256:0b1629b756d4cbc067c96b6c147a692510f4f4af3e023dbb4ef7d3bf99b84a86"
+ "digest": "sha256:5b714993ba47541e08d629fdf9f67602653270ecd9ee25b9e9ba0b023c02aa6f"
},
{
"name": "@narratage/render-hyperframes",
@@ -220,17 +220,17 @@
{
"name": "@narratage/speech-alignment",
"version": "0.0.0-dev",
- "digest": "sha256:91622c5ae381b522c097e63864f745dd41e171c673cb547e252b3fc9041baeb9"
+ "digest": "sha256:fda24a3d99eb883471147a2c3b1c5caa5526fb2721e0943a8146e1e8f3dd4ecc"
},
{
"name": "@narratage/speech-evidence",
"version": "0.0.0-dev",
- "digest": "sha256:a57d9b08cb1af8b378598cb7a918630e74613fa4c89c20e7515c72281e315a13"
+ "digest": "sha256:991b1dbcb2136186110bdb12885504f35cce1ade335151f4f0b3e5c552950196"
},
{
"name": "@narratage/speech",
"version": "0.0.0-dev",
- "digest": "sha256:226f2ca5cac5767b530234422b887227defed419a1d9da7fb915e75e81a132c2"
+ "digest": "sha256:079a337dfdb293f25dc7bb87a7644b3520524e165df913d8085c6c527ed25158"
},
{
"name": "@narratage/store-sqlite",
@@ -260,7 +260,7 @@
{
"name": "@narratage/whisperx",
"version": "0.0.0-dev",
- "digest": "sha256:7424e9d8c90e9f035a920df63b48d3417564d05f269292d450db114e8fca3a37"
+ "digest": "sha256:65555eac7342e1c538083be0bf627af10ca266c9d9a68221d78c2d88778c91c5"
},
{
"name": "@narratage/workspace",
@@ -963,7 +963,7 @@
}
],
"facetsDigest": "sha256:485fc29d1efcdaeb4e395980975b35090ea6aef0f78d02108d3bd9b3f11aeb91",
- "closureDigest": "sha256:3eeadfef22668fd042980c74765b60afee1a2dcb56eb4bc03e4fc34f5c07c5f9"
+ "closureDigest": "sha256:ffe3b8fb70b23459f76b73d1b57556cc37b1bda485b81cf28103f54bb05ef880"
},
{
"package": {
@@ -977,7 +977,7 @@
}
],
"facetsDigest": "sha256:e08dde3e4d38b167dca29cdc54eb2bb5e9ebc294e0e42b371099d53d6c44b402",
- "closureDigest": "sha256:e7e5c7e0df2caa6cef9519e24f4354174a7f3fbf088b08dd5ebce9474243ba5c"
+ "closureDigest": "sha256:1e7adb35e311a832b090b382a24ff229c6345332faf0eda17154032646f614f5"
},
{
"package": {
@@ -1005,7 +1005,7 @@
}
],
"facetsDigest": "sha256:998de772acb283828fee16310e77922bf43597bca1efa768e8398166a90af27d",
- "closureDigest": "sha256:fa4f32a69d3939caa21b397fdb6d7648bc15cd2a2d12d04d18c7d75fb2466902"
+ "closureDigest": "sha256:7f07403570eacf6eb80e237b099fd3622bca5a64014114253ee10f7266b30e40"
},
{
"package": {
@@ -1019,7 +1019,7 @@
}
],
"facetsDigest": "sha256:e7d4f1e2fd56f2bcb5dbc36e1faae48d2680bb839aed4a43eb0cb23207ffbe1d",
- "closureDigest": "sha256:23462c0d8308c2c479828edbc1799c74f54859401f708fd56af2af196cc20ecd"
+ "closureDigest": "sha256:c72b47aa38ec0f4686870752212830cb0be230fe855d1e0d96dbb912b32a838f"
},
{
"package": {
@@ -1036,5 +1036,5 @@
"closureDigest": "sha256:c866f01c2c6515940483a42be5808d45741e5622118e215e289460bb1203cf6d"
}
],
- "digest": "sha256:7f15b4a4629ef04902185e341fc99a6dcf61ff350b27ffad6ba9e85bc4b28a86"
+ "digest": "sha256:beced77ff804a0a659ba92c8254a2b39bd47fbf44105b9d735435e342921394f"
}
diff --git a/examples/talking-head-aroll/svml.packages.lock b/examples/talking-head-aroll/svml.packages.lock
index 1b429816..4c67beef 100644
--- a/examples/talking-head-aroll/svml.packages.lock
+++ b/examples/talking-head-aroll/svml.packages.lock
@@ -717,27 +717,27 @@
{
"name": "@narratage/speech-alignment",
"version": "0.0.0-dev",
- "digest": "sha256:91622c5ae381b522c097e63864f745dd41e171c673cb547e252b3fc9041baeb9"
+ "digest": "sha256:fda24a3d99eb883471147a2c3b1c5caa5526fb2721e0943a8146e1e8f3dd4ecc"
},
{
"name": "@narratage/speech-basis",
"version": "0.0.0-dev",
- "digest": "sha256:16a0c00b8b2cd5ce0bec02f9de28ee8bdd231fc148de4ae1ecfdf7d989756e22"
+ "digest": "sha256:5ba1ca2e8f0049c198a85e446c0847221113e9c89aafde7c5a8a0c8be86fdd96"
},
{
"name": "@narratage/speech-evidence",
"version": "0.0.0-dev",
- "digest": "sha256:a57d9b08cb1af8b378598cb7a918630e74613fa4c89c20e7515c72281e315a13"
+ "digest": "sha256:991b1dbcb2136186110bdb12885504f35cce1ade335151f4f0b3e5c552950196"
},
{
"name": "@narratage/speech-spine",
"version": "0.0.0-dev",
- "digest": "sha256:32fb8fc01ba28ea05e513b171751eba214af7821dcb8e8b047041a3ac14a3c3b"
+ "digest": "sha256:49ac56c325a71e794f5feb2f73d2051da9ebf2ee846761ca5542b75bdd6ea1d0"
},
{
"name": "@narratage/speech",
"version": "0.0.0-dev",
- "digest": "sha256:226f2ca5cac5767b530234422b887227defed419a1d9da7fb915e75e81a132c2"
+ "digest": "sha256:079a337dfdb293f25dc7bb87a7644b3520524e165df913d8085c6c527ed25158"
},
{
"name": "@narratage/svs",
@@ -762,7 +762,7 @@
{
"name": "@narratage/whisperx",
"version": "0.0.0-dev",
- "digest": "sha256:7424e9d8c90e9f035a920df63b48d3417564d05f269292d450db114e8fca3a37"
+ "digest": "sha256:65555eac7342e1c538083be0bf627af10ca266c9d9a68221d78c2d88778c91c5"
},
{
"name": "cmu-pronouncing-dictionary",
@@ -796,8 +796,8 @@
"name": "@narratage/caption@1"
}
],
- "facetsDigest": "sha256:256d177fe44bd1fde50e281ba617d8ef6c3837b1c661bb20c4d40756a7846a44",
- "closureDigest": "sha256:3377127ad1e2a1cdcbcfcceddf5c76c71d272c002c7f571004976ce5c87f01dd"
+ "facetsDigest": "sha256:1fbbf9bcdc39382ec612df3f29765b5d6ceb8598c0df2c7b5d6b8515f73367a3",
+ "closureDigest": "sha256:bd47658f06759f0b747685406e7af2e0176db405879dcab30321e7ed07780030"
},
{
"package": {
@@ -810,8 +810,8 @@
"name": "@narratage/caption-fine@1"
}
],
- "facetsDigest": "sha256:15398ca00f85ebc6692884616799e0656f5e2a62bc55126f6546e952f14bb15b",
- "closureDigest": "sha256:1b9a0a20db096122ba95c22110863ff53be480b9d93b5667d131341c1dc8e205"
+ "facetsDigest": "sha256:73e3f528a76fdc553ba2ee87fe5beb5fb2c91b7d4fcdff4ba9e6963d4740fdf6",
+ "closureDigest": "sha256:e91df235af196fcec367de95e84e4113444ae9bc1f3a6ed74f4906e566a3913e"
},
{
"package": {
@@ -824,8 +824,8 @@
"name": "@narratage/caption-gemini@1"
}
],
- "facetsDigest": "sha256:e8301a03017542e15a7dcc6bbfdfd1188d9ea6b1c4defa133069e056adc84416",
- "closureDigest": "sha256:47b73b5a61bd7130a9783efaa81973d1aaa130675a119d2c8af57f684a9525f0"
+ "facetsDigest": "sha256:894d546ffa3c9e221fc6b686d11bc117d15a250399332e3ae1c1d28a547e0f30",
+ "closureDigest": "sha256:51ab8d6ff0af88112c74b82c7b39a02672d1377e03e17e8ce968f27f53d0ce67"
},
{
"package": {
@@ -852,8 +852,8 @@
"name": "@narratage/estimate@1"
}
],
- "facetsDigest": "sha256:8faa63f635c9cd6d398f925616cdc979939f39eb72b3a8bbfa996a42c72163ba",
- "closureDigest": "sha256:d855f302d8a981eb5f50dcd9e1941d3748af21d8c174e7a166220f760fe8d8ef"
+ "facetsDigest": "sha256:c22a49bd60d2e8061f9cd57c57869a6bcdcfb1a39886ad68f5f4f62a8ddc663a",
+ "closureDigest": "sha256:fd09d4488aeea804582b7d995f4031e820110a7b7c73ece2184311e233d4832e"
},
{
"package": {
@@ -936,8 +936,8 @@
"name": "@narratage/media-pipeline@1"
}
],
- "facetsDigest": "sha256:1531e4a9513db82ec180df58b47ccd6a40316b6035987712eab2726b61b9bd2a",
- "closureDigest": "sha256:a09586c8ea87a69f113a937087dd5cf5316e40064aea29d8ad5ccd97fa956b27"
+ "facetsDigest": "sha256:a96ae88d7813e660bb496b77e53f66ed7e218b20d4e157632c9a4fd832580a92",
+ "closureDigest": "sha256:b91261dded4ffa419b5c8bcc4cc1f780ab1b5944b66d3b8122f499b6da50ed98"
},
{
"package": {
@@ -978,8 +978,8 @@
"name": "@narratage/render-hyperframes@1"
}
],
- "facetsDigest": "sha256:147c65c507be2e9fa260f34f9a86cd30a406549f635b7b0c209b154921c9c0eb",
- "closureDigest": "sha256:d61b84120b467e4011965625bdfdd113fe153575bbe03fdccdd2e6302c05a810"
+ "facetsDigest": "sha256:31b63a4bf10385d31feb50114811c5185b78a0caab6b7c1646c86a61c4958128",
+ "closureDigest": "sha256:a652fd059b0a9efd053f9f6bd2294c84ee33b9a9d3c3c395e9a6f682dc181d3e"
},
{
"package": {
@@ -1020,8 +1020,8 @@
"name": "@narratage/seedance@1"
}
],
- "facetsDigest": "sha256:943d0307a0d1141d326bedea224b7916051fedb79368ebe98304596237269e6d",
- "closureDigest": "sha256:4c92223092aa84575561ccf0eb69634232f0f7b50265ac085df3e50e7cbc8163"
+ "facetsDigest": "sha256:41615659f664d3480af5a50c81df188b6fc848628c60bc82d37eb7735071f111",
+ "closureDigest": "sha256:93b6f9ec6494f01f0074fb4bcaa261a5404f46019f941aca03039723ff0bca8b"
},
{
"package": {
@@ -1034,8 +1034,8 @@
"name": "@narratage/semantic-map@1"
}
],
- "facetsDigest": "sha256:798b50ac57d8f37718691422b81700e475e72e3d613a39f129474ae3772c2246",
- "closureDigest": "sha256:c790b3f2d1fe0f9e596a3d135d0cbdae00327959bb144d2c872364f019c975a5"
+ "facetsDigest": "sha256:80cad17557218021f4912919b0fc5bbfda61523fc40242e12dafd92a5ca52d28",
+ "closureDigest": "sha256:162ab0d26ba8577910bea38e44f39909bfd275033175e8c6892a21f8a298f8b0"
},
{
"package": {
@@ -1062,8 +1062,8 @@
"name": "@narratage/speech@1"
}
],
- "facetsDigest": "sha256:8d53b60008fcbda0a22280dc2b2bd5d1f23806c74e3df97dc88f4460fb423dcd",
- "closureDigest": "sha256:7e91611e8eef6bca0f798857b72b417e11159021c238e3e80acd7e31f4fec683"
+ "facetsDigest": "sha256:e86e5c90307ab183d2ea5d75e4ca0daab4f2b65905feb79c387871370d625e3a",
+ "closureDigest": "sha256:c41ea21f64a1d8d7cd3ceaada24c3c9ce2070384f36d1f4ee85e76d3957c09bc"
},
{
"package": {
@@ -1076,8 +1076,8 @@
"name": "@narratage/speech-alignment@1"
}
],
- "facetsDigest": "sha256:10194ffd5b8510c4fa0888dc8c568f98ce8de6c7826c5380b6e4990ad1161d28",
- "closureDigest": "sha256:5ad47f53c5ed538281b085436b0b0d1d5ac1527b5a5352ebcc1e71e90a875ffd"
+ "facetsDigest": "sha256:b55e20b4fdf24cbfaf956447ff62885e81638fdbcbb61f25483392a1c8fac8c9",
+ "closureDigest": "sha256:1b98679bc473bd0cd88ebfe9be0b181b2a7526038b687a14b4b316e55fab3cf0"
},
{
"package": {
@@ -1090,8 +1090,8 @@
"name": "@narratage/speech-basis@1"
}
],
- "facetsDigest": "sha256:19200ba7db7913ac23eeded9cd3fcd902d2f7af300fd8b8cb024c9b65783e151",
- "closureDigest": "sha256:51486c0da4f5691f9d5bfb04b4b814f8a5408206f88205f48ed8b195c8619d41"
+ "facetsDigest": "sha256:3e7413acb9b39071fad0ea2e99777ae9ab7a4b636041a8345ec1d08428859b3f",
+ "closureDigest": "sha256:3f38fd95d5d56cef65cec81d54fb80ae5945c95ca954c0d431c57b6636761b66"
},
{
"package": {
@@ -1104,8 +1104,8 @@
"name": "@narratage/speech-evidence@1"
}
],
- "facetsDigest": "sha256:6fcecb7fe6c7c20a7bdcfb3e0777d272f2756558bc65293cdfb742c7755fb021",
- "closureDigest": "sha256:8bd40d80bd26b9a8b5274db2915a9e62ef0178d5f4354b3564fa760ccbbccdb6"
+ "facetsDigest": "sha256:26f3ab63067b382373a9c4c24569bd672302370f5d0c64a055d61862959fc4d0",
+ "closureDigest": "sha256:0df8b3aa5a3c29300be9779b7f6f53fc44051665f956b20d554551d4100272f6"
},
{
"package": {
@@ -1118,8 +1118,8 @@
"name": "@narratage/speech-spine@1"
}
],
- "facetsDigest": "sha256:14dc318c2d3116235252969c752619710d7d65d84c705202b8bbdaa1724065f2",
- "closureDigest": "sha256:d662c9a496405c9254742c0b9665300b44a45272399b21bc6cd9ab14a6e93efb"
+ "facetsDigest": "sha256:340cf867483022b3896008781f24a04e22aa0abc76873cc44787aefdc0219522",
+ "closureDigest": "sha256:4893913844afa519f405281dd0d44e27f1dcfb8ef1bfc7f111bb00f8df9818d8"
},
{
"package": {
@@ -1182,9 +1182,9 @@
"name": "@narratage/whisperx@1"
}
],
- "facetsDigest": "sha256:6b656ef3121f69dc35ebf996ebf853d953233dd8eb61c5d7dcc0122ea5d8ad70",
- "closureDigest": "sha256:0bd7c18ea5edcdedd4b34699354f1bb32e99cdf2f829838552d8a788bfedd469"
+ "facetsDigest": "sha256:64c0e76dea02b19be28bc7425e11938fd9c1dbb9994dde3a6d98c41da819b85e",
+ "closureDigest": "sha256:037b8152a0fc865ccf121e25ad4a92c3a3497af1d340f532b49e6b446071f57d"
}
],
- "digest": "sha256:fa1813a01a7f8c3d4dd13a429110d31601c1007fdd1e76b7bec41413c5628245"
+ "digest": "sha256:1bdb158371743d9adb6e88c58109f2b89999937bdfea22d7779636ea1d04e435"
}
diff --git a/examples/talking-head-aroll/svml.runtime-packages.lock b/examples/talking-head-aroll/svml.runtime-packages.lock
index 5e2324e1..9e1437f6 100644
--- a/examples/talking-head-aroll/svml.runtime-packages.lock
+++ b/examples/talking-head-aroll/svml.runtime-packages.lock
@@ -180,7 +180,7 @@
{
"name": "@narratage/provider-whisperx-local",
"version": "0.0.0-dev",
- "digest": "sha256:0b1629b756d4cbc067c96b6c147a692510f4f4af3e023dbb4ef7d3bf99b84a86"
+ "digest": "sha256:5b714993ba47541e08d629fdf9f67602653270ecd9ee25b9e9ba0b023c02aa6f"
},
{
"name": "@narratage/render-hyperframes",
@@ -220,17 +220,17 @@
{
"name": "@narratage/speech-alignment",
"version": "0.0.0-dev",
- "digest": "sha256:91622c5ae381b522c097e63864f745dd41e171c673cb547e252b3fc9041baeb9"
+ "digest": "sha256:fda24a3d99eb883471147a2c3b1c5caa5526fb2721e0943a8146e1e8f3dd4ecc"
},
{
"name": "@narratage/speech-evidence",
"version": "0.0.0-dev",
- "digest": "sha256:a57d9b08cb1af8b378598cb7a918630e74613fa4c89c20e7515c72281e315a13"
+ "digest": "sha256:991b1dbcb2136186110bdb12885504f35cce1ade335151f4f0b3e5c552950196"
},
{
"name": "@narratage/speech",
"version": "0.0.0-dev",
- "digest": "sha256:226f2ca5cac5767b530234422b887227defed419a1d9da7fb915e75e81a132c2"
+ "digest": "sha256:079a337dfdb293f25dc7bb87a7644b3520524e165df913d8085c6c527ed25158"
},
{
"name": "@narratage/store-sqlite",
@@ -260,7 +260,7 @@
{
"name": "@narratage/whisperx",
"version": "0.0.0-dev",
- "digest": "sha256:7424e9d8c90e9f035a920df63b48d3417564d05f269292d450db114e8fca3a37"
+ "digest": "sha256:65555eac7342e1c538083be0bf627af10ca266c9d9a68221d78c2d88778c91c5"
},
{
"name": "@narratage/workspace",
@@ -963,7 +963,7 @@
}
],
"facetsDigest": "sha256:485fc29d1efcdaeb4e395980975b35090ea6aef0f78d02108d3bd9b3f11aeb91",
- "closureDigest": "sha256:3eeadfef22668fd042980c74765b60afee1a2dcb56eb4bc03e4fc34f5c07c5f9"
+ "closureDigest": "sha256:ffe3b8fb70b23459f76b73d1b57556cc37b1bda485b81cf28103f54bb05ef880"
},
{
"package": {
@@ -977,7 +977,7 @@
}
],
"facetsDigest": "sha256:e08dde3e4d38b167dca29cdc54eb2bb5e9ebc294e0e42b371099d53d6c44b402",
- "closureDigest": "sha256:e7e5c7e0df2caa6cef9519e24f4354174a7f3fbf088b08dd5ebce9474243ba5c"
+ "closureDigest": "sha256:1e7adb35e311a832b090b382a24ff229c6345332faf0eda17154032646f614f5"
},
{
"package": {
@@ -1005,7 +1005,7 @@
}
],
"facetsDigest": "sha256:998de772acb283828fee16310e77922bf43597bca1efa768e8398166a90af27d",
- "closureDigest": "sha256:fa4f32a69d3939caa21b397fdb6d7648bc15cd2a2d12d04d18c7d75fb2466902"
+ "closureDigest": "sha256:7f07403570eacf6eb80e237b099fd3622bca5a64014114253ee10f7266b30e40"
},
{
"package": {
@@ -1019,7 +1019,7 @@
}
],
"facetsDigest": "sha256:e7d4f1e2fd56f2bcb5dbc36e1faae48d2680bb839aed4a43eb0cb23207ffbe1d",
- "closureDigest": "sha256:23462c0d8308c2c479828edbc1799c74f54859401f708fd56af2af196cc20ecd"
+ "closureDigest": "sha256:c72b47aa38ec0f4686870752212830cb0be230fe855d1e0d96dbb912b32a838f"
},
{
"package": {
@@ -1036,5 +1036,5 @@
"closureDigest": "sha256:c866f01c2c6515940483a42be5808d45741e5622118e215e289460bb1203cf6d"
}
],
- "digest": "sha256:7f15b4a4629ef04902185e341fc99a6dcf61ff350b27ffad6ba9e85bc4b28a86"
+ "digest": "sha256:beced77ff804a0a659ba92c8254a2b39bd47fbf44105b9d735435e342921394f"
}
diff --git a/packages/caption/test/caption.test.ts b/packages/caption/test/caption.test.ts
index 0786aedf..9d5521ef 100644
--- a/packages/caption/test/caption.test.ts
+++ b/packages/caption/test/caption.test.ts
@@ -20,7 +20,6 @@ import { sealSpeechBasis } from "@narratage/speech";
import type { SpeechAudioBasis } from "@narratage/speech";
import { locateSpeechTiming } from "@narratage/speech-alignment";
import { sealAlignedTranscriptEvidence } from "@narratage/speech-evidence";
-import type { AlignedTranscriptSegment } from "@narratage/speech-evidence";
import {
captionCorrespondence,
captionDisplaySequence,
@@ -29,7 +28,17 @@ import {
} from "@narratage/script";
import type { StructuredElement, SurfaceResolvedReference } from "@narratage/markup";
-function locate(narrative: Narrative, durationSec: number, segments: readonly AlignedTranscriptSegment[]) {
+type TranscriptFixture = {
+ readonly words: readonly {
+ readonly text: string;
+ readonly startSec?: number;
+ readonly endSec?: number;
+ readonly score?: number;
+ }[];
+ readonly chars: readonly [];
+};
+
+function locate(narrative: Narrative, durationSec: number, segments: readonly TranscriptFixture[]) {
const space = sealProgramSpace({
durationSec,
frameRate: { numerator: 30, denominator: 1 },
@@ -37,8 +46,8 @@ function locate(narrative: Narrative, durationSec: number, segments: readonly Al
const audio = { kind: "blob" as const, digest: digestOf("caption:test-audio"), size: 1, mediaType: "audio/wav" };
const basisSegments = narrative.segments.map((segment, index) => ({
segmentId: segment.id,
- startSec: durationSec * index / narrative.segments.length,
- endSec: durationSec * (index + 1) / narrative.segments.length,
+ startFrame: Math.round(durationSec * 30 * index / narrative.segments.length),
+ endFrameExclusive: Math.round(durationSec * 30 * (index + 1) / narrative.segments.length),
}));
const basis = sealSpeechBasis({
programSpace: space,
@@ -46,7 +55,19 @@ function locate(narrative: Narrative, durationSec: number, segments: readonly Al
visualTrack: { clips: [] },
segments: basisSegments,
});
- const evidence = sealAlignedTranscriptEvidence({ segments });
+ const evidence = sealAlignedTranscriptEvidence({
+ passages: segments.map((segment) => ({
+ words: segment.words.map((word) => ({
+ text: word.text,
+ ...(word.startSec === undefined || word.endSec === undefined ? {} : {
+ startSample: Math.round(word.startSec * 16_000),
+ endSampleExclusive: Math.round(word.endSec * 16_000),
+ }),
+ ...(word.score === undefined ? {} : { score: word.score }),
+ })),
+ chars: [],
+ })),
+ });
const audioBasis: SpeechAudioBasis = {
programSpace: basis.programSpace,
audio: basis.audio,
@@ -148,7 +169,6 @@ test("Caption timing applies Mute after planning and preserves the original Cue
}],
});
const map = locate(parsed, 2, [{
- sourceSegmentId: "line",
words: ["Keep", "this", "hidden", "phrase", "visible"].map((text, index) => ({
text,
startSec: index * 0.3,
@@ -293,7 +313,6 @@ test("Dual Text exposes one whole timed display Atom and never invents internal
}] }],
});
const map = locate(parsed, 1, [{
- sourceSegmentId: "line",
words: [
{ text: "what", startSec: 0.1, endSec: 0.25 },
{ text: "the", startSec: 0.3, endSec: 0.42 },
diff --git a/packages/provider-whisperx-local/README.md b/packages/provider-whisperx-local/README.md
index 938a3a42..27e7201e 100644
--- a/packages/provider-whisperx-local/README.md
+++ b/packages/provider-whisperx-local/README.md
@@ -7,6 +7,10 @@ The Provider accepts only canonical 16 kHz mono PCM s16 WAV bytes produced by th
projection Need. It validates those bytes and stages them unchanged. WhisperX is therefore never
allowed to hide a second ffmpeg conversion or change the speech-master clock.
+WhisperX's wire response uses floating-point seconds. This adapter converts those boundaries once
+to integer positions in the 16 kHz evidence-audio sample domain. It preserves WhisperX acoustic
+passages but never receives or assigns authored Script Segment identities.
+
```ts
createLocalWhisperXProvider({
baseUrl: "http://127.0.0.1:8765",
diff --git a/packages/provider-whisperx-local/package.json b/packages/provider-whisperx-local/package.json
index cb4cbd76..a2132fe3 100644
--- a/packages/provider-whisperx-local/package.json
+++ b/packages/provider-whisperx-local/package.json
@@ -15,7 +15,6 @@
"@narratage/endpoint-kit": "workspace:*",
"@narratage/protocol": "workspace:*",
"@narratage/runtime-adapter": "workspace:*",
- "@narratage/speech": "workspace:*",
"@narratage/speech-evidence": "workspace:*",
"@narratage/whisperx": "workspace:*"
},
diff --git a/packages/provider-whisperx-local/src/provider.ts b/packages/provider-whisperx-local/src/provider.ts
index 280cb405..b897107b 100644
--- a/packages/provider-whisperx-local/src/provider.ts
+++ b/packages/provider-whisperx-local/src/provider.ts
@@ -1,9 +1,8 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import type { SpeechBasisSegment } from "@narratage/speech";
import { sealAlignedTranscriptEvidence, speechEvidenceTypes } from "@narratage/speech-evidence";
-import type { AlignedTranscriptEvidence, AlignedTranscriptSegment } from "@narratage/speech-evidence";
+import type { AlignedTranscriptEvidence, SpeechTranscriptPassage } from "@narratage/speech-evidence";
import type { EndpointInvocationContext, EndpointFulfillment } from "@narratage/endpoint-kit";
import { canonicalize, digestOf } from "@narratage/protocol";
import type { CanonicalValue } from "@narratage/protocol";
@@ -79,8 +78,7 @@ function alignmentRequest(value: CanonicalValue): WhisperXAlignmentRequest {
assert(item.audio?.kind === "blob"
&& item.audio.mediaType === "audio/wav"
&& Number.isSafeInteger(item.sampleFrames)
- && item.sampleFrames > 0
- && item.segments.length > 0,
+ && item.sampleFrames > 0,
"WhisperX alignment request is invalid");
return item;
}
@@ -121,47 +119,33 @@ function assertCanonicalEvidenceWav(bytes: Uint8Array, sampleFrames: number): vo
assert(dataBytes === sampleFrames * 2, "WhisperX evidence sample count differs from its contract");
}
-function sourceSegmentFor(
- segments: readonly SpeechBasisSegment[],
- start: number | undefined,
- end: number | undefined,
- fallbackStart: number | undefined,
- fallbackEnd: number | undefined,
-): SpeechBasisSegment {
- const point = start !== undefined && end !== undefined
- ? (start + end) / 2
- : fallbackStart !== undefined && fallbackEnd !== undefined ? (fallbackStart + fallbackEnd) / 2 : 0;
- return segments.find((segment, index) => point >= segment.startSec
- && (point < segment.endSec || index === segments.length - 1)) ?? segments[0]!;
+function sampleWindow(
+ startSec: unknown,
+ endSec: unknown,
+ sampleFrames: number,
+): { readonly startSample: number; readonly endSampleExclusive: number } | undefined {
+ if (!finite(startSec) || !finite(endSec) || startSec < 0 || endSec < startSec) return undefined;
+ const startSample = Math.round(startSec * 16_000);
+ const endSampleExclusive = Math.round(endSec * 16_000);
+ if (!Number.isSafeInteger(startSample) || !Number.isSafeInteger(endSampleExclusive)
+ || startSample > sampleFrames || endSampleExclusive > sampleFrames) return undefined;
+ return { startSample, endSampleExclusive };
}
-/**
- * Lower service-specific pauses/segments onto authored structural Segments without inventing time.
- * A word crossing a structural cut keeps its lexical evidence but loses its per-word time; the
- * downstream locator may derive that uncertainty explicitly instead of receiving a clipped lie.
- */
+/** Lower WhisperX's wire-level seconds once into exact 16 kHz evidence-sample boundaries. */
export function interpretWhisperXResponse(
response: WhisperXServiceResponse,
- sourceSegments: readonly SpeechBasisSegment[],
- durationSec: number,
-): readonly AlignedTranscriptSegment[] {
- assert(sourceSegments.length > 0, "WhisperX request has no source Segment");
+ sampleFrames: number,
+): readonly SpeechTranscriptPassage[] {
+ positiveInteger(sampleFrames, "WhisperX evidence sample count");
assert(Array.isArray(response.segments), "WhisperX response has no Segment array");
- const buckets = new Map(sourceSegments.map((segment) => [segment.segmentId, [] as Array<{
- readonly text: string;
- readonly startSec?: number;
- readonly endSec?: number;
- readonly score?: number;
- }>]));
- const previousEnd = new Map(sourceSegments.map((segment) => [segment.segmentId, segment.startSec]));
-
- for (const rawSegmentValue of response.segments) {
+ return response.segments.map((rawSegmentValue): SpeechTranscriptPassage => {
assert(rawSegmentValue !== null && typeof rawSegmentValue === "object" && !Array.isArray(rawSegmentValue),
"WhisperX response Segment is invalid");
const rawSegment = rawSegmentValue as RawSegment;
- const segmentStart = finite(rawSegment.start) ? rawSegment.start : undefined;
- const segmentEnd = finite(rawSegment.end) ? rawSegment.end : undefined;
+ const passageWindow = sampleWindow(rawSegment.start, rawSegment.end, sampleFrames);
assert(Array.isArray(rawSegment.words), "WhisperX response Segment has no Word array");
+ const words: Array<SpeechTranscriptPassage["words"][number]> = [];
for (const rawWordValue of rawSegment.words) {
assert(rawWordValue !== null && typeof rawWordValue === "object" && !Array.isArray(rawWordValue),
"WhisperX response Word is invalid");
@@ -170,36 +154,22 @@ export function interpretWhisperXResponse(
? rawWord.text.trim()
: typeof rawWord.word === "string" ? rawWord.word.trim() : "";
if (text.length === 0) continue;
- const start = finite(rawWord.start) ? rawWord.start : undefined;
- const end = finite(rawWord.end) ? rawWord.end : undefined;
- const source = sourceSegmentFor(sourceSegments, start, end, segmentStart, segmentEnd);
- const prior = previousEnd.get(source.segmentId)!;
- // A word whose timing cannot be proved keeps its text and loses its clock,
- // exactly as the service already does for words its aligner could not place.
- const timingIsProvable = start !== undefined && end !== undefined
- && start >= 0
- && end >= start
- && end <= durationSec + 1e-3
- && start >= source.startSec - 1e-6
- && end <= source.endSec + 1e-6
- && start >= prior - 1e-6;
+ const wordWindow = sampleWindow(rawWord.start, rawWord.end, sampleFrames);
const score = finite(rawWord.score) && rawWord.score >= 0 && rawWord.score <= 1
? rawWord.score
: undefined;
- buckets.get(source.segmentId)!.push({
+ words.push({
text,
- ...(timingIsProvable ? { startSec: start, endSec: end } : {}),
+ ...wordWindow,
...(score === undefined ? {} : { score }),
});
- if (timingIsProvable) previousEnd.set(source.segmentId, end);
}
- }
-
- return sourceSegments.map((segment) => ({
- sourceSegmentId: segment.segmentId,
- words: buckets.get(segment.segmentId)!,
- chars: [],
- }));
+ return {
+ ...passageWindow,
+ words,
+ chars: [],
+ };
+ });
}
async function limitedJson(response: Response, maxBytes: number, subject: string): Promise<{
@@ -341,9 +311,9 @@ export function createLocalWhisperXProvider(config: CreateLocalWhisperXProviderO
assert(raw.value !== null && typeof raw.value === "object" && !Array.isArray(raw.value),
"WhisperX transcription response is invalid");
const response = raw.value as WhisperXServiceResponse;
- const segments = interpretWhisperXResponse(response, request.segments, request.sampleFrames / 16_000);
+ const passages = interpretWhisperXResponse(response, request.sampleFrames);
const evidence: AlignedTranscriptEvidence = sealAlignedTranscriptEvidence({
- segments,
+ passages,
});
return result(canonicalize(evidence));
} finally {
diff --git a/packages/provider-whisperx-local/test/provider.test.ts b/packages/provider-whisperx-local/test/provider.test.ts
index 564b97ce..bb817261 100644
--- a/packages/provider-whisperx-local/test/provider.test.ts
+++ b/packages/provider-whisperx-local/test/provider.test.ts
@@ -2,7 +2,6 @@ import { createServer } from "node:http";
import { readFile } from "node:fs/promises";
import test from "node:test";
import { sealSpeechEvidenceAudio, speechTypes } from "@narratage/speech";
-import type { SpeechAudioBasis, SpeechEvidenceAudio } from "@narratage/speech";
import assert from "node:assert/strict";
import { MemoryArtifactStore, EndpointRegistry } from "@narratage/driver-node";
import { digestOf } from "@narratage/protocol";
@@ -18,10 +17,6 @@ import {
interpretWhisperXResponse,
} from "../src/index.js";
-const sourceSegments = [
- { segmentId: "opening", startSec: 0, endSec: 1 },
- { segmentId: "answer", startSec: 1, endSec: 2 },
-];
function wav(sampleFrames: number): Uint8Array {
const bytes = new Uint8Array(44 + sampleFrames * 2);
const view = new DataView(bytes.buffer);
@@ -57,7 +52,7 @@ test("local WhisperX Provider pins the complete service runtime and is independe
);
});
-test("service pauses are projected onto authored Segments without clipping a crossing word into fake evidence", () => {
+test("wire seconds are lowered once to exact evidence samples without authored Segment knowledge", () => {
const evidence = interpretWhisperXResponse({
language: "en",
segments: [{
@@ -69,11 +64,17 @@ test("service pauses are projected onto authored Segments without clipping a cro
{ text: "world", start: 1.2, end: 1.6 },
],
}],
- }, sourceSegments, 2);
- assert.deepEqual(evidence.map((segment) => segment.sourceSegmentId), ["opening", "answer"]);
- assert.deepEqual(evidence[0]!.words[0], { text: "hello", startSec: 0.1, endSec: 0.4, score: 0.99 });
- assert.deepEqual(evidence[1]!.words[0], { text: "crossing", score: 0.8 });
- assert.deepEqual(evidence[1]!.words[1], { text: "world", startSec: 1.2, endSec: 1.6 });
+ }, 32_000);
+ assert.equal(evidence.length, 1);
+ assert.deepEqual(
+ { startSample: evidence[0]!.startSample, endSampleExclusive: evidence[0]!.endSampleExclusive },
+ { startSample: 1_600, endSampleExclusive: 28_800 },
+ );
+ assert.deepEqual(evidence[0]!.words, [
+ { text: "hello", startSample: 1_600, endSampleExclusive: 6_400, score: 0.99 },
+ { text: "crossing", startSample: 14_400, endSampleExclusive: 17_600, score: 0.8 },
+ { text: "world", startSample: 19_200, endSampleExclusive: 25_600 },
+ ]);
});
test("local Provider stages canonical evidence bytes unchanged and returns sealed alignment evidence", async () => {
@@ -127,15 +128,7 @@ test("local Provider stages canonical evidence bytes unchanged and returns seale
artifact,
sampleFrames: 32_000,
});
- const audioBasis: SpeechAudioBasis = {
- programSpace: {
- durationSec: 2,
- frameRate: { numerator: 30, denominator: 1 },
- },
- audio: artifact,
- segments: sourceSegments,
- };
- const constraints = whisperXRequestForEvidenceAudio(evidenceAudio, audioBasis, { language: "en" });
+ const constraints = whisperXRequestForEvidenceAudio(evidenceAudio, { language: "en" });
const need: Need = {
id: "need:whisperx-loopback",
capability: whisperXCapabilities.alignment,
@@ -167,7 +160,7 @@ test("local Provider stages canonical evidence bytes unchanged and returns seale
assert.equal(stagedMatches, true);
assert.equal(output.value.kind, "inline");
const value = output.value.kind === "inline" ? output.value.value : null;
- assert.equal((value as { readonly segments?: readonly unknown[] }).segments?.length, 2);
+ assert.equal((value as { readonly passages?: readonly unknown[] }).passages?.length, 1);
assert.equal(speechTypes.evidenceAudio.name, "SpeechEvidenceAudio");
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
diff --git a/packages/speech-alignment/README.md b/packages/speech-alignment/README.md
index 58f0088f..bfffd66b 100644
--- a/packages/speech-alignment/README.md
+++ b/packages/speech-alignment/README.md
@@ -3,18 +3,22 @@
Provider-neutral deterministic timing locator for one aligned transcript. The package does not
invoke Python, read audio, call a speech provider, use an LLM, infer speakers or retain multiple
candidate paths. A runtime adapter preserves its raw provider artifact and supplies the normalized
-word, character, score, speech-activity and Segment evidence used here.
+word, character, score and speech-activity evidence used here in exact 16 kHz sample coordinates.
-The locator aligns each known Script Segment independently. A bounded monotonic M:N dynamic
+Using the explicitly connected SpeechAudioBasis, the locator first projects exact Segment frame
+cuts into the same evidence-sample domain and assigns acoustic passages there. It then aligns each
+known Script Segment independently. A bounded monotonic M:N dynamic
program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.
Within a selected group, timed evidence characters determine Script token boundaries; word times
and speech-activity bounds are fallbacks. Missing token runs receive continuous weighted windows
between their measured neighbors instead of invented point timestamps. The result is quantized once
into the selected ProgramSpace and contains one timing for every Script token and all `2M + 2N`
-semantic anchors.
+semantic anchors. Segment anchors copy the Basis frame cuts directly; they never round-trip through
+floating-point seconds.
The locator accepts Narrative, SpeechAudioBasis and AlignedTranscriptEvidence through three explicit
-graph edges. It checks Segment identity, order and measured windows against the supplied Basis.
+graph edges. Evidence carries no authored Segment identity; the locator checks its sample windows
+against the supplied Basis and performs that association locally.
Artifact/Basis lineage stays in the upstream request edges and Derivations instead of being copied
through every evidence value.
diff --git a/packages/speech-alignment/src/locate.ts b/packages/speech-alignment/src/locate.ts
index 05920839..247c663f 100644
--- a/packages/speech-alignment/src/locate.ts
+++ b/packages/speech-alignment/src/locate.ts
@@ -1,8 +1,14 @@
import { digestOf, isDigest } from "@narratage/protocol";
import type { Narrative, NarrativeToken } from "@narratage/narrative";
-import type { ProgramSpace } from "@narratage/program-space";
-import type { SpeechAudioBasis, SpeechBasis } from "@narratage/speech";
-import type { AlignedTranscriptEvidence, AlignedTranscriptSegment, SpeechCharacterEvidence, SpeechWordEvidence } from "@narratage/speech-evidence";
+import { programFrameSampleBoundary, programSpaceFrameCount } from "@narratage/program-space";
+import type { SpeechAudioBasis } from "@narratage/speech";
+import type {
+ AlignedTranscriptEvidence,
+ SpeechActivitySpan,
+ SpeechCharacterEvidence,
+ SpeechTranscriptPassage,
+ SpeechWordEvidence,
+} from "@narratage/speech-evidence";
import type { CompleteSemanticMap, SemanticTimePoint, TimedSpeechToken } from "@narratage/semantic-map";
import { alignWordGroups } from "./align.js";
@@ -13,47 +19,67 @@ import type { AlignmentGroup, TimedSpeechSegment } from "./types.js";
export const speechLocatorDigest = digestOf("@narratage/speech-alignment/locate@1");
type MutableTiming = {
- startSec: number;
- endSec: number;
+ startSample: number;
+ endSampleExclusive: number;
};
type TimedEvidenceChar = {
readonly value: string;
readonly wordIndex: number;
- readonly startSec?: number;
- readonly endSec?: number;
+ readonly startSample?: number;
+ readonly endSampleExclusive?: number;
+};
+
+type SegmentEvidence = {
+ readonly sourceSegmentId: string;
+ readonly words: SpeechWordEvidence[];
+ readonly chars: SpeechCharacterEvidence[];
+ readonly speechActivity: SpeechActivitySpan[];
};
function fail(code: string, message: string): never {
throw new SpeechAlignmentError(code, message);
}
-function finite(value: number | undefined): value is number {
- return value !== undefined && Number.isFinite(value);
+function sampleWindowPresent(value: {
+ readonly startSample?: number;
+ readonly endSampleExclusive?: number;
+}): value is { readonly startSample: number; readonly endSampleExclusive: number } {
+ return value.startSample !== undefined && value.endSampleExclusive !== undefined;
}
-/**
- * A time is rejected only when no frame can be derived from it. Whether a window
- * runs backwards, leaves its Segment or reaches past the programme is a fact
- * about the recording, reported as measured.
- */
-function validateWindow(start: number, end: number, _limit: number, label: string): void {
- if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < 0) {
- fail("SPEECH_WINDOW", `${label} has an unusable time ${start}..${end}.`);
+/** Provider measurements may overlap or run backwards; only unusable sample coordinates are rejected. */
+function validateSampleWindow(
+ value: { readonly startSample?: number; readonly endSampleExclusive?: number },
+ limit: number,
+ label: string,
+): void {
+ if ((value.startSample === undefined) !== (value.endSampleExclusive === undefined)) {
+ fail("SPEECH_WINDOW", `${label} must provide both sample boundaries or neither.`);
}
+ if (value.startSample === undefined || value.endSampleExclusive === undefined) return;
+ if (!Number.isSafeInteger(value.startSample) || !Number.isSafeInteger(value.endSampleExclusive)
+ || value.startSample < 0 || value.endSampleExclusive < 0
+ || value.startSample > limit || value.endSampleExclusive > limit) {
+ fail("SPEECH_WINDOW", `${label} has an unusable sample window ${value.startSample}..${value.endSampleExclusive}.`);
+ }
+}
+
+function evidenceSampleFrames(basis: SpeechAudioBasis): number {
+ return programFrameSampleBoundary(
+ basis.programSpace,
+ programSpaceFrameCount(basis.programSpace),
+ 16_000,
+ );
}
function validateBasis(narrative: Narrative, basis: SpeechAudioBasis): void {
const { numerator, denominator } = basis.programSpace.frameRate;
- if (!Number.isSafeInteger(numerator) || numerator <= 0 || !Number.isSafeInteger(denominator) || denominator <= 0) {
+ if (!Number.isSafeInteger(numerator) || numerator <= 0
+ || !Number.isSafeInteger(denominator) || denominator <= 0) {
fail("SPEECH_FRAME_RATE", "ProgramSpace frame rate must be a positive rational number.");
}
- if (
- !Number.isFinite(basis.programSpace.durationSec)
- || basis.programSpace.durationSec <= 0
- ) {
- fail("SPEECH_BASIS_DURATION", "SpeechAudioBasis ProgramSpace must have a positive duration.");
- }
+ const frameCount = programSpaceFrameCount(basis.programSpace);
if (basis.audio.kind !== "blob" || !isDigest(basis.audio.digest)
|| basis.audio.mediaType !== "audio/wav" || !Number.isSafeInteger(basis.audio.size) || basis.audio.size < 0) {
fail("SPEECH_AUDIO_DIGEST", "SpeechBasis audio BlobRef is invalid.");
@@ -61,52 +87,96 @@ function validateBasis(narrative: Narrative, basis: SpeechAudioBasis): void {
if (basis.segments.length !== narrative.segments.length) {
fail("SPEECH_BASIS_SEGMENTS", "SpeechAudioBasis must cover every Narrative Segment exactly once.");
}
+ let previousEnd = 0;
for (const [index, segment] of basis.segments.entries()) {
const expected = narrative.segments[index]!;
if (segment.segmentId !== expected.id) {
fail("SPEECH_BASIS_SEGMENTS", `SpeechAudioBasis Segment ${segment.segmentId} does not match ${expected.id}.`);
}
- validateWindow(segment.startSec, segment.endSec, basis.programSpace.durationSec, `Basis Segment ${segment.segmentId}`);
+ if (!Number.isSafeInteger(segment.startFrame) || !Number.isSafeInteger(segment.endFrameExclusive)
+ || segment.startFrame !== previousEnd || segment.endFrameExclusive <= segment.startFrame
+ || segment.endFrameExclusive > frameCount) {
+ fail("SPEECH_BASIS_SEGMENTS", `SpeechAudioBasis Segment ${segment.segmentId} has an invalid frame window.`);
+ }
+ previousEnd = segment.endFrameExclusive;
+ }
+ if (previousEnd !== frameCount) {
+ fail("SPEECH_BASIS_SEGMENTS", "SpeechAudioBasis Segments must cover ProgramSpace exactly.");
}
}
-function validateEvidence(
- narrative: Narrative,
- basis: SpeechAudioBasis,
- evidence: AlignedTranscriptEvidence,
-): void {
- const durationSec = basis.programSpace.durationSec;
- const expected = new Set(narrative.segments.map((segment) => segment.id));
- const seen = new Set<string>();
- for (const segment of evidence.segments) {
- if (!expected.has(segment.sourceSegmentId)) {
- fail("SPEECH_SEGMENT_UNKNOWN", `Aligned transcript references unknown Segment ${segment.sourceSegmentId}.`);
- }
- if (seen.has(segment.sourceSegmentId)) {
- fail("SPEECH_SEGMENT_DUPLICATE", `Aligned transcript repeats Segment ${segment.sourceSegmentId}.`);
+function validateEvidence(basis: SpeechAudioBasis, evidence: AlignedTranscriptEvidence): void {
+ const limit = evidenceSampleFrames(basis);
+ for (const [passageIndex, passage] of evidence.passages.entries()) {
+ validateSampleWindow(passage, limit, `Passage ${passageIndex + 1}`);
+ for (const [wordIndex, word] of passage.words.entries()) {
+ if (typeof word.text !== "string") fail("SPEECH_WORD_TEXT", `Word ${wordIndex + 1} has no text.`);
+ validateSampleWindow(word, limit, `Word ${wordIndex + 1}`);
}
- seen.add(segment.sourceSegmentId);
- for (const [index, word] of segment.words.entries()) {
- if (typeof word.text !== "string") fail("SPEECH_WORD_TEXT", `Word ${index + 1} has no text.`);
- if (finite(word.startSec) && finite(word.endSec)) {
- validateWindow(word.startSec, word.endSec, durationSec, `Word ${index + 1}`);
+ for (const [charIndex, char] of passage.chars.entries()) {
+ if (!Number.isInteger(char.wordIndex) || char.wordIndex < 0 || char.wordIndex >= passage.words.length) {
+ fail("SPEECH_CHAR_WORD", `Character ${charIndex + 1} has an invalid wordIndex.`);
}
+ validateSampleWindow(char, limit, `Character ${charIndex + 1}`);
}
- for (const [index, char] of segment.chars.entries()) {
- if (!Number.isInteger(char.wordIndex) || char.wordIndex < 0 || char.wordIndex >= segment.words.length) {
- fail("SPEECH_CHAR_WORD", `Character ${index + 1} has an invalid wordIndex.`);
- }
- if (finite(char.startSec) && finite(char.endSec)) {
- validateWindow(char.startSec, char.endSec, durationSec, `Character ${index + 1}`);
- }
- }
- for (const [index, span] of (segment.speechActivity ?? []).entries()) {
- validateWindow(span.startSec, span.endSec, durationSec, `VAD span ${index + 1}`);
+ for (const [spanIndex, span] of (passage.speechActivity ?? []).entries()) {
+ validateSampleWindow(span, limit, `Speech activity span ${spanIndex + 1}`);
}
}
- for (const id of expected) {
- if (!seen.has(id)) fail("SPEECH_SEGMENT_MISSING", `Aligned transcript is missing Segment ${id}.`);
+}
+
+function sampleMidpoint(value: {
+ readonly startSample?: number;
+ readonly endSampleExclusive?: number;
+}): number | undefined {
+ return sampleWindowPresent(value)
+ ? (value.startSample + value.endSampleExclusive) / 2
+ : undefined;
+}
+
+/** Assign acoustic evidence to authored Segments only where both clocks are explicitly connected. */
+function partitionEvidence(
+ basis: SpeechAudioBasis,
+ evidence: AlignedTranscriptEvidence,
+): readonly SegmentEvidence[] {
+ const ranges = basis.segments.map((segment) => ({
+ segment,
+ startSample: programFrameSampleBoundary(basis.programSpace, segment.startFrame, 16_000),
+ endSampleExclusive: programFrameSampleBoundary(basis.programSpace, segment.endFrameExclusive, 16_000),
+ }));
+ const buckets = ranges.map(({ segment }) => ({
+ sourceSegmentId: segment.segmentId,
+ words: [] as SpeechWordEvidence[],
+ chars: [] as SpeechCharacterEvidence[],
+ speechActivity: [] as SpeechActivitySpan[],
+ }));
+ const bucketIndex = (point: number | undefined): number => {
+ const measured = point ?? 0;
+ const index = ranges.findIndex((range, rangeIndex) => measured >= range.startSample
+ && (measured < range.endSampleExclusive || rangeIndex === ranges.length - 1));
+ if (index >= 0) return index;
+ return measured < ranges[0]!.startSample ? 0 : ranges.length - 1;
+ };
+
+ for (const passage of evidence.passages) {
+ const passagePoint = sampleMidpoint(passage);
+ for (const [wordIndex, word] of passage.words.entries()) {
+ const target = buckets[bucketIndex(sampleMidpoint(word) ?? passagePoint)]!;
+ const targetWordIndex = target.words.length;
+ target.words.push(word);
+ target.chars.push(...passage.chars
+ .filter((char) => char.wordIndex === wordIndex)
+ .map((char) => ({ ...char, wordIndex: targetWordIndex })));
+ }
+ for (const span of passage.speechActivity ?? []) {
+ buckets[bucketIndex(sampleMidpoint(span) ?? passagePoint)]!.speechActivity.push(span);
+ }
}
+ return buckets;
+}
+
+function interpolatedBoundary(start: number, end: number, index: number, count: number): number {
+ return start + Math.round((end - start) * index / count);
}
function syntheticCharTimes(
@@ -114,14 +184,19 @@ function syntheticCharTimes(
wordIndex: number,
word: SpeechWordEvidence,
): TimedEvidenceChar[] {
- const hasWindow = finite(word.startSec) && finite(word.endSec);
+ const hasWindow = sampleWindowPresent(word);
return characters.map((value, index) => ({
value,
wordIndex,
...(hasWindow
? {
- startSec: word.startSec! + (word.endSec! - word.startSec!) * index / characters.length,
- endSec: word.startSec! + (word.endSec! - word.startSec!) * (index + 1) / characters.length,
+ startSample: interpolatedBoundary(word.startSample, word.endSampleExclusive, index, characters.length),
+ endSampleExclusive: interpolatedBoundary(
+ word.startSample,
+ word.endSampleExclusive,
+ index + 1,
+ characters.length,
+ ),
}
: {}),
}));
@@ -139,8 +214,8 @@ function timedCharsForWord(
.flatMap((char) => alignmentCharacters(char.char).map((value) => ({
value,
wordIndex,
- ...(finite(char.startSec) && finite(char.endSec)
- ? { startSec: char.startSec, endSec: char.endSec }
+ ...(sampleWindowPresent(char)
+ ? { startSample: char.startSample, endSampleExclusive: char.endSampleExclusive }
: {}),
})));
if (!supplied.length) return syntheticCharTimes(expected, wordIndex, word);
@@ -149,22 +224,22 @@ function timedCharsForWord(
const pairs = alignCharacters(expected, supplied.map((char) => char.value));
for (const pair of pairs) {
const suppliedChar = supplied[pair.evidenceIndex]!;
- if (!finite(suppliedChar.startSec) || !finite(suppliedChar.endSec)) continue;
+ if (!sampleWindowPresent(suppliedChar)) continue;
fallback[pair.sourceIndex] = {
value: expected[pair.sourceIndex]!,
wordIndex,
- startSec: suppliedChar.startSec,
- endSec: suppliedChar.endSec,
+ startSample: suppliedChar.startSample,
+ endSampleExclusive: suppliedChar.endSampleExclusive,
};
}
return fallback;
}
-function timedChars(segment: AlignedTranscriptSegment): TimedEvidenceChar[] {
+function timedChars(segment: SegmentEvidence): TimedEvidenceChar[] {
return segment.words.flatMap((word, index) => timedCharsForWord(word, index, segment.chars));
}
-function scoredWords(segment: AlignedTranscriptSegment): SpeechWordEvidence[] {
+function scoredWords(segment: SegmentEvidence): SpeechWordEvidence[] {
return segment.words.map((word, wordIndex) => {
const scores = [
...(word.score === undefined ? [] : [word.score]),
@@ -182,21 +257,20 @@ function groupWindow(
group: AlignmentGroup,
words: readonly SpeechWordEvidence[],
chars: readonly TimedEvidenceChar[],
-): { readonly startSec: number; readonly endSec: number } | undefined {
+): MutableTiming | undefined {
const wordRun = words.slice(group.evidenceWordStart, group.evidenceWordEndExclusive);
const charRun = chars.filter((char) =>
char.wordIndex >= group.evidenceWordStart && char.wordIndex < group.evidenceWordEndExclusive);
const starts = [
- ...wordRun.map((word) => word.startSec),
- ...charRun.map((char) => char.startSec),
- ].filter(finite);
+ ...wordRun.map((word) => word.startSample),
+ ...charRun.map((char) => char.startSample),
+ ].filter((value): value is number => value !== undefined);
const ends = [
- ...wordRun.map((word) => word.endSec),
- ...charRun.map((char) => char.endSec),
- ].filter(finite);
+ ...wordRun.map((word) => word.endSampleExclusive),
+ ...charRun.map((char) => char.endSampleExclusive),
+ ].filter((value): value is number => value !== undefined);
if (!starts.length || !ends.length) return undefined;
- // The envelope of a run of evidence, not a correction of any one measurement.
- return { startSec: Math.min(...starts), endSec: Math.max(...ends) };
+ return { startSample: Math.min(...starts), endSampleExclusive: Math.max(...ends) };
}
function locatePairedGroup(
@@ -217,37 +291,42 @@ function locatePairedGroup(
sourceCharacters.map((char) => char.value),
evidenceCharacters.map((char) => char.value),
);
- const exactOneToOne = group.relation === "exact";
const window = groupWindow(group, words, chars);
for (let tokenOffset = 0; tokenOffset < sourceRun.length; tokenOffset += 1) {
const evidenceForToken = pairs
.filter((pair) => sourceCharacters[pair.sourceIndex]!.tokenOffset === tokenOffset)
.map((pair) => evidenceCharacters[pair.evidenceIndex]!)
- .filter((char) => finite(char.startSec) && finite(char.endSec));
- let startSec = evidenceForToken[0]?.startSec;
- let endSec = evidenceForToken.at(-1)?.endSec;
+ .filter(sampleWindowPresent);
+ let startSample = evidenceForToken[0]?.startSample;
+ let endSampleExclusive = evidenceForToken.at(-1)?.endSampleExclusive;
if (sourceRun.length === 1 && window) {
- startSec = window.startSec;
- endSec = window.endSec;
+ startSample = window.startSample;
+ endSampleExclusive = window.endSampleExclusive;
} else if (window) {
- if (tokenOffset === 0 && finite(startSec)) startSec = window.startSec;
- if (tokenOffset === sourceRun.length - 1 && finite(endSec)) endSec = window.endSec;
+ if (tokenOffset === 0 && startSample !== undefined) startSample = window.startSample;
+ if (tokenOffset === sourceRun.length - 1 && endSampleExclusive !== undefined) {
+ endSampleExclusive = window.endSampleExclusive;
+ }
}
- if (!finite(startSec) || !finite(endSec)) continue;
+ if (startSample === undefined || endSampleExclusive === undefined) continue;
const sourceIndex = sourceIndexById.get(sourceRun[tokenOffset]!.id)!;
- output[sourceIndex] = { startSec, endSec };
+ output[sourceIndex] = { startSample, endSampleExclusive };
}
}
function speechBounds(
- segment: AlignedTranscriptSegment,
- basisSegment: { readonly startSec: number; readonly endSec: number },
+ basis: SpeechAudioBasis,
+ segment: SegmentEvidence,
+ basisSegment: SpeechAudioBasis["segments"][number],
): { readonly start: number; readonly end: number } {
- const spans = segment.speechActivity ?? [];
+ const spans = segment.speechActivity;
return spans.length
- ? { start: spans[0]!.startSec, end: spans.at(-1)!.endSec }
- : { start: basisSegment.startSec, end: basisSegment.endSec };
+ ? { start: spans[0]!.startSample, end: spans.at(-1)!.endSampleExclusive }
+ : {
+ start: programFrameSampleBoundary(basis.programSpace, basisSegment.startFrame, 16_000),
+ end: programFrameSampleBoundary(basis.programSpace, basisSegment.endFrameExclusive, 16_000),
+ };
}
function fillMissingTiming(
@@ -265,10 +344,8 @@ function fillMissingTiming(
const runStart = cursor;
while (cursor < values.length && !values[cursor]) cursor += 1;
const runEnd = cursor;
- const left = runStart > 0 ? values[runStart - 1]!.endSec : startBound;
- const right = runEnd < values.length ? values[runEnd]!.startSec : endBound;
- // These tokens have no measurement at all, so this bound shapes an invention
- // rather than editing anything that was measured.
+ const left = runStart > 0 ? values[runStart - 1]!.endSampleExclusive : startBound;
+ const right = runEnd < values.length ? values[runEnd]!.startSample : endBound;
const usableRight = Math.max(left, right);
const weights = source
.slice(runStart, runEnd)
@@ -277,46 +354,33 @@ function fillMissingTiming(
let consumed = 0;
for (let index = runStart; index < runEnd; index += 1) {
const weight = weights[index - runStart]!;
- const startSec = left + (usableRight - left) * consumed / totalWeight;
+ const startSample = interpolatedBoundary(left, usableRight, consumed, totalWeight);
consumed += weight;
- const endSec = left + (usableRight - left) * consumed / totalWeight;
- values[index] = {
- startSec,
- endSec,
- };
+ const endSampleExclusive = interpolatedBoundary(left, usableRight, consumed, totalWeight);
+ values[index] = { startSample, endSampleExclusive };
}
}
return values as MutableTiming[];
}
-function frameFor(basis: SpeechAudioBasis, timeSec: number): number {
- const { numerator, denominator } = basis.programSpace.frameRate;
- return Math.round(timeSec * numerator / denominator);
+function frameForEvidenceSample(basis: SpeechAudioBasis, sample: number): number {
+ const numerator = BigInt(sample) * BigInt(basis.programSpace.frameRate.numerator);
+ const denominator = 16_000n * BigInt(basis.programSpace.frameRate.denominator);
+ const frame = (numerator * 2n + denominator) / (denominator * 2n);
+ if (frame > BigInt(Number.MAX_SAFE_INTEGER)) fail("SPEECH_FRAME", "Speech evidence exceeds the frame domain.");
+ return Number(frame);
}
-/**
- * Locate every Script token against one recording.
- *
- * The result is total: every token of every Segment carries a window, whether it
- * was measured, derived from a neighbouring character run, or interpolated
- * because the transcript never reached it. Whether a window runs backwards,
- * overlaps its neighbour or leaves its Segment is reported as measured — those
- * are facts about the recording, and deciding what they mean belongs to whoever
- * projects them onto a timeline.
- *
- * Locating fails only when the Script, the audio and the transcript are not the
- * same three things: a missing or repeated Segment, a Segment the Script never
- * declared, or a transcript answering a different segmentation. It never fails
- * merely because a provider timestamp is awkward.
- */
+/** Locate every Script token from one provider-neutral acoustic evidence pass. */
export function locateSpeechTiming(
narrative: Narrative,
basis: SpeechAudioBasis,
evidence: AlignedTranscriptEvidence,
): CompleteSemanticMap {
validateBasis(narrative, basis);
- validateEvidence(narrative, basis, evidence);
- const evidenceBySegment = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));
+ validateEvidence(basis, evidence);
+ const evidenceBySegment = new Map(partitionEvidence(basis, evidence)
+ .map((segment) => [segment.sourceSegmentId, segment]));
const timedSegments: TimedSpeechSegment[] = [];
const timedTokens: TimedSpeechToken[] = [];
@@ -328,26 +392,21 @@ export function locateSpeechTiming(
const located: Array<MutableTiming | undefined> = Array(source.length).fill(undefined);
for (const group of segmentGroups) locatePairedGroup(group, source, aligned.words, chars, located);
const basisSegment = basis.segments.find((item) => item.segmentId === segment.id)!;
- const bounds = speechBounds(aligned, basisSegment);
+ const bounds = speechBounds(basis, aligned, basisSegment);
const complete = fillMissingTiming(located, source, bounds.start, bounds.end);
- const segmentTokens = source.map((token, index): TimedSpeechToken => {
+ timedTokens.push(...source.map((token, index): TimedSpeechToken => {
const timing = complete[index]!;
- const startFrame = frameFor(basis, timing.startSec);
- const endFrame = frameFor(basis, timing.endSec);
return {
tokenId: token.id,
segmentId: segment.id,
- startFrame,
- endFrameExclusive: endFrame,
+ startFrame: frameForEvidenceSample(basis, timing.startSample),
+ endFrameExclusive: frameForEvidenceSample(basis, timing.endSampleExclusive),
};
- });
- timedTokens.push(...segmentTokens);
- const startFrame = frameFor(basis, basisSegment.startSec);
- const endFrame = frameFor(basis, basisSegment.endSec);
+ }));
timedSegments.push({
segmentId: segment.id,
- startFrame,
- endFrameExclusive: endFrame,
+ startFrame: basisSegment.startFrame,
+ endFrameExclusive: basisSegment.endFrameExclusive,
});
}
@@ -365,8 +424,5 @@ export function locateSpeechTiming(
? { identity: anchor.id, frame: token.startFrame }
: { identity: anchor.id, frame: token.endFrameExclusive };
});
- return {
- tokens: timedTokens,
- anchors,
- };
+ return { tokens: timedTokens, anchors };
}
diff --git a/packages/speech-alignment/src/types.ts b/packages/speech-alignment/src/types.ts
index 714375a2..5a188ee3 100644
--- a/packages/speech-alignment/src/types.ts
+++ b/packages/speech-alignment/src/types.ts
@@ -1,8 +1,8 @@
export type {
AlignedTranscriptEvidence,
- AlignedTranscriptSegment,
SpeechActivitySpan,
SpeechCharacterEvidence,
+ SpeechTranscriptPassage,
SpeechWordEvidence,
} from "@narratage/speech-evidence";
export type {
diff --git a/packages/speech-alignment/test/speech-align.test.ts b/packages/speech-alignment/test/speech-align.test.ts
index 486d82f6..f3a08fee 100644
--- a/packages/speech-alignment/test/speech-align.test.ts
+++ b/packages/speech-alignment/test/speech-align.test.ts
@@ -16,37 +16,58 @@ import {
locateSpeechTiming,
} from "@narratage/speech-alignment";
+type WordFixture = {
+ readonly text: string;
+ readonly startSec?: number;
+ readonly endSec?: number;
+ readonly score?: number;
+};
+
+function wordEvidence(word: WordFixture): SpeechWordEvidence {
+ return {
+ text: word.text,
+ ...(word.startSec === undefined || word.endSec === undefined ? {} : {
+ startSample: Math.round(word.startSec * 16_000),
+ endSampleExclusive: Math.round(word.endSec * 16_000),
+ }),
+ ...(word.score === undefined ? {} : { score: word.score }),
+ };
+}
+
/** The alignment classification is a property of alignWordGroups, tested at its own level. */
function relations(
narrative: Narrative,
- words: readonly SpeechWordEvidence[],
+ words: readonly WordFixture[],
segmentId = "line",
): string[] {
const segment = narrative.segments.find((item) => item.id === segmentId)!;
return alignWordGroups(
segmentId,
narrative.tokens.slice(segment.tokenStart, segment.tokenEndExclusive),
- words,
+ words.map(wordEvidence),
).map((group) => group.relation);
}
function evidence(args: {
readonly basis: SpeechAudioBasis;
- readonly segmentId?: string;
readonly durationSec?: number;
readonly startSec?: number;
readonly endSec?: number;
- readonly words: readonly SpeechWordEvidence[];
+ readonly words: readonly WordFixture[];
readonly chars?: readonly SpeechCharacterEvidence[];
readonly vad?: readonly { readonly startSec: number; readonly endSec: number }[];
}): AlignedTranscriptEvidence {
return sealAlignedTranscriptEvidence({
- segments: [
+ passages: [
{
- sourceSegmentId: args.segmentId ?? "line",
- words: args.words,
+ words: args.words.map(wordEvidence),
chars: args.chars ?? [],
- ...(args.vad === undefined ? {} : { speechActivity: args.vad }),
+ ...(args.vad === undefined ? {} : {
+ speechActivity: args.vad.map((span) => ({
+ startSample: Math.round(span.startSec * 16_000),
+ endSampleExclusive: Math.round(span.endSec * 16_000),
+ })),
+ }),
},
],
});
@@ -64,8 +85,8 @@ function speechBasis(
const audioDigest = digestOf(`fixture:audio:${narrative.segments.map((segment) => segment.id).join("+")}:${durationSec}`);
const segments = narrative.segments.map((segment, index) => ({
segmentId: segment.id,
- startSec: windows?.[index]?.startSec ?? durationSec * index / narrative.segments.length,
- endSec: windows?.[index]?.endSec ?? durationSec * (index + 1) / narrative.segments.length,
+ startFrame: Math.round((windows?.[index]?.startSec ?? durationSec * index / narrative.segments.length) * 1_000),
+ endFrameExclusive: Math.round((windows?.[index]?.endSec ?? durationSec * (index + 1) / narrative.segments.length) * 1_000),
}));
const take = sealSpeechBasis({
programSpace,
@@ -105,8 +126,8 @@ function characters(
return [...text].map((char, index) => ({
char,
wordIndex,
- startSec: starts[index]!,
- endSec: ends[index]!,
+ startSample: Math.round(starts[index]! * 16_000),
+ endSampleExclusive: Math.round(ends[index]! * 16_000),
score: 0.95,
}));
}
@@ -244,15 +265,13 @@ test("multiple Script Segments stay independent even when evidence records arriv
{ startSec: 1, endSec: 2 },
]);
const map = locateSpeechTiming(narrative, basis, sealAlignedTranscriptEvidence({
- segments: [
+ passages: [
{
- sourceSegmentId: "two",
- words: [{ text: "Goodbye", startSec: 1.2, endSec: 1.6 }],
+ words: [wordEvidence({ text: "Goodbye", startSec: 1.2, endSec: 1.6 })],
chars: [],
},
{
- sourceSegmentId: "one",
- words: [{ text: "Hello", startSec: 0.2, endSec: 0.55 }],
+ words: [wordEvidence({ text: "Hello", startSec: 0.2, endSec: 0.55 })],
chars: [],
},
],
@@ -264,6 +283,38 @@ test("multiple Script Segments stay independent even when evidence records arriv
]);
});
+test("Segment anchors preserve exact frame cuts without a seconds round trip", () => {
+ const narrative = parseScript("frame-cuts.svml", "<one>Alpha.</one><two>Beta.</two>");
+ const take = sealSpeechBasis({
+ programSpace: sealProgramSpace({
+ durationSec: 1,
+ frameRate: { numerator: 24, denominator: 1 },
+ }),
+ audio: { kind: "blob", digest: digestOf("frame-cuts:audio"), size: 1, mediaType: "audio/wav" },
+ visualTrack: { clips: [] },
+ segments: [
+ { segmentId: "one", startFrame: 0, endFrameExclusive: 7 },
+ { segmentId: "two", startFrame: 7, endFrameExclusive: 24 },
+ ],
+ });
+ const map = locateSpeechTiming(narrative, audioProjection(take), sealAlignedTranscriptEvidence({
+ passages: [{
+ startSample: 0,
+ endSampleExclusive: 16_000,
+ words: [
+ { text: "Alpha", startSample: 1_000, endSampleExclusive: 3_000 },
+ { text: "Beta", startSample: 6_000, endSampleExclusive: 10_000 },
+ ],
+ chars: [],
+ }],
+ }));
+ const frameByAnchor = new Map(map.anchors.map((anchor) => [anchor.identity, anchor.frame]));
+ const structuralFrames = narrative.semanticIndex.anchors
+ .filter((anchor) => anchor.kind === "segment-start" || anchor.kind === "segment-end")
+ .map((anchor) => frameByAnchor.get(anchor.id));
+ assert.deepEqual(structuralFrames, [0, 7, 7, 24]);
+});
+
test("overlapping evidence word windows reach the map overlapping", () => {
// Two spoken words whose measured windows overlap. Nothing here knows whether
// that is a real overlap or a wobble, and no consumer of the map is bound to
@@ -306,7 +357,11 @@ test("Evidence is interpreted only through the explicitly connected SpeechAudioB
durationSec: 2,
frameRate: { numerator: 30, denominator: 1 },
});
- const anotherBasis = { ...basis, programSpace: anotherSpace };
+ const anotherBasis = {
+ ...basis,
+ programSpace: anotherSpace,
+ segments: [{ segmentId: "line", startFrame: 0, endFrameExclusive: 60 }],
+ };
const map = locateSpeechTiming(narrative, anotherBasis, mismatched);
assert.equal(map.tokens[0]?.startFrame, 3);
});
@@ -321,6 +376,7 @@ test("the final map is quantized once into the selected ProgramSpace", () => {
const basis: SpeechAudioBasis = {
...original,
programSpace,
+ segments: [{ segmentId: "line", startFrame: 0, endFrameExclusive: 30 }],
};
const map = locateSpeechTiming(narrative, basis, evidence({
basis,
diff --git a/packages/speech-basis/src/projection.ts b/packages/speech-basis/src/projection.ts
index 15b89b53..04803098 100644
--- a/packages/speech-basis/src/projection.ts
+++ b/packages/speech-basis/src/projection.ts
@@ -40,12 +40,9 @@ export function projectSpeechAudio(basis: SpeechBasis): SpeechAudioBasis {
export function projectSpeechVisual(basis: SpeechBasis): VisualTrack {
assertSpeechBasisIdentity(basis);
const trackId = `speech-visual:${basis.segments.map((segment) => segment.segmentId).join("+")}`;
- const frameAt = (seconds: number): number => Math.round(
- seconds * basis.programSpace.frameRate.numerator / basis.programSpace.frameRate.denominator,
- );
const spans = new Map(basis.segments.map((segment) => [segment.segmentId, {
- startFrame: frameAt(segment.startSec),
- endFrameExclusive: frameAt(segment.endSec),
+ startFrame: segment.startFrame,
+ endFrameExclusive: segment.endFrameExclusive,
}]));
return sealVisualTrack({
visualIr: "svml.visual-ir@1",
diff --git a/packages/speech-basis/test/product.test.ts b/packages/speech-basis/test/product.test.ts
index fa6b6e4b..7ce972b8 100644
--- a/packages/speech-basis/test/product.test.ts
+++ b/packages/speech-basis/test/product.test.ts
@@ -116,8 +116,8 @@ function sampleTake(label = "generated"): SpeechBasis {
},
segments: [{
segmentId: "opening",
- startSec: 0,
- endSec: durationSec,
+ startFrame: 0,
+ endFrameExclusive: durationSec * 30,
}],
});
}
diff --git a/packages/speech-evidence/src/index.ts b/packages/speech-evidence/src/index.ts
index 77676db6..aefbc338 100644
--- a/packages/speech-evidence/src/index.ts
+++ b/packages/speech-evidence/src/index.ts
@@ -1,25 +1,105 @@
import { digestOf } from "@narratage/protocol";
import type { ModuleManifest, TypeRef, ValueSchema } from "@narratage/protocol";
-export type SpeechWordEvidence = { readonly text: string; readonly startSec?: number; readonly endSec?: number; readonly score?: number };
-export type SpeechCharacterEvidence = { readonly char: string; readonly wordIndex: number; readonly startSec?: number; readonly endSec?: number; readonly score?: number };
-export type SpeechActivitySpan = { readonly startSec: number; readonly endSec: number };
-export type AlignedTranscriptSegment = { readonly sourceSegmentId: string;
- readonly words: readonly SpeechWordEvidence[]; readonly chars: readonly SpeechCharacterEvidence[]; readonly speechActivity?: readonly SpeechActivitySpan[] };
-export type AlignedTranscriptEvidence = { readonly segments: readonly AlignedTranscriptSegment[] };
-const string = { kind: "string", minLength: 1 } as const; const number = { kind: "number", minimum: 0 } as const; const integer = { kind: "number", integer: true, minimum: 0 } as const;
-const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>): ValueSchema => ({ kind: "object", fields });
-const word = object({ text: { schema: { kind: "string" } }, startSec: { schema: number, optional: true }, endSec: { schema: number, optional: true }, score: { schema: { kind: "number", minimum: 0, maximum: 1 }, optional: true } });
-const char = object({ char: { schema: { kind: "string" } }, wordIndex: { schema: integer }, startSec: { schema: number, optional: true }, endSec: { schema: number, optional: true }, score: { schema: { kind: "number", minimum: 0, maximum: 1 }, optional: true } });
+
+/** Exact boundaries in the fixed 16 kHz Speech Evidence Audio sample domain. */
+export type SpeechWordEvidence = {
+ readonly text: string;
+ readonly startSample?: number;
+ readonly endSampleExclusive?: number;
+ readonly score?: number;
+};
+
+export type SpeechCharacterEvidence = {
+ readonly char: string;
+ /** Index into the containing passage's words. */
+ readonly wordIndex: number;
+ readonly startSample?: number;
+ readonly endSampleExclusive?: number;
+ readonly score?: number;
+};
+
+export type SpeechActivitySpan = {
+ readonly startSample: number;
+ readonly endSampleExclusive: number;
+};
+
+/**
+ * One acoustic passage reported by the evidence provider. It deliberately has
+ * no authored Segment identity; assigning evidence to Script Segments belongs
+ * to Speech Alignment, where the connected SpeechAudioBasis is available.
+ */
+export type SpeechTranscriptPassage = {
+ readonly startSample?: number;
+ readonly endSampleExclusive?: number;
+ readonly words: readonly SpeechWordEvidence[];
+ readonly chars: readonly SpeechCharacterEvidence[];
+ readonly speechActivity?: readonly SpeechActivitySpan[];
+};
+
+export type AlignedTranscriptEvidence = {
+ readonly passages: readonly SpeechTranscriptPassage[];
+};
+
+const sample = { kind: "number", integer: true, minimum: 0 } as const;
+const integer = { kind: "number", integer: true, minimum: 0 } as const;
+const score = { kind: "number", minimum: 0, maximum: 1 } as const;
+const object = (
+ fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>,
+): ValueSchema => ({ kind: "object", fields });
+
+const word = object({
+ text: { schema: { kind: "string" } },
+ startSample: { schema: sample, optional: true },
+ endSampleExclusive: { schema: sample, optional: true },
+ score: { schema: score, optional: true },
+});
+const char = object({
+ char: { schema: { kind: "string" } },
+ wordIndex: { schema: integer },
+ startSample: { schema: sample, optional: true },
+ endSampleExclusive: { schema: sample, optional: true },
+ score: { schema: score, optional: true },
+});
+const activity = object({
+ startSample: { schema: sample },
+ endSampleExclusive: { schema: sample },
+});
+
export const alignedTranscriptEvidenceFields = {
- segments: { schema: { kind: "array", minItems: 1, items: object({ sourceSegmentId: { schema: string },
- words: { schema: { kind: "array", items: word } }, chars: { schema: { kind: "array", items: char } },
- speechActivity: { schema: { kind: "array", items: object({ startSec: { schema: number }, endSec: { schema: number } }) }, optional: true } }) } },
+ passages: {
+ schema: {
+ kind: "array",
+ items: object({
+ startSample: { schema: sample, optional: true },
+ endSampleExclusive: { schema: sample, optional: true },
+ words: { schema: { kind: "array", items: word } },
+ chars: { schema: { kind: "array", items: char } },
+ speechActivity: { schema: { kind: "array", items: activity }, optional: true },
+ }),
+ },
+ },
} as const satisfies Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>;
+
export const alignedTranscriptEvidenceSchema: ValueSchema = object(alignedTranscriptEvidenceFields);
export const speechEvidenceModuleRef = { name: "@narratage/speech-evidence", version: "1" } as const;
-export const speechEvidenceTypes = { alignedTranscript: { module: speechEvidenceModuleRef, name: "AlignedTranscriptEvidence" } } satisfies Record<string, TypeRef>;
-export const speechEvidenceManifest: ModuleManifest = { format: "svml.module@1", name: speechEvidenceModuleRef.name, version: speechEvidenceModuleRef.version,
- dependencies: [], types: [{ name: speechEvidenceTypes.alignedTranscript.name, schema: alignedTranscriptEvidenceSchema }], capabilities: [], producers: [] };
+export const speechEvidenceTypes = {
+ alignedTranscript: { module: speechEvidenceModuleRef, name: "AlignedTranscriptEvidence" },
+} satisfies Record<string, TypeRef>;
+export const speechEvidenceManifest: ModuleManifest = {
+ format: "svml.module@1",
+ name: speechEvidenceModuleRef.name,
+ version: speechEvidenceModuleRef.version,
+ dependencies: [],
+ types: [{ name: speechEvidenceTypes.alignedTranscript.name, schema: alignedTranscriptEvidenceSchema }],
+ capabilities: [],
+ producers: [],
+};
export const speechEvidenceManifestDigest = digestOf(speechEvidenceManifest);
-export const speechEvidenceDependency = { module: speechEvidenceModuleRef, digest: speechEvidenceManifestDigest } as const;
-export function sealAlignedTranscriptEvidence(value: AlignedTranscriptEvidence): AlignedTranscriptEvidence { return structuredClone(value); }
+export const speechEvidenceDependency = {
+ module: speechEvidenceModuleRef,
+ digest: speechEvidenceManifestDigest,
+} as const;
+
+export function sealAlignedTranscriptEvidence(value: AlignedTranscriptEvidence): AlignedTranscriptEvidence {
+ return structuredClone(value);
+}
diff --git a/packages/speech-spine/src/program.ts b/packages/speech-spine/src/program.ts
index 06bf18df..c9e09095 100644
--- a/packages/speech-spine/src/program.ts
+++ b/packages/speech-spine/src/program.ts
@@ -240,11 +240,10 @@ export function assembleSpeechBasis(
program.frameRate,
), "TimelineAudio does not cover this Speech Spine");
let frame = 0;
- const seconds = (value: number): number => value * space.frameRate.denominator / space.frameRate.numerator;
const segments = set.takes.map((take) => {
const startFrame = frame;
frame += take.media.timeline.frameCount;
- return { segmentId: take.segment.id, startSec: seconds(startFrame), endSec: seconds(frame) };
+ return { segmentId: take.segment.id, startFrame, endFrameExclusive: frame };
});
frame = 0;
const visualClips = set.takes.flatMap((take) => {
diff --git a/packages/speech/README.md b/packages/speech/README.md
index f4c9b1b1..4e43eaaa 100644
--- a/packages/speech/README.md
+++ b/packages/speech/README.md
@@ -1,3 +1,6 @@
# `@narratage/speech`
-Public contracts for speech duration, assembled bases and alignment audio projections.
+Public contracts for speech duration, assembled bases and alignment audio projections. SpeechBasis
+Segments use exact ProgramSpace frame boundaries. The separate SpeechEvidenceAudio contract carries
+canonical 16 kHz WAV bytes and their exact sample count; neither contract duplicates floating-point
+Segment seconds.
diff --git a/packages/speech/src/identity.ts b/packages/speech/src/identity.ts
index 9e21e98b..c2351738 100644
--- a/packages/speech/src/identity.ts
+++ b/packages/speech/src/identity.ts
@@ -30,20 +30,19 @@ export function assertSpeechBasisIdentity(basis: SpeechBasis): void {
assertAudioBlob(basis.audio, "SpeechBasis audio");
if (basis.segments.length === 0) throw new Error("SpeechBasis must contain at least one Segment.");
let previousEnd = 0;
- const frameAt = (seconds: number): number => Math.round(
- seconds * basis.programSpace.frameRate.numerator / basis.programSpace.frameRate.denominator,
- );
const segmentIds = new Set<string>();
for (const segment of basis.segments) {
- if (!segment.segmentId || segment.startSec !== previousEnd || !Number.isFinite(segment.endSec)
- || segment.endSec <= segment.startSec || segment.endSec > basis.programSpace.durationSec) {
+ if (!segment.segmentId || segment.startFrame !== previousEnd
+ || !Number.isSafeInteger(segment.endFrameExclusive)
+ || segment.endFrameExclusive <= segment.startFrame
+ || segment.endFrameExclusive > programSpaceFrameCount(basis.programSpace)) {
throw new Error("SpeechBasis Segment is invalid or non-contiguous.");
}
if (segmentIds.has(segment.segmentId)) throw new Error(`SpeechBasis repeats Segment ${segment.segmentId}.`);
segmentIds.add(segment.segmentId);
- previousEnd = segment.endSec;
+ previousEnd = segment.endFrameExclusive;
}
- if (frameAt(previousEnd) !== programSpaceFrameCount(basis.programSpace)) {
+ if (previousEnd !== programSpaceFrameCount(basis.programSpace)) {
throw new Error("SpeechBasis Segments do not cover ProgramSpace.");
}
const seenVisuals = new Set<string>();
@@ -66,9 +65,13 @@ export function assertSpeechAudioBasisIdentity(basis: SpeechAudioBasis): void {
assertAudioBlob(basis.audio, "SpeechAudioBasis audio");
if (basis.segments.length === 0) throw new Error("SpeechAudioBasis must contain at least one Segment.");
let previousEnd = 0;
+ const frameCount = programSpaceFrameCount(basis.programSpace);
for (const segment of basis.segments) {
- if (!segment.segmentId || !Number.isFinite(segment.startSec) || !Number.isFinite(segment.endSec) || segment.startSec < previousEnd
- || segment.endSec < segment.startSec || segment.endSec > basis.programSpace.durationSec) throw new Error("SpeechAudioBasis Segment is invalid or out of order.");
- previousEnd = segment.endSec;
+ if (!segment.segmentId || !Number.isSafeInteger(segment.startFrame)
+ || !Number.isSafeInteger(segment.endFrameExclusive) || segment.startFrame < previousEnd
+ || segment.endFrameExclusive < segment.startFrame || segment.endFrameExclusive > frameCount) {
+ throw new Error("SpeechAudioBasis Segment is invalid or out of order.");
+ }
+ previousEnd = segment.endFrameExclusive;
}
}
diff --git a/packages/speech/src/schema.ts b/packages/speech/src/schema.ts
index 069eb1be..42671062 100644
--- a/packages/speech/src/schema.ts
+++ b/packages/speech/src/schema.ts
@@ -12,7 +12,11 @@ const audioBlobRef = object({
size: { schema: integer },
mediaType: { schema: { kind: "literal", value: "audio/wav" } },
});
-const segment = object({ segmentId: { schema: string }, startSec: { schema: number }, endSec: { schema: number } });
+const segment = object({
+ segmentId: { schema: string },
+ startFrame: { schema: integer },
+ endFrameExclusive: { schema: integer },
+});
export const speechDurationSchema: ValueSchema = number;
export const speechBasisSchema: ValueSchema = object({
programSpace: { schema: programSpaceSchema }, audio: { schema: audioBlobRef },
diff --git a/packages/speech/src/types.ts b/packages/speech/src/types.ts
index 2ecb3f6a..26eb9f3c 100644
--- a/packages/speech/src/types.ts
+++ b/packages/speech/src/types.ts
@@ -3,7 +3,11 @@ import type { BlobRef } from "@narratage/protocol";
import type { ContentFit, IntrinsicExtent, SpatialFrame } from "@narratage/spatial";
/** Positive seconds. The graph port's TypeRef already carries the SpeechDuration identity. */
export type SpeechDuration = number;
-export type SpeechBasisSegment = { readonly segmentId: string; readonly startSec: number; readonly endSec: number };
+export type SpeechBasisSegment = {
+ readonly segmentId: string;
+ readonly startFrame: number;
+ readonly endFrameExclusive: number;
+};
export type SpeechBasis = {
readonly programSpace: ProgramSpace; readonly audio: BlobRef;
readonly visualTrack: { readonly clips: readonly {
diff --git a/packages/whisperx/README.md b/packages/whisperx/README.md
index 0dd7f7f8..4dea4501 100644
--- a/packages/whisperx/README.md
+++ b/packages/whisperx/README.md
@@ -5,14 +5,14 @@ selects WhisperX; Runtime registration only binds the resulting alignment Need t
a concrete WhisperX execution endpoint such as a trusted local worker or hosted service.
The package contains no credentials, Python environment or queue. Providers translate the typed
-request constraints directly into provider-neutral `AlignedTranscriptEvidence`. There is no
-vendor-shaped Evidence wrapper or pass-through normalization node in the graph.
+request directly into provider-neutral `AlignedTranscriptEvidence`. There is no vendor-shaped
+Evidence wrapper or pass-through normalization node in the graph.
-The request Producer consumes canonical `SpeechEvidenceAudio` bytes plus a separate
-`SpeechAudioBasis` edge. `@narratage/media-pipeline` first requests the explicit 48 kHz stereo
-speech-master → 16 kHz mono evidence projection. The evidence Record carries only the normalized
-Artifact and exact sample count; Segment truth comes from the Basis edge instead of being copied
-through an intermediate media value. The Derivation and Need bind all three steps.
+The request Producer consumes only canonical `SpeechEvidenceAudio` bytes. `@narratage/media-pipeline`
+first requests the explicit 48 kHz stereo speech-master → 16 kHz mono evidence projection. Authored
+Segment identity never enters the WhisperX Need or Provider. The later Speech Alignment operation
+receives `SpeechAudioBasis` through its own graph edge and assigns acoustic passages to its exact
+Segment frame windows there.
`@narratage/provider-whisperx-local` is the first concrete adapter; it validates and stages those bytes
unchanged for a warm loopback service.
diff --git a/packages/whisperx/src/component.ts b/packages/whisperx/src/component.ts
index 376b395c..dce87687 100644
--- a/packages/whisperx/src/component.ts
+++ b/packages/whisperx/src/component.ts
@@ -1,5 +1,5 @@
import type { ComponentPackage } from "@narratage/component-kit";
-import type { SpeechAudioBasis, SpeechEvidenceAudio } from "@narratage/speech";
+import type { SpeechEvidenceAudio } from "@narratage/speech";
import type { CanonicalValue, StoredValue } from "@narratage/protocol";
import { canonicalize } from "@narratage/protocol";
@@ -21,10 +21,9 @@ export const whisperXComponent = {
implementationDigest: whisperXImplementationDigests.request,
handler: ({ inputs }) => {
const evidence = inline(inputs.evidence!.value, "SpeechEvidenceAudio") as unknown as SpeechEvidenceAudio;
- const audio = inline(inputs.audio!.value, "SpeechAudioBasis") as unknown as SpeechAudioBasis;
return {
outputs: {},
- needs: { alignment: canonicalize(whisperXRequestForEvidenceAudio(evidence, audio)) },
+ needs: { alignment: canonicalize(whisperXRequestForEvidenceAudio(evidence)) },
};
},
},
diff --git a/packages/whisperx/src/evidence.ts b/packages/whisperx/src/evidence.ts
index 66628f06..5f965ed7 100644
--- a/packages/whisperx/src/evidence.ts
+++ b/packages/whisperx/src/evidence.ts
@@ -1,19 +1,15 @@
import { assertSpeechEvidenceAudioIdentity } from "@narratage/speech";
-import { assertSpeechAudioBasisIdentity } from "@narratage/speech";
-import type { SpeechAudioBasis, SpeechEvidenceAudio } from "@narratage/speech";
+import type { SpeechEvidenceAudio } from "@narratage/speech";
import type { WhisperXAlignmentRequest } from "./types.js";
export function whisperXRequestForEvidenceAudio(
evidence: SpeechEvidenceAudio,
- basis: SpeechAudioBasis,
options: { readonly language?: string } = {},
): WhisperXAlignmentRequest {
assertSpeechEvidenceAudioIdentity(evidence);
- assertSpeechAudioBasisIdentity(basis);
return {
audio: evidence.artifact,
sampleFrames: evidence.sampleFrames,
- segments: basis.segments,
...(options.language === undefined ? {} : { language: options.language }),
};
}
diff --git a/packages/whisperx/src/fragment.ts b/packages/whisperx/src/fragment.ts
index fab0bd9a..65b80fcd 100644
--- a/packages/whisperx/src/fragment.ts
+++ b/packages/whisperx/src/fragment.ts
@@ -27,7 +27,7 @@ export const whisperXSpeechAlignmentFragment = sealGraphFragment({
{
id: "request-whisperx",
producer: whisperXProducers.request,
- inputs: { evidence: operation("prepare-evidence-audio"), audio: input("audio") },
+ inputs: { evidence: operation("prepare-evidence-audio") },
result: { kind: "need", name: "alignment" },
},
{
diff --git a/packages/whisperx/src/manifest.ts b/packages/whisperx/src/manifest.ts
index 2aafb9fc..250459cb 100644
--- a/packages/whisperx/src/manifest.ts
+++ b/packages/whisperx/src/manifest.ts
@@ -48,10 +48,7 @@ export const whisperXManifest: ModuleManifest = {
producers: [
{
name: whisperXProducers.request.name,
- inputs: [
- { name: "evidence", type: speechTypes.evidenceAudio },
- { name: "audio", type: speechTypes.audioBasis },
- ],
+ inputs: [{ name: "evidence", type: speechTypes.evidenceAudio }],
outputs: [],
needs: [{
name: "alignment",
diff --git a/packages/whisperx/src/types.ts b/packages/whisperx/src/types.ts
index c57397ad..6022499f 100644
--- a/packages/whisperx/src/types.ts
+++ b/packages/whisperx/src/types.ts
@@ -1,9 +1,7 @@
-import type { SpeechBasisSegment, SpeechEvidenceAudio } from "@narratage/speech";
-import type { AlignedTranscriptSegment } from "@narratage/speech-evidence";
+import type { SpeechEvidenceAudio } from "@narratage/speech";
export type WhisperXAlignmentRequest = {
readonly audio: SpeechEvidenceAudio["artifact"];
readonly sampleFrames: number;
- readonly segments: readonly SpeechBasisSegment[];
readonly language?: string;
};
diff --git a/packages/whisperx/test/whisperx.test.ts b/packages/whisperx/test/whisperx.test.ts
index b88f9245..2677137e2 100644
--- a/packages/whisperx/test/whisperx.test.ts
+++ b/packages/whisperx/test/whisperx.test.ts
@@ -4,7 +4,7 @@ import {
import type { ProducerRegistrar } from "@narratage/component-kit";
import { sealProgramSpace } from "@narratage/program-space";
import { sealSpeechBasis, sealSpeechEvidenceAudio } from "@narratage/speech";
-import type { SpeechAudioBasis, SpeechBasis, SpeechEvidenceAudio } from "@narratage/speech";
+import type { SpeechBasis, SpeechEvidenceAudio } from "@narratage/speech";
import assert from "node:assert/strict";
import test from "node:test";
@@ -25,7 +25,7 @@ function basis() {
programSpace,
audio: { kind: "blob", digest: digestOf("whisperx-test:audio"), size: 1, mediaType: "audio/wav" },
visualTrack: { clips: [] },
- segments: [{ segmentId: "line", startSec: 0, endSec: 1 }],
+ segments: [{ segmentId: "line", startFrame: 0, endFrameExclusive: 30 }],
});
}
@@ -41,20 +41,12 @@ function evidenceAudio(basis: SpeechBasis): SpeechEvidenceAudio {
});
}
-function audioBasis(value: SpeechBasis): SpeechAudioBasis {
- return {
- programSpace: value.programSpace,
- audio: value.audio,
- segments: value.segments,
- };
-}
-
-test("WhisperX receives normalized bytes and Segment truth through separate graph inputs", () => {
+test("WhisperX receives normalized bytes without authored Segment truth", () => {
const source = basis();
const valid = evidenceAudio(source);
- const request = whisperXRequestForEvidenceAudio(valid, audioBasis(source));
+ const request = whisperXRequestForEvidenceAudio(valid);
assert.equal(request.audio.digest, valid.artifact.digest);
- assert.deepEqual(request.segments, source.segments);
+ assert.equal("segments" in request, false);
assert.equal(request.sampleFrames, 16_000);
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index e6c05e77..344b6903 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -2038,9 +2038,6 @@ importers:
'@narratage/runtime-adapter':
specifier: workspace:*
version: link:../runtime-adapter
- '@narratage/speech':
- specifier: workspace:*
- version: link:../speech
'@narratage/speech-evidence':
specifier: workspace:*
version: link:../speech-evidence