Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

refactor(runtime): finish portable local execution

kashorinff72abd5650f
diff --git a/.gitignore b/.gitignore
index 8608dd74..31527f1d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ coverage/
*.log
*.tmp
*.temp
+*.lock.editing
# Local editor and agent state is never part of the distribution.
.claude/
@@ -36,6 +37,7 @@ playwright-report/
test-results/
docs/.vitepress/cache/
docs/.vitepress/dist/
+*.tsbuildinfo
.svml-cache/
examples/**/assets/
examples/**/build/
diff --git a/examples/talking-film-graph-check/svml.packages.lock b/examples/talking-film-graph-check/svml.packages.lock
index 2ce3439a..eaa02f72 100644
--- a/examples/talking-film-graph-check/svml.packages.lock
+++ b/examples/talking-film-graph-check/svml.packages.lock
@@ -637,12 +637,12 @@
{
"name": "@narratage/host",
"version": "0.0.0-dev",
- "digest": "sha256:3cbc10d27b16c44ba82a29db27b0a394574d9088184e889558f2b10f1d143e08"
+ "digest": "sha256:daf8c9f8cb8e380a8bcf3000d96fca8431da37ae5e6b3f2321fed5e4cd8fcf17"
},
{
"name": "@narratage/hyperframes",
"version": "0.0.0-dev",
- "digest": "sha256:dde9143aed4fa8731a1a91f860bb2a8a9f426c7515b5c12bed8b2fac24c0a416"
+ "digest": "sha256:586ea608ae520e5b5b0a5950a251fe8a4c67ec8f3c89fcec32d968e3cf3a6a17"
},
{
"name": "@narratage/markup",
@@ -802,7 +802,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:6a2991ec75f9fd30e8d286213fc7993aae2c570b85a8faae5d3c2af661c9fdfc",
- "closureDigest": "sha256:45c0adc0dbc045f12283db7da9b97a6d230813f508f512f91dfee5aedabdb401"
+ "closureDigest": "sha256:ee79cd66af6e561432e52f1bf78f597f9dc01cd09d12d68c26e7cf61e7361aac"
},
{
"specifier": "@narratage/caption-fine",
@@ -811,7 +811,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:c5616408dd56ce9c903523319b2a697ff6b8af5bfe1dc66fe793a11e4629f3cf",
- "closureDigest": "sha256:9331f6dcf950a7bd001cf28a4f866f36f9f6a0a97910c5f8355bba9724144c71"
+ "closureDigest": "sha256:782a9f2c0345431e06c8ab009d518933c964465b34d790b4064f87170e0495ff"
},
{
"specifier": "@narratage/caption-gemini",
@@ -820,7 +820,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:22fed1f1748349c11b6f28448be38cbea68b1dd5a51a3dc2d548ae0a376b9c71",
- "closureDigest": "sha256:0206e5a701c6feb1b60a9b73cf8bb74dfed7bc1e321b3e38b495590cdcf4ac24"
+ "closureDigest": "sha256:3cc6fd7d5250e5457db548fa2432f33d62012db8f77e1c8fe4f590e38a3439ab"
},
{
"specifier": "@narratage/composition",
@@ -829,7 +829,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:0a988b665baf0846246e309d9faf39ffb3ef6ec455bf37f7c438eb30429ff46e",
- "closureDigest": "sha256:37f34e3ddef9f2e86c8a9d062fd5a4523874841db3a5a51444b5c516a0c63817"
+ "closureDigest": "sha256:15c9088a176b04a6051b2398e308bde8fab6fb15d0147786e9e40abfce772662"
},
{
"specifier": "@narratage/film",
@@ -838,7 +838,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:8e58b5630b9a68a8ce7314620f5f1c7cda973a05c179c18d61e10d68002964f0",
- "closureDigest": "sha256:2d0ca309cdfb053c6fc47d3f3464c8cb3e50ee601e3c6951484e34529aeab8a6"
+ "closureDigest": "sha256:2a4b2b66d9f3913bc839f052e35e6c5cd6afed08821a3308c643b16dcd011f44"
},
{
"specifier": "@narratage/fonts-open",
@@ -847,7 +847,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:31b51352edcb4f34beccc9d9ac8cf7397125a840971a72c912df3d34d1f3feda",
- "closureDigest": "sha256:6c3f7a28db8159c5ba03528fd4c85f1b97718fe719d3a087312f1a1d8f18b160"
+ "closureDigest": "sha256:4594c947a3425c6c42738e2e7d0f7ce56a6efaced91fcf6085e282c3ad98b797"
},
{
"specifier": "@narratage/generation",
@@ -856,7 +856,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:d74a57a3210267e665366513324db5c313b3ca0f770da66e74cc38aee5048efb",
- "closureDigest": "sha256:d8c63f5908ba26809179c0e1ccf04b87b9c945270733947d11262bb2b0f7916c"
+ "closureDigest": "sha256:3bd4bfdd124001b83602627b0305d6652e96f43cbb9345e3b9bece51a3d701ce"
},
{
"specifier": "@narratage/hyperframes",
@@ -865,7 +865,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:b8b8773d03f19844cdc8d714650380145056649245e33c607f30220fff194157",
- "closureDigest": "sha256:9077ffd048007617708b988f285dcef6241b277cabc1ba472ccf30ac20b2356f"
+ "closureDigest": "sha256:88d7a9fedb614166e149e29d064be1dca18f1c4a102759f1a2600afba4953b8b"
},
{
"specifier": "@narratage/media",
@@ -874,7 +874,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:6c7cf563a92aa8e48db6da224697fd5f56c50b636b0327efc17fbbc264502b3f",
- "closureDigest": "sha256:2d56835dce5d6c4018cb8b6499771bba422f2e95ab67aa62500ca59f53672e23"
+ "closureDigest": "sha256:e8edbbfb7bec2efa7ad05fa273549a3b033861eb9ef20345430d9be617b26674"
},
{
"specifier": "@narratage/media-pipeline",
@@ -883,7 +883,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:21f817f8e6eea9e9e4361005ed5ab9e69117fd77ec237fd467fb11afe9965019",
- "closureDigest": "sha256:b5462108e446bc5a22c3888a24bf0f99652115cc3d69f3cb868b6c7305b8e324"
+ "closureDigest": "sha256:3555f1939c82dfa216902fd600e2616ecfce6a244e240b060080621367333d97"
},
{
"specifier": "@narratage/media-track",
@@ -892,7 +892,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:bb26c54c0ad8802ee8c7cd6f13314c90dd7de7e937ded1c501367e19693fe3b1",
- "closureDigest": "sha256:1619d3e8c8ec147e6ac1355f409e0c81f0ee66db46d876fc78cea925593e95d0"
+ "closureDigest": "sha256:5b8e32d627c9f55e5dbe8a4a85348b44852070f2e6796e3d19bbd7a0e42a49b6"
},
{
"specifier": "@narratage/narrative",
@@ -919,7 +919,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:aa47f04cf0652ee3fa2e114eb6d9a80b0522828a47869103ca33e95027b21634",
- "closureDigest": "sha256:b1c4fd45e70759f2597a980c1ca2097d44fc0a0c8e596b35317be04fa96fa7ba"
+ "closureDigest": "sha256:1c9dcb543a3afa6a94c9757fd47a3e0bfb68ed7d5f3c87b3ef0f0e52ed286332"
},
{
"specifier": "@narratage/run-markup",
@@ -928,7 +928,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:fb3a8a5be9a0ef0aa914026a199be641f4e20df27f9b284be196f52fd260be55",
- "closureDigest": "sha256:e1a5faf90e0925ff8100b2e32c188d1741aac3bc376fbc94183c5c9eeafc195b"
+ "closureDigest": "sha256:20b6fcbacf36352d6c180e1a466eaf368c0bef183eec09396456e2e2375ea16f"
},
{
"specifier": "@narratage/script",
@@ -937,7 +937,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:69e1c08a5f07c73a506ca8286426a0386485738649e86463cca0b291f319ee4e",
- "closureDigest": "sha256:070562efce768b46aca3960561b2bc3a592997192f7a8e9214bf8ec289df83b9"
+ "closureDigest": "sha256:696bb2e69c89d9e1079a76ce68d26fe2c99615275a4266bf24fa905ad77c1bb4"
},
{
"specifier": "@narratage/seedance",
@@ -946,7 +946,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:e889ca3808a19f544e7aa5e27453beb5a09464c645633b0b1b1d9a4b7ce6591a",
- "closureDigest": "sha256:55d4043e4ca8f85cd5eea781eb07801f1869443541d260f2ab427c78e93f5b2b"
+ "closureDigest": "sha256:e1c45192a92620ec8ea03d9cf1b19073bae2dbfde12475f8436cd5c008c854bd"
},
{
"specifier": "@narratage/semantic-map",
@@ -955,7 +955,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:a9d81ddfa09dfa23c9c3c0817b60299431ca9ec640252a659d490be55d92ba68",
- "closureDigest": "sha256:658b7ef5306d5ec68c172ce19543eb05fd8e1f616a10223acfb87223012e440e"
+ "closureDigest": "sha256:3ac5deb912231d3092666c692d73a379129d1cf2ed035d43a95bef6d5101d525"
},
{
"specifier": "@narratage/spatial",
@@ -964,7 +964,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:5062281ffadb46495ec12e1a67185b57fe13879e90f03bcf544e93fd1d273716",
- "closureDigest": "sha256:0bfb0066a25c344a6e74aa8e2439300179e55ffbf6a8d13e84e526d4123a59ce"
+ "closureDigest": "sha256:3209c05b608c20aedab35299235974d74ee4374ead722ec125f2b6ea70829c77"
},
{
"specifier": "@narratage/speech",
@@ -973,7 +973,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:75b1537c9b9729c2e9197445c34b295bc9f72dbef754a4702e1ca792dceea472",
- "closureDigest": "sha256:e9a563703c2f7c72777c617067444ef4cb036ff857fbbe65161314255fb102f1"
+ "closureDigest": "sha256:a5be05750141d51e4f4460945b779bf0196548d7b7c908eca34d8c63c2ec7b70"
},
{
"specifier": "@narratage/speech-alignment",
@@ -982,7 +982,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:bdfe219081310041d11c6d54440e6ccb80398bfb97f93afc28e9d1ed101b4a79",
- "closureDigest": "sha256:400256232c983f05e8ceec145ea790e2ec15b5dad91d10ca340f15514b6067ff"
+ "closureDigest": "sha256:de73bc9d1411af8319a099df658dfc51f6e7b44dcbf3173cb5f9db18edb449cf"
},
{
"specifier": "@narratage/speech-basis",
@@ -991,7 +991,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:f01f21badb0ccdd5cc62b131be05f46030bffee96042c7323a0c572786d012ef",
- "closureDigest": "sha256:bd0ad1414559f1b6da7c8c5aa60c16c3f3b2634b9151d7281bc4ebb274abd82b"
+ "closureDigest": "sha256:843a7922a70d5df180ade4ec3a8becf8fb110f6137fa6a38d86cc5aaa14830e1"
},
{
"specifier": "@narratage/speech-evidence",
@@ -1000,7 +1000,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:60f366de074a7fba20c71dcbd825083fddc54ea7de602a903eb144eab5555425",
- "closureDigest": "sha256:319c2bc5bcda4734e4c975304ae9d24796e011c4dabe245dfedb96e4b06228b0"
+ "closureDigest": "sha256:918172938ecc1e9b6b1cff9465048bdfe2dcbfe8748971101f78f12ee36f1d75"
},
{
"specifier": "@narratage/speech-spine",
@@ -1009,7 +1009,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:a27dd4349d6bb1aaa7f382ad09f883e9894157579e48285b00e8fa24456190f7",
- "closureDigest": "sha256:8c2c4fe30acea7d039e53aac7cc512c765e81e92cb210b538afe0cc3c2c253a5"
+ "closureDigest": "sha256:f056e14a44f4f2cc0751b28e1ecb9831b7ff472034a02cac8089b9ff45d67854"
},
{
"specifier": "@narratage/svs",
@@ -1027,7 +1027,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:2e9c4804f142e08e87b2e11af79aa17770109fb573e865397bed14ed7a4f1d0e",
- "closureDigest": "sha256:5762aa27d8add5f1dcf8f18f368d83e53cd26ee50480071e67b1e31924458fc3"
+ "closureDigest": "sha256:7a2df8dba92e1005388b373be07f4fe3065bb3d4b7106e19139a4fe1cadc090b"
},
{
"specifier": "@narratage/text",
@@ -1036,7 +1036,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:4435dd4bd38e5e017013f775e28e379e3e235211649ee8f3913378f7f9e60b58",
- "closureDigest": "sha256:82d2119cfc67634753bafcdb26003953f5d85519fbada9a5ce88b5a312e31a28"
+ "closureDigest": "sha256:c6ba3301621d63598421b7d3f0dc2e1bcc8967f16624407d37a20c2c0703a246"
},
{
"specifier": "@narratage/typography-track",
@@ -1045,7 +1045,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:cc8349fec027f43e01a8832ba8a7050490e924851cee3325d3568cb5bc5fd57f",
- "closureDigest": "sha256:2ca5685baa5d2fabd12ca79f86f7c2b79c8fa74c8d32f7d664bc4d5af3627c35"
+ "closureDigest": "sha256:840cec885e08123174809ef2dcf357120203b3fb5e2af3b22e0c6177d2e61af1"
},
{
"specifier": "@narratage/visual-ir",
@@ -1063,8 +1063,8 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:c8cb487d25e567a081c74a7abc97ce2b94dc0c47c36c90ba6b44d2ec21a9e80b",
- "closureDigest": "sha256:afc3d88b322d8c5aeebd520d5d691531d5314db0e87b19f0564941c3477f9e8a"
+ "closureDigest": "sha256:98e68de949bf7cfb9e1c483de66a38de6f347fc5032122d10e9af971a28fa534"
}
],
- "digest": "sha256:3df2354df92469461a8107654dc5319637f8f05be7ce89fca17a22c89694db87"
+ "digest": "sha256:c2d6ceb3e801c9045047d59040be4e3f9a297f3a245f51cc8425ce7e5d9c3043"
}
diff --git a/examples/talking-film-live/svml.packages.lock b/examples/talking-film-live/svml.packages.lock
index dab0c538..71244e89 100644
--- a/examples/talking-film-live/svml.packages.lock
+++ b/examples/talking-film-live/svml.packages.lock
@@ -643,12 +643,12 @@
{
"name": "@narratage/host",
"version": "0.0.0-dev",
- "digest": "sha256:3cbc10d27b16c44ba82a29db27b0a394574d9088184e889558f2b10f1d143e08"
+ "digest": "sha256:daf8c9f8cb8e380a8bcf3000d96fca8431da37ae5e6b3f2321fed5e4cd8fcf17"
},
{
"name": "@narratage/hyperframes",
"version": "0.0.0-dev",
- "digest": "sha256:dde9143aed4fa8731a1a91f860bb2a8a9f426c7515b5c12bed8b2fac24c0a416"
+ "digest": "sha256:586ea608ae520e5b5b0a5950a251fe8a4c67ec8f3c89fcec32d968e3cf3a6a17"
},
{
"name": "@narratage/markup",
@@ -813,7 +813,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:6a2991ec75f9fd30e8d286213fc7993aae2c570b85a8faae5d3c2af661c9fdfc",
- "closureDigest": "sha256:45c0adc0dbc045f12283db7da9b97a6d230813f508f512f91dfee5aedabdb401"
+ "closureDigest": "sha256:ee79cd66af6e561432e52f1bf78f597f9dc01cd09d12d68c26e7cf61e7361aac"
},
{
"specifier": "@narratage/caption-fine",
@@ -822,7 +822,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:c5616408dd56ce9c903523319b2a697ff6b8af5bfe1dc66fe793a11e4629f3cf",
- "closureDigest": "sha256:9331f6dcf950a7bd001cf28a4f866f36f9f6a0a97910c5f8355bba9724144c71"
+ "closureDigest": "sha256:782a9f2c0345431e06c8ab009d518933c964465b34d790b4064f87170e0495ff"
},
{
"specifier": "@narratage/caption-gemini",
@@ -831,7 +831,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:22fed1f1748349c11b6f28448be38cbea68b1dd5a51a3dc2d548ae0a376b9c71",
- "closureDigest": "sha256:0206e5a701c6feb1b60a9b73cf8bb74dfed7bc1e321b3e38b495590cdcf4ac24"
+ "closureDigest": "sha256:3cc6fd7d5250e5457db548fa2432f33d62012db8f77e1c8fe4f590e38a3439ab"
},
{
"specifier": "@narratage/composition",
@@ -840,7 +840,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:0a988b665baf0846246e309d9faf39ffb3ef6ec455bf37f7c438eb30429ff46e",
- "closureDigest": "sha256:37f34e3ddef9f2e86c8a9d062fd5a4523874841db3a5a51444b5c516a0c63817"
+ "closureDigest": "sha256:15c9088a176b04a6051b2398e308bde8fab6fb15d0147786e9e40abfce772662"
},
{
"specifier": "@narratage/estimate",
@@ -849,7 +849,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:9975815b3109520eb6bba276554a90767a9aebff6498c2768d9faf4763219bef",
- "closureDigest": "sha256:811c0d891e6754d5553f6e3f7f04f5a77a6bc0134fdb8803319b9502a7073bad"
+ "closureDigest": "sha256:7bbdc658746e65e2bb4199b3de84d1197376f16120708e2eea31558898de662f"
},
{
"specifier": "@narratage/film",
@@ -858,7 +858,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:8e58b5630b9a68a8ce7314620f5f1c7cda973a05c179c18d61e10d68002964f0",
- "closureDigest": "sha256:2d0ca309cdfb053c6fc47d3f3464c8cb3e50ee601e3c6951484e34529aeab8a6"
+ "closureDigest": "sha256:2a4b2b66d9f3913bc839f052e35e6c5cd6afed08821a3308c643b16dcd011f44"
},
{
"specifier": "@narratage/fonts-open",
@@ -867,7 +867,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:31b51352edcb4f34beccc9d9ac8cf7397125a840971a72c912df3d34d1f3feda",
- "closureDigest": "sha256:6c3f7a28db8159c5ba03528fd4c85f1b97718fe719d3a087312f1a1d8f18b160"
+ "closureDigest": "sha256:4594c947a3425c6c42738e2e7d0f7ce56a6efaced91fcf6085e282c3ad98b797"
},
{
"specifier": "@narratage/generation",
@@ -876,7 +876,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:d74a57a3210267e665366513324db5c313b3ca0f770da66e74cc38aee5048efb",
- "closureDigest": "sha256:d8c63f5908ba26809179c0e1ccf04b87b9c945270733947d11262bb2b0f7916c"
+ "closureDigest": "sha256:3bd4bfdd124001b83602627b0305d6652e96f43cbb9345e3b9bece51a3d701ce"
},
{
"specifier": "@narratage/hyperframes",
@@ -885,7 +885,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:b8b8773d03f19844cdc8d714650380145056649245e33c607f30220fff194157",
- "closureDigest": "sha256:9077ffd048007617708b988f285dcef6241b277cabc1ba472ccf30ac20b2356f"
+ "closureDigest": "sha256:88d7a9fedb614166e149e29d064be1dca18f1c4a102759f1a2600afba4953b8b"
},
{
"specifier": "@narratage/media",
@@ -894,7 +894,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:6c7cf563a92aa8e48db6da224697fd5f56c50b636b0327efc17fbbc264502b3f",
- "closureDigest": "sha256:2d56835dce5d6c4018cb8b6499771bba422f2e95ab67aa62500ca59f53672e23"
+ "closureDigest": "sha256:e8edbbfb7bec2efa7ad05fa273549a3b033861eb9ef20345430d9be617b26674"
},
{
"specifier": "@narratage/media-pipeline",
@@ -903,7 +903,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:21f817f8e6eea9e9e4361005ed5ab9e69117fd77ec237fd467fb11afe9965019",
- "closureDigest": "sha256:b5462108e446bc5a22c3888a24bf0f99652115cc3d69f3cb868b6c7305b8e324"
+ "closureDigest": "sha256:3555f1939c82dfa216902fd600e2616ecfce6a244e240b060080621367333d97"
},
{
"specifier": "@narratage/media-track",
@@ -912,7 +912,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:bb26c54c0ad8802ee8c7cd6f13314c90dd7de7e937ded1c501367e19693fe3b1",
- "closureDigest": "sha256:1619d3e8c8ec147e6ac1355f409e0c81f0ee66db46d876fc78cea925593e95d0"
+ "closureDigest": "sha256:5b8e32d627c9f55e5dbe8a4a85348b44852070f2e6796e3d19bbd7a0e42a49b6"
},
{
"specifier": "@narratage/narrative",
@@ -939,7 +939,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:aa47f04cf0652ee3fa2e114eb6d9a80b0522828a47869103ca33e95027b21634",
- "closureDigest": "sha256:b1c4fd45e70759f2597a980c1ca2097d44fc0a0c8e596b35317be04fa96fa7ba"
+ "closureDigest": "sha256:1c9dcb543a3afa6a94c9757fd47a3e0bfb68ed7d5f3c87b3ef0f0e52ed286332"
},
{
"specifier": "@narratage/run-markup",
@@ -948,7 +948,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:fb3a8a5be9a0ef0aa914026a199be641f4e20df27f9b284be196f52fd260be55",
- "closureDigest": "sha256:e1a5faf90e0925ff8100b2e32c188d1741aac3bc376fbc94183c5c9eeafc195b"
+ "closureDigest": "sha256:20b6fcbacf36352d6c180e1a466eaf368c0bef183eec09396456e2e2375ea16f"
},
{
"specifier": "@narratage/script",
@@ -957,7 +957,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:69e1c08a5f07c73a506ca8286426a0386485738649e86463cca0b291f319ee4e",
- "closureDigest": "sha256:070562efce768b46aca3960561b2bc3a592997192f7a8e9214bf8ec289df83b9"
+ "closureDigest": "sha256:696bb2e69c89d9e1079a76ce68d26fe2c99615275a4266bf24fa905ad77c1bb4"
},
{
"specifier": "@narratage/seedance",
@@ -966,7 +966,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:e889ca3808a19f544e7aa5e27453beb5a09464c645633b0b1b1d9a4b7ce6591a",
- "closureDigest": "sha256:55d4043e4ca8f85cd5eea781eb07801f1869443541d260f2ab427c78e93f5b2b"
+ "closureDigest": "sha256:e1c45192a92620ec8ea03d9cf1b19073bae2dbfde12475f8436cd5c008c854bd"
},
{
"specifier": "@narratage/semantic-map",
@@ -975,7 +975,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:a9d81ddfa09dfa23c9c3c0817b60299431ca9ec640252a659d490be55d92ba68",
- "closureDigest": "sha256:658b7ef5306d5ec68c172ce19543eb05fd8e1f616a10223acfb87223012e440e"
+ "closureDigest": "sha256:3ac5deb912231d3092666c692d73a379129d1cf2ed035d43a95bef6d5101d525"
},
{
"specifier": "@narratage/spatial",
@@ -984,7 +984,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:5062281ffadb46495ec12e1a67185b57fe13879e90f03bcf544e93fd1d273716",
- "closureDigest": "sha256:0bfb0066a25c344a6e74aa8e2439300179e55ffbf6a8d13e84e526d4123a59ce"
+ "closureDigest": "sha256:3209c05b608c20aedab35299235974d74ee4374ead722ec125f2b6ea70829c77"
},
{
"specifier": "@narratage/speech",
@@ -993,7 +993,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:75b1537c9b9729c2e9197445c34b295bc9f72dbef754a4702e1ca792dceea472",
- "closureDigest": "sha256:e9a563703c2f7c72777c617067444ef4cb036ff857fbbe65161314255fb102f1"
+ "closureDigest": "sha256:a5be05750141d51e4f4460945b779bf0196548d7b7c908eca34d8c63c2ec7b70"
},
{
"specifier": "@narratage/speech-alignment",
@@ -1002,7 +1002,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:bdfe219081310041d11c6d54440e6ccb80398bfb97f93afc28e9d1ed101b4a79",
- "closureDigest": "sha256:400256232c983f05e8ceec145ea790e2ec15b5dad91d10ca340f15514b6067ff"
+ "closureDigest": "sha256:de73bc9d1411af8319a099df658dfc51f6e7b44dcbf3173cb5f9db18edb449cf"
},
{
"specifier": "@narratage/speech-basis",
@@ -1011,7 +1011,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:f01f21badb0ccdd5cc62b131be05f46030bffee96042c7323a0c572786d012ef",
- "closureDigest": "sha256:bd0ad1414559f1b6da7c8c5aa60c16c3f3b2634b9151d7281bc4ebb274abd82b"
+ "closureDigest": "sha256:843a7922a70d5df180ade4ec3a8becf8fb110f6137fa6a38d86cc5aaa14830e1"
},
{
"specifier": "@narratage/speech-evidence",
@@ -1020,7 +1020,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:60f366de074a7fba20c71dcbd825083fddc54ea7de602a903eb144eab5555425",
- "closureDigest": "sha256:319c2bc5bcda4734e4c975304ae9d24796e011c4dabe245dfedb96e4b06228b0"
+ "closureDigest": "sha256:918172938ecc1e9b6b1cff9465048bdfe2dcbfe8748971101f78f12ee36f1d75"
},
{
"specifier": "@narratage/speech-spine",
@@ -1029,7 +1029,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:a27dd4349d6bb1aaa7f382ad09f883e9894157579e48285b00e8fa24456190f7",
- "closureDigest": "sha256:8c2c4fe30acea7d039e53aac7cc512c765e81e92cb210b538afe0cc3c2c253a5"
+ "closureDigest": "sha256:f056e14a44f4f2cc0751b28e1ecb9831b7ff472034a02cac8089b9ff45d67854"
},
{
"specifier": "@narratage/svs",
@@ -1047,7 +1047,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:2e9c4804f142e08e87b2e11af79aa17770109fb573e865397bed14ed7a4f1d0e",
- "closureDigest": "sha256:5762aa27d8add5f1dcf8f18f368d83e53cd26ee50480071e67b1e31924458fc3"
+ "closureDigest": "sha256:7a2df8dba92e1005388b373be07f4fe3065bb3d4b7106e19139a4fe1cadc090b"
},
{
"specifier": "@narratage/text",
@@ -1056,7 +1056,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:4435dd4bd38e5e017013f775e28e379e3e235211649ee8f3913378f7f9e60b58",
- "closureDigest": "sha256:82d2119cfc67634753bafcdb26003953f5d85519fbada9a5ce88b5a312e31a28"
+ "closureDigest": "sha256:c6ba3301621d63598421b7d3f0dc2e1bcc8967f16624407d37a20c2c0703a246"
},
{
"specifier": "@narratage/typography-track",
@@ -1065,7 +1065,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:cc8349fec027f43e01a8832ba8a7050490e924851cee3325d3568cb5bc5fd57f",
- "closureDigest": "sha256:2ca5685baa5d2fabd12ca79f86f7c2b79c8fa74c8d32f7d664bc4d5af3627c35"
+ "closureDigest": "sha256:840cec885e08123174809ef2dcf357120203b3fb5e2af3b22e0c6177d2e61af1"
},
{
"specifier": "@narratage/visual-ir",
@@ -1083,8 +1083,8 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:c8cb487d25e567a081c74a7abc97ce2b94dc0c47c36c90ba6b44d2ec21a9e80b",
- "closureDigest": "sha256:afc3d88b322d8c5aeebd520d5d691531d5314db0e87b19f0564941c3477f9e8a"
+ "closureDigest": "sha256:98e68de949bf7cfb9e1c483de66a38de6f347fc5032122d10e9af971a28fa534"
}
],
- "digest": "sha256:bcd2e047a42c40e065bbb8d444e8f88f8c28888c6315572dabd7a569c6e2371e"
+ "digest": "sha256:a224a78bd87743c777eadf978452e86d4cba400b1cf5075057cf0aafac26a406"
}
diff --git a/examples/talking-film-live/svml.runtime-packages.lock b/examples/talking-film-live/svml.runtime-packages.lock
index 7092057f..ef94a1d9 100644
--- a/examples/talking-film-live/svml.runtime-packages.lock
+++ b/examples/talking-film-live/svml.runtime-packages.lock
@@ -40,7 +40,7 @@
{
"name": "@narratage/artifact-store-fs",
"version": "0.0.0-dev",
- "digest": "sha256:a9d1cd5699df848cf89887897ece565cedd25af1eef1cac453e5b35caa52c3f2"
+ "digest": "sha256:ee4542f46f20c58f66b35c8683cd192026fb32a19157fd237fb365ab5a77ab39"
},
{
"name": "@narratage/artifact",
@@ -65,7 +65,7 @@
{
"name": "@narratage/compiler-node",
"version": "0.0.0-dev",
- "digest": "sha256:019d7603c28d726e9fc8127e5786c2cd22575a8faa84d91dfd77139fda0927af"
+ "digest": "sha256:4626882f405a083281ce7ffec73fa598640223669908b36715d8281fa19be92e"
},
{
"name": "@narratage/component-kit",
@@ -90,7 +90,7 @@
{
"name": "@narratage/driver-node",
"version": "0.0.0-dev",
- "digest": "sha256:9f7df9d719422b91ac6d51995a6657d2b01cc32f80204a98240c9102ac278791"
+ "digest": "sha256:0e1209c7f242956f55f90e712a7921ba63fe0e9a20a6164461afa5215d8bcba2"
},
{
"name": "@narratage/elaborator",
@@ -100,7 +100,7 @@
{
"name": "@narratage/endpoint-kit",
"version": "0.0.0-dev",
- "digest": "sha256:94b455390d37d675a55c1888cc4b665d777f017e279510852cb5033436aee3ed"
+ "digest": "sha256:833d4c2b40cbf889c4790476a92a7e3eb9ddf42b748fd17bc7f9ca9bc8596c14"
},
{
"name": "@narratage/generation",
@@ -110,17 +110,17 @@
{
"name": "@narratage/host",
"version": "0.0.0-dev",
- "digest": "sha256:3cbc10d27b16c44ba82a29db27b0a394574d9088184e889558f2b10f1d143e08"
+ "digest": "sha256:daf8c9f8cb8e380a8bcf3000d96fca8431da37ae5e6b3f2321fed5e4cd8fcf17"
},
{
"name": "@narratage/hyperframes",
"version": "0.0.0-dev",
- "digest": "sha256:dde9143aed4fa8731a1a91f860bb2a8a9f426c7515b5c12bed8b2fac24c0a416"
+ "digest": "sha256:586ea608ae520e5b5b0a5950a251fe8a4c67ec8f3c89fcec32d968e3cf3a6a17"
},
{
"name": "@narratage/local",
"version": "0.0.0-dev",
- "digest": "sha256:f833c604b93c46f7596951d41e89ddb630dad1404bee6029c44ed541c437d151"
+ "digest": "sha256:cfbd3c13a81dbc21378bebd22662e93a201480e1bfe725f04e78534764f86c0e"
},
{
"name": "@narratage/markup",
@@ -130,7 +130,7 @@
{
"name": "@narratage/media-execution",
"version": "0.0.0-dev",
- "digest": "sha256:2bca4da3fe0325847811888d2ded010761d4dc447b8ec79bec9710bca5011c3e"
+ "digest": "sha256:8fe6aca65ca4ebcc1efa71376e986e97741310d836f85647fcb223f39fda9754"
},
{
"name": "@narratage/media-pipeline",
@@ -150,7 +150,7 @@
{
"name": "@narratage/package-loader-node",
"version": "0.0.0-dev",
- "digest": "sha256:1296c0ef3170e77cfdaa8cde0e1753fad2822b254507e0978429498d84c73ed0"
+ "digest": "sha256:db78f8abd9a08a7655f7ed2d24860b1f848b3763421984847c303afa316b9003"
},
{
"name": "@narratage/program-space",
@@ -165,27 +165,27 @@
{
"name": "@narratage/provider-google-vertex",
"version": "0.0.0-dev",
- "digest": "sha256:b9de2ae61195991b65626a51053e71d79747b37274dd24dfbfdd0b855df455bd"
+ "digest": "sha256:e6f2594a0636b527fa7472c66fb3134eaa92d7e9903a8cebf0065fc7b4af4377"
},
{
"name": "@narratage/provider-hyperframes-local",
"version": "0.0.0-dev",
- "digest": "sha256:218adf5c62f11a4490b9a02a9cc1087f5eb2d493c3d99b16682cf70dbb2704f7"
+ "digest": "sha256:1d3c090bc9d13fc266b65120f3e0439be824b1d05d4746a670aa929da49a560b"
},
{
"name": "@narratage/provider-kie",
"version": "0.0.0-dev",
- "digest": "sha256:22612e3da4f12a5b780bcca5de5bf5585370772343aea749329715afe1988a88"
+ "digest": "sha256:47fd2c4c57dc381b21f9622002d77f7db7003c52de8c70e1ca295ae13cebdb18"
},
{
"name": "@narratage/provider-media-local",
"version": "0.0.0-dev",
- "digest": "sha256:241bfaeacef8ec59694c127bc2dacf8cbac4fd48d10cbe04d1f2d0aa25f84d36"
+ "digest": "sha256:9bfc485ddbdeca9cd747925b4ef04f85315e41746a2cf4a99ac0d96baf237014"
},
{
"name": "@narratage/provider-whisperx-local",
"version": "0.0.0-dev",
- "digest": "sha256:2aea383b431858033dd8a27ede705d99cbafb3d55cce0971c7bca09af20d3df2"
+ "digest": "sha256:3fea223b5b0d1569b2c04e7949c82b6eeaa770caa41b19b461ed035e30d8c29e"
},
{
"name": "@narratage/render-hyperframes",
@@ -205,12 +205,12 @@
{
"name": "@narratage/runtime-adapter",
"version": "0.0.0-dev",
- "digest": "sha256:357fc3c81876d25e553eaa555dd3308ff3db51715d230e6dd44573667058add2"
+ "digest": "sha256:7bb0b1ef43eb3fa2e67725c990b17f585a2fccd98fc71eae6c7a8da29b3e06a4"
},
{
"name": "@narratage/runtime",
"version": "0.0.0-dev",
- "digest": "sha256:0f383c247e08abb30ab69ef6ce9d6bbf8b0f8fabc72b057ac97970c7f99ed4b2"
+ "digest": "sha256:b7954bf727125baedc31377d3a1d9066b5bd9c019c72fe6468fcc4864243aa31"
},
{
"name": "@narratage/semantic-map",
@@ -245,7 +245,7 @@
{
"name": "@narratage/store-sqlite",
"version": "0.0.0-dev",
- "digest": "sha256:cf7e739e38b5d7094488cd3da0aaae16f66cdb229ed48ec59999ccbe381aedcd"
+ "digest": "sha256:a6063abc900ba2e608db91ab9b587306af3bb843627f5f95c242668ca9abdd7d"
},
{
"name": "@narratage/svs",
@@ -275,7 +275,7 @@
{
"name": "@narratage/workspace-fs-node",
"version": "0.0.0-dev",
- "digest": "sha256:3ed827c6a586ccb5a9bc023ccea5c05a1fbe1a317a79df88314073c18cfa8ea3"
+ "digest": "sha256:1c2ec730e80c5cb9ca504f433076aeccf855ee9a32cb4fdcef68d8914b24349d"
},
{
"name": "@protobufjs/aspromise",
@@ -926,7 +926,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:914018276b9218afc13940ca5c362dc95fa19d574fc07b06b6fe005c5eb30315",
- "closureDigest": "sha256:486228772cda98bfe09c6262bc6f06f6cc5601928856c184087d1719d7497e49"
+ "closureDigest": "sha256:fadbd7bdb4fb5af49e762cb92c1e22d0cf3e2bf10fd5f0de3d594ef3f0ca8d93"
},
{
"specifier": "@narratage/credential-store-env",
@@ -935,7 +935,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:ee33ba638bbf2f88eee8f34a2f3179f435a0af9ccbae3fbb7d2a81bd6aeac612",
- "closureDigest": "sha256:dee6b5e342b981185fce3dd86112d7a30bc11e0aa7132f4c4cbfa9ccfd086dda"
+ "closureDigest": "sha256:d416ef4d798b0948dcc2cd16c962784297b6e2f20daf8626f28c5d90e37b1dec"
},
{
"specifier": "@narratage/local",
@@ -944,7 +944,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:6689aed79b0947c7315aef65e46806ce73da01de3d73b454e1497b8d847ffa61",
- "closureDigest": "sha256:4a22531c658b24975a7b1c083102bc90bd107d97c20286da818a9f964c6bbc86"
+ "closureDigest": "sha256:2e6704a8d53b79c750c1dc1ada0196737e13cb1e41bec5fac233abbd62b70326"
},
{
"specifier": "@narratage/provider-google-vertex",
@@ -953,7 +953,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:b2636492ab0b0b500d674899ab2ff14fc61c7f2b22ef44eb8f816f141b21ec4d",
- "closureDigest": "sha256:8b92e7871fa616ed3a6adce14a6bb7d705532dc0807e453492cd8a878558d9a7"
+ "closureDigest": "sha256:fdfd1b6d25ae82b3eca28bc7f1e94ca2a4c2930f6171388afb861ca9a06990da"
},
{
"specifier": "@narratage/provider-hyperframes-local",
@@ -962,7 +962,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:b14b721bdc0c9d5d9ced1c345107c92a3a5878f3383dea2ecd3e310442c2a40c",
- "closureDigest": "sha256:1102b66d7613c516cbd67ded83961b8bd96ad0d45772fe185c28962084692daf"
+ "closureDigest": "sha256:22819933ee6b109e794ca94cee669cddf3288c8fe5c3d4e7e2bd727fdeab8bda"
},
{
"specifier": "@narratage/provider-kie",
@@ -971,7 +971,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:e61ebb911a33d4c5d0e4e4ee4fb69cdb790f4ced050ef35f7472bfbd72013d9b",
- "closureDigest": "sha256:b6d39b1697767893f05c27d7d3c2b5e683e9c559d020c8de0a412ec76226cea0"
+ "closureDigest": "sha256:90589aebf0631d19b59f258bd1e4c20638f4325f946260aa71968350e9e4efc0"
},
{
"specifier": "@narratage/provider-media-local",
@@ -980,7 +980,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:701abcc923bf8a711c20b6e308d8cc8be27bf560f249b8c5a801e1dbd9cb3979",
- "closureDigest": "sha256:8704836ca23624ca38023f5a3b0d7427e817a38412880e97dc5932fb7268d493"
+ "closureDigest": "sha256:f40546545b842197e77982ceba94c7f52f2389bad33d68ff98ad62e28eb576e1"
},
{
"specifier": "@narratage/provider-whisperx-local",
@@ -989,7 +989,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:3686b84c42cc2b1bcaccf57cbdcda25a15c7d9d3778fb92d25437f9880ed8f5d",
- "closureDigest": "sha256:7ff51a768a942fbe72ec4ef11b2d19c3066cc38c77f1a94261aa3cacd6a8e56d"
+ "closureDigest": "sha256:8c03021a0c809fbc076cd0c09c797151ec6cb48d473453f48993a8807ceb216f"
},
{
"specifier": "@narratage/store-sqlite",
@@ -998,8 +998,8 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:5bccf73cd953710fc31ef44672fef1bfc396fd4aa2eff4b53108e8ff87e749c3",
- "closureDigest": "sha256:46d6044b5fee1777c27718094480b5b6e2606d35bfe8347a4f9c0a5034419bb8"
+ "closureDigest": "sha256:d512ec9a51a70ce2c020a4a1cb4d1f67b66b484615c6212a4a85918983415718"
}
],
- "digest": "sha256:a7d0d90fe7ab45a8a8bf113e08f7348ab53d91396f7e08955cb90b7db6444524"
+ "digest": "sha256:2ef941c08a6fde4e0fc9b9527358186402e5bffbc262b1c35e98a6678b3f9b99"
}
diff --git a/examples/talking-film-live/svml.runtime.json b/examples/talking-film-live/svml.runtime.json
index 99a51ce0..dcca556a 100644
--- a/examples/talking-film-live/svml.runtime.json
+++ b/examples/talking-film-live/svml.runtime.json
@@ -4,7 +4,7 @@
"runtimePackageLock": "./svml.runtime-packages.lock",
"runtimeServices": [
{ "use": "@narratage/local", "instance": "execution" },
- { "use": "@narratage/store-sqlite", "instance": "state", "config": { "path": ".svml/runtime.sqlite" } },
+ { "use": "@narratage/store-sqlite", "instance": "state", "config": { "path": ".svml/state.sqlite" } },
{ "use": "@narratage/artifact-store-fs", "instance": "artifacts", "config": { "path": ".svml/artifacts" } },
{ "use": "@narratage/credential-store-env", "instance": "credentials.env", "config": {} }
],
diff --git a/examples/talking-film-live/svml.runtime.ts b/examples/talking-film-live/svml.runtime.ts
index 60ace792..db60cc01 100644
--- a/examples/talking-film-live/svml.runtime.ts
+++ b/examples/talking-film-live/svml.runtime.ts
@@ -52,7 +52,7 @@ export default async function createTalkingFilmRuntime() {
const execution = createLocalExecutionPackage("execution");
const state = createSqliteRuntimeServicePackage({
- path: join(root, ".svml", "runtime.sqlite"),
+ path: join(root, ".svml", "state.sqlite"),
name: "state",
buildInstance: "state.builds",
operationInstance: "state.operations",
diff --git a/examples/talking-head-aroll/svml.packages.lock b/examples/talking-head-aroll/svml.packages.lock
index ed137ccd..8c11d380 100644
--- a/examples/talking-head-aroll/svml.packages.lock
+++ b/examples/talking-head-aroll/svml.packages.lock
@@ -640,12 +640,12 @@
{
"name": "@narratage/host",
"version": "0.0.0-dev",
- "digest": "sha256:3cbc10d27b16c44ba82a29db27b0a394574d9088184e889558f2b10f1d143e08"
+ "digest": "sha256:daf8c9f8cb8e380a8bcf3000d96fca8431da37ae5e6b3f2321fed5e4cd8fcf17"
},
{
"name": "@narratage/hyperframes",
"version": "0.0.0-dev",
- "digest": "sha256:dde9143aed4fa8731a1a91f860bb2a8a9f426c7515b5c12bed8b2fac24c0a416"
+ "digest": "sha256:586ea608ae520e5b5b0a5950a251fe8a4c67ec8f3c89fcec32d968e3cf3a6a17"
},
{
"name": "@narratage/markup",
@@ -805,7 +805,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:6a2991ec75f9fd30e8d286213fc7993aae2c570b85a8faae5d3c2af661c9fdfc",
- "closureDigest": "sha256:45c0adc0dbc045f12283db7da9b97a6d230813f508f512f91dfee5aedabdb401"
+ "closureDigest": "sha256:ee79cd66af6e561432e52f1bf78f597f9dc01cd09d12d68c26e7cf61e7361aac"
},
{
"specifier": "@narratage/caption-fine",
@@ -814,7 +814,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:c5616408dd56ce9c903523319b2a697ff6b8af5bfe1dc66fe793a11e4629f3cf",
- "closureDigest": "sha256:9331f6dcf950a7bd001cf28a4f866f36f9f6a0a97910c5f8355bba9724144c71"
+ "closureDigest": "sha256:782a9f2c0345431e06c8ab009d518933c964465b34d790b4064f87170e0495ff"
},
{
"specifier": "@narratage/caption-gemini",
@@ -823,7 +823,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:22fed1f1748349c11b6f28448be38cbea68b1dd5a51a3dc2d548ae0a376b9c71",
- "closureDigest": "sha256:0206e5a701c6feb1b60a9b73cf8bb74dfed7bc1e321b3e38b495590cdcf4ac24"
+ "closureDigest": "sha256:3cc6fd7d5250e5457db548fa2432f33d62012db8f77e1c8fe4f590e38a3439ab"
},
{
"specifier": "@narratage/composition",
@@ -832,7 +832,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:0a988b665baf0846246e309d9faf39ffb3ef6ec455bf37f7c438eb30429ff46e",
- "closureDigest": "sha256:37f34e3ddef9f2e86c8a9d062fd5a4523874841db3a5a51444b5c516a0c63817"
+ "closureDigest": "sha256:15c9088a176b04a6051b2398e308bde8fab6fb15d0147786e9e40abfce772662"
},
{
"specifier": "@narratage/estimate",
@@ -841,7 +841,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:9975815b3109520eb6bba276554a90767a9aebff6498c2768d9faf4763219bef",
- "closureDigest": "sha256:811c0d891e6754d5553f6e3f7f04f5a77a6bc0134fdb8803319b9502a7073bad"
+ "closureDigest": "sha256:7bbdc658746e65e2bb4199b3de84d1197376f16120708e2eea31558898de662f"
},
{
"specifier": "@narratage/film",
@@ -850,7 +850,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:8e58b5630b9a68a8ce7314620f5f1c7cda973a05c179c18d61e10d68002964f0",
- "closureDigest": "sha256:2d0ca309cdfb053c6fc47d3f3464c8cb3e50ee601e3c6951484e34529aeab8a6"
+ "closureDigest": "sha256:2a4b2b66d9f3913bc839f052e35e6c5cd6afed08821a3308c643b16dcd011f44"
},
{
"specifier": "@narratage/fonts-open",
@@ -859,7 +859,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:31b51352edcb4f34beccc9d9ac8cf7397125a840971a72c912df3d34d1f3feda",
- "closureDigest": "sha256:6c3f7a28db8159c5ba03528fd4c85f1b97718fe719d3a087312f1a1d8f18b160"
+ "closureDigest": "sha256:4594c947a3425c6c42738e2e7d0f7ce56a6efaced91fcf6085e282c3ad98b797"
},
{
"specifier": "@narratage/generation",
@@ -868,7 +868,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:d74a57a3210267e665366513324db5c313b3ca0f770da66e74cc38aee5048efb",
- "closureDigest": "sha256:d8c63f5908ba26809179c0e1ccf04b87b9c945270733947d11262bb2b0f7916c"
+ "closureDigest": "sha256:3bd4bfdd124001b83602627b0305d6652e96f43cbb9345e3b9bece51a3d701ce"
},
{
"specifier": "@narratage/hyperframes",
@@ -877,7 +877,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:b8b8773d03f19844cdc8d714650380145056649245e33c607f30220fff194157",
- "closureDigest": "sha256:9077ffd048007617708b988f285dcef6241b277cabc1ba472ccf30ac20b2356f"
+ "closureDigest": "sha256:88d7a9fedb614166e149e29d064be1dca18f1c4a102759f1a2600afba4953b8b"
},
{
"specifier": "@narratage/media",
@@ -886,7 +886,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:6c7cf563a92aa8e48db6da224697fd5f56c50b636b0327efc17fbbc264502b3f",
- "closureDigest": "sha256:2d56835dce5d6c4018cb8b6499771bba422f2e95ab67aa62500ca59f53672e23"
+ "closureDigest": "sha256:e8edbbfb7bec2efa7ad05fa273549a3b033861eb9ef20345430d9be617b26674"
},
{
"specifier": "@narratage/media-pipeline",
@@ -895,7 +895,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:21f817f8e6eea9e9e4361005ed5ab9e69117fd77ec237fd467fb11afe9965019",
- "closureDigest": "sha256:b5462108e446bc5a22c3888a24bf0f99652115cc3d69f3cb868b6c7305b8e324"
+ "closureDigest": "sha256:3555f1939c82dfa216902fd600e2616ecfce6a244e240b060080621367333d97"
},
{
"specifier": "@narratage/narrative",
@@ -922,7 +922,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:aa47f04cf0652ee3fa2e114eb6d9a80b0522828a47869103ca33e95027b21634",
- "closureDigest": "sha256:b1c4fd45e70759f2597a980c1ca2097d44fc0a0c8e596b35317be04fa96fa7ba"
+ "closureDigest": "sha256:1c9dcb543a3afa6a94c9757fd47a3e0bfb68ed7d5f3c87b3ef0f0e52ed286332"
},
{
"specifier": "@narratage/run-markup",
@@ -931,7 +931,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:fb3a8a5be9a0ef0aa914026a199be641f4e20df27f9b284be196f52fd260be55",
- "closureDigest": "sha256:e1a5faf90e0925ff8100b2e32c188d1741aac3bc376fbc94183c5c9eeafc195b"
+ "closureDigest": "sha256:20b6fcbacf36352d6c180e1a466eaf368c0bef183eec09396456e2e2375ea16f"
},
{
"specifier": "@narratage/script",
@@ -940,7 +940,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:69e1c08a5f07c73a506ca8286426a0386485738649e86463cca0b291f319ee4e",
- "closureDigest": "sha256:070562efce768b46aca3960561b2bc3a592997192f7a8e9214bf8ec289df83b9"
+ "closureDigest": "sha256:696bb2e69c89d9e1079a76ce68d26fe2c99615275a4266bf24fa905ad77c1bb4"
},
{
"specifier": "@narratage/seedance",
@@ -949,7 +949,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:e889ca3808a19f544e7aa5e27453beb5a09464c645633b0b1b1d9a4b7ce6591a",
- "closureDigest": "sha256:55d4043e4ca8f85cd5eea781eb07801f1869443541d260f2ab427c78e93f5b2b"
+ "closureDigest": "sha256:e1c45192a92620ec8ea03d9cf1b19073bae2dbfde12475f8436cd5c008c854bd"
},
{
"specifier": "@narratage/semantic-map",
@@ -958,7 +958,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:a9d81ddfa09dfa23c9c3c0817b60299431ca9ec640252a659d490be55d92ba68",
- "closureDigest": "sha256:658b7ef5306d5ec68c172ce19543eb05fd8e1f616a10223acfb87223012e440e"
+ "closureDigest": "sha256:3ac5deb912231d3092666c692d73a379129d1cf2ed035d43a95bef6d5101d525"
},
{
"specifier": "@narratage/spatial",
@@ -967,7 +967,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:5062281ffadb46495ec12e1a67185b57fe13879e90f03bcf544e93fd1d273716",
- "closureDigest": "sha256:0bfb0066a25c344a6e74aa8e2439300179e55ffbf6a8d13e84e526d4123a59ce"
+ "closureDigest": "sha256:3209c05b608c20aedab35299235974d74ee4374ead722ec125f2b6ea70829c77"
},
{
"specifier": "@narratage/speech",
@@ -976,7 +976,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:75b1537c9b9729c2e9197445c34b295bc9f72dbef754a4702e1ca792dceea472",
- "closureDigest": "sha256:e9a563703c2f7c72777c617067444ef4cb036ff857fbbe65161314255fb102f1"
+ "closureDigest": "sha256:a5be05750141d51e4f4460945b779bf0196548d7b7c908eca34d8c63c2ec7b70"
},
{
"specifier": "@narratage/speech-alignment",
@@ -985,7 +985,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:bdfe219081310041d11c6d54440e6ccb80398bfb97f93afc28e9d1ed101b4a79",
- "closureDigest": "sha256:400256232c983f05e8ceec145ea790e2ec15b5dad91d10ca340f15514b6067ff"
+ "closureDigest": "sha256:de73bc9d1411af8319a099df658dfc51f6e7b44dcbf3173cb5f9db18edb449cf"
},
{
"specifier": "@narratage/speech-basis",
@@ -994,7 +994,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:f01f21badb0ccdd5cc62b131be05f46030bffee96042c7323a0c572786d012ef",
- "closureDigest": "sha256:bd0ad1414559f1b6da7c8c5aa60c16c3f3b2634b9151d7281bc4ebb274abd82b"
+ "closureDigest": "sha256:843a7922a70d5df180ade4ec3a8becf8fb110f6137fa6a38d86cc5aaa14830e1"
},
{
"specifier": "@narratage/speech-evidence",
@@ -1003,7 +1003,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:60f366de074a7fba20c71dcbd825083fddc54ea7de602a903eb144eab5555425",
- "closureDigest": "sha256:319c2bc5bcda4734e4c975304ae9d24796e011c4dabe245dfedb96e4b06228b0"
+ "closureDigest": "sha256:918172938ecc1e9b6b1cff9465048bdfe2dcbfe8748971101f78f12ee36f1d75"
},
{
"specifier": "@narratage/speech-spine",
@@ -1012,7 +1012,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:a27dd4349d6bb1aaa7f382ad09f883e9894157579e48285b00e8fa24456190f7",
- "closureDigest": "sha256:8c2c4fe30acea7d039e53aac7cc512c765e81e92cb210b538afe0cc3c2c253a5"
+ "closureDigest": "sha256:f056e14a44f4f2cc0751b28e1ecb9831b7ff472034a02cac8089b9ff45d67854"
},
{
"specifier": "@narratage/svs",
@@ -1030,7 +1030,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:4435dd4bd38e5e017013f775e28e379e3e235211649ee8f3913378f7f9e60b58",
- "closureDigest": "sha256:82d2119cfc67634753bafcdb26003953f5d85519fbada9a5ce88b5a312e31a28"
+ "closureDigest": "sha256:c6ba3301621d63598421b7d3f0dc2e1bcc8967f16624407d37a20c2c0703a246"
},
{
"specifier": "@narratage/visual-ir",
@@ -1048,8 +1048,8 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:c8cb487d25e567a081c74a7abc97ce2b94dc0c47c36c90ba6b44d2ec21a9e80b",
- "closureDigest": "sha256:afc3d88b322d8c5aeebd520d5d691531d5314db0e87b19f0564941c3477f9e8a"
+ "closureDigest": "sha256:98e68de949bf7cfb9e1c483de66a38de6f347fc5032122d10e9af971a28fa534"
}
],
- "digest": "sha256:2832029345fa93fdc3e49662f8bda2a313fb5a61609e5477c7f964b9178ddad7"
+ "digest": "sha256:828557259e164093a055111cff451aaf48d1d8519cc02352804066c5841347ee"
}
diff --git a/examples/talking-head-aroll/svml.runtime-packages.lock b/examples/talking-head-aroll/svml.runtime-packages.lock
index 7092057f..ef94a1d9 100644
--- a/examples/talking-head-aroll/svml.runtime-packages.lock
+++ b/examples/talking-head-aroll/svml.runtime-packages.lock
@@ -40,7 +40,7 @@
{
"name": "@narratage/artifact-store-fs",
"version": "0.0.0-dev",
- "digest": "sha256:a9d1cd5699df848cf89887897ece565cedd25af1eef1cac453e5b35caa52c3f2"
+ "digest": "sha256:ee4542f46f20c58f66b35c8683cd192026fb32a19157fd237fb365ab5a77ab39"
},
{
"name": "@narratage/artifact",
@@ -65,7 +65,7 @@
{
"name": "@narratage/compiler-node",
"version": "0.0.0-dev",
- "digest": "sha256:019d7603c28d726e9fc8127e5786c2cd22575a8faa84d91dfd77139fda0927af"
+ "digest": "sha256:4626882f405a083281ce7ffec73fa598640223669908b36715d8281fa19be92e"
},
{
"name": "@narratage/component-kit",
@@ -90,7 +90,7 @@
{
"name": "@narratage/driver-node",
"version": "0.0.0-dev",
- "digest": "sha256:9f7df9d719422b91ac6d51995a6657d2b01cc32f80204a98240c9102ac278791"
+ "digest": "sha256:0e1209c7f242956f55f90e712a7921ba63fe0e9a20a6164461afa5215d8bcba2"
},
{
"name": "@narratage/elaborator",
@@ -100,7 +100,7 @@
{
"name": "@narratage/endpoint-kit",
"version": "0.0.0-dev",
- "digest": "sha256:94b455390d37d675a55c1888cc4b665d777f017e279510852cb5033436aee3ed"
+ "digest": "sha256:833d4c2b40cbf889c4790476a92a7e3eb9ddf42b748fd17bc7f9ca9bc8596c14"
},
{
"name": "@narratage/generation",
@@ -110,17 +110,17 @@
{
"name": "@narratage/host",
"version": "0.0.0-dev",
- "digest": "sha256:3cbc10d27b16c44ba82a29db27b0a394574d9088184e889558f2b10f1d143e08"
+ "digest": "sha256:daf8c9f8cb8e380a8bcf3000d96fca8431da37ae5e6b3f2321fed5e4cd8fcf17"
},
{
"name": "@narratage/hyperframes",
"version": "0.0.0-dev",
- "digest": "sha256:dde9143aed4fa8731a1a91f860bb2a8a9f426c7515b5c12bed8b2fac24c0a416"
+ "digest": "sha256:586ea608ae520e5b5b0a5950a251fe8a4c67ec8f3c89fcec32d968e3cf3a6a17"
},
{
"name": "@narratage/local",
"version": "0.0.0-dev",
- "digest": "sha256:f833c604b93c46f7596951d41e89ddb630dad1404bee6029c44ed541c437d151"
+ "digest": "sha256:cfbd3c13a81dbc21378bebd22662e93a201480e1bfe725f04e78534764f86c0e"
},
{
"name": "@narratage/markup",
@@ -130,7 +130,7 @@
{
"name": "@narratage/media-execution",
"version": "0.0.0-dev",
- "digest": "sha256:2bca4da3fe0325847811888d2ded010761d4dc447b8ec79bec9710bca5011c3e"
+ "digest": "sha256:8fe6aca65ca4ebcc1efa71376e986e97741310d836f85647fcb223f39fda9754"
},
{
"name": "@narratage/media-pipeline",
@@ -150,7 +150,7 @@
{
"name": "@narratage/package-loader-node",
"version": "0.0.0-dev",
- "digest": "sha256:1296c0ef3170e77cfdaa8cde0e1753fad2822b254507e0978429498d84c73ed0"
+ "digest": "sha256:db78f8abd9a08a7655f7ed2d24860b1f848b3763421984847c303afa316b9003"
},
{
"name": "@narratage/program-space",
@@ -165,27 +165,27 @@
{
"name": "@narratage/provider-google-vertex",
"version": "0.0.0-dev",
- "digest": "sha256:b9de2ae61195991b65626a51053e71d79747b37274dd24dfbfdd0b855df455bd"
+ "digest": "sha256:e6f2594a0636b527fa7472c66fb3134eaa92d7e9903a8cebf0065fc7b4af4377"
},
{
"name": "@narratage/provider-hyperframes-local",
"version": "0.0.0-dev",
- "digest": "sha256:218adf5c62f11a4490b9a02a9cc1087f5eb2d493c3d99b16682cf70dbb2704f7"
+ "digest": "sha256:1d3c090bc9d13fc266b65120f3e0439be824b1d05d4746a670aa929da49a560b"
},
{
"name": "@narratage/provider-kie",
"version": "0.0.0-dev",
- "digest": "sha256:22612e3da4f12a5b780bcca5de5bf5585370772343aea749329715afe1988a88"
+ "digest": "sha256:47fd2c4c57dc381b21f9622002d77f7db7003c52de8c70e1ca295ae13cebdb18"
},
{
"name": "@narratage/provider-media-local",
"version": "0.0.0-dev",
- "digest": "sha256:241bfaeacef8ec59694c127bc2dacf8cbac4fd48d10cbe04d1f2d0aa25f84d36"
+ "digest": "sha256:9bfc485ddbdeca9cd747925b4ef04f85315e41746a2cf4a99ac0d96baf237014"
},
{
"name": "@narratage/provider-whisperx-local",
"version": "0.0.0-dev",
- "digest": "sha256:2aea383b431858033dd8a27ede705d99cbafb3d55cce0971c7bca09af20d3df2"
+ "digest": "sha256:3fea223b5b0d1569b2c04e7949c82b6eeaa770caa41b19b461ed035e30d8c29e"
},
{
"name": "@narratage/render-hyperframes",
@@ -205,12 +205,12 @@
{
"name": "@narratage/runtime-adapter",
"version": "0.0.0-dev",
- "digest": "sha256:357fc3c81876d25e553eaa555dd3308ff3db51715d230e6dd44573667058add2"
+ "digest": "sha256:7bb0b1ef43eb3fa2e67725c990b17f585a2fccd98fc71eae6c7a8da29b3e06a4"
},
{
"name": "@narratage/runtime",
"version": "0.0.0-dev",
- "digest": "sha256:0f383c247e08abb30ab69ef6ce9d6bbf8b0f8fabc72b057ac97970c7f99ed4b2"
+ "digest": "sha256:b7954bf727125baedc31377d3a1d9066b5bd9c019c72fe6468fcc4864243aa31"
},
{
"name": "@narratage/semantic-map",
@@ -245,7 +245,7 @@
{
"name": "@narratage/store-sqlite",
"version": "0.0.0-dev",
- "digest": "sha256:cf7e739e38b5d7094488cd3da0aaae16f66cdb229ed48ec59999ccbe381aedcd"
+ "digest": "sha256:a6063abc900ba2e608db91ab9b587306af3bb843627f5f95c242668ca9abdd7d"
},
{
"name": "@narratage/svs",
@@ -275,7 +275,7 @@
{
"name": "@narratage/workspace-fs-node",
"version": "0.0.0-dev",
- "digest": "sha256:3ed827c6a586ccb5a9bc023ccea5c05a1fbe1a317a79df88314073c18cfa8ea3"
+ "digest": "sha256:1c2ec730e80c5cb9ca504f433076aeccf855ee9a32cb4fdcef68d8914b24349d"
},
{
"name": "@protobufjs/aspromise",
@@ -926,7 +926,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:914018276b9218afc13940ca5c362dc95fa19d574fc07b06b6fe005c5eb30315",
- "closureDigest": "sha256:486228772cda98bfe09c6262bc6f06f6cc5601928856c184087d1719d7497e49"
+ "closureDigest": "sha256:fadbd7bdb4fb5af49e762cb92c1e22d0cf3e2bf10fd5f0de3d594ef3f0ca8d93"
},
{
"specifier": "@narratage/credential-store-env",
@@ -935,7 +935,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:ee33ba638bbf2f88eee8f34a2f3179f435a0af9ccbae3fbb7d2a81bd6aeac612",
- "closureDigest": "sha256:dee6b5e342b981185fce3dd86112d7a30bc11e0aa7132f4c4cbfa9ccfd086dda"
+ "closureDigest": "sha256:d416ef4d798b0948dcc2cd16c962784297b6e2f20daf8626f28c5d90e37b1dec"
},
{
"specifier": "@narratage/local",
@@ -944,7 +944,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:6689aed79b0947c7315aef65e46806ce73da01de3d73b454e1497b8d847ffa61",
- "closureDigest": "sha256:4a22531c658b24975a7b1c083102bc90bd107d97c20286da818a9f964c6bbc86"
+ "closureDigest": "sha256:2e6704a8d53b79c750c1dc1ada0196737e13cb1e41bec5fac233abbd62b70326"
},
{
"specifier": "@narratage/provider-google-vertex",
@@ -953,7 +953,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:b2636492ab0b0b500d674899ab2ff14fc61c7f2b22ef44eb8f816f141b21ec4d",
- "closureDigest": "sha256:8b92e7871fa616ed3a6adce14a6bb7d705532dc0807e453492cd8a878558d9a7"
+ "closureDigest": "sha256:fdfd1b6d25ae82b3eca28bc7f1e94ca2a4c2930f6171388afb861ca9a06990da"
},
{
"specifier": "@narratage/provider-hyperframes-local",
@@ -962,7 +962,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:b14b721bdc0c9d5d9ced1c345107c92a3a5878f3383dea2ecd3e310442c2a40c",
- "closureDigest": "sha256:1102b66d7613c516cbd67ded83961b8bd96ad0d45772fe185c28962084692daf"
+ "closureDigest": "sha256:22819933ee6b109e794ca94cee669cddf3288c8fe5c3d4e7e2bd727fdeab8bda"
},
{
"specifier": "@narratage/provider-kie",
@@ -971,7 +971,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:e61ebb911a33d4c5d0e4e4ee4fb69cdb790f4ced050ef35f7472bfbd72013d9b",
- "closureDigest": "sha256:b6d39b1697767893f05c27d7d3c2b5e683e9c559d020c8de0a412ec76226cea0"
+ "closureDigest": "sha256:90589aebf0631d19b59f258bd1e4c20638f4325f946260aa71968350e9e4efc0"
},
{
"specifier": "@narratage/provider-media-local",
@@ -980,7 +980,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:701abcc923bf8a711c20b6e308d8cc8be27bf560f249b8c5a801e1dbd9cb3979",
- "closureDigest": "sha256:8704836ca23624ca38023f5a3b0d7427e817a38412880e97dc5932fb7268d493"
+ "closureDigest": "sha256:f40546545b842197e77982ceba94c7f52f2389bad33d68ff98ad62e28eb576e1"
},
{
"specifier": "@narratage/provider-whisperx-local",
@@ -989,7 +989,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:3686b84c42cc2b1bcaccf57cbdcda25a15c7d9d3778fb92d25437f9880ed8f5d",
- "closureDigest": "sha256:7ff51a768a942fbe72ec4ef11b2d19c3066cc38c77f1a94261aa3cacd6a8e56d"
+ "closureDigest": "sha256:8c03021a0c809fbc076cd0c09c797151ec6cb48d473453f48993a8807ceb216f"
},
{
"specifier": "@narratage/store-sqlite",
@@ -998,8 +998,8 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:5bccf73cd953710fc31ef44672fef1bfc396fd4aa2eff4b53108e8ff87e749c3",
- "closureDigest": "sha256:46d6044b5fee1777c27718094480b5b6e2606d35bfe8347a4f9c0a5034419bb8"
+ "closureDigest": "sha256:d512ec9a51a70ce2c020a4a1cb4d1f67b66b484615c6212a4a85918983415718"
}
],
- "digest": "sha256:a7d0d90fe7ab45a8a8bf113e08f7348ab53d91396f7e08955cb90b7db6444524"
+ "digest": "sha256:2ef941c08a6fde4e0fc9b9527358186402e5bffbc262b1c35e98a6678b3f9b99"
}
diff --git a/examples/talking-head-aroll/svml.runtime.json b/examples/talking-head-aroll/svml.runtime.json
index c9c2e702..958034d0 100644
--- a/examples/talking-head-aroll/svml.runtime.json
+++ b/examples/talking-head-aroll/svml.runtime.json
@@ -4,7 +4,7 @@
"runtimePackageLock": "./svml.runtime-packages.lock",
"runtimeServices": [
{ "use": "@narratage/local", "instance": "execution" },
- { "use": "@narratage/store-sqlite", "instance": "state", "config": { "path": ".svml/runtime.sqlite" } },
+ { "use": "@narratage/store-sqlite", "instance": "state", "config": { "path": ".svml/state.sqlite" } },
{ "use": "@narratage/artifact-store-fs", "instance": "artifacts", "config": { "path": ".svml/artifacts" } },
{ "use": "@narratage/credential-store-env", "instance": "credentials.env", "config": {} }
],
diff --git a/narratage b/narratage
new file mode 100755
index 00000000..cb344720
--- /dev/null
+++ b/narratage
@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+
+launcher_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+loader="$launcher_dir/node_modules/tsx/dist/loader.mjs"
+if [ ! -f "$loader" ]; then
+ echo "Narratage dependencies are not installed in $launcher_dir; run: pnpm install --frozen-lockfile" >&2
+ exit 1
+fi
+exec node --disable-warning=ExperimentalWarning \
+ --import "$loader" \
+ "$launcher_dir/packages/video-cli/src/cli.ts" "$@"
diff --git a/package.json b/package.json
index b7a2a736..e75e28c5 100644
--- a/package.json
+++ b/package.json
@@ -6,7 +6,6 @@
"packageManager": "[email protected]",
"scripts": {
"check": "tsc -p tsconfig.json --noEmit",
- "postinstall": "node tools/postinstall-services.mjs",
"docs:dev": "vitepress dev docs",
"docs:build": "vitepress build docs",
"docs:preview": "vitepress preview docs",
@@ -14,8 +13,8 @@
"test:whisperx-service": "uv run --project services/whisperx --frozen python -m unittest discover -s services/whisperx/tests -p 'test_*.py'",
"test:image-opencv": "SVML_OPENCV_TESTS=1 SVML_OPENCV_PYTHON=services/image-opencv/.venv/bin/python node --import tsx --test packages/provider-image-opencv-local/test/provider.test.ts",
"smoke:kie": "node --import tsx packages/provider-kie/test/live-smoke.ts",
- "narratage": "node --import tsx packages/video-cli/src/cli.ts",
- "svml": "node --import tsx packages/video-cli/src/cli.ts",
+ "narratage": "node --disable-warning=ExperimentalWarning --import tsx packages/video-cli/src/cli.ts",
+ "svml": "node --disable-warning=ExperimentalWarning --import tsx packages/video-cli/src/cli.ts",
"caption:playground": "node --import tsx tools/caption-playground/start.ts",
"build:media-lambda": "pnpm --filter @narratage/media-lambda build",
"build:media-lambda-layer": "pnpm --filter @narratage/media-lambda build:layer"
@@ -27,6 +26,7 @@
"devDependencies": {
"@narratage/artifact": "workspace:*",
"@narratage/artifact-store-fs": "workspace:*",
+ "@narratage/artifact-store-s3": "workspace:*",
"@narratage/audio-track": "workspace:*",
"@narratage/background-removal": "workspace:*",
"@narratage/caption": "workspace:*",
@@ -40,7 +40,10 @@
"@narratage/estimate": "workspace:*",
"@narratage/film": "workspace:*",
"@narratage/fonts-open": "workspace:*",
+ "@narratage/gemini-omni": "workspace:*",
"@narratage/generation": "workspace:*",
+ "@narratage/gpt-image": "workspace:*",
+ "@narratage/grok-imagine": "workspace:*",
"@narratage/hyperframes": "workspace:*",
"@narratage/image-compose": "workspace:*",
"@narratage/image-transform": "workspace:*",
@@ -48,15 +51,19 @@
"@narratage/media": "workspace:*",
"@narratage/media-track": "workspace:*",
"@narratage/media-pipeline": "workspace:*",
+ "@narratage/minimax-h3": "workspace:*",
"@narratage/mimo-tts": "workspace:*",
+ "@narratage/nano-banana": "workspace:*",
"@narratage/narrative": "workspace:*",
"@narratage/program-space": "workspace:*",
"@narratage/protocol": "workspace:*",
"@narratage/ranking": "workspace:*",
"@narratage/provider-google-vertex": "workspace:*",
+ "@narratage/provider-hyperframes-aws-lambda": "workspace:*",
"@narratage/provider-hyperframes-local": "workspace:*",
"@narratage/provider-image-opencv-local": "workspace:*",
"@narratage/provider-kie": "workspace:*",
+ "@narratage/provider-media-aws-lambda": "workspace:*",
"@narratage/provider-media-local": "workspace:*",
"@narratage/provider-xiaomi-mimo": "workspace:*",
"@narratage/provider-whisperx-local": "workspace:*",
@@ -69,6 +76,7 @@
"@narratage/screen-overlay": "workspace:*",
"@narratage/seedance": "workspace:*",
"@narratage/seedance-kits": "workspace:*",
+ "@narratage/seedream": "workspace:*",
"@narratage/semantic-map": "workspace:*",
"@narratage/speech": "workspace:*",
"@narratage/speech-alignment": "workspace:*",
diff --git a/packages/artifact-store-fs/src/store.ts b/packages/artifact-store-fs/src/store.ts
index a7322b08..f327b90d 100644
--- a/packages/artifact-store-fs/src/store.ts
+++ b/packages/artifact-store-fs/src/store.ts
@@ -94,9 +94,6 @@ export class FileArtifactStore implements ArtifactStore {
}
}
await unlink(temporary).catch(() => undefined);
- if ((await this.get(digest)) === undefined) {
- throw new Error(`Artifact ${digest} disappeared after storage`);
- }
return { kind: "blob", digest, size, mediaType };
}
@@ -149,7 +146,12 @@ export class FileArtifactStore implements ArtifactStore {
}
async has(digest: Digest): Promise<boolean> {
- return (await this.get(digest)) !== undefined;
+ try {
+ return (await stat(digestPath(this.root, digest))).isFile();
+ } catch (error) {
+ if (isNodeError(error, "ENOENT")) return false;
+ throw error;
+ }
}
async list(): Promise<readonly Digest[]> {
diff --git a/packages/artifact-store-fs/test/store.test.ts b/packages/artifact-store-fs/test/store.test.ts
index 65fa403f..9bf862fb 100644
--- a/packages/artifact-store-fs/test/store.test.ts
+++ b/packages/artifact-store-fs/test/store.test.ts
@@ -22,6 +22,7 @@ test("filesystem artifacts are content-addressed and survive adapter restart", a
const hex = left.digest.slice("sha256:".length);
await writeFile(join(directory, "sha256", hex.slice(0, 2), hex), "tampered");
+ assert.equal(await reopened.has(left.digest), true, "presence is not a second full integrity read");
await assert.rejects(reopened.get(left.digest), /content digest differs/u);
} finally {
await rm(directory, { recursive: true, force: true });
diff --git a/packages/artifact-store-s3/src/store.ts b/packages/artifact-store-s3/src/store.ts
index 743eba84..f7f3cd7a 100644
--- a/packages/artifact-store-s3/src/store.ts
+++ b/packages/artifact-store-s3/src/store.ts
@@ -163,8 +163,25 @@ export class S3ArtifactStore implements ArtifactStore {
const status = statusCode(error);
if (status === 409 && attempt < this.#maxConflictRetries) continue;
if (status !== 412) throw error;
- const existing = await this.get(digest);
- if (existing === undefined) throw new Error(`S3 reported existing Artifact ${digest}, but it cannot be read`);
+ const existing = this.#client.head === undefined
+ ? await this.#client.get({
+ Bucket: this.#bucket,
+ Key: this.key(digest),
+ ...(this.#expectedBucketOwner === undefined
+ ? {}
+ : { ExpectedBucketOwner: this.#expectedBucketOwner }),
+ }).then((value) => value === undefined ? undefined : { size: value.byteLength })
+ : await this.#client.head({
+ Bucket: this.#bucket,
+ Key: this.key(digest),
+ ...(this.#expectedBucketOwner === undefined
+ ? {}
+ : { ExpectedBucketOwner: this.#expectedBucketOwner }),
+ });
+ if (existing === undefined) throw new Error(`S3 reported existing Artifact ${digest}, but it is absent`);
+ if (existing.size !== copy.byteLength) {
+ throw new Error(`S3 Artifact ${digest} size differs from the bytes being stored`);
+ }
return { kind: "blob", digest, size: copy.byteLength, mediaType };
}
}
@@ -187,8 +204,20 @@ export class S3ArtifactStore implements ArtifactStore {
}
async has(digest: Digest): Promise<boolean> {
- // Existence by key is insufficient: ArtifactStore promises content identity, so verify bytes.
- return (await this.get(digest)) !== undefined;
+ if (this.#client.head === undefined) return (await this.#client.get({
+ Bucket: this.#bucket,
+ Key: this.key(digest),
+ ...(this.#expectedBucketOwner === undefined
+ ? {}
+ : { ExpectedBucketOwner: this.#expectedBucketOwner }),
+ })) !== undefined;
+ return (await this.#client.head({
+ Bucket: this.#bucket,
+ Key: this.key(digest),
+ ...(this.#expectedBucketOwner === undefined
+ ? {}
+ : { ExpectedBucketOwner: this.#expectedBucketOwner }),
+ })) !== undefined;
}
/**
diff --git a/packages/artifact-store-s3/test/store.test.ts b/packages/artifact-store-s3/test/store.test.ts
index 31cb2d8e..747f57c2 100644
--- a/packages/artifact-store-s3/test/store.test.ts
+++ b/packages/artifact-store-s3/test/store.test.ts
@@ -11,6 +11,7 @@ import { isManagedArtifactStore, isStreamingArtifactStore } from "@narratage/run
class FakeS3 implements S3ObjectClient {
readonly values = new Map<string, Uint8Array>();
conflicts = 0;
+ gets = 0;
async put(input: Parameters<S3ObjectClient["put"]>[0]): Promise<void> {
const key = input.Key!;
@@ -26,6 +27,7 @@ class FakeS3 implements S3ObjectClient {
}
async get(input: Parameters<S3ObjectClient["get"]>[0]): Promise<Uint8Array | undefined> {
+ this.gets += 1;
const value = this.values.get(input.Key!);
return value === undefined ? undefined : Uint8Array.from(value);
}
@@ -64,6 +66,7 @@ test("configured S3 service locks location without putting AWS credentials in th
class FullFakeS3 extends FakeS3 {
readonly uploads = new Map<string, Uint8Array[]>();
copies = 0;
+ heads = 0;
async open(input: Parameters<NonNullable<S3ObjectClient["open"]>>[0]) {
const value = this.values.get(input.Key!);
@@ -77,6 +80,7 @@ class FullFakeS3 extends FakeS3 {
}
async head(input: Parameters<NonNullable<S3ObjectClient["head"]>>[0]) {
+ this.heads += 1;
const value = this.values.get(input.Key!);
return value === undefined ? undefined : { size: value.byteLength };
}
@@ -184,6 +188,16 @@ test("a streamed read hands back bytes as they arrive, and refuses tampered cont
}, /content digest differs/u);
});
+test("presence uses object metadata while byte reads retain integrity verification", async () => {
+ const client = new FullFakeS3();
+ const store = new S3ArtifactStore({ client, bucket: "fixture" });
+ const ref = await store.put(new TextEncoder().encode("present"), "text/plain");
+ const gets = client.gets;
+ assert.equal(await store.has(ref.digest), true);
+ assert.equal(client.heads, 1);
+ assert.equal(client.gets, gets, "has did not download the object");
+});
+
test("retention reports the Artifacts it stored, and whether a delete found one", async () => {
const client = new FullFakeS3();
const store = new S3ArtifactStore({ client, bucket: "fixture", prefix: "svml" });
diff --git a/packages/cli/src/archive.ts b/packages/cli/src/archive.ts
index 2a009b16..d8c7d641 100644
--- a/packages/cli/src/archive.ts
+++ b/packages/cli/src/archive.ts
@@ -1,4 +1,5 @@
-import { mkdir, writeFile } from "node:fs/promises";
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, open, rename, rm, writeFile } from "node:fs/promises";
import { dirname } from "node:path";
import type { LocalRuntime } from "@narratage/local";
@@ -17,6 +18,15 @@ export type ArchivedArtifactReference = ArtifactIdentity & {
readonly path: string;
};
+export type ArchivedLogicalOutput = {
+ readonly name: string;
+ readonly type: BuildCatalogEntry["aliases"][number]["type"];
+ readonly output: string;
+ readonly candidate: BuildState["plan"]["selections"][number]["candidate"];
+ readonly fidelity: BuildState["plan"]["selections"][number]["fidelity"];
+ readonly record: ReturnType<typeof summarizeRecord>;
+};
+
function artifactIdentity(value: unknown): ArtifactIdentity | undefined {
if (value === null || Array.isArray(value) || typeof value !== "object") return undefined;
const candidate = value as Readonly<Record<string, unknown>>;
@@ -32,7 +42,7 @@ function recordArtifact(record: TypedRecord): ArtifactIdentity | undefined {
}
export async function materializeRecord(
- runtime: Pick<LocalRuntime, "readArtifact">,
+ runtime: Pick<LocalRuntime, "openArtifact">,
record: TypedRecord,
destination: string,
): Promise<
@@ -51,16 +61,36 @@ export async function materializeRecord(
}
export async function materializeArtifact(
- runtime: Pick<LocalRuntime, "readArtifact">,
+ runtime: Pick<LocalRuntime, "openArtifact">,
artifact: ArtifactIdentity,
destination: string,
subject = "Build archive",
): Promise<{ readonly kind: "artifact"; readonly path: string; readonly digest: string; readonly size: number; readonly mediaType: string }> {
- const bytes = await runtime.readArtifact(artifact.digest);
- if (bytes === undefined) throw new Error(`Artifact ${artifact.digest} is absent from the selected ArtifactStore`);
- if (bytes.byteLength !== artifact.size) throw new Error(`Artifact ${artifact.digest} size differs from ${subject}`);
await mkdir(dirname(destination), { recursive: true });
- await writeFile(destination, bytes);
+ const source = await runtime.openArtifact(artifact.digest);
+ if (source === undefined) throw new Error(`Artifact ${artifact.digest} is absent from the selected ArtifactStore`);
+ const temporary = `${destination}.narratage-${randomUUID()}.part`;
+ const output = await open(temporary, "wx");
+ const hash = createHash("sha256");
+ let size = 0;
+ try {
+ for await (const chunk of source) {
+ size += chunk.byteLength;
+ hash.update(chunk);
+ await output.write(chunk);
+ }
+ await output.sync();
+ await output.close();
+ if (size !== artifact.size) throw new Error(`Artifact ${artifact.digest} size differs from ${subject}`);
+ const digest = `sha256:${hash.digest("hex")}`;
+ if (digest !== artifact.digest) throw new Error(`Artifact ${artifact.digest} bytes differ from ${subject}`);
+ // Validate the complete temporary file before replacing an explicitly selected destination.
+ await rename(temporary, destination);
+ } catch (error) {
+ await output.close().catch(() => undefined);
+ await rm(temporary, { force: true });
+ throw error;
+ }
return { kind: "artifact", path: destination, ...artifact };
}
@@ -107,6 +137,37 @@ function summarizeRecord(record: TypedRecord) {
};
}
+/**
+ * Public Logical Outputs that this exact Build actually selected and accepted.
+ *
+ * Catalog aliases alone are insufficient: a source can publish many Outputs while a Run targets
+ * only a small subgraph. Authored Record aliases are useful to `get`, but are not historical
+ * `build-record` Candidate roots, so this view intentionally excludes them as well.
+ */
+export function acceptedArchivedOutputs(
+ state: BuildState,
+ catalog: BuildCatalogEntry,
+): readonly ArchivedLogicalOutput[] {
+ if (catalog.core !== state.id) throw new Error("Build Catalog entry names another Core Build");
+ const records = new Map(state.records.map((record) => [record.id, record]));
+ const selections = new Map(state.plan.selections.map((selection) => [selection.output, selection]));
+ return catalog.aliases.flatMap((alias) => {
+ if (alias.ref.kind !== "logical-output") return [];
+ const selection = selections.get(alias.ref.id);
+ if (selection === undefined) return [];
+ const record = records.get(selection.record);
+ if (record === undefined) return [];
+ return [{
+ name: alias.name,
+ type: alias.type,
+ output: alias.ref.id,
+ candidate: selection.candidate,
+ fidelity: selection.fidelity,
+ record: summarizeRecord(record),
+ }];
+ });
+}
+
export function selectArchivedRecord(
state: BuildState,
options: {
diff --git a/packages/cli/src/distribution.ts b/packages/cli/src/distribution.ts
index 91964be0..f629371f 100644
--- a/packages/cli/src/distribution.ts
+++ b/packages/cli/src/distribution.ts
@@ -1,7 +1,9 @@
import type { NodeCompiler } from "@narratage/compiler-node";
-import type { LocalRuntime } from "@narratage/local";
-import type { ExternalServiceReport, RuntimeConfigDoctorResult } from "@narratage/local";
+import type { LocalCredentialControl, LocalRuntime, LocalRuntimeControl } from "@narratage/local";
+import type { ExternalServiceProgress, ExternalServiceReport, RuntimeConfigDoctorResult } from "@narratage/local";
import type { NodePackageContribution } from "@narratage/package-loader-node";
+import type { LoadedNodePackageSet } from "@narratage/package-loader-node";
+import type { CapabilityRef } from "@narratage/protocol";
import type { RunFrontend } from "@narratage/run";
export type CliCompilerOptions = {
@@ -24,13 +26,33 @@ export type CliDistribution = {
/** Explicitly trusted Run Frontends; source Headers select among them without suffix defaults. */
readonly runFrontends: readonly RunFrontend[];
createCompiler(options: CliCompilerOptions): NodeCompiler;
- createRuntimeFromConfig(path: string): Promise<LocalRuntime>;
+ /**
+ * Resolve the deterministic implementation lock named by a declarative
+ * Runtime Profile without constructing that Runtime. Trusted executable
+ * Runtime modules may decline and require an explicit CLI package lock.
+ */
+ resolveCompilationPackages?(path: string): Promise<{
+ readonly packageLock?: string;
+ readonly packageRoot?: string;
+ }>;
+ createRuntimeFromConfig(path: string, options?: {
+ /** Same-process package set already verified for compilation. */
+ readonly implementationPackages?: LoadedNodePackageSet;
+ }): Promise<LocalRuntime>;
+ /** Open only durable Stores for observation, cancellation, egress and maintenance. */
+ createRuntimeControlFromConfig(path: string): Promise<LocalRuntimeControl>;
+ /** Open only the selected Endpoint declaration and configured CredentialStores. */
+ createRuntimeCredentialsFromConfig(path: string, endpoint: string): Promise<LocalCredentialControl>;
/** Re-enter this exact Distribution as the hidden durable Worker process. */
runtimeWorkerLaunch(): { readonly command: string; readonly args: readonly string[] };
doctorRuntimeConfig(path: string): Promise<RuntimeConfigDoctorResult>;
/** The external programs a Runtime Profile implies: probe, prepare and start them. */
readonly externalServices: {
- up(path: string, options: { maxWaitMs?: number }): Promise<ExternalServiceResult>;
+ up(path: string, options: {
+ readonly maxWaitMs?: number;
+ readonly onProgress?: (event: ExternalServiceProgress) => void;
+ readonly capabilities?: readonly CapabilityRef[];
+ }): Promise<ExternalServiceResult>;
down(path: string): Promise<ExternalServiceResult>;
report(path: string): Promise<ExternalServiceResult>;
};
diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts
index c1a0ccdb..6c2d049d 100644
--- a/packages/cli/src/main.ts
+++ b/packages/cli/src/main.ts
@@ -2,18 +2,28 @@ import { dirname, extname, resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { readFile } from "node:fs/promises";
-import type { ExternalServiceReport, LocalRuntime } from "@narratage/local";
+import type {
+ ExternalServiceProgress,
+ ExternalServiceReport,
+ LocalBuildSubmission,
+ LocalRuntime,
+ LocalRuntimeControl,
+} from "@narratage/local";
import type { NodeCompiledSourceClosure } from "@narratage/compiler-node";
-import type { BuildCatalogDescriptor } from "@narratage/runtime";
+import type { BuildCatalogDescriptor, OperationProgress } from "@narratage/runtime";
import { isDigest } from "@narratage/protocol";
+import type { BuildState, CapabilityRef } from "@narratage/protocol";
import { parseSourceHeader } from "@narratage/source";
import {
createNodePackageLock,
loadNodePackageSet,
+ readNodePackageLock,
writeNodePackageLock,
} from "@narratage/package-loader-node";
+import type { LoadedNodePackageSet, NodePackageLock } from "@narratage/package-loader-node";
import {
+ acceptedArchivedOutputs,
collectArtifacts,
findArchivedArtifact,
inspectBuild,
@@ -26,6 +36,7 @@ import { collectRunFrontends, loadRunFile } from "./run-file.js";
import type { CliDistribution } from "./distribution.js";
import { writeCliHelp, writeCliOutput } from "./output.js";
import type { CliColorMode, CliIo } from "./output.js";
+import { withPackageLockEdit } from "./package-lock-edit.js";
import {
ensureRuntimeProcess,
markRuntimeProcessReady,
@@ -51,6 +62,10 @@ type ParsedArgs = {
readonly maxWaitMs: number | undefined;
readonly packageLock: string | undefined;
readonly packages: readonly string[];
+ readonly addPackages: readonly string[];
+ readonly removePackages: readonly string[];
+ readonly refresh: boolean;
+ readonly verifyPackageLock: boolean;
readonly record: string | undefined;
readonly output: string | undefined;
readonly name: string | undefined;
@@ -69,6 +84,10 @@ type ParsedArgs = {
readonly reason: string | undefined;
readonly slot: string | undefined;
readonly from: string | undefined;
+ /** Exact historical source path filter. It is Host presentation, never Build identity. */
+ readonly source: string | undefined;
+ /** Exact option spellings seen after positional dispatch. */
+ readonly seenOptions: readonly string[];
};
function parseArgs(argv: readonly string[]): ParsedArgs {
@@ -77,8 +96,9 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
const action = scoped ? tail[0] : undefined;
const positional = scoped ? tail.slice(1) : tail;
const noFile = command === "builds" || command === "queue";
- const file = noFile ? undefined : positional[0];
- const rest = noFile ? positional : positional.slice(1);
+ const hasFile = !noFile && positional[0] !== undefined && !positional[0]!.startsWith("--");
+ const file = hasFile ? positional[0] : undefined;
+ const rest = noFile || !hasFile ? positional : positional.slice(1);
const targets: string[] = [];
let workspaceRoot: string | undefined;
let packageRoot: string | undefined;
@@ -88,6 +108,10 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
let maxWaitMs: number | undefined;
let packageLock: string | undefined;
const packages: string[] = [];
+ const addPackages: string[] = [];
+ const removePackages: string[] = [];
+ let refresh = false;
+ let verifyPackageLock = false;
let substitute = false;
let record: string | undefined;
let output: string | undefined;
@@ -106,8 +130,18 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
let reason: string | undefined;
let slot: string | undefined;
let from: string | undefined;
+ let source: string | undefined;
+ const seenOptions = new Set<string>();
for (let index = 0; index < rest.length; index += 1) {
- const item = rest[index];
+ const item = rest[index]!;
+ if (item.startsWith("--")) {
+ const repeatable = [
+ "--package", "--add", "--remove", "--target", "--pin", "--json", "--jsonl", "--watch", "--verbose", "--debug",
+ "--no-color", "--refresh", "--accept-substitute", "--follow", "--apply", "--no-services",
+ ].includes(item);
+ if (!repeatable && seenOptions.has(item)) throw new Error(`${item} cannot be repeated`);
+ seenOptions.add(item);
+ }
if (item === "--json") {
json = true;
continue;
@@ -175,6 +209,28 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
index += 1;
continue;
}
+ if (item === "--add") {
+ const value = rest[index + 1];
+ if (value === undefined || value.startsWith("--")) throw new Error("--add requires an installed package name");
+ addPackages.push(value);
+ index += 1;
+ continue;
+ }
+ if (item === "--remove") {
+ const value = rest[index + 1];
+ if (value === undefined || value.startsWith("--")) throw new Error("--remove requires a selected package name");
+ removePackages.push(value);
+ index += 1;
+ continue;
+ }
+ if (item === "--refresh") {
+ refresh = true;
+ continue;
+ }
+ if (item === "--verify") {
+ verifyPackageLock = true;
+ continue;
+ }
if (item === "--accept-substitute") {
substitute = true;
continue;
@@ -290,6 +346,13 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
index += 1;
continue;
}
+ if (item === "--source") {
+ const value = rest[index + 1];
+ if (value === undefined || value.startsWith("--")) throw new Error("--source requires a source path");
+ source = resolve(value);
+ index += 1;
+ continue;
+ }
throw new Error(`unknown option ${item}`);
}
return {
@@ -306,6 +369,10 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
maxWaitMs,
packageLock,
packages,
+ addPackages,
+ removePackages,
+ refresh,
+ verifyPackageLock,
record,
output,
name,
@@ -323,13 +390,77 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
reason,
slot,
from,
+ source,
+ seenOptions: [...seenOptions],
};
}
+function assertCommandOptions(args: ParsedArgs): void {
+ const common = ["--json", "--color", "--no-color", "--verbose", "--debug"];
+ const allowed = new Set(common);
+ const add = (...items: readonly string[]): void => { for (const item of items) allowed.add(item); };
+ switch (args.command) {
+ case "lock-packages":
+ add("--package", "--add", "--remove", "--refresh", "--verify", "--package-root");
+ break;
+ case "services":
+ // This command has older, more specific diagnostics for deployment-selection
+ // flags and waiting on status/down; let its handler render those repairs.
+ add("--max-wait-ms", "--runtime", "--package-lock", "--package-root", "--package", "--apply");
+ break;
+ case "runtime":
+ if (args.action === "up" || args.action === "down") add("--max-wait-ms");
+ break;
+ case "gc":
+ add("--apply");
+ break;
+ case "auth":
+ add("--runtime", "--slot");
+ if (args.action === "login") add("--from");
+ break;
+ case "queue":
+ add("--runtime", "--watch", "--jsonl");
+ break;
+ case "get":
+ add("--runtime", "--name", "--record", "--output", "--artifact", "--to");
+ break;
+ case "cancel":
+ add("--runtime", "--reason");
+ break;
+ case "status":
+ case "builds":
+ case "history":
+ case "inspect":
+ case "operations":
+ case "operation":
+ add("--runtime");
+ if (args.command === "history") add("--source");
+ break;
+ case "check":
+ case "plan":
+ add("--runtime", "--package-lock", "--package-root", "--root");
+ break;
+ case "build":
+ add("--runtime", "--package-lock", "--package-root", "--root", "--build-id", "--follow",
+ "--max-wait-ms", "--no-services");
+ break;
+ }
+ const invalid = args.seenOptions.find((item) => !allowed.has(item));
+ if (invalid !== undefined) {
+ const command = args.action === undefined ? args.command : `${args.command} ${args.action}`;
+ throw new Error(`${invalid} does not apply to ${command}`);
+ }
+ if (args.seenOptions.includes("--color") && args.seenOptions.includes("--no-color")) {
+ throw new Error("--color and --no-color are mutually exclusive");
+ }
+}
+
function usage(): string {
return [
"usage:",
- " narratage lock-packages <svml.packages.lock> --package installed-name [--package installed-name] [--package-root directory]",
+ " narratage lock-packages <lock> --package name [...] [--package-root directory] # exact create/replace",
+ " narratage lock-packages <lock> (--add name [...] | --remove name [...]) [--package-root directory]",
+ " narratage lock-packages <lock> (--refresh | --verify) [--package-root directory]",
" narratage doctor <runtime-profile.json>",
" narratage services up|down|status <runtime-profile.json> [--max-wait-ms milliseconds]",
" narratage runtime up|status|logs|down <runtime-profile.json>",
@@ -337,9 +468,10 @@ function usage(): string {
" narratage gc <runtime-profile.json> [--apply]",
" narratage check <self-described-source> [--runtime profile.json] [--package-lock file] [--package-root directory] [--root workspace]",
" narratage plan <run-source> [--runtime profile.json] [--package-lock file] [--package-root directory] [--root workspace]",
- " narratage build <run-source> --runtime profile.json|./svml.runtime.ts [--package-lock file] [--package-root directory] [--root workspace] [--follow] [--no-services]",
+ " narratage build <run-source> --runtime profile.json|./svml.runtime.ts [--root workspace] [--follow] [--no-services]",
" narratage status <build-id> --runtime profile.json|./svml.runtime.ts",
" narratage builds --runtime profile.json|./svml.runtime.ts",
+ " narratage history [source-output-name] --runtime profile.json|./svml.runtime.ts [--source author.svml]",
" narratage inspect <build-id> --runtime profile.json|./svml.runtime.ts",
" narratage get <build-id> --runtime profile.json|./svml.runtime.ts [--name source-name|--record record-id|--output logical-output-id|--artifact digest] [--to path]",
" narratage operations <build-id> --runtime profile.json|./svml.runtime.ts",
@@ -362,6 +494,8 @@ function isLocalRuntime(value: unknown): value is LocalRuntime {
&& typeof value.builds === "function"
&& "readArtifact" in value
&& typeof value.readArtifact === "function"
+ && "openArtifact" in value
+ && typeof value.openArtifact === "function"
&& "credentials" in value
&& typeof value.credentials === "function"
&& "putCredential" in value
@@ -416,9 +550,14 @@ function createCatalogDescriptor(options: {
async function startDeclaredServices(
path: string,
distribution: CliDistribution,
+ capabilities: readonly CapabilityRef[],
+ onProgress?: (event: ExternalServiceProgress) => void,
): Promise<readonly ExternalServiceReport[] | undefined> {
if (extname(path) !== ".json") return undefined;
- const result = await distribution.externalServices.up(resolve(path), {});
+ const result = await distribution.externalServices.up(resolve(path), {
+ capabilities,
+ ...(onProgress === undefined ? {} : { onProgress }),
+ });
const unavailable = result.services.filter((item) => item.state.state !== "ready");
if (unavailable.length > 0) {
throw new Error([
@@ -435,8 +574,33 @@ async function startDeclaredServices(
return result.services;
}
-async function loadLocalRuntime(path: string, distribution: CliDistribution): Promise<LocalRuntime> {
- if (extname(path) === ".json") return await distribution.createRuntimeFromConfig(path);
+function demandedCapabilities(state: BuildState): readonly CapabilityRef[] {
+ const found = new Map<string, CapabilityRef>();
+ for (const step of state.plan.steps) {
+ const module = state.program.closure.modules.find((item) =>
+ item.ref.name === step.producer.module.name && item.ref.version === step.producer.module.version);
+ const producer = module?.manifest.producers.find((item) => item.name === step.producer.name);
+ if (producer === undefined) {
+ throw new Error(`BuildPlan refers to undeclared Producer ${step.producer.module.name}@${step.producer.module.version}#${step.producer.name}`);
+ }
+ for (const need of producer.needs) {
+ const key = `${need.capability.module.name}@${need.capability.module.version}#${need.capability.name}`;
+ found.set(key, need.capability);
+ }
+ }
+ return [...found.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([, value]) => value);
+}
+
+async function loadLocalRuntime(
+ path: string,
+ distribution: CliDistribution,
+ implementationPackages?: LoadedNodePackageSet,
+): Promise<LocalRuntime> {
+ if (extname(path) === ".json") {
+ return await distribution.createRuntimeFromConfig(path, {
+ ...(implementationPackages === undefined ? {} : { implementationPackages }),
+ });
+ }
// A Runtime config is trusted executable deployment code, never an Author Frontend or .svml import.
const imported = await import(pathToFileURL(path).href) as {
readonly default?: unknown;
@@ -451,6 +615,163 @@ async function loadLocalRuntime(path: string, distribution: CliDistribution): Pr
return candidate;
}
+async function loadRuntimeControl(
+ path: string,
+ distribution: CliDistribution,
+): Promise<LocalRuntimeControl> {
+ if (extname(path) === ".json") {
+ return await distribution.createRuntimeControlFromConfig(path);
+ }
+ // Trusted executable Runtime modules cannot be partially inspected without
+ // executing their own assembly. They retain the full-Runtime fallback.
+ return await loadLocalRuntime(path, distribution);
+}
+
+function submissionStatus(
+ dispatch: NonNullable<Awaited<ReturnType<LocalRuntime["status"]>>["dispatch"]>,
+): LocalBuildSubmission["status"] {
+ return dispatch.phase === "terminal"
+ ? dispatch.terminal!
+ : dispatch.phase === "leased" ? "running" : dispatch.phase;
+}
+
+function formatOperationProgress(progress: OperationProgress): string {
+ if (progress.completed === undefined) return progress.phase;
+ const amount = progress.total === undefined
+ ? String(progress.completed)
+ : `${progress.completed}/${progress.total}`;
+ return `${progress.phase} · ${amount}${progress.unit === undefined ? "" : ` ${progress.unit}`}`;
+}
+
+function inlineValuePreview(value: unknown, limit = 240): string {
+ const encoded = JSON.stringify(value);
+ if (encoded.length <= limit) return encoded;
+ return `${encoded.slice(0, Math.max(0, limit - 3))}...`;
+}
+
+async function observeBuild(
+ runtime: LocalRuntime,
+ initial: LocalBuildSubmission,
+ options: {
+ readonly maxWaitMs?: number;
+ readonly workerProfile?: string;
+ readonly onProgress?: (value: {
+ readonly build: string;
+ readonly phase: string;
+ readonly operations: Readonly<Record<string, number>>;
+ readonly activity: readonly string[];
+ }) => void;
+ },
+): Promise<LocalBuildSubmission> {
+ let current = initial;
+ let fingerprint: string | undefined;
+ let pollDelayMs = 100;
+ let observedActivity = false;
+ const startedAt = Date.now();
+ while (current.status !== "complete" && current.status !== "failed" && current.status !== "cancelled") {
+ const elapsedMs = Date.now() - startedAt;
+ const remainingMs = options.maxWaitMs === undefined
+ ? undefined
+ : options.maxWaitMs - elapsedMs;
+ if (remainingMs !== undefined && remainingMs <= 0) break;
+ const waitMs = remainingMs === undefined
+ ? pollDelayMs
+ : Math.min(pollDelayMs, remainingMs);
+ await new Promise((resolveWait) => setTimeout(resolveWait, waitMs));
+ const status = await runtime.activity(current.id);
+ observedActivity = true;
+ if (status.dispatch === undefined) {
+ throw new Error(`Build ${current.id} disappeared while it was being observed`);
+ }
+ current = {
+ id: current.id,
+ state: current.state,
+ status: submissionStatus(status.dispatch),
+ dispatch: status.dispatch,
+ };
+ if (!["complete", "failed", "cancelled"].includes(current.status)
+ && options.workerProfile !== undefined) {
+ const worker = await runtimeProcessStatus(options.workerProfile);
+ if (worker.state !== "running") {
+ throw new Error(
+ `Runtime Worker is ${worker.state}; Build ${current.id} remains durable. `
+ + `Run narratage runtime up ${options.workerProfile}, then observe the same Build again`,
+ );
+ }
+ }
+ const operations = Object.fromEntries([...new Set(status.operations.map((item) => item.status))]
+ .sort().map((state) => [state, status.operations.filter((item) => item.status === state).length]));
+ const activity = status.operations
+ .filter((item) => item.status === "created" || item.status === "pending")
+ .map((item) => `${item.endpoint}: ${item.progress === undefined
+ ? item.status
+ : formatOperationProgress(item.progress)}`);
+ const nextFingerprint = JSON.stringify({
+ phase: status.dispatch.phase,
+ terminal: status.dispatch.terminal,
+ operations,
+ activity: status.operations.map((item) => ({
+ id: item.id,
+ revision: item.revision,
+ status: item.status,
+ progress: item.progress,
+ cancellation: item.cancellation?.status,
+ })),
+ });
+ if (nextFingerprint !== fingerprint) {
+ fingerprint = nextFingerprint;
+ pollDelayMs = 100;
+ options.onProgress?.({
+ build: current.id,
+ phase: status.dispatch.phase,
+ operations,
+ activity,
+ });
+ } else {
+ pollDelayMs = Math.min(1_000, pollDelayMs * 2);
+ }
+ }
+ if (observedActivity) {
+ const status = await runtime.status(current.id);
+ if (status.build === undefined || status.dispatch === undefined) {
+ throw new Error(`Build ${current.id} disappeared while it was being observed`);
+ }
+ current = {
+ id: current.id,
+ state: status.build.state,
+ status: submissionStatus(status.dispatch),
+ dispatch: status.dispatch,
+ };
+ }
+ return current;
+}
+
+type RuntimeArchiveView = Pick<LocalRuntimeControl, "status" | "close">;
+
+/**
+ * A Run Source needs the archive only when it names a historical Build Candidate.
+ * Delay Store assembly until that edge is actually resolved, while still letting
+ * the Runtime Profile single-source the author package lock for every plan.
+ */
+function lazyRuntimeArchive(
+ path: string,
+ distribution: CliDistribution,
+): RuntimeArchiveView {
+ let loading: Promise<LocalRuntimeControl> | undefined;
+ const open = (): Promise<LocalRuntimeControl> => {
+ loading ??= loadRuntimeControl(path, distribution);
+ return loading;
+ };
+ return {
+ async status(build) {
+ return await (await open()).status(build);
+ },
+ async close() {
+ if (loading !== undefined) await (await loading).close();
+ },
+ };
+}
+
export async function runCli(
argv: readonly string[],
io: CliIo,
@@ -473,6 +794,18 @@ export async function runCli(
color: args.color,
verbose: args.verbose,
}, { kind: "operational", machine, title, status, facts, lines });
+ const reportServiceProgress = args.json || args.jsonl
+ ? undefined
+ : (event: ExternalServiceProgress): void => {
+ const verb = {
+ checking: "Checking",
+ preparing: "Preparing",
+ starting: "Starting",
+ waiting: "Waiting for",
+ ready: "Ready",
+ }[event.phase];
+ io.write(` · ${verb} ${event.id}\n`);
+ };
if (args.command === "_worker") {
if (args.file === undefined || args.readyFile === undefined) throw new Error("internal Worker launch is incomplete");
const runtime = await loadLocalRuntime(resolve(args.file), distribution);
@@ -497,12 +830,13 @@ export async function runCli(
}
const known = args.command === "lock-packages" || args.command === "check" || args.command === "plan"
|| args.command === "build" || args.command === "status" || args.command === "builds"
+ || args.command === "history"
|| args.command === "inspect" || args.command === "get" || args.command === "cancel"
|| args.command === "doctor" || args.command === "gc" || args.command === "services"
|| args.command === "runtime" || args.command === "queue" || args.command === "operations"
|| args.command === "operation";
const operational = known || args.command === "auth";
- const fileOptional = args.command === "builds" || args.command === "queue";
+ const fileOptional = args.command === "builds" || args.command === "history" || args.command === "queue";
if (!operational || (!fileOptional && args.file === undefined)) {
throw new Error(usage());
}
@@ -511,20 +845,146 @@ export async function runCli(
throw new Error("--jsonl applies only to queue --watch");
}
if (args.watch && args.json) throw new Error("queue --watch is a stream; use --jsonl instead of --json");
+ if (args.noServices && args.command !== "build") {
+ throw new Error("--no-services is only valid for build; no other command starts an external program");
+ }
+ if (args.command === "history" && args.file === undefined && args.source === undefined) {
+ throw new Error("history requires an output name or --source path");
+ }
+ if (args.targets.length > 0 || args.substitute || args.pins.length > 0) {
+ throw new Error("Targets, Candidate selections and fidelity belong in a self-described Run Source; CLI --target, --pin and --accept-substitute are not supported");
+ }
+ assertCommandOptions(args);
if (args.command === "lock-packages") {
- if (args.packages.length === 0) throw new Error("lock-packages requires at least one --package");
+ const exact = args.packages.length > 0;
+ const mutation = args.addPackages.length > 0 || args.removePackages.length > 0;
+ const modes = [exact, mutation, args.refresh, args.verifyPackageLock].filter(Boolean).length;
+ if (modes !== 1) {
+ throw new Error(
+ "lock-packages requires exactly one operation: an exact --package selection, --add/--remove, --refresh, or --verify",
+ );
+ }
if (args.packageLock !== undefined) throw new Error("lock-packages does not accept --package-lock");
if (args.workspaceRoot !== undefined) {
throw new Error("lock-packages does not compile a Workspace; use --package-root to locate installed packages");
}
const output = resolve(args.file!);
const packageRoot = args.packageRoot ?? distribution.packageRoot ?? dirname(output);
- const lock = await createNodePackageLock(args.packages, packageRoot);
- await writeNodePackageLock(output, lock);
- writeOperational({ ok: true, packageLock: output, digest: lock.digest, packages: lock.packages },
- "Package lock written", "success", [
- ["Path", output], ["Packages", String(lock.packages.length)], ["Digest", lock.digest],
+
+ if (args.verifyPackageLock) {
+ const loaded = await loadNodePackageSet(output, packageRoot);
+ writeOperational({
+ ok: true,
+ mode: "verify",
+ packageLock: output,
+ digest: loaded.lock.digest,
+ selected: loaded.lock.selected,
+ packages: loaded.lock.packages,
+ }, "Package lock verified", "success", [
+ ["Path", output], ["Selected", String(loaded.lock.selected.length)],
+ ["Activated", String(loaded.lock.packages.length)], ["Digest", loaded.lock.digest],
]);
+ return;
+ }
+
+ await withPackageLockEdit(output, async () => {
+ let mode: "replace" | "mutate" | "refresh";
+ let selected: readonly string[];
+ let added: readonly string[] = [];
+ let removed: readonly string[] = [];
+ let previous: NodePackageLock | undefined;
+ let retainedSelection: readonly string[] | undefined;
+ if (exact) {
+ mode = "replace";
+ selected = [...new Set(args.packages)].sort();
+ } else {
+ const current = await readNodePackageLock(output);
+ if (args.refresh) {
+ mode = "refresh";
+ selected = current.selected;
+ } else {
+ mode = "mutate";
+ const additions = new Set(args.addPackages);
+ const removals = new Set(args.removePackages);
+ const conflict = [...additions].find((name) => removals.has(name));
+ if (conflict !== undefined) {
+ throw new Error(`${conflict} cannot be added and removed in the same package-lock transaction`);
+ }
+ const currentSet = new Set(current.selected);
+ const missing = [...removals].find((name) => !currentSet.has(name));
+ if (missing !== undefined) {
+ throw new Error(`${missing} is not selected by ${output}; no package authority was removed`);
+ }
+ added = [...additions].filter((name) => !currentSet.has(name)).sort();
+ removed = [...removals].sort();
+ if (added.length === 0 && removed.length === 0) {
+ writeOperational({
+ ok: true,
+ changed: false,
+ mode,
+ packageLock: output,
+ digest: current.digest,
+ selected: current.selected,
+ added,
+ removed,
+ packages: current.packages,
+ }, "Package lock unchanged", "info", [
+ ["Path", output], ["Selected", String(current.selected.length)], ["Digest", current.digest],
+ ]);
+ return;
+ }
+ previous = current;
+ retainedSelection = current.selected.filter((name) => !removals.has(name));
+ for (const name of removals) currentSet.delete(name);
+ for (const name of additions) currentSet.add(name);
+ selected = [...currentSet].sort();
+ }
+ }
+ const lock = await createNodePackageLock(selected, packageRoot,
+ previous === undefined || retainedSelection === undefined
+ ? {}
+ : { retain: { from: previous, selected: retainedSelection } });
+ const artifactKey = (item: { readonly name: string; readonly version: string }): string =>
+ `${item.name}@${item.version}`;
+ const packageKey = (item: NodePackageLock["packages"][number]): string =>
+ `${item.package.name}@${item.package.version}`;
+ const physicalBefore = new Set(previous?.artifacts.map(artifactKey) ?? []);
+ const physicalAfter = new Set(lock.artifacts.map(artifactKey));
+ const activatedBefore = new Set(previous?.packages.map(packageKey) ?? []);
+ const activatedAfter = new Set(lock.packages.map(packageKey));
+ const closureChanges = previous === undefined ? undefined : {
+ physical: {
+ added: [...physicalAfter].filter((key) => !physicalBefore.has(key)).sort(),
+ removed: [...physicalBefore].filter((key) => !physicalAfter.has(key)).sort(),
+ },
+ activated: {
+ added: [...activatedAfter].filter((key) => !activatedBefore.has(key)).sort(),
+ removed: [...activatedBefore].filter((key) => !activatedAfter.has(key)).sort(),
+ },
+ };
+ await writeNodePackageLock(output, lock);
+ writeOperational({
+ ok: true,
+ changed: true,
+ mode,
+ packageLock: output,
+ digest: lock.digest,
+ selected: lock.selected,
+ added,
+ removed,
+ ...(closureChanges === undefined ? {} : { closureChanges }),
+ packages: lock.packages,
+ },
+ "Package lock written", "success", [
+ ["Path", output], ["Mode", mode], ["Selected", String(lock.selected.length)],
+ ["Activated", String(lock.packages.length)],
+ ...(closureChanges === undefined ? [] : [
+ ["Physical + / -", `${closureChanges.physical.added.length} / ${closureChanges.physical.removed.length}`] as const,
+ ["Activated + / -", `${closureChanges.activated.added.length} / ${closureChanges.activated.removed.length}`] as const,
+ ]),
+ ["Digest", lock.digest],
+ ]);
+ });
return;
}
if (args.command === "doctor") {
@@ -541,6 +1001,7 @@ export async function runCli(
diagnostics: result.diagnostics,
};
writeCliOutput(io, args, { kind: "doctor", machine, profile });
+ if (!machine.ok) io.setExitCode?.(1);
return;
}
if (args.command === "services") {
@@ -555,9 +1016,10 @@ export async function runCli(
throw new Error("--max-wait-ms applies to services up");
}
const profile = resolve(args.file!);
- const result = args.action === "up"
+ const result = args.action === "up"
? await distribution.externalServices.up(profile, {
...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
+ ...(reportServiceProgress === undefined ? {} : { onProgress: reportServiceProgress }),
})
: args.action === "down"
? await distribution.externalServices.down(profile)
@@ -572,6 +1034,7 @@ export async function runCli(
["Services", String(result.services.length)],
["Ready", String(result.services.filter((item) => item.state.state === "ready").length)],
]);
+ if (!machine.ok) io.setExitCode?.(1);
return;
}
if (args.command === "runtime") {
@@ -582,16 +1045,21 @@ export async function runCli(
if (args.action === "up") {
const external = await distribution.externalServices.up(profile, {
...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
+ ...(reportServiceProgress === undefined ? {} : { onProgress: reportServiceProgress }),
});
const processState = await ensureRuntimeProcess(
profile,
distribution.runtimeWorkerLaunch(),
args.maxWaitMs ?? 10_000,
);
- writeOperational({ ok: true, worker: processState, services: external.services }, "Runtime is up", "success", [
+ const ok = processState.state === "running"
+ && external.services.every((item) => item.state.state === "ready");
+ writeOperational({ ok, worker: processState, services: external.services }, "Runtime is up",
+ ok ? "success" : "warning", [
["Worker", String(processState.pid)],
["External services", String(external.services.length)],
]);
+ if (!ok) io.setExitCode?.(1);
return;
}
if (args.action === "logs") {
@@ -619,7 +1087,7 @@ export async function runCli(
}
const worker = await runtimeProcessStatus(profile);
const external = await distribution.externalServices.report(profile);
- const runtime = await loadLocalRuntime(profile, distribution);
+ const runtime = await loadRuntimeControl(profile, distribution);
try {
const queue = await runtime.queue();
const counts = Object.fromEntries(["queued", "leased", "waiting", "blocked", "settling", "terminal"]
@@ -635,8 +1103,9 @@ export async function runCli(
["Queued", String(counts.queued ?? 0)],
["Running", String(counts.leased ?? 0)],
["Waiting", String(counts.waiting ?? 0)],
- ["Capacity", String(queue.capacity.length)],
+ ["Capacity reservations", String(queue.capacity.length)],
]);
+ if (!machine.ok) io.setExitCode?.(1);
} finally {
await runtime.close();
}
@@ -647,7 +1116,7 @@ export async function runCli(
|| args.packages.length > 0) {
throw new Error("gc reads all deployment selection from the Runtime Profile itself");
}
- const runtime = await loadLocalRuntime(resolve(args.file!), distribution);
+ const runtime = await loadRuntimeControl(resolve(args.file!), distribution);
try {
if (!("garbageCollectArtifacts" in runtime) || typeof runtime.garbageCollectArtifacts !== "function") {
throw new Error("selected Runtime does not expose Artifact maintenance");
@@ -676,7 +1145,10 @@ export async function runCli(
}
if (args.runtime === undefined) throw new Error("auth requires --runtime");
if (args.from !== undefined && args.action !== "login") throw new Error("--from applies only to auth login");
- const runtime = await loadLocalRuntime(args.runtime, distribution);
+ if (extname(args.runtime) !== ".json") {
+ throw new Error("auth requires a declarative JSON Runtime Profile so credentials can be opened without executing the whole deployment");
+ }
+ const runtime = await distribution.createRuntimeCredentialsFromConfig(args.runtime, args.file!);
try {
let credentials = await runtime.credentials(args.file!);
if (args.slot !== undefined) credentials = credentials.filter((item) => item.slot === args.slot);
@@ -720,6 +1192,7 @@ export async function runCli(
return;
}
if (args.apply) throw new Error("--apply is only valid for gc");
+ if (args.refresh) throw new Error("--refresh is only valid for lock-packages");
if (args.noServices && args.command !== "build") {
throw new Error("--no-services is only valid for build; no other command starts an external program");
}
@@ -728,41 +1201,90 @@ export async function runCli(
&& args.command !== "get") {
throw new Error("--name, --record, --output, --artifact and --to are only valid for get");
}
- if (args.targets.length > 0 || args.substitute || args.pins.length > 0) {
- throw new Error("Targets, Candidate selections and fidelity belong in a self-described Run Source; CLI --target, --pin and --accept-substitute are not supported");
+ if (args.source !== undefined && args.command !== "history") {
+ throw new Error("--source is only valid for history");
}
if (args.command === "status" || args.command === "builds" || args.command === "inspect"
- || args.command === "get" || args.command === "cancel" || args.command === "queue"
+ || args.command === "history" || args.command === "get" || args.command === "cancel" || args.command === "queue"
|| args.command === "operations" || args.command === "operation") {
if (args.runtime === undefined) throw new Error(`${args.command} requires --runtime`);
- const runtime = await loadLocalRuntime(args.runtime, distribution);
+ const runtime = await loadRuntimeControl(args.runtime, distribution);
try {
if (args.command === "queue") {
- const writeQueue = async (): Promise<boolean> => {
- const queue = await runtime.queue();
+ let previous: string | undefined;
+ const writeQueue = async (): Promise<void> => {
+ const [queue, worker] = await Promise.all([
+ runtime.queue(),
+ runtimeProcessStatus(args.runtime!),
+ ]);
+ const fingerprint = JSON.stringify({
+ dispatches: queue.dispatches,
+ capacity: queue.capacity,
+ worker,
+ operations: queue.operations.map((item) => ({
+ id: item.id,
+ revision: item.revision,
+ status: item.status,
+ progress: item.progress,
+ cancellation: item.cancellation?.status,
+ })),
+ });
+ if (args.watch && fingerprint === previous) return;
+ previous = fingerprint;
const value = {
format: "narratage.cli-queue@1",
at: Date.now(),
+ worker,
dispatches: queue.dispatches,
capacity: queue.capacity,
+ operations: queue.operations.map((item) => ({
+ id: item.id,
+ build: item.build,
+ endpoint: item.endpoint,
+ attempt: item.attempt,
+ revision: item.revision,
+ status: item.status,
+ ...(item.progress === undefined ? {} : { progress: item.progress }),
+ ...(item.cancellation === undefined ? {} : { cancellation: item.cancellation }),
+ })),
};
const active = queue.dispatches.filter((item) => item.phase !== "terminal");
+ const activeOperations = queue.operations.filter((item) =>
+ item.status === "created" || item.status === "pending");
+ const buildLines = active.slice(0, args.verbose ? undefined : 12).map((item) =>
+ `${item.build}: ${item.phase}${item.admission === "open" ? "" : ` · ${item.admission}`}`);
+ const capacityLines = queue.capacity.slice(0, args.verbose ? undefined : 12).map((item) =>
+ `${item.lane}: ${item.inFlight ? "in flight" : "parked"} · ${item.build}`);
+ const operationLines = activeOperations.slice(0, args.verbose ? undefined : 12).map((item) =>
+ `${item.build} · ${item.endpoint}: ${item.progress === undefined
+ ? item.status
+ : formatOperationProgress(item.progress)}`);
writeOperational(value, "Runtime queue", active.length === 0 ? "success" : "info", [
["Active Builds", String(active.length)],
- ["Total Builds", String(queue.dispatches.length)],
+ ["Active Operations", String(activeOperations.length)],
+ ["Worker", worker.pid === undefined ? worker.state : `${worker.state} · ${worker.pid}`],
+ ["Queued Builds", String(queue.dispatches.length)],
["Reserved capacity", String(queue.capacity.length)],
- ], active.slice(0, args.verbose ? undefined : 12).map((item) =>
- `${item.build}: ${item.phase}${item.admission === "open" ? "" : ` · ${item.admission}`}`));
- return queue.dispatches.every((item) => item.phase === "terminal");
+ ], [
+ ...buildLines,
+ ...(operationLines.length === 0 ? [] : ["Operations:", ...operationLines]),
+ ...(capacityLines.length === 0 ? [] : ["Capacity:", ...capacityLines]),
+ ]);
};
if (!args.watch) await writeQueue();
- else while (!(await writeQueue())) await new Promise((resolveWait) => setTimeout(resolveWait, 1_000));
+ else while (true) {
+ await writeQueue();
+ await new Promise((resolveWait) => setTimeout(resolveWait, 1_000));
+ }
} else if (args.command === "operations") {
const status = await runtime.status(args.file!);
const machine = { build: args.file, operations: status.operations };
writeOperational(machine, "Build operations", "info", [
["Build", args.file!], ["Operations", String(status.operations.length)],
- ], status.operations.map((item) => `${item.id}: ${item.status} · ${item.endpoint}`));
+ ], status.operations.map((item) => `${item.id}: ${item.status} · ${item.endpoint}`
+ + `${item.progress === undefined ? "" : ` · ${formatOperationProgress(item.progress)}`}`
+ + `${item.failure === undefined ? "" : ` · ${item.failure.code}: ${item.failure.message}`}`
+ + `${item.cancellation === undefined ? "" : ` · cancellation ${item.cancellation.status}`}`));
} else if (args.command === "operation") {
if (!isDigest(args.file!)) throw new Error("operation id must be a content digest");
const operation = await runtime.operation(args.file!);
@@ -770,7 +1292,11 @@ export async function runCli(
operation === undefined ? "warning" : "info", operation === undefined ? [] : [
["Operation", operation.id], ["Status", operation.status], ["Endpoint", operation.endpoint],
["Attempt", String(operation.attempt)],
+ ...(operation.progress === undefined ? [] : [["Progress", formatOperationProgress(operation.progress)] as const]),
+ ], operation?.failure === undefined ? [] : [
+ `${operation.failure.code}: ${operation.failure.message}`,
]);
+ if (operation === undefined) io.setExitCode?.(1);
} else if (args.command === "builds") {
const entries = await runtime.builds();
const builds = await Promise.all(entries.map(async (entry) => {
@@ -786,6 +1312,46 @@ export async function runCli(
}));
writeOperational({ builds }, "Build archive", "info", [["Builds", String(builds.length)]],
builds.slice(0, args.verbose ? undefined : 20).map((item) => `${item.build}: ${item.status ?? "unknown"}`));
+ } else if (args.command === "history") {
+ const catalogs = await runtime.builds();
+ const entries = (await Promise.all(catalogs.map(async (catalog) => {
+ if (args.source !== undefined && catalog.source.path !== args.source) return [];
+ const status = await runtime.status(catalog.build);
+ if (status.build === undefined) return [];
+ return acceptedArchivedOutputs(status.build.state, catalog)
+ .filter((output) => args.file === undefined || output.name === args.file)
+ .map((output) => ({
+ build: catalog.build,
+ core: catalog.core,
+ createdAt: catalog.createdAt,
+ updatedAt: catalog.updatedAt,
+ status: status.dispatch === undefined
+ ? status.build!.state.status
+ : submissionStatus(status.dispatch),
+ source: catalog.source,
+ ...(catalog.run === undefined ? {} : { run: catalog.run }),
+ output,
+ }));
+ }))).flat()
+ .sort((left, right) => right.createdAt - left.createdAt
+ || left.output.name.localeCompare(right.output.name)
+ || left.build.localeCompare(right.build));
+ const query = {
+ ...(args.file === undefined ? {} : { output: args.file }),
+ ...(args.source === undefined ? {} : { source: args.source }),
+ };
+ const shown = entries.slice(0, args.verbose ? undefined : 20);
+ writeOperational({ query, entries }, entries.length === 0 ? "No accepted output history" : "Output history",
+ entries.length === 0 ? "warning" : "info", [
+ ...(args.file === undefined ? [] : [["Output", args.file] as const]),
+ ...(args.source === undefined ? [] : [["Source", args.source] as const]),
+ ["Records", String(entries.length)],
+ ], shown.map((item) => {
+ const created = new Date(item.createdAt).toISOString();
+ const digest = item.output.record.digest;
+ const shortDigest = `${digest.slice(0, 18)}…`;
+ return `${item.build}: ${item.output.name} · ${created} · ${shortDigest}`;
+ }));
} else if (args.command === "status") {
const status = await runtime.status(args.file!);
const machine = {
@@ -805,20 +1371,27 @@ export async function runCli(
endpoint: operation.endpoint,
attempt: operation.attempt,
status: operation.status,
+ revision: operation.revision,
...(operation.wakeAt === undefined ? {} : { wakeAt: operation.wakeAt }),
+ ...(operation.progress === undefined ? {} : { progress: operation.progress }),
...(operation.failure === undefined ? {} : { failure: operation.failure }),
...(operation.cancellation === undefined ? {} : { cancellation: operation.cancellation }),
})),
dispatch: status.dispatch,
};
const phase = status.dispatch?.phase ?? "missing";
+ const notableOperations = status.operations.filter((operation) =>
+ operation.status === "failed" || operation.status === "cancelled");
writeOperational(machine, status.build === undefined ? "Build not found" : "Build status",
status.build === undefined ? "warning" : status.dispatch?.terminal === "failed" ? "error" : "info", [
["Build", args.file!],
["Core", status.build?.state.status ?? "missing"],
["Dispatch", phase],
["Operations", String(status.operations.length)],
- ]);
+ ], notableOperations.map((operation) => operation.failure === undefined
+ ? `${operation.endpoint}: ${operation.status}`
+ : `${operation.endpoint}: ${operation.failure.code} — ${operation.failure.message}`));
+ if (status.build === undefined || status.dispatch?.terminal === "failed") io.setExitCode?.(1);
} else if (args.command === "inspect") {
const status = await runtime.status(args.file!);
if (status.build === undefined) throw new Error(`Build ${args.file} does not exist`);
@@ -832,7 +1405,9 @@ export async function runCli(
endpoint: operation.endpoint,
attempt: operation.attempt,
status: operation.status,
+ revision: operation.revision,
...(operation.wakeAt === undefined ? {} : { wakeAt: operation.wakeAt }),
+ ...(operation.progress === undefined ? {} : { progress: operation.progress }),
...(operation.failure === undefined ? {} : { failure: operation.failure }),
})),
};
@@ -848,7 +1423,6 @@ export async function runCli(
}
if (args.artifact !== undefined) {
const references = findArchivedArtifact(status.build.state, args.artifact);
- if (references.length === 0) throw new Error(`Build ${args.file} does not reference Artifact ${args.artifact}`);
if (args.to === undefined) {
writeOperational({ build: status.build.build, artifact: args.artifact, references }, "Artifact references", "info", [
["Build", status.build.build], ["Artifact", args.artifact], ["References", String(references.length)],
@@ -884,6 +1458,9 @@ export async function runCli(
};
writeOperational(machine, "Archived Record", "info", [
["Build", status.build.build], ["Record", record.id],
+ ...(record.value.kind === "inline"
+ ? [["Value", inlineValuePreview(record.value.value)] as const]
+ : []),
["Artifacts", String(machine.artifacts.length)],
]);
} else {
@@ -912,6 +1489,7 @@ export async function runCli(
result === undefined ? "warning" : "success", [
["Build", args.file!], ["Admission", result?.admission ?? "missing"], ["Phase", result?.phase ?? "missing"],
]);
+ if (result === undefined) io.setExitCode?.(1);
} else if (args.action === "operation") {
if (args.file === undefined || !isDigest(args.file)) {
throw new Error("cancel operation requires an Operation digest");
@@ -930,6 +1508,7 @@ export async function runCli(
["Operation", args.file!], ["Build", result?.build ?? "missing"],
["Execution", result?.status ?? "missing"], ["Control", result?.cancellation?.status ?? "missing"],
]);
+ if (result === undefined) io.setExitCode?.(1);
} else {
throw new Error("cancel must name its scope: cancel build <build-id> or cancel operation <operation-id>");
}
@@ -940,14 +1519,25 @@ export async function runCli(
return;
}
if (args.packages.length > 0) throw new Error("--package is only valid for lock-packages");
- if (args.packageRoot !== undefined && args.packageLock === undefined) {
+ const runtimePackageSelection = args.runtime === undefined
+ ? undefined
+ : await distribution.resolveCompilationPackages?.(args.runtime);
+ if (args.packageLock !== undefined && runtimePackageSelection?.packageLock !== undefined
+ && resolve(args.packageLock) !== resolve(runtimePackageSelection.packageLock)) {
+ throw new Error(
+ `--package-lock ${args.packageLock} differs from the deterministic package lock selected by ${args.runtime}`,
+ );
+ }
+ const effectivePackageLock = args.packageLock ?? runtimePackageSelection?.packageLock;
+ const effectivePackageRoot = args.packageRoot ?? runtimePackageSelection?.packageRoot;
+ if (effectivePackageRoot !== undefined && effectivePackageLock === undefined) {
throw new Error("--package-root locates the installed packages named by --package-lock; provide both options");
}
- const loadedPackageSet = args.packageLock === undefined
+ const loadedPackageSet = effectivePackageLock === undefined
? undefined
: await loadNodePackageSet(
- args.packageLock,
- args.packageRoot ?? distribution.packageRoot ?? dirname(args.packageLock),
+ effectivePackageLock,
+ effectivePackageRoot ?? distribution.packageRoot ?? dirname(effectivePackageLock),
);
const packageContributions = loadedPackageSet?.contributions ?? distribution.builtInPackageContributions;
const runFrontends = collectRunFrontends(distribution.runFrontends, packageContributions);
@@ -968,9 +1558,11 @@ export async function runCli(
throw new Error(`${args.command} requires a self-described Run Source; check Author Sources independently`);
}
if (args.command === "check") {
- let runtime: LocalRuntime | undefined;
+ let runtime: RuntimeArchiveView | undefined;
try {
- runtime = args.runtime === undefined ? undefined : await loadLocalRuntime(args.runtime, distribution);
+ runtime = args.runtime === undefined || !runMode
+ ? undefined
+ : lazyRuntimeArchive(args.runtime, distribution);
if (runMode) {
const loaded = await loadRunFile({
workspace,
@@ -1036,7 +1628,7 @@ export async function runCli(
}
if (args.command === "build") {
if (args.runtime === undefined) throw new Error("build requires --runtime with a Runtime Profile or trusted local config module");
- const runtime = await loadLocalRuntime(args.runtime, distribution);
+ const runtime = await loadLocalRuntime(args.runtime, distribution, loadedPackageSet);
try {
const loadedRun = await loadRunFile({
workspace,
@@ -1046,33 +1638,67 @@ export async function runCli(
runtime,
});
const result = loadedRun.compiler.planCompilation(loadedRun, loadedPackageSet?.lock.digest);
+ const catalog = createCatalogDescriptor({
+ core: result.state.id,
+ source: loadedRun.authorSource,
+ compilation: result.compilation.author,
+ run: {
+ path: loadedRun.path,
+ targetSet: loadedRun.run.graph.selectedTargets,
+ },
+ });
const request = {
id: args.buildId ?? result.state.id,
state: result.state,
- catalog: createCatalogDescriptor({
- core: result.state.id,
- source: loadedRun.authorSource,
- compilation: result.compilation.author,
- run: {
- path: loadedRun.path,
- targetSet: loadedRun.run.graph.selectedTargets,
- },
- }),
+ catalog,
attachments: result.compilation.attachments,
} as const;
- const services = args.noServices ? undefined : await startDeclaredServices(args.runtime, distribution);
- let built = await runtime.build(request);
+ const services = args.noServices
+ ? undefined
+ : await startDeclaredServices(
+ args.runtime,
+ distribution,
+ demandedCapabilities(result.state),
+ reportServiceProgress,
+ );
const worker = await ensureRuntimeProcess(
args.runtime,
distribution.runtimeWorkerLaunch(),
args.maxWaitMs ?? 10_000,
);
+ let built = await runtime.build(request);
if (args.follow) {
- built = await runtime.build(request, {
- follow: true,
+ built = await observeBuild(runtime, built, {
...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
+ workerProfile: args.runtime,
+ ...(args.json || args.jsonl ? {} : {
+ onProgress: (progress) => {
+ const operations = Object.entries(progress.operations)
+ .map(([status, count]) => `${count} ${status}`)
+ .join(", ");
+ io.write(` · ${progress.build}: ${progress.phase}`
+ + `${operations.length === 0 ? "" : ` · ${operations}`}\n`);
+ for (const line of progress.activity) io.write(` ${line}\n`);
+ },
+ }),
});
}
+ const targetOutputs = new Set(built.state.request.targets.map((target) => target.output));
+ const targetAliases = catalog.aliases.filter((alias) =>
+ alias.ref.kind === "logical-output" && targetOutputs.has(alias.ref.id));
+ const targetPresentations = targetAliases.map((alias) => {
+ const selection = built.state.plan.selections.find((item) => item.output === alias.ref.id);
+ const record = selection === undefined
+ ? undefined
+ : built.state.records.find((item) => item.id === selection.record);
+ return {
+ alias,
+ record,
+ ...(record?.value.kind === "inline"
+ ? { inline: inlineValuePreview(record.value.value) }
+ : {}),
+ };
+ });
const machine = {
build: built.id,
core: built.state.id,
@@ -1096,21 +1722,46 @@ export async function runCli(
...(built.dispatch.reason === undefined ? {} : { reason: built.dispatch.reason }),
},
};
- writeOperational(machine, args.follow ? "Build observation finished" : "Build submitted",
- built.status === "failed" ? "error" : built.status === "cancelled" ? "warning" : "success", [
+ const terminalLines = targetPresentations.length === 0
+ ? [`Inspect narratage inspect ${built.id} --runtime ${args.runtime}`]
+ : [
+ `Inspect narratage inspect ${built.id} --runtime ${args.runtime}`,
+ ...targetPresentations
+ .filter((item) => item.inline !== undefined)
+ .slice(0, args.verbose ? undefined : 8)
+ .map((item) => `Result ${item.alias.name} = ${item.inline}`),
+ ...targetPresentations
+ .filter((item) => item.inline === undefined)
+ .slice(0, args.verbose ? undefined : 4)
+ .map((item) =>
+ `Export narratage get ${built.id} --runtime ${args.runtime} --name ${item.alias.name} --to <path>`),
+ ];
+ const terminal = built.dispatch.phase === "terminal";
+ writeOperational(machine, args.follow
+ ? terminal ? "Build finished" : "Build still running"
+ : "Build submitted",
+ built.status === "failed" ? "error"
+ : built.status === "cancelled" || (args.follow && !terminal) ? "warning" : "success", [
["Build", built.id],
["Status", built.status],
["Worker", String(worker.pid)],
["Goals", String(machine.goals.length)],
+ ], terminal ? terminalLines : [
+ `Status narratage status ${built.id} --runtime ${args.runtime}`,
+ `Watch narratage queue --runtime ${args.runtime} --watch`,
+ `Cancel narratage cancel build ${built.id} --runtime ${args.runtime}`,
]);
+ if (built.status === "failed") io.setExitCode?.(1);
} finally {
await runtime.close();
}
return;
}
- let runtime: LocalRuntime | undefined;
+ let runtime: RuntimeArchiveView | undefined;
try {
- runtime = args.runtime === undefined ? undefined : await loadLocalRuntime(args.runtime, distribution);
+ runtime = args.runtime === undefined
+ ? undefined
+ : lazyRuntimeArchive(args.runtime, distribution);
const loaded = await loadRunFile({
workspace,
authorCompiler: compiler,
diff --git a/packages/cli/src/output.ts b/packages/cli/src/output.ts
index c3e2df6a..e46b437c 100644
--- a/packages/cli/src/output.ts
+++ b/packages/cli/src/output.ts
@@ -11,6 +11,8 @@ export type CliTerminal = {
export type CliIo = {
readonly write: (text: string) => void;
+ /** Concrete command shells expose process status without coupling the engine to Node globals. */
+ readonly setExitCode?: (code: number) => void;
/** Interactive secret input supplied by the concrete CLI shell; never echoed or logged. */
readonly readSecret?: (prompt: string) => Promise<string>;
readonly terminal?: CliTerminal;
@@ -368,13 +370,21 @@ export function writeCliHelp(io: CliIo): void {
colors.accent(colors.strong("Narratage")),
colors.dim("Write the story. Compile the result."),
"",
+ colors.strong("Typical flow"),
+ " doctor <profile> verify packages, credentials and services",
+ " check <source> --runtime <profile> verify author or run intent",
+ " plan <run-source> --runtime <profile> inspect the exact demanded subgraph",
+ " build <run-source> --runtime <profile> submit durable work and ensure a Worker",
+ " queue --runtime <profile> --watch observe shared work without owning it",
+ "",
colors.strong("Authoring"),
" check <source> verify one self-described Author or Run source",
" plan <run-source> freeze and inspect a Build plan",
- " build <run-source> execute a Build through a Runtime Profile",
+ " build <run-source> submit a durable Build; --follow only observes",
"",
colors.strong("Archive"),
" builds list known Builds",
+ " history [output] find accepted historical Logical Outputs",
" status <build-id> show Build and Operation status",
" inspect <build-id> inspect accepted Records and demanded outputs",
" get <build-id> read or materialize one archived result",
@@ -392,7 +402,7 @@ export function writeCliHelp(io: CliIo): void {
" gc <profile> report unreachable Artifacts; --apply deletes",
"",
colors.strong("Packages"),
- " lock-packages <file> lock explicitly installed packages",
+ " lock-packages <file> set, add, remove, refresh or verify local package trust",
"",
colors.strong("Output"),
" --json complete machine-readable result",
diff --git a/packages/cli/src/package-lock-edit.ts b/packages/cli/src/package-lock-edit.ts
new file mode 100644
index 00000000..20b6a7c1
--- /dev/null
+++ b/packages/cli/src/package-lock-edit.ts
@@ -0,0 +1,45 @@
+import { open, readFile, rm } from "node:fs/promises";
+import { hostname } from "node:os";
+import { resolve } from "node:path";
+
+function isAlreadyExists(error: unknown): boolean {
+ return error !== null && typeof error === "object" && "code" in error
+ && (error as { readonly code?: unknown }).code === "EEXIST";
+}
+
+/**
+ * Serialize local package-lock writers with the same deliberately simple protocol Git uses for
+ * its index: one sibling sentinel, no daemon and no hidden stale-lock recovery.
+ */
+export async function withPackageLockEdit<T>(
+ packageLock: string,
+ action: () => Promise<T>,
+): Promise<T> {
+ const target = resolve(packageLock);
+ const sentinel = `${target}.editing`;
+ let handle: Awaited<ReturnType<typeof open>>;
+ try {
+ handle = await open(sentinel, "wx", 0o600);
+ } catch (error) {
+ if (!isAlreadyExists(error)) throw error;
+ const owner = await readFile(sentinel, "utf8").catch(() => "unknown owner");
+ throw new Error(
+ `package lock ${target} is already being edited (${owner.trim().slice(0, 300)}); `
+ + `if that process no longer exists, remove ${sentinel}`,
+ );
+ }
+ try {
+ await handle.writeFile(`${JSON.stringify({
+ format: "narratage.package-lock-edit@1",
+ pid: process.pid,
+ host: hostname(),
+ startedAt: new Date().toISOString(),
+ target,
+ })}\n`, "utf8");
+ await handle.sync();
+ return await action();
+ } finally {
+ await handle.close().catch(() => undefined);
+ await rm(sentinel, { force: true }).catch(() => undefined);
+ }
+}
diff --git a/packages/cli/src/runtime-process.ts b/packages/cli/src/runtime-process.ts
index 0842b09a..70f7f137 100644
--- a/packages/cli/src/runtime-process.ts
+++ b/packages/cli/src/runtime-process.ts
@@ -1,6 +1,6 @@
import { createHash } from "node:crypto";
import { spawn } from "node:child_process";
-import { mkdir, open, readFile, stat, unlink, writeFile } from "node:fs/promises";
+import { mkdir, open, readFile, rename, stat, unlink, writeFile } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
export type RuntimeWorkerLaunch = {
@@ -27,6 +27,9 @@ type ProcessRecord = {
readonly profileDigest?: string;
};
+const LOG_TAIL_BYTES = 1024 * 1024;
+const LOG_ROTATE_BYTES = 10 * 1024 * 1024;
+
function paths(profile: string) {
const absolute = resolve(profile);
const id = createHash("sha256").update(absolute).digest("hex").slice(0, 16);
@@ -36,6 +39,7 @@ function paths(profile: string) {
pid: join(root, "worker.json"),
ready: join(root, "ready"),
log: join(root, "worker.log"),
+ lock: join(root, "lifecycle.lock"),
};
}
@@ -58,6 +62,47 @@ function alive(pid: number): boolean {
}
}
+async function withLifecycleLock<T>(profile: string, timeoutMs: number, run: () => Promise<T>): Promise<T> {
+ const location = paths(profile);
+ await mkdir(location.root, { recursive: true });
+ const deadline = Date.now() + timeoutMs;
+ while (true) {
+ let lock;
+ try {
+ lock = await open(location.lock, "wx");
+ } catch (error) {
+ if (!(error instanceof Error && "code" in error && error.code === "EEXIST")) throw error;
+ const owner = await readFile(location.lock, "utf8").then((text) => JSON.parse(text) as {
+ readonly pid?: unknown;
+ }).catch(() => undefined);
+ if (owner === undefined || !Number.isSafeInteger(owner.pid) || !alive(owner.pid as number)) {
+ await unlink(location.lock).catch(() => undefined);
+ continue;
+ }
+ if (Date.now() >= deadline) {
+ throw new Error(`Runtime lifecycle is busy in process ${String(owner.pid)}; lock: ${location.lock}`);
+ }
+ await new Promise((resolveWait) => setTimeout(resolveWait, 25));
+ continue;
+ }
+ try {
+ await lock.writeFile(JSON.stringify({ pid: process.pid, acquiredAt: Date.now() }), "utf8");
+ return await run();
+ } finally {
+ await lock.close();
+ await unlink(location.lock).catch(() => undefined);
+ }
+ }
+}
+
+async function rotateLog(path: string): Promise<void> {
+ const size = await stat(path).then((value) => value.size).catch(() => 0);
+ if (size <= LOG_ROTATE_BYTES) return;
+ const previous = `${path}.previous`;
+ await unlink(previous).catch(() => undefined);
+ await rename(path, previous);
+}
+
async function digestProfile(profile: string): Promise<string> {
const absolute = resolve(profile);
const bytes = await readFile(absolute);
@@ -145,44 +190,45 @@ export async function ensureRuntimeProcess(
launch: RuntimeWorkerLaunch,
timeoutMs = 10_000,
): Promise<RuntimeProcessState> {
- const current = await runtimeProcessStatus(profile);
- if (current.state === "running") return current;
- if (current.state === "stale") {
- await stopRuntimeProcess(profile, timeoutMs);
- }
- const absolute = resolve(profile);
- const location = paths(absolute);
- const profileDigest = await digestProfile(absolute);
- await mkdir(location.root, { recursive: true });
- await unlink(location.ready).catch(() => undefined);
- const log = await open(location.log, "a");
- const child = spawn(launch.command, [
- ...launch.args,
- "_worker",
- absolute,
- "--ready-file",
- location.ready,
- ], {
- cwd: process.cwd(),
- detached: true,
- stdio: ["ignore", log.fd, log.fd],
- env: process.env,
+ return await withLifecycleLock(profile, timeoutMs, async () => {
+ const current = await runtimeProcessStatus(profile);
+ if (current.state === "running") return current;
+ if (current.state === "stale") await stopRuntimeProcessUnlocked(profile, timeoutMs);
+ const absolute = resolve(profile);
+ const location = paths(absolute);
+ const profileDigest = await digestProfile(absolute);
+ await mkdir(location.root, { recursive: true });
+ await unlink(location.ready).catch(() => undefined);
+ await rotateLog(location.log);
+ const log = await open(location.log, "a");
+ const child = spawn(launch.command, [
+ ...launch.args,
+ "_worker",
+ absolute,
+ "--ready-file",
+ location.ready,
+ ], {
+ cwd: process.cwd(),
+ detached: true,
+ stdio: ["ignore", log.fd, log.fd],
+ env: process.env,
+ });
+ if (child.pid === undefined) throw new Error("Runtime Worker process has no pid");
+ const startedAt = Date.now();
+ await writeFile(location.pid, JSON.stringify({
+ format: "narratage.runtime-process@1",
+ profile: absolute,
+ pid: child.pid,
+ startedAt,
+ profileDigest,
+ } satisfies ProcessRecord), "utf8");
+ child.unref();
+ await log.close();
+ return await waitForReady(absolute, timeoutMs);
});
- if (child.pid === undefined) throw new Error("Runtime Worker process has no pid");
- const startedAt = Date.now();
- await writeFile(location.pid, JSON.stringify({
- format: "narratage.runtime-process@1",
- profile: absolute,
- pid: child.pid,
- startedAt,
- profileDigest,
- } satisfies ProcessRecord), "utf8");
- child.unref();
- await log.close();
- return await waitForReady(absolute, timeoutMs);
}
-export async function stopRuntimeProcess(profile: string, timeoutMs = 10_000): Promise<RuntimeProcessState> {
+async function stopRuntimeProcessUnlocked(profile: string, timeoutMs: number): Promise<RuntimeProcessState> {
const current = await record(profile);
const location = paths(profile);
if (current === undefined || !alive(current.pid)) {
@@ -201,9 +247,26 @@ export async function stopRuntimeProcess(profile: string, timeoutMs = 10_000): P
return { state: "stopped", profile: resolve(profile), logPath: location.log };
}
+export async function stopRuntimeProcess(profile: string, timeoutMs = 10_000): Promise<RuntimeProcessState> {
+ return await withLifecycleLock(profile, timeoutMs, async () => await stopRuntimeProcessUnlocked(profile, timeoutMs));
+}
+
export async function runtimeProcessLogs(profile: string): Promise<{ readonly path: string; readonly text: string }> {
const path = paths(profile).log;
- return { path, text: await readFile(path, "utf8").catch(() => "") };
+ let file;
+ try {
+ file = await open(path, "r");
+ const metadata = await file.stat();
+ const length = Math.min(metadata.size, LOG_TAIL_BYTES);
+ const buffer = Buffer.alloc(length);
+ const { bytesRead } = await file.read(buffer, 0, length, metadata.size - length);
+ return { path, text: buffer.subarray(0, bytesRead).toString("utf8") };
+ } catch (error) {
+ if (error instanceof Error && "code" in error && error.code === "ENOENT") return { path, text: "" };
+ throw error;
+ } finally {
+ await file?.close();
+ }
}
export async function markRuntimeProcessReady(path: string): Promise<void> {
diff --git a/packages/cli/test/control-commands.test.ts b/packages/cli/test/control-commands.test.ts
new file mode 100644
index 00000000..78dce6ef
--- /dev/null
+++ b/packages/cli/test/control-commands.test.ts
@@ -0,0 +1,109 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import type { LocalCredentialControl, LocalRuntimeControl } from "@narratage/local";
+
+import type { CliDistribution } from "../src/distribution.js";
+import { runCli } from "../src/main.js";
+
+test("queue opens durable control without constructing execution Providers", async () => {
+ const calls: string[] = [];
+ const control = {
+ async queue() {
+ calls.push("control.queue");
+ return { dispatches: [], capacity: [], operations: [] };
+ },
+ async close() {
+ calls.push("control.close");
+ },
+ } as unknown as LocalRuntimeControl;
+ const distribution = {
+ createRuntimeControlFromConfig: async () => {
+ calls.push("control.create");
+ return control;
+ },
+ createRuntimeFromConfig: async () => {
+ calls.push("execution.create");
+ throw new Error("execution Providers must not be constructed");
+ },
+ } as unknown as CliDistribution;
+ let output = "";
+
+ await runCli(
+ ["queue", "--runtime", "/tmp/narratage-control-profile.json", "--json"],
+ { write: (text) => { output += text; } },
+ distribution,
+ );
+
+ assert.deepEqual(calls, ["control.create", "control.queue", "control.close"]);
+ const result = JSON.parse(output) as {
+ readonly format: string;
+ readonly at: number;
+ readonly dispatches: readonly unknown[];
+ readonly capacity: readonly unknown[];
+ readonly operations: readonly unknown[];
+ };
+ assert.equal(result.format, "narratage.cli-queue@1");
+ assert.equal(typeof result.at, "number");
+ assert.deepEqual(result.dispatches, []);
+ assert.deepEqual(result.capacity, []);
+ assert.deepEqual(result.operations, []);
+});
+
+test("command options fail closed instead of being silently ignored", async () => {
+ const distribution = {} as CliDistribution;
+ const io = { write() {} };
+ await assert.rejects(
+ async () => await runCli([
+ "status", "build-1", "--runtime", "/tmp/runtime.json", "--root", "/tmp",
+ ], io, distribution),
+ /--root does not apply to status/u,
+ );
+ await assert.rejects(
+ async () => await runCli([
+ "queue", "--runtime", "/tmp/one.json", "--runtime", "/tmp/two.json",
+ ], io, distribution),
+ /--runtime cannot be repeated/u,
+ );
+});
+
+test("auth opens only one Endpoint credential control, never the execution Runtime", async () => {
+ const calls: string[] = [];
+ const credentials = {
+ async credentials(endpoint?: string) {
+ calls.push(`credentials.status:${endpoint}`);
+ return [{
+ endpoint: "kie.project",
+ slot: "apiKey",
+ label: "KIE API key",
+ kind: "secret",
+ ref: { format: "svml.credential-ref@1", store: "env", key: "KIE_API_KEY" },
+ configured: false,
+ writable: false,
+ }];
+ },
+ async close() { calls.push("credentials.close"); },
+ } as unknown as LocalCredentialControl;
+ const distribution = {
+ createRuntimeCredentialsFromConfig: async (_path: string, endpoint: string) => {
+ calls.push(`credentials.create:${endpoint}`);
+ return credentials;
+ },
+ createRuntimeFromConfig: async () => {
+ calls.push("execution.create");
+ throw new Error("auth must not construct execution");
+ },
+ } as unknown as CliDistribution;
+ let output = "";
+
+ await runCli([
+ "auth", "status", "kie.project", "--runtime", "/tmp/runtime.json", "--json",
+ ], { write: (text) => { output += text; } }, distribution);
+
+ assert.deepEqual(calls, [
+ "credentials.create:kie.project",
+ "credentials.status:kie.project",
+ "credentials.close",
+ ]);
+ assert.equal((JSON.parse(output) as { readonly endpoint?: string }).endpoint, "kie.project");
+});
diff --git a/packages/cli/test/runtime-build-process.test.ts b/packages/cli/test/runtime-build-process.test.ts
index 7536eb11..ca2f53ac 100644
--- a/packages/cli/test/runtime-build-process.test.ts
+++ b/packages/cli/test/runtime-build-process.test.ts
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { execFile, spawn } from "node:child_process";
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
@@ -159,8 +159,6 @@ test("CLI exits after durable submission and a restarted detached Worker complet
"build", run,
"--build-id", build,
"--runtime", profile,
- "--package-lock", authorLockPath,
- "--package-root", process.cwd(),
"--root", project,
"--no-services",
]);
@@ -175,8 +173,6 @@ test("CLI exits after durable submission and a restarted detached Worker complet
"build", run,
"--build-id", build,
"--runtime", profile,
- "--package-lock", authorLockPath,
- "--package-root", process.cwd(),
"--root", project,
"--no-services",
"--follow",
@@ -210,6 +206,20 @@ test("CLI exits after durable submission and a restarted detached Worker complet
const completed = await waitForBuild(project, profile, build);
assert.equal((completed.build as { readonly status?: string }).status, "complete");
assert.equal((completed.dispatch as { readonly terminal?: string }).terminal, "complete");
+
+ const queue = await cli(project, ["queue", "--runtime", profile]);
+ assert.equal((queue.dispatches as readonly { readonly phase: string }[])
+ .filter((item) => item.phase !== "terminal").length, 0);
+ const archive = await cli(project, ["builds", "--runtime", profile]);
+ assert.deepEqual((archive.builds as readonly { readonly build: string }[]).map((item) => item.build), [build]);
+ const inspected = await cli(project, ["inspect", build, "--runtime", profile]);
+ assert.equal((inspected.archive as { readonly status?: string }).status, "complete");
+ const exported = join(project, "output", "message.json");
+ const materialized = await cli(project, [
+ "get", build, "--runtime", profile, "--name", "message-004", "--to", exported,
+ ]);
+ assert.equal((materialized.materialized as { readonly kind?: string }).kind, "json");
+ assert.notEqual(JSON.parse(await readFile(exported, "utf8")), undefined);
} finally {
await stopRuntimeProcess(profile, 10_000).catch(() => undefined);
await rm(project, { recursive: true, force: true });
diff --git a/packages/cli/test/runtime-process.test.ts b/packages/cli/test/runtime-process.test.ts
index 9dc36732..5d93fde8 100644
--- a/packages/cli/test/runtime-process.test.ts
+++ b/packages/cli/test/runtime-process.test.ts
@@ -1,5 +1,5 @@
import assert from "node:assert/strict";
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { appendFile, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
@@ -39,8 +39,17 @@ test("one detached Runtime Worker is observable, reusable and explicitly stoppab
assert.ok(first.pid);
const second = await ensureRuntimeProcess(profile, { command: "must-not-run", args: [] }, 5_000);
assert.equal(second.pid, first.pid);
+ const concurrent = await Promise.all(Array.from({ length: 8 }, async () =>
+ await ensureRuntimeProcess(profile, { command: "must-not-run", args: [] }, 5_000)));
+ assert.deepEqual([...new Set(concurrent.map((item) => item.pid))], [first.pid],
+ "concurrent clients share one atomically launched Worker");
assert.equal((await runtimeProcessStatus(profile)).state, "running");
assert.match((await runtimeProcessLogs(profile)).text, /worker-ready/u);
+ const logPath = (await runtimeProcessLogs(profile)).path;
+ await appendFile(logPath, `${"x".repeat(1024 * 1024 + 64)}\ntail-marker\n`, "utf8");
+ const bounded = await runtimeProcessLogs(profile);
+ assert.ok(Buffer.byteLength(bounded.text) <= 1024 * 1024);
+ assert.match(bounded.text, /tail-marker/u);
await writeFile(runtimeLock, "runtime-v2\n", "utf8");
assert.equal((await runtimeProcessStatus(profile)).state, "stale");
const replaced = await ensureRuntimeProcess(profile, { command: process.execPath, args: ["-e", program] }, 5_000);
diff --git a/packages/compiler-node/src/compiler.ts b/packages/compiler-node/src/compiler.ts
index fc503274..a01f220a 100644
--- a/packages/compiler-node/src/compiler.ts
+++ b/packages/compiler-node/src/compiler.ts
@@ -38,20 +38,16 @@ function attachmentKey(artifact: BlobRef): string {
return `${artifact.digest}\u0000${artifact.mediaType}`;
}
-function sameBytes(left: Uint8Array, right: Uint8Array): boolean {
- return left.byteLength === right.byteLength && left.every((value, index) => value === right[index]);
-}
-
function mergeAttachments(groups: readonly (readonly ArtifactAttachment[])[]): readonly ArtifactAttachment[] {
const merged = new Map<string, ArtifactAttachment>();
for (const item of groups.flat()) {
const key = attachmentKey(item.artifact);
const existing = merged.get(key);
if (existing !== undefined) {
- if (existing.artifact.size !== item.artifact.size || !sameBytes(existing.bytes, item.bytes)) {
+ if (existing.artifact.size !== item.artifact.size) {
throw new NodeCompilerError(
"SOURCE_ATTACHMENT_CONFLICT",
- `Artifact attachment ${item.artifact.digest} carries conflicting bytes`,
+ `Artifact attachment ${item.artifact.digest} carries conflicting sizes`,
item.artifact.digest,
);
}
@@ -59,7 +55,7 @@ function mergeAttachments(groups: readonly (readonly ArtifactAttachment[])[]): r
}
merged.set(key, {
artifact: { ...item.artifact },
- bytes: Uint8Array.from(item.bytes),
+ open: item.open,
});
}
return [...merged.values()]
@@ -183,14 +179,17 @@ export class NodeCompiler {
};
const key = attachmentKey(artifact);
const existing = embeddedAttachments.get(key);
- if (existing !== undefined && !sameBytes(existing.bytes, bytes)) {
+ if (existing !== undefined && existing.artifact.size !== bytes.byteLength) {
throw new NodeCompilerError(
"SOURCE_ATTACHMENT_CONFLICT",
`Embedded asset ${request.from} conflicts with ${artifact.digest}`,
request.from,
);
}
- embeddedAttachments.set(key, { artifact, bytes });
+ embeddedAttachments.set(key, {
+ artifact,
+ open: async () => (async function* () { yield Uint8Array.from(bytes); })(),
+ });
return { artifact: { ...artifact } };
},
admitRecord: this.#admitRecord,
diff --git a/packages/compiler-node/src/run.ts b/packages/compiler-node/src/run.ts
index b8ee514f..a324f538 100644
--- a/packages/compiler-node/src/run.ts
+++ b/packages/compiler-node/src/run.ts
@@ -1,7 +1,12 @@
import type { Workspace, WorkspaceSession } from "@narratage/host";
import {
+ compileBuild,
+ createResolvedClosure,
+ EMPTY_REALIZATION_DIGEST,
+ link,
sealBuildRequest,
sealCompiledGraph,
+ sealTypedModule,
start,
} from "@narratage/core";
import type {
@@ -13,6 +18,7 @@ import type {
BuildState,
BuildState as ArchivedBuildState,
Digest,
+ ModuleRef,
StoredValue,
} from "@narratage/protocol";
import { resolveRealization } from "@narratage/run";
@@ -64,6 +70,118 @@ export type PlannedBuild = {
readonly state: BuildState;
};
+function moduleKey(ref: ModuleRef): string {
+ return `${ref.name}\u0000${ref.version}`;
+}
+
+/**
+ * Materialize the exact execution slice selected by one BuildRequest.
+ *
+ * Author compilation deliberately keeps the whole document available. A durable
+ * Build does not need unrelated Tracks, renderers and their schemas merely
+ * because the Author Source imported them. Keeping those bytes in every state
+ * revision made a four-node estimate Build carry megabytes of unrelated video
+ * declarations. The Build graph below retains the selected Candidates plus the
+ * primary Candidates required to verify their Logical Output promises, and no
+ * other executable branch.
+ */
+function executionSlice(
+ program: LinkedProgram,
+ graph: ReturnType<typeof sealCompiledGraph>,
+ request: BuildRequest,
+): { readonly program: LinkedProgram; readonly graph: ReturnType<typeof sealCompiledGraph> } {
+ const preliminary = compileBuild(program, graph, request);
+ const outputById = new Map(graph.outputs.map((item) => [item.id, item]));
+ const candidateById = new Map(graph.candidates.map((item) => [item.id, item]));
+ const operationById = new Map(graph.operations.map((item) => [item.id, item]));
+ const satisfactionByOutput = new Map(request.satisfactions.map((item) => [item.output, item]));
+ const selectionByOutput = new Map(preliminary.selections.map((item) => [item.output, item]));
+ const outputs = new Set<string>();
+ const candidates = new Set<string>();
+ const operations = new Set<string>();
+ const records = new Set<string>();
+
+ const visitRef = (ref: import("@narratage/protocol").GraphValueRef): void => {
+ if (ref.kind === "record") {
+ records.add(ref.id);
+ } else if (ref.kind === "logical-output") {
+ visitOutput(ref.id);
+ } else {
+ visitOperation(ref.operation);
+ }
+ };
+ const visitOperation = (id: string): void => {
+ if (operations.has(id)) return;
+ const operation = operationById.get(id);
+ if (operation === undefined) throw new Error(`execution slice refers to absent Operation ${id}`);
+ operations.add(id);
+ Object.values(operation.inputs).forEach(visitRef);
+ };
+ const visitCandidate = (id: string): void => {
+ if (candidates.has(id)) return;
+ const candidate = candidateById.get(id);
+ if (candidate === undefined) throw new Error(`execution slice refers to absent Candidate ${id}`);
+ candidates.add(id);
+ if (candidate.root.kind === "operation") visitOperation(candidate.root.result.operation);
+ };
+ const visitOutput = (id: string): void => {
+ if (outputs.has(id)) return;
+ const output = outputById.get(id);
+ if (output === undefined) throw new Error(`execution slice refers to absent Logical Output ${id}`);
+ outputs.add(id);
+ // The primary remains the exact promise witness even when this Run chooses
+ // an explicit substitute Candidate.
+ visitCandidate(output.primary);
+ visitCandidate(
+ selectionByOutput.get(id)?.candidate
+ ?? satisfactionByOutput.get(id)?.candidate
+ ?? output.primary,
+ );
+ output.semanticInputs.forEach(visitRef);
+ };
+ request.targets.forEach((target) => visitOutput(target.output));
+
+ const selectedOutputs = graph.outputs.filter((item) => outputs.has(item.id));
+ const selectedCandidates = graph.candidates.filter((item) => candidates.has(item.id));
+ const selectedOperations = graph.operations.filter((item) => operations.has(item.id));
+ const authoredRecords = program.records.filter((record) => records.has(record.id));
+
+ const requiredModules = new Set<string>();
+ const requireModule = (ref: ModuleRef): void => {
+ const key = moduleKey(ref);
+ if (requiredModules.has(key)) return;
+ const resolved = program.closure.modules.find((item) => moduleKey(item.ref) === key);
+ if (resolved === undefined) throw new Error(`execution slice refers to absent module ${ref.name}@${ref.version}`);
+ requiredModules.add(key);
+ resolved.manifest.dependencies.forEach((item) => requireModule(item.module));
+ };
+ selectedOperations.forEach((item) => requireModule(item.producer.module));
+ selectedOutputs.forEach((item) => requireModule(item.type.module));
+ selectedCandidates.forEach((item) => requireModule(item.type.module));
+ authoredRecords.forEach((item) => requireModule(item.type.module));
+
+ const closure = createResolvedClosure(program.closure.modules
+ .filter((item) => requiredModules.has(moduleKey(item.ref)))
+ .map((item) => item.manifest));
+ const typedModules = program.modules.flatMap((item) => {
+ const selected = item.records.filter((record) => records.has(record.id));
+ return selected.length === 0
+ ? []
+ : [sealTypedModule({ id: item.id, closureDigest: closure.digest, records: selected })];
+ });
+ const slicedProgram = link(closure, typedModules);
+ const slicedGraph = sealCompiledGraph({
+ program: slicedProgram.semanticDigest,
+ outputs: selectedOutputs,
+ candidates: selectedCandidates,
+ operations: selectedOperations,
+ ...(graph.realization === EMPTY_REALIZATION_DIGEST
+ ? {}
+ : { source: graph.source, realization: graph.realization }),
+ });
+ return { program: slicedProgram, graph: slicedGraph };
+}
+
function decodeStoredValue(bytes: Uint8Array, from: string): StoredValue {
let parsed: unknown;
try {
@@ -77,6 +195,22 @@ function decodeStoredValue(bytes: Uint8Array, from: string): StoredValue {
return parsed as StoredValue;
}
+async function attachmentBytes(attachment: ArtifactAttachment): Promise<Uint8Array> {
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ for await (const chunk of await attachment.open()) {
+ chunks.push(Uint8Array.from(chunk));
+ size += chunk.byteLength;
+ }
+ const bytes = new Uint8Array(size);
+ let offset = 0;
+ for (const chunk of chunks) {
+ bytes.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return bytes;
+}
+
async function storedValueFromWorkspace(
workspace: WorkspaceSession,
source: RunSourceUnit,
@@ -85,7 +219,7 @@ async function storedValueFromWorkspace(
const resolved = await workspace.resolveAsset(source, { from, mediaType: "application/json" });
const attachment = (await workspace.attachments()).find((item) => item.artifact.digest === resolved.artifact.digest);
if (attachment === undefined) throw new Error(`Workspace did not retain bytes for ${from}`);
- return decodeStoredValue(attachment.bytes, from);
+ return decodeStoredValue(await attachmentBytes(attachment), from);
}
/** Node Host for the second, mandatory source graph. It never guesses a Frontend from a suffix. */
@@ -161,7 +295,7 @@ export class NodeRunCompiler {
authorGraph,
[compilation.run.overlay],
).graph;
- const graph = sealCompiledGraph({
+ const fullGraph = sealCompiledGraph({
program: realized.program,
source: authorGraph.id,
realization: compilation.run.graph.id,
@@ -173,13 +307,21 @@ export class NodeRunCompiler {
(item) => item.id === compilation.run.graph.selectedTargets,
);
if (selected === undefined) throw new Error(`Run Graph selects absent Target Set ${compilation.run.graph.selectedTargets}`);
- const request = sealBuildRequest({
- graph: graph.id,
+ const fullRequest = sealBuildRequest({
+ graph: fullGraph.id,
...(implementationClosure === undefined ? {} : { implementationClosure }),
targets: selected.targets,
satisfactions: compilation.run.graph.satisfactions,
});
- const state = start(compilation.program, graph, request);
+ const sliced = executionSlice(compilation.program, fullGraph, fullRequest);
+ const slicedOutputs = new Set(sliced.graph.outputs.map((item) => item.id));
+ const request = sealBuildRequest({
+ graph: sliced.graph.id,
+ ...(implementationClosure === undefined ? {} : { implementationClosure }),
+ targets: selected.targets,
+ satisfactions: compilation.run.graph.satisfactions.filter((item) => slicedOutputs.has(item.output)),
+ });
+ const state = start(sliced.program, sliced.graph, request);
return { compilation, request, plan: state.plan, state };
}
diff --git a/packages/compiler-node/test/compiler.test.ts b/packages/compiler-node/test/compiler.test.ts
index 55261d0c..42018816 100644
--- a/packages/compiler-node/test/compiler.test.ts
+++ b/packages/compiler-node/test/compiler.test.ts
@@ -44,7 +44,7 @@ import {
RunFrontendRegistry,
} from "@narratage/run";
import { runMarkupFrontend } from "@narratage/run-markup";
-import type { Workspace } from "@narratage/host";
+import type { ArtifactAttachment, Workspace } from "@narratage/host";
import { WorkspaceError } from "@narratage/host";
import {
createMarkupAuthorFrontend,
@@ -290,7 +290,7 @@ function memoryWorkspace(sourceText: string, assetBytes: Uint8Array): Workspace
return {
async open(entryLocator) {
const entry: AuthorSourceUnit = { id: entryLocator, name: "main.svml", text: sourceText };
- let attachment: { readonly artifact: BlobRef; readonly bytes: Uint8Array } | undefined;
+ let attachment: { readonly artifact: BlobRef; readonly open: () => AsyncIterable<Uint8Array> } | undefined;
return {
entry,
async resolveSource(_importer: AuthorSourceUnit, request: AuthorSourceImport) {
@@ -307,19 +307,36 @@ function memoryWorkspace(sourceText: string, assetBytes: Uint8Array): Workspace
size: bytes.byteLength,
mediaType: request.mediaType,
};
- attachment = { artifact, bytes };
+ attachment = { artifact, open: async function* () { yield Uint8Array.from(bytes); } };
return { artifact: { ...artifact } };
},
attachments() {
return attachment === undefined
? []
- : [{ artifact: { ...attachment.artifact }, bytes: Uint8Array.from(attachment.bytes) }];
+ : [{ artifact: { ...attachment.artifact }, open: attachment.open }];
},
};
},
};
}
+async function readAttachment(attachment: ArtifactAttachment | undefined): Promise<Uint8Array | undefined> {
+ if (attachment === undefined) return undefined;
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ for await (const chunk of await attachment.open()) {
+ chunks.push(Uint8Array.from(chunk));
+ size += chunk.byteLength;
+ }
+ const bytes = new Uint8Array(size);
+ let offset = 0;
+ for (const chunk of chunks) {
+ bytes.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return bytes;
+}
+
test("Node Compiler discovers real imports and emits a named public Author Graph export", async () => {
const root = await mkdtemp(join(tmpdir(), "svml-compiler-node-"));
const file = join(root, "main.svml");
@@ -361,11 +378,13 @@ test("Author Frontend identity comes only from the mandatory Source Header, neve
test("Run-only Fragment modules extend the execution closure without polluting the Author Graph", async () => {
const root = await mkdtemp(join(tmpdir(), "svml-dual-graph-closure-"));
+ const unused = emptyManifest("example.unused-video-feature");
const authorFile = join(root, "main.svml");
const runFile = join(root, "build.svrun");
await writeFile(authorFile, `<?svml using="@narratage/markup@1"?>
<svml>
<import as="lab" from="example.compiler-lab@1"/>
+ <import as="unused" from="example.unused-video-feature@1"/>
<lab:Result id="hello"/>
</svml>`, "utf8");
await writeFile(runFile, `<?svml using="@narratage/run-markup@1"?>
@@ -377,7 +396,7 @@ test("Run-only Fragment modules extend the execution closure without polluting t
<satisfy output="hello.result" candidate="one.result" fidelity="substitute"/>
</svrun>`, "utf8");
- const authorCompiler = compiler(root, [previewManifest]);
+ const authorCompiler = compiler(root, [previewManifest, unused]);
const frontends = new RunFrontendRegistry();
frontends.register(runMarkupFrontend);
const fragments = new RunFragmentRegistry();
@@ -388,11 +407,16 @@ test("Run-only Fragment modules extend the execution closure without polluting t
assert.deepEqual(
compiled.author.program.closure.modules.map((item) => item.ref.name),
- [laboratory.name],
+ [laboratory.name, unused.name],
);
assert.deepEqual(
compiled.program.closure.modules.map((item) => item.ref.name).sort(),
+ [laboratory.name, previewModule.name, unused.name].sort(),
+ );
+ assert.deepEqual(
+ planned.state.program.closure.modules.map((item) => item.ref.name).sort(),
[laboratory.name, previewModule.name].sort(),
+ "the durable Build keeps only modules needed by its selected execution slice",
);
assert.equal(compiled.run.graph.authorGraph, compiled.author.elaboration.graph.id);
assert.equal(planned.plan.steps.length, 1);
@@ -415,7 +439,7 @@ test("source assets are content addressed, closure-bound and returned as a Host
const attachment = first.attachments[0];
assert.equal(unit?.assets[0]?.from, "./reference.bin");
assert.equal(unit?.assets[0]?.artifact.digest, attachment?.artifact.digest);
- assert.deepEqual(attachment?.bytes, new Uint8Array([1, 2, 3, 4]));
+ assert.deepEqual(await readAttachment(attachment), new Uint8Array([1, 2, 3, 4]));
assert.equal(first.module.records[0]?.value.kind, "blob");
assert.equal(first.module.records[0]?.value.kind === "blob" ? first.module.records[0].value.digest : undefined, attachment?.artifact.digest);
const tamperedUnit = first.closure.units.map((item) => item === unit
@@ -449,7 +473,7 @@ test("an installed package Surface can contribute locked bytes without an author
const compiled = await assetCompiler({ root }).compileFile(file);
const attachment = compiled.attachments[0];
- assert.deepEqual(attachment?.bytes, new Uint8Array([8, 6, 7, 5, 3, 0, 9]));
+ assert.deepEqual(await readAttachment(attachment), new Uint8Array([8, 6, 7, 5, 3, 0, 9]));
assert.equal(compiled.closure.units[0]?.assets[0]?.from, "package:example.asset-lab/embedded.bin");
assert.equal(compiled.closure.units[0]?.assets[0]?.artifact.digest, attachment?.artifact.digest);
assert.equal(compiled.module.records[0]?.value.kind, "blob");
@@ -458,7 +482,7 @@ test("an installed package Surface can contribute locked bytes without an author
: undefined, attachment?.artifact.digest);
});
-test("filesystem Workspace contains symlinks and reads each canonical source only once", async () => {
+test("filesystem Workspace contains symlinks and locks source text plus asset identity once", async () => {
const parent = await mkdtemp(join(tmpdir(), "svml-source-host-"));
const root = join(parent, "project");
await mkdir(root);
@@ -487,8 +511,9 @@ test("filesystem Workspace contains symlinks and reads each canonical source onl
const lockedAsset = await workspace.resolveAsset(entry, { from: "./asset.bin", mediaType: "application/octet-stream" });
assert.deepEqual(lockedAsset, firstAsset, "one Host locks an asset edge to the first bytes read");
const detached = await workspace.attachments();
- detached[0]?.bytes.fill(0);
- assert.deepEqual((await workspace.attachments())[0]?.bytes, new Uint8Array([1, 2, 3]));
+ assert.deepEqual(await readAttachment(detached[0]), new Uint8Array([4, 5, 6, 7]),
+ "attachment bytes are opened lazily; Runtime rejects them if they no longer match the locked identity");
+ assert.equal((await workspace.attachments())[0]?.artifact.digest, firstAsset.artifact.digest);
await assert.rejects(
async () => await workspace.resolveSource(entry, {
from: "./escaped.svs",
@@ -523,5 +548,8 @@ test("filesystem and in-memory Workspaces compile identical source and bytes to
assert.equal(memory.closure.id, filesystem.closure.id);
assert.equal(memory.module.semanticDigest, filesystem.module.semanticDigest);
assert.equal(memory.elaboration.graph.id, filesystem.elaboration.graph.id);
- assert.deepEqual(memory.attachments, filesystem.attachments);
+ assert.deepEqual(memory.attachments.map((item) => item.artifact),
+ filesystem.attachments.map((item) => item.artifact));
+ assert.deepEqual(await readAttachment(memory.attachments[0]),
+ await readAttachment(filesystem.attachments[0]));
});
diff --git a/packages/driver-node/src/driver.ts b/packages/driver-node/src/driver.ts
index 8c7b90d4..62e7520c 100644
--- a/packages/driver-node/src/driver.ts
+++ b/packages/driver-node/src/driver.ts
@@ -419,6 +419,7 @@ export class NodeDriver {
status: "pending",
checkpoint: outcome.checkpoint,
...(outcome.wakeAt === undefined ? {} : { wakeAt: outcome.wakeAt }),
+ ...(outcome.progress === undefined ? {} : { progress: outcome.progress }),
});
return await this.#completedOperation(
state,
@@ -671,6 +672,7 @@ export class NodeDriver {
status: "pending",
checkpoint: observed.checkpoint,
...(observed.wakeAt === undefined ? {} : { wakeAt: observed.wakeAt }),
+ ...(observed.progress === undefined ? {} : { progress: observed.progress }),
});
}
if (observed.status === "failed") {
diff --git a/packages/endpoint-kit/src/index.ts b/packages/endpoint-kit/src/index.ts
index acdf34fd..8e4d9ebc 100644
--- a/packages/endpoint-kit/src/index.ts
+++ b/packages/endpoint-kit/src/index.ts
@@ -21,6 +21,7 @@ import type {
CredentialValue,
OperationFailure,
OperationIdentity,
+ OperationProgress,
RuntimeEndpointBinding,
RuntimeFacetRef,
RuntimeModuleManifest,
@@ -50,7 +51,12 @@ export type ImmediateEndpointHandler = (
) => Awaitable<EndpointFulfillment>;
export type EndpointOutcome =
- | { readonly status: "pending"; readonly checkpoint: CanonicalValue; readonly wakeAt?: number }
+ | {
+ readonly status: "pending";
+ readonly checkpoint: CanonicalValue;
+ readonly wakeAt?: number;
+ readonly progress?: OperationProgress;
+ }
| { readonly status: "completed"; readonly result: EndpointFulfillment }
| { readonly status: "failed"; readonly failure: OperationFailure };
@@ -327,8 +333,19 @@ export function wakeAfter(
checkpoint: CanonicalValue,
delayMs: number,
now = Date.now(),
-): { readonly status: "pending"; readonly checkpoint: CanonicalValue; readonly wakeAt: number } {
+ progress?: OperationProgress,
+): {
+ readonly status: "pending";
+ readonly checkpoint: CanonicalValue;
+ readonly wakeAt: number;
+ readonly progress?: OperationProgress;
+} {
assert(Number.isSafeInteger(delayMs) && delayMs >= 0, "wake delay must be a non-negative safe integer");
assert(Number.isSafeInteger(now) && now >= 0, "current time must be a non-negative epoch millisecond");
- return { status: "pending", checkpoint, wakeAt: now + delayMs };
+ return {
+ status: "pending",
+ checkpoint,
+ wakeAt: now + delayMs,
+ ...(progress === undefined ? {} : { progress }),
+ };
}
diff --git a/packages/host/README.md b/packages/host/README.md
index ba1da6df..38c6cb21 100644
--- a/packages/host/README.md
+++ b/packages/host/README.md
@@ -5,6 +5,9 @@ only the definition-time `Workspace` session and generic `ArtifactAttachment` tr
It contains no filesystem, parser, scheduler, store, Provider or domain implementation.
A `Workspace` opens one isolated compilation session. The session supplies one entry SourceUnit,
-relative Source/Asset resolution and the exact content-addressed bytes requested while decoding.
+relative Source/Asset resolution and reopenable streams for exact content-addressed assets requested
+while decoding. The Host fixes identity during compilation; the Runtime verifies the streamed size
+and digest while admitting bytes to its selected ArtifactStore, so source mutation cannot silently
+change a Build and large media is not copied through compiler memory.
Filesystem, browser, Git, memory and remote workspaces implement the same interface. Compiler and
Runtime never infer environment selection from author imports.
diff --git a/packages/host/src/index.ts b/packages/host/src/index.ts
index 7ac0d0c9..a47eec81 100644
--- a/packages/host/src/index.ts
+++ b/packages/host/src/index.ts
@@ -8,10 +8,14 @@ import type { BlobRef } from "@narratage/protocol";
export type { HostFacet } from "./facet.js";
-/** Host transfer bytes. Their origin is deliberately absent from the Runtime contract. */
+/**
+ * Repeatably openable Host transfer bytes. Their origin is deliberately absent
+ * from the Runtime contract; Runtime verifies the opened stream against the
+ * declared BlobRef before admitting a Build.
+ */
export type ArtifactAttachment = {
readonly artifact: BlobRef;
- readonly bytes: Uint8Array;
+ open(): Awaitable<AsyncIterable<Uint8Array>>;
};
/** One isolated, read-once definition session for exactly one compilation. */
diff --git a/packages/hyperframes/src/project.ts b/packages/hyperframes/src/project.ts
index b23ba01f..33f9b87e 100644
--- a/packages/hyperframes/src/project.ts
+++ b/packages/hyperframes/src/project.ts
@@ -1,5 +1,5 @@
import { createHash } from "node:crypto";
-import { mkdir, writeFile } from "node:fs/promises";
+import { mkdir, open, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { CompositableSurfaceRef } from "@narratage/media";
@@ -9,7 +9,9 @@ import { assertHyperframesDocument, materializeHyperframesHtml } from "./documen
import type { HyperframesDocument } from "./types.js";
/** Reads one content-addressed dependency. Whose store it comes from is the caller's business. */
-export type HyperframesArtifactReader = (artifact: BlobRef) => Promise<Uint8Array>;
+export type HyperframesArtifactReader = (
+ artifact: BlobRef,
+) => Promise<Uint8Array | AsyncIterable<Uint8Array>>;
export type HyperframesSurfaceValidator = (
surface: CompositableSurfaceRef,
bytes: Uint8Array,
@@ -61,19 +63,44 @@ export async function stageHyperframesProject(options: {
const surfaces = new Map(document.surfaces.map((surface) => [surface.artifact.digest, surface]));
const validations = await Promise.all(document.artifacts.map(async (artifact) => {
const name = `${artifact.digest.slice("sha256:".length)}${extension(artifact.mediaType)}`;
- const bytes = await read(artifact);
- if (bytes.byteLength !== artifact.size) {
+ const opened = await read(artifact);
+ const chunks = opened instanceof Uint8Array
+ ? (async function* () { yield opened; })()
+ : opened;
+ const hash = createHash("sha256");
+ const surface = surfaces.get(artifact.digest);
+ const retained: Uint8Array[] = [];
+ let size = 0;
+ const target = await open(join(artifactDirectory, name), "w");
+ try {
+ for await (const chunk of chunks) {
+ await target.write(chunk);
+ hash.update(chunk);
+ size += chunk.byteLength;
+ if (surface !== undefined) retained.push(Uint8Array.from(chunk));
+ }
+ } finally {
+ await target.close();
+ }
+ if (size !== artifact.size) {
throw new Error(`HyperFrames Artifact ${artifact.digest} size differs`);
}
- const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+ const digest = `sha256:${hash.digest("hex")}`;
if (digest !== artifact.digest) {
throw new Error(`HyperFrames Artifact ${artifact.digest} bytes differ`);
}
- const surface = surfaces.get(artifact.digest);
+ const bytes = surface === undefined ? undefined : (() => {
+ const value = new Uint8Array(size);
+ let offset = 0;
+ for (const chunk of retained) {
+ value.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return value;
+ })();
const validation = surface === undefined
? undefined
- : await options.validateSurface!(structuredClone(surface), bytes.slice());
- await writeFile(join(artifactDirectory, name), bytes);
+ : await options.validateSurface!(structuredClone(surface), bytes!.slice());
paths.set(artifact.digest, `./artifacts/${name}`);
return validation;
}));
diff --git a/packages/local/README.md b/packages/local/README.md
index 99fae7eb..570dfded 100644
--- a/packages/local/README.md
+++ b/packages/local/README.md
@@ -69,9 +69,17 @@ or too-late requests continue reconciling the same Operation and never select an
`LocalBuildRequest.attachments` is explicit Host ingress into the selected ArtifactStore. The Store's
returned digest, media type and size must equal the claimed `BlobRef` before Core execution. Artifact
-retention is explicit maintenance: `narratage gc` previews by default and `--apply` removes only bytes
+attachments are reopenable streams; a streaming Store receives them without a whole-file compiler or
+Runtime buffer. A non-streaming replacement Store remains valid but explicitly pays the buffering cost.
+Operation progress is a small generic `{ phase, completed?, total?, unit? }` fact owned by the
+Endpoint, independent from its opaque recovery checkpoint.
+Artifact retention is explicit maintenance: `narratage gc` previews by default and `--apply` removes only bytes
unreachable from retained Build and Operation facts.
+Worker and managed-service lifecycle changes are serialized by profile-scoped filesystem locks. Two
+simultaneous `build` commands therefore reuse one detached Worker rather than launching orphan
+processes. CLI log reads are tail-bounded and oversized logs rotate on the next managed start.
+
`BuildCatalog` is optional Host presentation metadata. SQLite may expose one beside its execution
facets, but Local never creates a second hidden database or treats aliases and source paths as Build
truth.
diff --git a/packages/local/src/config.ts b/packages/local/src/config.ts
index dea60ae0..d32cc18e 100644
--- a/packages/local/src/config.ts
+++ b/packages/local/src/config.ts
@@ -2,21 +2,33 @@ import { readFile, stat } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import type { ComponentPackage } from "@narratage/component-kit";
-import { loadNodePackageSet } from "@narratage/package-loader-node";
+import {
+ collectNodePackageComponents,
+ loadNodePackageSet,
+ readNodePackageLock,
+} from "@narratage/package-loader-node";
+import type { LoadedNodePackageSet } from "@narratage/package-loader-node";
import { canonicalize } from "@narratage/protocol";
-import type { CanonicalValue } from "@narratage/protocol";
+import type { CanonicalValue, CapabilityRef } from "@narratage/protocol";
import {
isRuntimeAdapterHostFacet,
RuntimeAdapterRegistry,
} from "@narratage/runtime-adapter";
import type {
RuntimeDoctorDiagnostic,
+ RuntimeEndpointActivation,
RuntimeExternalService,
RuntimeServiceState,
} from "@narratage/runtime-adapter";
-import type { RuntimeServiceSelection } from "@narratage/runtime";
+import {
+ CompositeCredentialStore,
+ verifyRuntimeServicePackage,
+} from "@narratage/runtime";
+import type { RuntimeServicePackage, RuntimeServiceSelection } from "@narratage/runtime";
import type { LocalRuntime } from "./types.js";
+import type { LocalCredentialControl } from "./types.js";
+import type { LocalRuntimeControl } from "./types.js";
export type RuntimeConfigEntry = {
readonly use: string;
@@ -179,6 +191,8 @@ export type LoadRuntimeConfigOptions = {
readonly components?: readonly ComponentPackage[];
/** Host package installation used when the Profile does not explicitly select packageRoot. */
readonly packageRoot?: string;
+ /** Package set already verified by this same Host command. */
+ readonly implementationPackages?: LoadedNodePackageSet;
};
export type RuntimeConfigDoctorResult = {
@@ -186,6 +200,36 @@ export type RuntimeConfigDoctorResult = {
readonly diagnostics: readonly RuntimeDoctorDiagnostic[];
};
+export type RuntimeConfigPackageSelection = {
+ readonly root: string;
+ readonly packageRoot: string;
+ readonly packageLock?: string;
+};
+
+/**
+ * Resolve the deterministic implementation lock selected by a JSON Runtime
+ * Profile without constructing the Runtime or executing any adapter code.
+ *
+ * The CLI uses this to keep `build --runtime profile.json` single-sourced: the
+ * same author package lock drives compilation and later Worker execution.
+ */
+export async function runtimeConfigPackageSelection(
+ path: string,
+ options: LoadRuntimeConfigOptions = {},
+): Promise<RuntimeConfigPackageSelection> {
+ const absolute = resolve(path);
+ const document = parseRuntimeConfig(JSON.parse(await readFile(absolute, "utf8")));
+ const root = resolve(dirname(absolute), document.root ?? ".");
+ const packageRoot = resolve(dirname(absolute), document.packageRoot ?? options.packageRoot ?? document.root ?? ".");
+ return {
+ root,
+ packageRoot,
+ ...(document.packageLock === undefined
+ ? {}
+ : { packageLock: resolve(root, document.packageLock) }),
+ };
+}
+
async function installLockedRuntimeAdapters(
registry: RuntimeAdapterRegistry,
path: string | undefined,
@@ -218,6 +262,12 @@ export type DeclaredExternalService = {
readonly service: RuntimeExternalService;
};
+function sameCapability(left: CapabilityRef, right: CapabilityRef): boolean {
+ return left.name === right.name
+ && left.module.name === right.module.name
+ && left.module.version === right.module.version;
+}
+
/**
* The external programs a Runtime Profile implies, in the order it declares
* them. Two Endpoints may name the same program — one WhisperX serves every
@@ -226,11 +276,17 @@ export type DeclaredExternalService = {
*/
export async function declaredExternalServices(
path: string,
- options: LoadRuntimeConfigOptions = {},
+ options: LoadRuntimeConfigOptions & { readonly capabilities?: readonly CapabilityRef[] } = {},
): Promise<{ readonly root: string; readonly services: readonly DeclaredExternalService[] }> {
const absolute = resolve(path);
const document = parseRuntimeConfig(JSON.parse(await readFile(absolute, "utf8")));
const root = resolve(dirname(absolute), document.root ?? ".");
+ // A Build with no demanded external capability has no program to discover.
+ // Keep validating the Profile document and its root, but do not load the
+ // privileged Runtime package closure merely to prove that the empty set is
+ // empty. `services up/status/down` omit this filter and still inspect every
+ // declared service.
+ if (options.capabilities?.length === 0) return { root, services: [] };
const packageRoot = resolve(dirname(absolute), document.packageRoot ?? options.packageRoot ?? document.root ?? ".");
const registry = options.registry ?? new RuntimeAdapterRegistry();
await installLockedRuntimeAdapters(registry, document.runtimePackageLock, root, packageRoot);
@@ -239,12 +295,17 @@ export async function declaredExternalServices(
if (!registry.has(item.use, "endpoint")) {
throw new Error(`Runtime Adapter ${item.use} is not registered`);
}
- const service = registry.service(item.use, {
+ const activation = await registry.activateEndpoint(item.use, {
root,
instance: item.instance,
...(item.lane === undefined ? {} : { lane: item.lane }),
config: item.config ?? {},
});
+ if (options.capabilities !== undefined && !activation.endpoint.bindings.some((binding) =>
+ options.capabilities!.some((capability) => sameCapability(binding.capability, capability)))) {
+ continue;
+ }
+ const service = activation.externalService;
if (service !== undefined) services.push({ instance: item.instance, service });
}
return { root, services };
@@ -269,6 +330,10 @@ export async function doctorRuntimeConfig(
const root = resolve(dirname(absolute), document.root ?? ".");
const packageRoot = resolve(dirname(absolute), document.packageRoot ?? options.packageRoot ?? document.root ?? ".");
const diagnostics: RuntimeDoctorDiagnostic[] = [];
+ const credentialEndpoints: Array<{
+ readonly entry: RuntimeConfigEntry;
+ readonly activation: RuntimeEndpointActivation;
+ }> = [];
try {
if (!(await stat(root)).isDirectory()) throw new Error(`Runtime root ${root} is not a directory`);
} catch (error) {
@@ -276,18 +341,17 @@ export async function doctorRuntimeConfig(
return { root, diagnostics };
}
const registry = options.registry ?? new RuntimeAdapterRegistry();
- try {
- await installLockedRuntimeAdapters(registry, document.runtimePackageLock, root, packageRoot);
- } catch (error) {
- diagnostics.push(diagnostic(error, "RUNTIME_PACKAGE_LOCK_INVALID", document.runtimePackageLock));
+ const runtimeLock = installLockedRuntimeAdapters(registry, document.runtimePackageLock, root, packageRoot);
+ const implementationLock = document.packageLock === undefined
+ ? Promise.resolve(undefined)
+ : loadNodePackageSet(resolve(root, document.packageLock), packageRoot);
+ const [runtimeResult, implementationResult] = await Promise.allSettled([runtimeLock, implementationLock]);
+ if (runtimeResult.status === "rejected") {
+ diagnostics.push(diagnostic(runtimeResult.reason, "RUNTIME_PACKAGE_LOCK_INVALID", document.runtimePackageLock));
return { root, diagnostics };
}
- if (document.packageLock !== undefined) {
- try {
- await loadNodePackageSet(resolve(root, document.packageLock), packageRoot);
- } catch (error) {
- diagnostics.push(diagnostic(error, "IMPLEMENTATION_PACKAGE_LOCK_INVALID", document.packageLock));
- }
+ if (implementationResult.status === "rejected") {
+ diagnostics.push(diagnostic(implementationResult.reason, "IMPLEMENTATION_PACKAGE_LOCK_INVALID", document.packageLock));
}
for (const item of document.runtimeServices) {
const context = { root, instance: item.instance, config: item.config ?? {} };
@@ -312,17 +376,16 @@ export async function doctorRuntimeConfig(
...(item.lane === undefined ? {} : { lane: item.lane }),
config: item.config ?? {},
};
+ let activation: RuntimeEndpointActivation;
try {
- const selection = registry.validate(item.use, "endpoint", context);
- diagnostics.push(...selection);
- if (selection.some((entry) => entry.severity === "error")) continue;
+ activation = await registry.activateEndpoint(item.use, context);
} catch (error) {
diagnostics.push(diagnostic(error, "RUNTIME_ENDPOINT_CONFIG_INVALID", item.instance));
continue;
}
let adapterHasError = false;
try {
- const adapterDiagnostics = await registry.doctor(item.use, "endpoint", context);
+ const adapterDiagnostics = activation.diagnose === undefined ? [] : await activation.diagnose();
diagnostics.push(...adapterDiagnostics);
adapterHasError = adapterDiagnostics.some((entry) => entry.severity === "error");
} catch (error) {
@@ -330,13 +393,14 @@ export async function doctorRuntimeConfig(
continue;
}
if (adapterHasError) continue;
+ credentialEndpoints.push({ entry: item, activation });
// An external program is a prerequisite a Build cannot supply for itself, so
// name the command that supplies it here rather than failing mid-Build on a
// socket. `RUNTIME_SERVICE_` above is the Runtime's own part; this is the
// separate program a deployment must have installed or running.
let service: RuntimeExternalService | undefined;
try {
- service = registry.service(item.use, context);
+ service = activation.externalService;
} catch (error) {
diagnostics.push(diagnostic(error, "EXTERNAL_SERVICE_INVALID", item.instance));
}
@@ -367,6 +431,52 @@ export async function doctorRuntimeConfig(
});
}
}
+ if (credentialEndpoints.length > 0) {
+ let packages: readonly RuntimeServicePackage[] = [];
+ try {
+ const endpoints = credentialEndpoints.map(({ entry, activation }) => ({
+ entry,
+ credentials: activation.endpoint.credentials,
+ }));
+ if (endpoints.every((item) => item.credentials.length === 0)) return { root, diagnostics };
+ const serviceEntries = runtimeEntriesForServices(
+ document.runtimeServices,
+ document.services.stores.credentials,
+ );
+ packages = await Promise.all(serviceEntries.map(async (item) => await registry.createService(item.use, {
+ root,
+ instance: item.instance,
+ config: item.config ?? {},
+ })));
+ for (const item of packages) verifyRuntimeServicePackage(item);
+ const byId = new Map(packages.flatMap((item) =>
+ item.services.map((service) => [service.instance.id, service] as const)));
+ const stores = document.services.stores.credentials.map((id) => {
+ const service = byId.get(id);
+ if (service === undefined) throw new Error(`CredentialStore selection refers to unknown instance ${id}`);
+ if (service.role !== "credential-store") {
+ throw new Error(`Runtime service ${id} is ${service.role}, not credential-store`);
+ }
+ return service.service;
+ });
+ const credentials = new CompositeCredentialStore(stores);
+ for (const { entry, credentials: slots } of endpoints) {
+ for (const slot of slots) {
+ if (await credentials.resolve(slot.ref) !== undefined) continue;
+ diagnostics.push({
+ severity: "error",
+ code: "RUNTIME_CREDENTIAL_MISSING",
+ message: `${slot.label} for Endpoint ${entry.instance} is not configured in CredentialStore ${slot.ref.store}`,
+ subject: `${entry.instance}.${slot.slot}`,
+ });
+ }
+ }
+ } catch (error) {
+ diagnostics.push(diagnostic(error, "RUNTIME_CREDENTIAL_CHECK_FAILED"));
+ } finally {
+ await closeRuntimeServicePackages(packages);
+ }
+ }
return { root, diagnostics };
}
@@ -381,8 +491,26 @@ export async function createRuntimeFromConfig(
const document = parseRuntimeConfig(JSON.parse(await readFile(absolute, "utf8")));
const root = resolve(dirname(absolute), document.root ?? ".");
const packageRoot = resolve(dirname(absolute), document.packageRoot ?? options.packageRoot ?? document.root ?? ".");
+ if (options.implementationPackages !== undefined) {
+ if (document.packageLock === undefined) {
+ throw new Error("Runtime Profile has no packageLock for the supplied implementation packages");
+ }
+ const declared = await readNodePackageLock(resolve(root, document.packageLock));
+ if (declared.digest !== options.implementationPackages.lock.digest) {
+ throw new Error("Runtime Profile packageLock differs from the implementation packages already verified by this Host");
+ }
+ }
const registry = options.registry ?? new RuntimeAdapterRegistry();
- await installLockedRuntimeAdapters(registry, document.runtimePackageLock, root, packageRoot);
+ const implementationPackages = options.implementationPackages
+ ?? (document.packageLock === undefined
+ ? undefined
+ : await Promise.all([
+ installLockedRuntimeAdapters(registry, document.runtimePackageLock, root, packageRoot),
+ loadNodePackageSet(resolve(root, document.packageLock), packageRoot),
+ ]).then(([, loaded]) => loaded));
+ if (options.implementationPackages !== undefined || document.packageLock === undefined) {
+ await installLockedRuntimeAdapters(registry, document.runtimePackageLock, root, packageRoot);
+ }
const runtimeServices = await Promise.all(document.runtimeServices.map(async (item) => {
return await registry.createService(item.use, {
root,
@@ -401,12 +529,129 @@ export async function createRuntimeFromConfig(
return await createProjectLocalRuntime({
root,
packageRoot,
- ...(document.packageLock === undefined ? {} : { packageLock: document.packageLock }),
+ ...(implementationPackages === undefined
+ ? {}
+ : { implementationClosure: implementationPackages.lock.digest }),
runtimeServices,
runtimeSelection: document.services,
- components: options.components ?? [],
+ components: [
+ ...collectNodePackageComponents(implementationPackages?.contributions ?? []),
+ ...(options.components ?? []),
+ ],
endpoints,
allowedPermissions: document.permissions,
scheduling: document.scheduling,
});
}
+
+/**
+ * Open only the durable control Stores selected by a Runtime Profile. Endpoint
+ * construction, credential resolution, author components and Worker instances
+ * are unnecessary for archive inspection, queue visibility, egress and cancellation.
+ */
+export async function createRuntimeControlFromConfig(
+ path: string,
+ options: LoadRuntimeConfigOptions = {},
+): Promise<LocalRuntimeControl> {
+ const { createProjectLocalRuntimeControl } = await import("./control.js");
+ const absolute = resolve(path);
+ const document = parseRuntimeConfig(JSON.parse(await readFile(absolute, "utf8")));
+ const root = resolve(dirname(absolute), document.root ?? ".");
+ const packageRoot = resolve(dirname(absolute), document.packageRoot ?? options.packageRoot ?? document.root ?? ".");
+ const registry = options.registry ?? new RuntimeAdapterRegistry();
+ await installLockedRuntimeAdapters(registry, document.runtimePackageLock, root, packageRoot);
+ const runtimeServices = await Promise.all(document.runtimeServices.map(async (item) => {
+ return await registry.createService(item.use, {
+ root,
+ instance: item.instance,
+ config: item.config ?? {},
+ });
+ }));
+ return await createProjectLocalRuntimeControl({
+ root,
+ runtimeServices,
+ runtimeSelection: document.services,
+ allowedPermissions: document.permissions,
+ });
+}
+
+function ownsSelectedService(entry: RuntimeConfigEntry, service: string): boolean {
+ return service === entry.instance || service.startsWith(`${entry.instance}.`);
+}
+
+function runtimeEntriesForServices(
+ entries: readonly RuntimeConfigEntry[],
+ services: readonly string[],
+): readonly RuntimeConfigEntry[] {
+ const selected = new Set<RuntimeConfigEntry>();
+ for (const service of services) {
+ const owners = entries.filter((entry) => ownsSelectedService(entry, service));
+ if (owners.length !== 1) {
+ throw new Error(owners.length === 0
+ ? `Runtime service ${service} is outside every configured adapter namespace`
+ : `Runtime service ${service} has overlapping configured adapter namespaces`);
+ }
+ selected.add(owners[0]!);
+ }
+ return [...selected];
+}
+
+async function closeRuntimeServicePackages(packages: readonly RuntimeServicePackage[]): Promise<void> {
+ for (const item of [...packages].reverse()) await item.close?.();
+}
+
+/**
+ * Open the smallest deployment slice required to manage one exact Endpoint's
+ * credentials. Build Stores, scheduler, Worker, author packages and unrelated
+ * Endpoints are deliberately not constructed.
+ */
+export async function createRuntimeCredentialsFromConfig(
+ path: string,
+ endpointInstance: string,
+ options: LoadRuntimeConfigOptions = {},
+): Promise<LocalCredentialControl> {
+ const { createLocalCredentialControl } = await import("./credentials.js");
+ const absolute = resolve(path);
+ const document = parseRuntimeConfig(JSON.parse(await readFile(absolute, "utf8")));
+ const root = resolve(dirname(absolute), document.root ?? ".");
+ const packageRoot = resolve(dirname(absolute), document.packageRoot ?? options.packageRoot ?? document.root ?? ".");
+ const endpointEntry = document.endpoints.find((item) => item.instance === endpointInstance);
+ if (endpointEntry === undefined) throw new Error(`Runtime Profile has no Endpoint instance ${endpointInstance}`);
+ const registry = options.registry ?? new RuntimeAdapterRegistry();
+ await installLockedRuntimeAdapters(registry, document.runtimePackageLock, root, packageRoot);
+ const serviceEntries = runtimeEntriesForServices(
+ document.runtimeServices,
+ document.services.stores.credentials,
+ );
+ const packages = await Promise.all(serviceEntries.map(async (item) => await registry.createService(item.use, {
+ root,
+ instance: item.instance,
+ config: item.config ?? {},
+ })));
+ try {
+ for (const item of packages) verifyRuntimeServicePackage(item);
+ const byId = new Map(packages.flatMap((item) => item.services.map((service) => [service.instance.id, service] as const)));
+ const credentialStores = document.services.stores.credentials.map((id) => {
+ const service = byId.get(id);
+ if (service === undefined) throw new Error(`CredentialStore selection refers to unknown instance ${id}`);
+ if (service.role !== "credential-store") {
+ throw new Error(`Runtime service ${id} is ${service.role}, not credential-store`);
+ }
+ return service.service;
+ });
+ const endpoint = await registry.createEndpoint(endpointEntry.use, {
+ root,
+ instance: endpointEntry.instance,
+ ...(endpointEntry.lane === undefined ? {} : { lane: endpointEntry.lane }),
+ config: endpointEntry.config ?? {},
+ });
+ return createLocalCredentialControl({
+ credentialStore: new CompositeCredentialStore(credentialStores),
+ endpoints: [endpoint],
+ close: async () => await closeRuntimeServicePackages(packages),
+ });
+ } catch (error) {
+ await closeRuntimeServicePackages(packages);
+ throw error;
+ }
+}
diff --git a/packages/local/src/control.ts b/packages/local/src/control.ts
new file mode 100644
index 00000000..39955272
--- /dev/null
+++ b/packages/local/src/control.ts
@@ -0,0 +1,184 @@
+import { isDigest } from "@narratage/protocol";
+import {
+ isEnumerableBuildStore,
+ isManagedArtifactStore,
+ isStreamingArtifactStore,
+ operationCancellationRequestId,
+} from "@narratage/runtime";
+
+import type {
+ CreateLocalRuntimeControlOptions,
+ LocalRuntimeControl,
+ ProjectLocalRuntimeControlOptions,
+} from "./types.js";
+import { createProjectRuntimeServices } from "./project-services.js";
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function collectArtifactDigests(
+ value: unknown,
+ digests: Set<import("@narratage/protocol").Digest>,
+ seen = new WeakSet<object>(),
+): void {
+ if (value === null || typeof value !== "object") return;
+ if (seen.has(value)) return;
+ seen.add(value);
+ if (!Array.isArray(value)) {
+ const item = value as Record<string, unknown>;
+ if (item.kind === "blob" && typeof item.digest === "string" && isDigest(item.digest)) {
+ digests.add(item.digest);
+ }
+ }
+ for (const nested of Array.isArray(value) ? value : Object.values(value)) {
+ collectArtifactDigests(nested, digests, seen);
+ }
+}
+
+/** Store-only control surface shared by a full executor and read-only CLI commands. */
+export function createLocalRuntimeControl(
+ options: CreateLocalRuntimeControlOptions,
+): LocalRuntimeControl {
+ const buildCatalog = options.buildCatalog;
+ return {
+ async activity(build) {
+ const [operations, dispatch] = await Promise.all([
+ options.operationStore.list({ build }),
+ options.dispatchStore.read(build),
+ ]);
+ return { operations, dispatch };
+ },
+ async status(build) {
+ const [snapshot, catalog, activity] = await Promise.all([
+ options.buildStore.read(build),
+ buildCatalog?.read(build),
+ Promise.all([
+ options.operationStore.list({ build }),
+ options.dispatchStore.read(build),
+ ]),
+ ]);
+ return {
+ build: snapshot,
+ catalog,
+ operations: activity[0],
+ dispatch: activity[1],
+ };
+ },
+ async queue() {
+ const [dispatches, capacity] = await Promise.all([
+ options.dispatchStore.list({ phases: ["queued", "leased", "waiting", "blocked", "settling"] }),
+ options.dispatchStore.listCapacity(),
+ ]);
+ const operationHistory = (await Promise.all(dispatches.map(async (item) =>
+ await options.operationStore.list({ build: item.build })))).flat();
+ return {
+ dispatches,
+ capacity,
+ operations: operationHistory,
+ };
+ },
+ async operation(id) {
+ return await options.operationStore.read(id);
+ },
+ async journal(query) {
+ return await options.journal.list(query);
+ },
+ async builds() {
+ return await buildCatalog?.list() ?? [];
+ },
+ async cancel(build, reason) {
+ if (await options.dispatchStore.read(build) === undefined) return undefined;
+ const dispatch = await options.dispatchStore.requestCancellation(build, reason);
+ await options.journal.append({
+ at: Date.now(),
+ kind: "cancellation-requested",
+ build,
+ detail: { ...(reason === undefined ? {} : { reason }) },
+ });
+ return dispatch;
+ },
+ async cancelOperation(id, reason) {
+ let current = await options.operationStore.read(id);
+ if (current === undefined) return undefined;
+ if (current.cancellation === undefined) {
+ const requestedAt = Date.now();
+ while (current.cancellation === undefined) {
+ const status = current.status === "completed" || current.status === "failed" || current.status === "cancelled"
+ ? "too-late" as const
+ : "requested" as const;
+ const written = await options.operationStore.compareAndSwap(current.id, current.revision, {
+ status: "control",
+ cancellation: {
+ requestedAt,
+ requestId: operationCancellationRequestId(current.id, requestedAt),
+ status,
+ attempts: 0,
+ },
+ });
+ current = written.status === "stored" ? written.snapshot : written.current;
+ }
+ await options.journal.append({
+ at: requestedAt,
+ kind: "cancellation-requested",
+ build: current.build,
+ operation: current.id,
+ detail: { scope: "operation", ...(reason === undefined ? {} : { reason }) },
+ });
+ }
+ await options.dispatchStore.wake(current.build);
+ return current;
+ },
+ async readArtifact(digest) {
+ return await options.artifactStore.get(digest);
+ },
+ async openArtifact(digest) {
+ if (isStreamingArtifactStore(options.artifactStore)) return await options.artifactStore.open(digest);
+ const bytes = await options.artifactStore.get(digest);
+ return bytes === undefined ? undefined : (async function* () { yield bytes; })();
+ },
+ async garbageCollectArtifacts(gc = {}) {
+ assert(isManagedArtifactStore(options.artifactStore),
+ "selected ArtifactStore does not expose explicit retention management");
+ assert(isEnumerableBuildStore(options.buildStore),
+ "selected BuildStore does not expose the maintenance index required for Artifact GC");
+ const reachable = new Set<import("@narratage/protocol").Digest>();
+ for (const snapshot of await options.buildStore.list()) collectArtifactDigests(snapshot.state, reachable);
+ for (const operation of await options.operationStore.list({})) collectArtifactDigests(operation, reachable);
+ const stored = await options.artifactStore.list();
+ const unreachable = stored.filter((digest) => !reachable.has(digest)).sort();
+ const deleted: import("@narratage/protocol").Digest[] = [];
+ if (gc.apply === true) {
+ for (const digest of unreachable) {
+ if (await options.artifactStore.delete(digest)) deleted.push(digest);
+ }
+ }
+ return {
+ reachable: [...reachable].sort(),
+ unreachable,
+ deleted,
+ };
+ },
+ close() {
+ return options.close?.();
+ },
+ };
+}
+
+/** Assemble selected service packages, then expose only their exact durable Store facets. */
+export async function createProjectLocalRuntimeControl(
+ options: ProjectLocalRuntimeControlOptions,
+): Promise<LocalRuntimeControl> {
+ const root = options.root ?? process.cwd();
+ const projectServices = await createProjectRuntimeServices(root, options);
+ const services = projectServices.assembly;
+ return createLocalRuntimeControl({
+ buildStore: services.buildStore,
+ ...(projectServices.catalog === undefined ? {} : { buildCatalog: projectServices.catalog }),
+ operationStore: services.operationStore,
+ dispatchStore: services.dispatchStore,
+ journal: services.journal,
+ artifactStore: services.artifactStore,
+ close: projectServices.close,
+ });
+}
diff --git a/packages/local/src/credentials.ts b/packages/local/src/credentials.ts
new file mode 100644
index 00000000..dea44c50
--- /dev/null
+++ b/packages/local/src/credentials.ts
@@ -0,0 +1,61 @@
+import {
+ writableCredentialStore,
+} from "@narratage/runtime";
+
+import type {
+ CreateLocalCredentialControlOptions,
+ LocalCredentialControl,
+} from "./types.js";
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+/**
+ * Credential management is deployment control, not execution. It needs only
+ * the selected CredentialStores and the declaration of the selected Endpoint;
+ * opening Build state, a Worker, unrelated Endpoints or author components here
+ * would make logging into one Provider depend on the entire deployment.
+ */
+export function createLocalCredentialControl(
+ options: CreateLocalCredentialControlOptions,
+): LocalCredentialControl {
+ const descriptions = options.endpoints.flatMap((item) => item.credentials);
+ const credential = (endpoint: string, slot: string) => {
+ const matches = descriptions.filter((item) => item.endpoint === endpoint && item.slot === slot);
+ assert(matches.length === 1, matches.length === 0
+ ? `Endpoint ${endpoint} has no credential slot ${slot}`
+ : `Endpoint ${endpoint} repeats credential slot ${slot}`);
+ return matches[0]!;
+ };
+ const status = async (item: typeof descriptions[number]) => ({
+ ...structuredClone(item),
+ configured: await options.credentialStore.resolve(item.ref) !== undefined,
+ writable: await writableCredentialStore(options.credentialStore, item.ref) !== undefined,
+ });
+
+ return {
+ async credentials(endpoint) {
+ const selected = descriptions.filter((item) => endpoint === undefined || item.endpoint === endpoint);
+ return await Promise.all(selected.map(status));
+ },
+ async putCredential(endpoint, slot, secret) {
+ assert(secret.length > 0, "credential secret is empty");
+ const item = credential(endpoint, slot);
+ const store = await writableCredentialStore(options.credentialStore, item.ref);
+ assert(store !== undefined, `CredentialStore ${item.ref.store} is not writable`);
+ await store.put(item.ref, { secret });
+ return await status(item);
+ },
+ async deleteCredential(endpoint, slot) {
+ const item = credential(endpoint, slot);
+ const store = await writableCredentialStore(options.credentialStore, item.ref);
+ assert(store !== undefined, `CredentialStore ${item.ref.store} is not writable`);
+ const deleted = await store.delete(item.ref);
+ return { deleted, credential: await status(item) };
+ },
+ close() {
+ return options.close?.();
+ },
+ };
+}
diff --git a/packages/local/src/external-services.ts b/packages/local/src/external-services.ts
index cf771806..3b97585c 100644
--- a/packages/local/src/external-services.ts
+++ b/packages/local/src/external-services.ts
@@ -1,8 +1,9 @@
import { spawn } from "node:child_process";
-import { mkdir, open, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
+import type { CapabilityRef } from "@narratage/protocol";
import type { RuntimeExternalService, RuntimeServiceCommand, RuntimeServiceState } from "@narratage/runtime-adapter";
import { declaredExternalServices } from "./config.js";
@@ -30,9 +31,18 @@ export type ExternalServiceReport = {
readonly pid?: number;
};
+export type ExternalServiceProgress = {
+ readonly id: string;
+ readonly phase: "checking" | "preparing" | "starting" | "waiting" | "ready";
+};
+
export type ExternalServiceOptions = LoadRuntimeConfigOptions & {
/** How long to wait for a started program to answer its probe. Default 300000. */
readonly maxWaitMs?: number;
+ /** Human-facing progress only; never changes service selection or lifecycle. */
+ readonly onProgress?: (event: ExternalServiceProgress) => void;
+ /** When present, operate only programs backing at least one demanded capability. */
+ readonly capabilities?: readonly CapabilityRef[];
};
/** Two Endpoints may drive the same program; it is brought up once. */
@@ -50,6 +60,54 @@ function directory(root: string): string {
return join(root, ".svml", "services");
}
+const LOG_ROTATE_BYTES = 10 * 1024 * 1024;
+
+async function withServiceLifecycleLock<T>(
+ root: string,
+ id: string,
+ timeoutMs: number,
+ runLocked: () => Promise<T>,
+): Promise<T> {
+ const path = join(directory(root), `${id}.lifecycle.lock`);
+ await mkdir(directory(root), { recursive: true });
+ const deadline = Date.now() + timeoutMs;
+ while (true) {
+ let lock;
+ try {
+ lock = await open(path, "wx");
+ } catch (error) {
+ if (!(error instanceof Error && "code" in error && error.code === "EEXIST")) throw error;
+ const owner = await readFile(path, "utf8").then((text) => JSON.parse(text) as {
+ readonly pid?: unknown;
+ }).catch(() => undefined);
+ if (owner === undefined || !Number.isSafeInteger(owner.pid) || !alive(owner.pid as number)) {
+ await rm(path, { force: true });
+ continue;
+ }
+ if (Date.now() >= deadline) {
+ throw new Error(`External service ${id} lifecycle is busy in process ${String(owner.pid)}; lock: ${path}`);
+ }
+ await sleep(25);
+ continue;
+ }
+ try {
+ await lock.writeFile(JSON.stringify({ pid: process.pid, acquiredAt: Date.now() }), "utf8");
+ return await runLocked();
+ } finally {
+ await lock.close();
+ await rm(path, { force: true });
+ }
+ }
+}
+
+async function rotateLog(path: string): Promise<void> {
+ const size = await stat(path).then((value) => value.size).catch(() => 0);
+ if (size <= LOG_ROTATE_BYTES) return;
+ const previous = `${path}.previous`;
+ await rm(previous, { force: true });
+ await rename(path, previous);
+}
+
async function readPid(root: string, id: string): Promise<number | undefined> {
try {
const text = await readFile(join(directory(root), `${id}.pid`), "utf8");
@@ -103,16 +161,22 @@ async function bringUp(
service: RuntimeExternalService,
instances: readonly string[],
maxWaitMs: number,
+ onProgress?: (event: ExternalServiceProgress) => void,
): Promise<ExternalServiceReport> {
const base = { id: service.id, instances };
+ onProgress?.({ id: service.id, phase: "checking" });
const initial = await service.probe();
- if (initial.state === "ready") return { ...base, action: "already-running", state: initial };
+ if (initial.state === "ready") {
+ onProgress?.({ id: service.id, phase: "ready" });
+ return { ...base, action: "already-running", state: initial };
+ }
if (initial.state === "mismatch") {
// Never start a second copy beside a program that is already answering.
return { ...base, action: "unchanged", state: initial };
}
if (service.prepare !== undefined) {
+ onProgress?.({ id: service.id, phase: "preparing" });
const prepared = await run(root, service.prepare);
if (!prepared.ok) {
return { ...base, action: "unchanged", state: initial, detail: `${service.prepare.command} failed: ${prepared.detail}` };
@@ -122,11 +186,14 @@ async function bringUp(
// Nothing to keep running: preparing was the whole job, and the probe says
// whether it worked.
const state = await service.probe();
+ if (state.state === "ready") onProgress?.({ id: service.id, phase: "ready" });
return { ...base, action: state.state === "ready" ? "prepared" : "unchanged", state };
}
+ onProgress?.({ id: service.id, phase: "starting" });
await mkdir(directory(root), { recursive: true });
const logPath = join(directory(root), `${service.id}.log`);
+ await rotateLog(logPath);
const log = await open(logPath, "a");
try {
const child = spawn(service.start.command, [...service.start.args], {
@@ -140,7 +207,9 @@ async function bringUp(
return { ...base, action: "unchanged", state: initial, detail: `${service.start.command} did not start`, logPath };
}
await writeFile(join(directory(root), `${service.id}.pid`), `${child.pid}\n`);
+ onProgress?.({ id: service.id, phase: "waiting" });
const state = await waitForReady(service, maxWaitMs);
+ if (state.state === "ready") onProgress?.({ id: service.id, phase: "ready" });
return {
...base,
action: state.state === "ready" ? "started" : "unchanged",
@@ -160,10 +229,9 @@ export async function bringExternalServicesUp(
options: ExternalServiceOptions = {},
): Promise<{ readonly root: string; readonly services: readonly ExternalServiceReport[] }> {
const { root, services } = await declaredExternalServices(path, options);
- const reports: ExternalServiceReport[] = [];
- for (const { service, instances } of distinct(services)) {
- reports.push(await bringUp(root, service, instances, options.maxWaitMs ?? 300_000));
- }
+ const reports = await Promise.all(distinct(services).map(async ({ service, instances }) =>
+ await withServiceLifecycleLock(root, service.id, options.maxWaitMs ?? 300_000, async () =>
+ await bringUp(root, service, instances, options.maxWaitMs ?? 300_000, options.onProgress))));
return { root, services: reports };
}
@@ -173,27 +241,26 @@ export async function takeExternalServicesDown(
options: ExternalServiceOptions = {},
): Promise<{ readonly root: string; readonly services: readonly ExternalServiceReport[] }> {
const { root, services } = await declaredExternalServices(path, options);
- const reports: ExternalServiceReport[] = [];
- for (const { service, instances } of distinct(services)) {
+ const reports = await Promise.all(distinct(services).map(async ({ service, instances }): Promise<ExternalServiceReport> =>
+ await withServiceLifecycleLock(root, service.id, 30_000, async () => {
const base = { id: service.id, instances };
const pid = await readPid(root, service.id);
if (pid === undefined || !alive(pid)) {
if (pid !== undefined) await rm(join(directory(root), `${service.id}.pid`), { force: true });
const state = await service.probe();
- reports.push(state.state === "down"
+ return state.state === "down"
? { ...base, action: "nothing-to-stop", state }
// Someone else's process, or one started by hand. Killing it is not this
// command's business; saying so is.
- : { ...base, action: "not-ours", state, detail: `${service.id} is running but this project did not start it` });
- continue;
+ : { ...base, action: "not-ours", state, detail: `${service.id} is running but this project did not start it` };
}
process.kill(pid, "SIGTERM");
const deadline = Date.now() + 15_000;
while (alive(pid) && Date.now() < deadline) await sleep(200);
if (alive(pid)) process.kill(pid, "SIGKILL");
await rm(join(directory(root), `${service.id}.pid`), { force: true });
- reports.push({ ...base, action: "stopped", state: await service.probe(), pid });
- }
+ return { ...base, action: "stopped", state: await service.probe(), pid };
+ })));
return { root, services: reports };
}
@@ -203,16 +270,15 @@ export async function reportExternalServices(
options: ExternalServiceOptions = {},
): Promise<{ readonly root: string; readonly services: readonly ExternalServiceReport[] }> {
const { root, services } = await declaredExternalServices(path, options);
- const reports: ExternalServiceReport[] = [];
- for (const { service, instances } of distinct(services)) {
+ const reports = await Promise.all(distinct(services).map(async ({ service, instances }): Promise<ExternalServiceReport> => {
const state = await service.probe();
const pid = await readPid(root, service.id);
- reports.push({
+ return {
id: service.id,
instances,
state,
...(pid !== undefined && alive(pid) ? { pid } : {}),
- });
- }
+ };
+ }));
return { root, services: reports };
}
diff --git a/packages/local/src/index.ts b/packages/local/src/index.ts
index 364b43fb..80986e05 100644
--- a/packages/local/src/index.ts
+++ b/packages/local/src/index.ts
@@ -5,15 +5,23 @@ export {
localSchedulerImplementationDigest,
localWorkerImplementationDigest,
} from "./scheduler-package.js";
+export {
+ createLocalRuntimeControl,
+ createProjectLocalRuntimeControl,
+} from "./control.js";
+export { createLocalCredentialControl } from "./credentials.js";
export {
createLocalRuntime,
createProjectLocalRuntime,
} from "./runtime.js";
export {
createRuntimeFromConfig,
+ createRuntimeControlFromConfig,
+ createRuntimeCredentialsFromConfig,
declaredExternalServices,
doctorRuntimeConfig,
parseRuntimeConfig,
+ runtimeConfigPackageSelection,
} from "./config.js";
export {
bringExternalServicesUp,
diff --git a/packages/local/src/project-services.ts b/packages/local/src/project-services.ts
index 0a8aff4c..db934a6a 100644
--- a/packages/local/src/project-services.ts
+++ b/packages/local/src/project-services.ts
@@ -9,7 +9,7 @@ import type {
RuntimeServiceSelection,
} from "@narratage/runtime";
-import type { ProjectLocalRuntimeOptions } from "./types.js";
+import type { ProjectLocalRuntimeControlOptions } from "./types.js";
export type ProjectRuntimeServiceSelection = RuntimeServiceSelection;
export type ProjectRuntimeServiceAssembly = RuntimeServiceAssembly;
@@ -49,7 +49,7 @@ function packageCatalog(
*/
export async function createProjectRuntimeServices(
_root: string,
- options: ProjectLocalRuntimeOptions,
+ options: ProjectLocalRuntimeControlOptions,
): Promise<AssembledProjectRuntimeServices> {
const packages = [...options.runtimeServices];
try {
diff --git a/packages/local/src/runtime.ts b/packages/local/src/runtime.ts
index d3cc3486..43be9fd5 100644
--- a/packages/local/src/runtime.ts
+++ b/packages/local/src/runtime.ts
@@ -13,17 +13,12 @@ import {
loadNodePackageSet,
collectNodePackageComponents,
} from "@narratage/package-loader-node";
-import { isDigest } from "@narratage/protocol";
import {
RuntimeModuleRegistry,
createBuildDispatchIdentity,
- isEnumerableBuildStore,
- isManagedArtifactStore,
isStreamingArtifactStore,
- operationCancellationRequestId,
resolveRuntimeProfile,
sealRuntimeProfile,
- writableCredentialStore,
} from "@narratage/runtime";
import { TypeValidatorRegistry } from "@narratage/validation";
import type {
@@ -32,6 +27,8 @@ import type {
} from "@narratage/runtime";
import { createProjectRuntimeServices } from "./project-services.js";
+import { createLocalRuntimeControl } from "./control.js";
+import { createLocalCredentialControl } from "./credentials.js";
import type {
CreateLocalRuntimeOptions,
LocalBuildOptions,
@@ -64,25 +61,6 @@ function nonNegativeInteger(value: number, subject: string): number {
return value;
}
-function collectArtifactDigests(
- value: unknown,
- digests: Set<import("@narratage/protocol").Digest>,
- seen = new WeakSet<object>(),
-): void {
- if (value === null || typeof value !== "object") return;
- if (seen.has(value)) return;
- seen.add(value);
- if (!Array.isArray(value)) {
- const item = value as Record<string, unknown>;
- if (item.kind === "blob" && typeof item.digest === "string" && isDigest(item.digest)) {
- digests.add(item.digest);
- }
- }
- for (const nested of Array.isArray(value) ? value : Object.values(value)) {
- collectArtifactDigests(nested, digests, seen);
- }
-}
-
async function wait(delayMs: number, signal: AbortSignal | undefined): Promise<void> {
if (signal?.aborted === true) throw signal.reason ?? new Error("Local Runtime follow was aborted");
await new Promise<void>((resolveWait, reject) => {
@@ -192,22 +170,38 @@ export async function createLocalRuntime(
scheduling,
runtimeClosure: options.closure.value,
});
- const credentialDescriptions = (options.endpoints ?? []).flatMap((item) => item.credentials);
- const credential = (endpoint: string, slot: string) => {
- const matches = credentialDescriptions.filter((item) => item.endpoint === endpoint && item.slot === slot);
- assert(matches.length === 1, matches.length === 0
- ? `Endpoint ${endpoint} has no credential slot ${slot}`
- : `Endpoint ${endpoint} repeats credential slot ${slot}`);
- return matches[0]!;
- };
- const credentialStatus = async (item: typeof credentialDescriptions[number]) => ({
- ...structuredClone(item),
- configured: await options.credentialStore.resolve(item.ref) !== undefined,
- writable: await writableCredentialStore(options.credentialStore, item.ref) !== undefined,
+ const credentialControl = createLocalCredentialControl({
+ credentialStore: options.credentialStore,
+ endpoints: options.endpoints ?? [],
+ });
+ const control = createLocalRuntimeControl({
+ buildStore: options.buildStore,
+ ...(buildCatalog === undefined ? {} : { buildCatalog }),
+ operationStore: options.operationStore,
+ dispatchStore: options.dispatchStore,
+ journal: options.journal,
+ artifactStore: options.artifactStore,
});
const stageAttachments = async (request: LocalBuildRequest): Promise<void> => {
for (const item of request.attachments ?? []) {
- const stored = await options.artifactStore.put(Uint8Array.from(item.bytes), item.artifact.mediaType);
+ const stream = await item.open();
+ const stored = isStreamingArtifactStore(options.artifactStore)
+ ? await options.artifactStore.putStream(stream, item.artifact.mediaType)
+ : await options.artifactStore.put(await (async () => {
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ for await (const chunk of stream) {
+ chunks.push(Uint8Array.from(chunk));
+ size += chunk.byteLength;
+ }
+ const bytes = new Uint8Array(size);
+ let offset = 0;
+ for (const chunk of chunks) {
+ bytes.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return bytes;
+ })(), item.artifact.mediaType);
assert(
stored.digest === item.artifact.digest
&& stored.size === item.artifact.size
@@ -241,7 +235,8 @@ export async function createLocalRuntime(
}
}
assert(stored.state.id === request.state.id,
- `Build ${request.id} already names another Core Build`);
+ `Build ${request.id} already names another Core Build. Choose a new --build-id for a new Run, `
+ + "or restore the original Author/Run Sources to resume this Build");
const created = await options.dispatchStore.create(createBuildDispatchIdentity({
build: request.id,
core: request.state.id,
@@ -282,131 +277,19 @@ export async function createLocalRuntime(
return result;
};
return {
+ ...control,
+ ...credentialControl,
build: runBuild,
async buildMany(requests) {
return await Promise.all(requests.map(submit));
},
- async status(build) {
- return {
- build: await options.buildStore.read(build),
- catalog: await buildCatalog?.read(build),
- operations: await options.operationStore.list({ build }),
- dispatch: await options.dispatchStore.read(build),
- };
- },
- async queue() {
- return {
- dispatches: await options.dispatchStore.list(),
- capacity: await options.dispatchStore.listCapacity(),
- };
- },
- async operation(id) {
- return await options.operationStore.read(id);
- },
- async journal(query) {
- return await options.journal.list(query);
- },
- async credentials(endpoint) {
- const selected = credentialDescriptions.filter((item) => endpoint === undefined || item.endpoint === endpoint);
- return await Promise.all(selected.map(credentialStatus));
- },
- async putCredential(endpoint, slot, secret) {
- assert(secret.length > 0, "credential secret is empty");
- const item = credential(endpoint, slot);
- const store = await writableCredentialStore(options.credentialStore, item.ref);
- assert(store !== undefined, `CredentialStore ${item.ref.store} is not writable`);
- await store.put(item.ref, { secret });
- return await credentialStatus(item);
- },
- async deleteCredential(endpoint, slot) {
- const item = credential(endpoint, slot);
- const store = await writableCredentialStore(options.credentialStore, item.ref);
- assert(store !== undefined, `CredentialStore ${item.ref.store} is not writable`);
- const deleted = await store.delete(item.ref);
- return { deleted, credential: await credentialStatus(item) };
- },
- async builds() {
- return await buildCatalog?.list() ?? [];
- },
- async cancel(build, reason) {
- if (await options.dispatchStore.read(build) === undefined) return undefined;
- const dispatch = await options.dispatchStore.requestCancellation(build, reason);
- await options.journal.append({
- at: Date.now(),
- kind: "cancellation-requested",
- build,
- detail: { ...(reason === undefined ? {} : { reason }) },
- });
- return dispatch;
- },
- async cancelOperation(id, reason) {
- let current = await options.operationStore.read(id);
- if (current === undefined) return undefined;
- if (current.cancellation === undefined) {
- const requestedAt = Date.now();
- while (current.cancellation === undefined) {
- const status = current.status === "completed" || current.status === "failed" || current.status === "cancelled"
- ? "too-late" as const
- : "requested" as const;
- const written = await options.operationStore.compareAndSwap(current.id, current.revision, {
- status: "control",
- cancellation: {
- requestedAt,
- requestId: operationCancellationRequestId(current.id, requestedAt),
- status,
- attempts: 0,
- },
- });
- current = written.status === "stored" ? written.snapshot : written.current;
- }
- await options.journal.append({
- at: requestedAt,
- kind: "cancellation-requested",
- build: current.build,
- operation: current.id,
- detail: { scope: "operation", ...(reason === undefined ? {} : { reason }) },
- });
- }
- await options.dispatchStore.wake(current.build);
- return current;
- },
async workOnce(workOptions) {
return await worker.runOnce(workOptions);
},
async work(workOptions) {
await worker.run(workOptions);
},
- async readArtifact(digest) {
- return await options.artifactStore.get(digest);
- },
- async openArtifact(digest) {
- if (isStreamingArtifactStore(options.artifactStore)) return await options.artifactStore.open(digest);
- const bytes = await options.artifactStore.get(digest);
- return bytes === undefined ? undefined : (async function* () { yield bytes; })();
- },
- async garbageCollectArtifacts(gc = {}) {
- assert(isManagedArtifactStore(options.artifactStore),
- "selected ArtifactStore does not expose explicit retention management");
- assert(isEnumerableBuildStore(options.buildStore),
- "selected BuildStore does not expose the maintenance index required for Artifact GC");
- const reachable = new Set<import("@narratage/protocol").Digest>();
- for (const snapshot of await options.buildStore.list()) collectArtifactDigests(snapshot.state, reachable);
- for (const operation of await options.operationStore.list({})) collectArtifactDigests(operation, reachable);
- const stored = await options.artifactStore.list();
- const unreachable = stored.filter((digest) => !reachable.has(digest)).sort();
- const deleted: import("@narratage/protocol").Digest[] = [];
- if (gc.apply === true) {
- for (const digest of unreachable) {
- if (await options.artifactStore.delete(digest)) deleted.push(digest);
- }
- }
- return {
- reachable: [...reachable].sort(),
- unreachable,
- deleted,
- };
- },
- close() {},
+ close: control.close,
};
}
@@ -423,6 +306,8 @@ export async function createProjectLocalRuntime(
const lockedPackageSet = options.packageLock === undefined
? undefined
: await loadNodePackageSet(resolve(root, options.packageLock), packageRoot);
+ assert(lockedPackageSet === undefined || options.implementationClosure === undefined,
+ "packageLock and implementationClosure are two sources for one implementation identity");
const lockedComponents = lockedPackageSet === undefined
? []
: collectNodePackageComponents(lockedPackageSet.contributions);
@@ -488,12 +373,15 @@ export async function createProjectLocalRuntime(
: { maxEventsPerBuild: options.scheduling.maxEventsPerBuild }),
},
...(options.validators === undefined ? {} : { validators: options.validators }),
- ...(lockedPackageSet === undefined ? {} : { implementationClosure: lockedPackageSet.lock.digest }),
+ ...(lockedPackageSet === undefined && options.implementationClosure === undefined
+ ? {}
+ : { implementationClosure: lockedPackageSet?.lock.digest ?? options.implementationClosure }),
});
return {
build: runtime.build,
buildMany: runtime.buildMany,
status: runtime.status,
+ activity: runtime.activity,
queue: runtime.queue,
operation: runtime.operation,
journal: runtime.journal,
diff --git a/packages/local/src/types.ts b/packages/local/src/types.ts
index 72ba52ab..cd15ce62 100644
--- a/packages/local/src/types.ts
+++ b/packages/local/src/types.ts
@@ -12,6 +12,7 @@ import type {
BuildSchedulerOptions,
BuildSnapshot,
BuildStore,
+ BuildDispatchStore,
BuildDispatchSnapshot,
CredentialStore,
OperationStore,
@@ -57,6 +58,23 @@ export type CreateLocalRuntimeOptions = {
readonly implementationClosure?: import("@narratage/protocol").Digest;
};
+export type CreateLocalRuntimeControlOptions = {
+ readonly buildStore: BuildStore;
+ readonly buildCatalog?: BuildCatalog;
+ readonly operationStore: OperationStore;
+ readonly dispatchStore: BuildDispatchStore;
+ readonly journal: RuntimeJournal;
+ readonly artifactStore: ArtifactStore;
+ /** Optional owner supplied by the project service assembly. */
+ readonly close?: () => Awaitable<void>;
+};
+
+export type CreateLocalCredentialControlOptions = {
+ readonly credentialStore: CredentialStore;
+ readonly endpoints: readonly EndpointPackage[];
+ readonly close?: () => Awaitable<void>;
+};
+
export type ProjectLocalRuntimeOptions = {
/** Project directory containing the private .svml Runtime directory. Defaults to cwd. */
readonly root?: string;
@@ -65,6 +83,8 @@ export type ProjectLocalRuntimeOptions = {
readonly buildCatalog?: BuildCatalog;
/** Exact installed implementation package lock. Source imports cannot change this selection. */
readonly packageLock?: string;
+ /** Already verified implementation identity when components are supplied by this Host process. */
+ readonly implementationClosure?: import("@narratage/protocol").Digest;
/**
* Replaceable parts of the Runtime itself. One package may fill several roles;
* every selected role is explicit, and two packages exposing the same instance
@@ -84,6 +104,14 @@ export type ProjectLocalRuntimeOptions = {
readonly validators?: LocalTypeValidatorRegistry;
};
+export type ProjectLocalRuntimeControlOptions = Pick<ProjectLocalRuntimeOptions,
+ | "root"
+ | "buildCatalog"
+ | "runtimeServices"
+ | "runtimeSelection"
+ | "allowedPermissions"
+>;
+
export type LocalBuildRequest = {
/** Stable user/run identity. Reusing it resumes only the same Core Build identity. */
readonly id: string;
@@ -109,9 +137,17 @@ export type LocalRuntimeStatus = {
readonly dispatch: BuildDispatchSnapshot | undefined;
};
+/** Lightweight execution facts. Unlike status(), this does not read or verify the BuildState. */
+export type LocalRuntimeActivity = {
+ readonly operations: readonly OperationSnapshot[];
+ readonly dispatch: BuildDispatchSnapshot | undefined;
+};
+
export type LocalRuntimeQueue = {
readonly dispatches: readonly BuildDispatchSnapshot[];
readonly capacity: readonly import("@narratage/runtime").CapacityReservation[];
+ /** Execution facts belonging to non-terminal queued Builds. */
+ readonly operations: readonly OperationSnapshot[];
};
export type LocalCredentialStatus = import("@narratage/endpoint-kit").EndpointCredentialDescription & {
@@ -136,6 +172,7 @@ export type LocalRuntime = {
build(request: LocalBuildRequest, options?: LocalBuildOptions): Promise<LocalBuildSubmission>;
buildMany(requests: readonly LocalBuildRequest[]): Promise<readonly LocalBuildSubmission[]>;
status(build: string): Promise<LocalRuntimeStatus>;
+ activity(build: string): Promise<LocalRuntimeActivity>;
queue(): Promise<LocalRuntimeQueue>;
operation(id: Digest): Promise<OperationSnapshot | undefined>;
journal(query?: import("@narratage/runtime").RuntimeJournalQuery): Promise<readonly import("@narratage/runtime").RuntimeJournalEntry[]>;
@@ -154,3 +191,27 @@ export type LocalRuntime = {
garbageCollectArtifacts(options?: { readonly apply?: boolean }): Promise<ArtifactGarbageCollection>;
close(): Awaitable<void>;
};
+
+/** Durable project control that needs Stores but no Producer, Endpoint, Driver or Worker. */
+export type LocalRuntimeControl = Pick<LocalRuntime,
+ | "status"
+ | "activity"
+ | "queue"
+ | "operation"
+ | "journal"
+ | "builds"
+ | "cancel"
+ | "cancelOperation"
+ | "readArtifact"
+ | "openArtifact"
+ | "garbageCollectArtifacts"
+ | "close"
+>;
+
+/** Credential control for one or more exact Endpoint declarations; no execution state is opened. */
+export type LocalCredentialControl = Pick<LocalRuntime,
+ | "credentials"
+ | "putCredential"
+ | "deleteCredential"
+ | "close"
+>;
diff --git a/packages/local/test/config.test.ts b/packages/local/test/config.test.ts
index 5722a892..0743612f 100644
--- a/packages/local/test/config.test.ts
+++ b/packages/local/test/config.test.ts
@@ -8,6 +8,8 @@ import {
createRuntimeEndpointAdapterFacet,
createRuntimeServiceAdapterFacet,
} from "@narratage/runtime-adapter";
+import { digestOf } from "@narratage/protocol";
+import { credentialRef, defineRuntimeServicePackage } from "@narratage/runtime";
import {
createRuntimeFromConfig,
@@ -31,6 +33,17 @@ const services = {
const required = { runtimeServices: [], services, scheduling: { maxConcurrency: 3 } } as const;
+function endpointPackage(instance: string, credentials: readonly unknown[] = []) {
+ return {
+ name: instance,
+ manifest: { facets: [] },
+ instance: { id: instance },
+ bindings: [],
+ credentials,
+ install() {},
+ } as never;
+}
+
test("declarative Runtime config has no implicit local services", () => {
assert.throws(() => parseRuntimeConfig({
format: "svml.runtime-config@1",
@@ -126,9 +139,7 @@ test("doctor names the external program a Provider needs, and the command that s
const declare = (use: string, service: unknown) =>
registry.registerFacet(createRuntimeEndpointAdapterFacet({
use,
- validate() {},
- create: () => ({}) as never,
- service: () => service as never,
+ activate: (context) => ({ endpoint: endpointPackage(context.instance), externalService: service as never }),
}));
declare("example.absent", {
id: "absent-one",
@@ -159,7 +170,7 @@ test("doctor names the external program a Provider needs, and the command that s
await rm(root, { recursive: true, force: true });
});
-test("doctor validates closed config without constructing adapters or cascading one root failure", async () => {
+test("doctor activates one pure Endpoint declaration without constructing Runtime services", async () => {
const root = await mkdtemp(join(tmpdir(), "svml-read-only-doctor-"));
const path = join(root, "svml.runtime.json");
await writeFile(path, JSON.stringify({
@@ -190,33 +201,26 @@ test("doctor validates closed config without constructing adapters or cascading
}));
registry.registerFacet(createRuntimeEndpointAdapterFacet({
use: "example.invalid",
- validate() {
+ activate() {
throw new Error("mode is invalid");
},
- create() {
- constructed += 1;
- throw new Error("doctor constructed the invalid Endpoint");
- },
- doctor() {
- invalidDoctorCalls += 1;
- return [{ severity: "error", code: "SECOND_ERROR", message: "must never be reported" }];
- },
}));
registry.registerFacet(createRuntimeEndpointAdapterFacet({
use: "example.missing-credential",
- validate() {},
- create() {
- constructed += 1;
- throw new Error("missing credential also prevents construction");
- },
- doctor: () => [{
- severity: "error",
- code: "RUNTIME_CREDENTIAL_MISSING",
- message: "one declared credential is absent",
- }],
- service() {
+ activate(context) {
serviceDeclarations += 1;
- return { id: "should-not-be-probed", probe: async () => ({ state: "ready" }) };
+ return {
+ endpoint: endpointPackage(context.instance),
+ diagnose: () => {
+ invalidDoctorCalls += 1;
+ return [{
+ severity: "error" as const,
+ code: "RUNTIME_CREDENTIAL_MISSING",
+ message: "one declared credential is absent",
+ }];
+ },
+ externalService: { id: "should-not-be-probed", probe: async () => ({ state: "ready" as const }) },
+ };
},
}));
@@ -229,7 +233,67 @@ test("doctor validates closed config without constructing adapters or cascading
{ severity: "error", code: "RUNTIME_CREDENTIAL_MISSING", message: "one declared credential is absent" },
]);
assert.equal(constructed, 0);
- assert.equal(invalidDoctorCalls, 0);
- assert.equal(serviceDeclarations, 0);
+ assert.equal(invalidDoctorCalls, 1);
+ assert.equal(serviceDeclarations, 1);
+ await rm(root, { recursive: true, force: true });
+});
+
+test("doctor resolves credentials from the same Endpoint declaration used by execution", async () => {
+ const root = await mkdtemp(join(tmpdir(), "svml-credential-doctor-"));
+ const path = join(root, "svml.runtime.json");
+ await writeFile(path, JSON.stringify({
+ format: "svml.runtime-config@1",
+ ...required,
+ runtimeServices: [{ use: "example.credentials", instance: "credentials", config: {} }],
+ endpoints: [{ use: "example.provider", instance: "provider", config: {} }],
+ permissions: [],
+ }));
+
+ let endpointActivations = 0;
+ let credentialStoreConstructions = 0;
+ let credentialStoreCloses = 0;
+ const registry = new RuntimeAdapterRegistry();
+ registry.registerFacet(createRuntimeEndpointAdapterFacet({
+ use: "example.provider",
+ activate(context) {
+ endpointActivations += 1;
+ return { endpoint: endpointPackage(context.instance, [{
+ endpoint: context.instance,
+ slot: "token",
+ label: "Example token",
+ kind: "secret",
+ ref: credentialRef("env", "EXAMPLE_TOKEN_THAT_IS_NOT_SET"),
+ }]) };
+ },
+ }));
+ registry.registerFacet(createRuntimeServiceAdapterFacet({
+ use: "example.credentials",
+ validate() {},
+ create(context) {
+ credentialStoreConstructions += 1;
+ return defineRuntimeServicePackage({
+ name: context.instance,
+ module: { name: "example.credentials", version: "1" },
+ services: [{
+ role: "credential-store",
+ facet: "credentials",
+ instance: context.instance,
+ implementation: { locator: "example.credentials", digest: digestOf("example.credentials@1") },
+ service: { async resolve() { return undefined; } },
+ }],
+ close() { credentialStoreCloses += 1; },
+ });
+ },
+ }));
+
+ const { diagnostics } = await doctorRuntimeConfig(path, { registry });
+ assert.deepEqual(diagnostics.map(({ code, message, subject }) => ({ code, message, subject })), [{
+ code: "RUNTIME_CREDENTIAL_MISSING",
+ message: "Example token for Endpoint provider is not configured in CredentialStore env",
+ subject: "provider.token",
+ }]);
+ assert.equal(endpointActivations, 1);
+ assert.equal(credentialStoreConstructions, 1);
+ assert.equal(credentialStoreCloses, 1);
await rm(root, { recursive: true, force: true });
});
diff --git a/packages/local/test/external-services.test.ts b/packages/local/test/external-services.test.ts
index 05589051..f62852ab 100644
--- a/packages/local/test/external-services.test.ts
+++ b/packages/local/test/external-services.test.ts
@@ -7,14 +7,21 @@ import { setTimeout as sleep } from "node:timers/promises";
import { createRuntimeEndpointAdapterFacet } from "@narratage/runtime-adapter";
import type { RuntimeExternalService } from "@narratage/runtime-adapter";
+import type { CapabilityRef } from "@narratage/protocol";
import {
bringExternalServicesUp,
+ declaredExternalServices,
reportExternalServices,
RuntimeAdapterRegistry,
takeExternalServicesDown,
} from "@narratage/local";
+const requiredCapability = {
+ module: { name: "example.capabilities", version: "1" },
+ name: "Required",
+} as const satisfies CapabilityRef;
+
/**
* A stand-in program: `start` writes a file and sleeps, and the probe reads that
* file back. That is enough to exercise detaching, the pid file, waiting for
@@ -49,13 +56,60 @@ async function project(service: (root: string) => RuntimeExternalService) {
const registry = new RuntimeAdapterRegistry();
registry.registerFacet(createRuntimeEndpointAdapterFacet({
use: "example.program",
- validate() {},
- create: () => ({}) as never,
- service: () => service(root),
+ activate: (context) => ({
+ endpoint: {
+ name: context.instance,
+ manifest: { facets: [] },
+ instance: { id: context.instance },
+ bindings: [{
+ capability: requiredCapability,
+ returns: { module: { name: "example.values", version: "1" }, name: "Value" },
+ endpoint: context.instance,
+ }],
+ credentials: [],
+ install() {},
+ } as never,
+ externalService: service(root),
+ }),
}));
return { root, path, options: { registry } };
}
+test("a Build service selection ignores programs outside its demanded capabilities", async () => {
+ let probes = 0;
+ const configured = await project(() => ({
+ id: "unused",
+ async probe() {
+ probes += 1;
+ return { state: "ready" as const };
+ },
+ }));
+ const result = await bringExternalServicesUp(configured.path, {
+ ...configured.options,
+ capabilities: [{
+ module: { name: "another.capabilities", version: "1" },
+ name: "Other",
+ }],
+ });
+ assert.deepEqual(result.services, []);
+ assert.equal(probes, 0);
+});
+
+test("an empty Build capability set loads no Runtime Adapter closure", async () => {
+ const configured = await project(() => ({
+ id: "must-not-load",
+ async probe() {
+ throw new Error("an empty capability set must not activate or probe an Endpoint");
+ },
+ }));
+ const result = await declaredExternalServices(configured.path, {
+ // Deliberately omit the registry. Reaching adapter activation would fail
+ // because `example.program` is not installed in a package lock.
+ capabilities: [],
+ });
+ assert.deepEqual(result, { root: configured.root, services: [] });
+});
+
function fileBackedService(root: string, marker: string): RuntimeExternalService {
return {
id: "example",
@@ -74,12 +128,18 @@ function fileBackedService(root: string, marker: string): RuntimeExternalService
test("up starts the program once for every Endpoint that drives it, and down stops it", async () => {
const marker = join(await mkdtemp(join(tmpdir(), "svml-marker-")), "ready");
const { root, path, options } = await project((projectRoot) => fileBackedService(projectRoot, marker));
+ const progress: string[] = [];
- const started = await bringExternalServicesUp(path, { ...options, maxWaitMs: 20_000 });
+ const started = await bringExternalServicesUp(path, {
+ ...options,
+ maxWaitMs: 20_000,
+ onProgress: (event) => progress.push(`${event.id}:${event.phase}`),
+ });
assert.equal(started.services.length, 1, "one program, not one per Endpoint");
assert.deepEqual(started.services[0]!.instances, ["one", "two"]);
assert.equal(started.services[0]!.action, "started");
assert.deepEqual(started.services[0]!.state, { state: "ready" });
+ assert.deepEqual(progress, ["example:checking", "example:starting", "example:waiting", "example:ready"]);
const pid = started.services[0]!.pid!;
assert.equal(await readFile(join(root, ".svml", "services", "example.pid"), "utf8"), `${pid}\n`);
@@ -99,6 +159,27 @@ test("up starts the program once for every Endpoint that drives it, and down sto
await rm(root, { recursive: true, force: true });
});
+test("concurrent up calls atomically share one external service process", async () => {
+ const markerRoot = await mkdtemp(join(tmpdir(), "svml-marker-concurrent-"));
+ const marker = join(markerRoot, "ready");
+ const { root, path, options } = await project((projectRoot) => fileBackedService(projectRoot, marker));
+ try {
+ const results = await Promise.all(Array.from({ length: 6 }, async () =>
+ await bringExternalServicesUp(path, { ...options, maxWaitMs: 20_000 })));
+ assert.equal(results.filter((item) => item.services[0]!.action === "started").length, 1);
+ assert.equal(results.filter((item) => item.services[0]!.action === "already-running").length, 5);
+ const pid = Number.parseInt(await readFile(join(root, ".svml", "services", "example.pid"), "utf8"), 10);
+ assert.ok(Number.isSafeInteger(pid) && pid > 0);
+ await takeExternalServicesDown(path, options);
+ await sleep(100);
+ assert.throws(() => process.kill(pid, 0));
+ } finally {
+ await takeExternalServicesDown(path, options).catch(() => undefined);
+ await rm(root, { recursive: true, force: true });
+ await rm(markerRoot, { recursive: true, force: true });
+ }
+});
+
test("a program answering with another identity is never joined by a second copy", async () => {
const { path, options } = await project(() => ({
id: "example",
diff --git a/packages/local/test/local.test.ts b/packages/local/test/local.test.ts
index 140cb8c5..a4a7c3ed 100644
--- a/packages/local/test/local.test.ts
+++ b/packages/local/test/local.test.ts
@@ -730,7 +730,7 @@ test("project local runtime accepts a permission-checked replacement ArtifactSto
await runtime.build({
id: "source-artifact-staging",
state: createGreetingBuild(),
- attachments: [{ artifact: sourceArtifact, bytes }],
+ attachments: [{ artifact: sourceArtifact, open: async () => (async function* () { yield bytes; })() }],
});
const blocked = await runtime.workOnce({ owner: "artifact-test", leaseMs: 5_000 });
assert.equal(blocked?.terminal, "failed");
@@ -740,7 +740,10 @@ test("project local runtime accepts a permission-checked replacement ArtifactSto
runtime.build({
id: "tampered-source-artifact",
state: createGreetingBuild(),
- attachments: [{ artifact: sourceArtifact, bytes: new Uint8Array([0]) }],
+ attachments: [{
+ artifact: sourceArtifact,
+ open: async () => (async function* () { yield new Uint8Array([0]); })(),
+ }],
}),
/does not match its staged bytes/u,
);
diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.ts
index 69b9a561..d13a3254 100644
--- a/packages/media-execution/src/execute.ts
+++ b/packages/media-execution/src/execute.ts
@@ -1,6 +1,7 @@
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { createReadStream } from "node:fs";
+import { mkdtemp, open, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { mediaTypes, sealMediaInspection, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyRenderedVisual, verifySynchronizedMedia, verifyTimelineAudio } from "@narratage/media";
@@ -50,7 +51,9 @@ import type { AnimatedWebp } from "./webp.js";
*/
export type MediaArtifactGateway = {
get(source: BlobRef): Promise<Uint8Array | undefined>;
+ open(source: BlobRef): Promise<AsyncIterable<Uint8Array> | undefined>;
put(bytes: Uint8Array, mediaType: string): Promise<BlobRef>;
+ putFile(path: string, mediaType: string): Promise<BlobRef>;
};
export type MediaExecutionEnvironment = {
@@ -176,7 +179,22 @@ async function stageArtifact(
source: BlobRef,
path: string,
): Promise<void> {
- await writeFile(path, await sourceBytes(env, source));
+ const chunks = await env.artifacts.open(source);
+ assert(chunks !== undefined, `Media source ${source.digest} is unavailable`);
+ const file = await open(path, "w");
+ const hash = createHash("sha256");
+ let size = 0;
+ try {
+ for await (const chunk of chunks) {
+ await file.write(chunk);
+ hash.update(chunk);
+ size += chunk.byteLength;
+ }
+ } finally {
+ await file.close();
+ }
+ assert(size === source.size, `Media source ${source.digest} size differs`);
+ assert(`sha256:${hash.digest("hex")}` === source.digest, `Media source ${source.digest} digest differs`);
}
function timestampFraction(value: MediaTimestamp): { numerator: bigint; denominator: bigint } {
@@ -420,11 +438,16 @@ async function outputInspection(args: {
readonly maxProbeOutputBytes: number;
readonly sharedLibraryPath?: string;
}): Promise<MediaInspection> {
- const bytes = await readFile(args.path);
+ const hash = createHash("sha256");
+ let size = 0;
+ for await (const chunk of createReadStream(args.path)) {
+ hash.update(chunk);
+ size += chunk.byteLength;
+ }
const source: BlobRef = {
kind: "blob",
- digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` as BlobRef["digest"],
- size: bytes.byteLength,
+ digest: `sha256:${hash.digest("hex")}` as BlobRef["digest"],
+ size,
mediaType: args.mediaType,
};
return await inspectFile({ source, input: args.path, ffprobePath: args.ffprobePath,
@@ -608,9 +631,10 @@ export async function executeInspectMedia(
const work = await mkdtemp(join(tmpdir(), "svml-media-inspect-"));
try {
const input = join(work, "source.bin");
- const bytes = await sourceBytes(env, need.source);
- await writeFile(input, bytes);
- const animation = parseAnimatedWebp(bytes);
+ const animationBytes = need.source.mediaType === "image/webp" ? await sourceBytes(env, need.source) : undefined;
+ if (animationBytes === undefined) await stageArtifact(env, need.source, input);
+ else await writeFile(input, animationBytes);
+ const animation = animationBytes === undefined ? undefined : parseAnimatedWebp(animationBytes);
const inspection = animation === undefined
? await inspectFile({
source: need.source,
@@ -636,9 +660,10 @@ export async function executeNormalizeMedia(
const work = await mkdtemp(join(tmpdir(), "svml-media-normalize-"));
try {
const input = join(work, "source.bin");
- const bytes = await sourceBytes(env, need.source);
- await writeFile(input, bytes);
- const animation = parseAnimatedWebp(bytes);
+ const animationBytes = need.source.mediaType === "image/webp" ? await sourceBytes(env, need.source) : undefined;
+ if (animationBytes === undefined) await stageArtifact(env, need.source, input);
+ else await writeFile(input, animationBytes);
+ const animation = animationBytes === undefined ? undefined : parseAnimatedWebp(animationBytes);
let visualArtifact: BlobRef | undefined;
let visualWidth: number | undefined;
let visualHeight: number | undefined;
@@ -682,8 +707,7 @@ export async function executeNormalizeMedia(
assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1
&& visual.rotationDegrees === 0,
"Normalized visual retains non-square samples or display rotation");
- const bytes = await readFile(output);
- visualArtifact = await env.artifacts.put(bytes, "video/mp4");
+ visualArtifact = await env.artifacts.putFile(output, "video/mp4");
visualWidth = visual.width;
visualHeight = visual.height;
}
@@ -720,7 +744,7 @@ export async function executeNormalizeMedia(
const audio = inspected.streams.find((item): item is MediaAudioStream => item.kind === "audio");
assert(audio?.decodedSampleFrames === plan.sampleFrames && audio.sampleRate === 48_000 && audio.channels === 2,
"Normalized audio sample shape differs from its plan");
- audioArtifact = await env.artifacts.put(await readFile(output), "audio/wav");
+ audioArtifact = await env.artifacts.putFile(output, "audio/wav");
}
const media = sealSynchronizedMedia({
contract: "svml.synchronized-media@1",
@@ -886,7 +910,7 @@ export async function executeTransformMedia(
assert(audios[0].sampleRate === 48_000 && audios[0].channels === 2,
"Transformed media audio is not 48 kHz stereo");
}
- const artifact = await env.artifacts.put(await readFile(output), "video/mp4");
+ const artifact = await env.artifacts.putFile(output, "video/mp4");
return artifactResult(artifact, canonicalize({
provider: env.label,
operation: "transform",
@@ -921,8 +945,7 @@ export async function executeExtractAudio(
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
- const bytes = await readFile(output);
- const artifact = await env.artifacts.put(bytes, "audio/wav");
+ const artifact = await env.artifacts.putFile(output, "audio/wav");
await assertCanonicalWav({
path: output,
source: artifact,
@@ -1045,7 +1068,7 @@ export async function executeProjectSpeechEvidenceAudio(
&& streams[0]!.channels === 1
&& streams[0]!.decodedSampleFrames === need.evidenceSampleFrames,
"Alignment evidence must be exact 16 kHz mono PCM s16");
- const artifact = await env.artifacts.put(await readFile(output), "audio/wav");
+ const artifact = await env.artifacts.putFile(output, "audio/wav");
const evidence: SpeechEvidenceAudio = sealSpeechEvidenceAudio({
contract: "svml.speech-evidence-audio@1",
artifact,
@@ -1143,8 +1166,7 @@ export async function executeRenderTimelineAudio(
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
- const bytes = await readFile(output);
- const artifact = await env.artifacts.put(bytes, "audio/wav");
+ const artifact = await env.artifacts.putFile(output, "audio/wav");
const audio = await assertCanonicalWav({
path: output,
source: artifact,
@@ -1254,7 +1276,7 @@ export async function executeMuxProgramMedia(
assert(roundPositive(finalAudioSpan.numerator * 48_000n, finalAudioSpan.denominator)
=== need.audio.sampleFrames,
"Final mux audio presentation span differs from TimelineAudio");
- const artifact = await env.artifacts.put(await readFile(output), "video/mp4");
+ const artifact = await env.artifacts.putFile(output, "video/mp4");
const value: MuxedMedia = sealMuxedMedia({
contract: "svml.muxed-media@1",
frameRate: need.visual.frameRate,
diff --git a/packages/mimo-tts/README.md b/packages/mimo-tts/README.md
index b737e76c..3336a5c9 100644
--- a/packages/mimo-tts/README.md
+++ b/packages/mimo-tts/README.md
@@ -37,7 +37,7 @@ Official API reference: <https://mimo.mi.com/docs/zh-CN/quick-start/usage-guide/
Add the author package to the project's explicit package lock before compiling:
```sh
-pnpm narratage lock-packages svml.packages.lock \
+node --run narratage -- lock-packages svml.packages.lock \
--package @narratage/script \
--package @narratage/mimo-tts \
--root .
diff --git a/packages/package-loader-node/src/index.ts b/packages/package-loader-node/src/index.ts
index c1563144..0b74ac5d 100644
--- a/packages/package-loader-node/src/index.ts
+++ b/packages/package-loader-node/src/index.ts
@@ -7,6 +7,7 @@ export {
createNodePackageLock,
loadNodePackageSet,
loadNodePackageContributions,
+ readNodePackageLock,
writeNodePackageLock,
} from "./lock.js";
export type * from "./types.js";
diff --git a/packages/package-loader-node/src/lock.ts b/packages/package-loader-node/src/lock.ts
index 933df226..1752d0c3 100644
--- a/packages/package-loader-node/src/lock.ts
+++ b/packages/package-loader-node/src/lock.ts
@@ -1,13 +1,16 @@
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
import {
+ open,
readdir,
readFile,
realpath,
+ rename,
+ rm,
stat,
- writeFile,
} from "node:fs/promises";
import { createRequire } from "node:module";
import {
+ basename,
dirname,
isAbsolute,
join,
@@ -25,6 +28,7 @@ import type {
LockedPackageArtifact,
LoadedNodePackageSet,
NodePackageContribution,
+ NodePackageLockCreateOptions,
NodePackageLock,
} from "./types.js";
@@ -45,6 +49,12 @@ type ResolvedPhysicalPackage = {
readonly json: PackageJson;
};
+type ReachablePhysicalPackage = {
+ readonly physical: ResolvedPhysicalPackage;
+ /** Direct selection roots that can reach this physical package. Ephemeral; never serialized. */
+ readonly roots: ReadonlySet<string>;
+};
+
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
@@ -150,27 +160,41 @@ async function packageClosure(entries: readonly string[], root: string): Promise
async function packageClosureFromPhysical(
entries: readonly ResolvedPhysicalPackage[],
): Promise<readonly ResolvedPhysicalPackage[]> {
- const resolved = new Map<string, ResolvedPhysicalPackage>();
- const visit = async (item: ResolvedPhysicalPackage): Promise<void> => {
+ return (await packageClosureFromRoots(entries.map((physical, index) => ({
+ root: `anonymous:${index}`,
+ physical,
+ })))).map((item) => item.physical);
+}
+
+async function packageClosureFromRoots(
+ entries: readonly { readonly root: string; readonly physical: ResolvedPhysicalPackage }[],
+): Promise<readonly ReachablePhysicalPackage[]> {
+ const resolved = new Map<string, {
+ readonly physical: ResolvedPhysicalPackage;
+ readonly roots: Set<string>;
+ }>();
+ const visit = async (item: ResolvedPhysicalPackage, sourceRoot: string): Promise<void> => {
const key = `${item.json.name}@${item.json.version}`;
const existing = resolved.get(key);
if (existing !== undefined) {
- assert(await realpath(existing.root) === await realpath(item.root), `${key} resolves to two physical packages`);
- return;
+ assert(await realpath(existing.physical.root) === await realpath(item.root), `${key} resolves to two physical packages`);
+ if (existing.roots.has(sourceRoot)) return;
+ existing.roots.add(sourceRoot);
+ } else {
+ resolved.set(key, { physical: item, roots: new Set([sourceRoot]) });
}
- resolved.set(key, item);
for (const dependency of dependencyNames(item.json)) {
try {
- await visit(await resolvePhysicalPackage(dependency, item.root));
+ await visit(await resolvePhysicalPackage(dependency, item.root), sourceRoot);
} catch (error) {
if (optionalDependency(item.json, dependency)) continue;
throw error;
}
}
};
- for (const entry of entries) await visit(entry);
+ for (const entry of entries) await visit(entry.physical, entry.root);
return [...resolved.values()].sort((left, right) =>
- `${left.json.name}@${left.json.version}`.localeCompare(`${right.json.name}@${right.json.version}`),
+ artifactKey(left.physical.json).localeCompare(artifactKey(right.physical.json)),
);
}
@@ -328,6 +352,11 @@ function parseLock(value: unknown): NodePackageLock {
return lock;
}
+/** Parse and authenticate a lock without resolving or executing its packages. */
+export async function readNodePackageLock(path: string): Promise<NodePackageLock> {
+ return parseLock(JSON.parse(await readFile(path, "utf8")));
+}
+
async function resolvedArtifacts(closure: readonly ResolvedPhysicalPackage[]): Promise<readonly LockedPackageArtifact[]> {
return await Promise.all(closure.map(async (item) => ({
name: item.json.name,
@@ -336,6 +365,30 @@ async function resolvedArtifacts(closure: readonly ResolvedPhysicalPackage[]): P
})));
}
+function artifactKey(item: { readonly name: string; readonly version: string }): string {
+ return `${item.name}@${item.version}`;
+}
+
+/**
+ * Select one package closure from a set whose physical bytes were already hashed.
+ *
+ * A lock load used to call `resolvedArtifacts()` for the complete closure and then
+ * call it again for every activated package's dependency closure. Shared packages
+ * were therefore read and hashed many times in one command. The complete physical
+ * closure has already rejected two locations for the same name/version, so its
+ * artifact digest is safe to reuse here without weakening byte verification.
+ */
+function artifactsForClosure(
+ closure: readonly ResolvedPhysicalPackage[],
+ verified: ReadonlyMap<string, LockedPackageArtifact>,
+): readonly LockedPackageArtifact[] {
+ return closure.map((item) => {
+ const artifact = verified.get(artifactKey(item.json));
+ assert(artifact !== undefined, `verified Artifact for ${artifactKey(item.json)} is missing`);
+ return artifact;
+ });
+}
+
function sameArtifacts(left: readonly LockedPackageArtifact[], right: readonly LockedPackageArtifact[]): boolean {
return JSON.stringify([...left].sort((a, b) => `${a.name}@${a.version}`.localeCompare(`${b.name}@${b.version}`)))
=== JSON.stringify([...right].sort((a, b) => `${a.name}@${a.version}`.localeCompare(`${b.name}@${b.version}`)));
@@ -345,12 +398,41 @@ function sameArtifacts(left: readonly LockedPackageArtifact[], right: readonly L
export async function createNodePackageLock(
specifiers: readonly string[],
root: string,
+ options: NodePackageLockCreateOptions = {},
): Promise<NodePackageLock> {
const unique = [...new Set(specifiers)].sort();
- assert(unique.length > 0, "at least one Node package is required");
- const selectedPhysical = await Promise.all(unique.map(async (specifier) =>
- await resolvePhysicalPackage(specifier, root)));
- const closure = await packageClosureFromPhysical(selectedPhysical);
+ const selectedPhysical = await Promise.all(unique.map(async (specifier) => ({
+ root: specifier,
+ physical: await resolvePhysicalPackage(specifier, root),
+ })));
+ const reachable = await packageClosureFromRoots(selectedPhysical);
+ const closure = reachable.map((item) => item.physical);
+
+ // Hash this candidate physical closure once, before any activation code executes. A selection
+ // mutation may admit new roots, but every package still reachable from a retained root must be
+ // byte-identical to the old authenticated lock.
+ const artifacts = await resolvedArtifacts(closure);
+ const artifactsByPackage = new Map(artifacts.map((item) => [artifactKey(item), item]));
+ if (options.retain !== undefined) {
+ const oldSelected = new Set(options.retain.from.selected);
+ const newSelected = new Set(unique);
+ const retained = new Set(options.retain.selected);
+ for (const specifier of retained) {
+ assert(oldSelected.has(specifier), `${specifier} is not a selected root in the existing package lock`);
+ assert(newSelected.has(specifier), `${specifier} is not retained by the new package selection`);
+ }
+ const locked = new Map(options.retain.from.artifacts.map((artifact) => [artifactKey(artifact), artifact.digest]));
+ for (const item of reachable) {
+ if (![...item.roots].some((specifier) => retained.has(specifier))) continue;
+ const artifact = artifactsByPackage.get(artifactKey(item.physical.json))!;
+ const expected = locked.get(artifactKey(item.physical.json));
+ assert(expected !== undefined,
+ `retained package closure introduced ${artifactKey(item.physical.json)}; use --refresh or an exact --package selection`);
+ assert(expected === artifact.digest,
+ `retained package bytes changed for ${artifactKey(item.physical.json)}; use --refresh or an exact --package selection`);
+ }
+ }
+
const candidates = new Map<string, {
readonly physical: ResolvedPhysicalPackage;
readonly contribution: NodePackageContribution;
@@ -363,7 +445,7 @@ export async function createNodePackageLock(
});
}
- const selectedKeys = new Set(selectedPhysical.map((item) => `${item.json.name}@${item.json.version}`));
+ const selectedKeys = new Set(selectedPhysical.map((item) => artifactKey(item.physical.json)));
const activated = new Map<string, {
readonly physical: ResolvedPhysicalPackage;
readonly contribution: NodePackageContribution;
@@ -431,7 +513,10 @@ export async function createNodePackageLock(
const packages: LockedNodePackage[] = [];
for (const { physical, contribution } of activated.values()) {
- const ownArtifacts = await resolvedArtifacts(await packageClosureFromPhysical([physical]));
+ const ownArtifacts = artifactsForClosure(
+ await packageClosureFromPhysical([physical]),
+ artifactsByPackage,
+ );
packages.push({
specifier: physical.json.name,
package: { name: physical.json.name, version: physical.json.version },
@@ -442,13 +527,27 @@ export async function createNodePackageLock(
return sealLock({
format: "svml.node-package-lock@1",
selected: unique,
- artifacts: await resolvedArtifacts(closure),
+ artifacts,
packages,
});
}
export async function writeNodePackageLock(path: string, lock: NodePackageLock): Promise<void> {
- await writeFile(path, `${JSON.stringify(lock, null, 2)}\n`, { encoding: "utf8", flag: "w" });
+ const target = resolve(path);
+ const temporary = join(dirname(target), `.${basename(target)}.${process.pid}.${randomUUID()}.tmp`);
+ let handle: Awaited<ReturnType<typeof open>> | undefined;
+ try {
+ handle = await open(temporary, "wx");
+ await handle.writeFile(`${JSON.stringify(lock, null, 2)}\n`, { encoding: "utf8" });
+ await handle.sync();
+ await handle.close();
+ handle = undefined;
+ await rename(temporary, target);
+ } catch (error) {
+ await handle?.close().catch(() => undefined);
+ await rm(temporary, { force: true }).catch(() => undefined);
+ throw error;
+ }
}
/** Verify every selected physical artifact before executing any package activation code. */
@@ -456,10 +555,11 @@ export async function loadNodePackageSet(
path: string,
root = dirname(resolve(path)),
): Promise<LoadedNodePackageSet> {
- const lock = parseLock(JSON.parse(await readFile(path, "utf8")));
+ const lock = await readNodePackageLock(path);
const closure = await packageClosure(lock.selected, root);
const artifacts = await resolvedArtifacts(closure);
assert(sameArtifacts(artifacts, lock.artifacts), "installed Node package bytes do not match the lock");
+ const artifactsByPackage = new Map(artifacts.map((item) => [artifactKey(item), item]));
const byName = new Map(closure.map((item) => [`${item.json.name}@${item.json.version}`, item]));
const values: NodePackageContribution[] = [];
for (const expected of lock.packages) {
@@ -469,7 +569,10 @@ export async function loadNodePackageSet(
const contribution = await importContribution(physical);
assert(digestOf(contributionMetadata(contribution)) === expected.facetsDigest,
`${expected.specifier} facets do not match the lock`);
- const ownArtifacts = await resolvedArtifacts(await packageClosureFromPhysical([physical]));
+ const ownArtifacts = artifactsForClosure(
+ await packageClosureFromPhysical([physical]),
+ artifactsByPackage,
+ );
assert(digestOf({ format: "svml.package-closure@1", artifacts: ownArtifacts }) === expected.closureDigest,
`${expected.specifier} dependency closure does not match the lock`);
values.push(contribution);
diff --git a/packages/package-loader-node/src/types.ts b/packages/package-loader-node/src/types.ts
index 7fd8d378..8d6a13fc 100644
--- a/packages/package-loader-node/src/types.ts
+++ b/packages/package-loader-node/src/types.ts
@@ -50,6 +50,14 @@ export type NodePackageLock = {
readonly packages: readonly LockedNodePackage[];
};
+/** Guard used by a direct-selection edit: retained roots may not acquire different bytes. */
+export type NodePackageLockCreateOptions = {
+ readonly retain?: {
+ readonly from: NodePackageLock;
+ readonly selected: readonly string[];
+ };
+};
+
export type LoadedNodePackageSet = {
readonly lock: NodePackageLock;
readonly contributions: readonly NodePackageContribution[];
diff --git a/packages/package-loader-node/test/loader.test.ts b/packages/package-loader-node/test/loader.test.ts
index 74555bd2..a6304f20 100644
--- a/packages/package-loader-node/test/loader.test.ts
+++ b/packages/package-loader-node/test/loader.test.ts
@@ -79,6 +79,7 @@ async function fixture(): Promise<{
version: "1.0.0",
type: "module",
exports: "./activation.mjs",
+ dependencies: { "example-types": "1.0.0" },
svml: { activation: "./activation.mjs" },
}, null, 2), "utf8");
await writeFile(join(extraRoot, "activation.mjs"), `export default {
@@ -240,6 +241,17 @@ test("an installed locked package carries inert Host facets and activatable comp
}]);
});
+test("an empty local trust selection is a valid authenticated package lock", async () => {
+ const root = await mkdtemp(join(tmpdir(), "svml-empty-package-lock-"));
+ const path = join(root, "svml.packages.lock");
+ const lock = await createNodePackageLock([], root);
+ assert.deepEqual(lock.selected, []);
+ assert.deepEqual(lock.artifacts, []);
+ assert.deepEqual(lock.packages, []);
+ await writeNodePackageLock(path, lock);
+ assert.deepEqual(await loadNodePackageContributions(path, root), []);
+});
+
test("dependency bytes are rejected before a locked contribution entry is reused", async () => {
const item = await fixture();
const lock = await createNodePackageLock(["example-card"], item.root);
@@ -305,6 +317,17 @@ test("an unrelated selected package does not contaminate another package's imple
assert.notEqual(alone.digest, together.digest);
});
+test("selection mutation keeps a shared dependency only through its retained root", async () => {
+ const item = await fixture();
+ const old = await createNodePackageLock(["example-card", "example-extra"], item.root);
+ const next = await createNodePackageLock(["example-extra"], item.root, {
+ retain: { from: old, selected: ["example-extra"] },
+ });
+ assert.deepEqual(next.selected, ["example-extra"]);
+ assert.deepEqual(next.artifacts.map((artifact) => artifact.name), ["example-extra", "example-types"]);
+ assert.deepEqual(next.packages.map((value) => value.specifier), ["example-extra"]);
+});
+
test("a compute facet cannot claim another implementation than its static Manifest", async () => {
const item = await fixture();
const source = await readFile(item.activation, "utf8");
@@ -352,41 +375,42 @@ async function runtimeAdapterFixture(marker: string): Promise<{
kind: "endpoint",
},
implementation: {
- validate() {},
- create(context) {
+ activate(context) {
const facet = { module, name: "endpoint" };
- return {
- name: context.instance,
- manifest: {
- format: "svml.runtime-module@1",
- name: module.name,
- version: module.version,
- facets: [{
- name: facet.name,
- role: "capability-endpoint",
- implementation: { locator: "example-runtime-adapter/endpoint", digest: declared },
- permissions: [],
- fulfills: [{ capability, returns }],
- lifecycle: "immediate",
- defaultConcurrency: 1,
- credentialSlots: [],
- }],
- },
- instance: { id: context.instance, facet, configurationDigest: configuration },
- bindings: [{ capability, returns, endpoint: context.instance }],
- install(registry) {
- registry.registerImmediateEndpoint(
- context.instance,
- capability,
- returns,
- () => ({
- value: { kind: "inline", value: "external" },
- conformance: "exact",
- delivery: { kind: "inline" },
- metadata: null,
- }),
- { runtimeImplementation: { facet, digest: declared, configurationDigest: configuration } },
- );
+ return { endpoint: {
+ name: context.instance,
+ manifest: {
+ format: "svml.runtime-module@1",
+ name: module.name,
+ version: module.version,
+ facets: [{
+ name: facet.name,
+ role: "capability-endpoint",
+ implementation: { locator: "example-runtime-adapter/endpoint", digest: declared },
+ permissions: [],
+ fulfills: [{ capability, returns }],
+ lifecycle: "immediate",
+ defaultConcurrency: 1,
+ credentialSlots: [],
+ }],
+ },
+ instance: { id: context.instance, facet, configurationDigest: configuration },
+ bindings: [{ capability, returns, endpoint: context.instance }],
+ credentials: [],
+ install(registry) {
+ registry.registerImmediateEndpoint(
+ context.instance,
+ capability,
+ returns,
+ () => ({
+ value: { kind: "inline", value: "external" },
+ conformance: "exact",
+ delivery: { kind: "inline" },
+ metadata: null,
+ }),
+ { runtimeImplementation: { facet, digest: declared, configurationDigest: configuration } },
+ );
+ },
},
};
},
diff --git a/packages/provider-google-vertex/src/activation.ts b/packages/provider-google-vertex/src/activation.ts
index 24cb589f..0db08a5e 100644
--- a/packages/provider-google-vertex/src/activation.ts
+++ b/packages/provider-google-vertex/src/activation.ts
@@ -12,26 +12,7 @@ import { createGoogleVertexCaptionProvider } from "./provider.js";
const googleVertexRuntimeAdapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-google-vertex",
- validate(context) {
- const config = runtimeConfigObject(context.config, "Google Vertex");
- runtimeConfigExact(config, [
- "project", "projectEnv", "location", "credentials", "defaultConcurrency",
- "requestTimeoutMs", "maxResponseBytes",
- ], "Google Vertex");
- const project = runtimeConfigString(config.project, "Google Vertex project");
- const projectEnv = runtimeConfigString(config.projectEnv, "Google Vertex projectEnv");
- if ((project === undefined) === (projectEnv === undefined)) {
- throw new Error("Google Vertex requires exactly one of project or projectEnv");
- }
- runtimeConfigString(config.location, "Google Vertex location");
- if (runtimeConfigCredentialRef(config.credentials, "Google Vertex credentials") === undefined) {
- throw new Error("Google Vertex credentials CredentialRef is required");
- }
- runtimeConfigPositiveInteger(config.defaultConcurrency, "Google Vertex defaultConcurrency");
- runtimeConfigPositiveInteger(config.requestTimeoutMs, "Google Vertex requestTimeoutMs");
- runtimeConfigPositiveInteger(config.maxResponseBytes, "Google Vertex maxResponseBytes");
- },
- create(context) {
+ activate(context) {
const config = runtimeConfigObject(context.config, "Google Vertex");
runtimeConfigExact(config, [
"project", "projectEnv", "location", "credentials", "defaultConcurrency",
@@ -42,29 +23,27 @@ const googleVertexRuntimeAdapter = createRuntimeEndpointAdapterFacet({
if ((projectValue === undefined) === (projectEnv === undefined)) {
throw new Error("Google Vertex requires exactly one of project or projectEnv");
}
- const project = projectValue ?? process.env[projectEnv!]?.trim();
- if (project === undefined || project.length === 0) throw new Error(`Google Vertex project environment ${projectEnv} is empty`);
const credentials = runtimeConfigCredentialRef(config.credentials, "Google Vertex credentials");
if (credentials === undefined) throw new Error("Google Vertex credentials CredentialRef is required");
- return createGoogleVertexCaptionProvider({
- project,
- instance: context.instance,
- ...(context.lane === undefined ? {} : { lane: context.lane }),
- ...(runtimeConfigString(config.location, "Google Vertex location") === undefined
- ? {} : { location: config.location as string }),
- credentialsJson: credentials,
- ...(runtimeConfigPositiveInteger(config.defaultConcurrency, "Google Vertex defaultConcurrency") === undefined
- ? {} : { defaultConcurrency: config.defaultConcurrency as number }),
- ...(runtimeConfigPositiveInteger(config.requestTimeoutMs, "Google Vertex requestTimeoutMs") === undefined
- ? {} : { requestTimeoutMs: config.requestTimeoutMs as number }),
- ...(runtimeConfigPositiveInteger(config.maxResponseBytes, "Google Vertex maxResponseBytes") === undefined
- ? {} : { maxResponseBytes: config.maxResponseBytes as number }),
- });
- },
- doctor(context) {
- const config = runtimeConfigObject(context.config, "Google Vertex");
- const projectEnv = runtimeConfigString(config.projectEnv, "Google Vertex projectEnv");
- return projectEnv === undefined ? [] : diagnoseRuntimeEnvironmentCredential(projectEnv, "Google Vertex project");
+ const location = runtimeConfigString(config.location, "Google Vertex location");
+ const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "Google Vertex defaultConcurrency");
+ const requestTimeoutMs = runtimeConfigPositiveInteger(config.requestTimeoutMs, "Google Vertex requestTimeoutMs");
+ const maxResponseBytes = runtimeConfigPositiveInteger(config.maxResponseBytes, "Google Vertex maxResponseBytes");
+ return {
+ endpoint: createGoogleVertexCaptionProvider({
+ ...(projectValue === undefined ? { projectEnv: projectEnv! } : { project: projectValue }),
+ instance: context.instance,
+ ...(context.lane === undefined ? {} : { lane: context.lane }),
+ ...(location === undefined ? {} : { location }),
+ credentialsJson: credentials,
+ ...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
+ ...(requestTimeoutMs === undefined ? {} : { requestTimeoutMs }),
+ ...(maxResponseBytes === undefined ? {} : { maxResponseBytes }),
+ }),
+ ...(projectEnv === undefined ? {} : {
+ diagnose: () => diagnoseRuntimeEnvironmentCredential(projectEnv, "Google Vertex project"),
+ }),
+ };
},
});
diff --git a/packages/provider-google-vertex/src/provider.ts b/packages/provider-google-vertex/src/provider.ts
index e2913de0..605e71ba 100644
--- a/packages/provider-google-vertex/src/provider.ts
+++ b/packages/provider-google-vertex/src/provider.ts
@@ -39,7 +39,10 @@ export type GenerateCaptionContent = (
) => Promise<{ readonly text: string; readonly totalTokenCount?: number; readonly modelVersion?: string }>;
export type CreateGoogleVertexCaptionProviderOptions = {
- readonly project: string;
+ /** Literal deployment project, mutually exclusive with projectEnv. */
+ readonly project?: string;
+ /** Environment variable resolved only when this Endpoint actually handles a Need. */
+ readonly projectEnv?: string;
readonly location?: string;
readonly instance?: string;
readonly lane?: string;
@@ -187,8 +190,19 @@ function fulfillment(value: CanonicalValue, metadata: CanonicalValue): EndpointF
}
export function createGoogleVertexCaptionProvider(options: CreateGoogleVertexCaptionProviderOptions) {
- assert(options.project.trim().length > 0, "Google Vertex project is empty");
- const project = options.project.trim();
+ const projectValue = options.project?.trim();
+ const projectEnv = options.projectEnv?.trim();
+ assert((projectValue === undefined) !== (projectEnv === undefined),
+ "Google Vertex requires exactly one of project or projectEnv");
+ if (projectValue !== undefined) assert(projectValue.length > 0, "Google Vertex project is empty");
+ if (projectEnv !== undefined) assert(projectEnv.length > 0, "Google Vertex projectEnv is empty");
+ const resolveProject = (): string => {
+ if (projectValue !== undefined) return projectValue;
+ const value = process.env[projectEnv!]?.trim();
+ assert(value !== undefined && value.length > 0,
+ `Google Vertex project environment ${projectEnv} is empty`);
+ return value;
+ };
const location = (options.location ?? "global").trim();
assert(location.length > 0, "Google Vertex location is empty");
const requestTimeoutMs = positiveInteger(options.requestTimeoutMs ?? 120_000, "requestTimeoutMs");
@@ -213,7 +227,15 @@ export function createGoogleVertexCaptionProvider(options: CreateGoogleVertexCap
digest: googleVertexProviderImplementationDigest,
},
permissions: ["network:aiplatform.googleapis.com"],
- configuration: canonicalize({ project, location, requestTimeoutMs, maxResponseBytes, transportDigest }),
+ configuration: canonicalize({
+ project: projectValue === undefined
+ ? { source: "environment", name: projectEnv! }
+ : { source: "literal", value: projectValue },
+ location,
+ requestTimeoutMs,
+ maxResponseBytes,
+ transportDigest,
+ }),
credentials: {
googleCredentials: options.credentialsJson ?? credentialRef("env", "GOOGLE_APPLICATION_CREDENTIALS_JSON"),
},
@@ -235,6 +257,7 @@ export function createGoogleVertexCaptionProvider(options: CreateGoogleVertexCap
},
handler: async (context) => {
const request = requestValue(context.need.constraints);
+ const project = resolveProject();
const response = await withTimeout(requestTimeoutMs, (abortSignal) => generateContent({
model: request.model,
systemInstruction: request.systemInstruction,
diff --git a/packages/provider-google-vertex/test/provider.test.ts b/packages/provider-google-vertex/test/provider.test.ts
index 29d3d53b..8699c2a8 100644
--- a/packages/provider-google-vertex/test/provider.test.ts
+++ b/packages/provider-google-vertex/test/provider.test.ts
@@ -126,3 +126,34 @@ test("Vertex configuration exposes credential/queue policy without changing the
}]);
assert.equal(facet.implementation.digest, googleVertexProviderImplementationDigest);
});
+
+test("Vertex defers projectEnv resolution until the Endpoint handles a Need", async () => {
+ const variable = "NARRATAGE_TEST_MISSING_VERTEX_PROJECT";
+ const previous = process.env[variable];
+ delete process.env[variable];
+ try {
+ const provider = createGoogleVertexCaptionProvider({
+ projectEnv: variable,
+ generateContentImplementationDigest: digestOf("caption-gemini:unreached-transport"),
+ generateContent: async () => {
+ throw new Error("transport must not be reached");
+ },
+ });
+ const registry = new EndpointRegistry();
+ await provider.install(registry);
+ const requestNeed = need(request());
+ const resolution = registry.resolve(requestNeed);
+ assert.equal(resolution.status, "resolved");
+ assert.equal(resolution.registration.kind, "immediate");
+ const handler: ImmediateEndpointHandler = resolution.registration.handler;
+ await assert.rejects(async () => await handler({
+ command: { kind: "fulfill-need", id: "command:caption-gemini-missing-project", need: requestNeed },
+ need: requestNeed,
+ artifacts: new MemoryArtifactStore(),
+ credentials: { googleCredentials: { secret: JSON.stringify({ type: "service_account" }) } },
+ }), new RegExp(`Google Vertex project environment ${variable} is empty`));
+ } finally {
+ if (previous === undefined) delete process.env[variable];
+ else process.env[variable] = previous;
+ }
+});
diff --git a/packages/provider-hyperframes-aws-lambda/src/activation.ts b/packages/provider-hyperframes-aws-lambda/src/activation.ts
index c7942337..f92f540a 100644
--- a/packages/provider-hyperframes-aws-lambda/src/activation.ts
+++ b/packages/provider-hyperframes-aws-lambda/src/activation.ts
@@ -100,11 +100,8 @@ function providerOptions(context: RuntimeAdapterFactoryContext): RuntimeProvider
const awsLambdaHyperframesRuntimeAdapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-hyperframes-aws-lambda",
- validate(context) {
- providerOptions(context);
- },
- create(context) {
- return createAwsLambdaHyperframesProvider(providerOptions(context));
+ activate(context) {
+ return { endpoint: createAwsLambdaHyperframesProvider(providerOptions(context)) };
},
});
diff --git a/packages/provider-hyperframes-aws-lambda/src/provider.ts b/packages/provider-hyperframes-aws-lambda/src/provider.ts
index c318068e..1cdc7d1d 100644
--- a/packages/provider-hyperframes-aws-lambda/src/provider.ts
+++ b/packages/provider-hyperframes-aws-lambda/src/provider.ts
@@ -227,7 +227,15 @@ function alreadyStopped(error: unknown): boolean {
.includes(String((error as { readonly name?: unknown }).name));
}
-async function artifactBytes(context: EndpointStartContext, artifact: BlobRef): Promise<Uint8Array> {
+async function artifactBytes(
+ context: EndpointStartContext,
+ artifact: BlobRef,
+): Promise<Uint8Array | AsyncIterable<Uint8Array>> {
+ if (isStreamingArtifactStore(context.artifacts)) {
+ const chunks = await context.artifacts.open(artifact.digest);
+ assert(chunks !== undefined, `HyperFrames Artifact ${artifact.digest} is unavailable`);
+ return chunks;
+ }
const bytes = await context.artifacts.get(artifact.digest);
assert(bytes !== undefined, `HyperFrames Artifact ${artifact.digest} is unavailable`);
assert(bytes.byteLength === artifact.size, `HyperFrames Artifact ${artifact.digest} size differs`);
@@ -536,13 +544,17 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
});
} catch (error) {
if (duplicateExecution(error)) {
- return wakeAfter(canonicalize({ ...checkpoint, submission: "confirmed" }), pollIntervalMs, now());
+ return wakeAfter(canonicalize({ ...checkpoint, submission: "confirmed" }), pollIntervalMs, now(), {
+ phase: "submitted",
+ });
}
const rejected = rejectedSubmission(error);
if (rejected !== undefined) return implementationFailure(rejected.code, error, rejected.retryable);
// StartExecution may have reached AWS even when its response did not reach us. The
// deterministic execution name makes polling this derived ARN safer than submitting a new job.
- return wakeAfter(canonicalize(checkpoint), pollIntervalMs, now());
+ return wakeAfter(canonicalize(checkpoint), pollIntervalMs, now(), {
+ phase: "confirming-submission",
+ });
}
try {
sameRender(handle, {
@@ -556,7 +568,9 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
} catch (error) {
return implementationFailure("HYPERFRAMES_SUBMISSION_IDENTITY_MISMATCH", error, false);
}
- return wakeAfter(canonicalize({ ...checkpoint, submission: "confirmed" }), pollIntervalMs, now());
+ return wakeAfter(canonicalize({ ...checkpoint, submission: "confirmed" }), pollIntervalMs, now(), {
+ phase: "submitted",
+ });
};
const safeSubmit = async (
@@ -609,7 +623,9 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
if (failures >= maxPollFailures) {
return implementationFailure("HYPERFRAMES_PROGRESS_UNAVAILABLE", error, true);
}
- return wakeAfter(canonicalize({ ...checkpoint, pollFailures: failures }), pollIntervalMs, now());
+ return wakeAfter(canonicalize({ ...checkpoint, pollFailures: failures }), pollIntervalMs, now(), {
+ phase: "progress-retry",
+ });
}
try {
verifyProgress(progress, document);
@@ -618,7 +634,14 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
}
const next = { ...checkpoint, submission: "confirmed" as const,
polls: checkpoint.polls + 1, pollFailures: 0 };
- if (progress.status === "RUNNING") return wakeAfter(canonicalize(next), pollIntervalMs, now());
+ if (progress.status === "RUNNING") {
+ return wakeAfter(canonicalize(next), pollIntervalMs, now(), {
+ phase: "rendering",
+ completed: progress.framesRendered,
+ ...(progress.totalFrames === null ? {} : { total: progress.totalFrames }),
+ unit: "frames",
+ });
+ }
if (progress.status !== "SUCCEEDED") {
const details = progress.errors.map((item) => `${item.state}: ${item.error}: ${item.cause}`).join("; ");
return implementationFailure(`HYPERFRAMES_${progress.status}`,
diff --git a/packages/provider-hyperframes-aws-lambda/test/provider.test.ts b/packages/provider-hyperframes-aws-lambda/test/provider.test.ts
index 9b7b7f34..6453f8ff 100644
--- a/packages/provider-hyperframes-aws-lambda/test/provider.test.ts
+++ b/packages/provider-hyperframes-aws-lambda/test/provider.test.ts
@@ -22,6 +22,7 @@ import type {
} from "@narratage/provider-hyperframes-aws-lambda";
import { canonicalize, digestOf } from "@narratage/protocol";
import type { CanonicalValue, Need } from "@narratage/protocol";
+import type { RuntimeEndpointAdapterImplementation } from "@narratage/runtime-adapter";
import {
hyperframesVisualRequest,
renderHyperframesCapabilities,
@@ -272,7 +273,8 @@ test("the Lambda Endpoint declines frame domains and requirements it cannot pres
test("the Runtime adapter refuses a hardware-GPU deployment wish instead of ignoring it", () => {
const facet = awsLambdaActivation.hostFacets[0]!;
- assert.throws(() => facet.implementation.create({
+ const implementation = facet.implementation as RuntimeEndpointAdapterImplementation;
+ assert.throws(() => implementation.activate({
root: "/tmp",
instance: "hyperframes.lambda.test",
config: canonicalize({
@@ -381,6 +383,34 @@ test("an ambiguous StartExecution is recovered by the same name without redeploy
}
});
+test("running render progress is projected without exposing the recovery checkpoint", async () => {
+ const request = requestNeed();
+ const { client } = fakeClient({
+ async progress() {
+ return successfulProgress("s3://unused/while-running", {
+ status: "RUNNING",
+ overallProgress: 0.4,
+ framesRendered: 24,
+ totalFrames: 60,
+ outputFile: null,
+ endedAt: null,
+ });
+ },
+ });
+ const { endpoint } = await endpointFor(request, client);
+ const common = context(request, new MemoryArtifactStore());
+ const started = await endpoint.start(common);
+ const running = await endpoint.resume({
+ ...common,
+ checkpoint: started.status === "pending" ? started.checkpoint : undefined,
+ });
+ assert.equal(running.status, "pending");
+ if (running.status === "pending") {
+ assert.deepEqual(running.progress,
+ { phase: "rendering", completed: 24, total: 60, unit: "frames" });
+ }
+});
+
test("a successful execution with another frame domain is refused before its bytes are opened", async () => {
const request = requestNeed();
let outputUri = "";
diff --git a/packages/provider-hyperframes-local/package.json b/packages/provider-hyperframes-local/package.json
index 26e70e8b..f504c869 100644
--- a/packages/provider-hyperframes-local/package.json
+++ b/packages/provider-hyperframes-local/package.json
@@ -16,6 +16,7 @@
"@narratage/media-execution": "workspace:*",
"@narratage/protocol": "workspace:*",
"@narratage/render-hyperframes": "workspace:*",
+ "@narratage/runtime": "workspace:*",
"@narratage/runtime-adapter": "workspace:*",
"@narratage/runtime-adapter-node": "workspace:*",
"hyperframes": "0.7.101"
diff --git a/packages/provider-hyperframes-local/src/activation.ts b/packages/provider-hyperframes-local/src/activation.ts
index 3b2b29cf..ae3b676c 100644
--- a/packages/provider-hyperframes-local/src/activation.ts
+++ b/packages/provider-hyperframes-local/src/activation.ts
@@ -16,7 +16,7 @@ import { localHyperframesBrowserService } from "./service.js";
const localHyperframesRuntimeAdapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-hyperframes-local",
- validate(context) {
+ activate(context) {
const config = runtimeConfigObject(context.config, "local HyperFrames");
runtimeConfigExact(config, [
"nodePath", "hyperframesCliPath", "ffprobePath", "workers", "quality", "browserGpu",
@@ -38,70 +38,48 @@ const localHyperframesRuntimeAdapter = createRuntimeEndpointAdapterFacet({
if (browserGpu !== undefined && browserGpu !== "auto" && browserGpu !== "software" && browserGpu !== "hardware") {
throw new Error("HyperFrames browserGpu is invalid");
}
- runtimeConfigPositiveInteger(config.defaultConcurrency, "HyperFrames defaultConcurrency");
- runtimeConfigPositiveInteger(config.processTimeoutMs, "HyperFrames processTimeoutMs");
- runtimeConfigPositiveInteger(config.maxProcessOutputBytes, "HyperFrames maxProcessOutputBytes");
- runtimeConfigPositiveInteger(config.maxRenderedBytes, "HyperFrames maxRenderedBytes");
- },
- create(context) {
- const config = runtimeConfigObject(context.config, "local HyperFrames");
- runtimeConfigExact(config, [
- "nodePath", "hyperframesCliPath", "ffprobePath", "workers", "quality", "browserGpu",
- "defaultConcurrency", "processTimeoutMs", "maxProcessOutputBytes", "maxRenderedBytes",
- ], "local HyperFrames");
- const workers = config.workers;
- if (workers !== undefined && workers !== "auto") runtimeConfigPositiveInteger(workers, "HyperFrames workers");
- const quality = runtimeConfigString(config.quality, "HyperFrames quality");
- if (quality !== undefined && quality !== "draft" && quality !== "standard" && quality !== "high") {
- throw new Error("HyperFrames quality is invalid");
- }
- const browserGpu = runtimeConfigString(config.browserGpu, "HyperFrames browserGpu");
- if (browserGpu !== undefined && browserGpu !== "auto" && browserGpu !== "software" && browserGpu !== "hardware") {
- throw new Error("HyperFrames browserGpu is invalid");
- }
const configuredNode = runtimeConfigString(config.nodePath, "HyperFrames nodePath");
const configuredCli = runtimeConfigString(config.hyperframesCliPath, "HyperFrames hyperframesCliPath");
const configuredFfprobe = runtimeConfigString(config.ffprobePath, "HyperFrames ffprobePath");
const nodePath = configuredNode === undefined ? undefined : resolveRuntimeExecutable(context.root, configuredNode);
const hyperframesCliPath = configuredCli === undefined ? undefined : resolveRuntimeExecutable(context.root, configuredCli);
const ffprobePath = configuredFfprobe === undefined ? undefined : resolveRuntimeExecutable(context.root, configuredFfprobe);
- return createLocalHyperframesProvider({
- instance: context.instance,
- ...(context.lane === undefined ? {} : { lane: context.lane }),
- ...(nodePath === undefined ? {} : { nodePath }),
- ...(hyperframesCliPath === undefined ? {} : { hyperframesCliPath }),
- ...(ffprobePath === undefined ? {} : { ffprobePath }),
- ...(workers === undefined ? {} : { workers: workers as HyperframesWorkers }),
- ...(quality === undefined ? {} : { quality: quality as HyperframesQuality }),
- ...(browserGpu === undefined ? {} : { browserGpu: browserGpu as HyperframesBrowserGpu }),
- ...(runtimeConfigPositiveInteger(config.defaultConcurrency, "HyperFrames defaultConcurrency") === undefined
- ? {} : { defaultConcurrency: config.defaultConcurrency as number }),
- ...(runtimeConfigPositiveInteger(config.processTimeoutMs, "HyperFrames processTimeoutMs") === undefined
- ? {} : { processTimeoutMs: config.processTimeoutMs as number }),
- ...(runtimeConfigPositiveInteger(config.maxProcessOutputBytes, "HyperFrames maxProcessOutputBytes") === undefined
- ? {} : { maxProcessOutputBytes: config.maxProcessOutputBytes as number }),
- ...(runtimeConfigPositiveInteger(config.maxRenderedBytes, "HyperFrames maxRenderedBytes") === undefined
- ? {} : { maxRenderedBytes: config.maxRenderedBytes as number }),
- });
- },
- async doctor(context) {
- const config = runtimeConfigObject(context.config, "local HyperFrames");
- return [
- ...await diagnoseRuntimeExecutable({
- root: context.root,
- configured: runtimeConfigString(config.nodePath, "HyperFrames nodePath"),
- fallback: process.execPath,
- subject: "Node.js",
- }),
- ...await diagnoseRuntimeExecutable({
- root: context.root,
- configured: runtimeConfigString(config.ffprobePath, "HyperFrames ffprobePath"),
- fallback: "ffprobe",
- subject: "FFprobe",
+ const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "HyperFrames defaultConcurrency");
+ const processTimeoutMs = runtimeConfigPositiveInteger(config.processTimeoutMs, "HyperFrames processTimeoutMs");
+ const maxProcessOutputBytes = runtimeConfigPositiveInteger(config.maxProcessOutputBytes, "HyperFrames maxProcessOutputBytes");
+ const maxRenderedBytes = runtimeConfigPositiveInteger(config.maxRenderedBytes, "HyperFrames maxRenderedBytes");
+ return {
+ endpoint: createLocalHyperframesProvider({
+ instance: context.instance,
+ ...(context.lane === undefined ? {} : { lane: context.lane }),
+ ...(nodePath === undefined ? {} : { nodePath }),
+ ...(hyperframesCliPath === undefined ? {} : { hyperframesCliPath }),
+ ...(ffprobePath === undefined ? {} : { ffprobePath }),
+ ...(workers === undefined ? {} : { workers: workers as HyperframesWorkers }),
+ ...(quality === undefined ? {} : { quality: quality as HyperframesQuality }),
+ ...(browserGpu === undefined ? {} : { browserGpu: browserGpu as HyperframesBrowserGpu }),
+ ...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
+ ...(processTimeoutMs === undefined ? {} : { processTimeoutMs }),
+ ...(maxProcessOutputBytes === undefined ? {} : { maxProcessOutputBytes }),
+ ...(maxRenderedBytes === undefined ? {} : { maxRenderedBytes }),
}),
- ];
+ externalService: localHyperframesBrowserService(context),
+ diagnose: async () => [
+ ...await diagnoseRuntimeExecutable({
+ root: context.root,
+ configured: configuredNode,
+ fallback: process.execPath,
+ subject: "Node.js",
+ }),
+ ...await diagnoseRuntimeExecutable({
+ root: context.root,
+ configured: configuredFfprobe,
+ fallback: "ffprobe",
+ subject: "FFprobe",
+ }),
+ ],
+ };
},
- service: localHyperframesBrowserService,
});
export const svmlPackage = {
diff --git a/packages/provider-hyperframes-local/src/provider.ts b/packages/provider-hyperframes-local/src/provider.ts
index 79c24025..c70ecd48 100644
--- a/packages/provider-hyperframes-local/src/provider.ts
+++ b/packages/provider-hyperframes-local/src/provider.ts
@@ -14,6 +14,7 @@ import type { HyperframesDocument } from "@narratage/hyperframes";
import { stageHyperframesProject } from "@narratage/hyperframes/project";
import { renderHyperframesCapabilities } from "@narratage/render-hyperframes";
import { canonicalize, digestOf } from "@narratage/protocol";
+import { isStreamingArtifactStore } from "@narratage/runtime";
import type { BlobRef, CanonicalValue, Digest } from "@narratage/protocol";
import { defineEndpointPackage } from "@narratage/endpoint-kit";
@@ -143,7 +144,15 @@ async function runProcess(args: {
});
}
-async function artifactBytes(context: EndpointInvocationContext, artifact: BlobRef): Promise<Uint8Array> {
+async function artifactBytes(
+ context: EndpointInvocationContext,
+ artifact: BlobRef,
+): Promise<Uint8Array | AsyncIterable<Uint8Array>> {
+ if (isStreamingArtifactStore(context.artifacts)) {
+ const chunks = await context.artifacts.open(artifact.digest);
+ assert(chunks !== undefined, `HyperFrames Artifact ${artifact.digest} is unavailable`);
+ return chunks;
+ }
const bytes = await context.artifacts.get(artifact.digest);
assert(bytes !== undefined, `HyperFrames Artifact ${artifact.digest} is unavailable`);
assert(bytes.byteLength === artifact.size, `HyperFrames Artifact ${artifact.digest} size differs`);
@@ -368,7 +377,9 @@ export function createLocalHyperframesProvider(config: CreateLocalHyperframesPro
timeoutMs: processTimeoutMs,
maxOutputBytes: maxProcessOutputBytes,
});
- const artifact = await context.artifacts.put(await readFile(output), "video/mp4");
+ const artifact = isStreamingArtifactStore(context.artifacts)
+ ? await context.artifacts.putStream(createReadStream(output), "video/mp4")
+ : await context.artifacts.put(await readFile(output), "video/mp4");
const value: RenderedVisual = sealRenderedVisual({
contract: "svml.rendered-visual@1",
frameRate: document.frameRate,
diff --git a/packages/provider-image-opencv-local/src/activation.ts b/packages/provider-image-opencv-local/src/activation.ts
index 2e580e2c..72e2c739 100644
--- a/packages/provider-image-opencv-local/src/activation.ts
+++ b/packages/provider-image-opencv-local/src/activation.ts
@@ -15,47 +15,36 @@ import { localOpenCvService } from "./service.js";
const localOpenCvRuntimeAdapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-image-opencv-local",
- validate(context) {
+ activate(context) {
const config = runtimeConfigObject(context.config, "local OpenCV image");
runtimeConfigExact(config, [
"pythonExecutable", "defaultConcurrency", "processTimeoutMs", "maxInputBytes", "maxOutputBytes",
], "local OpenCV image");
runtimeConfigString(config.pythonExecutable, "OpenCV pythonExecutable");
- runtimeConfigPositiveInteger(config.defaultConcurrency, "OpenCV defaultConcurrency");
- runtimeConfigPositiveInteger(config.processTimeoutMs, "OpenCV processTimeoutMs");
- runtimeConfigPositiveInteger(config.maxInputBytes, "OpenCV maxInputBytes");
- runtimeConfigPositiveInteger(config.maxOutputBytes, "OpenCV maxOutputBytes");
- },
- create(context) {
- const config = runtimeConfigObject(context.config, "local OpenCV image");
- runtimeConfigExact(config, [
- "pythonExecutable", "defaultConcurrency", "processTimeoutMs", "maxInputBytes", "maxOutputBytes",
- ], "local OpenCV image");
- const pythonExecutable = resolveLocalOpenCvDeployment(context).pythonExecutable;
- return createLocalOpenCvImageProvider({
- instance: context.instance,
- ...(context.lane === undefined ? {} : { lane: context.lane }),
- pythonExecutable,
- ...(runtimeConfigPositiveInteger(config.defaultConcurrency, "OpenCV defaultConcurrency") === undefined
- ? {} : { defaultConcurrency: config.defaultConcurrency as number }),
- ...(runtimeConfigPositiveInteger(config.processTimeoutMs, "OpenCV processTimeoutMs") === undefined
- ? {} : { processTimeoutMs: config.processTimeoutMs as number }),
- ...(runtimeConfigPositiveInteger(config.maxInputBytes, "OpenCV maxInputBytes") === undefined
- ? {} : { maxInputBytes: config.maxInputBytes as number }),
- ...(runtimeConfigPositiveInteger(config.maxOutputBytes, "OpenCV maxOutputBytes") === undefined
- ? {} : { maxOutputBytes: config.maxOutputBytes as number }),
- });
- },
- async doctor(context) {
const deployment = resolveLocalOpenCvDeployment(context);
- return await diagnoseRuntimeExecutable({
- root: context.root,
- configured: deployment.pythonExecutable,
- fallback: deployment.pythonExecutable,
- subject: "OpenCV Python",
- });
+ const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "OpenCV defaultConcurrency");
+ const processTimeoutMs = runtimeConfigPositiveInteger(config.processTimeoutMs, "OpenCV processTimeoutMs");
+ const maxInputBytes = runtimeConfigPositiveInteger(config.maxInputBytes, "OpenCV maxInputBytes");
+ const maxOutputBytes = runtimeConfigPositiveInteger(config.maxOutputBytes, "OpenCV maxOutputBytes");
+ return {
+ endpoint: createLocalOpenCvImageProvider({
+ instance: context.instance,
+ ...(context.lane === undefined ? {} : { lane: context.lane }),
+ pythonExecutable: deployment.pythonExecutable,
+ ...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
+ ...(processTimeoutMs === undefined ? {} : { processTimeoutMs }),
+ ...(maxInputBytes === undefined ? {} : { maxInputBytes }),
+ ...(maxOutputBytes === undefined ? {} : { maxOutputBytes }),
+ }),
+ externalService: localOpenCvService(context),
+ diagnose: () => diagnoseRuntimeExecutable({
+ root: context.root,
+ configured: deployment.pythonExecutable,
+ fallback: deployment.pythonExecutable,
+ subject: "OpenCV Python",
+ }),
+ };
},
- service: localOpenCvService,
});
export const svmlPackage = {
diff --git a/packages/provider-kie/src/activation.ts b/packages/provider-kie/src/activation.ts
index ece34993..4e1d5ad2 100644
--- a/packages/provider-kie/src/activation.ts
+++ b/packages/provider-kie/src/activation.ts
@@ -11,59 +11,44 @@ import { createKieProvider } from "./provider.js";
const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-kie",
- validate(context) {
+ activate(context) {
const config = runtimeConfigObject(context.config, "KIE");
runtimeConfigExact(config, [
"apiBaseUrl", "uploadBaseUrl", "apiKey", "defaultConcurrency", "pollIntervalMs",
"submissionIntervalMs", "requestTimeoutMs", "maxOperationMs", "maxArtifactBytes",
], "KIE");
- for (const [key, subject] of [["apiBaseUrl", "KIE apiBaseUrl"], ["uploadBaseUrl", "KIE uploadBaseUrl"]] as const) {
- const value = runtimeConfigString(config[key], subject);
+ const apiBaseUrl = runtimeConfigString(config.apiBaseUrl, "KIE apiBaseUrl");
+ const uploadBaseUrl = runtimeConfigString(config.uploadBaseUrl, "KIE uploadBaseUrl");
+ for (const [value, subject] of [[apiBaseUrl, "KIE apiBaseUrl"], [uploadBaseUrl, "KIE uploadBaseUrl"]] as const) {
if (value === undefined) continue;
const url = new URL(value);
if (url.protocol !== "https:" && url.hostname !== "localhost") {
throw new Error(`${subject} must use HTTPS or localhost`);
}
}
- if (runtimeConfigCredentialRef(config.apiKey, "KIE apiKey") === undefined) {
- throw new Error("KIE apiKey CredentialRef is required");
- }
- runtimeConfigPositiveInteger(config.defaultConcurrency, "KIE defaultConcurrency");
- runtimeConfigPositiveInteger(config.pollIntervalMs, "KIE pollIntervalMs");
- runtimeConfigPositiveInteger(config.submissionIntervalMs, "KIE submissionIntervalMs");
- runtimeConfigPositiveInteger(config.requestTimeoutMs, "KIE requestTimeoutMs");
- runtimeConfigPositiveInteger(config.maxOperationMs, "KIE maxOperationMs");
- runtimeConfigPositiveInteger(config.maxArtifactBytes, "KIE maxArtifactBytes");
- },
- create(context) {
- const config = runtimeConfigObject(context.config, "KIE");
- runtimeConfigExact(config, [
- "apiBaseUrl", "uploadBaseUrl", "apiKey", "defaultConcurrency", "pollIntervalMs",
- "submissionIntervalMs", "requestTimeoutMs", "maxOperationMs", "maxArtifactBytes",
- ], "KIE");
const apiKey = runtimeConfigCredentialRef(config.apiKey, "KIE apiKey");
if (apiKey === undefined) throw new Error("KIE apiKey CredentialRef is required");
- return createKieProvider({
- instance: context.instance,
- ...(context.lane === undefined ? {} : { lane: context.lane }),
- ...(runtimeConfigString(config.apiBaseUrl, "KIE apiBaseUrl") === undefined
- ? {} : { apiBaseUrl: config.apiBaseUrl as string }),
- ...(runtimeConfigString(config.uploadBaseUrl, "KIE uploadBaseUrl") === undefined
- ? {} : { uploadBaseUrl: config.uploadBaseUrl as string }),
- apiKey,
- ...(runtimeConfigPositiveInteger(config.defaultConcurrency, "KIE defaultConcurrency") === undefined
- ? {} : { defaultConcurrency: config.defaultConcurrency as number }),
- ...(runtimeConfigPositiveInteger(config.pollIntervalMs, "KIE pollIntervalMs") === undefined
- ? {} : { pollIntervalMs: config.pollIntervalMs as number }),
- ...(runtimeConfigPositiveInteger(config.submissionIntervalMs, "KIE submissionIntervalMs") === undefined
- ? {} : { submissionIntervalMs: config.submissionIntervalMs as number }),
- ...(runtimeConfigPositiveInteger(config.requestTimeoutMs, "KIE requestTimeoutMs") === undefined
- ? {} : { requestTimeoutMs: config.requestTimeoutMs as number }),
- ...(runtimeConfigPositiveInteger(config.maxOperationMs, "KIE maxOperationMs") === undefined
- ? {} : { maxOperationMs: config.maxOperationMs as number }),
- ...(runtimeConfigPositiveInteger(config.maxArtifactBytes, "KIE maxArtifactBytes") === undefined
- ? {} : { maxArtifactBytes: config.maxArtifactBytes as number }),
- });
+ const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "KIE defaultConcurrency");
+ const pollIntervalMs = runtimeConfigPositiveInteger(config.pollIntervalMs, "KIE pollIntervalMs");
+ const submissionIntervalMs = runtimeConfigPositiveInteger(config.submissionIntervalMs, "KIE submissionIntervalMs");
+ const requestTimeoutMs = runtimeConfigPositiveInteger(config.requestTimeoutMs, "KIE requestTimeoutMs");
+ const maxOperationMs = runtimeConfigPositiveInteger(config.maxOperationMs, "KIE maxOperationMs");
+ const maxArtifactBytes = runtimeConfigPositiveInteger(config.maxArtifactBytes, "KIE maxArtifactBytes");
+ return {
+ endpoint: createKieProvider({
+ instance: context.instance,
+ ...(context.lane === undefined ? {} : { lane: context.lane }),
+ ...(apiBaseUrl === undefined ? {} : { apiBaseUrl }),
+ ...(uploadBaseUrl === undefined ? {} : { uploadBaseUrl }),
+ apiKey,
+ ...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
+ ...(pollIntervalMs === undefined ? {} : { pollIntervalMs }),
+ ...(submissionIntervalMs === undefined ? {} : { submissionIntervalMs }),
+ ...(requestTimeoutMs === undefined ? {} : { requestTimeoutMs }),
+ ...(maxOperationMs === undefined ? {} : { maxOperationMs }),
+ ...(maxArtifactBytes === undefined ? {} : { maxArtifactBytes }),
+ }),
+ };
},
});
diff --git a/packages/provider-kie/src/provider.ts b/packages/provider-kie/src/provider.ts
index ab1ac24e..75559927 100644
--- a/packages/provider-kie/src/provider.ts
+++ b/packages/provider-kie/src/provider.ts
@@ -19,7 +19,7 @@ import {
defineEndpointPackage,
wakeAfter,
} from "@narratage/endpoint-kit";
-import { credentialRef } from "@narratage/runtime";
+import { credentialRef, isStreamingArtifactStore } from "@narratage/runtime";
import type { ArtifactStore, CredentialRef } from "@narratage/runtime";
import {
@@ -189,8 +189,33 @@ class KieClient {
if (!response.ok) {
throw new KieError("KIE_HTTP_ERROR", `KIE returned HTTP ${response.status}`, { status: response.status });
}
- const text = await response.text();
- if (text.length > 2_000_000) throw new KieError("KIE_RESPONSE_TOO_LARGE", "KIE JSON response is too large");
+ const limit = 2_000_000;
+ const reader = response.body?.getReader();
+ let text: string;
+ if (reader === undefined) {
+ const bytes = new Uint8Array(await response.arrayBuffer());
+ if (bytes.byteLength > limit) throw new KieError("KIE_RESPONSE_TOO_LARGE", "KIE JSON response is too large");
+ text = new TextDecoder().decode(bytes);
+ } else {
+ const decoder = new TextDecoder();
+ let size = 0;
+ let decoded = "";
+ try {
+ while (true) {
+ const item = await reader.read();
+ if (item.done) break;
+ size += item.value.byteLength;
+ if (size > limit) {
+ await reader.cancel();
+ throw new KieError("KIE_RESPONSE_TOO_LARGE", "KIE JSON response is too large");
+ }
+ decoded += decoder.decode(item.value, { stream: true });
+ }
+ text = decoded + decoder.decode();
+ } finally {
+ reader.releaseLock();
+ }
+ }
try {
return object(JSON.parse(text), "KIE response");
} catch (error) {
@@ -200,25 +225,60 @@ class KieClient {
}
async upload(artifact: BlobRef, artifacts: ArtifactStore, apiKey: string): Promise<string> {
- const bytes = await artifacts.get(artifact.digest);
- if (bytes === undefined) throw new KieError("KIE_ARTIFACT_MISSING", `Artifact ${artifact.digest} is unavailable`, { retryable: false });
- if (bytes.byteLength !== artifact.size) throw new KieError("KIE_ARTIFACT_SIZE_MISMATCH", `Artifact ${artifact.digest} size differs`);
- if (bytes.byteLength > this.#options.maxArtifactBytes) {
+ if (artifact.size > this.#options.maxArtifactBytes) {
throw new KieError("KIE_ARTIFACT_TOO_LARGE", `Artifact ${artifact.digest} exceeds the configured KIE upload limit`);
}
const hex = artifact.digest.slice("sha256:".length);
- const form = new FormData();
- const body = Uint8Array.from(bytes).buffer;
- form.append("file", new Blob([body], { type: artifact.mediaType }), `${hex}.${mediaExtension(artifact.mediaType)}`);
- form.append("uploadPath", `svml/${hex.slice(0, 2)}`);
- form.append("fileName", `${hex}.${mediaExtension(artifact.mediaType)}`);
+ const fileName = `${hex}.${mediaExtension(artifact.mediaType)}`;
+ const boundary = `narratage-${hex}`;
+ const encode = (value: string) => new TextEncoder().encode(value);
+ const fileHead = encode(
+ `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${fileName}"\r\n`
+ + `Content-Type: ${artifact.mediaType}\r\n\r\n`,
+ );
+ const fields = encode(
+ `\r\n--${boundary}\r\nContent-Disposition: form-data; name="uploadPath"\r\n\r\nsvml/${hex.slice(0, 2)}`
+ + `\r\n--${boundary}\r\nContent-Disposition: form-data; name="fileName"\r\n\r\n${fileName}`
+ + `\r\n--${boundary}--\r\n`,
+ );
+ const source = isStreamingArtifactStore(artifacts)
+ ? await artifacts.open(artifact.digest)
+ : await artifacts.get(artifact.digest).then((bytes) => bytes === undefined
+ ? undefined
+ : (async function* () { yield bytes; })());
+ if (source === undefined) {
+ throw new KieError("KIE_ARTIFACT_MISSING", `Artifact ${artifact.digest} is unavailable`, { retryable: false });
+ }
+ const maximum = this.#options.maxArtifactBytes;
+ const multipart = (async function* () {
+ yield fileHead;
+ const hash = createHash("sha256");
+ let size = 0;
+ for await (const chunk of source) {
+ size += chunk.byteLength;
+ if (size > artifact.size || size > maximum) {
+ throw new KieError("KIE_ARTIFACT_SIZE_MISMATCH", `Artifact ${artifact.digest} size differs`);
+ }
+ hash.update(chunk);
+ yield chunk;
+ }
+ if (size !== artifact.size || `sha256:${hash.digest("hex")}` !== artifact.digest) {
+ throw new KieError("KIE_ARTIFACT_SIZE_MISMATCH", `Artifact ${artifact.digest} bytes differ`);
+ }
+ yield fields;
+ })();
let response: Record<string, unknown>;
try {
response = await this.#json(`${this.#options.uploadBaseUrl}/api/file-stream-upload`, {
method: "POST",
- headers: { authorization: `Bearer ${apiKey}` },
- body: form,
- });
+ headers: {
+ authorization: `Bearer ${apiKey}`,
+ "content-type": `multipart/form-data; boundary=${boundary}`,
+ "content-length": String(fileHead.byteLength + artifact.size + fields.byteLength),
+ },
+ body: multipart as unknown as BodyInit,
+ duplex: "half",
+ } as RequestInit & { duplex: "half" });
} catch (error) {
if (error instanceof KieError) {
throw new KieError("KIE_UPLOAD_FAILED", error.message, {
@@ -300,7 +360,10 @@ class KieClient {
return response.data;
}
- async download(original: string, expected: "image" | "video", apiKey: string): Promise<{ bytes: Uint8Array; mediaType: string }> {
+ async download(original: string, expected: "image" | "video", apiKey: string): Promise<{
+ chunks: AsyncIterable<Uint8Array>;
+ mediaType: string;
+ }> {
const url = await this.#downloadUrl(original, apiKey);
const response = await this.#fetch(url, { method: "GET" });
if (!response.ok) throw new KieError("KIE_DOWNLOAD_FAILED", `KIE artifact download returned HTTP ${response.status}`);
@@ -308,36 +371,38 @@ class KieClient {
if (Number.isFinite(declared) && declared > this.#options.maxArtifactBytes) {
throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result exceeds the configured artifact limit");
}
- const reader = response.body?.getReader();
- const chunks: Uint8Array[] = [];
- let size = 0;
- if (reader === undefined) {
- const bytes = new Uint8Array(await response.arrayBuffer());
- if (bytes.byteLength > this.#options.maxArtifactBytes) throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result is too large");
- chunks.push(bytes);
- size = bytes.byteLength;
- } else {
- while (true) {
- const item = await reader.read();
- if (item.done) break;
- size += item.value.byteLength;
- if (size > this.#options.maxArtifactBytes) {
- await reader.cancel();
- throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result is too large");
- }
- chunks.push(item.value);
- }
- }
- const bytes = new Uint8Array(size);
- let offset = 0;
- chunks.forEach((chunk) => { bytes.set(chunk, offset); offset += chunk.byteLength; });
const header = response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase();
const fallback = expected === "image" ? "image/png" : "video/mp4";
const mediaType = header === undefined || header === "application/octet-stream" ? fallback : header;
if (!mediaType.startsWith(`${expected}/`)) {
throw new KieError("KIE_RESULT_MEDIA_MISMATCH", `KIE returned ${mediaType} for ${expected} generation`);
}
- return { bytes, mediaType };
+ const maximum = this.#options.maxArtifactBytes;
+ const chunks = response.body === null
+ ? (async function* () {
+ const bytes = new Uint8Array(await response.arrayBuffer());
+ if (bytes.byteLength > maximum) throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result is too large");
+ yield bytes;
+ })()
+ : (async function* () {
+ const reader = response.body!.getReader();
+ let size = 0;
+ try {
+ while (true) {
+ const item = await reader.read();
+ if (item.done) break;
+ size += item.value.byteLength;
+ if (size > maximum) {
+ await reader.cancel();
+ throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result is too large");
+ }
+ yield item.value;
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ })();
+ return { chunks, mediaType };
}
}
@@ -406,7 +471,9 @@ function endpoint(options: {
return failure(new KieError("KIE_OPERATION_TIMEOUT", "KIE task did not become durably available before the deadline"));
}
const delayMs = Math.min(30_000, options.pollIntervalMs * (2 ** Math.min(failures, 5)));
- return wakeAfter(canonicalize({ ...checkpoint, pollFailures: failures }), delayMs, options.now());
+ return wakeAfter(canonicalize({ ...checkpoint, pollFailures: failures }), delayMs, options.now(), {
+ phase: "poll-retry",
+ });
};
return {
async start(context) {
@@ -436,7 +503,9 @@ function endpoint(options: {
polls: 0,
pollFailures: 0,
};
- return wakeAfter(canonicalize(checkpoint), options.pollIntervalMs, options.now());
+ return wakeAfter(canonicalize(checkpoint), options.pollIntervalMs, options.now(), {
+ phase: "submitted",
+ });
} catch (error) {
return failure(error);
}
@@ -470,7 +539,9 @@ function endpoint(options: {
const state = data.state;
if (state === "waiting" || state === "queuing" || state === "generating") {
const next = { ...checkpoint, polls: checkpoint.polls + 1, pollFailures: 0 };
- return wakeAfter(canonicalize(next), options.pollIntervalMs, options.now());
+ return wakeAfter(canonicalize(next), options.pollIntervalMs, options.now(), {
+ phase: state,
+ });
}
if (state === "fail") {
const vendorCode = typeof data.failCode === "string" && data.failCode.length > 0 ? data.failCode : "unknown";
@@ -484,7 +555,23 @@ function endpoint(options: {
const artifacts: BlobRef[] = [];
for (const url of urls) {
const downloaded = await options.client.download(url, route.media, key);
- artifacts.push(await context.artifacts.put(downloaded.bytes, downloaded.mediaType));
+ if (isStreamingArtifactStore(context.artifacts)) {
+ artifacts.push(await context.artifacts.putStream(downloaded.chunks, downloaded.mediaType));
+ } else {
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ for await (const chunk of downloaded.chunks) {
+ chunks.push(Uint8Array.from(chunk));
+ size += chunk.byteLength;
+ }
+ const bytes = new Uint8Array(size);
+ let offset = 0;
+ for (const chunk of chunks) {
+ bytes.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ artifacts.push(await context.artifacts.put(bytes, downloaded.mediaType));
+ }
}
const result = route.packageResult(artifacts);
const vendorMetrics: Record<string, CanonicalValue> = {};
@@ -593,3 +680,4 @@ export function createKieProvider(config: CreateKieProviderOptions = {}) {
})),
});
}
+import { createHash } from "node:crypto";
diff --git a/packages/provider-kie/test/provider.test.ts b/packages/provider-kie/test/provider.test.ts
index 9dea3f01..ed119706 100644
--- a/packages/provider-kie/test/provider.test.ts
+++ b/packages/provider-kie/test/provider.test.ts
@@ -117,8 +117,12 @@ test("KIE uploads content-addressed references, resumes one task, and persists g
calls.push(url);
if (url.endsWith("/api/file-stream-upload")) {
assert.equal(init?.method, "POST");
- assert.ok(init?.body instanceof FormData);
- assert.equal(init.body.get("fileName"), `${firstFrame.digest.slice(7)}.png`);
+ assert.match(String((init?.headers as Record<string, string>)["content-type"]), /^multipart\/form-data; boundary=/u);
+ const chunks: Uint8Array[] = [];
+ for await (const chunk of init?.body as unknown as AsyncIterable<Uint8Array>) chunks.push(chunk);
+ const multipart = Buffer.concat(chunks).toString("latin1");
+ assert.match(multipart, /name="fileName"\r\n\r\n[0-9a-f]{64}\.png/u);
+ assert.ok(Buffer.concat(chunks).includes(Buffer.from([1, 2, 3])));
return Response.json({
success: true,
code: 200,
@@ -164,6 +168,7 @@ test("KIE uploads content-addressed references, resumes one task, and persists g
};
const started = await endpoint.start(common);
assert.equal(started.status, "pending");
+ assert.deepEqual(started.status === "pending" && started.progress, { phase: "submitted" });
assert.equal(calls.length, 2);
assert.equal(started.status === "pending" && (started.checkpoint as Record<string, unknown>).taskId, "task_seedance_test");
const completed = await endpoint.resume({
diff --git a/packages/provider-media-aws-lambda/src/activation.ts b/packages/provider-media-aws-lambda/src/activation.ts
index 2babfef7..1836d494 100644
--- a/packages/provider-media-aws-lambda/src/activation.ts
+++ b/packages/provider-media-aws-lambda/src/activation.ts
@@ -51,11 +51,8 @@ function providerOptions(context: RuntimeAdapterFactoryContext): CreateAwsLambda
const awsLambdaMediaRuntimeAdapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-media-aws-lambda",
- validate(context) {
- providerOptions(context);
- },
- create(context) {
- return createAwsLambdaMediaProvider(providerOptions(context));
+ activate(context) {
+ return { endpoint: createAwsLambdaMediaProvider(providerOptions(context)) };
},
// ARN, region and bucket relationships are closed configuration facts. Live
// Lambda availability remains an operation-time fact until a dedicated
diff --git a/packages/provider-media-local/package.json b/packages/provider-media-local/package.json
index 524a14f5..a4195aed 100644
--- a/packages/provider-media-local/package.json
+++ b/packages/provider-media-local/package.json
@@ -17,6 +17,7 @@
"@narratage/media-pipeline": "workspace:*",
"@narratage/program-space": "workspace:*",
"@narratage/protocol": "workspace:*",
+ "@narratage/runtime": "workspace:*",
"@narratage/runtime-adapter": "workspace:*",
"@narratage/runtime-adapter-node": "workspace:*",
"@narratage/speech": "workspace:*"
diff --git a/packages/provider-media-local/src/activation.ts b/packages/provider-media-local/src/activation.ts
index 59a4c782..585ea41a 100644
--- a/packages/provider-media-local/src/activation.ts
+++ b/packages/provider-media-local/src/activation.ts
@@ -15,18 +15,7 @@ import { localMediaToolchainService } from "./service.js";
const localMediaRuntimeAdapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-media-local",
- validate(context) {
- const config = runtimeConfigObject(context.config, "local media");
- runtimeConfigExact(config, [
- "ffmpegPath", "ffprobePath", "defaultConcurrency", "processTimeoutMs", "maxProbeOutputBytes",
- ], "local media");
- runtimeConfigString(config.ffmpegPath, "media ffmpegPath");
- runtimeConfigString(config.ffprobePath, "media ffprobePath");
- runtimeConfigPositiveInteger(config.defaultConcurrency, "media defaultConcurrency");
- runtimeConfigPositiveInteger(config.processTimeoutMs, "media processTimeoutMs");
- runtimeConfigPositiveInteger(config.maxProbeOutputBytes, "media maxProbeOutputBytes");
- },
- create(context) {
+ activate(context) {
const config = runtimeConfigObject(context.config, "local media");
runtimeConfigExact(config, [
"ffmpegPath", "ffprobePath", "defaultConcurrency", "processTimeoutMs", "maxProbeOutputBytes",
@@ -35,37 +24,36 @@ const localMediaRuntimeAdapter = createRuntimeEndpointAdapterFacet({
const configuredFfprobe = runtimeConfigString(config.ffprobePath, "media ffprobePath");
const ffmpegPath = configuredFfmpeg === undefined ? undefined : resolveRuntimeExecutable(context.root, configuredFfmpeg);
const ffprobePath = configuredFfprobe === undefined ? undefined : resolveRuntimeExecutable(context.root, configuredFfprobe);
- return createLocalMediaProvider({
- instance: context.instance,
- ...(context.lane === undefined ? {} : { lane: context.lane }),
- ...(ffmpegPath === undefined ? {} : { ffmpegPath }),
- ...(ffprobePath === undefined ? {} : { ffprobePath }),
- ...(runtimeConfigPositiveInteger(config.defaultConcurrency, "media defaultConcurrency") === undefined
- ? {} : { defaultConcurrency: config.defaultConcurrency as number }),
- ...(runtimeConfigPositiveInteger(config.processTimeoutMs, "media processTimeoutMs") === undefined
- ? {} : { processTimeoutMs: config.processTimeoutMs as number }),
- ...(runtimeConfigPositiveInteger(config.maxProbeOutputBytes, "media maxProbeOutputBytes") === undefined
- ? {} : { maxProbeOutputBytes: config.maxProbeOutputBytes as number }),
- });
- },
- async doctor(context) {
- const config = runtimeConfigObject(context.config, "local media");
- return [
- ...await diagnoseRuntimeExecutable({
- root: context.root,
- configured: runtimeConfigString(config.ffmpegPath, "media ffmpegPath"),
- fallback: "ffmpeg",
- subject: "FFmpeg",
- }),
- ...await diagnoseRuntimeExecutable({
- root: context.root,
- configured: runtimeConfigString(config.ffprobePath, "media ffprobePath"),
- fallback: "ffprobe",
- subject: "FFprobe",
+ const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "media defaultConcurrency");
+ const processTimeoutMs = runtimeConfigPositiveInteger(config.processTimeoutMs, "media processTimeoutMs");
+ const maxProbeOutputBytes = runtimeConfigPositiveInteger(config.maxProbeOutputBytes, "media maxProbeOutputBytes");
+ return {
+ endpoint: createLocalMediaProvider({
+ instance: context.instance,
+ ...(context.lane === undefined ? {} : { lane: context.lane }),
+ ...(ffmpegPath === undefined ? {} : { ffmpegPath }),
+ ...(ffprobePath === undefined ? {} : { ffprobePath }),
+ ...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
+ ...(processTimeoutMs === undefined ? {} : { processTimeoutMs }),
+ ...(maxProbeOutputBytes === undefined ? {} : { maxProbeOutputBytes }),
}),
- ];
+ externalService: localMediaToolchainService(context),
+ diagnose: async () => [
+ ...await diagnoseRuntimeExecutable({
+ root: context.root,
+ configured: configuredFfmpeg,
+ fallback: "ffmpeg",
+ subject: "FFmpeg",
+ }),
+ ...await diagnoseRuntimeExecutable({
+ root: context.root,
+ configured: configuredFfprobe,
+ fallback: "ffprobe",
+ subject: "FFprobe",
+ }),
+ ],
+ };
},
- service: localMediaToolchainService,
});
export const svmlPackage = {
diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.ts
index 9dddab91..0df2f1f4 100644
--- a/packages/provider-media-local/src/provider.ts
+++ b/packages/provider-media-local/src/provider.ts
@@ -15,6 +15,7 @@ import {
import type { MediaExecutionEnvironment, MediaOperationResult } from "@narratage/media-execution";
import { mediaPipelineCapabilities } from "@narratage/media-pipeline";
import { canonicalize, digestOf } from "@narratage/protocol";
+import { isStreamingArtifactStore } from "@narratage/runtime";
import { speechTypes } from "@narratage/speech";
import { defineEndpointPackage } from "@narratage/endpoint-kit";
import type { EndpointFulfillment, EndpointInvocationContext } from "@narratage/endpoint-kit";
@@ -67,7 +68,15 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
label: "media.local",
artifacts: {
get: async (source) => await context.artifacts.get(source.digest),
+ open: async (source) => isStreamingArtifactStore(context.artifacts)
+ ? await context.artifacts.open(source.digest)
+ : await context.artifacts.get(source.digest).then((bytes) => bytes === undefined
+ ? undefined
+ : (async function* () { yield bytes; })()),
put: async (bytes, mediaType) => await context.artifacts.put(bytes, mediaType),
+ putFile: async (path, mediaType) => isStreamingArtifactStore(context.artifacts)
+ ? await context.artifacts.putStream(createReadStream(path), mediaType)
+ : await context.artifacts.put(await readFile(path), mediaType),
},
});
const operation = (
@@ -148,3 +157,5 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
],
});
}
+import { createReadStream } from "node:fs";
+import { readFile } from "node:fs/promises";
diff --git a/packages/provider-whisperx-local/src/activation.ts b/packages/provider-whisperx-local/src/activation.ts
index b5111695..20f778e6 100644
--- a/packages/provider-whisperx-local/src/activation.ts
+++ b/packages/provider-whisperx-local/src/activation.ts
@@ -11,7 +11,7 @@ import { localWhisperXService } from "./service.js";
const localWhisperXRuntimeAdapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-whisperx-local",
- validate(context) {
+ activate(context) {
const config = runtimeConfigObject(context.config, "local WhisperX");
runtimeConfigExact(config, [
"baseUrl", "expectedModel", "expectedDevice", "expectedCompute", "expectedBatchSize",
@@ -19,27 +19,27 @@ const localWhisperXRuntimeAdapter = createRuntimeEndpointAdapterFacet({
"defaultConcurrency", "requestTimeoutMs", "maxResponseBytes",
"serviceCommand", "servicePrepareCommand",
], "local WhisperX");
- const baseUrlValue = runtimeConfigString(config.baseUrl, "WhisperX baseUrl");
- if (baseUrlValue !== undefined) {
- const baseUrl = new URL(baseUrlValue);
- if (baseUrl.protocol !== "http:"
- || !["127.0.0.1", "localhost", "::1", "[::1]"].includes(baseUrl.hostname)) {
+ const baseUrl = runtimeConfigString(config.baseUrl, "WhisperX baseUrl");
+ if (baseUrl !== undefined) {
+ const parsed = new URL(baseUrl);
+ if (parsed.protocol !== "http:"
+ || !["127.0.0.1", "localhost", "::1", "[::1]"].includes(parsed.hostname)) {
throw new Error("local WhisperX Provider requires a loopback HTTP service");
}
}
- runtimeConfigString(config.expectedModel, "WhisperX expectedModel");
- runtimeConfigString(config.expectedDevice, "WhisperX expectedDevice");
- runtimeConfigString(config.expectedCompute, "WhisperX expectedCompute");
- runtimeConfigPositiveInteger(config.expectedBatchSize, "WhisperX expectedBatchSize");
- runtimeConfigString(config.expectedServiceVersion, "WhisperX expectedServiceVersion");
- runtimeConfigString(config.expectedWhisperXVersion, "WhisperX expectedWhisperXVersion");
- const punkt = runtimeConfigString(config.expectedPunktTabDigest, "WhisperX expectedPunktTabDigest");
- if (punkt !== undefined && !/^[0-9a-f]{64}$/u.test(punkt)) {
+ const expectedModel = runtimeConfigString(config.expectedModel, "WhisperX expectedModel");
+ const expectedDevice = runtimeConfigString(config.expectedDevice, "WhisperX expectedDevice");
+ const expectedCompute = runtimeConfigString(config.expectedCompute, "WhisperX expectedCompute");
+ const expectedBatchSize = runtimeConfigPositiveInteger(config.expectedBatchSize, "WhisperX expectedBatchSize");
+ const expectedServiceVersion = runtimeConfigString(config.expectedServiceVersion, "WhisperX expectedServiceVersion");
+ const expectedWhisperXVersion = runtimeConfigString(config.expectedWhisperXVersion, "WhisperX expectedWhisperXVersion");
+ const expectedPunktTabDigest = runtimeConfigString(config.expectedPunktTabDigest, "WhisperX expectedPunktTabDigest");
+ if (expectedPunktTabDigest !== undefined && !/^[0-9a-f]{64}$/u.test(expectedPunktTabDigest)) {
throw new Error("WhisperX expectedPunktTabDigest is invalid");
}
- runtimeConfigPositiveInteger(config.defaultConcurrency, "WhisperX defaultConcurrency");
- runtimeConfigPositiveInteger(config.requestTimeoutMs, "WhisperX requestTimeoutMs");
- runtimeConfigPositiveInteger(config.maxResponseBytes, "WhisperX maxResponseBytes");
+ const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "WhisperX defaultConcurrency");
+ const requestTimeoutMs = runtimeConfigPositiveInteger(config.requestTimeoutMs, "WhisperX requestTimeoutMs");
+ const maxResponseBytes = runtimeConfigPositiveInteger(config.maxResponseBytes, "WhisperX maxResponseBytes");
for (const key of ["serviceCommand", "servicePrepareCommand"] as const) {
const value = config[key];
if (value !== undefined
@@ -48,43 +48,25 @@ const localWhisperXRuntimeAdapter = createRuntimeEndpointAdapterFacet({
throw new Error(`WhisperX ${key} must be a non-empty array of non-empty strings`);
}
}
+ return {
+ endpoint: createLocalWhisperXProvider({
+ instance: context.instance,
+ ...(context.lane === undefined ? {} : { lane: context.lane }),
+ ...(baseUrl === undefined ? {} : { baseUrl }),
+ ...(expectedModel === undefined ? {} : { expectedModel }),
+ ...(expectedDevice === undefined ? {} : { expectedDevice }),
+ ...(expectedCompute === undefined ? {} : { expectedCompute }),
+ ...(expectedBatchSize === undefined ? {} : { expectedBatchSize }),
+ ...(expectedServiceVersion === undefined ? {} : { expectedServiceVersion }),
+ ...(expectedWhisperXVersion === undefined ? {} : { expectedWhisperXVersion }),
+ ...(expectedPunktTabDigest === undefined ? {} : { expectedPunktTabDigest }),
+ ...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
+ ...(requestTimeoutMs === undefined ? {} : { requestTimeoutMs }),
+ ...(maxResponseBytes === undefined ? {} : { maxResponseBytes }),
+ }),
+ externalService: localWhisperXService(context),
+ };
},
- create(context) {
- const config = runtimeConfigObject(context.config, "local WhisperX");
- runtimeConfigExact(config, [
- "baseUrl", "expectedModel", "expectedDevice", "expectedCompute", "expectedBatchSize",
- "expectedServiceVersion", "expectedWhisperXVersion", "expectedPunktTabDigest",
- "defaultConcurrency", "requestTimeoutMs", "maxResponseBytes",
- "serviceCommand", "servicePrepareCommand",
- ], "local WhisperX");
- return createLocalWhisperXProvider({
- instance: context.instance,
- ...(context.lane === undefined ? {} : { lane: context.lane }),
- ...(runtimeConfigString(config.baseUrl, "WhisperX baseUrl") === undefined
- ? {} : { baseUrl: config.baseUrl as string }),
- ...(runtimeConfigString(config.expectedModel, "WhisperX expectedModel") === undefined
- ? {} : { expectedModel: config.expectedModel as string }),
- ...(runtimeConfigString(config.expectedDevice, "WhisperX expectedDevice") === undefined
- ? {} : { expectedDevice: config.expectedDevice as string }),
- ...(runtimeConfigString(config.expectedCompute, "WhisperX expectedCompute") === undefined
- ? {} : { expectedCompute: config.expectedCompute as string }),
- ...(runtimeConfigPositiveInteger(config.expectedBatchSize, "WhisperX expectedBatchSize") === undefined
- ? {} : { expectedBatchSize: config.expectedBatchSize as number }),
- ...(runtimeConfigString(config.expectedServiceVersion, "WhisperX expectedServiceVersion") === undefined
- ? {} : { expectedServiceVersion: config.expectedServiceVersion as string }),
- ...(runtimeConfigString(config.expectedWhisperXVersion, "WhisperX expectedWhisperXVersion") === undefined
- ? {} : { expectedWhisperXVersion: config.expectedWhisperXVersion as string }),
- ...(runtimeConfigString(config.expectedPunktTabDigest, "WhisperX expectedPunktTabDigest") === undefined
- ? {} : { expectedPunktTabDigest: config.expectedPunktTabDigest as string }),
- ...(runtimeConfigPositiveInteger(config.defaultConcurrency, "WhisperX defaultConcurrency") === undefined
- ? {} : { defaultConcurrency: config.defaultConcurrency as number }),
- ...(runtimeConfigPositiveInteger(config.requestTimeoutMs, "WhisperX requestTimeoutMs") === undefined
- ? {} : { requestTimeoutMs: config.requestTimeoutMs as number }),
- ...(runtimeConfigPositiveInteger(config.maxResponseBytes, "WhisperX maxResponseBytes") === undefined
- ? {} : { maxResponseBytes: config.maxResponseBytes as number }),
- });
- },
- service: localWhisperXService,
});
export const svmlPackage = {
diff --git a/packages/provider-whisperx-local/src/provider.ts b/packages/provider-whisperx-local/src/provider.ts
index 9af1ddba..098aa702 100644
--- a/packages/provider-whisperx-local/src/provider.ts
+++ b/packages/provider-whisperx-local/src/provider.ts
@@ -214,8 +214,35 @@ async function limitedJson(response: Response, maxBytes: number, subject: string
readonly value: unknown;
readonly bytes: Uint8Array;
}> {
- const bytes = new Uint8Array(await response.arrayBuffer());
- assert(bytes.byteLength <= maxBytes, `${subject} exceeded the configured response limit`);
+ const reader = response.body?.getReader();
+ let bytes: Uint8Array;
+ if (reader === undefined) {
+ bytes = new Uint8Array(await response.arrayBuffer());
+ assert(bytes.byteLength <= maxBytes, `${subject} exceeded the configured response limit`);
+ } else {
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ try {
+ while (true) {
+ const item = await reader.read();
+ if (item.done) break;
+ size += item.value.byteLength;
+ if (size > maxBytes) {
+ await reader.cancel();
+ throw new Error(`${subject} exceeded the configured response limit`);
+ }
+ chunks.push(item.value);
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ bytes = new Uint8Array(size);
+ let offset = 0;
+ for (const chunk of chunks) {
+ bytes.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ }
assert(response.ok, `${subject} failed with HTTP ${response.status}: ${Buffer.from(bytes).toString("utf8").slice(0, 500)}`);
try {
return { value: JSON.parse(Buffer.from(bytes).toString("utf8")), bytes };
diff --git a/packages/provider-xiaomi-mimo/README.md b/packages/provider-xiaomi-mimo/README.md
index 2d5588f5..f6774e2a 100644
--- a/packages/provider-xiaomi-mimo/README.md
+++ b/packages/provider-xiaomi-mimo/README.md
@@ -27,7 +27,7 @@ not part of the author model.
Add the Provider independently to the Runtime package lock:
```sh
-pnpm narratage lock-packages svml.runtime-packages.lock \
+node --run narratage -- lock-packages svml.runtime-packages.lock \
--package @narratage/provider-xiaomi-mimo \
--package-root .
```
diff --git a/packages/provider-xiaomi-mimo/src/activation.ts b/packages/provider-xiaomi-mimo/src/activation.ts
index 5ce74adc..2ce41b39 100644
--- a/packages/provider-xiaomi-mimo/src/activation.ts
+++ b/packages/provider-xiaomi-mimo/src/activation.ts
@@ -11,46 +11,40 @@ import { createXiaomiMimoProvider } from "./provider.js";
const adapter = createRuntimeEndpointAdapterFacet({
use: "@narratage/provider-xiaomi-mimo",
- validate(context) {
+ activate(context) {
const config = runtimeConfigObject(context.config, "Xiaomi MiMo");
runtimeConfigExact(config, [
"apiBaseUrl", "apiKey", "defaultConcurrency", "requestTimeoutMs",
"maxResponseBytes", "maxVoiceSampleBase64Bytes",
], "Xiaomi MiMo");
- const base = runtimeConfigString(config.apiBaseUrl, "Xiaomi MiMo apiBaseUrl");
- if (base !== undefined) {
- const url = new URL(base);
+ const apiBaseUrl = runtimeConfigString(config.apiBaseUrl, "Xiaomi MiMo apiBaseUrl");
+ if (apiBaseUrl !== undefined) {
+ const url = new URL(apiBaseUrl);
if (url.protocol !== "https:" && url.hostname !== "localhost" && url.hostname !== "127.0.0.1") {
throw new Error("Xiaomi MiMo apiBaseUrl must use HTTPS or loopback");
}
}
- if (runtimeConfigCredentialRef(config.apiKey, "Xiaomi MiMo apiKey") === undefined) {
- throw new Error("Xiaomi MiMo apiKey CredentialRef is required");
- }
- runtimeConfigPositiveInteger(config.defaultConcurrency, "Xiaomi MiMo defaultConcurrency");
- runtimeConfigPositiveInteger(config.requestTimeoutMs, "Xiaomi MiMo requestTimeoutMs");
- runtimeConfigPositiveInteger(config.maxResponseBytes, "Xiaomi MiMo maxResponseBytes");
- runtimeConfigPositiveInteger(config.maxVoiceSampleBase64Bytes, "Xiaomi MiMo maxVoiceSampleBase64Bytes");
- },
- create(context) {
- const config = runtimeConfigObject(context.config, "Xiaomi MiMo");
const apiKey = runtimeConfigCredentialRef(config.apiKey, "Xiaomi MiMo apiKey");
if (apiKey === undefined) throw new Error("Xiaomi MiMo apiKey CredentialRef is required");
- return createXiaomiMimoProvider({
- instance: context.instance,
- ...(context.lane === undefined ? {} : { lane: context.lane }),
- ...(runtimeConfigString(config.apiBaseUrl, "Xiaomi MiMo apiBaseUrl") === undefined
- ? {} : { apiBaseUrl: config.apiBaseUrl as string }),
- apiKey,
- ...(runtimeConfigPositiveInteger(config.defaultConcurrency, "Xiaomi MiMo defaultConcurrency") === undefined
- ? {} : { defaultConcurrency: config.defaultConcurrency as number }),
- ...(runtimeConfigPositiveInteger(config.requestTimeoutMs, "Xiaomi MiMo requestTimeoutMs") === undefined
- ? {} : { requestTimeoutMs: config.requestTimeoutMs as number }),
- ...(runtimeConfigPositiveInteger(config.maxResponseBytes, "Xiaomi MiMo maxResponseBytes") === undefined
- ? {} : { maxResponseBytes: config.maxResponseBytes as number }),
- ...(runtimeConfigPositiveInteger(config.maxVoiceSampleBase64Bytes, "Xiaomi MiMo maxVoiceSampleBase64Bytes") === undefined
- ? {} : { maxVoiceSampleBase64Bytes: config.maxVoiceSampleBase64Bytes as number }),
- });
+ const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "Xiaomi MiMo defaultConcurrency");
+ const requestTimeoutMs = runtimeConfigPositiveInteger(config.requestTimeoutMs, "Xiaomi MiMo requestTimeoutMs");
+ const maxResponseBytes = runtimeConfigPositiveInteger(config.maxResponseBytes, "Xiaomi MiMo maxResponseBytes");
+ const maxVoiceSampleBase64Bytes = runtimeConfigPositiveInteger(
+ config.maxVoiceSampleBase64Bytes,
+ "Xiaomi MiMo maxVoiceSampleBase64Bytes",
+ );
+ return {
+ endpoint: createXiaomiMimoProvider({
+ instance: context.instance,
+ ...(context.lane === undefined ? {} : { lane: context.lane }),
+ ...(apiBaseUrl === undefined ? {} : { apiBaseUrl }),
+ apiKey,
+ ...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
+ ...(requestTimeoutMs === undefined ? {} : { requestTimeoutMs }),
+ ...(maxResponseBytes === undefined ? {} : { maxResponseBytes }),
+ ...(maxVoiceSampleBase64Bytes === undefined ? {} : { maxVoiceSampleBase64Bytes }),
+ }),
+ };
},
});
diff --git a/packages/runtime-adapter/README.md b/packages/runtime-adapter/README.md
index 5dc92a7c..b3ce8620 100644
--- a/packages/runtime-adapter/README.md
+++ b/packages/runtime-adapter/README.md
@@ -8,9 +8,15 @@ One adapter has a unique `use` name and exactly one kind:
- `endpoint` constructs an external capability Endpoint;
- `runtime-service` constructs a Scheduler, Store or credential service package.
-Every adapter separates pure closed-data `validate()` from environment-touching `create()`.
-`doctor()` may report diagnostics without constructing the selected service, and an Endpoint adapter
-may describe one warm external service with bounded prepare/start commands and a health probe.
+An Endpoint adapter has one pure `activate()` declaration. It parses the selected configuration once
+and returns the Endpoint package, optional diagnostics and an optional warm external service. The
+Endpoint package is the only source of credentials, permissions, capabilities and scheduling facts.
+Activation must not resolve credentials or environment-sourced deployment values, access the network,
+start a process or mutate durable state. Environment references are resolved only when the Endpoint
+actually handles a Need; `doctor` may diagnose them explicitly.
+
+Runtime-service adapters retain a separate pure `validate()` gate because constructing SQLite or a
+Store can itself mutate deployment state; `doctor()` may diagnose them without calling `create()`.
The registry binds adapter implementations to the actual locked package and dependency-closure
digests. Source imports cannot select these facets or grant their permissions; only the Runtime
diff --git a/packages/runtime-adapter/src/index.ts b/packages/runtime-adapter/src/index.ts
index e04cc546..3567e6a8 100644
--- a/packages/runtime-adapter/src/index.ts
+++ b/packages/runtime-adapter/src/index.ts
@@ -62,12 +62,23 @@ export type RuntimeExternalService = {
readonly start?: RuntimeServiceCommand;
};
+/**
+ * The complete, side-effect-free declaration of one configured Endpoint.
+ *
+ * `activate` may parse configuration and construct handlers, but it must not
+ * resolve credentials or environment-sourced deployment values, access the
+ * network, start a process or mutate durable state. That gives normal execution, `doctor`, credential management and
+ * service lifecycle one source of truth without making a second manifest just
+ * for diagnostics.
+ */
+export type RuntimeEndpointActivation = {
+ readonly endpoint: EndpointPackage;
+ readonly externalService?: RuntimeExternalService;
+ readonly diagnose?: () => readonly RuntimeDoctorDiagnostic[] | Promise<readonly RuntimeDoctorDiagnostic[]>;
+};
+
export type RuntimeEndpointAdapterImplementation = {
- /** Pure, closed-data validation. Must not construct an Endpoint or touch the environment. */
- validate(context: RuntimeAdapterFactoryContext): void;
- create(context: RuntimeAdapterFactoryContext): EndpointPackage | Promise<EndpointPackage>;
- doctor?(context: RuntimeAdapterFactoryContext): readonly RuntimeDoctorDiagnostic[] | Promise<readonly RuntimeDoctorDiagnostic[]>;
- service?(context: RuntimeAdapterFactoryContext): RuntimeExternalService | undefined;
+ activate(context: RuntimeAdapterFactoryContext): RuntimeEndpointActivation | Promise<RuntimeEndpointActivation>;
};
export type RuntimeServiceAdapterImplementation = {
@@ -110,22 +121,27 @@ function identity(value: CanonicalValue): RuntimeAdapterIdentity {
return canonicalize(item) as unknown as RuntimeAdapterIdentity;
}
-function implementation(value: unknown, subject: string): RuntimeAdapterHostFacet["implementation"] {
- assert(value !== null && typeof value === "object" && typeof (value as { create?: unknown }).create === "function",
- `${subject} does not implement create()`);
- assert(typeof (value as { validate?: unknown }).validate === "function",
- `${subject} does not implement validate()`);
+function implementation(
+ value: unknown,
+ subject: string,
+ kind: RuntimeAdapterKind,
+): RuntimeAdapterHostFacet["implementation"] {
+ assert(value !== null && typeof value === "object", `${subject} implementation is not an object`);
+ if (kind === "endpoint") {
+ assert(typeof (value as { activate?: unknown }).activate === "function",
+ `${subject} does not implement activate()`);
+ return value as RuntimeEndpointAdapterImplementation;
+ }
+ assert(typeof (value as { create?: unknown }).create === "function", `${subject} does not implement create()`);
+ assert(typeof (value as { validate?: unknown }).validate === "function", `${subject} does not implement validate()`);
const doctor = (value as { doctor?: unknown }).doctor;
assert(doctor === undefined || typeof doctor === "function", `${subject}.doctor must be a function`);
- return value as RuntimeAdapterHostFacet["implementation"];
+ return value as RuntimeServiceAdapterImplementation;
}
export function createRuntimeEndpointAdapterFacet(options: {
readonly use: string;
- readonly validate: RuntimeEndpointAdapterImplementation["validate"];
- readonly create: RuntimeEndpointAdapterImplementation["create"];
- readonly doctor?: RuntimeEndpointAdapterImplementation["doctor"];
- readonly service?: RuntimeEndpointAdapterImplementation["service"];
+ readonly activate: RuntimeEndpointAdapterImplementation["activate"];
}): RuntimeAdapterHostFacet {
const facet: RuntimeAdapterHostFacet = {
abi: runtimeAdapterHostAbi,
@@ -134,16 +150,7 @@ export function createRuntimeEndpointAdapterFacet(options: {
use: options.use,
kind: "endpoint",
})),
- // Validated here, not only at the loading boundary. A facet that fails
- // isRuntimeAdapterHostFacet is skipped by the loader, so the author of a
- // broken adapter would otherwise learn about it as "adapter X is not
- // registered", three layers from the mistake.
- implementation: implementation({
- validate: options.validate,
- create: options.create,
- ...(options.doctor === undefined ? {} : { doctor: options.doctor }),
- ...(options.service === undefined ? {} : { service: options.service }),
- }, `Runtime Adapter ${options.use}`),
+ implementation: implementation({ activate: options.activate }, `Runtime Adapter ${options.use}`, "endpoint"),
};
return facet;
}
@@ -165,7 +172,7 @@ export function createRuntimeServiceAdapterFacet(options: {
validate: options.validate,
create: options.create,
...(options.doctor === undefined ? {} : { doctor: options.doctor }),
- }, `Runtime Adapter ${options.use}`),
+ }, `Runtime Adapter ${options.use}`, "runtime-service"),
};
}
@@ -173,7 +180,7 @@ export function isRuntimeAdapterHostFacet(value: HostFacet): value is RuntimeAda
if (value.abi !== runtimeAdapterHostAbi) return false;
try {
identity(value.identity);
- implementation(value.implementation, "Runtime Adapter");
+ implementation(value.implementation, "Runtime Adapter", identity(value.identity).kind);
return true;
} catch {
return false;
@@ -272,7 +279,7 @@ export class RuntimeAdapterRegistry {
#register(facet: RuntimeAdapterHostFacet, binding?: RuntimeAdapterPackageBinding): void {
assert(facet.abi === runtimeAdapterHostAbi, `Runtime Adapter ${facet.abi} ABI is unsupported`);
const normalized = identity(facet.identity);
- implementation(facet.implementation, `Runtime Adapter ${normalized.use}`);
+ implementation(facet.implementation, `Runtime Adapter ${normalized.use}`, normalized.kind);
assert(!this.#registrations.has(normalized.use), `Runtime Adapter ${normalized.use} is already registered`);
if (binding !== undefined) {
assert(binding.packageName.trim().length > 0, "Runtime Adapter physical package name is empty");
@@ -291,8 +298,16 @@ export class RuntimeAdapterRegistry {
}
/** Trusted embedding compatibility. Locked installed packages should use registerFacet(). */
- registerEndpoint(use: string, create: RuntimeEndpointAdapterImplementation["create"]): void {
- this.#register(createRuntimeEndpointAdapterFacet({ use, validate() {}, create }));
+ registerEndpoint(
+ use: string,
+ create: (context: RuntimeAdapterFactoryContext) => EndpointPackage | Promise<EndpointPackage>,
+ ): void {
+ this.#register(createRuntimeEndpointAdapterFacet({
+ use,
+ async activate(context) {
+ return { endpoint: await create(context) };
+ },
+ }));
}
/** Trusted embedding compatibility. Locked installed packages should use registerFacet(). */
@@ -328,18 +343,37 @@ export class RuntimeAdapterRegistry {
message: `Runtime Adapter ${use} is ${value.facet.identity.kind}, not ${kind}`,
subject: use,
}];
- value.facet.implementation.validate(context);
+ assert(kind === "runtime-service", "Endpoint configuration is validated by its activation");
+ (value.facet.implementation as RuntimeServiceAdapterImplementation).validate(context);
return [];
}
- async createEndpoint(use: string, context: RuntimeAdapterFactoryContext): Promise<EndpointPackage> {
+ async activateEndpoint(
+ use: string,
+ context: RuntimeAdapterFactoryContext,
+ ): Promise<RuntimeEndpointActivation> {
const value = this.#registrations.get(use);
assert(value !== undefined, `Runtime Endpoint adapter ${use} is not registered`);
assert(value.facet.identity.kind === "endpoint", `Runtime Adapter ${use} is not an Endpoint adapter`);
const implementation = value.facet.implementation as RuntimeEndpointAdapterImplementation;
- implementation.validate(context);
- const created = await implementation.create(context);
- return bindEndpointPackage(created, value.facet.identity, value.binding);
+ const activation = await implementation.activate(context);
+ assert(activation !== null && typeof activation === "object", `Runtime Endpoint adapter ${use} returned no activation`);
+ assert(activation.endpoint !== null && typeof activation.endpoint === "object",
+ `Runtime Endpoint adapter ${use} activation has no Endpoint package`);
+ assert(activation.endpoint.instance.id === context.instance,
+ `Runtime Endpoint adapter ${use} created Endpoint ${activation.endpoint.instance.id} outside configured instance ${context.instance}`);
+ assert(activation.externalService === undefined || activation.externalService.id.trim().length > 0,
+ `Runtime Endpoint adapter ${use} external service id is empty`);
+ assert(activation.diagnose === undefined || typeof activation.diagnose === "function",
+ `Runtime Endpoint adapter ${use} activation diagnose must be a function`);
+ return {
+ ...activation,
+ endpoint: bindEndpointPackage(activation.endpoint, value.facet.identity, value.binding),
+ };
+ }
+
+ async createEndpoint(use: string, context: RuntimeAdapterFactoryContext): Promise<EndpointPackage> {
+ return (await this.activateEndpoint(use, context)).endpoint;
}
async createService(use: string, context: RuntimeAdapterFactoryContext): Promise<RuntimeServicePackage> {
@@ -349,24 +383,19 @@ export class RuntimeAdapterRegistry {
const implementation = value.facet.implementation as RuntimeServiceAdapterImplementation;
implementation.validate(context);
const created = await implementation.create(context);
+ for (const service of created.services) {
+ assert(service.instance.id === context.instance || service.instance.id.startsWith(`${context.instance}.`),
+ `Runtime Adapter ${use} created service ${service.instance.id} outside configured namespace ${context.instance}`);
+ }
return bindServicePackage(created, value.facet.identity, value.binding);
}
- /** The external program this Endpoint adapter needs running, when it declares one. */
- service(use: string, context: RuntimeAdapterFactoryContext): RuntimeExternalService | undefined {
- const value = this.#registrations.get(use);
- if (value === undefined || value.facet.identity.kind !== "endpoint") return undefined;
- const implementation = value.facet.implementation as RuntimeEndpointAdapterImplementation;
- implementation.validate(context);
- const declare = implementation.service;
- return declare === undefined ? undefined : declare(context);
- }
-
async doctor(use: string, kind: RuntimeAdapterKind, context: RuntimeAdapterFactoryContext): Promise<readonly RuntimeDoctorDiagnostic[]> {
const value = this.#registrations.get(use);
if (value === undefined) return [{ severity: "error", code: "RUNTIME_ADAPTER_MISSING", message: `Runtime Adapter ${use} is not registered`, subject: use }];
if (value.facet.identity.kind !== kind) return [{ severity: "error", code: "RUNTIME_ADAPTER_KIND", message: `Runtime Adapter ${use} is ${value.facet.identity.kind}, not ${kind}`, subject: use }];
- const diagnose = (value.facet.implementation as RuntimeEndpointAdapterImplementation).doctor;
+ assert(kind === "runtime-service", "Endpoint diagnostics belong to its activation");
+ const diagnose = (value.facet.implementation as RuntimeServiceAdapterImplementation).doctor;
return diagnose === undefined ? [] : await diagnose(context);
}
}
diff --git a/packages/runtime-adapter/test/registry.test.ts b/packages/runtime-adapter/test/registry.test.ts
index a2f84a3a..bbdcab5f 100644
--- a/packages/runtime-adapter/test/registry.test.ts
+++ b/packages/runtime-adapter/test/registry.test.ts
@@ -14,10 +14,22 @@ import {
runtimeConfigString,
RuntimeAdapterRegistry,
} from "@narratage/runtime-adapter";
+import type { RuntimeAdapterFactoryContext } from "@narratage/runtime-adapter";
const context = { root: "/tmp", instance: "one", config: {} };
+const endpointPackage = (instance: string, credentials: readonly unknown[] = []) => ({
+ name: instance,
+ manifest: { facets: [] },
+ instance: { id: instance },
+ bindings: [],
+ credentials,
+ install() {},
+}) as never;
const endpoint = (use: string, extra: Record<string, unknown> = {}) =>
- createRuntimeEndpointAdapterFacet({ use, validate() {}, create: () => ({ use }) as never, ...extra } as never);
+ createRuntimeEndpointAdapterFacet({
+ use,
+ activate: (value: RuntimeAdapterFactoryContext) => ({ endpoint: endpointPackage(value.instance), ...extra }),
+ } as never);
test("a facet declares which kind it is, and the registry keeps the two apart", () => {
const registry = new RuntimeAdapterRegistry();
@@ -42,76 +54,94 @@ test("asking an Endpoint adapter for a Runtime Service is refused, not coerced",
await assert.rejects(async () => await registry.createEndpoint("example.absent", context));
});
+test("a Runtime Service adapter cannot escape its configured instance namespace", async () => {
+ const registry = new RuntimeAdapterRegistry();
+ registry.registerFacet(createRuntimeServiceAdapterFacet({
+ use: "example.escaping-service",
+ validate() {},
+ create: () => ({ services: [{ instance: { id: "someone-else" } }] }) as never,
+ }));
+ await assert.rejects(
+ async () => await registry.createService("example.escaping-service", context),
+ /outside configured namespace one/u,
+ );
+});
+
test("one name is one adapter, so a second registration is an error rather than a winner", () => {
const registry = new RuntimeAdapterRegistry();
registry.registerFacet(endpoint("example.endpoint"));
assert.throws(() => registry.registerFacet(endpoint("example.endpoint")), /already registered/u);
});
-test("a facet without create() is not a facet", () => {
+test("an Endpoint facet without activate() is not a facet", () => {
assert.throws(
() => createRuntimeEndpointAdapterFacet({ use: "x" } as never),
- /does not implement create\(\)/u,
- );
- assert.throws(
- () => createRuntimeEndpointAdapterFacet({ use: "x", validate() {}, create: () => ({}), doctor: 1 } as never),
- /doctor must be a function/u,
+ /does not implement activate\(\)/u,
);
assert.throws(
- () => createRuntimeEndpointAdapterFacet({ use: "x", create: () => ({}) } as never),
- /does not implement validate\(\)/u,
+ () => createRuntimeEndpointAdapterFacet({ use: "x", activate: 1 } as never),
+ /does not implement activate\(\)/u,
);
assert.throws(
- () => createRuntimeEndpointAdapterFacet({ use: " ", validate() {}, create: () => ({}) } as never),
+ () => createRuntimeEndpointAdapterFacet({ use: " ", activate: () => ({}) } as never),
/use name is empty/u,
);
});
-test("validation is a separate gate and never constructs the selected adapter", async () => {
- let created = 0;
+test("one activation validates and declares the selected Endpoint", async () => {
+ let activated = 0;
const registry = new RuntimeAdapterRegistry();
registry.registerFacet(createRuntimeEndpointAdapterFacet({
use: "example.validated",
- validate(value) {
+ activate(value) {
+ activated += 1;
if (value.config !== null && typeof value.config === "object" && "bad" in value.config) {
throw new Error("bad config");
}
- },
- create: () => {
- created += 1;
- return {} as never;
+ return { endpoint: endpointPackage(value.instance) };
},
}));
- assert.deepEqual(registry.validate("example.validated", "endpoint", context), []);
- assert.equal(created, 0);
- assert.throws(
- () => registry.validate("example.validated", "endpoint", { ...context, config: { bad: true } }),
+ await assert.rejects(
+ async () => await registry.activateEndpoint("example.validated", { ...context, config: { bad: true } }),
/bad config/u,
);
- assert.equal(created, 0);
- await registry.createEndpoint("example.validated", context);
- assert.equal(created, 1);
+ const activation = await registry.activateEndpoint("example.validated", context);
+ assert.equal(activation.endpoint.instance.id, "one");
+ assert.equal(activated, 2);
+});
+
+test("credentials have one source: the activated Endpoint package", async () => {
+ const registry = new RuntimeAdapterRegistry();
+ registry.registerFacet(createRuntimeEndpointAdapterFacet({
+ use: "example.credentials",
+ activate(value) {
+ return { endpoint: endpointPackage(value.instance, [{ slot: "token" }]) };
+ },
+ }));
+ assert.deepEqual((await registry.activateEndpoint("example.credentials", context)).endpoint.credentials, [{ slot: "token" }]);
});
-test("an external service is present only when the adapter declares one", () => {
+test("an external service is present only when the activation declares one", async () => {
const registry = new RuntimeAdapterRegistry();
registry.registerFacet(endpoint("example.bare"));
registry.registerFacet(endpoint("example.declaring", {
- service: () => ({ id: "thing", probe: async () => ({ state: "ready" as const }) }),
+ externalService: { id: "thing", probe: async () => ({ state: "ready" as const }) },
}));
- assert.equal(registry.service("example.bare", context), undefined);
- assert.equal(registry.service("example.declaring", context)?.id, "thing");
+ assert.equal((await registry.activateEndpoint("example.bare", context)).externalService, undefined);
+ assert.equal((await registry.activateEndpoint("example.declaring", context)).externalService?.id, "thing");
});
-test("an adapter with no doctor reports nothing rather than failing", async () => {
+test("an activation may add diagnostics without another configuration declaration", async () => {
const registry = new RuntimeAdapterRegistry();
registry.registerFacet(endpoint("example.bare"));
registry.registerFacet(endpoint("example.diagnosing", {
- doctor: () => [{ severity: "warning" as const, code: "X", message: "m" }],
+ diagnose: () => [{ severity: "warning" as const, code: "X", message: "m" }],
}));
- assert.deepEqual(await registry.doctor("example.bare", "endpoint", context), []);
- assert.equal((await registry.doctor("example.diagnosing", "endpoint", context)).length, 1);
+ const bare = await registry.activateEndpoint("example.bare", context);
+ const diagnosing = await registry.activateEndpoint("example.diagnosing", context);
+ assert.equal(bare.diagnose, undefined);
+ assert.equal((await diagnosing.diagnose!()).length, 1);
});
test("a produced facet is recognisable as a Runtime Adapter host facet", () => {
diff --git a/packages/runtime/src/operations.ts b/packages/runtime/src/operations.ts
index 798a65b5..613cd69a 100644
--- a/packages/runtime/src/operations.ts
+++ b/packages/runtime/src/operations.ts
@@ -45,18 +45,38 @@ export type OperationCancellationControl = {
readonly lastError?: { readonly code: string; readonly message: string };
};
+/**
+ * Provider-neutral, human-observable progress for one recoverable Operation.
+ *
+ * The opaque checkpoint remains the Provider's private recovery state. This
+ * deliberately small projection is the only part Runtime tools may display;
+ * it must not carry credentials, request bodies or vendor-specific payloads.
+ */
+export type OperationProgress = {
+ readonly phase: string;
+ readonly completed?: number;
+ readonly total?: number;
+ readonly unit?: string;
+};
+
export type OperationSnapshot = OperationIdentity & {
readonly revision: number;
readonly status: "created" | "pending" | "completed" | "failed" | "cancelled";
readonly checkpoint?: CanonicalValue;
readonly wakeAt?: number;
+ readonly progress?: OperationProgress;
readonly completion?: OperationCompletion;
readonly failure?: OperationFailure;
readonly cancellation?: OperationCancellationControl;
};
export type OperationUpdate =
- | { readonly status: "pending"; readonly checkpoint: CanonicalValue; readonly wakeAt?: number }
+ | {
+ readonly status: "pending";
+ readonly checkpoint: CanonicalValue;
+ readonly wakeAt?: number;
+ readonly progress?: OperationProgress;
+ }
| {
readonly status: "completed";
readonly completion: Omit<OperationCompletion, "digest">;
@@ -188,8 +208,27 @@ export function verifyOperationSnapshot(snapshot: OperationSnapshot): void {
if (snapshot.status === "pending") {
assert(snapshot.checkpoint !== undefined && presence === 1, "pending Operation requires only a checkpoint");
if (snapshot.wakeAt !== undefined) epochMillisecond(snapshot.wakeAt, "Operation wakeAt");
+ if (snapshot.progress !== undefined) {
+ assert(snapshot.progress.phase.trim().length > 0, "Operation progress phase is empty");
+ if (snapshot.progress.completed !== undefined) {
+ assert(Number.isFinite(snapshot.progress.completed) && snapshot.progress.completed >= 0,
+ "Operation progress completed value is invalid");
+ }
+ if (snapshot.progress.total !== undefined) {
+ assert(Number.isFinite(snapshot.progress.total) && snapshot.progress.total >= 0,
+ "Operation progress total value is invalid");
+ }
+ if (snapshot.progress.completed !== undefined && snapshot.progress.total !== undefined) {
+ assert(snapshot.progress.completed <= snapshot.progress.total,
+ "Operation progress exceeds its total");
+ }
+ if (snapshot.progress.unit !== undefined) {
+ assert(snapshot.progress.unit.trim().length > 0, "Operation progress unit is empty");
+ }
+ }
} else {
assert(snapshot.wakeAt === undefined, `${snapshot.status} Operation cannot carry wakeAt`);
+ assert(snapshot.progress === undefined, `${snapshot.status} Operation cannot carry progress`);
}
if (snapshot.status === "completed") {
assert(snapshot.completion !== undefined && presence === 1, "completed Operation requires only completion");
@@ -290,6 +329,7 @@ export class MemoryOperationStore implements OperationStore {
status: current.status,
...(current.checkpoint === undefined ? {} : { checkpoint: structuredClone(current.checkpoint) }),
...(current.wakeAt === undefined ? {} : { wakeAt: current.wakeAt }),
+ ...(current.progress === undefined ? {} : { progress: structuredClone(current.progress) }),
...(current.completion === undefined ? {} : { completion: structuredClone(current.completion) }),
...(current.failure === undefined ? {} : { failure: structuredClone(current.failure) }),
cancellation: structuredClone(update.cancellation),
@@ -299,6 +339,7 @@ export class MemoryOperationStore implements OperationStore {
status: "pending" as const,
checkpoint: structuredClone(update.checkpoint),
...(update.wakeAt === undefined ? {} : { wakeAt: epochMillisecond(update.wakeAt, "Operation wakeAt") }),
+ ...(update.progress === undefined ? {} : { progress: structuredClone(update.progress) }),
}
: update.status === "completed"
? { status: "completed" as const, completion: sealOperationCompletion(update.completion) }
diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts
index 82278352..64d31ba9 100644
--- a/packages/runtime/src/types.ts
+++ b/packages/runtime/src/types.ts
@@ -99,14 +99,18 @@ export type RuntimeCommandExecutor = {
/** Content-addressed bytes. Location, retention and remote transport are adapter policy. */
export type ArtifactStore = {
+ /** Admit bytes and compute their identity in the same pass. */
put(bytes: Uint8Array, mediaType: string): Promise<BlobRef>;
+ /** Return verified bytes; a digest mismatch is an error, not a cache miss. */
get(digest: Digest): Promise<Uint8Array | undefined>;
+ /** Cheap presence query only. Integrity is checked when bytes cross get/open. */
has(digest: Digest): Promise<boolean>;
};
/** Optional transfer capability. Core and components never require storage to expose it. */
export type StreamingArtifactStore = ArtifactStore & {
putStream(chunks: AsyncIterable<Uint8Array>, mediaType: string): Promise<BlobRef>;
+ /** Stream bytes once and reject completion when their digest does not match. */
open(digest: Digest): Promise<AsyncIterable<Uint8Array> | undefined>;
};
diff --git a/packages/runtime/test/operations.test.ts b/packages/runtime/test/operations.test.ts
index c6b1b052..bba4dc94 100644
--- a/packages/runtime/test/operations.test.ts
+++ b/packages/runtime/test/operations.test.ts
@@ -47,12 +47,15 @@ test("pending checkpoints and completion advance by CAS without entering BuildSt
status: "pending",
checkpoint: { remoteJob: "job-123", pollAfterMs: 5_000 },
wakeAt: 10_000,
+ progress: { phase: "rendering", completed: 12, total: 60, unit: "frames" },
});
assert.equal(pending.status, "stored");
if (pending.status !== "stored") return;
assert.equal(pending.snapshot.status, "pending");
assert.deepEqual(pending.snapshot.checkpoint, { remoteJob: "job-123", pollAfterMs: 5_000 });
assert.equal(pending.snapshot.wakeAt, 10_000);
+ assert.deepEqual(pending.snapshot.progress,
+ { phase: "rendering", completed: 12, total: 60, unit: "frames" });
assert.deepEqual((await store.list({ build: "video-42" })).map((item) => item.id), [operation.id]);
const stale = await store.compareAndSwap(operation.id, 0, {
@@ -86,6 +89,17 @@ test("pending checkpoints and completion advance by CAS without entering BuildSt
);
});
+test("pending progress is bounded and cannot leak into terminal execution facts", async () => {
+ const store = new MemoryOperationStore();
+ const operation = identity();
+ await store.create(operation);
+ await assert.rejects(store.compareAndSwap(operation.id, 0, {
+ status: "pending",
+ checkpoint: { remoteJob: "job-123" },
+ progress: { phase: "rendering", completed: 61, total: 60, unit: "frames" },
+ }), /progress exceeds/u);
+});
+
test("OperationStore returns defensive snapshots", async () => {
const store = new MemoryOperationStore();
const operation = identity();
diff --git a/packages/store-sqlite/src/activation.ts b/packages/store-sqlite/src/activation.ts
index 86eb2257..f0198a3d 100644
--- a/packages/store-sqlite/src/activation.ts
+++ b/packages/store-sqlite/src/activation.ts
@@ -7,8 +7,6 @@ import {
} from "@narratage/runtime-adapter";
import { resolve } from "node:path";
-import { createSqliteRuntimeServicePackage } from "./store.js";
-
const sqliteStateAdapter = createRuntimeServiceAdapterFacet({
use: "@narratage/store-sqlite",
validate(context) {
@@ -17,10 +15,11 @@ const sqliteStateAdapter = createRuntimeServiceAdapterFacet({
if (runtimeConfigString(config.path, "SQLite path") === undefined) throw new Error("SQLite path is required");
runtimeConfigPositiveInteger(config.busyTimeoutMs, "SQLite busyTimeoutMs");
},
- create(context) {
+ async create(context) {
const config = runtimeConfigObject(context.config, "SQLite Runtime state");
const path = runtimeConfigString(config.path, "SQLite path");
if (path === undefined) throw new Error("SQLite path is required");
+ const { createSqliteRuntimeServicePackage } = await import("./store.js");
return createSqliteRuntimeServicePackage({
name: context.instance,
path: resolve(context.root, path),
diff --git a/packages/store-sqlite/src/store.ts b/packages/store-sqlite/src/store.ts
index ff1b0778..a9e847b5 100644
--- a/packages/store-sqlite/src/store.ts
+++ b/packages/store-sqlite/src/store.ts
@@ -163,6 +163,7 @@ function parseOperationSnapshot(row: Row): OperationSnapshot {
readonly format?: string;
readonly checkpoint?: unknown;
readonly wakeAt?: number;
+ readonly progress?: OperationSnapshot["progress"];
}
: undefined;
const mutable = row.status === "pending"
@@ -170,6 +171,7 @@ function parseOperationSnapshot(row: Row): OperationSnapshot {
? {
checkpoint: pending.checkpoint,
...(pending.wakeAt === undefined ? {} : { wakeAt: pending.wakeAt }),
+ ...(pending.progress === undefined ? {} : { progress: pending.progress }),
}
: { checkpoint: pending }
: row.status === "completed"
@@ -357,17 +359,26 @@ class SqliteOperationStore implements OperationStore {
}
async list(query: OperationQuery): Promise<readonly OperationSnapshot[]> {
+ const predicates: string[] = [];
+ const values: string[] = [];
+ for (const [field, value] of [
+ ["build", query.build],
+ ["command", query.command],
+ ["endpoint", query.endpoint],
+ ["runtimeClosure", query.runtimeClosure],
+ ["requestDigest", query.requestDigest],
+ ] as const) {
+ if (value === undefined) continue;
+ predicates.push(`json_extract(identity_json, '$.${field}') = ?`);
+ values.push(value);
+ }
const rows = this.#database.prepare(`
SELECT identity_json, revision, status, checkpoint_json, completion_json, failure_json, cancellation_json
FROM svml_operations
+ ${predicates.length === 0 ? "" : `WHERE ${predicates.join(" AND ")}`}
ORDER BY operation_id ASC
- `).all() as Row[];
+ `).all(...values) as Row[];
return rows.map(parseOperationSnapshot)
- .filter((snapshot) => query.build === undefined || snapshot.build === query.build)
- .filter((snapshot) => query.command === undefined || snapshot.command === query.command)
- .filter((snapshot) => query.endpoint === undefined || snapshot.endpoint === query.endpoint)
- .filter((snapshot) => query.runtimeClosure === undefined || snapshot.runtimeClosure === query.runtimeClosure)
- .filter((snapshot) => query.requestDigest === undefined || snapshot.requestDigest === query.requestDigest)
.sort((left, right) => left.attempt - right.attempt || left.id.localeCompare(right.id));
}
@@ -403,6 +414,7 @@ class SqliteOperationStore implements OperationStore {
format: "svml.operation-pending@1",
checkpoint: update.checkpoint,
...(update.wakeAt === undefined ? {} : { wakeAt: update.wakeAt }),
+ ...(update.progress === undefined ? {} : { progress: update.progress }),
}) : null;
const completion = update.status === "completed"
? canonicalStringify(sealOperationCompletion(update.completion))
@@ -514,13 +526,23 @@ class SqliteBuildDispatchStore implements BuildDispatchStore {
}
async list(query: DispatchQuery = {}): Promise<readonly BuildDispatchSnapshot[]> {
+ const predicates: string[] = [];
+ const values: string[] = [];
+ if (query.phases !== undefined) {
+ if (query.phases.length === 0) return [];
+ predicates.push(`phase IN (${query.phases.map(() => "?").join(", ")})`);
+ values.push(...query.phases);
+ }
+ if (query.admission !== undefined) {
+ predicates.push("admission = ?");
+ values.push(query.admission);
+ }
const rows = this.#database.prepare(`
SELECT * FROM svml_dispatches
+ ${predicates.length === 0 ? "" : `WHERE ${predicates.join(" AND ")}`}
ORDER BY priority DESC, available_at ASC, created_at ASC, build_id ASC
- `).all() as Row[];
- return rows.map(parseDispatchSnapshot)
- .filter((item) => query.phases === undefined || query.phases.includes(item.phase))
- .filter((item) => query.admission === undefined || item.admission === query.admission);
+ `).all(...values) as Row[];
+ return rows.map(parseDispatchSnapshot);
}
async claim(request: BuildDispatchClaim): Promise<BuildDispatchSnapshot | undefined> {
@@ -875,6 +897,24 @@ export class SqliteRuntimeState {
this.#database.exec(`PRAGMA busy_timeout = ${positiveInteger(options.busyTimeoutMs ?? 5_000, "busyTimeoutMs")}`);
this.#database.exec("PRAGMA journal_mode = WAL");
this.#database.exec("PRAGMA synchronous = FULL");
+ const alreadyInitialized = this.#database.prepare(`
+ SELECT 1 AS present FROM sqlite_master
+ WHERE type = 'table' AND name = 'svml_store_meta'
+ `).get() as Row | undefined;
+ if (alreadyInitialized !== undefined) {
+ const version = this.#database.prepare(
+ "SELECT schema_version FROM svml_store_meta WHERE singleton = 1",
+ ).get() as Row | undefined;
+ if (version === undefined || version.schema_version !== databaseSchemaVersion) {
+ const found = version === undefined ? "an incomplete schema" : `schema ${String(version.schema_version)}`;
+ this.#database.close();
+ throw new Error(
+ `@narratage/store-sqlite found ${found} at ${absolute}; expected schema ${databaseSchemaVersion}. `
+ + "Pre-release Runtime state is not migrated automatically. Archive that database (including its -wal and -shm files) "
+ + "or select a new SQLite path in the Runtime Profile; Artifact files are separate and are not deleted.",
+ );
+ }
+ }
this.#database.exec(`
CREATE TABLE IF NOT EXISTS svml_store_meta (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
@@ -952,12 +992,8 @@ export class SqliteRuntimeState {
CREATE INDEX IF NOT EXISTS svml_runtime_journal_build ON svml_runtime_journal (build_id, sequence);
CREATE INDEX IF NOT EXISTS svml_runtime_journal_operation ON svml_runtime_journal (operation_id, sequence);
`);
- const version = this.#database.prepare("SELECT schema_version FROM svml_store_meta WHERE singleton = 1").get() as Row | undefined;
- if (version === undefined) {
+ if (alreadyInitialized === undefined) {
this.#database.prepare("INSERT INTO svml_store_meta (singleton, schema_version) VALUES (1, ?)").run(databaseSchemaVersion);
- } else {
- assert(version.schema_version === databaseSchemaVersion,
- `unsupported @narratage/store-sqlite schema ${String(version.schema_version)}`);
}
this.builds = new SqliteBuildStore(database);
this.operations = new SqliteOperationStore(database);
diff --git a/packages/store-sqlite/test/store.test.ts b/packages/store-sqlite/test/store.test.ts
index 4a913cf8..79e74d7e 100644
--- a/packages/store-sqlite/test/store.test.ts
+++ b/packages/store-sqlite/test/store.test.ts
@@ -50,6 +50,7 @@ test("SQLite stores verified Build facts and Operation checkpoints across reopen
status: "pending",
checkpoint: { remoteJob: "job-1" },
wakeAt: 12_345,
+ progress: { phase: "generating" },
});
assert.equal(pending.status, "stored");
first.close();
@@ -62,6 +63,7 @@ test("SQLite stores verified Build facts and Operation checkpoints across reopen
assert.equal(restoredOperation?.status, "pending");
assert.deepEqual(restoredOperation?.checkpoint, { remoteJob: "job-1" });
assert.equal(restoredOperation?.wakeAt, 12_345);
+ assert.deepEqual(restoredOperation?.progress, { phase: "generating" });
assert.deepEqual((await second.operations.list({ build: "video" })).map((item) => item.id), [operation.id]);
const updated = await second.builds.compareAndSwap("video", 0, initial);
diff --git a/packages/video-cli/README.md b/packages/video-cli/README.md
index ed262845..f4b874b4 100644
--- a/packages/video-cli/README.md
+++ b/packages/video-cli/README.md
@@ -11,27 +11,33 @@ the corresponding physical package. They never grant network, credential or proc
From the repository:
```bash
-pnpm narratage lock-packages ./svml.packages.lock \
+node --run narratage -- lock-packages ./svml.packages.lock \
--package @narratage/run-markup --package @narratage/script --package @example/cards --package-root .
-pnpm narratage lock-packages ./svml.runtime-packages.lock \
+node --run narratage -- lock-packages ./svml.runtime-packages.lock \
--package @narratage/local --package @narratage/store-sqlite \
--package @narratage/artifact-store-fs --package @narratage/credential-store-env \
--package @narratage/provider-kie --package @narratage/provider-media-local --package-root .
-pnpm narratage check path/to/main.svml --package-lock ./svml.packages.lock --root .
-pnpm narratage check path/to/build.svrun --package-lock ./svml.packages.lock --root .
-pnpm narratage plan path/to/build.svrun --package-lock ./svml.packages.lock --root .
-pnpm narratage build path/to/build.svrun --package-lock ./svml.packages.lock \
+node --run narratage -- check path/to/main.svml --runtime ./svml.runtime.json --root .
+node --run narratage -- check path/to/build.svrun --runtime ./svml.runtime.json --root .
+node --run narratage -- plan path/to/build.svrun --runtime ./svml.runtime.json --root .
+node --run narratage -- build path/to/build.svrun \
--runtime ./svml.runtime.json --build-id delivery-01 --follow --root .
-pnpm narratage status <build-id> --runtime ./svml.runtime.json
-pnpm narratage builds --runtime ./svml.runtime.json
-pnpm narratage inspect <build-id> --runtime ./svml.runtime.json
-pnpm narratage get <build-id> --name final.video --runtime ./svml.runtime.json --to ./final.mp4
-pnpm narratage cancel build <build-id> --runtime ./svml.runtime.json
-pnpm narratage cancel operation <operation-id> --runtime ./svml.runtime.json
-pnpm narratage doctor ./svml.runtime.json
-pnpm narratage gc ./svml.runtime.json
+node --run narratage -- status <build-id> --runtime ./svml.runtime.json
+node --run narratage -- builds --runtime ./svml.runtime.json
+node --run narratage -- history [source-output-name] --runtime ./svml.runtime.json [--source ./main.svml]
+node --run narratage -- inspect <build-id> --runtime ./svml.runtime.json
+node --run narratage -- get <build-id> --name final.video --runtime ./svml.runtime.json --to ./final.mp4
+node --run narratage -- cancel build <build-id> --runtime ./svml.runtime.json
+node --run narratage -- cancel operation <operation-id> --runtime ./svml.runtime.json
+node --run narratage -- doctor ./svml.runtime.json
+node --run narratage -- gc ./svml.runtime.json
```
+From a separate project directory during source development, run
+`/path/to/svml/narratage ... --package-root /path/to/svml`. The root launcher resolves its own
+installed TypeScript loader and CLI, so it neither invokes pnpm nor requires the current directory
+to contain Narratage's `package.json`.
+
`--root` is only the Source Workspace containment boundary. `--package-root` is only the Host
override used to resolve the installed packages named by a lock; by default this Distribution uses
its own installation location. Keeping the two concepts separate lets a video project live outside
@@ -66,10 +72,16 @@ without a selector it requires one distinct target, while `--name`, `--record` a
a source output alias, accepted Record or demanded Logical Output. `--artifact` selects any nested
BlobRef that the Build actually references. `--to` copies an Artifact or writes an inline structured
value as JSON. It is Host egress only and never changes Build identity or retention.
+Blob egress uses the Store's streaming facet, verifies the declared size and content digest while
+writing a temporary file, then atomically replaces the requested destination. A large video is not
+loaded into CLI memory and a corrupt stream cannot overwrite an existing export.
`builds` and source aliases come from an optional Host `BuildCatalog`. The Catalog contains paths
and presentation names only. It is not Core truth or a Runtime Closure facet; `inspect` and `get`
always resolve the alias back through the verified BuildState before accepting it.
+`history` searches those frozen Catalog names across Builds, but reports only Logical Outputs whose
+selected Record is present in verified BuildState. It neither infers renames nor lists unbuilt
+aliases; an old Catalog name and a current output name are connected explicitly in the Run Source.
Historical Records, fixed files and generated previews are declared as ordinary Candidates in the
Run Source and selected by explicit Satisfaction edges. The Host verifies a referenced prior Build
diff --git a/packages/video-cli/src/cli.ts b/packages/video-cli/src/cli.ts
index bc7d1091..4f77787b 100644
--- a/packages/video-cli/src/cli.ts
+++ b/packages/video-cli/src/cli.ts
@@ -46,6 +46,7 @@ async function readSecret(prompt: string): Promise<string> {
const io: CliIo = {
write: (text) => process.stdout.write(text),
+ setExitCode: (code) => { process.exitCode = code; },
readSecret,
terminal: {
isTTY: process.stdout.isTTY === true,
diff --git a/packages/video-cli/src/distribution.ts b/packages/video-cli/src/distribution.ts
index 11dfcb73..2f0bfdb8 100644
--- a/packages/video-cli/src/distribution.ts
+++ b/packages/video-cli/src/distribution.ts
@@ -1,5 +1,6 @@
import type { CliDistribution } from "@narratage/cli";
import { fileURLToPath } from "node:url";
+import { extname } from "node:path";
import {
createVideoCompiler,
videoBuiltInPackageContributions,
@@ -14,13 +15,30 @@ export const videoCliDistribution: CliDistribution = {
builtInPackageContributions: videoBuiltInPackageContributions,
runFrontends: [],
createCompiler: createVideoCompiler,
+ resolveCompilationPackages: async (path) => {
+ if (extname(path) !== ".json") return {};
+ const { runtimeConfigPackageSelection } = await import("@narratage/local/config");
+ const selection = await runtimeConfigPackageSelection(path, { packageRoot });
+ return {
+ ...(selection.packageLock === undefined ? {} : { packageLock: selection.packageLock }),
+ packageRoot: selection.packageRoot,
+ };
+ },
runtimeWorkerLaunch: () => ({
command: process.execPath,
args: [...process.execArgv, fileURLToPath(new URL("./cli.ts", import.meta.url))],
}),
- createRuntimeFromConfig: async (path) => {
+ createRuntimeFromConfig: async (path, options) => {
const { createVideoRuntimeFromConfig } = await import("./runtime-config.js");
- return await createVideoRuntimeFromConfig(path);
+ return await createVideoRuntimeFromConfig(path, packageRoot, options?.implementationPackages);
+ },
+ createRuntimeControlFromConfig: async (path) => {
+ const { createRuntimeControlFromConfig } = await import("@narratage/local/config");
+ return await createRuntimeControlFromConfig(path, { packageRoot });
+ },
+ createRuntimeCredentialsFromConfig: async (path, endpoint) => {
+ const { createRuntimeCredentialsFromConfig } = await import("@narratage/local/config");
+ return await createRuntimeCredentialsFromConfig(path, endpoint, { packageRoot });
},
doctorRuntimeConfig: async (path) => {
const { doctorRuntimeConfig } = await import("@narratage/local/config");
diff --git a/packages/video-cli/src/runtime-config.ts b/packages/video-cli/src/runtime-config.ts
index e331840f..9a4283a8 100644
--- a/packages/video-cli/src/runtime-config.ts
+++ b/packages/video-cli/src/runtime-config.ts
@@ -1,9 +1,17 @@
import { createRuntimeFromConfig } from "@narratage/local";
+import type { LoadedNodePackageSet } from "@narratage/package-loader-node";
/**
* Video CLI assembly adds no privileged Provider registry. The Runtime Profile selects
* locked Runtime Adapter packages, so installing a new adapter never changes this CLI.
*/
-export async function createVideoRuntimeFromConfig(path: string, packageRoot = import.meta.dirname) {
- return await createRuntimeFromConfig(path, { packageRoot });
+export async function createVideoRuntimeFromConfig(
+ path: string,
+ packageRoot = import.meta.dirname,
+ implementationPackages?: LoadedNodePackageSet,
+) {
+ return await createRuntimeFromConfig(path, {
+ packageRoot,
+ ...(implementationPackages === undefined ? {} : { implementationPackages }),
+ });
}
diff --git a/packages/video-cli/test/cli.test.ts b/packages/video-cli/test/cli.test.ts
index 329f064b..4326d0fb 100644
--- a/packages/video-cli/test/cli.test.ts
+++ b/packages/video-cli/test/cli.test.ts
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
-import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
@@ -321,8 +321,11 @@ test("CLI accepts a declarative Runtime Profile without an executable config mod
readonly ok: boolean;
readonly diagnostics: readonly { readonly code: string; readonly subject?: string }[];
};
- assert.equal(diagnosis.ok, true);
- assert.equal(diagnosis.diagnostics.length, 0);
+ assert.equal(diagnosis.ok, false);
+ assert.deepEqual(diagnosis.diagnostics.map((item) => ({ code: item.code, subject: item.subject })), [{
+ code: "RUNTIME_CREDENTIAL_MISSING",
+ subject: "kie.cli-test.apiKey",
+ }]);
output = "";
await runCli(["auth", "status", "kie.cli-test", "--runtime", profile], {
write: (text) => { output += text; },
@@ -380,9 +383,11 @@ test("CLI package lock activates an installed package without changing the offic
const directory = await mkdtemp(join(tmpdir(), "svml-cli-package-lock-"));
const projectRoot = join(directory, "external-video-project");
const installedPackage = join(directory, "narratage-install", "node_modules", "example-empty");
+ const installedExtra = join(directory, "narratage-install", "node_modules", "example-extra");
const packageRoot = join(directory, "narratage-install");
await mkdir(projectRoot, { recursive: true });
await mkdir(installedPackage, { recursive: true });
+ await mkdir(installedExtra, { recursive: true });
const implementationDigest = `sha256:${"2".repeat(64)}`;
await writeFile(join(installedPackage, "package.json"), JSON.stringify({
name: "example-empty",
@@ -411,6 +416,16 @@ test("CLI package lock activates an installed package without changing the offic
}],
};
`, "utf8");
+ await writeFile(join(installedExtra, "package.json"), JSON.stringify({
+ name: "example-extra",
+ version: "1.0.0",
+ type: "module",
+ exports: "./activation.mjs",
+ svml: { activation: "./activation.mjs" },
+ }), "utf8");
+ await writeFile(join(installedExtra, "activation.mjs"), `
+ export default { format: "svml.node-package@1", name: "example-extra" };
+ `, "utf8");
const file = join(projectRoot, "main.svml");
const lockPath = join(projectRoot, "svml.packages.lock");
await writeFile(file, `<?svml using="@narratage/markup@1"?>
@@ -437,6 +452,139 @@ test("CLI package lock activates an installed package without changing the offic
const checked = JSON.parse(checkOutput) as { readonly ok: boolean; readonly modules: readonly string[] };
assert.equal(checked.ok, true);
assert.deepEqual(checked.modules, ["example.empty@1"]);
+
+ const editSentinel = `${lockPath}.editing`;
+ await writeFile(editSentinel, JSON.stringify({ pid: 999, host: "test" }), "utf8");
+ await assert.rejects(
+ async () => await runCli(["lock-packages", lockPath, "--add", "example-extra", "--package-root", packageRoot], {
+ write() {},
+ }),
+ /already being edited.*pid.*999/u,
+ );
+ await runCli(["lock-packages", lockPath, "--verify", "--package-root", packageRoot], { write() {} });
+ await rm(editSentinel);
+
+ const trustedActivation = await readFile(join(installedPackage, "activation.mjs"), "utf8");
+ const trustedLockDigest = (JSON.parse(await readFile(lockPath, "utf8")) as { readonly digest: string }).digest;
+ await writeFile(join(installedPackage, "activation.mjs"), `${trustedActivation}\n// unreviewed retained change\n`, "utf8");
+ await assert.rejects(
+ async () => await runCli(["lock-packages", lockPath, "--add", "example-extra", "--package-root", packageRoot], {
+ write() {},
+ }),
+ /retained package bytes changed.*use --refresh or an exact --package selection/u,
+ );
+ assert.equal((JSON.parse(await readFile(lockPath, "utf8")) as { readonly digest: string }).digest, trustedLockDigest);
+ await writeFile(join(installedPackage, "activation.mjs"), trustedActivation, "utf8");
+
+ let mutationOutput = "";
+ await runCli(["lock-packages", lockPath, "--add", "example-extra", "--package-root", packageRoot], {
+ write: (text) => { mutationOutput += text; },
+ });
+ const added = JSON.parse(mutationOutput) as {
+ readonly changed: boolean;
+ readonly mode: string;
+ readonly selected: readonly string[];
+ readonly added: readonly string[];
+ readonly closureChanges: {
+ readonly physical: { readonly added: readonly string[]; readonly removed: readonly string[] };
+ readonly activated: { readonly added: readonly string[]; readonly removed: readonly string[] };
+ };
+ };
+ assert.equal(added.changed, true);
+ assert.equal(added.mode, "mutate");
+ assert.deepEqual(added.selected, ["example-empty", "example-extra"]);
+ assert.deepEqual(added.added, ["example-extra"]);
+ assert.deepEqual(added.closureChanges, {
+ physical: { added: ["[email protected]"], removed: [] },
+ activated: { added: ["[email protected]"], removed: [] },
+ });
+ assert.equal((await readdir(projectRoot)).some((name) => name.endsWith(".editing")), false);
+
+ mutationOutput = "";
+ await runCli(["lock-packages", lockPath, "--add", "example-extra", "--package-root", packageRoot], {
+ write: (text) => { mutationOutput += text; },
+ });
+ const unchanged = JSON.parse(mutationOutput) as { readonly changed: boolean; readonly digest: string };
+ assert.equal(unchanged.changed, false);
+
+ let verifyOutput = "";
+ await runCli(["lock-packages", lockPath, "--verify", "--package-root", packageRoot], {
+ write: (text) => { verifyOutput += text; },
+ });
+ const verified = JSON.parse(verifyOutput) as { readonly ok: boolean; readonly mode: string; readonly digest: string };
+ assert.equal(verified.ok, true);
+ assert.equal(verified.mode, "verify");
+ assert.equal(verified.digest, unchanged.digest);
+
+ await assert.rejects(
+ async () => await runCli(["lock-packages", lockPath, "--remove", "example-missing", "--package-root", packageRoot], {
+ write() {},
+ }),
+ /example-missing is not selected.*no package authority was removed/u,
+ );
+ assert.equal((JSON.parse(await readFile(lockPath, "utf8")) as { readonly digest: string }).digest, unchanged.digest);
+
+ mutationOutput = "";
+ await writeFile(join(installedExtra, "activation.mjs"), `
+ export default { format: "svml.node-package@1", name: "example-extra" };
+ // Changed bytes do not prevent revoking this now-unreachable root.
+ `, "utf8");
+ await runCli(["lock-packages", lockPath, "--remove", "example-extra", "--package-root", packageRoot], {
+ write: (text) => { mutationOutput += text; },
+ });
+ const removed = JSON.parse(mutationOutput) as {
+ readonly changed: boolean;
+ readonly selected: readonly string[];
+ readonly removed: readonly string[];
+ };
+ assert.equal(removed.changed, true);
+ assert.deepEqual(removed.selected, ["example-empty"]);
+ assert.deepEqual(removed.removed, ["example-extra"]);
+
+ await assert.rejects(
+ async () => await runCli([
+ "lock-packages", lockPath,
+ "--add", "example-extra", "--remove", "example-extra",
+ "--package-root", packageRoot,
+ ], { write() {} }),
+ /cannot be added and removed in the same package-lock transaction/u,
+ );
+
+ let replaceOutput = "";
+ await runCli([
+ "lock-packages", lockPath,
+ "--package", "example-empty", "--package", "example-extra",
+ "--package-root", packageRoot,
+ ], { write: (text) => { replaceOutput += text; } });
+ const replaced = JSON.parse(replaceOutput) as { readonly mode: string; readonly selected: readonly string[] };
+ assert.equal(replaced.mode, "replace");
+ assert.deepEqual(replaced.selected, ["example-empty", "example-extra"]);
+
+ await runCli([
+ "lock-packages", lockPath,
+ "--package", "example-empty",
+ "--package-root", packageRoot,
+ ], { write() {} });
+
+ const originalLock = JSON.parse(await readFile(lockPath, "utf8")) as { readonly digest: string };
+ await writeFile(join(installedPackage, "activation.mjs"),
+ `${await readFile(join(installedPackage, "activation.mjs"), "utf8")}\n// refreshed physical bytes\n`, "utf8");
+ let refreshOutput = "";
+ await runCli(["lock-packages", lockPath, "--refresh", "--package-root", packageRoot], {
+ write: (text) => { refreshOutput += text; },
+ });
+ const refreshed = JSON.parse(refreshOutput) as { readonly ok: boolean; readonly digest: string };
+ assert.equal(refreshed.ok, true);
+ assert.notEqual(refreshed.digest, originalLock.digest);
+
+ let emptyOutput = "";
+ await runCli(["lock-packages", lockPath, "--remove", "example-empty", "--package-root", packageRoot], {
+ write: (text) => { emptyOutput += text; },
+ });
+ const empty = JSON.parse(emptyOutput) as { readonly selected: readonly string[]; readonly packages: readonly unknown[] };
+ assert.deepEqual(empty.selected, []);
+ assert.deepEqual(empty.packages, []);
+ await runCli(["lock-packages", lockPath, "--verify", "--package-root", packageRoot], { write() {} });
});
test("materializeRecord copies an archived Artifact without requiring a completed Build", async () => {
@@ -445,7 +593,9 @@ test("materializeRecord copies an archived Artifact without requiring a complete
const artifactDigest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
const output = join(root, "out", "final.mp4");
const runtime = {
- async readArtifact(digest: string) { return digest === artifactDigest ? bytes : undefined; },
+ async openArtifact(digest: string) {
+ return digest === artifactDigest ? (async function* () { yield bytes; })() : undefined;
+ },
} as Parameters<typeof materializeRecord>[0];
const record = {
id: "final.video",
@@ -469,12 +619,32 @@ test("materializeRecord writes structured intermediate facts as JSON", async ()
id: "whisperx.evidence",
value: { kind: "inline", value: { words: [{ text: "hello", start: 0, end: 0.4 }] } },
} as unknown as Parameters<typeof materializeRecord>[1];
- const result = await materializeRecord({ async readArtifact() { return undefined; } }, record, output);
+ const result = await materializeRecord({ async openArtifact() { return undefined; } }, record, output);
assert.equal(result.kind, "json");
if (record.value.kind !== "inline") assert.fail("fixture must be inline");
assert.deepEqual(JSON.parse(await readFile(output, "utf8")), record.value.value);
});
+test("streaming materialization verifies bytes before replacing an existing destination", async () => {
+ const root = await mkdtemp(join(tmpdir(), "svml-cli-get-verified-"));
+ const output = join(root, "final.mp4");
+ const expected = Buffer.from("expected");
+ const corrupt = Buffer.from("corrupt!");
+ const artifactDigest = `sha256:${createHash("sha256").update(expected).digest("hex")}`;
+ await writeFile(output, "keep-me", "utf8");
+ const record = {
+ id: "final.video",
+ value: { kind: "inline", value: { digest: artifactDigest, size: expected.byteLength, mediaType: "video/mp4" } },
+ } as unknown as Parameters<typeof materializeRecord>[1];
+
+ await assert.rejects(async () => await materializeRecord({
+ async openArtifact() { return (async function* () { yield corrupt; })(); },
+ }, record, output), /bytes differ/u);
+
+ assert.equal(await readFile(output, "utf8"), "keep-me");
+ assert.deepEqual(await readdir(root), ["final.mp4"]);
+});
+
test("CLI inspect and get read the durable Build archive independently of build execution", async () => {
const root = await mkdtemp(join(tmpdir(), "svml-cli-archive-"));
const runtimePath = join(root, "runtime.mjs");
@@ -541,6 +711,10 @@ test("CLI inspect and get read the durable Build archive independently of build
name: "timing.rawEvidence",
type: state.records[1]!.type,
ref: { kind: "record", id: "record:whisperx" },
+ }, {
+ name: "unused.image",
+ type: state.records[0]!.type,
+ ref: { kind: "logical-output", id: "logical:unused" },
}],
createdAt: 100,
updatedAt: 100,
@@ -565,7 +739,7 @@ export default {
async credentials() { return []; },
async putCredential() { throw new Error("not used"); },
async deleteCredential() { throw new Error("not used"); },
- async queue() { return { dispatches: [dispatch], capacity: [] }; },
+ async queue() { return { dispatches: [dispatch], capacity: [], operations: [] }; },
async operation(id) { return id === operation.id ? operation : undefined; },
async cancelOperation(id) { return id === operation.id ? { ...operation, cancellation: { status: "requested" } } : undefined; },
async status(id) { return { build: id === "archive-1" ? { build: id, revision: 7, state: ${JSON.stringify(state)} } : undefined, catalog: id === "archive-1" ? catalog : undefined, operations: id === "archive-1" ? [operation] : [], dispatch: id === "archive-1" ? dispatch : undefined }; },
@@ -574,6 +748,11 @@ export default {
if (digest === ${JSON.stringify(rawDigest)}) return rawBytes;
return undefined;
},
+ async openArtifact(digest) {
+ const value = digest === ${JSON.stringify(artifactDigest)} ? bytes
+ : digest === ${JSON.stringify(rawDigest)} ? rawBytes : undefined;
+ return value === undefined ? undefined : (async function* () { yield value; })();
+ },
async close() {},
};\n`, "utf8");
@@ -595,6 +774,7 @@ export default {
assert.deepEqual(inspected.archive.presentation.aliases.map((item) => [item.name, item.accepted]), [
["final.video", true],
["timing.rawEvidence", true],
+ ["unused.image", false],
]);
let buildsOutput = "";
@@ -614,10 +794,44 @@ export default {
status: "active",
source: catalog.source,
run: catalog.run,
- aliasCount: 2,
+ aliasCount: 3,
targets: ["final.video"],
}]);
+ let historyOutput = "";
+ await runCli(["history", "final.video", "--runtime", runtimePath], {
+ write: (text) => { historyOutput += text; },
+ });
+ const history = JSON.parse(historyOutput) as {
+ readonly query: { readonly output: string };
+ readonly entries: readonly {
+ readonly build: string;
+ readonly source: { readonly path: string };
+ readonly output: { readonly name: string; readonly record: { readonly digest: string } };
+ }[];
+ };
+ assert.equal(history.query.output, "final.video");
+ assert.equal(history.entries.length, 1);
+ assert.equal(history.entries[0]?.build, "archive-1");
+ assert.equal(history.entries[0]?.source.path, catalog.source.path);
+ assert.equal(history.entries[0]?.output.name, "final.video");
+ assert.equal(history.entries[0]?.output.record.digest, recordDigest);
+
+ let sourceHistoryOutput = "";
+ await runCli(["history", "--source", catalog.source.path, "--runtime", runtimePath], {
+ write: (text) => { sourceHistoryOutput += text; },
+ });
+ const sourceHistory = JSON.parse(sourceHistoryOutput) as {
+ readonly entries: readonly { readonly output: { readonly name: string } }[];
+ };
+ assert.deepEqual(sourceHistory.entries.map((item) => item.output.name), ["final.video"]);
+
+ let absentHistoryOutput = "";
+ await runCli(["history", "unused.image", "--runtime", runtimePath], {
+ write: (text) => { absentHistoryOutput += text; },
+ });
+ assert.deepEqual((JSON.parse(absentHistoryOutput) as { readonly entries: readonly unknown[] }).entries, []);
+
let statusOutput = "";
await runCli(["status", "archive-1", "--runtime", runtimePath], {
write: (text) => { statusOutput += text; },
@@ -628,7 +842,7 @@ export default {
assert.deepEqual(status.catalog, {
source: catalog.source,
run: catalog.run,
- aliasCount: 2,
+ aliasCount: 3,
targets: ["final.video"],
});
diff --git a/packages/workspace-fs-node/src/workspace.ts b/packages/workspace-fs-node/src/workspace.ts
index 5a4d5c2d..f48fc208 100644
--- a/packages/workspace-fs-node/src/workspace.ts
+++ b/packages/workspace-fs-node/src/workspace.ts
@@ -1,4 +1,5 @@
import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
import { readFile, realpath } from "node:fs/promises";
import { dirname, isAbsolute, relative, resolve } from "node:path";
@@ -25,7 +26,7 @@ class NodeFilesystemWorkspaceSession implements WorkspaceSession {
readonly entry: AuthorSourceUnit;
readonly #sourceCache = new Map<string, AuthorSourceUnit>();
readonly #sourceEdges = new Map<string, AuthorSourceUnit>();
- readonly #assetBytes = new Map<string, Uint8Array>();
+ readonly #assetIdentity = new Map<string, { readonly digest: BlobRef["digest"]; readonly size: number }>();
readonly #assetEdges = new Map<string, string>();
readonly #attachments = new Map<string, ArtifactAttachment>();
@@ -122,20 +123,29 @@ class NodeFilesystemWorkspaceSession implements WorkspaceSession {
}
this.#assetEdges.set(edge, canonical);
}
- let bytes = this.#assetBytes.get(canonical);
- if (bytes === undefined) {
- bytes = Uint8Array.from(await readFile(canonical));
- this.#assetBytes.set(canonical, bytes);
+ let identity = this.#assetIdentity.get(canonical);
+ if (identity === undefined) {
+ const hash = createHash("sha256");
+ let size = 0;
+ for await (const chunk of createReadStream(canonical)) {
+ hash.update(chunk);
+ size += chunk.byteLength;
+ }
+ identity = { digest: `sha256:${hash.digest("hex")}`, size };
+ this.#assetIdentity.set(canonical, identity);
}
const artifact: BlobRef = {
kind: "blob",
- digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
- size: bytes.byteLength,
+ digest: identity.digest,
+ size: identity.size,
mediaType: request.mediaType,
};
const attachmentKey = `${artifact.digest}\u0000${artifact.mediaType}`;
if (!this.#attachments.has(attachmentKey)) {
- this.#attachments.set(attachmentKey, { artifact: { ...artifact }, bytes: Uint8Array.from(bytes) });
+ this.#attachments.set(attachmentKey, {
+ artifact: { ...artifact },
+ open: () => createReadStream(canonical),
+ });
}
return { artifact: { ...artifact } };
};
@@ -146,7 +156,7 @@ class NodeFilesystemWorkspaceSession implements WorkspaceSession {
const byDigest = left.artifact.digest.localeCompare(right.artifact.digest);
return byDigest === 0 ? left.artifact.mediaType.localeCompare(right.artifact.mediaType) : byDigest;
})
- .map((item) => ({ artifact: { ...item.artifact }, bytes: Uint8Array.from(item.bytes) }));
+ .map((item) => ({ artifact: { ...item.artifact }, open: item.open }));
}
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d828eb55..58a3b309 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -14,6 +14,9 @@ importers:
'@narratage/artifact-store-fs':
specifier: workspace:*
version: link:packages/artifact-store-fs
+ '@narratage/artifact-store-s3':
+ specifier: workspace:*
+ version: link:packages/artifact-store-s3
'@narratage/audio-track':
specifier: workspace:*
version: link:packages/audio-track
@@ -53,9 +56,18 @@ importers:
'@narratage/fonts-open':
specifier: workspace:*
version: link:packages/fonts-open
+ '@narratage/gemini-omni':
+ specifier: workspace:*
+ version: link:packages/gemini-omni
'@narratage/generation':
specifier: workspace:*
version: link:packages/generation
+ '@narratage/gpt-image':
+ specifier: workspace:*
+ version: link:packages/gpt-image
+ '@narratage/grok-imagine':
+ specifier: workspace:*
+ version: link:packages/grok-imagine
'@narratage/hyperframes':
specifier: workspace:*
version: link:packages/hyperframes
@@ -80,6 +92,12 @@ importers:
'@narratage/mimo-tts':
specifier: workspace:*
version: link:packages/mimo-tts
+ '@narratage/minimax-h3':
+ specifier: workspace:*
+ version: link:packages/minimax-h3
+ '@narratage/nano-banana':
+ specifier: workspace:*
+ version: link:packages/nano-banana
'@narratage/narrative':
specifier: workspace:*
version: link:packages/narrative
@@ -92,6 +110,9 @@ importers:
'@narratage/provider-google-vertex':
specifier: workspace:*
version: link:packages/provider-google-vertex
+ '@narratage/provider-hyperframes-aws-lambda':
+ specifier: workspace:*
+ version: link:packages/provider-hyperframes-aws-lambda
'@narratage/provider-hyperframes-local':
specifier: workspace:*
version: link:packages/provider-hyperframes-local
@@ -101,6 +122,9 @@ importers:
'@narratage/provider-kie':
specifier: workspace:*
version: link:packages/provider-kie
+ '@narratage/provider-media-aws-lambda':
+ specifier: workspace:*
+ version: link:packages/provider-media-aws-lambda
'@narratage/provider-media-local':
specifier: workspace:*
version: link:packages/provider-media-local
@@ -140,6 +164,9 @@ importers:
'@narratage/seedance-kits':
specifier: workspace:*
version: link:packages/seedance-kits
+ '@narratage/seedream':
+ specifier: workspace:*
+ version: link:packages/seedream
'@narratage/semantic-map':
specifier: workspace:*
version: link:packages/semantic-map
@@ -1708,6 +1735,9 @@ importers:
'@narratage/render-hyperframes':
specifier: workspace:*
version: link:../render-hyperframes
+ '@narratage/runtime':
+ specifier: workspace:*
+ version: link:../runtime
'@narratage/runtime-adapter':
specifier: workspace:*
version: link:../runtime-adapter
@@ -1898,6 +1928,9 @@ importers:
'@narratage/protocol':
specifier: workspace:*
version: link:../protocol
+ '@narratage/runtime':
+ specifier: workspace:*
+ version: link:../runtime
'@narratage/runtime-adapter':
specifier: workspace:*
version: link:../runtime-adapter
diff --git a/services/media-lambda/src/handler.ts b/services/media-lambda/src/handler.ts
index 3b476684..607c5332 100644
--- a/services/media-lambda/src/handler.ts
+++ b/services/media-lambda/src/handler.ts
@@ -1,5 +1,4 @@
-import { s3ArtifactKey } from "@narratage/artifact-store-s3";
-import { AwsS3ObjectClient } from "@narratage/artifact-store-s3";
+import { AwsS3ObjectClient, S3ArtifactStore } from "@narratage/artifact-store-s3";
import type { S3ObjectClient } from "@narratage/artifact-store-s3";
import {
executeInspectMedia,
@@ -13,7 +12,8 @@ import {
} from "@narratage/media-execution";
import type { MediaExecutionEnvironment, MediaOperationResult } from "@narratage/media-execution";
import { execFile } from "node:child_process";
-import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { readFile } from "node:fs/promises";
import { promisify } from "node:util";
import type { BlobRef, CanonicalValue, Digest } from "@narratage/protocol";
import {
@@ -62,30 +62,27 @@ function positiveInteger(value: string | undefined, fallback: number): number {
* agree rather than conflict.
*/
function gateway(location: MediaLambdaArtifactLocation, client: S3ObjectClient) {
+ const store = new S3ArtifactStore({
+ bucket: location.bucket,
+ ...(location.prefix === undefined ? {} : { prefix: location.prefix }),
+ client,
+ });
return {
async get(source: BlobRef): Promise<Uint8Array | undefined> {
- return await client.get({ Bucket: location.bucket, Key: s3ArtifactKey(location.prefix, source.digest) });
+ return await store.get(source.digest);
+ },
+ async open(source: BlobRef): Promise<AsyncIterable<Uint8Array> | undefined> {
+ if (store.open !== undefined) return await store.open(source.digest);
+ const bytes = await store.get(source.digest);
+ return bytes === undefined ? undefined : (async function* () { yield bytes; })();
},
async put(bytes: Uint8Array, mediaType: string): Promise<BlobRef> {
- const copy = Uint8Array.from(bytes);
- const digest = `sha256:${createHash("sha256").update(copy).digest("hex")}` as Digest;
- try {
- await client.put({
- Bucket: location.bucket,
- Key: s3ArtifactKey(location.prefix, digest),
- Body: copy,
- ContentLength: copy.byteLength,
- ContentType: mediaType,
- IfNoneMatch: "*",
- Metadata: { "svml-digest": digest, "svml-size": String(copy.byteLength) },
- });
- } catch (error) {
- const status = (error as { $metadata?: { httpStatusCode?: number } })?.$metadata?.httpStatusCode;
- // 412: these exact bytes are already stored. Under content addressing
- // that is agreement, not conflict.
- if (status !== 412) throw error;
- }
- return { kind: "blob", digest, size: copy.byteLength, mediaType };
+ return await store.put(bytes, mediaType);
+ },
+ async putFile(path: string, mediaType: string): Promise<BlobRef> {
+ return store.putStream === undefined
+ ? await store.put(await readFile(path), mediaType)
+ : await store.putStream(createReadStream(path), mediaType);
},
};
}
diff --git a/tools/caption-playground/src/server.ts b/tools/caption-playground/src/server.ts
index 12fa746e..c7467738 100644
--- a/tools/caption-playground/src/server.ts
+++ b/tools/caption-playground/src/server.ts
@@ -343,7 +343,8 @@ export function captionPlaygroundPlugin(options: CaptionPlaygroundOptions): Plug
if (attachment === undefined) { response.statusCode = 404; response.end(); return; }
response.setHeader("content-type", attachment.artifact.mediaType);
response.setHeader("cache-control", "no-store");
- response.end(Buffer.from(attachment.bytes));
+ for await (const chunk of await attachment.open()) response.write(chunk);
+ response.end();
return;
}
const familyId = /^\/__caption\/font-css\/([a-z0-9-]+)$/u.exec(url.pathname)?.[1];
diff --git a/tools/postinstall-services.mjs b/tools/postinstall-services.mjs
deleted file mode 100644
index 9e529452..00000000
--- a/tools/postinstall-services.mjs
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * Prepares the external programs that installed Providers declare.
- *
- * A Provider that needs a separate program — WhisperX, OpenCV — states that in
- * its own `service.ts`, and marks itself with `"service": true` under `svml` in
- * its package.json. Nothing here names a Provider, so adding one is a change to
- * that package alone.
- *
- * The prepare command is read from the declaration rather than restated here.
- * Two statements of "how this environment is built" drift, and the copy that
- * drifts is the one nobody runs during a Build.
- *
- * This runs from `pnpm install` and must never fail it. A missing `uv`, a
- * Provider whose activation will not import, a prepare that exits non-zero —
- * each is reported and stepped over. What it prepares is a convenience; what it
- * must not do is stop someone from installing the repository.
- *
- * Reading a declaration means importing TypeScript, so that half runs in a
- * child under tsx (`--declare`, printing JSON). A child that dies takes its
- * failure with it.
- */
-import { spawn } from "node:child_process";
-import { readdir, readFile } from "node:fs/promises";
-import { fileURLToPath } from "node:url";
-
-const packageRoot = new URL("../packages/", import.meta.url);
-const repositoryRoot = fileURLToPath(new URL("../", import.meta.url));
-
-/** Packages that declare an external program, as `{ name, activation }`. */
-async function declaringPackages() {
- const found = [];
- for (const entry of await readdir(packageRoot, { withFileTypes: true })) {
- if (!entry.isDirectory()) continue;
- let manifest;
- try {
- manifest = JSON.parse(await readFile(new URL(`./${entry.name}/package.json`, packageRoot), "utf8"));
- } catch {
- continue;
- }
- if (manifest.svml?.service !== true) continue;
- const activation = manifest.svml?.activation;
- if (typeof activation !== "string") {
- console.warn(` ${manifest.name} marks itself as a service but declares no activation`);
- continue;
- }
- found.push({
- name: manifest.name,
- activation: fileURLToPath(new URL(`./${entry.name}/${activation.replace(/^\.\//u, "")}`, packageRoot)),
- });
- }
- return found.sort((a, b) => a.name.localeCompare(b.name));
-}
-
-/**
- * Streams stay separate: the declaration child answers on stdout, and Node's
- * own deprecation notices arrive on stderr. Merging them would leave the JSON
- * unparseable for a reason that has nothing to do with what was declared.
- */
-function run(command, args, options = {}) {
- return new Promise((resolve) => {
- const child = spawn(command, args, { cwd: repositoryRoot, shell: false, ...options });
- let out = "";
- let err = "";
- if (options.stdio === undefined) {
- child.stdout.on("data", (chunk) => { out += chunk.toString(); });
- child.stderr.on("data", (chunk) => { err += chunk.toString(); });
- }
- child.on("error", (error) => resolve({ ok: false, out: "", err: error.message }));
- child.on("close", (code) => resolve({ ok: code === 0, out: out.trim(), err: err.trim(), code }));
- });
-}
-
-function lastLine(text) {
- return text.split("\n").filter((line) => line.trim().length > 0).at(-1) ?? "unknown error";
-}
-
-/**
- * Asks a child running under tsx what each package declares. The parent stays
- * plain JavaScript so that a repository without tsx built yet still installs.
- */
-async function declarations(packages) {
- const result = await run(process.execPath, [
- "--import", "tsx",
- fileURLToPath(import.meta.url),
- "--declare",
- ...packages.map((item) => `${item.name}=${item.activation}`),
- ]);
- if (!result.ok) {
- console.warn(` could not read service declarations: ${lastLine(result.err)}`);
- return [];
- }
- try {
- return JSON.parse(result.out);
- } catch {
- console.warn(" service declarations were not readable JSON");
- return [];
- }
-}
-
-/** Imports the activations named on argv and prints what they declare. */
-async function emitDeclarations(specifiers) {
- const declared = [];
- for (const specifier of specifiers) {
- const separator = specifier.indexOf("=");
- const name = specifier.slice(0, separator);
- try {
- const module = await import(specifier.slice(separator + 1));
- const facets = (module.default ?? module.svmlPackage)?.hostFacets ?? [];
- for (const facet of facets) {
- // A Provider states its program for a Runtime Profile it has not seen
- // yet. An empty config is what a fresh clone has, and yields the
- // defaults this repository ships.
- const service = facet.implementation?.service?.({
- root: repositoryRoot,
- instance: "",
- config: {},
- });
- if (service?.prepare === undefined) continue;
- declared.push({
- package: name,
- id: service.id,
- prepare: { command: service.prepare.command, args: [...service.prepare.args] },
- });
- }
- } catch (error) {
- declared.push({ package: name, failure: error instanceof Error ? error.message : String(error) });
- }
- }
- process.stdout.write(JSON.stringify(declared));
-}
-
-async function main() {
- const packages = await declaringPackages();
- if (packages.length === 0) return;
-
- console.log(`Preparing external services for ${packages.length} Provider${packages.length === 1 ? "" : "s"}`);
- const declared = await declarations(packages);
-
- // One program may be declared by more than one Provider; prepare it once.
- const unique = new Map();
- for (const item of declared) {
- if (item.failure !== undefined) {
- console.warn(` ${item.package}: ${item.failure}`);
- continue;
- }
- if (!unique.has(item.id)) unique.set(item.id, item);
- }
-
- const missing = new Set();
- let prepared = 0;
- for (const item of unique.values()) {
- const { command, args } = item.prepare;
- if (missing.has(command) || !(await run(command, ["--version"])).ok) {
- missing.add(command);
- console.warn(` ${item.id}: skipped, ${command} is not on PATH`);
- continue;
- }
- console.log(` ${item.id}: ${command} ${args.join(" ")}`);
- const result = await run(command, args, { stdio: "inherit" });
- if (result.ok) prepared += 1;
- else console.warn(` ${item.id}: prepare exited ${result.code ?? "abnormally"}; run it by hand to see why`);
- }
-
- const skipped = unique.size - prepared;
- console.log(`Prepared ${prepared} service${prepared === 1 ? "" : "s"}${skipped > 0 ? `, skipped ${skipped}` : ""}`);
- if (missing.size > 0) {
- console.log(`Install ${[...missing].join(", ")} and run pnpm install again to finish setup`);
- }
-}
-
-const declareAt = process.argv.indexOf("--declare");
-try {
- if (declareAt === -1) await main();
- else await emitDeclarations(process.argv.slice(declareAt + 1));
-} catch (error) {
- // Nothing this script does is worth failing an install over.
- console.warn(`Service preparation stopped: ${error instanceof Error ? error.message : String(error)}`);
-}