Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(api): a JSON Schema for every request and edit command, generated from the types (#337)

Jareercf7e0d23ed45
diff --git a/src/Cargo.toml b/src/Cargo.toml
index cd95d77b..a4a1d0de 100644
--- a/src/Cargo.toml
+++ b/src/Cargo.toml
@@ -17,6 +17,7 @@ concat-export = { path = "crates/concat-export" }
concat-host = { path = "crates/concat-host" }
concat-api = { path = "crates/concat-api" }
concat-server = { path = "crates/concat-server" }
+concat-mcp = { path = "crates/concat-mcp" }
concat-speech = { path = "crates/concat-speech" }
concat-effects = { path = "crates/concat-effects" }
concat-text = { path = "crates/concat-text" }
@@ -26,6 +27,10 @@ concat = { path = "crates/concat" }
serde_json = "1"
thiserror = "2"
clap = { version = "4", features = ["derive", "env"] }
+# JSON Schemas for the API's request types, behind the `schema` features of
+# concat-project and concat-api: what concat-mcp hands an agent as each
+# tool's input schema, generated from the types so it cannot drift.
+schemars = "1"
# The GPU compositor, behind concat-render's `gpu` feature. A heavy tree, which
# is exactly why it is optional: engine-only work never pays for it. Pinned to
diff --git a/src/crates/concat-api/Cargo.toml b/src/crates/concat-api/Cargo.toml
index bd3b6dad..481c58dd 100644
--- a/src/crates/concat-api/Cargo.toml
+++ b/src/crates/concat-api/Cargo.toml
@@ -19,6 +19,12 @@ serde_json.workspace = true
png = "0.17"
# A frame handed back inline crosses as text.
base64 = "0.22"
+schemars = { workspace = true, optional = true }
+
+[features]
+# A JSON Schema for every request, generated from the types and their doc
+# comments: what concat-mcp gives an agent for each tool's arguments.
+schema = ["dep:schemars", "concat-project/schema"]
[dev-dependencies]
tempfile = "3"
diff --git a/src/crates/concat-api/src/lib.rs b/src/crates/concat-api/src/lib.rs
index a23bd9f8..e4b463e5 100644
--- a/src/crates/concat-api/src/lib.rs
+++ b/src/crates/concat-api/src/lib.rs
@@ -67,6 +67,18 @@ const DEFAULT_PRESET: &str = "medium";
/// build or the embedder is added; see [`VersionInfo::capabilities`].
const CAPABILITIES: &[&str] = &["events"];
+/// The JSON Schema of [`Request`]: one alternative per method, each the
+/// object its params are with `method` naming it, and every type a request
+/// carries - an edit [`Command`] and all it holds - under `$defs`. The
+/// descriptions are this crate's and `concat_project`'s doc comments, so
+/// what a caller is told about a field is what its maintainer wrote next
+/// to it. Generated, never written by hand: a method or a command field
+/// added to the types is in the schema on the next build.
+#[cfg(feature = "schema")]
+pub fn request_schema() -> serde_json::Value {
+ serde_json::to_value(schemars::schema_for!(Request)).unwrap_or_default()
+}
+
/// Where a job's events go. Called from the job's thread, so a transport
/// that writes them to a caller locks its writer inside.
pub type EventSink = Arc<dyn Fn(Event) + Send + Sync>;
diff --git a/src/crates/concat-api/src/message.rs b/src/crates/concat-api/src/message.rs
index 0e0117b5..cc4ec121 100644
--- a/src/crates/concat-api/src/message.rs
+++ b/src/crates/concat-api/src/message.rs
@@ -46,6 +46,7 @@ pub const API_VERSION: &str = "0.2";
/// path [`Request::ProjectOpen`] took; the API keeps one session per folder
/// and a method on a folder that is not open is an error, not a silent
/// open, so a caller always knows what it is editing.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(tag = "method", rename_all_fields = "camelCase")]
pub enum Request {
@@ -246,6 +247,7 @@ pub enum Request {
}
/// One template slot and the file that takes it.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Fill {
@@ -257,6 +259,7 @@ pub struct Fill {
/// How an export is rendered. Everything but the output is optional and
/// defaults to what the window's sheet defaults to.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ExportSpec {
diff --git a/src/crates/concat-project/Cargo.toml b/src/crates/concat-project/Cargo.toml
index 532232fc..69a6cd97 100644
--- a/src/crates/concat-project/Cargo.toml
+++ b/src/crates/concat-project/Cargo.toml
@@ -14,6 +14,13 @@ concat-core.workspace = true
serde = { version = "1", features = ["derive", "rc"] }
serde_json.workspace = true
thiserror.workspace = true
+schemars = { workspace = true, optional = true }
+
+[features]
+# A JSON Schema for every type a command carries, from the same doc
+# comments, for a caller that hands the schema to an agent (concat-mcp).
+# Off by default: the engine and the window do not need it.
+schema = ["dep:schemars"]
[lints]
workspace = true
diff --git a/src/crates/concat-project/src/commands/mod.rs b/src/crates/concat-project/src/commands/mod.rs
index 94423328..8694d2ff 100644
--- a/src/crates/concat-project/src/commands/mod.rs
+++ b/src/crates/concat-project/src/commands/mod.rs
@@ -44,6 +44,7 @@ pub(crate) use crate::model::ranges::{
/// Which end of a clip a trim drags. The two are not symmetric: see
/// [`Command::TrimClip`].
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TrimEdge {
@@ -55,6 +56,7 @@ pub enum TrimEdge {
}
/// Which per-track toggle a [`Command::SetTrackFlag`] flips.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TrackFlag {
@@ -65,6 +67,7 @@ pub enum TrackFlag {
}
/// Where one clip is going, in a multi-clip move.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ClipMove {
@@ -81,6 +84,7 @@ pub struct ClipMove {
/// A partial update to one clip. Every field optional; `transition_in` and
/// `text` are double-optional so "clear it" and "leave it alone" stay
/// distinct on the wire (absent = untouched, null = cleared).
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Default, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ClipPatch {
@@ -120,6 +124,7 @@ pub struct ClipPatch {
with = "double_option",
skip_serializing_if = "Option::is_none"
)]
+ #[cfg_attr(feature = "schema", schemars(with = "Option<Crop>"))]
pub crop: Option<Option<Crop>>,
/// The mask, same three-way wire semantics as `crop`: absent leaves it
/// alone, null takes it off, a value replaces it, tidied on the way in.
@@ -143,6 +148,7 @@ pub struct ClipPatch {
with = "double_option",
skip_serializing_if = "Option::is_none"
)]
+ #[cfg_attr(feature = "schema", schemars(with = "Option<Transition>"))]
pub transition_in: Option<Option<Transition>>,
/// The title styling, same three-way wire semantics as `transition_in`.
/// Setting a style also renames the clip after its first line.
@@ -151,6 +157,7 @@ pub struct ClipPatch {
with = "double_option",
skip_serializing_if = "Option::is_none"
)]
+ #[cfg_attr(feature = "schema", schemars(with = "Option<TextStyle>"))]
pub text: Option<Option<TextStyle>>,
/// Which of the media's audio streams the clip plays, by stream index:
/// absent leaves it alone, null goes back to the file's first, a value
@@ -160,6 +167,7 @@ pub struct ClipPatch {
with = "double_option",
skip_serializing_if = "Option::is_none"
)]
+ #[cfg_attr(feature = "schema", schemars(with = "Option<u32>"))]
pub audio_stream: Option<Option<u32>>,
/// The clip's own background over the timeline's: absent leaves it
/// alone, null goes back to following the timeline's, a value - black
@@ -194,6 +202,7 @@ mod double_option {
}
/// A media item as probed by the host, before the model mints its id.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct NewMedia {
@@ -241,6 +250,7 @@ pub struct NewMedia {
/// fields. [`apply`] is the single place each variant's meaning lives; the
/// notes here state the contract - clamps, tolerances, what gets minted -
/// so a caller need not read `apply` to know what a command will do.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(tag = "op", rename_all = "camelCase", rename_all_fields = "camelCase")]
// A clip patch carries a whole text style, which is the largest variant by
diff --git a/src/crates/concat-project/src/model.rs b/src/crates/concat-project/src/model.rs
index 845565db..24e80854 100644
--- a/src/crates/concat-project/src/model.rs
+++ b/src/crates/concat-project/src/model.rs
@@ -19,6 +19,7 @@ use serde_json::{Map, Value};
/// What a piece of media is. A still is not a video with one frame: it has no
/// intrinsic duration, so its length on a timeline is editorial.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum MediaKind {
@@ -34,6 +35,7 @@ pub enum MediaKind {
/// What made a media file, when it was the editor and not the user's
/// import. The bin shelves such a file under Generated, by origin, and
/// keeps it out of the import shelves.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum MediaOrigin {
@@ -78,6 +80,7 @@ impl ClipKind {
/// One audio stream of a media file, as the probe reported it. A screen
/// recording keeps the desktop's sound and the microphone as two of these;
/// a clip names the one it plays by `index`.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AudioTrack {
@@ -108,6 +111,7 @@ pub struct AudioTrack {
/// full crushes its shadows until it is read as `limited`. Set with
/// `Command::SetMediaColorRange`; absent means the file's own tag is read.
/// https://github.com/jub0t/Concat/issues/103
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ColorRange {
@@ -234,6 +238,7 @@ impl Default for Track {
/// One applied audio filter or video effect: a catalogue id plus whatever
/// parameters were set. The catalogues themselves (and the FFmpeg strings
/// they build) live in the UI today; the model only stores the numbers.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AppliedFilter {
@@ -264,6 +269,7 @@ fn yes() -> bool {
}
/// One key on one parameter of an applied effect; see `AppliedFilter::keys`.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ParamKey {
@@ -414,6 +420,7 @@ impl AppliedFilter {
/// How a picture's background is taken away when there is no key colour
/// to remove: a person mask found by the cutout model, alone or corrected
/// by hand.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Cutout {
@@ -431,6 +438,7 @@ pub struct Cutout {
}
/// The two ways a cutout decides what stays.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum CutoutMode {
@@ -443,6 +451,7 @@ pub enum CutoutMode {
/// What a cutout keeps: the person the matting model finds, the one
/// thing the picture is of whatever it is, or whichever of those the
/// analysis decides fits the footage.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Subject {
@@ -474,6 +483,7 @@ impl Subject {
/// One brush stroke over a cutout: which tool, how wide, and where it went.
/// Points are fractions of the source picture, `(0, 0)` its top-left, so a
/// stroke survives a change of output size, a crop or a flip.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Stroke {
@@ -491,6 +501,7 @@ pub struct Stroke {
}
/// The four brushes of a custom cutout.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum BrushTool {
@@ -634,6 +645,7 @@ impl Stabilize {
}
/// A crop: what is taken off each edge, as fractions of the source.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Crop {
@@ -1014,6 +1026,7 @@ impl ClipMask {
/// carry them and the document that stores them do not care which side of
/// the clip a value ends up on, and only the thing that finally reads them
/// does.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum KeyProperty {
@@ -1131,6 +1144,7 @@ impl KeyProperty {
/// Serialised as a bare array: `"ease": [0.42, 0, 0.58, 1]`. Documents
/// written before the curve editor spell it `"linear"` / `"in"` / `"out"` /
/// `"inOut"` instead, and `from_value` still reads those.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Debug, Serialize)]
#[serde(transparent)]
pub struct KeyEase(pub [f64; 4]);
@@ -1257,6 +1271,7 @@ pub struct ClipKey {
pub const KEY_EPSILON: f64 = 0.002;
/// One point of a speed curve.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SpeedPoint {
@@ -1279,6 +1294,7 @@ pub struct Enhanced {
}
/// A transition on the cut into a clip.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Transition {
@@ -1294,6 +1310,7 @@ pub struct Transition {
/// left-aligned one by its block's left edge and a right-aligned one by its
/// right edge. Typing more into a left-aligned title grows it to the right
/// and leaves its left edge where it was, as alignment means everywhere.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TextAlign {
@@ -1308,6 +1325,7 @@ pub enum TextAlign {
/// Which figure a shape clip draws. The names are written into project
/// files, so they are stable for ever.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ShapeKind {
@@ -1361,6 +1379,7 @@ impl ShapeKind {
/// How a shape clip is painted. Sizes are fractions of the frame's height,
/// as a title's are, so a shape composed against 1080p is the same shape at
/// 4K. A style arriving in a command needs only the fields it changes.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct ShapeStyle {
@@ -1423,6 +1442,7 @@ impl Default for ShapeStyle {
/// command needs only the fields it changes; the rest are the default's, so
/// a caller can say `{ "content": "Hello" }` and get a title that looks like
/// one the window would place.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct TextStyle {
@@ -2404,6 +2424,7 @@ impl Default for Timeline {
/// The colour a picture is in: SDR on Rec. 709, or HDR on Rec. 2020 with
/// the hybrid log-gamma or the perceptual quantiser transfer. A timeline's
/// is what it is output in; a media file's, what it was recorded in.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ColorSpace {
@@ -2437,6 +2458,7 @@ impl ColorSpace {
/// rate, so a build that predates this reads the document it always did,
/// and a document from such a build gives every timeline that block on the
/// way in.
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VideoSettings {