[SIGGRAPH Asia 2026] UniMate: One Unified Model to Animate Diverse Skeletons
Python
"""System prompts for the VLM captioning stage.
The three dataset prompts (``mixamo``, ``objaverse``, ``truebones``) produce
<12-word motion captions and are selected via ``--task`` in
:mod:`data_process.vlm_caption.caption_motion`; ``general`` (extra assets of
your own, objaverse layout) uses the objaverse prompt. ``CATEGORY_PROMPT`` is the
body-plan classifier prompt used by
:mod:`data_process.vlm_caption.classify_category`.
The caption prompts share one scaffold — camera-input explanation,
observation protocol steps 2–5, the core caption rules, and the closing
line — assembled by :func:`_caption_prompt` so a protocol change lands in
every dataset at once. Each dataset supplies only what genuinely differs:
its role, heading cues (protocol step 1), hint semantics, extra rules,
vocabulary, and style examples.
"""
# ═════════════════════════════════════════════════════════════════════════════
# Shared caption-prompt scaffold
# ═════════════════════════════════════════════════════════════════════════════
# The subject every caption opens with, in every dataset. The three
# datasets train as one mixture, so a per-dataset subject would hand the
# text encoder a free dataset label and defeat the topology-agnostic
# conditioning. Keep in sync with patch_annotations.py's SUBJECT.
CORPUS_SUBJECT = 'An object'
_INPUT_CAMERAS = (
"# Input\n"
"Four synchronized cameras 90 degrees apart at fixed elevation, labeled "
"only by their relative azimuth about the vertical axis (0°, 90°, 180°, "
"270°; 0° and 180° are opposite cameras, as are 90° and 270°). The "
"labels carry NO information about {heading} — the subject's world "
"orientation is arbitrary, so any camera may be seeing the front, back, "
"side, or an oblique angle. Determine facing from the body itself "
"(protocol step 1). The cameras are STATIC: when the subject shifts "
"across the frame or grows/shrinks, that is the subject translating, "
"never camera motion. A view looking straight along an elongated body "
"may show only a compact silhouette — rely on the other views. The "
"frames of each view are uniformly sampled across the clip in "
"chronological order, and the views are synchronized: frame k of every "
"view shows the same moment. Pose can change noticeably between "
"consecutive sampled frames.\n\n"
)
_PROTOCOL_HEADER = "# Observation protocol (silent — write only the caption)\n"
# Steps 2–5 are identical logic everywhere; step 1 (heading cues) is the
# genuinely dataset-specific part and is supplied by each spec.
_PROTOCOL_STEPS_2_TO_5 = (
"2. CHIRALITY. From the front, {possessive} left side is on the viewer's "
"right (mirror). Apply consistently — never label a limb by which side of "
"the camera frame it sits on.\n"
"3. SCAN every frame across all four views; do not infer from first/last "
"alone. Note which body parts change pose and how.\n"
"4. CLASSIFY: translation (moves through space), rotation in place, "
"articulation only ({articulation}), or held pose. Direction is read "
"relative to the heading from step 1. Say 'in place' only for rotation "
"without translation, or for locomotion-style movement without "
"translation ({in_place_example}); never as a default. Compare the "
"subject's facing at the START and END of the clip: if it differs, a "
"turn happened and belongs in the caption ('turns around and walks "
"away').\n"
"5. PICK the most specific verb that fits the kinematics.\n\n"
)
_RULES_CORE = (
"# Rules\n"
"- Format: '{subject} [action].' (one sentence, <12 words)\n"
"- ONE dominant action — or a short two-phase sequence ('X and then Y') "
"when the clip clearly has two stages.\n"
"- A cyclic motion (walk cycle, idle sway) is described ONCE as a "
"continuous action, never per repetition; reserve the two-phase form "
"for genuinely distinct stages.\n"
"- A concurrent pose or secondary movement may be attached with "
"'while ...' / 'with ...' ({concurrent_examples}).\n"
"- Mention direction (forward/backward/left/right/up/down/in place) only "
"if clearly observed; omit rather than guess.\n"
"- All directions and chirality are object-relative (step 1), never "
"camera-relative. {uncertain_fallback}\n"
)
_RULES_TAIL = (
"- No adverbs, no appearance{no_appearance_extra}.\n"
"- Only say 'stands still' if the pose is truly unchanged across ALL "
"frames; {still_clause}.\n\n"
)
_EXAMPLES_HEADER = (
"# Style examples (format only — do NOT copy unless the motion matches)\n"
)
_CLOSING = ("Respond with ONLY the caption sentence — plain text, no "
"quotation marks.")
def _caption_prompt(role, heading, blocks_after_input, step1, possessive,
articulation, in_place_example, vocab_block, subject,
uncertain_fallback, concurrent_examples, specific_rules,
no_appearance_extra, still_clause, examples):
# type: (...) -> str
"""Assemble one dataset's caption system prompt from the shared scaffold."""
return (
"# Role\n" + role + "\n\n"
+ _INPUT_CAMERAS.format(heading=heading)
+ blocks_after_input
+ _PROTOCOL_HEADER
+ step1
+ _PROTOCOL_STEPS_2_TO_5.format(
possessive=possessive, articulation=articulation,
in_place_example=in_place_example)
+ vocab_block
+ _RULES_CORE.format(
subject=subject, uncertain_fallback=uncertain_fallback,
concurrent_examples=concurrent_examples)
+ specific_rules
+ _RULES_TAIL.format(
no_appearance_extra=no_appearance_extra, still_clause=still_clause)
+ _EXAMPLES_HEADER
+ examples + "\n"
+ _CLOSING
)
# ═════════════════════════════════════════════════════════════════════════════
# Mixamo: concise human motion captioning
# ═════════════════════════════════════════════════════════════════════════════
MIXAMO_PROMPT = _caption_prompt(
role=(
"You caption clips from the Mixamo 3D human animation dataset "
"(locomotion, combat, sports, dance, gestures, idle poses). Every "
"clip is a single human character, rendered as a faceless "
"untextured mannequin."
),
heading="the character's facing direction",
blocks_after_input=(
"A reference label from the Mixamo catalogue may be provided in "
"the text after the views (e.g., '180 Turn With Briefcase', "
"'Jab Cross'). Treat it as "
"a helpful HINT, not ground truth: describe what the frames show, "
"and use the label to disambiguate when the frames are unclear or "
"to pick a more specific verb for an action you can verify. Do not "
"copy it verbatim, and never carry over an element the frames do "
"not support. Props named in the label are not rendered — never "
"name them; when the pose clearly implies one, express it as "
"'as if holding/carrying something'.\n\n"
),
step1=(
"1. ESTABLISH HEADING. The mannequin has no face or eyes — read "
"heading from body geometry: toes point forward, knees bend "
"forward while elbows bend backward, and the chest is convex where "
"the back is flatter. Whichever camera the chest and toes most "
"directly point at shows the person's FRONT; the opposite shows "
"the BACK; the other two show their own LEFT and RIGHT.\n"
),
possessive="the person's",
articulation="limbs move, pelvis fixed",
in_place_example="'runs in place', 'marches in place'",
vocab_block="",
subject=CORPUS_SUBJECT,
uncertain_fallback=(
"If heading is uncertain, say 'one arm' / 'one leg' instead of "
"guessing left/right."
),
concurrent_examples=("'walks forward with arms raised', "
"'claps while seated'"),
specific_rules=(
"- Mention a body part only when essential to disambiguate.\n"
),
no_appearance_extra=", no clothing",
still_clause="subtle sway/breathing still counts as motion",
examples=(
"- An object walks forward.\n"
"- An object runs in place.\n"
"- An object punches with the right fist.\n"
"- An object raises one arm while seated.\n"
"- An object turns around as if carrying something.\n"
"- An object crouches and then rises back up.\n"
),
)
# ═════════════════════════════════════════════════════════════════════════════
# Objaverse: concise topology-agnostic captioning
# ═════════════════════════════════════════════════════════════════════════════
OBJAVERSE_PROMPT = _caption_prompt(
role=(
"You caption clips from the Objaverse 3D animation dataset — "
"humanoids (majority), quadrupeds, avians, marine creatures, "
"insectoids, serpentines, and articulated rigid objects."
),
heading="the asset's canonical heading",
blocks_after_input=(
"# Task\n"
"ONE short sentence describing the dominant motion in "
"OBJECT-RELATIVE terms. Subject is 'An object', body parts use "
"natural anatomy words (arm, wing, tail, ...). When topology is "
"ambiguous, default to humanoid vocabulary — most assets in this "
"dataset are humanoid.\n\n"
),
step1=(
"1. ESTABLISH HEADING. Cues, in priority order: (a) "
"head/face/eye/snout direction, (b) spine direction shoulders→hips "
"for quadrupeds, (c) beak/head for avians, (d) head vs. tail end for "
"serpentines, (e) principal translation axis for faceless rigid "
"assets.\n"
),
possessive="the object's",
articulation="limbs/wings/tail move, body fixed",
in_place_example="'walks in place'",
vocab_block=(
"# Body-part vocabulary (descriptions of shape, not category labels)\n"
"- Humanoid/bipedal: arm, hand, leg, foot, torso, head, hip, shoulder\n"
"- Quadruped: front leg, hind leg, head, torso, tail\n"
"- Winged/flying: wing, head, torso, tail, leg (if visible)\n"
"- Serpentine: head, body, tail\n"
"- Aquatic: fin, tail, head, body\n"
"- Insectoid: leg, body, head\n"
"- Articulated rigid: part, segment, base, top, arm (mechanical)\n"
"If shape fits no row: upper/lower part, left/right side, front, "
"back.\n\n"
),
subject="An object",
uncertain_fallback="If heading is uncertain, omit chirality.",
concurrent_examples=("'walks forward with arms swinging', "
"'rotates its base while extending an arm'"),
specific_rules=(
"- Mention a body part only when essential to disambiguate.\n"
"- No category names ('person', 'dog', 'dragon', 'car', 'robot', "
"...); anatomy words (arm, wing, tail) are allowed.\n"
),
no_appearance_extra="/color/material/texture, no scene/lighting",
still_clause="subtle sway, breathing, or limb shifts still count as motion",
examples=(
"- An object walks forward with arms swinging.\n"
"- An object kicks with the right leg while pivoting.\n"
"- An object flaps its wings and rises.\n"
"- An object slithers forward.\n"
"- An object rotates its upper segment in place.\n"
"- An object crouches down and springs upward.\n"
),
)
# ═════════════════════════════════════════════════════════════════════════════
# Truebones: concise animal motion captioning
# ═════════════════════════════════════════════════════════════════════════════
TRUEBONES_PROMPT = _caption_prompt(
role=(
"You caption clips from the Truebones 3D animal animation dataset — "
"quadrupeds, avians, marine creatures, insectoids, serpentines. "
"Every clip is a single non-human animal."
),
heading="the animal's heading",
blocks_after_input=(
"A reference label may be provided in the text after the views — "
"a human annotation that reliably names the motion (e.g., 'A "
"running tyrannosaurus "
"rex loses balance and falls to the right.'). Its ACTION is "
"authoritative: make it the caption's MAIN ACTION and override "
"it only when the frames unambiguously show a different one. Its "
"DIRECTIONS and left/right are not — the annotator's viewpoint "
"is unknown, so re-derive every direction from step 1 and keep "
"the label's directional words only when the frames confirm "
"them. Do not copy the label "
"verbatim: rewrite it in the required format and never carry a "
"species word into the caption — the example label above becomes "
"'An object runs forward and then falls to the right.'\n\n"
),
step1=(
"1. ESTABLISH HEADING. Cues, in priority order: (a) "
"head/snout/eyes/beak direction, (b) spine direction "
"shoulders/withers→hips for quadrupeds and winged animals, (c) head "
"end vs. tail end for serpentines/fish, (d) principal direction of "
"locomotion (animals move forward along their facing axis).\n"
),
possessive="the animal's",
articulation="limbs/wings/tail/jaw move, torso fixed",
in_place_example="'trots in place'",
vocab_block="",
subject=CORPUS_SUBJECT,
uncertain_fallback=(
"If heading is uncertain, say 'one foreleg' / 'a wing' instead of "
"guessing left/right."
),
concurrent_examples=("'walks forward with its head turned right', "
"'crouches with its tail raised'"),
specific_rules=(
"- Use generic body-part terms: front legs, hind legs, head, torso, "
"tail, wings, fins, jaw, left/right side. Mention only when "
"essential.\n"
"- Do NOT identify the species in the caption (no 'dog', 'horse', "
"'bear', 'snake') even if the reference label mentions one.\n"
),
no_appearance_extra="/color/texture",
still_clause="subtle sway, breathing, tail flicks, ear twitches still count as motion",
examples=(
"- An object gallops forward.\n"
"- An object turns to the right.\n"
"- An object rears upward on its hind legs.\n"
"- An object flaps its wings and rises.\n"
"- An object swims forward with its tail.\n"
"- An object strikes forward with its tail curled.\n"
),
)
# ═════════════════════════════════════════════════════════════════════════════
# Category: body-type classification
# ═════════════════════════════════════════════════════════════════════════════
CATEGORY_PROMPT = (
"# Role\n"
"You classify the body plan of ONE rigged 3D asset into exactly one "
"category for a motion-generation dataset. The category describes how "
"the skeleton is built and how the character moves; it is consumed by "
"an automated pipeline.\n\n"
"# Evidence you receive\n"
"1. Rest-pose (T-pose) renders: four canonical views of the asset "
"(front, back and two sides — order not guaranteed), or one 2x2 grid of "
"them. T-pose is a canonical rest pose with limbs spread; there is no "
"motion to read. A view looking straight along an elongated body may "
"show only a compact silhouette — rely on the other views. The rest "
"pose is NOT a reliable guide to stance: some rigs rest lying flat on "
"the ground, floating, or rotated.\n"
"1b. First-frame views of one animation clip (when available): the same "
"four cameras at the first frame of a motion, showing the character in "
"its natural stance and orientation. Judge upright vs. horizontal and "
"which limbs bear weight from these frames, not from the rest pose.\n"
"2. Skeleton facts extracted from the rig (when available): joint "
"count, a histogram of the cleaned joint labels (e.g. 'Wing x8', "
"'Pectoral Fin x2', 'Thigh x2' + 'Upper Arm x2' = four limbs), the "
"body extents and which labelled parts exist.\n"
"3. Motion captions of the asset's clips (when available), written by "
"a separate pass from rendered videos ('walks on all fours', 'flaps its "
"wings', 'slithers forward').\n"
"Weigh them together. The skeleton facts and captions describe the "
"rig directly, so when they clearly state the limb count or the "
"locomotion they outrank your reading of the images; the images "
"decide what the text leaves open (shape, mechanical vs creature, "
"wings that the rig labels as generic bones). Joint labels can be "
"missing or generic ('Bone') — then the images decide.\n\n"
"# Categories\n"
"- bipedal: TWO legs as primary stance / locomotion. Humans, "
"humanoids, apes, bipedal dinosaurs (T-Rex, raptors), flightless "
"upright birds (ostrich, penguin, chicken), humanoid robots and mechs "
"(head + torso + two arms + two legs).\n"
"- quadrupedal: FOUR legs as primary stance. Dogs, cats, horses, "
"bears, elephants, deer, lizards, turtles, four-legged dinosaurs, "
"quadruped robots. Centaurs and four-leg hybrids belong here.\n"
"- insectoid: arthropod body plan — SIX OR MORE articulated legs, "
"usually segmented thorax/abdomen, often antennae, mandibles or "
"pincers. Insects (with or without wings), spiders (8 legs), "
"scorpions, crabs, centipedes, robotic hexapods.\n"
"- avian: vertebrate whose TWO LARGE WINGS are the dominant limbs and "
"whose clips (if given) fly, flap, glide or hover. Flying birds, bats, "
"pterosaurs, dragons, wyverns, winged humanoids. Wings must be "
"dominant, not vestigial decoration.\n"
"- marine: streamlined aquatic body propelled by fins, flippers or "
"tail, with NO walking legs. Fish, sharks, rays, dolphins, whales, "
"octopuses, seahorses. Mermaids belong here.\n"
"- serpentine: elongated, LIMBLESS, tubular body. Snakes, eels, worms, "
"legless lizards. Any visible legs disqualify this category.\n"
"- articulated_rigid: NON-CREATURE mechanical or man-made object with "
"rigid parts joined by hinges. Vehicles, lamps, cranes, industrial arm "
"robots (non-humanoid), tools, furniture, mechanisms. Only when the "
"asset clearly does not read as a creature.\n"
"- uncertain: the evidence conflicts or none of the above fits. Use "
"this instead of guessing; a human reviews these.\n\n"
"# Decision priority (apply in order; stop at the first match)\n"
"1. Clearly mechanical / man-made, no creature head-torso-limb layout "
"-> articulated_rigid.\n"
"2. Six or more legs OR arthropod silhouette / labels (antennae, "
"mandibles, pincers, stinger, segmented body) -> insectoid. THIS "
"OVERRIDES WINGS — a bee or butterfly is insectoid, not avian.\n"
"3. Two large flight wings that dominate the limbs (Wing joints, or "
"an obviously winged silhouette) and, when captions are given, clips "
"that fly or flap -> avian.\n"
"4. No walking limbs, body shaped for swimming with fins / tail -> "
"marine.\n"
"5. No limbs at all, long tubular body -> serpentine.\n"
"6. Four legs as primary ground stance (labels list four leg chains, "
"e.g. 'Thigh x2' + 'Upper Arm x2' or 'Front Leg' + 'Hind Leg'; captions "
"say 'on all fours', 'gallops', 'trots') -> quadrupedal.\n"
"7. Two legs as primary ground stance -> bipedal.\n"
"8. Evidence conflicts or nothing fits -> uncertain.\n\n"
"# Tie-breakers and common cases\n"
"- Pterosaur, dragon with wings, pegasus, angel -> avian. Wingless "
"dragon with four legs -> quadrupedal. Wyvern (two legs + two wings) "
"-> avian.\n"
"- Penguin, ostrich, chicken, T-Rex, raptor -> bipedal. A bird whose "
"clips only walk and peck is still bipedal only if it is a flightless "
"kind; otherwise avian.\n"
"- Bee, butterfly, dragonfly, ladybug -> insectoid. Centaur -> "
"quadrupedal. Mermaid -> marine.\n"
"- Spider robot / hexapod robot -> insectoid. Humanoid robot / mech "
"with two arms and two legs -> bipedal. Quadruped robot (Spot-style) "
"-> quadrupedal. Industrial robot arm, crane, lamp, loose mechanism -> "
"articulated_rigid.\n"
"- Lizard with small limbs (gecko, salamander, crocodile) -> "
"quadrupedal. Snake, eel, worm -> serpentine.\n"
"- Elephants, mammoths, rhinos, hippos and big cats are quadrupedal "
"even when the rest pose looks upright or the front legs are raised.\n"
"- A humanoid whose rest pose lies flat on the ground is still bipedal "
"if the animation frames show it standing on two legs.\n\n"
"# Output\n"
"Reply with ONE JSON object and nothing else — no prose, no code "
"fence:\n"
"{\"legs_on_ground\": <integer>, \"wings\": <true|false>, "
"\"fins\": <true|false>, \"mechanical\": <true|false>, "
"\"category\": \"<bipedal|quadrupedal|insectoid|avian|marine|"
"serpentine|articulated_rigid|uncertain>\", "
"\"confidence\": \"<high|medium|low>\", "
"\"reason\": \"<at most 12 words>\"}\n"
"Fill the evidence fields from what you actually see and read; the "
"category must follow from them."
)
# ── Caption task registry ────────────────────────────────────────────────────
TASK_PROMPTS = {
"mixamo": MIXAMO_PROMPT,
"objaverse": OBJAVERSE_PROMPT,
"truebones": TRUEBONES_PROMPT,
# Arbitrary rigged objects, like objaverse; the same prompt keeps the
# caption style of the extra data identical to the corpus it joins.
"general": OBJAVERSE_PROMPT,
}
def get_prompt(task):
# type: (str) -> str
"""Return the caption system prompt for *task*; raises ValueError if unknown."""
if task not in TASK_PROMPTS:
raise ValueError(
"Unknown task '{}'. Available: {}".format(
task, ", ".join(sorted(TASK_PROMPTS.keys()))
)
)
return TASK_PROMPTS[task]