Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: add Codex-native YouTube Shorts pipeline

alextavares5e60dbaa80c6
diff --git a/apps/daemon/src/creator/stickman/action-handler.ts b/apps/daemon/src/creator/stickman/action-handler.ts
index c3666df4..322d311d 100644
--- a/apps/daemon/src/creator/stickman/action-handler.ts
+++ b/apps/daemon/src/creator/stickman/action-handler.ts
@@ -8,6 +8,11 @@ import type {
CreatorJobStatus,
CreatorJson
} from '@opencreator/protocol';
+import {
+ stickmanOutputPresetDefaults,
+ stickmanOutputPresets,
+ stickmanRatios
+} from '@opencreator/protocol';
import type { CreatorRepository } from '../repository.js';
import { CreatorServiceError } from '../service.js';
import {
@@ -24,6 +29,18 @@ import {
readVisualAssetRef
} from './visual-assets.js';
+const stickmanFormatStateKeys = [
+ 'outputPreset',
+ 'ratio',
+ 'targetDurationSeconds',
+ 'sourceLanguage',
+ 'targetLanguage',
+ 'ttsProvider',
+ 'ttsModel',
+ 'voiceCode',
+ 'voiceName'
+] as const;
+
export type StickmanActionResult = {
handled: boolean;
state: Record<string, CreatorJson>;
@@ -51,7 +68,8 @@ export function handleStickmanAction(input: {
if (input.action === 'update-settings') {
const patch = readRecord(parsedInput.patch, 'patch');
- if (patch.characterAsset === undefined && patch.styleAsset === undefined) {
+ const hasFormatPatch = stickmanFormatStateKeys.some(key => patch[key] !== undefined);
+ if (patch.characterAsset === undefined && patch.styleAsset === undefined && !hasFormatPatch) {
return {
handled: false,
state: current.state,
@@ -59,6 +77,18 @@ export function handleStickmanAction(input: {
affectedArtifactIds: []
};
}
+ if (
+ patch.ratio !== undefined
+ && !stickmanRatios.includes(patch.ratio as (typeof stickmanRatios)[number])
+ ) {
+ throw new CreatorServiceError('creator_action_input_invalid', 'ratio must be 16:9 or 9:16');
+ }
+ if (
+ patch.outputPreset !== undefined
+ && !stickmanOutputPresets.includes(patch.outputPreset as (typeof stickmanOutputPresets)[number])
+ ) {
+ throw new CreatorServiceError('creator_action_input_invalid', 'outputPreset is invalid');
+ }
const nextCharacter = patch.characterAsset === undefined
? readVisualAssetRef(current.state.characterAsset, DEFAULT_STICKMAN_CHARACTER_ASSET)
: requireVisualAssetRef(patch.characterAsset, 'characterAsset');
@@ -73,21 +103,44 @@ export function handleStickmanAction(input: {
current.state.styleAsset,
DEFAULT_STICKMAN_STYLE_ASSET
);
- const visualProfileChanged = !sameAssetRef(previousCharacter, nextCharacter)
+ const outputPreset = patch.outputPreset === 'youtube-shorts' || patch.outputPreset === 'landscape'
+ ? patch.outputPreset
+ : current.state.outputPreset;
+ const normalizedFormat = outputPreset === 'youtube-shorts'
+ ? stickmanOutputPresetDefaults('youtube-shorts')
+ : outputPreset === 'landscape'
+ ? stickmanOutputPresetDefaults('landscape')
+ : {};
+ const nextState: Record<string, CreatorJson> = {
+ ...current.state,
+ ...patch,
+ ...normalizedFormat,
+ characterAsset: nextCharacter,
+ styleAsset: nextStyle
+ };
+ if (patch.outputPreset === 'landscape' && patch.ttsProvider === undefined) {
+ delete nextState.ttsProvider;
+ delete nextState.ttsModel;
+ delete nextState.voiceCode;
+ delete nextState.voiceName;
+ }
+ const formatChanged = stickmanFormatStateKeys.some(key => (
+ current.state[key] !== nextState[key]
+ ));
+ const visualAssetsChanged = !sameAssetRef(previousCharacter, nextCharacter)
|| !sameAssetRef(previousStyle, nextStyle);
- const affectedArtifactIds = visualProfileChanged
- ? staleVisualPipeline(repository, current)
- : [];
+ const affected = new Set<string>();
+ if (visualAssetsChanged) {
+ for (const artifactId of staleVisualPipeline(repository, current)) affected.add(artifactId);
+ }
+ if (formatChanged) {
+ for (const artifactId of staleContentPipeline(repository, current)) affected.add(artifactId);
+ }
return {
handled: true,
- state: {
- ...current.state,
- ...patch,
- characterAsset: nextCharacter,
- styleAsset: nextStyle
- },
+ state: nextState,
status: current.status,
- affectedArtifactIds
+ affectedArtifactIds: [...affected]
};
}
@@ -478,6 +531,37 @@ function staleVisualPipeline(repository: CreatorRepository, job: CreatorJob): st
return [...affected];
}
+function staleContentPipeline(repository: CreatorRepository, job: CreatorJob): string[] {
+ const rootKinds = new Set([
+ 'content_plan',
+ 'script_manifest',
+ 'narration_audio',
+ 'audio_timing',
+ 'shot_spec',
+ 'character_reference',
+ 'style_reference',
+ 'style_contract',
+ 'image_prompt_pack',
+ 'shot_image',
+ 'visual_validation',
+ 'timeline_manifest',
+ 'narration_subtitle',
+ 'clean_video',
+ 'media_validation',
+ 'thumbnail',
+ 'publish_copy',
+ 'delivery_manifest'
+ ]);
+ const roots = job.artifacts.filter(artifact => (
+ artifact.status !== 'stale' && rootKinds.has(artifact.kind)
+ ));
+ const affected = new Set<string>();
+ for (const root of roots) {
+ for (const artifactId of staleFrom(repository, job, root, true)) affected.add(artifactId);
+ }
+ return [...affected];
+}
+
function requireVisualAssetRef(
value: CreatorJson,
field: string
diff --git a/apps/daemon/src/creator/stickman/audio-executor.ts b/apps/daemon/src/creator/stickman/audio-executor.ts
index d0a2430f..b448bbe0 100644
--- a/apps/daemon/src/creator/stickman/audio-executor.ts
+++ b/apps/daemon/src/creator/stickman/audio-executor.ts
@@ -1,5 +1,6 @@
import { createHash } from 'node:crypto';
-import { readFile, writeFile } from 'node:fs/promises';
+import { execFile } from 'node:child_process';
+import { readFile, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import type { CreatorArtifact, CreatorServicesConfig } from '@opencreator/protocol';
import type { CreatorServicesConfigStore } from '../../creator-services/config-store.js';
@@ -12,8 +13,25 @@ import {
stickmanAudioTimingSchema,
stickmanScriptManifestSchema
} from './contracts.js';
+import { stickmanEdgeTtsVoiceForLanguage } from './tts.js';
type Synthesize = Pick<KrillinTtsService, 'synthesize'>['synthesize'];
+type EdgeTtsSynthesisResult = {
+ content: Buffer;
+ mime: 'audio/mpeg';
+ provider: 'edge-tts';
+ model: '';
+ voiceId: string;
+ format: 'mp3';
+};
+type EdgeTtsSynthesize = (input: {
+ text: string;
+ voiceId: string;
+ workdir: string;
+ scopeKey: string;
+ command?: string;
+ signal: AbortSignal;
+}) => Promise<EdgeTtsSynthesisResult>;
export function createStickmanAudioExecutor(input: {
configStore: Pick<CreatorServicesConfigStore, 'read'>;
@@ -21,6 +39,8 @@ export function createStickmanAudioExecutor(input: {
ledger: CreatorProviderRequestLedger;
ffprobePath: string;
synthesize?: Synthesize;
+ synthesizeEdgeTts?: EdgeTtsSynthesize;
+ edgeTtsCommand?: string;
probe?: (path: string, ffprobe: string) => Promise<MediaProbe>;
}): CreatorExecutor {
const synthesize = input.synthesize ?? (request => input.ttsService.synthesize(request));
@@ -48,6 +68,8 @@ async function synthesizeSegment(
configStore: Pick<CreatorServicesConfigStore, 'read'>;
ledger: CreatorProviderRequestLedger;
ffprobePath: string;
+ synthesizeEdgeTts?: EdgeTtsSynthesize;
+ edgeTtsCommand?: string;
},
synthesize: Synthesize,
probe: (path: string, ffprobe: string) => Promise<MediaProbe>
@@ -97,7 +119,7 @@ async function synthesizeSegment(
provider: selection.provider,
model: selection.model,
voiceId: selection.voiceId,
- format: 'wav'
+ format: selection.provider === 'edge-tts' ? 'mp3' : 'wav'
}
});
input.ledger.markSubmitting(ledger.id);
@@ -110,15 +132,24 @@ async function synthesizeSegment(
ledgerId: ledger.id
});
try {
- const result = await synthesize({
- text: segment.narration,
- provider: selection.provider,
- model: selection.model,
- voiceId: selection.voiceId,
- format: 'wav',
- signal: stage.signal
- });
- const path = join(stage.workdir, `${scopeKey}.wav`);
+ const result = selection.provider === 'edge-tts'
+ ? await (input.synthesizeEdgeTts ?? synthesizeWithEdgeTts)({
+ text: segment.narration,
+ voiceId: selection.voiceId,
+ workdir: stage.workdir,
+ scopeKey,
+ ...(input.edgeTtsCommand === undefined ? {} : { command: input.edgeTtsCommand }),
+ signal: stage.signal
+ })
+ : await synthesize({
+ text: segment.narration,
+ provider: selection.provider,
+ model: selection.model,
+ voiceId: selection.voiceId,
+ format: 'wav',
+ signal: stage.signal
+ });
+ const path = join(stage.workdir, `${scopeKey}.${result.format}`);
await writeFile(path, result.content);
const media = await probe(path, input.ffprobePath);
if (!media.hasAudio || media.duration <= 0) {
@@ -249,17 +280,25 @@ async function buildAudioTiming(
function resolveTtsSelection(stage: CreatorExecutorInput, config: CreatorServicesConfig) {
const provider = stage.job.state.ttsProvider === 'openai'
|| stage.job.state.ttsProvider === 'aliyun'
+ || stage.job.state.ttsProvider === 'edge-tts'
|| stage.job.state.ttsProvider === 'minimax'
? stage.job.state.ttsProvider
- : config.tts.provider === 'edge-tts'
- ? undefined
- : config.tts.provider;
+ : config.tts.provider;
if (provider === undefined) {
throw new CreatorExecutorError(
'creator_tts_config_missing',
'请先在 AI 服务的配音服务中配置可用的配音 Provider'
);
}
+ if (provider === 'edge-tts') {
+ return {
+ provider,
+ model: '',
+ voiceId: typeof stage.job.state.voiceCode === 'string' && stage.job.state.voiceCode.trim()
+ ? stage.job.state.voiceCode.trim()
+ : stickmanEdgeTtsVoiceForLanguage(stage.job.state.targetLanguage)
+ };
+ }
const providerConfig = config.tts[provider];
const model = typeof stage.job.state.ttsModel === 'string' && stage.job.state.ttsModel.trim()
? stage.job.state.ttsModel.trim()
@@ -276,6 +315,55 @@ function resolveTtsSelection(stage: CreatorExecutorInput, config: CreatorService
return { provider, model, voiceId };
}
+async function synthesizeWithEdgeTts(input: {
+ text: string;
+ voiceId: string;
+ workdir: string;
+ scopeKey: string;
+ command?: string;
+ signal: AbortSignal;
+}): Promise<EdgeTtsSynthesisResult> {
+ const outputPath = join(input.workdir, `${input.scopeKey}.edge-tts.mp3`);
+ const command = input.command ?? process.env.OPENCREATOR_EDGE_TTS_COMMAND ?? 'edge-tts';
+ try {
+ await execFileAsync(command, [
+ '--voice', input.voiceId,
+ '--text', input.text,
+ '--write-media', outputPath
+ ], input.signal);
+ const content = await readFile(outputPath);
+ if (content.length === 0) throw new Error('Edge TTS returned an empty audio file');
+ return {
+ content,
+ mime: 'audio/mpeg',
+ provider: 'edge-tts',
+ model: '',
+ voiceId: input.voiceId,
+ format: 'mp3'
+ };
+ } catch (error) {
+ throw new CreatorExecutorError(
+ 'creator_tts_runtime_unavailable',
+ `Edge TTS command failed: ${error instanceof Error ? error.message : String(error)}`
+ );
+ } finally {
+ await rm(outputPath, { force: true });
+ }
+}
+
+function execFileAsync(command: string, args: string[], signal: AbortSignal): Promise<void> {
+ return new Promise((resolve, reject) => {
+ execFile(command, args, {
+ signal,
+ timeout: 120_000,
+ windowsHide: true
+ }, error => {
+ if (error) reject(error);
+ else resolve();
+ });
+ });
+}
+
function requireArtifact(artifacts: CreatorArtifact[], kind: string): CreatorArtifact {
const artifact = artifacts.find(item => item.kind === kind && item.status === 'completed');
if (artifact === undefined) {
diff --git a/apps/daemon/src/creator/stickman/content-executor.ts b/apps/daemon/src/creator/stickman/content-executor.ts
index ab08573a..d02574f7 100644
--- a/apps/daemon/src/creator/stickman/content-executor.ts
+++ b/apps/daemon/src/creator/stickman/content-executor.ts
@@ -653,6 +653,7 @@ async function styleAssets(
);
const character = visualAssets.character(characterRef);
const style = visualAssets.style(styleRef);
+ const ratio = stage.job.state.ratio === '9:16' ? '9:16' : '16:9';
const characterReferences = visualAssets.referenceFiles(character);
const styleReferences = visualAssets.referenceFiles(style);
const primaryCharacter = characterReferences[0];
@@ -675,7 +676,7 @@ async function styleAssets(
await writeFile(characterPath, content);
const styleContract = stickmanStyleContractSchema.parse({
contract: 'stickman-visual-profile-v2',
- ratio: '16:9',
+ ratio,
character: {
assetId: character.id,
revision: character.revision,
@@ -771,6 +772,7 @@ async function promptPack(stage: CreatorExecutorInput) {
prompt: buildStickmanImagePrompt({
visualProfile,
shot,
+ ratio: visualProfile.ratio,
hasStyleReference: styleReference !== undefined,
hasPreviousShotReference: shouldUsePreviousShotReference(shot, shotIndex)
})
diff --git a/apps/daemon/src/creator/stickman/contracts.ts b/apps/daemon/src/creator/stickman/contracts.ts
index 2f81b259..e062bfda 100644
--- a/apps/daemon/src/creator/stickman/contracts.ts
+++ b/apps/daemon/src/creator/stickman/contracts.ts
@@ -1,4 +1,10 @@
import { z } from 'zod';
+import {
+ stickmanCanvasForRatio,
+ stickmanRatios
+} from '@opencreator/protocol';
+
+export const stickmanRatioSchema = z.enum(stickmanRatios);
export const stickmanSourceSpanSchema = z.object({
id: z.string().regex(/^source-\d{3}$/),
@@ -233,7 +239,7 @@ const stickmanRenderingContractSchema = z.object({
export const stickmanStyleContractSchema = z.object({
contract: z.literal('stickman-visual-profile-v2'),
- ratio: z.literal('16:9'),
+ ratio: stickmanRatioSchema,
character: z.object({
assetId: z.string().min(1),
revision: z.number().int().positive(),
@@ -260,6 +266,9 @@ export const stickmanStyleContractSchema = z.object({
export const stickmanVisualValidationSchema = z.object({
ok: z.literal(true),
validation: z.literal('automated_decode_aspect_nonblank_hash_and_ocr'),
+ ratio: stickmanRatioSchema,
+ width: z.number().int().positive(),
+ height: z.number().int().positive(),
approvedShotSpecArtifactId: z.string().min(1),
shotCount: z.number().int().positive(),
ocrStatus: z.enum(['passed', 'unavailable']),
@@ -277,6 +286,10 @@ export const stickmanVisualValidationSchema = z.object({
detectedText: z.array(z.string())
}).strict()).min(1).max(200)
}).strict().superRefine((value, context) => {
+ const canvas = stickmanCanvasForRatio(value.ratio);
+ if (value.width !== canvas.width || value.height !== canvas.height) {
+ context.addIssue({ code: 'custom', message: 'Visual validation dimensions do not match ratio' });
+ }
uniqueIds(value.shots.map(shot => shot.shotId), 'validated shot', context);
if (value.shotCount !== value.shots.length) {
context.addIssue({ code: 'custom', message: 'Visual validation shot count does not match' });
@@ -287,9 +300,10 @@ export const stickmanVisualValidationSchema = z.object({
});
export const stickmanTimelineSchema = z.object({
+ ratio: stickmanRatioSchema,
fps: z.number().int().positive(),
- width: z.literal(1280),
- height: z.literal(720),
+ width: z.number().int().positive(),
+ height: z.number().int().positive(),
totalFrames: z.number().int().positive(),
shots: z.array(z.object({
shotId: z.string().min(1),
@@ -302,49 +316,105 @@ export const stickmanTimelineSchema = z.object({
audioSha256: z.string().regex(/^[a-f0-9]{64}$/i),
imagePath: z.string().min(1).optional(),
audioPath: z.string().min(1).optional()
+ }).strict()).min(1),
+ captions: z.array(z.object({
+ segmentId: z.string().regex(/^segment-[a-z0-9-]+$/),
+ startFrame: z.number().int().nonnegative(),
+ endFrame: z.number().int().positive(),
+ text: z.string().trim().min(1)
}).strict()).min(1)
-}).strict();
+}).strict().superRefine((value, context) => {
+ const canvas = stickmanCanvasForRatio(value.ratio);
+ if (value.width !== canvas.width || value.height !== canvas.height) {
+ context.addIssue({ code: 'custom', message: 'Timeline dimensions do not match ratio' });
+ }
+ let cursor = 0;
+ for (const shot of value.shots) {
+ if (shot.startFrame !== cursor || shot.endFrame <= shot.startFrame) {
+ context.addIssue({ code: 'custom', message: `Invalid timeline shot range: ${shot.shotId}` });
+ }
+ cursor = shot.endFrame;
+ }
+ if (cursor !== value.totalFrames) {
+ context.addIssue({ code: 'custom', message: 'Timeline shots do not cover total frames' });
+ }
+ cursor = 0;
+ for (const caption of value.captions) {
+ if (caption.startFrame !== cursor || caption.endFrame <= caption.startFrame) {
+ context.addIssue({ code: 'custom', message: `Invalid timeline caption range: ${caption.segmentId}` });
+ }
+ cursor = caption.endFrame;
+ }
+ if (cursor !== value.totalFrames) {
+ context.addIssue({ code: 'custom', message: 'Timeline captions do not cover total frames' });
+ }
+});
export const stickmanMediaValidationSchema = z.object({
ok: z.literal(true),
validation: z.literal('ffprobe_and_three_frame_sampling'),
+ ratio: stickmanRatioSchema,
cleanVideoArtifactId: z.string().min(1),
cleanVideoSha256: z.string().regex(/^[a-f0-9]{64}$/i),
timelineArtifactId: z.string().min(1),
duration: z.number().positive(),
expectedDuration: z.number().positive(),
durationTolerance: z.number().positive(),
- width: z.literal(1280),
- height: z.literal(720),
+ width: z.number().int().positive(),
+ height: z.number().int().positive(),
hasVideo: z.literal(true),
hasAudio: z.literal(true),
sampledFrames: z.array(z.object({
index: z.number().int().min(1).max(3),
timestampSeconds: z.number().nonnegative(),
sha256: z.string().regex(/^[a-f0-9]{64}$/i),
- width: z.literal(1280),
- height: z.literal(720),
+ width: z.number().int().positive(),
+ height: z.number().int().positive(),
brightnessMean: z.number().nonnegative().max(255),
contrastStddev: z.number().nonnegative()
}).strict()).length(3)
-}).strict();
+}).strict().superRefine((value, context) => {
+ const canvas = stickmanCanvasForRatio(value.ratio);
+ if (value.width !== canvas.width || value.height !== canvas.height) {
+ context.addIssue({ code: 'custom', message: 'Media validation dimensions do not match ratio' });
+ }
+ if (value.sampledFrames.some(frame => frame.width !== canvas.width || frame.height !== canvas.height)) {
+ context.addIssue({ code: 'custom', message: 'Sampled frame dimensions do not match ratio' });
+ }
+});
export const stickmanDeliveryManifestSchema = z.object({
packageStatus: z.enum(['technical-draft', 'publishable']),
+ ratio: stickmanRatioSchema,
+ width: z.number().int().positive(),
+ height: z.number().int().positive(),
+ duration: z.number().positive(),
+ providers: z.object({
+ image: z.string().min(1),
+ video: z.string().min(1),
+ voice: z.string().min(1)
+ }).strict(),
placeholderAssets: z.array(z.string().min(1)),
blockingChecks: z.array(z.string().min(1)),
files: z.array(z.object({
name: z.enum([
- 'stickman-video.mp4',
- 'narration.srt'
+ 'short.mp4',
+ 'subtitles.srt',
+ 'thumbnail.png',
+ 'publish-copy.md'
]),
relativePath: z.string().min(1),
sha256: z.string().regex(/^[a-f0-9]{64}$/i),
bytes: z.number().int().nonnegative(),
mime: z.string().min(1),
sourceArtifactId: z.string().min(1)
- }).strict()).length(2)
-}).strict();
+ }).strict()).length(4)
+}).strict().superRefine((value, context) => {
+ const canvas = stickmanCanvasForRatio(value.ratio);
+ if (value.width !== canvas.width || value.height !== canvas.height) {
+ context.addIssue({ code: 'custom', message: 'Delivery dimensions do not match ratio' });
+ }
+});
export type StickmanSourceBrief = z.infer<typeof stickmanSourceBriefSchema>;
export type StickmanContentPlan = z.infer<typeof stickmanContentPlanSchema>;
diff --git a/apps/daemon/src/creator/stickman/delivery-executor.ts b/apps/daemon/src/creator/stickman/delivery-executor.ts
index 97d83d7d..18e6ec23 100644
--- a/apps/daemon/src/creator/stickman/delivery-executor.ts
+++ b/apps/daemon/src/creator/stickman/delivery-executor.ts
@@ -8,12 +8,13 @@ import {
readFile,
readdir,
realpath,
+ rm,
stat,
writeFile
} from 'node:fs/promises';
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
import sharp from 'sharp';
-import type { CreatorArtifact } from '@opencreator/protocol';
+import { stickmanCanvasForRatio, type CreatorArtifact } from '@opencreator/protocol';
import type { CreatorExecutor, CreatorExecutorOutput } from '../executor.js';
import { CreatorExecutorError } from '../executor.js';
import { validateMediaFile } from '../validators/media.js';
@@ -22,13 +23,15 @@ import {
stickmanAudioTimingSchema,
stickmanDeliveryManifestSchema,
stickmanMediaValidationSchema,
+ stickmanScriptManifestSchema,
stickmanTimelineSchema,
stickmanVisualValidationSchema
} from './contracts.js';
+import { renderStickmanPublishCopy } from './publish-copy.js';
const deliveryFiles = [
- { kind: 'clean_video', name: 'stickman-video.mp4', mime: 'video/mp4' },
- { kind: 'narration_subtitle', name: 'narration.srt', mime: 'application/x-subrip' }
+ { kind: 'clean_video', name: 'short.mp4', mime: 'video/mp4' },
+ { kind: 'narration_subtitle', name: 'subtitles.srt', mime: 'application/x-subrip' }
] as const;
type VideoValidator = typeof validateMediaFile;
@@ -40,11 +43,21 @@ type VideoFrameSampler = (input: {
ffmpegPath?: string;
}) => Promise<{ sampleCount: number }>;
+type ThumbnailGenerator = (input: {
+ path: string;
+ duration: number;
+ targetPath: string;
+ width: number;
+ height: number;
+ ffmpegPath?: string;
+}) => Promise<void>;
+
export function createStickmanDeliveryExecutor(input: {
ffprobePath: string;
ffmpegPath?: string;
validateVideo?: VideoValidator;
sampleVideoFrames?: VideoFrameSampler;
+ createThumbnail?: ThumbnailGenerator;
}): CreatorExecutor {
const validateVideo = input.validateVideo ?? validateMediaFile;
const sampleVideoFrames = input.sampleVideoFrames
@@ -62,42 +75,49 @@ export function createStickmanDeliveryExecutor(input: {
`Delivery directory is not empty: ${existing[0]}`
);
}
+ const timelineArtifact = requireSingleArtifact(stage.inputArtifacts, 'timeline_manifest');
+ const timelinePath = await validateSourcePath(jobRoot, timelineArtifact);
+ const timeline = stickmanTimelineSchema.parse(JSON.parse(await readFile(timelinePath, 'utf8')));
+ const canvas = stickmanCanvasForRatio(timeline.ratio);
+ const scriptArtifact = requireSingleArtifact(stage.inputArtifacts, 'script_manifest');
+ const scriptPath = await validateSourcePath(jobRoot, scriptArtifact);
+ const script = stickmanScriptManifestSchema.parse(JSON.parse(await readFile(scriptPath, 'utf8')));
const manifestFiles = [];
const outputs: CreatorExecutorOutput[] = [];
const sampledVideos = new Set<string>();
+ let cleanVideoMedia: Awaited<ReturnType<VideoValidator>> | undefined;
for (const definition of deliveryFiles) {
const artifact = requireSingleArtifact(stage.inputArtifacts, definition.kind);
const sourcePath = await validateSourcePath(jobRoot, artifact);
- const actualSha256 = await sha256File(sourcePath);
- if (artifact.sha256 === null || artifact.sha256.toLowerCase() !== actualSha256) {
- throw new CreatorExecutorError(
- 'creator_delivery_hash_mismatch',
- `Artifact hash mismatch: ${definition.kind}`
- );
- }
+ const actualSha256 = await assertArtifactHash(artifact, sourcePath);
const targetPath = join(deliveryRoot, definition.name);
await copyFile(sourcePath, targetPath, constants.COPYFILE_EXCL);
const info = await stat(targetPath);
- const media = await validateDeliveryFile(definition.kind, targetPath, input.ffprobePath, validateVideo);
- if (
- media !== undefined
- && sampleVideoFrames !== undefined
- && definition.kind === 'clean_video'
- ) {
- const samples = await sampleVideoFrames({
- path: targetPath,
- kind: definition.kind,
- duration: media.duration,
- workdir: join(stage.workdir, `frame-samples-${definition.kind}`),
- ...(input.ffmpegPath === undefined ? {} : { ffmpegPath: input.ffmpegPath })
- });
- if (samples.sampleCount < 3) {
- throw new CreatorExecutorError(
- 'creator_delivery_frame_sampling_failed',
- `${definition.kind} requires at least three decoded frame samples`
- );
+ const media = await validateDeliveryFile(
+ definition.kind,
+ targetPath,
+ input.ffprobePath,
+ validateVideo,
+ canvas
+ );
+ if (definition.kind === 'clean_video' && media !== undefined) {
+ cleanVideoMedia = media;
+ if (sampleVideoFrames !== undefined) {
+ const samples = await sampleVideoFrames({
+ path: targetPath,
+ kind: definition.kind,
+ duration: media.duration,
+ workdir: join(stage.workdir, `frame-samples-${definition.kind}`),
+ ...(input.ffmpegPath === undefined ? {} : { ffmpegPath: input.ffmpegPath })
+ });
+ if (samples.sampleCount < 3) {
+ throw new CreatorExecutorError(
+ 'creator_delivery_frame_sampling_failed',
+ `${definition.kind} requires at least three decoded frame samples`
+ );
+ }
+ sampledVideos.add(definition.kind);
}
- sampledVideos.add(definition.kind);
}
manifestFiles.push({
name: definition.name,
@@ -116,12 +136,99 @@ export function createStickmanDeliveryExecutor(input: {
fileName: definition.name,
mimeType: definition.mime,
bytes: info.size,
- delivery: true
+ delivery: true,
+ ratio: timeline.ratio,
+ width: canvas.width,
+ height: canvas.height
}
});
}
+ if (cleanVideoMedia === undefined) {
+ throw new CreatorExecutorError('creator_delivery_video_invalid', 'Clean video media was not validated');
+ }
+ const cleanVideoArtifact = requireSingleArtifact(stage.inputArtifacts, 'clean_video');
+ const thumbnailPath = join(deliveryRoot, 'thumbnail.png');
+ const thumbnailGenerator = input.createThumbnail
+ ?? (input.ffmpegPath === undefined ? undefined : renderThumbnailWithFfmpeg);
+ if (thumbnailGenerator === undefined) {
+ throw new CreatorExecutorError(
+ 'creator_delivery_thumbnail_failed',
+ 'FFmpeg is required to generate the delivery thumbnail'
+ );
+ }
+ await thumbnailGenerator({
+ path: join(deliveryRoot, 'short.mp4'),
+ duration: cleanVideoMedia.duration,
+ targetPath: thumbnailPath,
+ width: canvas.width,
+ height: canvas.height,
+ ...(input.ffmpegPath === undefined ? {} : { ffmpegPath: input.ffmpegPath })
+ });
+ const thumbnailInfo = await stat(thumbnailPath);
+ const thumbnailMetadata = await sharp(thumbnailPath).metadata();
+ if (thumbnailMetadata.width !== canvas.width || thumbnailMetadata.height !== canvas.height) {
+ throw new CreatorExecutorError(
+ 'creator_delivery_thumbnail_failed',
+ `Thumbnail must be ${canvas.width}x${canvas.height} media`
+ );
+ }
+ const thumbnailSha256 = await sha256File(thumbnailPath);
+ manifestFiles.push({
+ name: 'thumbnail.png',
+ relativePath: 'delivery/thumbnail.png',
+ sha256: thumbnailSha256,
+ bytes: thumbnailInfo.size,
+ mime: 'image/png',
+ sourceArtifactId: cleanVideoArtifact.id
+ });
+ outputs.push({
+ kind: 'thumbnail',
+ status: 'completed',
+ path: thumbnailPath,
+ sourceArtifactIds: [cleanVideoArtifact.id],
+ metadata: {
+ fileName: 'thumbnail.png',
+ mimeType: 'image/png',
+ bytes: thumbnailInfo.size,
+ delivery: true,
+ ratio: timeline.ratio,
+ width: canvas.width,
+ height: canvas.height
+ }
+ });
+ const publishCopyPath = join(deliveryRoot, 'publish-copy.md');
+ await writeFile(publishCopyPath, renderStickmanPublishCopy({
+ title: script.title,
+ language: script.language,
+ durationSeconds: cleanVideoMedia.duration,
+ ratio: timeline.ratio,
+ narration: script.segments.map(segment => segment.narration)
+ }), 'utf8');
+ const publishCopyInfo = await stat(publishCopyPath);
+ const publishCopySha256 = await sha256File(publishCopyPath);
+ manifestFiles.push({
+ name: 'publish-copy.md',
+ relativePath: 'delivery/publish-copy.md',
+ sha256: publishCopySha256,
+ bytes: publishCopyInfo.size,
+ mime: 'text/markdown',
+ sourceArtifactId: scriptArtifact.id
+ });
+ outputs.push({
+ kind: 'publish_copy',
+ status: 'completed',
+ path: publishCopyPath,
+ sourceArtifactIds: [scriptArtifact.id],
+ metadata: {
+ fileName: 'publish-copy.md',
+ mimeType: 'text/markdown',
+ bytes: publishCopyInfo.size,
+ delivery: true,
+ ratio: timeline.ratio
+ }
+ });
const actualFiles = await readdir(deliveryRoot);
- const expectedFiles = deliveryFiles.map(file => file.name).sort();
+ const expectedFiles = ['short.mp4', 'subtitles.srt', 'thumbnail.png', 'publish-copy.md'].sort();
if (JSON.stringify([...actualFiles].sort()) !== JSON.stringify(expectedFiles)) {
throw new CreatorExecutorError(
'creator_delivery_file_set_mismatch',
@@ -130,6 +237,7 @@ export function createStickmanDeliveryExecutor(input: {
}
const placeholderAssets = findPlaceholderAssets(stage.inputArtifacts);
const blockingChecks = await collectBlockingChecks({
+ jobRoot,
artifacts: stage.inputArtifacts,
sampledVideos
});
@@ -138,6 +246,15 @@ export function createStickmanDeliveryExecutor(input: {
: 'technical-draft' as const;
const manifest = stickmanDeliveryManifestSchema.parse({
packageStatus,
+ ratio: timeline.ratio,
+ width: canvas.width,
+ height: canvas.height,
+ duration: cleanVideoMedia.duration,
+ providers: {
+ image: readProvider(stage.inputArtifacts, 'shot_image'),
+ video: readVideoProvider(cleanVideoArtifact),
+ voice: readProvider(stage.inputArtifacts, 'narration_audio')
+ },
placeholderAssets,
blockingChecks,
files: manifestFiles
@@ -209,14 +326,15 @@ async function validateDeliveryFile(
kind: string,
path: string,
ffprobePath: string,
- validateVideo: VideoValidator
+ validateVideo: VideoValidator,
+ canvas: { width: number; height: number }
): Promise<Awaited<ReturnType<VideoValidator>> | undefined> {
if (kind === 'clean_video') {
const media = await validateVideo(path, ffprobePath);
- if (!media.hasVideo || !media.hasAudio || media.width !== 1280 || media.height !== 720) {
+ if (!media.hasVideo || !media.hasAudio || media.width !== canvas.width || media.height !== canvas.height) {
throw new CreatorExecutorError(
'creator_delivery_video_invalid',
- `${kind} must be 1280x720 audio/video media`
+ `${kind} must be ${canvas.width}x${canvas.height} audio/video media`
);
}
return media;
@@ -225,7 +343,66 @@ async function validateDeliveryFile(
return undefined;
}
+async function assertArtifactHash(artifact: CreatorArtifact, path: string): Promise<string> {
+ const actualSha256 = await sha256File(path);
+ if (artifact.sha256 === null || artifact.sha256.toLowerCase() !== actualSha256) {
+ throw new CreatorExecutorError(
+ 'creator_delivery_hash_mismatch',
+ `Artifact hash mismatch: ${artifact.kind}`
+ );
+ }
+ return actualSha256;
+}
+
+function readProvider(artifacts: CreatorArtifact[], kind: string): string {
+ const providers = artifacts
+ .filter(artifact => artifact.kind === kind && artifact.status === 'completed')
+ .map(artifact => artifact.metadata.provider)
+ .filter((provider): provider is string => typeof provider === 'string' && provider.trim().length > 0);
+ return providers[0] ?? 'unknown';
+}
+
+function readVideoProvider(artifact: CreatorArtifact): string {
+ const provider = artifact.metadata.provider;
+ if (typeof provider === 'string' && provider.trim().length > 0) return provider;
+ const renderEngine = artifact.metadata.renderEngine;
+ return typeof renderEngine === 'string' && renderEngine.trim().length > 0
+ ? renderEngine
+ : 'unknown';
+}
+
+async function renderThumbnailWithFfmpeg(input: {
+ path: string;
+ duration: number;
+ targetPath: string;
+ width: number;
+ height: number;
+ ffmpegPath?: string;
+}): Promise<void> {
+ if (input.ffmpegPath === undefined) {
+ throw new CreatorExecutorError(
+ 'creator_delivery_thumbnail_failed',
+ 'FFmpeg is required to generate the delivery thumbnail'
+ );
+ }
+ const sourcePath = `${input.targetPath}.source.png`;
+ const timestamp = Math.max(0, Math.min(input.duration / 2, Math.max(0, input.duration - 0.1)));
+ try {
+ await execFileAsync(input.ffmpegPath, [
+ '-y', '-ss', timestamp.toFixed(3), '-i', input.path,
+ '-frames:v', '1', '-f', 'image2', sourcePath
+ ]);
+ await sharp(sourcePath)
+ .resize(input.width, input.height, { fit: 'cover', position: 'centre' })
+ .png()
+ .toFile(input.targetPath);
+ } finally {
+ await rm(sourcePath, { force: true });
+ }
+}
+
async function collectBlockingChecks(input: {
+ jobRoot: string;
artifacts: CreatorArtifact[];
sampledVideos: Set<string>;
}): Promise<string[]> {
@@ -235,7 +412,11 @@ async function collectBlockingChecks(input: {
blocking.add('visual_validation_missing');
} else {
try {
- const report = stickmanVisualValidationSchema.parse(JSON.parse(await readFile(visual.path, 'utf8')));
+ const visualPath = await validateSourcePath(input.jobRoot, visual);
+ await assertArtifactHash(visual, visualPath);
+ const report = stickmanVisualValidationSchema.parse(
+ JSON.parse(await readFile(visualPath, 'utf8'))
+ );
if (!report.publishable || report.ocrStatus !== 'passed') blocking.add('visual_ocr_unverified');
} catch {
blocking.add('visual_validation_invalid');
@@ -247,7 +428,7 @@ async function collectBlockingChecks(input: {
));
if (
narration.length === 0
- || !(await everyArtifactHashMatches(narration))
+ || !(await everyArtifactHashMatches(input.jobRoot, narration))
|| narration.some(artifact => (
artifact.scopeKey === null
|| artifact.metadata.timingSource !== 'ffprobe'
@@ -262,7 +443,11 @@ async function collectBlockingChecks(input: {
blocking.add('audio_timing_missing');
} else {
try {
- const timing = stickmanAudioTimingSchema.parse(JSON.parse(await readFile(timingArtifact.path, 'utf8')));
+ const timingPath = await validateSourcePath(input.jobRoot, timingArtifact);
+ await assertArtifactHash(timingArtifact, timingPath);
+ const timing = stickmanAudioTimingSchema.parse(
+ JSON.parse(await readFile(timingPath, 'utf8'))
+ );
const narrationById = new Map(narration.map(artifact => [artifact.id, artifact]));
if (timing.segments.some(segment => {
const artifact = narrationById.get(segment.audioArtifactId);
@@ -278,7 +463,11 @@ async function collectBlockingChecks(input: {
blocking.add('timeline_missing');
} else {
try {
- stickmanTimelineSchema.parse(JSON.parse(await readFile(timelineArtifact.path, 'utf8')));
+ const timelinePath = await validateSourcePath(input.jobRoot, timelineArtifact);
+ await assertArtifactHash(timelineArtifact, timelinePath);
+ stickmanTimelineSchema.parse(
+ JSON.parse(await readFile(timelinePath, 'utf8'))
+ );
if (timelineArtifact.metadata.timingSource !== 'ffprobe_cumulative_tts_duration') {
blocking.add('timeline_unverified');
}
@@ -297,8 +486,10 @@ async function collectBlockingChecks(input: {
blocking.add('media_validation_missing');
} else {
try {
+ const mediaValidationPath = await validateSourcePath(input.jobRoot, mediaValidation);
+ await assertArtifactHash(mediaValidation, mediaValidationPath);
const report = stickmanMediaValidationSchema.parse(JSON.parse(
- await readFile(mediaValidation.path, 'utf8')
+ await readFile(mediaValidationPath, 'utf8')
));
if (
report.cleanVideoArtifactId !== cleanVideo.id
@@ -332,11 +523,15 @@ function singleCompleted(artifacts: CreatorArtifact[], kind: string): CreatorArt
return matches.length === 1 ? matches[0] : undefined;
}
-async function everyArtifactHashMatches(artifacts: CreatorArtifact[]): Promise<boolean> {
+async function everyArtifactHashMatches(
+ jobRoot: string,
+ artifacts: CreatorArtifact[]
+): Promise<boolean> {
for (const artifact of artifacts) {
if (artifact.path === null || artifact.sha256 === null) return false;
try {
- if ((await sha256File(artifact.path)) !== artifact.sha256.toLowerCase()) return false;
+ const path = await validateSourcePath(jobRoot, artifact);
+ if ((await sha256File(path)) !== artifact.sha256.toLowerCase()) return false;
} catch {
return false;
}
diff --git a/apps/daemon/src/creator/stickman/image-executor.ts b/apps/daemon/src/creator/stickman/image-executor.ts
index a7de5d5c..785b59b2 100644
--- a/apps/daemon/src/creator/stickman/image-executor.ts
+++ b/apps/daemon/src/creator/stickman/image-executor.ts
@@ -1,7 +1,14 @@
-import type { ImageGenerationProvider, ImageGenerationQuality } from '@opencreator/protocol';
+import {
+ readStickmanRatio,
+ stickmanCanvasForRatio,
+ stickmanImageSizeForRatio,
+ type ImageGenerationProvider,
+ type ImageGenerationQuality,
+ type StickmanCanvas
+} from '@opencreator/protocol';
import { execFile } from 'node:child_process';
import { createHash } from 'node:crypto';
-import { copyFile, readFile } from 'node:fs/promises';
+import { copyFile, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import sharp from 'sharp';
import type { CreatorServicesConfigStore } from '../../creator-services/config-store.js';
@@ -37,11 +44,12 @@ export function createStickmanImageExecutor(input: {
generate?: typeof generateImageContents;
codexNative?: CodexNativeImageRuntime;
tesseractPath?: string;
- validateCandidate?: (path: string) => Promise<StickmanImageCandidateQuality>;
+ validateCandidate?: (
+ path: string,
+ expectedCanvas?: StickmanCanvas
+ ) => Promise<StickmanImageCandidateQuality>;
}): CreatorExecutor {
const generate = input.generate ?? generateImageContents;
- const validateCandidate = input.validateCandidate
- ?? (path => inspectStickmanImageCandidate(path, input.tesseractPath));
return {
id: 'stickman-image',
async run(stage) {
@@ -50,6 +58,15 @@ export function createStickmanImageExecutor(input: {
}
const scopeKey = stage.stageRun.scopeKey;
const inputFingerprint = stage.stageRun.inputFingerprint;
+ const ratio = readStickmanRatio(stage.job.state.ratio);
+ const canvas = stickmanCanvasForRatio(ratio);
+ const imageSize = stickmanImageSizeForRatio(ratio);
+ const validateCandidate = input.validateCandidate
+ ?? ((path, expectedCanvas) => inspectStickmanImageCandidate(
+ path,
+ input.tesseractPath,
+ expectedCanvas ?? canvas
+ ));
if (scopeKey === null || inputFingerprint === null) {
throw new CreatorExecutorError('creator_stage_scope_missing', 'Shot scope and fingerprint are required');
}
@@ -231,6 +248,9 @@ export function createStickmanImageExecutor(input: {
const quality = readQuality(stage.job.state.quality);
assertReferenceImageSupport(provider, referenceImages.length);
const model = provider === 'codex-native' ? 'codex-native' : config.image[provider].model;
+ const codexNativeImagePaths = provider === 'codex-native'
+ ? await materializeCodexNativeReferences(referenceImages, stage.workdir)
+ : undefined;
if (
promptPackArtifact.metadata.contract !== STICKMAN_IMAGE_PROMPT_CONTRACT
|| promptPack.characterReferenceArtifactId !== characterReference.id
@@ -246,7 +266,7 @@ export function createStickmanImageExecutor(input: {
const request = {
prompt: generationPrompt,
provider,
- size: '1536x1024' as const,
+ size: imageSize,
quality,
count: 1
};
@@ -275,12 +295,16 @@ export function createStickmanImageExecutor(input: {
characterReferenceSha256,
requestReferenceSha256,
characterReferenceMime,
+ ratio,
+ width: canvas.width,
+ height: canvas.height,
referenceImages: referenceImages.map(reference => ({
role: reference.role,
sha256: reference.sha256,
mimeType: reference.mime,
artifactId: reference.artifactId
})),
+ ...(codexNativeImagePaths === undefined ? {} : { codexNativeImagePaths }),
characterAsset: selectedCharacter,
styleAsset: selectedStyle,
referenceImagePreparation: STICKMAN_CHARACTER_REFERENCE_PREPARATION,
@@ -306,6 +330,7 @@ export function createStickmanImageExecutor(input: {
content: reference.content,
mime: reference.mime
})),
+ ...(codexNativeImagePaths === undefined ? {} : { codexNativeImagePaths }),
...(input.codexNative === undefined
? {}
: { codexNative: { ...input.codexNative, cwd: stage.workdir } })
@@ -324,11 +349,11 @@ export function createStickmanImageExecutor(input: {
const extension = image.mime === 'image/jpeg' ? 'jpg' : image.mime === 'image/webp' ? 'webp' : 'png';
const candidatePath = join(stage.workdir, `${scopeKey}-candidate-${candidateAttempt}.${extension}`);
await sharp(image.content)
- .resize(1280, 720, { fit: 'cover', position: 'centre' })
+ .resize(canvas.width, canvas.height, { fit: 'cover', position: 'centre' })
.toFile(candidatePath);
let candidateQuality: StickmanImageCandidateQuality;
try {
- candidateQuality = await validateCandidate(candidatePath);
+ candidateQuality = await validateCandidate(candidatePath, canvas);
} catch (error) {
candidateFailures.push(
error instanceof Error ? error.message : `Candidate ${candidateAttempt} failed quality checks`
@@ -348,6 +373,7 @@ export function createStickmanImageExecutor(input: {
metadata: {
...metadata,
shotId: scopeKey,
+ ratio,
prompt: candidateRequest.prompt,
provider,
model: result.model,
@@ -368,6 +394,7 @@ export function createStickmanImageExecutor(input: {
mimeType: reference.mime,
artifactId: reference.artifactId
})),
+ ...(codexNativeImagePaths === undefined ? {} : { codexNativeImagePaths }),
characterAssetId: selectedCharacter.assetId,
characterRevision: selectedCharacter.revision,
styleAssetId: selectedStyle.assetId,
@@ -411,9 +438,31 @@ type StickmanImageCandidateQuality = {
detectedText: string[];
};
+async function materializeCodexNativeReferences(
+ references: Array<{
+ role: string;
+ content: Buffer;
+ mime: 'image/png' | 'image/jpeg' | 'image/webp';
+ }>,
+ workdir: string
+): Promise<string[]> {
+ return await Promise.all(references.map(async (reference, index) => {
+ const extension = reference.mime === 'image/jpeg'
+ ? 'jpg'
+ : reference.mime === 'image/webp'
+ ? 'webp'
+ : 'png';
+ const safeRole = reference.role.replace(/[^a-z0-9_-]/gi, '-');
+ const path = join(workdir, `codex-reference-${String(index + 1).padStart(2, '0')}-${safeRole}.${extension}`);
+ await writeFile(path, reference.content);
+ return path;
+ }));
+}
+
async function inspectStickmanImageCandidate(
path: string,
- tesseractPath?: string
+ tesseractPath?: string,
+ expectedCanvas: StickmanCanvas = stickmanCanvasForRatio('16:9')
): Promise<StickmanImageCandidateQuality> {
let metadata: Awaited<ReturnType<typeof sharp.prototype.metadata>>;
let stats: Awaited<ReturnType<typeof sharp.prototype.stats>>;
@@ -427,8 +476,11 @@ async function inspectStickmanImageCandidate(
}
const width = metadata.width ?? 0;
const height = metadata.height ?? 0;
- if (width <= 0 || height <= 0 || Math.abs(width / height - 16 / 9) > 0.03) {
- throw new CreatorExecutorError('creator_shot_image_invalid', 'Generated image is not 16:9');
+ if (width !== expectedCanvas.width || height !== expectedCanvas.height) {
+ throw new CreatorExecutorError(
+ 'creator_shot_image_invalid',
+ `Generated image is not ${expectedCanvas.width}x${expectedCanvas.height} ${expectedCanvas.ratio} media`
+ );
}
const brightnessMean = stats.channels[0]?.mean ?? 0;
const contrastStddev = stats.channels[0]?.stdev ?? 0;
@@ -492,7 +544,7 @@ function assertReferenceImageSupport(provider: ImageGenerationProvider, count: n
if (capabilities.supportsReferenceImage && capabilities.maxReferenceImages >= count) return;
throw new CreatorExecutorError(
'creator_stickman_image_provider_unsupported',
- `当前生图服务 ${provider} 不支持任务需要的 ${count} 张参考图。请在 AI 服务 -> 生图服务中切换到支持多参考图的 OpenAI 或 Gemini 服务`
+ `The ${provider} image provider does not support the ${count} reference images required by this job. Use Codex (subscription), OpenAI, or Gemini with multi-reference support`
);
}
diff --git a/apps/daemon/src/creator/stickman/image-prompt.ts b/apps/daemon/src/creator/stickman/image-prompt.ts
index 0a27e821..89e9ad87 100644
--- a/apps/daemon/src/creator/stickman/image-prompt.ts
+++ b/apps/daemon/src/creator/stickman/image-prompt.ts
@@ -1,3 +1,4 @@
+import type { StickmanRatio } from '@opencreator/protocol';
import type { StickmanShot, StickmanStyleContract } from './contracts.js';
export const STICKMAN_CHARACTER_REFERENCE_PREPARATION = 'original-bytes-v2';
@@ -10,6 +11,7 @@ export function shouldUsePreviousShotReference(shot: StickmanShot, shotIndex: nu
export function buildStickmanImagePrompt(input: {
visualProfile: StickmanStyleContract;
shot: StickmanShot;
+ ratio: StickmanRatio;
hasStyleReference: boolean;
hasPreviousShotReference: boolean;
}): string {
@@ -32,7 +34,13 @@ export function buildStickmanImagePrompt(input: {
}
return [
- 'Create exactly one full-frame 16:9 narrative image for a recurring stick-figure video.',
+ `Create exactly one full-frame ${input.ratio} narrative image for a recurring stick-figure video.`,
+ ...(input.ratio === '9:16'
+ ? [
+ 'This is a vertical 9:16 YouTube Short. Keep the protagonist and action inside the central safe area so captions can occupy the lower third.',
+ 'Use a clear vertical composition with readable foreground, middle ground, and background; do not letterbox, pillarbox, or split the frame.'
+ ]
+ : []),
'',
'[REFERENCE RESPONSIBILITIES]',
...referenceInstructions,
diff --git a/apps/daemon/src/creator/stickman/media-validation-executor.ts b/apps/daemon/src/creator/stickman/media-validation-executor.ts
index a6bc00b4..cb153f32 100644
--- a/apps/daemon/src/creator/stickman/media-validation-executor.ts
+++ b/apps/daemon/src/creator/stickman/media-validation-executor.ts
@@ -4,7 +4,10 @@ import { createReadStream } from 'node:fs';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
import sharp from 'sharp';
-import type { CreatorArtifact } from '@opencreator/protocol';
+import {
+ stickmanCanvasForRatio,
+ type CreatorArtifact
+} from '@opencreator/protocol';
import type { CreatorExecutor } from '../executor.js';
import { CreatorExecutorError } from '../executor.js';
import { validateMediaFile } from '../validators/media.js';
@@ -15,8 +18,8 @@ type FrameSample = {
index: number;
timestampSeconds: number;
sha256: string;
- width: 1280;
- height: 720;
+ width: number;
+ height: number;
brightnessMean: number;
contrastStddev: number;
};
@@ -28,8 +31,6 @@ export function createStickmanMediaValidationExecutor(input: {
sampleFrames?: (path: string, duration: number, workdir: string) => Promise<FrameSample[]>;
}): CreatorExecutor {
const validateVideo = input.validateVideo ?? validateMediaFile;
- const sampleFrames = input.sampleFrames
- ?? ((path, duration, workdir) => sampleVideoFrames(input.ffmpegPath, path, duration, workdir));
return {
id: 'stickman-media-validation',
async run(stage) {
@@ -64,26 +65,35 @@ export function createStickmanMediaValidationExecutor(input: {
const timeline = stickmanTimelineSchema.parse(JSON.parse(
await readFile(timelineArtifact.path, 'utf8')
));
+ const canvas = stickmanCanvasForRatio(timeline.ratio);
const expectedDuration = timeline.totalFrames / timeline.fps;
const durationTolerance = Math.max(0.15, 2 / timeline.fps);
const media = await validateVideo(cleanVideo.path, input.ffprobePath);
if (
- media.width !== 1280
- || media.height !== 720
+ media.width !== canvas.width
+ || media.height !== canvas.height
|| !media.hasVideo
|| !media.hasAudio
|| Math.abs(media.duration - expectedDuration) > durationTolerance
) {
throw new CreatorExecutorError(
'creator_media_validation_failed',
- 'Clean video must be decodable 1280x720 audio/video media with Timeline-matched duration'
+ `Clean video must be decodable ${canvas.width}x${canvas.height} audio/video media with Timeline-matched duration`
);
}
- const sampledFrames = await sampleFrames(
- cleanVideo.path,
- media.duration,
- join(stage.workdir, 'frame-samples')
- );
+ const sampledFrames = input.sampleFrames === undefined
+ ? await sampleVideoFrames(
+ input.ffmpegPath,
+ cleanVideo.path,
+ media.duration,
+ join(stage.workdir, 'frame-samples'),
+ canvas
+ )
+ : await input.sampleFrames(
+ cleanVideo.path,
+ media.duration,
+ join(stage.workdir, 'frame-samples')
+ );
if (sampledFrames.length !== 3) {
throw new CreatorExecutorError(
'creator_media_validation_frame_count',
@@ -99,8 +109,9 @@ export function createStickmanMediaValidationExecutor(input: {
duration: media.duration,
expectedDuration,
durationTolerance,
- width: 1280,
- height: 720,
+ ratio: timeline.ratio,
+ width: canvas.width,
+ height: canvas.height,
hasVideo: true,
hasAudio: true,
sampledFrames
@@ -117,6 +128,7 @@ export function createStickmanMediaValidationExecutor(input: {
validation: report.validation,
sampleCount: report.sampledFrames.length,
duration: report.duration,
+ ratio: report.ratio,
width: report.width,
height: report.height
}
@@ -131,7 +143,8 @@ async function sampleVideoFrames(
ffmpegPath: string,
videoPath: string,
duration: number,
- workdir: string
+ workdir: string,
+ canvas: { width: number; height: number }
): Promise<FrameSample[]> {
await mkdir(workdir, { recursive: true });
const timestamps = [
@@ -154,8 +167,8 @@ async function sampleVideoFrames(
const brightnessMean = stats.channels[0]?.mean ?? 0;
const contrastStddev = stats.channels[0]?.stdev ?? 0;
if (
- metadata.width !== 1280
- || metadata.height !== 720
+ metadata.width !== canvas.width
+ || metadata.height !== canvas.height
|| brightnessMean < 3
|| brightnessMean > 252
|| contrastStddev < 2
@@ -169,8 +182,8 @@ async function sampleVideoFrames(
index: position + 1,
timestampSeconds,
sha256,
- width: 1280,
- height: 720,
+ width: canvas.width,
+ height: canvas.height,
brightnessMean: roundMetric(brightnessMean),
contrastStddev: roundMetric(contrastStddev)
});
diff --git a/apps/daemon/src/creator/stickman/publish-copy.ts b/apps/daemon/src/creator/stickman/publish-copy.ts
new file mode 100644
index 00000000..b734aaf6
--- /dev/null
+++ b/apps/daemon/src/creator/stickman/publish-copy.ts
@@ -0,0 +1,30 @@
+import type { StickmanRatio } from '@opencreator/protocol';
+
+export function renderStickmanPublishCopy(input: {
+ title: string;
+ language: string;
+ durationSeconds: number;
+ ratio: StickmanRatio;
+ narration: string[];
+}): string {
+ const description = input.narration.join(' ').replace(/\s+/g, ' ').trim();
+ const roundedDuration = Math.max(1, Math.round(input.durationSeconds));
+ const minutes = Math.floor(roundedDuration / 60);
+ const seconds = roundedDuration % 60;
+ const format = input.ratio === '9:16' ? 'YouTube Short (9:16)' : 'YouTube video (16:9)';
+ const hashtags = input.ratio === '9:16'
+ ? '#Shorts #StickFigure #Explained'
+ : '#StickFigure #Explained';
+ return [
+ `# ${input.title.trim()}`,
+ '',
+ 'Description:',
+ description,
+ '',
+ `Language: ${input.language.trim()}`,
+ `Duration: ${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`,
+ `Format: ${format}`,
+ '',
+ `Hashtags: ${hashtags}`
+ ].join('\n') + '\n';
+}
diff --git a/apps/daemon/src/creator/stickman/remotion-executor.ts b/apps/daemon/src/creator/stickman/remotion-executor.ts
index a12270c8..41c5a823 100644
--- a/apps/daemon/src/creator/stickman/remotion-executor.ts
+++ b/apps/daemon/src/creator/stickman/remotion-executor.ts
@@ -3,6 +3,7 @@ import { createRequire } from 'node:module';
import { dirname, extname, join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { Worker } from 'node:worker_threads';
+import { readStickmanRatio, stickmanCanvasForRatio } from '@opencreator/protocol';
import type { CreatorExecutor } from '../executor.js';
import { CreatorExecutorError } from '../executor.js';
import { validateMediaFile } from '../validators/media.js';
@@ -31,8 +32,18 @@ export function createStickmanRemotionExecutor(input: {
if (timeline?.path === null || timeline?.path === undefined) {
throw new CreatorExecutorError('creator_stage_input_missing', 'Timeline manifest is required');
}
+ const timelineValue = JSON.parse(await readFile(timeline.path, 'utf8')) as {
+ ratio?: unknown;
+ totalFrames?: number;
+ fps?: number;
+ };
+ const ratio = readStickmanRatio(timelineValue.ratio);
+ const canvas = stickmanCanvasForRatio(ratio);
const runtime = input.runtime ?? readStickmanRemotionRuntime(input.runtimeRoot);
- const outputPath = join(stage.workdir, 'landscape-clean.mp4');
+ const outputPath = join(
+ stage.workdir,
+ ratio === '9:16' ? 'portrait-clean.mp4' : 'landscape-clean.mp4'
+ );
const requestPath = join(stage.workdir, 'remotion-request.json');
const resultPath = join(stage.workdir, 'remotion-result.json');
await writeFile(requestPath, `${JSON.stringify({
@@ -59,15 +70,11 @@ export function createStickmanRemotionExecutor(input: {
);
}
const media = await validateVideo(outputPath, input.ffprobePath);
- const timelineValue = JSON.parse(await readFile(timeline.path, 'utf8')) as {
- totalFrames?: number;
- fps?: number;
- };
const expectedDuration = Number(timelineValue.totalFrames) / Number(timelineValue.fps);
const durationTolerance = Math.max(0.15, 2 / Number(timelineValue.fps));
if (
- media.width !== 1280
- || media.height !== 720
+ media.width !== canvas.width
+ || media.height !== canvas.height
|| !media.hasVideo
|| !media.hasAudio
|| !Number.isFinite(expectedDuration)
@@ -75,7 +82,7 @@ export function createStickmanRemotionExecutor(input: {
) {
throw new CreatorExecutorError(
'stickman_render_invalid',
- 'Rendered video must be decodable 1280x720 media with matching audio duration'
+ `Rendered video must be decodable ${canvas.width}x${canvas.height} media with matching audio duration`
);
}
return {
@@ -86,7 +93,8 @@ export function createStickmanRemotionExecutor(input: {
sourceArtifactIds: [timeline.id],
metadata: {
...media,
- fileName: 'landscape-clean.mp4',
+ ratio,
+ fileName: ratio === '9:16' ? 'portrait-clean.mp4' : 'landscape-clean.mp4',
renderEngine: 'remotion',
renderKind: 'final',
mediaValidation: 'ffprobe'
diff --git a/apps/daemon/src/creator/stickman/remotion-worker.ts b/apps/daemon/src/creator/stickman/remotion-worker.ts
index 350cc1b8..310b32bc 100644
--- a/apps/daemon/src/creator/stickman/remotion-worker.ts
+++ b/apps/daemon/src/creator/stickman/remotion-worker.ts
@@ -35,6 +35,7 @@ async function main(): Promise<void> {
throw new Error('stickman_worker_browser_missing');
}
const timeline = JSON.parse(await readFile(resolve(request.timelinePath), 'utf8')) as {
+ ratio?: '16:9' | '9:16';
totalFrames: number;
fps: number;
shots?: Array<{ imagePath?: string; audioPath?: string }>;
@@ -60,7 +61,7 @@ async function main(): Promise<void> {
const renderer = await import('@remotion/renderer');
const composition = await renderer.selectComposition({
serveUrl: assetServer.url,
- id: 'StickmanLandscape',
+ id: timeline.ratio === '9:16' ? 'StickmanPortrait' : 'StickmanLandscape',
inputProps: renderTimeline,
browserExecutable: resolve(request.browserExecutable)
});
diff --git a/apps/daemon/src/creator/stickman/timeline-executor.ts b/apps/daemon/src/creator/stickman/timeline-executor.ts
index 1653950c..3ba9cf1e 100644
--- a/apps/daemon/src/creator/stickman/timeline-executor.ts
+++ b/apps/daemon/src/creator/stickman/timeline-executor.ts
@@ -1,6 +1,10 @@
import { realpath, readFile, writeFile } from 'node:fs/promises';
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
import type { CreatorArtifact } from '@opencreator/protocol';
+import {
+ readStickmanRatio,
+ stickmanCanvasForRatio
+} from '@opencreator/protocol';
import type { CreatorExecutor, CreatorExecutorOutput } from '../executor.js';
import { CreatorExecutorError } from '../executor.js';
import {
@@ -47,6 +51,8 @@ export function createStickmanTimelineExecutor(): CreatorExecutor {
const timingBySegment = new Map(timing.segments.map(segment => [segment.segmentId, segment]));
const fps = 30;
const totalFrames = Math.round(timing.totalDurationSeconds * fps);
+ const ratio = readStickmanRatio(stage.job.state?.ratio);
+ const canvas = stickmanCanvasForRatio(ratio);
if (totalFrames < shotSpec.shots.length) {
throw new CreatorExecutorError(
'creator_audio_too_short',
@@ -111,15 +117,40 @@ export function createStickmanTimelineExecutor(): CreatorExecutor {
audioPath: audio.path
});
}
+ const captions = script.segments.map(segment => {
+ const measured = timingBySegment.get(segment.id);
+ if (measured === undefined) {
+ throw new CreatorExecutorError(
+ 'creator_audio_timing_incomplete',
+ `Audio timing is missing ${segment.id}`
+ );
+ }
+ const startFrame = Math.round(measured.startSeconds * fps);
+ const endFrame = Math.round(measured.endSeconds * fps);
+ if (endFrame <= startFrame) {
+ throw new CreatorExecutorError(
+ 'creator_audio_too_short',
+ `Narration segment ${segment.id} is shorter than one render frame`
+ );
+ }
+ return {
+ segmentId: segment.id,
+ startFrame,
+ endFrame,
+ text: segment.narration
+ };
+ });
const timeline = stickmanTimelineSchema.parse({
+ ratio,
fps,
- width: 1280,
- height: 720,
+ width: canvas.width,
+ height: canvas.height,
totalFrames,
- shots
+ shots,
+ captions
});
const path = join(stage.workdir, 'timeline-manifest.json');
- await writeFile(path, `${JSON.stringify({ ...timeline, shots }, null, 2)}\n`, 'utf8');
+ await writeFile(path, `${JSON.stringify(timeline, null, 2)}\n`, 'utf8');
const subtitlePath = join(stage.workdir, 'narration.srt');
await writeFile(subtitlePath, renderStickmanTimedNarrationSrt(script, timing), 'utf8');
const outputs: CreatorExecutorOutput[] = [
@@ -135,9 +166,10 @@ export function createStickmanTimelineExecutor(): CreatorExecutor {
...images.map(image => image.id)
],
metadata: {
+ ratio,
fps,
- width: 1280,
- height: 720,
+ width: canvas.width,
+ height: canvas.height,
totalFrames,
duration: timing.totalDurationSeconds,
shotCount: shots.length,
diff --git a/apps/daemon/src/creator/stickman/tts.ts b/apps/daemon/src/creator/stickman/tts.ts
new file mode 100644
index 00000000..6b7a0e79
--- /dev/null
+++ b/apps/daemon/src/creator/stickman/tts.ts
@@ -0,0 +1,6 @@
+export function stickmanEdgeTtsVoiceForLanguage(language: unknown): string {
+ const value = typeof language === 'string' ? language.toLowerCase() : '';
+ if (value.startsWith('zh')) return 'zh-CN-XiaoxiaoNeural';
+ if (value.startsWith('ja')) return 'ja-JP-NanamiNeural';
+ return 'en-US-AriaNeural';
+}
diff --git a/apps/daemon/src/creator/stickman/validation-executor.ts b/apps/daemon/src/creator/stickman/validation-executor.ts
index 0677e26a..cf3d4ba7 100644
--- a/apps/daemon/src/creator/stickman/validation-executor.ts
+++ b/apps/daemon/src/creator/stickman/validation-executor.ts
@@ -3,6 +3,7 @@ import { execFile } from 'node:child_process';
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import sharp from 'sharp';
+import { readStickmanRatio, stickmanCanvasForRatio } from '@opencreator/protocol';
import type { CreatorExecutor } from '../executor.js';
import { CreatorExecutorError } from '../executor.js';
import { stickmanShotSpecSchema, stickmanVisualValidationSchema } from './contracts.js';
@@ -26,6 +27,8 @@ export function createStickmanValidationExecutor(input: {
if (shotSpec?.path === null || shotSpec?.path === undefined) {
throw new CreatorExecutorError('creator_stage_input_missing', 'Shot spec is required');
}
+ const ratio = readStickmanRatio(stage.job.state?.ratio);
+ const canvas = stickmanCanvasForRatio(ratio);
const value = stickmanShotSpecSchema.parse(JSON.parse(
await readFile(shotSpec.path, 'utf8')
));
@@ -69,10 +72,14 @@ export function createStickmanValidationExecutor(input: {
]);
const width = metadata.width ?? 0;
const height = metadata.height ?? 0;
- if (width <= 0 || height <= 0 || Math.abs(width / height - 16 / 9) > 0.03) {
+ if (
+ width <= 0
+ || height <= 0
+ || Math.abs(width / height - canvas.width / canvas.height) > 0.03
+ ) {
throw new CreatorExecutorError(
'creator_shot_image_invalid',
- `Shot ${shot.id} image must be decodable 16:9 media`
+ `Shot ${shot.id} image must be decodable ${ratio} media`
);
}
const brightnessMean = stats.channels[0]?.mean ?? 0;
@@ -122,6 +129,9 @@ export function createStickmanValidationExecutor(input: {
const result = stickmanVisualValidationSchema.parse({
ok: true,
validation: 'automated_decode_aspect_nonblank_hash_and_ocr',
+ ratio,
+ width: canvas.width,
+ height: canvas.height,
approvedShotSpecArtifactId: shotSpec.id,
shotCount: value.shots.length,
ocrStatus,
@@ -138,6 +148,9 @@ export function createStickmanValidationExecutor(input: {
path,
sourceArtifactIds: [shotSpec.id, ...images.map(image => image.id)],
metadata: {
+ ratio,
+ width: canvas.width,
+ height: canvas.height,
shotCount: value.shots.length,
warningCount: warnings.length,
ocrStatus,
diff --git a/apps/daemon/src/creator/templates/stickman-video-actions.ts b/apps/daemon/src/creator/templates/stickman-video-actions.ts
index 4acb6e24..3bfcd860 100644
--- a/apps/daemon/src/creator/templates/stickman-video-actions.ts
+++ b/apps/daemon/src/creator/templates/stickman-video-actions.ts
@@ -32,6 +32,7 @@ import {
DEFAULT_STICKMAN_STYLE_ASSET,
readVisualAssetRef
} from '../stickman/visual-assets.js';
+import { stickmanEdgeTtsVoiceForLanguage } from '../stickman/tts.js';
export type StickmanVideoWorkflow = ReturnType<typeof createStickmanVideoWorkflow>;
@@ -443,11 +444,18 @@ export function createStickmanVideoWorkflow(input: {
const config = await input.configStore.read();
const provider = job.state.ttsProvider === 'openai'
|| job.state.ttsProvider === 'aliyun'
+ || job.state.ttsProvider === 'edge-tts'
|| job.state.ttsProvider === 'minimax'
? job.state.ttsProvider
: config.tts.provider;
if (provider === 'edge-tts') {
- return { provider, model: '', voiceId: '' };
+ return {
+ provider,
+ model: '',
+ voiceId: typeof job.state.voiceCode === 'string' && job.state.voiceCode.trim()
+ ? job.state.voiceCode.trim()
+ : stickmanEdgeTtsVoiceForLanguage(job.state.targetLanguage)
+ };
}
const providerConfig = config.tts[provider];
return {
diff --git a/apps/daemon/src/creator/templates/stickman-video.ts b/apps/daemon/src/creator/templates/stickman-video.ts
index e75c4df3..a497f522 100644
--- a/apps/daemon/src/creator/templates/stickman-video.ts
+++ b/apps/daemon/src/creator/templates/stickman-video.ts
@@ -1,4 +1,9 @@
import { z } from 'zod';
+import {
+ stickmanOutputPresetDefaults,
+ stickmanOutputPresets,
+ stickmanRatios
+} from '@opencreator/protocol';
import type { CreatorTemplateAction, CreatorTemplateDefinition, CreatorTemplateStage } from './types.js';
const jsonRecord = z.record(z.string(), z.unknown());
@@ -25,6 +30,40 @@ const action = (
allowedStages: string[]
): CreatorTemplateAction => ({ id, inputSchema: schema as never, allowedStages });
+const stickmanInputSchema = z.preprocess(value => {
+ if (value === null || typeof value !== 'object' || Array.isArray(value)) return value;
+ const record = value as Record<string, unknown>;
+ if (record.outputPreset !== 'youtube-shorts') return value;
+ return {
+ ...record,
+ ...stickmanOutputPresetDefaults('youtube-shorts')
+ };
+}, z.object({
+ sourceType: z.enum(['url', 'text']).default('url'),
+ sourceUrl: z.string().default(''),
+ sourceText: z.string().max(50_000).default(''),
+ topic: z.string().default(''),
+ characterAsset: visualAssetRef.default({
+ assetId: 'stickman.character.default',
+ revision: 1
+ }),
+ styleAsset: visualAssetRef.default({
+ assetId: 'stickman.style.paper-pencil',
+ revision: 1
+ }),
+ outputPreset: z.enum(stickmanOutputPresets).default('landscape'),
+ ratio: z.enum(stickmanRatios).default('16:9'),
+ targetDurationSeconds: z.number().positive().max(600).default(30),
+ sourceLanguage: z.string().default('auto'),
+ targetLanguage: z.string().default('zh-CN'),
+ ttsProvider: z.enum(['openai', 'aliyun', 'edge-tts', 'minimax']).optional(),
+ ttsModel: z.string().optional(),
+ voiceCode: z.string().optional(),
+ voiceName: z.string().optional(),
+ workflowTarget: workflowTarget.default('script_ready'),
+ currentStage: z.string().nullable().default(null)
+}).passthrough());
+
const stage = (
definition: Omit<CreatorTemplateStage, 'allowedJobStatuses' | 'jobCompletionPolicy'>
& { final?: boolean }
@@ -63,30 +102,7 @@ export function createStickmanVideoTemplate(): CreatorTemplateDefinition {
id: 'stickman-video',
version: 2,
renderer: 'stickman-video',
- inputSchema: z.object({
- sourceType: z.enum(['url', 'text']).default('url'),
- sourceUrl: z.string().default(''),
- sourceText: z.string().max(50_000).default(''),
- topic: z.string().default(''),
- characterAsset: visualAssetRef.default({
- assetId: 'stickman.character.default',
- revision: 1
- }),
- styleAsset: visualAssetRef.default({
- assetId: 'stickman.style.paper-pencil',
- revision: 1
- }),
- ratio: z.literal('16:9').default('16:9'),
- targetDurationSeconds: z.number().positive().max(600).default(30),
- sourceLanguage: z.string().default('auto'),
- targetLanguage: z.string().default('zh-CN'),
- ttsProvider: z.enum(['openai', 'aliyun', 'edge-tts', 'minimax']).optional(),
- ttsModel: z.string().optional(),
- voiceCode: z.string().optional(),
- voiceName: z.string().optional(),
- workflowTarget: workflowTarget.default('script_ready'),
- currentStage: z.string().nullable().default(null)
- }).passthrough() as never,
+ inputSchema: stickmanInputSchema as never,
stages: [
stage({ id: 'ingest-text', executor: 'stickman-content', inputArtifacts: [], outputArtifacts: [{ kind: 'source_text', status: 'completed' }] }),
stage({ id: 'source-transcript', executor: 'krillinai', inputArtifacts: [], outputArtifacts: [{ kind: 'source_subtitle', status: 'completed' }] }),
@@ -106,6 +122,7 @@ export function createStickmanVideoTemplate(): CreatorTemplateDefinition {
stage({ id: 'package-validation', executor: 'stickman-delivery', dependsOn: ['media-validation'], final: true, invalidateDependentArtifacts: false, inputArtifacts: [
{ kind: 'clean_video', selector: 'latest-completed' },
{ kind: 'narration_subtitle', selector: 'latest-completed' },
+ { kind: 'script_manifest', selector: 'latest-completed' },
{ kind: 'visual_validation', selector: 'latest-completed' },
{ kind: 'audio_timing', selector: 'latest-completed' },
{ kind: 'timeline_manifest', selector: 'latest-completed' },
@@ -114,6 +131,8 @@ export function createStickmanVideoTemplate(): CreatorTemplateDefinition {
], outputArtifacts: [
{ kind: 'clean_video', status: 'completed' },
{ kind: 'narration_subtitle', status: 'completed' },
+ { kind: 'thumbnail', status: 'completed' },
+ { kind: 'publish_copy', status: 'completed' },
{ kind: 'delivery_manifest', status: 'completed' }
] })
],
@@ -139,6 +158,8 @@ export function createStickmanVideoTemplate(): CreatorTemplateDefinition {
outputs: [
{ kind: 'clean_video', required: true },
{ kind: 'narration_subtitle', required: true },
+ { kind: 'thumbnail', required: true },
+ { kind: 'publish_copy', required: true },
{ kind: 'delivery_manifest', required: true }
],
agentGuidance: '按脚本、配音、分镜画面、动画合成和成片交付的阶段边界推进;计费请求未知时只能建议用户显式处置。'
diff --git a/apps/daemon/src/image-generation/codex-native.ts b/apps/daemon/src/image-generation/codex-native.ts
index 7eacdeaa..fa7fc54e 100644
--- a/apps/daemon/src/image-generation/codex-native.ts
+++ b/apps/daemon/src/image-generation/codex-native.ts
@@ -43,6 +43,7 @@ export type CodexNativeImageInput = {
cwd: string;
codexBin: string;
codexHome: string;
+ imagePaths?: string[];
signal?: AbortSignal;
timeoutMs?: number;
createHost?: (input: CodexAppServerHostInput) => CodexAppServerHost;
@@ -68,6 +69,7 @@ export async function generateCodexNativeImage(
input: CodexNativeImageInput
): Promise<CodexNativeImageResult> {
await assertLocalConfiguration(input);
+ await assertLocalInputImages(input);
if (input.signal?.aborted) {
throw new CodexNativeImageError('upstream_error', 'Codex native image generation was canceled');
}
@@ -91,9 +93,11 @@ export async function generateCodexNativeImage(
prompt: buildImagePrompt(input),
developerInstructions: [
'Use the native image_generation tool for exactly one original image.',
+ 'The attached local images are reference images; preserve their responsibilities and do not invent replacement references.',
'Do not call an external image API, use an API key, or return a remote URL.',
'The completed image must be saved as the imageGeneration savedPath artifact.'
].join(' '),
+ ...(input.imagePaths === undefined ? {} : { imagePaths: input.imagePaths }),
timeoutMs: input.timeoutMs ?? DEFAULT_TIMEOUT_MS,
inactivityTimeoutMs: input.timeoutMs ?? DEFAULT_TIMEOUT_MS,
onThreadStarted(threadId) {
@@ -167,6 +171,51 @@ async function assertLocalConfiguration(input: CodexNativeImageInput): Promise<v
}
}
+async function assertLocalInputImages(input: Pick<CodexNativeImageInput, 'cwd' | 'imagePaths'>): Promise<void> {
+ const imagePaths = input.imagePaths ?? [];
+ if (imagePaths.length > 8) {
+ throw new CodexNativeImageError(
+ 'config_missing',
+ 'Codex native image generation supports at most eight local reference images'
+ );
+ }
+ if (imagePaths.length === 0) return;
+ const workspaceRoot = await realpath(input.cwd).catch(() => undefined);
+ if (workspaceRoot === undefined) {
+ throw new CodexNativeImageError('config_missing', 'Codex native image workspace is unavailable');
+ }
+ for (const imagePath of imagePaths) {
+ if (!isAbsolute(imagePath) || imagePath.startsWith('file:') || imagePath.includes('\0')) {
+ throw new CodexNativeImageError(
+ 'config_missing',
+ 'Codex native reference images must be absolute local filesystem paths'
+ );
+ }
+ const canonical = await realpath(imagePath).catch(() => undefined);
+ const metadata = canonical === undefined ? undefined : await stat(canonical).catch(() => undefined);
+ if (
+ canonical === undefined
+ || metadata === undefined
+ || !metadata.isFile()
+ || !isPathInside(workspaceRoot, canonical)
+ || metadata.size <= 0
+ || metadata.size > MAX_IMAGE_BYTES
+ ) {
+ throw new CodexNativeImageError(
+ 'config_missing',
+ `Codex native reference image is missing or outside the workspace: ${imagePath}`
+ );
+ }
+ const mime = detectImageMime(await readFile(canonical));
+ if (mime === undefined) {
+ throw new CodexNativeImageError(
+ 'config_missing',
+ `Codex native reference image has an unsupported signature: ${imagePath}`
+ );
+ }
+ }
+}
+
function isCodexExecutablePath(codexBin: string): boolean {
return isAbsolute(codexBin) || /[\\/]/.test(codexBin);
}
@@ -358,11 +407,13 @@ function discoveryRoots(
path: resolve(generatedRoot, codexThreadId),
boundary: codexHomeRoot,
recursive: true
- },
- { path: generatedRoot, boundary: codexHomeRoot, recursive: false }
+ }
];
}
- return [{ path: generatedRoot, boundary: codexHomeRoot, recursive: true }];
+ // A Codex build that does not report a thread can only be trusted for the
+ // legacy flat layout. Never scan other thread directories without a thread
+ // identity, because a concurrent turn could otherwise be mistaken for ours.
+ return [{ path: generatedRoot, boundary: codexHomeRoot, recursive: false }];
}
async function listLocalArtifactFiles(roots: LocalArtifactScanRoot[]): Promise<string[]> {
diff --git a/apps/daemon/src/image-generation/provider.ts b/apps/daemon/src/image-generation/provider.ts
index 14eb57d3..ee1e912d 100644
--- a/apps/daemon/src/image-generation/provider.ts
+++ b/apps/daemon/src/image-generation/provider.ts
@@ -41,8 +41,8 @@ export function imageGenerationCapabilities(
provider: CreateImageGenerationRequest['provider']
): ImageGenerationCapabilities {
return {
- supportsReferenceImage: provider === 'openai' || provider === 'gemini',
- maxReferenceImages: provider === 'openai' || provider === 'gemini' ? 8 : 0
+ supportsReferenceImage: provider === 'openai' || provider === 'gemini' || provider === 'codex-native',
+ maxReferenceImages: provider === 'openai' || provider === 'gemini' || provider === 'codex-native' ? 8 : 0
};
}
@@ -64,6 +64,7 @@ export async function generateImageContents(
signal?: AbortSignal;
referenceImage?: GeneratedImageContent;
referenceImages?: GeneratedImageContent[];
+ codexNativeImagePaths?: string[];
codexNative?: CodexNativeImageRuntime;
} = {}
): Promise<{ model: string; contents: GeneratedImageContent[] }> {
@@ -106,6 +107,9 @@ export async function generateImageContents(
prompt: request.prompt,
size: request.size,
quality: request.quality,
+ ...(options.codexNativeImagePaths === undefined
+ ? {}
+ : { imagePaths: options.codexNativeImagePaths }),
signal: controller.signal
});
return { model: generated.model, contents: [generated] };
diff --git a/apps/daemon/test/integration/creator-stickman-real-pipeline.test.ts b/apps/daemon/test/integration/creator-stickman-real-pipeline.test.ts
index e50ec497..a1470195 100644
--- a/apps/daemon/test/integration/creator-stickman-real-pipeline.test.ts
+++ b/apps/daemon/test/integration/creator-stickman-real-pipeline.test.ts
@@ -417,6 +417,7 @@ describe('stickman real provider pipeline', () => {
}>;
};
expect(timeline).toMatchObject({
+ ratio: '16:9',
width: 1280,
height: 720
});
@@ -449,7 +450,10 @@ describe('stickman real provider pipeline', () => {
};
expect(mediaValidation).toMatchObject({
ok: true,
- validation: 'ffprobe_and_three_frame_sampling'
+ validation: 'ffprobe_and_three_frame_sampling',
+ ratio: '16:9',
+ width: 1280,
+ height: 720
});
expect(mediaValidation.sampledFrames).toHaveLength(3);
expect(mediaValidation.sampledFrames.every(frame => (
@@ -458,7 +462,9 @@ describe('stickman real provider pipeline', () => {
const deliveryKinds = [
'clean_video',
- 'narration_subtitle'
+ 'narration_subtitle',
+ 'thumbnail',
+ 'publish_copy'
] as const;
const deliveries = deliveryKinds.map(kind => requireDelivery(completed, kind));
for (const artifact of deliveries) {
@@ -499,14 +505,14 @@ describe('stickman real provider pipeline', () => {
placeholderAssets: [],
blockingChecks: []
});
- expect(manifest.files).toHaveLength(2);
+ expect(manifest.files).toHaveLength(4);
for (const file of manifest.files) {
const source = completed.artifacts.find(item => item.id === file.sourceArtifactId);
- const delivery = deliveries.find(item => item.sha256 === file.sha256);
+ const delivery = deliveries.find(item => item.metadata.fileName === file.name);
expect(source, `Manifest 来源 ${file.sourceArtifactId}`).toBeDefined();
expect(delivery, `Manifest 交付 ${file.name}`).toBeDefined();
- expect(file.sha256).toBe(source!.sha256);
- expect(file.bytes).toBe(statSync(source!.path!).size);
+ expect(file.sha256).toBe(hashFile(delivery!.path!));
+ expect(file.bytes).toBe(statSync(delivery!.path!).size);
}
const summary = {
@@ -553,12 +559,15 @@ function readPrivateConfig(path: string): CreatorServicesConfig {
}
function configuredImage(config: CreatorServicesConfig): {
- provider: 'openai' | 'gemini';
+ provider: 'openai' | 'gemini' | 'codex-native';
model: string;
} {
const provider = config.image.provider;
+ if (provider === 'codex-native') {
+ return { provider, model: 'codex-native' };
+ }
if (provider !== 'openai' && provider !== 'gemini') {
- throw new Error('真实火柴人 E2E 的生图服务必须支持角色参考图(OpenAI 或 Gemini)');
+ throw new Error('真实火柴人 E2E 的生图服务必须支持角色参考图(Codex、OpenAI 或 Gemini)');
}
const providerConfig = config.image[provider];
if (!providerConfig.apiKey.trim() || !providerConfig.model.trim()) {
diff --git a/apps/daemon/test/unit/codex-native-image.test.ts b/apps/daemon/test/unit/codex-native-image.test.ts
index d66851f2..11eb211e 100644
--- a/apps/daemon/test/unit/codex-native-image.test.ts
+++ b/apps/daemon/test/unit/codex-native-image.test.ts
@@ -28,8 +28,10 @@ describe('Codex native image adapter', () => {
const root = await tempRoot();
const codexHome = join(root, 'codex-home');
const savedPath = join(codexHome, 'generated_images', 'result.png');
+ const referencePath = join(root, 'reference.png');
await mkdir(join(codexHome, 'generated_images'), { recursive: true });
await writeFile(savedPath, png);
+ await writeFile(referencePath, png);
const createHost = fakeHost({ savedPath });
const result = await generateCodexNativeImage({
@@ -39,6 +41,7 @@ describe('Codex native image adapter', () => {
cwd: root,
codexBin: 'codex',
codexHome,
+ imagePaths: [referencePath],
createHost
});
@@ -55,7 +58,8 @@ describe('Codex native image adapter', () => {
cwd: root,
sandbox: 'workspace-write',
prompt: expect.stringContaining('an original orange cat'),
- developerInstructions: expect.stringContaining('native image_generation')
+ developerInstructions: expect.stringContaining('native image_generation'),
+ imagePaths: [referencePath]
}));
expect(host.close).toHaveBeenCalledTimes(1);
});
@@ -66,6 +70,7 @@ describe('Codex native image adapter', () => {
const savedPath = join(codexHome, 'generated_images', 'thread-1', 'result.png');
await mkdir(codexHome, { recursive: true });
const createHost = fakeHost({
+ threadId: 'thread-1',
emitImageItem: false,
createArtifact: async () => {
await mkdir(join(codexHome, 'generated_images', 'thread-1'), { recursive: true });
@@ -92,6 +97,7 @@ describe('Codex native image adapter', () => {
const savedPath = join(codexHome, 'generated_images', 'result.png');
await mkdir(codexHome, { recursive: true });
const createHost = fakeHost({
+ threadId: null,
emitImageItem: false,
createArtifact: async () => {
await mkdir(join(codexHome, 'generated_images'), { recursive: true });
@@ -342,7 +348,7 @@ async function tempRoot(): Promise<string> {
function fakeHost(input: {
savedPath?: string;
- threadId?: string;
+ threadId?: string | null;
failure?: { message: string };
neverCompletes?: boolean;
emitImageItem?: boolean;
@@ -356,7 +362,9 @@ function fakeHost(input: {
const result = input.neverCompletes
? new Promise<CodexAppServerResult>(() => undefined)
: (async () => {
- await turnInput.onThreadStarted?.(input.threadId ?? 'thread-1');
+ if (input.threadId !== null) {
+ await turnInput.onThreadStarted?.(input.threadId ?? 'thread-1');
+ }
await input.createArtifact?.();
if (input.emitImageItem !== false) {
await turnInput.onNotification?.({
diff --git a/apps/daemon/test/unit/creator-stickman-action-handler.test.ts b/apps/daemon/test/unit/creator-stickman-action-handler.test.ts
new file mode 100644
index 00000000..42e632a2
--- /dev/null
+++ b/apps/daemon/test/unit/creator-stickman-action-handler.test.ts
@@ -0,0 +1,155 @@
+import { describe, expect, it } from 'vitest';
+import type { CreatorArtifact, CreatorJob } from '@opencreator/protocol';
+import type { CreatorRepository } from '../../src/creator/repository.js';
+import { handleStickmanAction } from '../../src/creator/stickman/action-handler.js';
+
+describe('stickman action handler', () => {
+ it('stales the content pipeline when a Shorts preset changes duration, language, and TTS', () => {
+ const artifacts = [
+ 'content_plan',
+ 'script_manifest',
+ 'narration_audio',
+ 'audio_timing',
+ 'shot_spec',
+ 'character_reference',
+ 'style_reference',
+ 'style_contract',
+ 'image_prompt_pack',
+ 'shot_image',
+ 'visual_validation',
+ 'timeline_manifest',
+ 'narration_subtitle',
+ 'clean_video',
+ 'media_validation',
+ 'thumbnail',
+ 'publish_copy',
+ 'delivery_manifest'
+ ].map((kind, index): CreatorArtifact => ({
+ id: `artifact-${index + 1}`,
+ jobId: 'job-1',
+ kind,
+ version: 1,
+ status: 'completed',
+ path: null,
+ scopeKey: null,
+ inputFingerprint: null,
+ sha256: null,
+ sourceArtifactIds: [],
+ metadata: {},
+ createdAt: '2026-09-20T00:00:00.000Z'
+ }));
+ const job = {
+ id: 'job-1',
+ projectId: 'project-1',
+ templateId: 'stickman-video',
+ templateVersion: 2,
+ status: 'running',
+ revision: 1,
+ state: {
+ outputPreset: 'landscape',
+ ratio: '16:9',
+ targetDurationSeconds: 60,
+ targetLanguage: 'zh-CN',
+ ttsProvider: 'openai'
+ },
+ stages: [],
+ artifacts,
+ providerRequests: [],
+ activities: [],
+ agentThreadId: null,
+ createdAt: '2026-09-20T00:00:00.000Z',
+ updatedAt: '2026-09-20T00:00:00.000Z'
+ } as CreatorJob;
+ const staleIds: string[] = [];
+ const repository = {
+ setArtifactStatus(id: string, status: string) {
+ if (status === 'stale') staleIds.push(id);
+ }
+ } as unknown as CreatorRepository;
+
+ const result = handleStickmanAction({
+ repository,
+ current: job,
+ action: 'update-settings',
+ parsedInput: { patch: { outputPreset: 'youtube-shorts' } },
+ actor: 'user',
+ newRevision: 2
+ });
+
+ expect(result.handled).toBe(true);
+ expect(result.state).toMatchObject({
+ outputPreset: 'youtube-shorts',
+ ratio: '9:16',
+ targetDurationSeconds: 30,
+ targetLanguage: 'en-US',
+ ttsProvider: 'edge-tts'
+ });
+ expect(result.affectedArtifactIds).toHaveLength(artifacts.length);
+ expect(new Set(staleIds)).toHaveLength(artifacts.length);
+ });
+
+ it('restores landscape defaults and global TTS selection when leaving Shorts', () => {
+ const artifacts: CreatorArtifact[] = ['script_manifest', 'clean_video'].map((kind, index) => ({
+ id: `artifact-${index + 1}`,
+ jobId: 'job-1',
+ kind,
+ version: 1,
+ status: 'completed',
+ path: null,
+ scopeKey: null,
+ inputFingerprint: null,
+ sha256: null,
+ sourceArtifactIds: [],
+ metadata: {},
+ createdAt: '2026-09-20T00:00:00.000Z'
+ }));
+ const job = {
+ id: 'job-1',
+ projectId: 'project-1',
+ templateId: 'stickman-video',
+ templateVersion: 2,
+ status: 'running',
+ revision: 1,
+ state: {
+ outputPreset: 'youtube-shorts',
+ ratio: '9:16',
+ sourceLanguage: 'auto',
+ targetDurationSeconds: 30,
+ targetLanguage: 'en-US',
+ ttsProvider: 'edge-tts',
+ ttsModel: '',
+ voiceCode: 'en-US-AriaNeural',
+ voiceName: 'Aria'
+ },
+ stages: [],
+ artifacts,
+ providerRequests: [],
+ activities: [],
+ agentThreadId: null,
+ createdAt: '2026-09-20T00:00:00.000Z',
+ updatedAt: '2026-09-20T00:00:00.000Z'
+ } as CreatorJob;
+ const repository = {
+ setArtifactStatus() {}
+ } as unknown as CreatorRepository;
+
+ const result = handleStickmanAction({
+ repository,
+ current: job,
+ action: 'update-settings',
+ parsedInput: { patch: { outputPreset: 'landscape' } },
+ actor: 'user',
+ newRevision: 2
+ });
+
+ expect(result.state).toMatchObject({
+ outputPreset: 'landscape',
+ ratio: '16:9',
+ sourceLanguage: 'auto',
+ targetDurationSeconds: 30,
+ targetLanguage: 'zh-CN'
+ });
+ expect(result.state).not.toHaveProperty('ttsProvider');
+ expect(result.state).not.toHaveProperty('voiceCode');
+ });
+});
diff --git a/apps/daemon/test/unit/creator-stickman-audio.test.ts b/apps/daemon/test/unit/creator-stickman-audio.test.ts
index 1f069105..29d07d39 100644
--- a/apps/daemon/test/unit/creator-stickman-audio.test.ts
+++ b/apps/daemon/test/unit/creator-stickman-audio.test.ts
@@ -94,6 +94,62 @@ describe('stickman audio executor', () => {
db.close();
});
+ it('uses the local Edge TTS command path without requiring a paid provider', async () => {
+ const { db, repository, service, templates, script } = setupConfiguredJob({ configured: false });
+ const config = createDefaultCreatorServicesConfig();
+ config.tts.provider = 'edge-tts';
+ const current = service.getJob(script.jobId)!;
+ repository.updateJob({
+ id: current.id,
+ status: current.status,
+ revision: current.revision,
+ state: { ...current.state, targetLanguage: 'en-US', ttsProvider: 'edge-tts' }
+ });
+ const synthesizeEdgeTts = vi.fn(async (request: { voiceId: string }) => ({
+ content: Buffer.from('edge-audio'),
+ mime: 'audio/mpeg' as const,
+ provider: 'edge-tts' as const,
+ model: '' as const,
+ voiceId: request.voiceId,
+ format: 'mp3' as const
+ }));
+ const run = repository.createStageRun({
+ jobId: script.jobId,
+ stageId: 'narration',
+ executor: 'stickman-audio',
+ status: 'queued',
+ scopeKey: 'segment-01',
+ inputFingerprint: '1'.repeat(64)
+ });
+ const runner = createCreatorStageRunner({
+ repository,
+ templates,
+ workRoot: join(tempDir, 'jobs'),
+ executors: [createStickmanAudioExecutor({
+ configStore: { read: async () => config },
+ ttsService: { synthesize: vi.fn() as never },
+ synthesizeEdgeTts,
+ ledger: new CreatorProviderRequestLedger(repository),
+ ffprobePath: 'ffprobe-test',
+ probe: async () => ({ duration: 1, hasVideo: false, hasAudio: true })
+ })]
+ });
+
+ expect(await runner.runStageRun(run.id)).toMatchObject({ status: 'succeeded' });
+ expect(synthesizeEdgeTts).toHaveBeenCalledWith(expect.objectContaining({
+ voiceId: 'en-US-AriaNeural'
+ }));
+ expect(service.getJob(script.jobId)!.artifacts).toEqual(
+ expect.arrayContaining([expect.objectContaining({
+ kind: 'narration_audio',
+ path: expect.stringMatching(/segment-01\.mp3$/),
+ metadata: expect.objectContaining({ provider: 'edge-tts', mimeType: 'audio/mpeg' })
+ })])
+ );
+ await runner.close();
+ db.close();
+ });
+
it('changes the narration fingerprint when the selected voice changes', () => {
const script = {
sha256: 'a'.repeat(64)
@@ -172,7 +228,7 @@ describe('stickman audio executor', () => {
db.close();
});
- it('fails before synthesis when no supported TTS provider is configured', async () => {
+ it('fails clearly when the local Edge TTS command is unavailable', async () => {
const { db, repository, templates, script } = setupConfiguredJob({ configured: false });
const synthesize = vi.fn();
const config = createDefaultCreatorServicesConfig();
@@ -194,6 +250,7 @@ describe('stickman audio executor', () => {
ttsService: { synthesize: synthesize as never },
ledger: new CreatorProviderRequestLedger(repository),
ffprobePath: 'ffprobe-test',
+ edgeTtsCommand: 'missing-opencreator-edge-tts-command',
synthesize: synthesize as never,
probe: async () => ({ duration: 1, hasVideo: false, hasAudio: true })
})]
@@ -201,7 +258,7 @@ describe('stickman audio executor', () => {
expect(await runner.runStageRun(run.id)).toMatchObject({
status: 'failed',
- errorCode: 'creator_tts_config_missing'
+ errorCode: 'creator_tts_runtime_unavailable'
});
expect(synthesize).not.toHaveBeenCalled();
await runner.close();
diff --git a/apps/daemon/test/unit/creator-stickman-delivery.test.ts b/apps/daemon/test/unit/creator-stickman-delivery.test.ts
index 366d5653..d72bda66 100644
--- a/apps/daemon/test/unit/creator-stickman-delivery.test.ts
+++ b/apps/daemon/test/unit/creator-stickman-delivery.test.ts
@@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'nod
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
+import sharp from 'sharp';
import type { CreatorArtifact } from '@opencreator/protocol';
import { createStickmanDeliveryExecutor } from '../../src/creator/stickman/delivery-executor.js';
@@ -20,6 +21,8 @@ describe('stickman delivery executor', () => {
expect(result.outputs.map(output => output.kind)).toEqual([
'clean_video',
'narration_subtitle',
+ 'thumbnail',
+ 'publish_copy',
'delivery_manifest'
]);
const manifest = JSON.parse(readFileSync(result.outputs.at(-1)!.path!, 'utf8'));
@@ -34,8 +37,10 @@ describe('stickman delivery executor', () => {
])
});
expect(manifest.files.map((file: { name: string }) => file.name)).toEqual([
- 'stickman-video.mp4',
- 'narration.srt'
+ 'short.mp4',
+ 'subtitles.srt',
+ 'thumbnail.png',
+ 'publish-copy.md'
]);
});
@@ -57,6 +62,38 @@ describe('stickman delivery executor', () => {
expect(delivery.metadata?.packageStatus).toBe('publishable');
});
+ it('does not read blocking evidence from outside the job root', async () => {
+ const fixture = await createFixture();
+ const outsidePath = join(tempRoot, 'outside-visual-validation.json');
+ writeFileSync(outsidePath, JSON.stringify({ publishable: true, ocrStatus: 'passed' }));
+ fixture.artifacts.push(artifact('artifact-visual-outside', 'visual_validation', outsidePath));
+
+ const result = await run(fixture.artifacts, fixture.workdir, {
+ sampleVideoFrames: async () => ({ sampleCount: 3 })
+ });
+ const manifest = JSON.parse(readFileSync(result.outputs.at(-1)!.path!, 'utf8'));
+
+ expect(manifest.blockingChecks).toContain('visual_validation_invalid');
+ });
+
+ it('rejects tampered blocking evidence even when the JSON remains valid', async () => {
+ const fixture = await createFixture();
+ addPublishableEvidence(fixture.artifacts, join(tempRoot, 'job-1', 'source'));
+ const visual = fixture.artifacts.find(artifact => artifact.kind === 'visual_validation');
+ if (visual?.path === null || visual?.path === undefined) throw new Error('visual evidence is missing');
+ writeFileSync(visual.path, JSON.stringify({
+ ...JSON.parse(readFileSync(visual.path, 'utf8')),
+ warnings: ['tampered-after-validation']
+ }));
+
+ const result = await run(fixture.artifacts, fixture.workdir, {
+ sampleVideoFrames: async () => ({ sampleCount: 3 })
+ });
+ const manifest = JSON.parse(readFileSync(result.outputs.at(-1)!.path!, 'utf8'));
+
+ expect(manifest.blockingChecks).toContain('visual_validation_invalid');
+ });
+
it('rejects extra, stale, hash-mismatched, and invalid subtitle inputs', async () => {
let fixture = await createFixture();
mkdirSync(join(fixture.workdir, 'delivery'), { recursive: true });
@@ -95,10 +132,53 @@ async function createFixture(): Promise<{ artifacts: CreatorArtifact[]; workdir:
mkdirSync(workdir, { recursive: true });
const files = {
clean_video: join(sourceRoot, 'clean.mp4'),
- narration_subtitle: join(sourceRoot, 'narration.srt')
+ narration_subtitle: join(sourceRoot, 'narration.srt'),
+ script_manifest: join(sourceRoot, 'script.json'),
+ timeline_manifest: join(sourceRoot, 'timeline.json')
};
writeFileSync(files.clean_video, 'clean-video');
writeFileSync(files.narration_subtitle, '1\n00:00:00,000 --> 00:00:01,000\n第一段旁白\n');
+ writeFileSync(files.script_manifest, JSON.stringify({
+ contract: 'stickman-narration-script-v2',
+ reviewStatus: 'approved',
+ contentLocked: true,
+ title: 'A deterministic stickman Short',
+ language: 'en-US',
+ targetDurationSeconds: 1,
+ narrationBudget: { unit: 'characters', unitsPerMinute: 60, minUnits: 1, maxUnits: 100 },
+ segmentCount: 1,
+ totalNarrationUnits: 4,
+ estimatedTotalDurationSeconds: 4,
+ segments: [{
+ id: 'segment-01',
+ order: 1,
+ narration: 'A clear first idea.',
+ claimIds: ['claim-001'],
+ sourceSpanIds: ['source-001'],
+ narrationUnits: 4,
+ estimatedDurationSeconds: 4
+ }]
+ }));
+ writeFileSync(files.timeline_manifest, JSON.stringify({
+ ratio: '16:9',
+ fps: 30,
+ width: 1280,
+ height: 720,
+ totalFrames: 30,
+ shots: [{
+ shotId: 'shot-01',
+ startFrame: 0,
+ endFrame: 30,
+ imageArtifactId: 'image-1',
+ audioArtifactId: 'audio-1',
+ motion: 'static',
+ imageSha256: 'a'.repeat(64),
+ audioSha256: 'b'.repeat(64),
+ imagePath: 'image.png',
+ audioPath: 'audio.wav'
+ }],
+ captions: [{ segmentId: 'segment-01', startFrame: 0, endFrame: 30, text: 'A clear first idea.' }]
+ }));
return {
workdir,
artifacts: Object.entries(files).map(([kind, path], index): CreatorArtifact => ({
@@ -114,7 +194,9 @@ async function createFixture(): Promise<{ artifacts: CreatorArtifact[]; workdir:
sourceArtifactIds: [],
metadata: kind === 'clean_video'
? { renderEngine: 'remotion', renderKind: 'final' }
- : {},
+ : kind === 'timeline_manifest'
+ ? { timingSource: 'ffprobe_cumulative_tts_duration' }
+ : {},
createdAt: '2026-08-31T00:00:00.000Z'
}))
};
@@ -138,7 +220,12 @@ function run(
}),
...(options.sampleVideoFrames === undefined
? {}
- : { sampleVideoFrames: options.sampleVideoFrames })
+ : { sampleVideoFrames: options.sampleVideoFrames }),
+ createThumbnail: async ({ targetPath, width, height }) => {
+ await sharp({ create: { width, height, channels: 3, background: '#202020' } })
+ .png()
+ .toFile(targetPath);
+ }
}).run({
stageRun: { id: 'stage-delivery', stageId: 'package-validation' },
job: { id: 'job-1' },
@@ -171,27 +258,14 @@ function addPublishableEvidence(artifacts: CreatorArtifact[], sourceRoot: string
}],
totalDurationSeconds: 1
}));
- const timelinePath = join(sourceRoot, 'timeline.json');
- writeFileSync(timelinePath, JSON.stringify({
- fps: 30,
- width: 1280,
- height: 720,
- totalFrames: 30,
- shots: [{
- shotId: 'shot-01',
- startFrame: 0,
- endFrame: 30,
- imageArtifactId: 'image-1',
- audioArtifactId: narration.id,
- motion: 'static',
- imageSha256: 'a'.repeat(64),
- audioSha256: narration.sha256
- }]
- }));
+ const timelineArtifact = artifacts.find(candidate => candidate.kind === 'timeline_manifest')!;
const visualPath = join(sourceRoot, 'visual-validation.json');
writeFileSync(visualPath, JSON.stringify({
ok: true,
validation: 'automated_decode_aspect_nonblank_hash_and_ocr',
+ ratio: '16:9',
+ width: 1280,
+ height: 720,
approvedShotSpecArtifactId: 'shot-spec-1',
shotCount: 1,
ocrStatus: 'passed',
@@ -209,9 +283,6 @@ function addPublishableEvidence(artifacts: CreatorArtifact[], sourceRoot: string
detectedText: []
}]
}));
- const timelineArtifact = artifact('artifact-timeline', 'timeline_manifest', timelinePath, {
- timingSource: 'ffprobe_cumulative_tts_duration'
- });
const cleanVideo = artifacts.find(candidate => candidate.kind === 'clean_video')!;
const mediaValidationPath = join(sourceRoot, 'media-validation.json');
writeFileSync(mediaValidationPath, JSON.stringify({
@@ -223,6 +294,7 @@ function addPublishableEvidence(artifacts: CreatorArtifact[], sourceRoot: string
duration: 1,
expectedDuration: 1,
durationTolerance: 0.15,
+ ratio: '16:9',
width: 1280,
height: 720,
hasVideo: true,
@@ -240,7 +312,6 @@ function addPublishableEvidence(artifacts: CreatorArtifact[], sourceRoot: string
artifacts.push(
narration,
artifact('artifact-timing', 'audio_timing', timingPath),
- timelineArtifact,
artifact('artifact-visual', 'visual_validation', visualPath),
artifact('artifact-media-validation', 'media_validation', mediaValidationPath)
);
diff --git a/apps/daemon/test/unit/creator-stickman-format.test.ts b/apps/daemon/test/unit/creator-stickman-format.test.ts
new file mode 100644
index 00000000..798df4df
--- /dev/null
+++ b/apps/daemon/test/unit/creator-stickman-format.test.ts
@@ -0,0 +1,135 @@
+import { describe, expect, it } from 'vitest';
+import { createStickmanVideoTemplate } from '../../src/creator/templates/stickman-video.js';
+import {
+ stickmanDeliveryManifestSchema,
+ stickmanMediaValidationSchema,
+ stickmanStyleContractSchema,
+ stickmanTimelineSchema
+} from '../../src/creator/stickman/contracts.js';
+
+describe('stickman format contracts', () => {
+ it('normalizes the YouTube Shorts preset without changing the landscape default', () => {
+ const template = createStickmanVideoTemplate();
+ const shorts = template.inputSchema.parse({
+ sourceType: 'text',
+ outputPreset: 'youtube-shorts'
+ }) as Record<string, unknown>;
+ const landscape = template.inputSchema.parse({}) as Record<string, unknown>;
+
+ expect(shorts).toMatchObject({
+ outputPreset: 'youtube-shorts',
+ ratio: '9:16',
+ targetDurationSeconds: 30,
+ targetLanguage: 'en-US',
+ ttsProvider: 'edge-tts'
+ });
+ expect(landscape).toMatchObject({ outputPreset: 'landscape', ratio: '16:9' });
+ });
+
+ it('accepts vertical visual, timeline, media, and four-file delivery contracts', () => {
+ expect(stickmanStyleContractSchema.parse({
+ contract: 'stickman-visual-profile-v2',
+ ratio: '9:16',
+ character: {
+ assetId: 'character',
+ revision: 1,
+ identity: { preserve: 'same protagonist', prohibit: 'identity changes' },
+ references: [{ role: 'character', sha256: 'a'.repeat(64), bytes: 10, mimeType: 'image/png' }]
+ },
+ style: {
+ assetId: 'style',
+ revision: 1,
+ rendering: {
+ medium: 'ink',
+ surface: 'paper',
+ linework: 'bold',
+ shading: 'flat',
+ palette: 'monochrome',
+ sceneDensity: 'low',
+ composition: 'clear',
+ characterRendering: 'stick figure'
+ },
+ semanticRenderingRules: { color: 'palette', light: 'soft', complexEnvironment: 'simplify' },
+ forbiddenDirections: ['text'],
+ references: []
+ }
+ })).toMatchObject({ ratio: '9:16' });
+
+ expect(stickmanTimelineSchema.parse({
+ ratio: '9:16',
+ fps: 30,
+ width: 720,
+ height: 1280,
+ totalFrames: 30,
+ shots: [{
+ shotId: 'shot-01',
+ startFrame: 0,
+ endFrame: 30,
+ imageArtifactId: 'image-01',
+ audioArtifactId: 'audio-01',
+ motion: 'static',
+ imageSha256: 'b'.repeat(64),
+ audioSha256: 'c'.repeat(64),
+ imagePath: 'image.png',
+ audioPath: 'audio.mp3'
+ }],
+ captions: [{ segmentId: 'segment-01', startFrame: 0, endFrame: 30, text: 'A clear hook.' }]
+ })).toMatchObject({ ratio: '9:16', width: 720, height: 1280 });
+
+ expect(stickmanMediaValidationSchema.parse({
+ ok: true,
+ validation: 'ffprobe_and_three_frame_sampling',
+ ratio: '9:16',
+ cleanVideoArtifactId: 'video-01',
+ cleanVideoSha256: 'd'.repeat(64),
+ timelineArtifactId: 'timeline-01',
+ duration: 1,
+ expectedDuration: 1,
+ durationTolerance: 0.15,
+ width: 720,
+ height: 1280,
+ hasVideo: true,
+ hasAudio: true,
+ sampledFrames: [1, 2, 3].map(index => ({
+ index,
+ timestampSeconds: index / 10,
+ sha256: String(index).repeat(64),
+ width: 720,
+ height: 1280,
+ brightnessMean: 100,
+ contrastStddev: 20
+ }))
+ })).toMatchObject({ ratio: '9:16', width: 720, height: 1280 });
+
+ const manifest = stickmanDeliveryManifestSchema.parse({
+ packageStatus: 'publishable',
+ ratio: '9:16',
+ width: 720,
+ height: 1280,
+ duration: 30,
+ providers: { image: 'codex-native', video: 'remotion', voice: 'edge-tts' },
+ placeholderAssets: [],
+ blockingChecks: [],
+ files: [
+ ['short.mp4', 'video/mp4'],
+ ['subtitles.srt', 'application/x-subrip'],
+ ['thumbnail.png', 'image/png'],
+ ['publish-copy.md', 'text/markdown']
+ ].map(([name, mime], index) => ({
+ name,
+ relativePath: `delivery/${name}`,
+ sha256: String(index + 1).repeat(64),
+ bytes: 10,
+ mime,
+ sourceArtifactId: `artifact-${index}`
+ }))
+ });
+ expect(manifest).toMatchObject({ ratio: '9:16', width: 720, height: 1280 });
+ expect(manifest.files.map(file => file.name)).toEqual([
+ 'short.mp4',
+ 'subtitles.srt',
+ 'thumbnail.png',
+ 'publish-copy.md'
+ ]);
+ });
+});
diff --git a/apps/daemon/test/unit/creator-stickman-image-prompt.test.ts b/apps/daemon/test/unit/creator-stickman-image-prompt.test.ts
index 5b8e20f7..8ac61252 100644
--- a/apps/daemon/test/unit/creator-stickman-image-prompt.test.ts
+++ b/apps/daemon/test/unit/creator-stickman-image-prompt.test.ts
@@ -10,6 +10,7 @@ describe('stickman image prompt compiler', () => {
const prompt = buildStickmanImagePrompt({
visualProfile: visualProfile(),
shot: shot(),
+ ratio: '16:9',
hasStyleReference: true,
hasPreviousShotReference: true
});
@@ -31,6 +32,7 @@ describe('stickman image prompt compiler', () => {
const prompt = buildStickmanImagePrompt({
visualProfile: visualProfile(),
shot: shot(),
+ ratio: '16:9',
hasStyleReference: false,
hasPreviousShotReference: false
});
diff --git a/apps/daemon/test/unit/creator-stickman-images.test.ts b/apps/daemon/test/unit/creator-stickman-images.test.ts
index 81a98c2b..1967b322 100644
--- a/apps/daemon/test/unit/creator-stickman-images.test.ts
+++ b/apps/daemon/test/unit/creator-stickman-images.test.ts
@@ -80,11 +80,12 @@ function insertImageContracts(input: {
character: { id: string; sha256: string | null };
characterAssetId: string;
prompts: Array<{ shotId: string; visualDescription: string; prompt: string }>;
+ ratio?: '16:9' | '9:16';
}): void {
const stylePath = join(tempDir, `style-contract-${input.shotSpecId}.json`);
writeFileSync(stylePath, JSON.stringify({
contract: 'stickman-visual-profile-v2',
- ratio: '16:9',
+ ratio: input.ratio ?? '16:9',
character: {
assetId: input.characterAssetId,
revision: 1,
@@ -464,6 +465,103 @@ describe('stickman scoped images', () => {
db.close();
});
+ it('uses the portrait canvas and materializes Codex-native reference paths for Shorts', async () => {
+ const { db, repository, templates, service } = setup();
+ const job = service.createJob({
+ projectId: 'p1',
+ templateId: 'stickman-video',
+ state: {
+ ratio: '9:16',
+ characterAsset: { assetId: 'stickman.character.default', revision: 1 },
+ styleAsset: { assetId: 'stickman.style.minimal-ink', revision: 1 }
+ }
+ });
+ const shotSpecPath = join(tempDir, 'portrait-shot.json');
+ writeFileSync(shotSpecPath, JSON.stringify({
+ scriptArtifactId: 'script-1',
+ audioTimingArtifactId: 'audio-timing-1',
+ timingSource: 'ffprobe_cumulative_tts_duration',
+ shots: [shot('shot-01', 'segment-01', 'portrait scene', 'static', 0)]
+ }));
+ const shotSpecArtifact = repository.insertArtifact({
+ jobId: job.id,
+ kind: 'shot_spec',
+ status: 'completed',
+ path: shotSpecPath,
+ sha256: 'a'.repeat(64),
+ sourceArtifactIds: [],
+ metadata: {}
+ });
+ const reference = await sharp({
+ create: { width: 100, height: 100, channels: 4, background: '#333333' }
+ }).png().toBuffer();
+ const referencePath = join(tempDir, 'portrait-character.png');
+ writeFileSync(referencePath, reference);
+ const characterArtifact = repository.insertArtifact({
+ jobId: job.id,
+ kind: 'character_reference',
+ status: 'completed',
+ path: referencePath,
+ sha256: createHash('sha256').update(reference).digest('hex'),
+ sourceArtifactIds: [],
+ metadata: { assetId: 'stickman.character.default', revision: 1, mimeType: 'image/png' }
+ });
+ insertImageContracts({
+ repository,
+ jobId: job.id,
+ shotSpecId: shotSpecArtifact.id,
+ character: characterArtifact,
+ characterAssetId: 'stickman.character.default',
+ ratio: '9:16',
+ prompts: [{ shotId: 'shot-01', visualDescription: 'portrait scene', prompt: 'portrait scene' }]
+ });
+ const stage = repository.createStageRun({
+ jobId: job.id,
+ stageId: 'images',
+ executor: 'stickman-image',
+ status: 'queued',
+ scopeKey: 'shot-01',
+ inputFingerprint: '1'.repeat(64)
+ });
+ const generatedImage = await sharp({
+ create: { width: 300, height: 200, channels: 4, background: '#ffffff' }
+ }).png().toBuffer();
+ const config = createDefaultCreatorServicesConfig();
+ config.image.provider = 'codex-native';
+ let submittedSize = '';
+ let submittedPaths: string[] = [];
+ const runner = createCreatorStageRunner({
+ repository,
+ templates,
+ workRoot: join(tempDir, 'jobs'),
+ executors: [createStickmanImageExecutor({
+ configStore: { read: async () => config },
+ ledger: new CreatorProviderRequestLedger(repository),
+ validateCandidate: acceptCandidate,
+ async generate(request, _config, options) {
+ submittedSize = request.size;
+ submittedPaths = options?.codexNativeImagePaths ?? [];
+ return {
+ model: 'codex-native',
+ contents: [{ mime: 'image/png' as const, content: generatedImage }]
+ };
+ }
+ })]
+ });
+
+ const completed = await runner.runStageRun(stage.id);
+
+ expect(completed.status).toBe('succeeded');
+ expect(submittedSize).toBe('1024x1536');
+ expect(submittedPaths).toHaveLength(1);
+ expect(submittedPaths[0]).not.toBe(referencePath);
+ expect(readFileSync(submittedPaths[0]!)).toEqual(reference);
+ const generated = service.getJob(job.id)!.artifacts.find(artifact => artifact.kind === 'shot_image');
+ expect(await sharp(generated!.path!).metadata()).toMatchObject({ width: 720, height: 1280 });
+ await runner.close();
+ db.close();
+ });
+
it('keeps successful shot artifacts when another scope fails', async () => {
const { db, repository, templates, service } = setup();
const job = service.createJob({
@@ -772,4 +870,95 @@ describe('stickman scoped images', () => {
})).not.toBe(fingerprint);
db.close();
});
+
+ it('invalidates content and all downstream artifacts when switching to Shorts', () => {
+ const { db, repository, service } = setup();
+ const created = service.createJob({
+ projectId: 'p1',
+ templateId: 'stickman-video',
+ state: {
+ outputPreset: 'landscape',
+ ratio: '16:9',
+ targetDurationSeconds: 60,
+ targetLanguage: 'zh-CN',
+ ttsProvider: 'openai'
+ }
+ });
+ const contentPlan = repository.insertArtifact({
+ jobId: created.id,
+ kind: 'content_plan',
+ status: 'completed',
+ path: null,
+ sourceArtifactIds: [],
+ metadata: {}
+ });
+ const script = repository.insertArtifact({
+ jobId: created.id,
+ kind: 'script_manifest',
+ status: 'completed',
+ path: null,
+ sourceArtifactIds: [contentPlan.id],
+ metadata: {}
+ });
+ const narration = repository.insertArtifact({
+ jobId: created.id,
+ kind: 'narration_audio',
+ status: 'completed',
+ path: null,
+ sourceArtifactIds: [script.id],
+ metadata: {}
+ });
+ const timing = repository.insertArtifact({
+ jobId: created.id,
+ kind: 'audio_timing',
+ status: 'completed',
+ path: null,
+ sourceArtifactIds: [script.id, narration.id],
+ metadata: {}
+ });
+ const shotSpec = repository.insertArtifact({
+ jobId: created.id,
+ kind: 'shot_spec',
+ status: 'completed',
+ path: null,
+ sourceArtifactIds: [timing.id],
+ metadata: {}
+ });
+ const image = repository.insertArtifact({
+ jobId: created.id,
+ kind: 'shot_image',
+ status: 'completed',
+ path: null,
+ sourceArtifactIds: [shotSpec.id],
+ metadata: {}
+ });
+ const timeline = repository.insertArtifact({
+ jobId: created.id,
+ kind: 'timeline_manifest',
+ status: 'completed',
+ path: null,
+ sourceArtifactIds: [image.id, timing.id],
+ metadata: {}
+ });
+
+ const response = service.applyAction(created.id, {
+ actor: 'user',
+ action: 'update-settings',
+ expectedRevision: created.revision,
+ input: { patch: { outputPreset: 'youtube-shorts' } }
+ });
+
+ expect(response.job.state).toMatchObject({
+ outputPreset: 'youtube-shorts',
+ ratio: '9:16',
+ targetDurationSeconds: 30,
+ targetLanguage: 'en-US',
+ ttsProvider: 'edge-tts'
+ });
+ for (const artifact of [contentPlan, script, narration, timing, shotSpec, image, timeline]) {
+ expect(response.job.artifacts.find(candidate => candidate.id === artifact.id)?.status)
+ .toBe('stale');
+ }
+ db.close();
+ });
});
diff --git a/apps/daemon/test/unit/creator-stickman-media-validation.test.ts b/apps/daemon/test/unit/creator-stickman-media-validation.test.ts
index 2fe56247..a5ad2bbe 100644
--- a/apps/daemon/test/unit/creator-stickman-media-validation.test.ts
+++ b/apps/daemon/test/unit/creator-stickman-media-validation.test.ts
@@ -46,6 +46,7 @@ describe('stickman media validation executor', () => {
});
expect(JSON.parse(readFileSync(result.outputs[0]!.path!, 'utf8'))).toMatchObject({
ok: true,
+ ratio: '16:9',
cleanVideoArtifactId: 'clean-1',
timelineArtifactId: 'timeline-1',
sampledFrames: [{ index: 1 }, { index: 2 }, { index: 3 }]
@@ -76,6 +77,7 @@ function setup() {
mkdirSync(workdir, { recursive: true });
const timelinePath = join(sourceRoot, 'timeline.json');
writeFileSync(timelinePath, JSON.stringify({
+ ratio: '16:9',
fps: 30,
width: 1280,
height: 720,
@@ -89,7 +91,8 @@ function setup() {
motion: 'static',
imageSha256: 'c'.repeat(64),
audioSha256: 'd'.repeat(64)
- }]
+ }],
+ captions: [{ segmentId: 'segment-01', startFrame: 0, endFrame: 60, text: 'Caption' }]
}));
const cleanPath = join(sourceRoot, 'clean.mp4');
writeFileSync(cleanPath, 'validated remotion video');
diff --git a/apps/daemon/test/unit/creator-stickman-publish-copy.test.ts b/apps/daemon/test/unit/creator-stickman-publish-copy.test.ts
new file mode 100644
index 00000000..1342bac8
--- /dev/null
+++ b/apps/daemon/test/unit/creator-stickman-publish-copy.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, it } from 'vitest';
+import { renderStickmanPublishCopy } from '../../src/creator/stickman/publish-copy.js';
+
+describe('stickman publish copy', () => {
+ it('renders deterministic Shorts metadata without a network call', () => {
+ expect(renderStickmanPublishCopy({
+ title: 'Why tiny habits compound',
+ language: 'en-US',
+ durationSeconds: 29.6,
+ ratio: '9:16',
+ narration: [
+ 'Small actions feel invisible at first.',
+ 'Repeated daily, they become the system that changes the result.'
+ ]
+ })).toBe([
+ '# Why tiny habits compound',
+ '',
+ 'Description:',
+ 'Small actions feel invisible at first. Repeated daily, they become the system that changes the result.',
+ '',
+ 'Language: en-US',
+ 'Duration: 00:30',
+ 'Format: YouTube Short (9:16)',
+ '',
+ 'Hashtags: #Shorts #StickFigure #Explained'
+ ].join('\n') + '\n');
+ });
+});
diff --git a/apps/daemon/test/unit/creator-stickman-results.test.ts b/apps/daemon/test/unit/creator-stickman-results.test.ts
index f713c6c6..708f2ee1 100644
--- a/apps/daemon/test/unit/creator-stickman-results.test.ts
+++ b/apps/daemon/test/unit/creator-stickman-results.test.ts
@@ -4,6 +4,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import Fastify from 'fastify';
import { afterEach, describe, expect, it } from 'vitest';
+import sharp from 'sharp';
import { readCreatorResultSnapshots } from '@opencreator/protocol';
import { registerCreatorRoutes } from '../../src/api/routes.creator.js';
import { createCreatorAgentRepository } from '../../src/creator/agent/repository.js';
@@ -45,7 +46,9 @@ describe('stickman result snapshots', () => {
expect(Object.keys(firstSnapshots[0]!.artifactRefs).sort()).toEqual([
'clean_video',
'delivery_manifest',
- 'narration_subtitle'
+ 'narration_subtitle',
+ 'publish_copy',
+ 'thumbnail'
]);
const firstCleanId = firstSnapshots[0]!.artifactRefs.clean_video![0]!;
@@ -109,7 +112,12 @@ function setup() {
height: 720,
hasVideo: true,
hasAudio: true
- })
+ }),
+ createThumbnail: async ({ targetPath, width, height }) => {
+ await sharp({ create: { width, height, channels: 3, background: '#202020' } })
+ .png()
+ .toFile(targetPath);
+ }
})],
workRoot: join(tempRoot, 'jobs')
});
@@ -126,6 +134,7 @@ async function insertInputs(
const files = {
clean_video: join(sourceRoot, 'clean.mp4'),
narration_subtitle: join(sourceRoot, 'narration.srt'),
+ script_manifest: join(sourceRoot, 'script.json'),
narration_audio: join(sourceRoot, 'segment-01.wav'),
audio_timing: join(sourceRoot, 'audio-timing.json'),
timeline_manifest: join(sourceRoot, 'timeline.json'),
@@ -134,6 +143,27 @@ async function insertInputs(
};
writeFileSync(files.clean_video, `clean-${label}`);
writeFileSync(files.narration_subtitle, `1\n00:00:00,000 --> 00:00:01,000\n${label}\n`);
+ writeFileSync(files.script_manifest, JSON.stringify({
+ contract: 'stickman-narration-script-v2',
+ reviewStatus: 'approved',
+ contentLocked: true,
+ title: `Snapshot ${label}`,
+ language: 'en-US',
+ targetDurationSeconds: 1,
+ narrationBudget: { unit: 'characters', unitsPerMinute: 60, minUnits: 1, maxUnits: 100 },
+ segmentCount: 1,
+ totalNarrationUnits: 4,
+ estimatedTotalDurationSeconds: 4,
+ segments: [{
+ id: 'segment-01',
+ order: 1,
+ narration: 'A clear first idea.',
+ claimIds: ['claim-001'],
+ sourceSpanIds: ['source-001'],
+ narrationUnits: 4,
+ estimatedDurationSeconds: 4
+ }]
+ }));
writeFileSync(files.narration_audio, `narration-${label}`);
const insert = (
@@ -156,13 +186,14 @@ async function insertInputs(
const cleanVideo = insert('clean_video', { renderEngine: 'remotion', renderKind: 'final' });
const narrationSubtitle = insert('narration_subtitle');
+ const script = insert('script_manifest');
const narrationAudio = insert('narration_audio', {
duration: 1,
provider: 'openai',
timingSource: 'ffprobe'
}, 'segment-01');
writeFileSync(files.audio_timing, JSON.stringify({
- scriptArtifactId: `script-${label}`,
+ scriptArtifactId: script.id,
timingSource: 'ffprobe_cumulative_tts_duration',
segments: [{
segmentId: 'segment-01',
@@ -176,6 +207,7 @@ async function insertInputs(
}));
const audioTiming = insert('audio_timing');
writeFileSync(files.timeline_manifest, JSON.stringify({
+ ratio: '16:9',
fps: 30,
width: 1280,
height: 720,
@@ -188,8 +220,11 @@ async function insertInputs(
audioArtifactId: narrationAudio.id,
motion: 'static',
imageSha256: 'a'.repeat(64),
- audioSha256: narrationAudio.sha256
- }]
+ audioSha256: narrationAudio.sha256,
+ imagePath: 'image.png',
+ audioPath: 'audio.wav'
+ }],
+ captions: [{ segmentId: 'segment-01', startFrame: 0, endFrame: 30, text: 'A clear first idea.' }]
}));
const timeline = insert('timeline_manifest', {
timingSource: 'ffprobe_cumulative_tts_duration'
@@ -197,6 +232,9 @@ async function insertInputs(
writeFileSync(files.visual_validation, JSON.stringify({
ok: true,
validation: 'automated_decode_aspect_nonblank_hash_and_ocr',
+ ratio: '16:9',
+ width: 1280,
+ height: 720,
approvedShotSpecArtifactId: `shot-spec-${label}`,
shotCount: 1,
ocrStatus: 'passed',
@@ -224,6 +262,7 @@ async function insertInputs(
duration: 1,
expectedDuration: 1,
durationTolerance: 0.15,
+ ratio: '16:9',
width: 1280,
height: 720,
hasVideo: true,
@@ -242,6 +281,7 @@ async function insertInputs(
return {
clean_video: cleanVideo,
narration_subtitle: narrationSubtitle,
+ script_manifest: script,
narration_audio: narrationAudio,
audio_timing: audioTiming,
timeline_manifest: timeline,
diff --git a/apps/daemon/test/unit/creator-stickman-timeline.test.ts b/apps/daemon/test/unit/creator-stickman-timeline.test.ts
index e36c54a3..c68b9cdd 100644
--- a/apps/daemon/test/unit/creator-stickman-timeline.test.ts
+++ b/apps/daemon/test/unit/creator-stickman-timeline.test.ts
@@ -81,6 +81,7 @@ describe('stickman timeline executor', () => {
const timeline = JSON.parse(readFileSync(result.outputs[0]!.path!, 'utf8'));
expect(timeline).toMatchObject({
+ ratio: '16:9',
fps: 30,
width: 1280,
height: 720,
@@ -88,6 +89,10 @@ describe('stickman timeline executor', () => {
shots: [
{ shotId: 'shot-01', startFrame: 0, endFrame: 60, imageArtifactId: 'shot-01-image' },
{ shotId: 'shot-02', startFrame: 60, endFrame: 150, imageArtifactId: 'shot-02-image' }
+ ],
+ captions: [
+ { segmentId: 'segment-01', startFrame: 0, endFrame: 60, text: '第一段' },
+ { segmentId: 'segment-02', startFrame: 60, endFrame: 150, text: '第二段' }
]
});
expect(readFileSync(result.outputs[1]!.path!, 'utf8')).toBe(
diff --git a/apps/web/src/features/dashboard/StickmanVideoWorkspace.tsx b/apps/web/src/features/dashboard/StickmanVideoWorkspace.tsx
index 20b3fc68..90f1a71f 100644
--- a/apps/web/src/features/dashboard/StickmanVideoWorkspace.tsx
+++ b/apps/web/src/features/dashboard/StickmanVideoWorkspace.tsx
@@ -108,6 +108,11 @@ type AudioTiming = {
type DeliveryManifest = {
packageStatus: 'technical-draft' | 'publishable';
+ ratio?: '16:9' | '9:16';
+ width?: number;
+ height?: number;
+ duration?: number;
+ providers?: { image: string; video: string; voice: string };
blockingChecks?: string[];
files?: Array<{
name: string;
@@ -133,9 +138,13 @@ const defaultStyleAsset: CreatorVisualAssetRef = {
const targetDurationPresets = [30, 60, 300, 600] as const;
+type StickmanOutputPreset = 'landscape' | 'youtube-shorts';
+
const deliveryKinds = [
'clean_video',
- 'narration_subtitle'
+ 'narration_subtitle',
+ 'thumbnail',
+ 'publish_copy'
] as const;
export default function StickmanVideoWorkspace(props: {
@@ -206,6 +215,10 @@ export default function StickmanVideoWorkspace(props: {
const voiceCode = typeof state.voiceCode === 'string' ? state.voiceCode : '';
const voiceName = typeof state.voiceName === 'string' ? state.voiceName : voiceCode;
const targetLanguage = typeof state.targetLanguage === 'string' ? state.targetLanguage : 'zh-CN';
+ const outputPreset: StickmanOutputPreset = state.outputPreset === 'youtube-shorts'
+ ? 'youtube-shorts'
+ : 'landscape';
+ const ratio = state.ratio === '9:16' ? '9:16' : '16:9';
const scriptDraft = scriptArtifact !== undefined && scriptDraftState?.artifactId === scriptArtifact.id
? scriptDraftState.value
: script;
@@ -284,7 +297,9 @@ export default function StickmanVideoWorkspace(props: {
));
setTtsConfigurationStatus(configured ? 'configured' : 'missing');
const imageProvider = response.config.image.provider;
- if (imageProvider !== 'openai' && imageProvider !== 'gemini') {
+ if (imageProvider === 'codex-native') {
+ setImageConfigurationStatus('configured');
+ } else if (imageProvider !== 'openai' && imageProvider !== 'gemini') {
setImageConfigurationStatus('unsupported');
} else {
const credential = imageProvider === 'openai'
@@ -417,7 +432,7 @@ export default function StickmanVideoWorkspace(props: {
}
if (imageConfigurationStatus !== 'configured') {
setNotice(imageConfigurationStatus === 'unsupported'
- ? l('当前生图服务不支持角色参考图,请在 AI 服务中切换到 OpenAI 或 Gemini', 'The current image provider does not support character references. Switch to OpenAI or Gemini in AI Services.')
+ ? l('当前生图服务不支持角色参考图,请切换到 Codex(订阅)、OpenAI 或 Gemini', 'The current image provider does not support character references. Switch to Codex (subscription), OpenAI, or Gemini in AI Services.')
: imageConfigurationStatus === 'unavailable'
? l('暂时无法读取生图服务配置,请检查 Runtime 后重试', 'Could not read image settings. Check the Runtime and retry.')
: l('请先前往 AI 服务配置生图服务', 'Configure an image provider in AI Services first.'));
@@ -754,6 +769,8 @@ export default function StickmanVideoWorkspace(props: {
styleAssets={styleAssets}
previewUrls={visualAssetPreviewUrls}
visualAssetCatalogStatus={visualAssetCatalogStatus}
+ outputPreset={outputPreset}
+ ratio={ratio}
targetDurationSeconds={targetDurationSeconds}
ttsProvider={ttsProvider}
ttsModel={ttsModel}
@@ -764,7 +781,19 @@ export default function StickmanVideoWorkspace(props: {
busy={isBusy}
canContinue={workbenchStep > 0}
l={l}
- onPatch={patch => session.updateDraft({ ratio: '16:9', ...patch }, { semantic: true })}
+ onPatch={patch => session.updateDraft(patch, { semantic: true })}
+ onFormatChange={preset => session.updateDraft(
+ preset === 'youtube-shorts'
+ ? {
+ outputPreset: preset,
+ ratio: '9:16',
+ targetDurationSeconds: 30,
+ targetLanguage: 'en-US',
+ ttsProvider: 'edge-tts'
+ }
+ : { outputPreset: preset, ratio: '16:9' },
+ { semantic: true }
+ )}
onStart={startWorkflow}
onContinue={() => setActiveStep(1)}
/>
@@ -867,6 +896,8 @@ function SourceAndCharacterStep(props: {
styleAssets: CreatorVisualAssetSummary[];
previewUrls: Record<string, string>;
visualAssetCatalogStatus: VisualAssetCatalogStatus;
+ outputPreset: StickmanOutputPreset;
+ ratio: '16:9' | '9:16';
targetDurationSeconds: number;
ttsProvider: CreatorTtsProvider;
ttsModel: string;
@@ -878,6 +909,7 @@ function SourceAndCharacterStep(props: {
canContinue: boolean;
l: ReturnType<typeof useLocalizedCopy>;
onPatch(patch: Record<string, CreatorJson>): void;
+ onFormatChange(preset: StickmanOutputPreset): void;
onStart(): void;
onContinue(): void;
}) {
@@ -955,6 +987,24 @@ function SourceAndCharacterStep(props: {
</div>
</div>
</div>
+ <div className="creator-tool-form-row">
+ <label className="creator-tool-field">
+ <span>{props.l('Formato de saída', 'Output format')}</span>
+ <select
+ value={props.outputPreset}
+ aria-label={props.l('Formato de saída', 'Output format')}
+ onChange={event => props.onFormatChange(
+ event.target.value === 'youtube-shorts' ? 'youtube-shorts' : 'landscape'
+ )}
+ >
+ <option value="youtube-shorts">{props.l('YouTube Short · 9:16 · 30 s', 'YouTube Short · 9:16 · 30 s')}</option>
+ <option value="landscape">{props.l('Vídeo horizontal · 16:9', 'Landscape video · 16:9')}</option>
+ </select>
+ <small>{props.ratio === '9:16'
+ ? props.l('Imagens pelo Codex e legendas serão compostas no formato vertical.', 'Codex images and burned-in captions will use the vertical canvas.')
+ : props.l('Mantém o fluxo horizontal existente.', 'Keeps the existing landscape workflow.')}</small>
+ </label>
+ </div>
<div className="stickman-voice-settings">
{props.ttsConfigurationStatus === 'configured' && props.ttsProvider !== 'edge-tts' ? (
<TtsVoicePicker
@@ -1012,7 +1062,7 @@ function SourceAndCharacterStep(props: {
? props.l('当前生图服务不支持角色参考图', 'The image provider does not support character references')
: props.l('尚未配置生图服务', 'Image service is not configured')}</strong>
<small>{props.imageConfigurationStatus === 'unsupported'
- ? props.l('火柴人必须把所选角色图片随每个镜头提交,请切换到 OpenAI 或 Gemini。', 'Stickman generation must attach the selected character to every shot. Switch to OpenAI or Gemini.')
+ ? props.l('火柴人必须把所选角色图片随每个镜头提交,请切换到 Codex(订阅)、OpenAI 或 Gemini。', 'Stickman generation must attach the selected character to every shot. Switch to Codex (subscription), OpenAI, or Gemini.')
: props.l('配置支持参考图的服务后,才能保证镜头使用所选角色。', 'Configure a reference-capable provider so every shot uses the selected character.')}</small>
</div>
{props.imageConfigurationStatus !== 'loading' ? (
@@ -2255,13 +2305,21 @@ function currentIssue(job: { status: string; stages: CreatorStageRun[] }, step:
}
function deliveryLabel(kind: typeof deliveryKinds[number], l: ReturnType<typeof useLocalizedCopy>): string {
- const labels = { clean_video: ['火柴人动画', 'Stickman video'], narration_subtitle: ['旁白字幕', 'Narration subtitles'] } as const;
+ const labels = {
+ clean_video: ['火柴人动画', 'Stickman video'],
+ narration_subtitle: ['旁白字幕', 'Narration subtitles'],
+ thumbnail: ['缩略图', 'Thumbnail'],
+ publish_copy: ['发布文案', 'Publish copy']
+ } as const;
const [zh, en] = labels[kind];
return l(zh, en);
}
function deliveryIcon(kind: typeof deliveryKinds[number]) {
- return kind === 'narration_subtitle' ? <Captions size={17} /> : <FileVideo size={17} />;
+ if (kind === 'narration_subtitle') return <Captions size={17} />;
+ if (kind === 'thumbnail') return <ImageIcon size={17} />;
+ if (kind === 'publish_copy') return <FileText size={17} />;
+ return <FileVideo size={17} />;
}
type CompositionStatus = 'waiting' | 'running' | 'completed' | 'failed';
diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts
index 613eb54f..0ec7611d 100644
--- a/packages/protocol/src/index.ts
+++ b/packages/protocol/src/index.ts
@@ -13,3 +13,4 @@ export * from './settings.js';
export * from './smart-dubbing.js';
export * from './video-metadata.js';
export * from './wechat-article.js';
+export * from './stickman.js';
diff --git a/packages/protocol/src/stickman.ts b/packages/protocol/src/stickman.ts
new file mode 100644
index 00000000..f5b9681d
--- /dev/null
+++ b/packages/protocol/src/stickman.ts
@@ -0,0 +1,61 @@
+export const stickmanRatios = ['16:9', '9:16'] as const;
+export type StickmanRatio = typeof stickmanRatios[number];
+
+export const stickmanOutputPresets = ['landscape', 'youtube-shorts'] as const;
+export type StickmanOutputPreset = typeof stickmanOutputPresets[number];
+
+export type StickmanCanvas = {
+ ratio: StickmanRatio;
+ width: 1280 | 720;
+ height: 720 | 1280;
+};
+
+const canvases: Record<StickmanRatio, StickmanCanvas> = {
+ '16:9': { ratio: '16:9', width: 1280, height: 720 },
+ '9:16': { ratio: '9:16', width: 720, height: 1280 }
+};
+
+export function stickmanCanvasForRatio(ratio: StickmanRatio): StickmanCanvas {
+ return { ...canvases[ratio] };
+}
+
+export function readStickmanRatio(value: unknown): StickmanRatio {
+ return value === '9:16' ? '9:16' : '16:9';
+}
+
+export type StickmanImageSize = '1536x1024' | '1024x1536';
+
+export function stickmanImageSizeForRatio(ratio: StickmanRatio): StickmanImageSize {
+ return ratio === '9:16' ? '1024x1536' : '1536x1024';
+}
+
+export type StickmanOutputPresetDefaults = {
+ outputPreset: StickmanOutputPreset;
+ ratio: StickmanRatio;
+ sourceLanguage?: string;
+ targetDurationSeconds?: number;
+ targetLanguage?: string;
+ ttsProvider?: 'edge-tts';
+};
+
+export function stickmanOutputPresetDefaults(
+ outputPreset: StickmanOutputPreset
+): StickmanOutputPresetDefaults {
+ if (outputPreset === 'youtube-shorts') {
+ return {
+ outputPreset,
+ ratio: '9:16',
+ sourceLanguage: 'auto',
+ targetDurationSeconds: 30,
+ targetLanguage: 'en-US',
+ ttsProvider: 'edge-tts'
+ };
+ }
+ return {
+ outputPreset,
+ ratio: '16:9',
+ sourceLanguage: 'auto',
+ targetDurationSeconds: 30,
+ targetLanguage: 'zh-CN'
+ };
+}
diff --git a/packages/protocol/test/stickman.test.ts b/packages/protocol/test/stickman.test.ts
new file mode 100644
index 00000000..397c2987
--- /dev/null
+++ b/packages/protocol/test/stickman.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, it } from 'vitest';
+import {
+ stickmanCanvasForRatio,
+ stickmanImageSizeForRatio,
+ stickmanOutputPresetDefaults
+} from '../src/stickman.js';
+
+describe('stickman rendering contract', () => {
+ it('maps both supported ratios to the canonical canvas', () => {
+ expect(stickmanCanvasForRatio('16:9')).toEqual({ ratio: '16:9', width: 1280, height: 720 });
+ expect(stickmanCanvasForRatio('9:16')).toEqual({ ratio: '9:16', width: 720, height: 1280 });
+ });
+
+ it('maps each ratio to an image generation size', () => {
+ expect(stickmanImageSizeForRatio('16:9')).toBe('1536x1024');
+ expect(stickmanImageSizeForRatio('9:16')).toBe('1024x1536');
+ });
+
+ it('defines a cost-free Shorts preset without changing the landscape default', () => {
+ expect(stickmanOutputPresetDefaults('youtube-shorts')).toMatchObject({
+ outputPreset: 'youtube-shorts',
+ ratio: '9:16',
+ targetDurationSeconds: 30,
+ targetLanguage: 'en-US',
+ ttsProvider: 'edge-tts'
+ });
+ expect(stickmanOutputPresetDefaults('landscape')).toMatchObject({
+ outputPreset: 'landscape',
+ ratio: '16:9',
+ sourceLanguage: 'auto',
+ targetDurationSeconds: 30,
+ targetLanguage: 'zh-CN'
+ });
+ });
+});
diff --git a/packages/stickman-remotion/package.json b/packages/stickman-remotion/package.json
index 142769f3..6e51945b 100644
--- a/packages/stickman-remotion/package.json
+++ b/packages/stickman-remotion/package.json
@@ -9,6 +9,7 @@
"test": "vitest run --passWithNoTests"
},
"dependencies": {
+ "@opencreator/protocol": "workspace:*",
"react": "18.3.1",
"react-dom": "18.3.1",
"remotion": "4.0.473"
diff --git a/packages/stickman-remotion/src/Root.tsx b/packages/stickman-remotion/src/Root.tsx
index f887d0c6..683522db 100644
--- a/packages/stickman-remotion/src/Root.tsx
+++ b/packages/stickman-remotion/src/Root.tsx
@@ -1,30 +1,56 @@
import React from 'react';
import { Composition } from 'remotion';
-import { StickmanLandscape } from './StickmanLandscape.js';
+import { StickmanLandscape, StickmanPortrait } from './StickmanLandscape.js';
import type { StickmanTimelineProps } from './timeline.js';
const defaultProps: StickmanTimelineProps = {
+ ratio: '16:9',
fps: 30,
width: 1280,
height: 720,
totalFrames: 30,
- shots: []
+ shots: [],
+ captions: []
+};
+
+const portraitDefaultProps: StickmanTimelineProps = {
+ ...defaultProps,
+ ratio: '9:16',
+ width: 720,
+ height: 1280
};
export const RemotionRoot: React.FC = () => (
- <Composition
- id="StickmanLandscape"
- component={StickmanLandscape}
- width={1280}
- height={720}
- fps={30}
- durationInFrames={30}
- defaultProps={defaultProps}
- calculateMetadata={({ props }) => ({
- durationInFrames: props.totalFrames,
- fps: props.fps,
- width: props.width,
- height: props.height
- })}
- />
+ <>
+ <Composition
+ id="StickmanLandscape"
+ component={StickmanLandscape}
+ width={1280}
+ height={720}
+ fps={30}
+ durationInFrames={30}
+ defaultProps={defaultProps}
+ calculateMetadata={({ props }: { props: StickmanTimelineProps }) => ({
+ durationInFrames: props.totalFrames,
+ fps: props.fps,
+ width: props.width,
+ height: props.height
+ })}
+ />
+ <Composition
+ id="StickmanPortrait"
+ component={StickmanPortrait}
+ width={720}
+ height={1280}
+ fps={30}
+ durationInFrames={30}
+ defaultProps={portraitDefaultProps}
+ calculateMetadata={({ props }: { props: StickmanTimelineProps }) => ({
+ durationInFrames: props.totalFrames,
+ fps: props.fps,
+ width: props.width,
+ height: props.height
+ })}
+ />
+ </>
);
diff --git a/packages/stickman-remotion/src/StickmanLandscape.tsx b/packages/stickman-remotion/src/StickmanLandscape.tsx
index 130d0cf5..9a3f17ae 100644
--- a/packages/stickman-remotion/src/StickmanLandscape.tsx
+++ b/packages/stickman-remotion/src/StickmanLandscape.tsx
@@ -7,7 +7,7 @@ import {
interpolate,
useCurrentFrame
} from 'remotion';
-import { assertTimeline, buildNarrationTrack, type StickmanTimelineProps } from './timeline.js';
+import { assertTimeline, buildNarrationTrack, motionTransform, type StickmanTimelineProps } from './timeline.js';
import notoSansBold from '../assets/fonts/NotoSans-Bold.woff2';
import notoSansScBold from '../assets/fonts/NotoSansSC-Bold.woff2';
@@ -28,7 +28,15 @@ const localFontFaces = `
}
`;
-export const StickmanLandscape: React.FC<StickmanTimelineProps> = props => {
+export const StickmanLandscape: React.FC<StickmanTimelineProps> = props => (
+ <StickmanVideo {...props} />
+);
+
+export const StickmanPortrait: React.FC<StickmanTimelineProps> = props => (
+ <StickmanVideo {...props} />
+);
+
+const StickmanVideo: React.FC<StickmanTimelineProps> = props => {
const timeline = assertTimeline(props);
const narrationTrack = buildNarrationTrack(timeline);
return (
@@ -55,6 +63,15 @@ export const StickmanLandscape: React.FC<StickmanTimelineProps> = props => {
<Shot shot={shot} />
</Sequence>
))}
+ {timeline.captions.map(caption => (
+ <Sequence
+ key={`caption-${caption.segmentId}`}
+ from={caption.startFrame}
+ durationInFrames={caption.endFrame - caption.startFrame}
+ >
+ <Caption ratio={timeline.ratio} text={caption.text} />
+ </Sequence>
+ ))}
</AbsoluteFill>
);
};
@@ -66,25 +83,42 @@ const Shot: React.FC<{ shot: StickmanTimelineProps['shots'][number] }> = ({ shot
extrapolateLeft: 'clamp',
extrapolateRight: 'clamp'
});
- const transform = motionTransform(shot.motion, progress);
return (
<AbsoluteFill style={{ overflow: 'hidden', backgroundColor: '#ffffff' }}>
<Img
src={fileUrl(shot.imagePath)}
- style={{ width: '100%', height: '100%', objectFit: 'cover', transform }}
+ style={{
+ width: '100%',
+ height: '100%',
+ objectFit: 'cover',
+ transform: motionTransform(shot.motion, progress)
+ }}
/>
<div style={{ position: 'absolute', inset: 24, border: '3px solid rgba(15,23,42,0.12)' }} />
</AbsoluteFill>
);
};
-function motionTransform(motion: StickmanTimelineProps['shots'][number]['motion'], progress: number): string {
- if (motion === 'push-in') return `scale(${1 + progress * 0.08})`;
- if (motion === 'zoom-out') return `scale(${1.08 - progress * 0.08})`;
- if (motion === 'pan-left') return `scale(1.06) translateX(${3 - progress * 6}%)`;
- if (motion === 'pan-right') return `scale(1.06) translateX(${-3 + progress * 6}%)`;
- return 'scale(1)';
-}
+const Caption: React.FC<{ ratio: StickmanTimelineProps['ratio']; text: string }> = ({ ratio, text }) => (
+ <div style={{
+ position: 'absolute',
+ left: ratio === '9:16' ? 32 : 48,
+ right: ratio === '9:16' ? 32 : 48,
+ bottom: ratio === '9:16' ? 112 : 48,
+ padding: ratio === '9:16' ? '18px 22px' : '14px 20px',
+ borderRadius: ratio === '9:16' ? 18 : 14,
+ backgroundColor: 'rgba(0, 0, 0, 0.78)',
+ color: '#ffffff',
+ fontSize: ratio === '9:16' ? 46 : 32,
+ fontWeight: 700,
+ lineHeight: 1.12,
+ textAlign: 'center',
+ whiteSpace: 'pre-wrap',
+ overflowWrap: 'anywhere'
+ }}>
+ {text}
+ </div>
+);
function fileUrl(path: string): string {
if (/^(?:https?:|data:|blob:)/i.test(path)) return path;
diff --git a/packages/stickman-remotion/src/timeline.test.ts b/packages/stickman-remotion/src/timeline.test.ts
index 7dd5ac08..8af6af43 100644
--- a/packages/stickman-remotion/src/timeline.test.ts
+++ b/packages/stickman-remotion/src/timeline.test.ts
@@ -4,6 +4,7 @@ import { assertTimeline, buildNarrationTrack } from './timeline.js';
describe('stickman remotion timeline', () => {
it('rejects gaps and accepts contiguous shots', () => {
const base = {
+ ratio: '16:9' as const,
fps: 30,
width: 1280 as const,
height: 720 as const,
@@ -11,6 +12,10 @@ describe('stickman remotion timeline', () => {
shots: [
{ shotId: 'shot-1', startFrame: 0, endFrame: 30, imageArtifactId: 'i1', audioArtifactId: 'a1', motion: 'static' as const, imageSha256: 'a'.repeat(64), audioSha256: 'b'.repeat(64), imagePath: 'a.png', audioPath: 'a.mp3' },
{ shotId: 'shot-2', startFrame: 30, endFrame: 60, imageArtifactId: 'i2', audioArtifactId: 'a1', motion: 'push-in' as const, imageSha256: 'c'.repeat(64), audioSha256: 'b'.repeat(64), imagePath: 'b.png', audioPath: 'a.mp3' }
+ ],
+ captions: [
+ { segmentId: 'segment-01', startFrame: 0, endFrame: 30, text: 'First caption' },
+ { segmentId: 'segment-02', startFrame: 30, endFrame: 60, text: 'Second caption' }
]
};
expect(assertTimeline(base)).toBe(base);
@@ -23,4 +28,28 @@ describe('stickman remotion timeline', () => {
{ shotId: 'shot-2', from: 30, durationInFrames: 30, audioPath: 'a.mp3' }
]);
});
+
+ it('accepts the portrait Shorts canvas and contiguous caption coverage', () => {
+ const base = {
+ ratio: '9:16' as const,
+ fps: 30,
+ width: 720,
+ height: 1280,
+ totalFrames: 30,
+ shots: [{
+ shotId: 'shot-1',
+ startFrame: 0,
+ endFrame: 30,
+ imageArtifactId: 'i1',
+ audioArtifactId: 'a1',
+ motion: 'static' as const,
+ imageSha256: 'a'.repeat(64),
+ audioSha256: 'b'.repeat(64),
+ imagePath: 'a.png',
+ audioPath: 'a.mp3'
+ }],
+ captions: [{ segmentId: 'segment-01', startFrame: 0, endFrame: 30, text: 'A vertical hook' }]
+ };
+ expect(assertTimeline(base)).toBe(base);
+ });
});
diff --git a/packages/stickman-remotion/src/timeline.ts b/packages/stickman-remotion/src/timeline.ts
index 9750a470..1d8a3b51 100644
--- a/packages/stickman-remotion/src/timeline.ts
+++ b/packages/stickman-remotion/src/timeline.ts
@@ -1,7 +1,13 @@
+import {
+ stickmanCanvasForRatio,
+ type StickmanRatio
+} from '@opencreator/protocol';
+
export type StickmanTimelineProps = {
+ ratio: StickmanRatio;
fps: number;
- width: 1280;
- height: 720;
+ width: number;
+ height: number;
totalFrames: number;
shots: Array<{
shotId: string;
@@ -15,10 +21,22 @@ export type StickmanTimelineProps = {
imagePath: string;
audioPath: string;
}>;
+ captions: Array<{
+ segmentId: string;
+ startFrame: number;
+ endFrame: number;
+ text: string;
+ }>;
};
export function assertTimeline(value: StickmanTimelineProps): StickmanTimelineProps {
- if (value.width !== 1280 || value.height !== 720 || value.fps <= 0 || value.totalFrames <= 0) {
+ const canvas = stickmanCanvasForRatio(value.ratio);
+ if (
+ value.width !== canvas.width
+ || value.height !== canvas.height
+ || value.fps <= 0
+ || value.totalFrames <= 0
+ ) {
throw new Error('stickman_timeline_invalid');
}
let cursor = 0;
@@ -30,6 +48,15 @@ export function assertTimeline(value: StickmanTimelineProps): StickmanTimelinePr
cursor = shot.endFrame;
}
if (cursor !== value.totalFrames) throw new Error('stickman_timeline_total_mismatch');
+ cursor = 0;
+ for (const caption of value.captions) {
+ if (caption.startFrame !== cursor || caption.endFrame <= caption.startFrame) {
+ throw new Error(`stickman_timeline_caption_gap_or_overlap:${caption.segmentId}`);
+ }
+ if (!caption.text.trim()) throw new Error(`stickman_timeline_caption_empty:${caption.segmentId}`);
+ cursor = caption.endFrame;
+ }
+ if (cursor !== value.totalFrames) throw new Error('stickman_timeline_caption_total_mismatch');
return value;
}
@@ -46,3 +73,14 @@ export function buildNarrationTrack(timeline: StickmanTimelineProps): Array<{
audioPath: shot.audioPath
}));
}
+
+export function motionTransform(
+ motion: StickmanTimelineProps['shots'][number]['motion'],
+ progress: number
+): string {
+ if (motion === 'push-in') return `scale(${1 + progress * 0.08})`;
+ if (motion === 'zoom-out') return `scale(${1.08 - progress * 0.08})`;
+ if (motion === 'pan-left') return `scale(1.06) translateX(${3 - progress * 6}%)`;
+ if (motion === 'pan-right') return `scale(1.06) translateX(${-3 + progress * 6}%)`;
+ return 'scale(1)';
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1e78c23b..f206f947 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -293,6 +293,9 @@ importers:
packages/stickman-remotion:
dependencies:
+ '@opencreator/protocol':
+ specifier: workspace:*
+ version: link:../protocol
react:
specifier: 18.3.1
version: 18.3.1