Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat: prepare Gemini media before upload
rpone7158a8b743b4
diff --git a/packages/gemini/package.json b/packages/gemini/package.jsonindex 700a48c4..f6b7867f 100644--- a/packages/gemini/package.json+++ b/packages/gemini/package.json@@ -12,6 +12,7 @@"@hypit/elaborator": "workspace:*","@hypit/markup": "workspace:*","@hypit/protocol": "workspace:*",- "@hypit/text": "workspace:*"+ "@hypit/text": "workspace:*",+ "@hypit/media-pipeline": "workspace:*"}}diff --git a/packages/gemini/src/fragment.ts b/packages/gemini/src/fragment.tsindex a442c80b..d7de5bab 100644--- a/packages/gemini/src/fragment.ts+++ b/packages/gemini/src/fragment.ts@@ -2,6 +2,7 @@ import { artifactTypes } from "@hypit/artifact";import { sealGraphFragment } from "@hypit/elaborator";import type { FragmentOperation } from "@hypit/elaborator";import { textTypes } from "@hypit/text";+import { mediaPipelineProducers } from "@hypit/media-pipeline";import type { GeminiModel } from "./manifest.js";import { geminiProducers, geminiTypes } from "./manifest.js";@@ -26,10 +27,17 @@ export function createGeminiFragment(model: GeminiModel, mediaCount: number) {}];let draft = operation("start-request");for (let index = 0; index < mediaCount; index += 1) {+ const prepareId = `prepare-media-${String(index + 1).padStart(4, "0")}`;+ operations.push({+ id: prepareId,+ producer: mediaPipelineProducers.prepare,+ inputs: { source: input(`media-${String(index + 1).padStart(4, "0")}`) },+ result: { kind: "need" as const, name: "artifact" },+ });const id = `bind-media-${String(index + 1).padStart(4, "0")}`;operations.push({id, producer: geminiProducers.bindMedia,- inputs: { draft, media: input(`media-${String(index + 1).padStart(4, "0")}`) },+ inputs: { draft, media: operation(prepareId) },result: { kind: "output" as const, name: "draft" },});draft = operation(id);diff --git a/packages/gemini/src/manifest.ts b/packages/gemini/src/manifest.tsindex 13a7b7e4..731799a2 100644--- a/packages/gemini/src/manifest.ts+++ b/packages/gemini/src/manifest.ts@@ -1,6 +1,7 @@import { artifactDependency, artifactTypes } from "@hypit/artifact";import type { CapabilityRef, ModuleManifest, ProducerRef, TypeRef } from "@hypit/protocol";import { textDependency, textTypes } from "@hypit/text";+import { mediaPipelineDependency } from "@hypit/media-pipeline";export const geminiModuleRef = { name: "@hypit/gemini", version: "1" } as const;@@ -62,7 +63,7 @@ export const geminiManifest: ModuleManifest = {format: "hypit.module@1",name: geminiModuleRef.name,version: geminiModuleRef.version,- dependencies: [artifactDependency, textDependency],+ dependencies: [artifactDependency, textDependency, mediaPipelineDependency],types: [{ name: geminiTypes.request.name }, { name: geminiTypes.draft.name }],capabilities: geminiModels.map((model) => ({ name: model, returns: textTypes.text })),producers: [diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.tsindex 674044ba..692fd33d 100644--- a/packages/media-execution/src/execute.ts+++ b/packages/media-execution/src/execute.ts@@ -18,6 +18,7 @@ import {verifyMediaTransformProgram,type ExtractAudioNeed,type ExtractFrameNeed,+ type PrepareMediaNeed,type InspectMediaNeed,type MuxMediaNeed,type NormalizeMediaNeed,@@ -93,6 +94,52 @@ export type RenderMockSilenceRequest = {readonly sampleFrames: number;};+function prepareNeed(value: CanonicalValue): PrepareMediaNeed {+ const item = object(value, "Media preparation need");+ const source = object(item.source, "Media preparation source");+ assert(source.kind === "blob" && typeof source.digest === "string"+ && Number.isSafeInteger(source.size) && (source.size as number) >= 0+ && typeof source.mediaType === "string", "Media preparation source must be a BlobRef");+ assert(item.profile === "gemini-reference", "Media preparation profile is unsupported");+ return { source: source as unknown as BlobRef, profile: "gemini-reference" };+}++export async function executePrepareMedia(env: MediaExecutionEnvironment, constraints: CanonicalValue): Promise<MediaOperationResult> {+ const need = prepareNeed(constraints);+ const source = await sourceBytes(env, need.source);+ if (need.source.mediaType.startsWith("image/")) {+ const work = await mkdtemp(join(tmpdir(), "hypit-media-prepare-"));+ try {+ const input = join(work, "source.bin");+ const output = join(work, "prepared.webp");+ await writeFile(input, source);+ await runProcess({ executable: env.ffmpegPath, argv: ["-y", "-i", input, "-frames:v", "1", "-vf", "scale='min(2048,iw)':-2", "-c:v", "libwebp", "-q:v", "75", output], timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024 });+ return artifactResult(await env.artifacts.putFile(output, "image/webp"));+ } finally { await rm(work, { recursive: true, force: true }).catch(() => {}); }+ }+ if (need.source.mediaType.startsWith("audio/")) {+ const work = await mkdtemp(join(tmpdir(), "hypit-media-prepare-"));+ try {+ const input = join(work, "source.bin");+ const output = join(work, "prepared.m4a");+ await writeFile(input, source);+ await runProcess({ executable: env.ffmpegPath, argv: ["-y", "-i", input, "-vn", "-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart", output], timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024 });+ return artifactResult(await env.artifacts.putFile(output, "audio/mp4"));+ } finally { await rm(work, { recursive: true, force: true }).catch(() => {}); }+ }+ if (need.source.mediaType.startsWith("video/")) {+ const work = await mkdtemp(join(tmpdir(), "hypit-media-prepare-"));+ try {+ const input = join(work, "source.bin");+ const output = join(work, "prepared.mp4");+ await writeFile(input, source);+ await runProcess({ executable: env.ffmpegPath, argv: ["-y", "-i", input, "-vf", "scale='min(1280,iw)':-2", "-c:v", "libx264", "-preset", "veryfast", "-crf", "28", "-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart", output], timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024 });+ return artifactResult(await env.artifacts.putFile(output, "video/mp4"));+ } finally { await rm(work, { recursive: true, force: true }).catch(() => {}); }+ }+ throw new Error(`Media preparation does not support ${need.source.mediaType}`);+}+function assert(condition: unknown, message: string): asserts condition {if (!condition) throw new Error(message);}diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.tsindex 4206ee57..ae5f351d 100644--- a/packages/media-execution/src/index.ts+++ b/packages/media-execution/src/index.ts@@ -4,6 +4,7 @@ export {executeExtractFrame,executeMuxProgramMedia,executeNormalizeMedia,+ executePrepareMedia,executeProjectSpeechEvidenceAudio,executeRenderTimelineAudio,executeRenderStillVideo,diff --git a/packages/media-pipeline/src/component.ts b/packages/media-pipeline/src/component.tsindex 2d8bab94..bb46c2dc 100644--- a/packages/media-pipeline/src/component.ts+++ b/packages/media-pipeline/src/component.ts@@ -42,6 +42,7 @@ import type {RenderStillVideoNeed,StillVideoRequest,TransformMediaNeed,+ PrepareMediaNeed,} from "./types.js";function inline(value: StoredValue, subject: string): CanonicalValue {@@ -102,6 +103,14 @@ export const mediaPipelineComponent = {return { outputs: {}, needs: { inspection: canonicalize(need) } };},},+ {+ producer: mediaPipelineProducers.prepare,+ handler: ({ inputs }) => {+ const source = blob(inputs.source!.value, "Media preparation source");+ const need: PrepareMediaNeed = { source, profile: "gemini-reference" };+ return { outputs: {}, needs: { artifact: canonicalize(need) } };+ },+ },{producer: mediaPipelineProducers.select,handler: ({ inputs }) => {diff --git a/packages/media-pipeline/src/fragment.ts b/packages/media-pipeline/src/fragment.tsindex c4a927b1..2ac00691 100644--- a/packages/media-pipeline/src/fragment.ts+++ b/packages/media-pipeline/src/fragment.ts@@ -49,6 +49,17 @@ export const synchronizedMediaFragment = sealGraphFragment({}],});+export const prepareMediaFragment = sealGraphFragment({+ inputs: [{ name: "source", type: artifactTypes.blob }],+ operations: [{+ id: "prepare",+ producer: mediaPipelineProducers.prepare,+ inputs: { source: input("source") },+ result: { kind: "need", name: "artifact" },+ }],+ exports: [{ name: "artifact", type: artifactTypes.blob, root: operation("prepare") }],+});+export const transformMediaFragment = sealGraphFragment({inputs: [{ name: "media", type: mediaTypes.synchronized },diff --git a/packages/media-pipeline/src/index.ts b/packages/media-pipeline/src/index.tsindex bc1d13ba..e119aef9 100644--- a/packages/media-pipeline/src/index.ts+++ b/packages/media-pipeline/src/index.ts@@ -5,6 +5,7 @@ export {export {extractAudioFragment,extractFrameFragment,+ prepareMediaFragment,synchronizedMediaFragment,stillVideoFragment,transformMediaFragment,@@ -14,7 +15,7 @@ export {sealAudioProgramPlan,verifyAudioProgramPlan,} from "./audio-plan.js";-export { mediaPipelineCapabilities, mediaPipelineManifest, mediaPipelineMarkupSurfaces, mediaPipelineModuleRef, mediaPipelineProducers, mediaPipelineTypes, audioProgramPlanSchema, mediaSelectionRequestSchema, audioExtractionRequestSchema, frameExtractionRequestSchema, stillVideoRequestSchema, mediaTransformProgramSchema } from "./manifest.js";+export { mediaPipelineCapabilities, mediaPipelineDependency, mediaPipelineManifest, mediaPipelineMarkupSurfaces, mediaPipelineModuleRef, mediaPipelineProducers, mediaPipelineTypes, audioProgramPlanSchema, mediaSelectionRequestSchema, audioExtractionRequestSchema, frameExtractionRequestSchema, stillVideoRequestSchema, mediaTransformProgramSchema } from "./manifest.js";export {sealMediaSelectionRequest,selectMediaStreams,diff --git a/packages/media-pipeline/src/manifest.ts b/packages/media-pipeline/src/manifest.tsindex e0809025..3e967af7 100644--- a/packages/media-pipeline/src/manifest.ts+++ b/packages/media-pipeline/src/manifest.ts@@ -24,6 +24,7 @@ export const mediaPipelineTypes = {} satisfies Record<string, TypeRef>;export const mediaPipelineCapabilities = {inspect: { module: mediaPipelineModuleRef, name: "inspect-media" },+ prepare: { module: mediaPipelineModuleRef, name: "prepare-media" },normalize: { module: mediaPipelineModuleRef, name: "normalize-media" },transform: { module: mediaPipelineModuleRef, name: "transform-media" },extractAudio: { module: mediaPipelineModuleRef, name: "extract-media-audio" },@@ -35,6 +36,7 @@ export const mediaPipelineCapabilities = {} satisfies Record<string, CapabilityRef>;export const mediaPipelineProducers = {inspect: { module: mediaPipelineModuleRef, name: "request-media-inspection" },+ prepare: { module: mediaPipelineModuleRef, name: "request-media-preparation" },select: { module: mediaPipelineModuleRef, name: "select-media-streams" },normalize: { module: mediaPipelineModuleRef, name: "request-media-normalization" },transform: { module: mediaPipelineModuleRef, name: "request-media-transform" },@@ -49,6 +51,8 @@ export const mediaPipelineProducers = {projectMuxed: { module: mediaPipelineModuleRef, name: "project-muxed-media" },} satisfies Record<string, ProducerRef>;+export const mediaPipelineDependency = { module: mediaPipelineModuleRef } as const;+const integer = { kind: "number", integer: true, minimum: 0 } as const;const mode = (name: string): ValueSchema => ({kind: "object",@@ -428,6 +432,7 @@ export const mediaPipelineManifest: ModuleManifest = {],capabilities: [{ name: mediaPipelineCapabilities.inspect.name, returns: mediaTypes.inspection },+ { name: mediaPipelineCapabilities.prepare.name, returns: artifactTypes.blob },{ name: mediaPipelineCapabilities.normalize.name, returns: mediaTypes.synchronized },{ name: mediaPipelineCapabilities.transform.name, returns: artifactTypes.blob },{ name: mediaPipelineCapabilities.extractAudio.name, returns: artifactTypes.blob },@@ -448,6 +453,12 @@ export const mediaPipelineManifest: ModuleManifest = {returns: mediaTypes.inspection,}],},+ {+ name: mediaPipelineProducers.prepare.name,+ inputs: [{ name: "source", type: artifactTypes.blob }],+ outputs: [],+ needs: [{ name: "artifact", capability: mediaPipelineCapabilities.prepare, returns: artifactTypes.blob }],+ },{name: mediaPipelineProducers.select.name,inputs: [diff --git a/packages/media-pipeline/src/types.ts b/packages/media-pipeline/src/types.tsindex 8c9c0acd..533ea390 100644--- a/packages/media-pipeline/src/types.ts+++ b/packages/media-pipeline/src/types.ts@@ -18,6 +18,12 @@ export type InspectMediaNeed = {readonly source: BlobRef;};+/** Deterministic model-reference preparation over one ordinary media Blob. */+export type PrepareMediaNeed = {+ readonly source: BlobRef;+ readonly profile: "gemini-reference";+};+export type MediaVideoSelector =| { readonly mode: "primary-moving" }| { readonly mode: "stream-index"; readonly streamIndex: number };diff --git a/packages/provider-media-aws-lambda/src/contract.ts b/packages/provider-media-aws-lambda/src/contract.tsindex bbf89425..8435304d 100644--- a/packages/provider-media-aws-lambda/src/contract.ts+++ b/packages/provider-media-aws-lambda/src/contract.ts@@ -14,6 +14,7 @@ export const MEDIA_LAMBDA_RESPONSE = "hypit.media-lambda-response@1";export const mediaLambdaOperations = ["inspect",+ "prepare","normalize","transform","extract-audio",diff --git a/packages/provider-media-aws-lambda/src/provider.ts b/packages/provider-media-aws-lambda/src/provider.tsindex 869dd925..02cc0a37 100644--- a/packages/provider-media-aws-lambda/src/provider.ts+++ b/packages/provider-media-aws-lambda/src/provider.ts@@ -127,6 +127,12 @@ export function createAwsLambdaMediaProvider(config: CreateAwsLambdaMediaProvidereturns: mediaTypes.synchronized,handler: operation("normalize"),},+ {+ lifecycle: "immediate" as const,+ capability: mediaPipelineCapabilities.prepare,+ returns: artifactTypes.blob,+ handler: operation("prepare"),+ },{lifecycle: "immediate" as const,capability: mediaPipelineCapabilities.transform,diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.tsindex 482bcace..c3ccb1e5 100644--- a/packages/provider-media-local/src/provider.ts+++ b/packages/provider-media-local/src/provider.ts@@ -6,6 +6,7 @@ import {executeInspectMedia,executeMuxProgramMedia,executeNormalizeMedia,+ executePrepareMedia,executeProjectSpeechEvidenceAudio,executeRenderTimelineAudio,executeRenderStillVideo,@@ -98,6 +99,12 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptionsreturns: mediaTypes.synchronized,handler: operation(executeNormalizeMedia),},+ {+ lifecycle: "immediate" as const,+ capability: mediaPipelineCapabilities.prepare,+ returns: artifactTypes.blob,+ handler: operation(executePrepareMedia),+ },{lifecycle: "immediate" as const,capability: mediaPipelineCapabilities.transform,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 1e38ea8d..c2e63cbc 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1333,6 +1333,9 @@ importers:'@hypit/markup':specifier: workspace:*version: link:../markup+ '@hypit/media-pipeline':+ specifier: workspace:*+ version: link:../media-pipeline'@hypit/protocol':specifier: workspace:*version: link:../protocoldiff --git a/services/media-lambda/src/handler.ts b/services/media-lambda/src/handler.tsindex f05387c2..81194db4 100644--- a/services/media-lambda/src/handler.ts+++ b/services/media-lambda/src/handler.ts@@ -6,6 +6,7 @@ import {executeExtractFrame,executeMuxProgramMedia,executeNormalizeMedia,+ executePrepareMedia,executeProjectSpeechEvidenceAudio,executeRenderTimelineAudio,executeRenderStillVideo,@@ -37,6 +38,7 @@ const OPERATIONS: Record<> = {"inspect": executeInspectMedia,"normalize": executeNormalizeMedia,+ "prepare": executePrepareMedia,"transform": executeTransformMedia,"extract-audio": executeExtractAudio,"extract-frame": executeExtractFrame,