Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat: prepare Gemini media before upload

rpone7158a8b743b4
diff --git a/packages/gemini/package.json b/packages/gemini/package.json
index 700a48c4..f6b7867f 100644
--- a/packages/gemini/package.json
+++ b/packages/gemini/package.json
@@ -12,6 +12,7 @@
"@hypit/elaborator": "workspace:*",
"@hypit/markup": "workspace:*",
"@hypit/protocol": "workspace:*",
- "@hypit/text": "workspace:*"
+ "@hypit/text": "workspace:*",
+ "@hypit/media-pipeline": "workspace:*"
}
}
diff --git a/packages/gemini/src/fragment.ts b/packages/gemini/src/fragment.ts
index a442c80b..d7de5bab 100644
--- a/packages/gemini/src/fragment.ts
+++ b/packages/gemini/src/fragment.ts
@@ -2,6 +2,7 @@ import { artifactTypes } from "@hypit/artifact";
import { sealGraphFragment } from "@hypit/elaborator";
import type { FragmentOperation } from "@hypit/elaborator";
import { textTypes } from "@hypit/text";
+import { mediaPipelineProducers } from "@hypit/media-pipeline";
import type { GeminiModel } from "./manifest.js";
import { geminiProducers, geminiTypes } from "./manifest.js";
@@ -26,10 +27,17 @@ export function createGeminiFragment(model: GeminiModel, mediaCount: number) {
}];
let draft = operation("start-request");
for (let index = 0; index < mediaCount; index += 1) {
+ const prepareId = `prepare-media-${String(index + 1).padStart(4, "0")}`;
+ operations.push({
+ id: prepareId,
+ producer: mediaPipelineProducers.prepare,
+ inputs: { source: input(`media-${String(index + 1).padStart(4, "0")}`) },
+ result: { kind: "need" as const, name: "artifact" },
+ });
const id = `bind-media-${String(index + 1).padStart(4, "0")}`;
operations.push({
id, producer: geminiProducers.bindMedia,
- inputs: { draft, media: input(`media-${String(index + 1).padStart(4, "0")}`) },
+ inputs: { draft, media: operation(prepareId) },
result: { kind: "output" as const, name: "draft" },
});
draft = operation(id);
diff --git a/packages/gemini/src/manifest.ts b/packages/gemini/src/manifest.ts
index 13a7b7e4..731799a2 100644
--- a/packages/gemini/src/manifest.ts
+++ b/packages/gemini/src/manifest.ts
@@ -1,6 +1,7 @@
import { artifactDependency, artifactTypes } from "@hypit/artifact";
import type { CapabilityRef, ModuleManifest, ProducerRef, TypeRef } from "@hypit/protocol";
import { textDependency, textTypes } from "@hypit/text";
+import { mediaPipelineDependency } from "@hypit/media-pipeline";
export const geminiModuleRef = { name: "@hypit/gemini", version: "1" } as const;
@@ -62,7 +63,7 @@ export const geminiManifest: ModuleManifest = {
format: "hypit.module@1",
name: geminiModuleRef.name,
version: geminiModuleRef.version,
- dependencies: [artifactDependency, textDependency],
+ dependencies: [artifactDependency, textDependency, mediaPipelineDependency],
types: [{ name: geminiTypes.request.name }, { name: geminiTypes.draft.name }],
capabilities: geminiModels.map((model) => ({ name: model, returns: textTypes.text })),
producers: [
diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.ts
index 674044ba..692fd33d 100644
--- a/packages/media-execution/src/execute.ts
+++ b/packages/media-execution/src/execute.ts
@@ -18,6 +18,7 @@ import {
verifyMediaTransformProgram,
type ExtractAudioNeed,
type ExtractFrameNeed,
+ type PrepareMediaNeed,
type InspectMediaNeed,
type MuxMediaNeed,
type NormalizeMediaNeed,
@@ -93,6 +94,52 @@ export type RenderMockSilenceRequest = {
readonly sampleFrames: number;
};
+function prepareNeed(value: CanonicalValue): PrepareMediaNeed {
+ const item = object(value, "Media preparation need");
+ const source = object(item.source, "Media preparation source");
+ assert(source.kind === "blob" && typeof source.digest === "string"
+ && Number.isSafeInteger(source.size) && (source.size as number) >= 0
+ && typeof source.mediaType === "string", "Media preparation source must be a BlobRef");
+ assert(item.profile === "gemini-reference", "Media preparation profile is unsupported");
+ return { source: source as unknown as BlobRef, profile: "gemini-reference" };
+}
+
+export async function executePrepareMedia(env: MediaExecutionEnvironment, constraints: CanonicalValue): Promise<MediaOperationResult> {
+ const need = prepareNeed(constraints);
+ const source = await sourceBytes(env, need.source);
+ if (need.source.mediaType.startsWith("image/")) {
+ const work = await mkdtemp(join(tmpdir(), "hypit-media-prepare-"));
+ try {
+ const input = join(work, "source.bin");
+ const output = join(work, "prepared.webp");
+ await writeFile(input, source);
+ await runProcess({ executable: env.ffmpegPath, argv: ["-y", "-i", input, "-frames:v", "1", "-vf", "scale='min(2048,iw)':-2", "-c:v", "libwebp", "-q:v", "75", output], timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024 });
+ return artifactResult(await env.artifacts.putFile(output, "image/webp"));
+ } finally { await rm(work, { recursive: true, force: true }).catch(() => {}); }
+ }
+ if (need.source.mediaType.startsWith("audio/")) {
+ const work = await mkdtemp(join(tmpdir(), "hypit-media-prepare-"));
+ try {
+ const input = join(work, "source.bin");
+ const output = join(work, "prepared.m4a");
+ await writeFile(input, source);
+ await runProcess({ executable: env.ffmpegPath, argv: ["-y", "-i", input, "-vn", "-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart", output], timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024 });
+ return artifactResult(await env.artifacts.putFile(output, "audio/mp4"));
+ } finally { await rm(work, { recursive: true, force: true }).catch(() => {}); }
+ }
+ if (need.source.mediaType.startsWith("video/")) {
+ const work = await mkdtemp(join(tmpdir(), "hypit-media-prepare-"));
+ try {
+ const input = join(work, "source.bin");
+ const output = join(work, "prepared.mp4");
+ await writeFile(input, source);
+ await runProcess({ executable: env.ffmpegPath, argv: ["-y", "-i", input, "-vf", "scale='min(1280,iw)':-2", "-c:v", "libx264", "-preset", "veryfast", "-crf", "28", "-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart", output], timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024 });
+ return artifactResult(await env.artifacts.putFile(output, "video/mp4"));
+ } finally { await rm(work, { recursive: true, force: true }).catch(() => {}); }
+ }
+ throw new Error(`Media preparation does not support ${need.source.mediaType}`);
+}
+
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.ts
index 4206ee57..ae5f351d 100644
--- a/packages/media-execution/src/index.ts
+++ b/packages/media-execution/src/index.ts
@@ -4,6 +4,7 @@ export {
executeExtractFrame,
executeMuxProgramMedia,
executeNormalizeMedia,
+ executePrepareMedia,
executeProjectSpeechEvidenceAudio,
executeRenderTimelineAudio,
executeRenderStillVideo,
diff --git a/packages/media-pipeline/src/component.ts b/packages/media-pipeline/src/component.ts
index 2d8bab94..bb46c2dc 100644
--- a/packages/media-pipeline/src/component.ts
+++ b/packages/media-pipeline/src/component.ts
@@ -42,6 +42,7 @@ import type {
RenderStillVideoNeed,
StillVideoRequest,
TransformMediaNeed,
+ PrepareMediaNeed,
} from "./types.js";
function inline(value: StoredValue, subject: string): CanonicalValue {
@@ -102,6 +103,14 @@ export const mediaPipelineComponent = {
return { outputs: {}, needs: { inspection: canonicalize(need) } };
},
},
+ {
+ producer: mediaPipelineProducers.prepare,
+ handler: ({ inputs }) => {
+ const source = blob(inputs.source!.value, "Media preparation source");
+ const need: PrepareMediaNeed = { source, profile: "gemini-reference" };
+ return { outputs: {}, needs: { artifact: canonicalize(need) } };
+ },
+ },
{
producer: mediaPipelineProducers.select,
handler: ({ inputs }) => {
diff --git a/packages/media-pipeline/src/fragment.ts b/packages/media-pipeline/src/fragment.ts
index c4a927b1..2ac00691 100644
--- a/packages/media-pipeline/src/fragment.ts
+++ b/packages/media-pipeline/src/fragment.ts
@@ -49,6 +49,17 @@ export const synchronizedMediaFragment = sealGraphFragment({
}],
});
+export const prepareMediaFragment = sealGraphFragment({
+ inputs: [{ name: "source", type: artifactTypes.blob }],
+ operations: [{
+ id: "prepare",
+ producer: mediaPipelineProducers.prepare,
+ inputs: { source: input("source") },
+ result: { kind: "need", name: "artifact" },
+ }],
+ exports: [{ name: "artifact", type: artifactTypes.blob, root: operation("prepare") }],
+});
+
export const transformMediaFragment = sealGraphFragment({
inputs: [
{ name: "media", type: mediaTypes.synchronized },
diff --git a/packages/media-pipeline/src/index.ts b/packages/media-pipeline/src/index.ts
index bc1d13ba..e119aef9 100644
--- a/packages/media-pipeline/src/index.ts
+++ b/packages/media-pipeline/src/index.ts
@@ -5,6 +5,7 @@ export {
export {
extractAudioFragment,
extractFrameFragment,
+ prepareMediaFragment,
synchronizedMediaFragment,
stillVideoFragment,
transformMediaFragment,
@@ -14,7 +15,7 @@ export {
sealAudioProgramPlan,
verifyAudioProgramPlan,
} from "./audio-plan.js";
-export { mediaPipelineCapabilities, mediaPipelineManifest, mediaPipelineMarkupSurfaces, mediaPipelineModuleRef, mediaPipelineProducers, mediaPipelineTypes, audioProgramPlanSchema, mediaSelectionRequestSchema, audioExtractionRequestSchema, frameExtractionRequestSchema, stillVideoRequestSchema, mediaTransformProgramSchema } from "./manifest.js";
+export { mediaPipelineCapabilities, mediaPipelineDependency, mediaPipelineManifest, mediaPipelineMarkupSurfaces, mediaPipelineModuleRef, mediaPipelineProducers, mediaPipelineTypes, audioProgramPlanSchema, mediaSelectionRequestSchema, audioExtractionRequestSchema, frameExtractionRequestSchema, stillVideoRequestSchema, mediaTransformProgramSchema } from "./manifest.js";
export {
sealMediaSelectionRequest,
selectMediaStreams,
diff --git a/packages/media-pipeline/src/manifest.ts b/packages/media-pipeline/src/manifest.ts
index e0809025..3e967af7 100644
--- a/packages/media-pipeline/src/manifest.ts
+++ b/packages/media-pipeline/src/manifest.ts
@@ -24,6 +24,7 @@ export const mediaPipelineTypes = {
} satisfies Record<string, TypeRef>;
export const mediaPipelineCapabilities = {
inspect: { module: mediaPipelineModuleRef, name: "inspect-media" },
+ prepare: { module: mediaPipelineModuleRef, name: "prepare-media" },
normalize: { module: mediaPipelineModuleRef, name: "normalize-media" },
transform: { module: mediaPipelineModuleRef, name: "transform-media" },
extractAudio: { module: mediaPipelineModuleRef, name: "extract-media-audio" },
@@ -35,6 +36,7 @@ export const mediaPipelineCapabilities = {
} satisfies Record<string, CapabilityRef>;
export const mediaPipelineProducers = {
inspect: { module: mediaPipelineModuleRef, name: "request-media-inspection" },
+ prepare: { module: mediaPipelineModuleRef, name: "request-media-preparation" },
select: { module: mediaPipelineModuleRef, name: "select-media-streams" },
normalize: { module: mediaPipelineModuleRef, name: "request-media-normalization" },
transform: { module: mediaPipelineModuleRef, name: "request-media-transform" },
@@ -49,6 +51,8 @@ export const mediaPipelineProducers = {
projectMuxed: { module: mediaPipelineModuleRef, name: "project-muxed-media" },
} satisfies Record<string, ProducerRef>;
+export const mediaPipelineDependency = { module: mediaPipelineModuleRef } as const;
+
const integer = { kind: "number", integer: true, minimum: 0 } as const;
const mode = (name: string): ValueSchema => ({
kind: "object",
@@ -428,6 +432,7 @@ export const mediaPipelineManifest: ModuleManifest = {
],
capabilities: [
{ name: mediaPipelineCapabilities.inspect.name, returns: mediaTypes.inspection },
+ { name: mediaPipelineCapabilities.prepare.name, returns: artifactTypes.blob },
{ name: mediaPipelineCapabilities.normalize.name, returns: mediaTypes.synchronized },
{ name: mediaPipelineCapabilities.transform.name, returns: artifactTypes.blob },
{ name: mediaPipelineCapabilities.extractAudio.name, returns: artifactTypes.blob },
@@ -448,6 +453,12 @@ export const mediaPipelineManifest: ModuleManifest = {
returns: mediaTypes.inspection,
}],
},
+ {
+ name: mediaPipelineProducers.prepare.name,
+ inputs: [{ name: "source", type: artifactTypes.blob }],
+ outputs: [],
+ needs: [{ name: "artifact", capability: mediaPipelineCapabilities.prepare, returns: artifactTypes.blob }],
+ },
{
name: mediaPipelineProducers.select.name,
inputs: [
diff --git a/packages/media-pipeline/src/types.ts b/packages/media-pipeline/src/types.ts
index 8c9c0acd..533ea390 100644
--- a/packages/media-pipeline/src/types.ts
+++ b/packages/media-pipeline/src/types.ts
@@ -18,6 +18,12 @@ export type InspectMediaNeed = {
readonly source: BlobRef;
};
+/** Deterministic model-reference preparation over one ordinary media Blob. */
+export type PrepareMediaNeed = {
+ readonly source: BlobRef;
+ readonly profile: "gemini-reference";
+};
+
export type MediaVideoSelector =
| { readonly mode: "primary-moving" }
| { readonly mode: "stream-index"; readonly streamIndex: number };
diff --git a/packages/provider-media-aws-lambda/src/contract.ts b/packages/provider-media-aws-lambda/src/contract.ts
index bbf89425..8435304d 100644
--- a/packages/provider-media-aws-lambda/src/contract.ts
+++ b/packages/provider-media-aws-lambda/src/contract.ts
@@ -14,6 +14,7 @@ export const MEDIA_LAMBDA_RESPONSE = "hypit.media-lambda-response@1";
export const mediaLambdaOperations = [
"inspect",
+ "prepare",
"normalize",
"transform",
"extract-audio",
diff --git a/packages/provider-media-aws-lambda/src/provider.ts b/packages/provider-media-aws-lambda/src/provider.ts
index 869dd925..02cc0a37 100644
--- a/packages/provider-media-aws-lambda/src/provider.ts
+++ b/packages/provider-media-aws-lambda/src/provider.ts
@@ -127,6 +127,12 @@ export function createAwsLambdaMediaProvider(config: CreateAwsLambdaMediaProvide
returns: mediaTypes.synchronized,
handler: operation("normalize"),
},
+ {
+ lifecycle: "immediate" as const,
+ capability: mediaPipelineCapabilities.prepare,
+ returns: artifactTypes.blob,
+ handler: operation("prepare"),
+ },
{
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.transform,
diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.ts
index 482bcace..c3ccb1e5 100644
--- a/packages/provider-media-local/src/provider.ts
+++ b/packages/provider-media-local/src/provider.ts
@@ -6,6 +6,7 @@ import {
executeInspectMedia,
executeMuxProgramMedia,
executeNormalizeMedia,
+ executePrepareMedia,
executeProjectSpeechEvidenceAudio,
executeRenderTimelineAudio,
executeRenderStillVideo,
@@ -98,6 +99,12 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
returns: mediaTypes.synchronized,
handler: operation(executeNormalizeMedia),
},
+ {
+ lifecycle: "immediate" as const,
+ capability: mediaPipelineCapabilities.prepare,
+ returns: artifactTypes.blob,
+ handler: operation(executePrepareMedia),
+ },
{
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.transform,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1e38ea8d..c2e63cbc 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1333,6 +1333,9 @@ importers:
'@hypit/markup':
specifier: workspace:*
version: link:../markup
+ '@hypit/media-pipeline':
+ specifier: workspace:*
+ version: link:../media-pipeline
'@hypit/protocol':
specifier: workspace:*
version: link:../protocol
diff --git a/services/media-lambda/src/handler.ts b/services/media-lambda/src/handler.ts
index f05387c2..81194db4 100644
--- a/services/media-lambda/src/handler.ts
+++ b/services/media-lambda/src/handler.ts
@@ -6,6 +6,7 @@ import {
executeExtractFrame,
executeMuxProgramMedia,
executeNormalizeMedia,
+ executePrepareMedia,
executeProjectSpeechEvidenceAudio,
executeRenderTimelineAudio,
executeRenderStillVideo,
@@ -37,6 +38,7 @@ const OPERATIONS: Record<
> = {
"inspect": executeInspectMedia,
"normalize": executeNormalizeMedia,
+ "prepare": executePrepareMedia,
"transform": executeTransformMedia,
"extract-audio": executeExtractAudio,
"extract-frame": executeExtractFrame,