Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat(pixverse): add the PixVerse V6 model package (#303)
Sharky Wange2f25ba998a5
diff --git a/package.json b/package.jsonindex 5134126e..067fa589 100644--- a/package.json+++ b/package.json@@ -232,6 +232,7 @@"@hypit/narrative": "workspace:*","@hypit/performance": "workspace:*","@hypit/performance-studio": "workspace:*",+ "@hypit/pixverse": "workspace:*","@hypit/program-space": "workspace:*","@hypit/project-context-node": "workspace:*","@hypit/protocol": "workspace:*",diff --git a/packages/pixverse/README.md b/packages/pixverse/README.mdnew file mode 100644index 00000000..50929101--- /dev/null+++ b/packages/pixverse/README.md@@ -0,0 +1,35 @@+# `@hypit/pixverse`++Exact author/compute contracts and a package-owned author Surface for PixVerse V6.++The Surface projects the primary result to an ordinary video Artifact. The package contains no+Provider selection, API key or network execution. The selected Provider implements its exact+capability.++Connect prompt and frames as ordinary graph edges:++```xml+<pix:Video+ id="opening"+ prompt={line}+ duration="5"+ quality="720p"+ aspect-ratio="9:16"+ generate-audio="true"+/>++<pix:Video id="bridge" prompt={motion} duration="5" quality="720p"+ first-frame={hero.image} last-frame={product.image}/>+```++The Surface only lowers this syntax into the package's exact model request. It does not select a+Provider.++The model renders 1 to 15 seconds at `360p`, `540p`, `720p` or `1080p`. A prompt-only run states its+`aspect-ratio`; a run that starts from a `first-frame` takes that frame's shape instead. A+`last-frame` bridges from the first frame into one continuous shot, so it is not combined with+`multi-clip`, which renders the prompt as several cuts.++`generate-audio` renders an audio track alongside the picture, including speech the prompt asks a+character to say. The model exposes no separate voice, language or dialogue field, so a spoken line+belongs in the prompt itself.diff --git a/packages/pixverse/package.json b/packages/pixverse/package.jsonnew file mode 100644index 00000000..2598604e--- /dev/null+++ b/packages/pixverse/package.json@@ -0,0 +1,21 @@+{+ "name": "@hypit/pixverse",+ "version": "0.0.0-dev",+ "license": "SEE LICENSE IN LICENSE",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts"+ },+ "hypit": {+ "activation": "./src/activation.ts"+ },+ "dependencies": {+ "@hypit/artifact": "workspace:*",+ "@hypit/generation": "workspace:*",+ "@hypit/markup": "workspace:*",+ "@hypit/model-kit": "workspace:*",+ "@hypit/protocol": "workspace:*",+ "@hypit/text": "workspace:*"+ }+}diff --git a/packages/pixverse/src/activation.ts b/packages/pixverse/src/activation.tsnew file mode 100644index 00000000..62fa59be--- /dev/null+++ b/packages/pixverse/src/activation.ts@@ -0,0 +1,11 @@+import { createMarkupSurfaceHostFacet } from "@hypit/markup";+import {+ pixverseComponent, pixverseDefinition, pixverseManifest, pixverseModuleRef, pixverseMarkupSurfaces,+} from "./index.js";+import { decodePixverseVideoSurface } from "./surface.js";+export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: pixverseManifest }], components: [pixverseComponent], hostFacets: [+ pixverseDefinition.hostFacet,+ createMarkupSurfaceHostFacet({ module: pixverseModuleRef,+ declaration: pixverseMarkupSurfaces.find((item) => item.name === "video")!, handler: decodePixverseVideoSurface }),+] };+export default hypitPackage;diff --git a/packages/pixverse/src/index.ts b/packages/pixverse/src/index.tsnew file mode 100644index 00000000..821710dd--- /dev/null+++ b/packages/pixverse/src/index.ts@@ -0,0 +1,122 @@+import { artifactTypes } from "@hypit/artifact";+import { sealGenerationPortRequest, sealGenerationPortTable } from "@hypit/generation";+import type { GenerationPortTable, GenerationPortValue, GenerationRequest } from "@hypit/generation";+import type { SurfaceAttributeVocabulary } from "@hypit/markup";+import { defineExactModelModule } from "@hypit/model-kit";+import { textTypes } from "@hypit/text";++export const pixverseModuleRef = { name: "@hypit/pixverse", version: "1" } as const;+export const pixverseModels = ["pixverse-v6"] as const;+export type PixverseModel = typeof pixverseModels[number];++const PIXVERSE_QUALITIES = ["360p", "540p", "720p", "1080p"] as const;+const PIXVERSE_ASPECT_RATIOS = ["16:9", "4:3", "1:1", "3:4", "9:16", "2:3", "3:2", "21:9"] as const;++export const pixverseV6Ports: GenerationPortTable = sealGenerationPortTable({+ model: "pixverse-v6",+ result: "video",+ ports: [+ { name: "prompt", value: { kind: "text", maxChars: 5_000 }, minItems: 1, maxItems: 1 },+ { name: "firstFrame", value: { kind: "media", accepts: ["image"] }, minItems: 0, maxItems: 1 },+ { name: "lastFrame", value: { kind: "media", accepts: ["image"] }, minItems: 0, maxItems: 1 },+ { name: "duration", value: { kind: "number", integer: true, minimum: 1, maximum: 15 }, minItems: 1, maxItems: 1 },+ { name: "quality", value: { kind: "enum", values: [...PIXVERSE_QUALITIES] }, minItems: 1, maxItems: 1 },+ { name: "aspectRatio", value: { kind: "enum", values: [...PIXVERSE_ASPECT_RATIOS] }, minItems: 0, maxItems: 1 },+ { name: "generateAudio", value: { kind: "boolean" }, minItems: 0, maxItems: 1 },+ { name: "multiClip", value: { kind: "boolean" }, minItems: 0, maxItems: 1 },+ { name: "seed", value: { kind: "number", integer: true, minimum: 0, maximum: 2_147_483_647 }, minItems: 0, maxItems: 1 },+ ],+ requires: [+ // A last frame states where a run that already has a first frame ends.+ { kind: "requiresPresent", port: "lastFrame", needs: ["firstFrame"] },+ // A run that starts from a frame inherits that frame's shape, and one that+ // bridges two frames renders a single continuous shot.+ { kind: "atMostOneOf", ports: ["aspectRatio", "firstFrame"] },+ { kind: "atMostOneOf", ports: ["multiClip", "lastFrame"] },+ ],+});++export const pixversePorts: Readonly<Record<PixverseModel, GenerationPortTable>> = {+ "pixverse-v6": pixverseV6Ports,+};++export function sealPixverseRequest(+ ports: Readonly<Record<string, readonly GenerationPortValue[]>>,+): GenerationRequest {+ return sealGenerationPortRequest(pixverseV6Ports, ports);+}++const pixverseBaseDefinition = defineExactModelModule({+ module: pixverseModuleRef,+ endpoints: [{+ key: "video",+ requestTypeName: "PixverseV6Request",+ producerName: "request-pixverse-v6",+ ports: pixverseV6Ports,+ }],+});++export const pixverseEndpoints = pixverseBaseDefinition.endpoints;+export const pixverseComponent = pixverseBaseDefinition.component;+const endpoint = pixverseEndpoints.video!;++const pixverseAttributes: readonly SurfaceAttributeVocabulary[] = [+ { name: "id", kind: "identifier", required: true,+ summary: "Names this generation so its video Artifact can be referenced elsewhere in the Source." },+ { name: "prompt", kind: "reference", required: true, accepts: [textTypes.text],+ summary: "The Text edge describing the shot, including any spoken line the model should voice." },+ { name: "first-frame", kind: "reference", required: false, accepts: [artifactTypes.blob],+ summary: "Starts the video from one image Artifact." },+ { name: "last-frame", kind: "reference", required: false, accepts: [artifactTypes.blob],+ summary: "Ends the video on one image Artifact, bridging from the first frame." },+ { name: "duration", kind: "literal", required: true,+ summary: "How many seconds of video to render, from 1 to 15." },+ { name: "quality", kind: "literal", required: true, values: [...PIXVERSE_QUALITIES],+ summary: "The size band the model renders at." },+ { name: "aspect-ratio", kind: "literal", required: false, values: [...PIXVERSE_ASPECT_RATIOS],+ summary: "The Frame shape of the generated video." },+ { name: "generate-audio", kind: "literal", required: false, values: ["true", "false"],+ summary: "Renders an audio track alongside the picture." },+ { name: "multi-clip", kind: "literal", required: false, values: ["true", "false"],+ summary: "Renders the prompt as several cuts instead of one continuous shot." },+ { name: "seed", kind: "literal", required: false,+ summary: "Seeds the model's sampling so a rerun stays close to this one." },+];++export const pixverseMarkupSurfaces = [{+ name: "video",+ tag: "Video",+ mode: "structured" as const,+ outputs: [endpoint.draftType, endpoint.mediaBindings.firstFrame!.type, endpoint.mediaBindings.lastFrame!.type],+ vocabulary: {+ summary: "Generates one video with the exact PixVerse V6 model from a Text prompt, optionally starting from a frame or bridging two.",+ attributes: pixverseAttributes,+ ports: [{+ name: "video",+ type: artifactTypes.blob,+ summary: "The generated video, addressed as `<id>.video`.",+ }],+ example: `<pix:Video+ id="opening"+ prompt={line}+ duration="5"+ quality="720p"+ aspect-ratio="9:16"+ generate-audio="true"+/>`,+ notes: [+ "A run that starts from a frame takes its shape from that frame, so `aspect-ratio` states the shape only for a prompt-only run.",+ "A `last-frame` bridges from the `first-frame` into one continuous shot, so it is not combined with `multi-clip`.",+ "A spoken line belongs in the prompt; the model exposes no separate voice, language or dialogue field.",+ "The Surface lowers the element into the package's exact model request and selects no Provider.",+ ],+ },+}] as const;++export const pixverseManifest = {+ ...pixverseBaseDefinition.manifest,+};+export const pixverseDefinition = {+ ...pixverseBaseDefinition,+ manifest: pixverseManifest,+};diff --git a/packages/pixverse/src/surface.ts b/packages/pixverse/src/surface.tsnew file mode 100644index 00000000..4b5b388c--- /dev/null+++ b/packages/pixverse/src/surface.ts@@ -0,0 +1,146 @@+import { artifactTypes } from "@hypit/artifact";+import { generationPort, sealGenerationMediaBinding, sealGenerationRequestDraft } from "@hypit/generation";+import type { GenerationMediaPort, GenerationPortValue } from "@hypit/generation";+import {+ createExactModelPrimaryGenerationFragment,+ exactModelMediaInputNames,+ exactModelTextInputName,+} from "@hypit/model-kit";+import type { ExactModelEndpoint } from "@hypit/model-kit";+import type {+ MarkupAttributeValue,+ StructuredElement,+ StructuredSurfaceHandler,+ SurfaceResolvedReference,+} from "@hypit/markup";+import type { CanonicalValue, TypeRef } from "@hypit/protocol";+import { textTypes, verifyText } from "@hypit/text";++import { pixverseEndpoints } from "./index.js";++function assert(condition: unknown, message: string): asserts condition {+ if (!condition) throw new Error(message);+}++function sameType(left: TypeRef, right: TypeRef): boolean {+ return left.name === right.name && left.module.name === right.module.name && left.module.version === right.module.version;+}++const attributes = ["id", "prompt", "first-frame", "last-frame", "duration", "quality",+ "aspect-ratio", "generate-audio", "multi-clip", "seed"] as const;++function exact(element: StructuredElement): void {+ const unknown = Object.keys(element.attributes).filter((name) => !attributes.includes(name as typeof attributes[number]));+ assert(unknown.length === 0, `${element.name} does not accept ${unknown[0]}`);+ const missing = ["id", "prompt", "duration", "quality"].filter((name) => element.attributes[name] === undefined);+ assert(missing.length === 0, `${element.name} requires ${missing.join(", ")}`);+}++function text(element: StructuredElement, name: string): string {+ const value = element.attributes[name];+ assert(typeof value === "string" && value.trim().length > 0, `${element.name}.${name} must be text`);+ return value.trim();+}++/** An absent optional attribute leaves its port unstated, which the model reads as its own default. */+function optionalText(element: StructuredElement, name: string): readonly GenerationPortValue[] | undefined {+ return element.attributes[name] === undefined ? undefined : [text(element, name)];+}++function optionalFlag(element: StructuredElement, name: string): readonly GenerationPortValue[] | undefined {+ if (element.attributes[name] === undefined) return undefined;+ const value = text(element, name);+ assert(value === "true" || value === "false", `${element.name}.${name} must be true or false`);+ return [value === "true"];+}++function integer(element: StructuredElement, name: string): readonly GenerationPortValue[] {+ const value = text(element, name);+ assert(/^\d+$/u.test(value), `${element.name}.${name} must be a whole number`);+ return [Number(value)];+}++function ref(+ element: StructuredElement,+ name: string,+ type: TypeRef,+ resolve: (path: string) => SurfaceResolvedReference | undefined,+): SurfaceResolvedReference {+ const value: MarkupAttributeValue | undefined = element.attributes[name];+ assert(typeof value === "object" && value.kind === "reference", `${element.name}.${name} must be a reference`);+ const result = resolve(value.path);+ assert(result !== undefined && sameType(result.type, type), `${element.name}.${name} has the wrong type`);+ return result;+}++function frame(+ element: StructuredElement,+ name: string,+ resolve: (path: string) => SurfaceResolvedReference | undefined,+): SurfaceResolvedReference | undefined {+ if (element.attributes[name] === undefined) return undefined;+ const image = ref(element, name, artifactTypes.blob, resolve);+ if (image.record !== undefined) {+ assert(image.record.value.kind === "blob" && image.record.value.mediaType.startsWith("image/"),+ `${element.name}.${name} must reference image media`);+ }+ return image;+}++function decoder(endpoint: ExactModelEndpoint): StructuredSurfaceHandler {+ return ({ element, resolveReference }) => {+ exact(element);+ assert(!element.children.some((item) => item.kind === "element" || item.value.trim()),+ `${element.name} accepts no children`);+ const id = text(element, "id");+ const prompt = ref(element, "prompt", textTypes.text, resolveReference);+ if (prompt.record !== undefined) {+ assert(prompt.record.value.kind === "inline", `${element.name}.prompt must reference Text`);+ verifyText(prompt.record.value.value);+ }+ const frames = ([["firstFrame", "first-frame"], ["lastFrame", "last-frame"]] as const)+ .flatMap(([port, attribute]) => {+ const image = frame(element, attribute, resolveReference);+ return image === undefined ? [] : [{ port, image }];+ });+ const stated: Record<string, readonly GenerationPortValue[] | undefined> = {+ duration: integer(element, "duration"),+ quality: [text(element, "quality")],+ aspectRatio: optionalText(element, "aspect-ratio"),+ generateAudio: optionalFlag(element, "generate-audio"),+ multiClip: optionalFlag(element, "multi-clip"),+ seed: element.attributes["seed"] === undefined ? undefined : integer(element, "seed"),+ };+ const draft = sealGenerationRequestDraft(endpoint.ports, Object.fromEntries(+ Object.entries(stated).filter((entry): entry is [string, readonly GenerationPortValue[]] => entry[1] !== undefined),+ ));+ const records: Array<{ id: string; type: TypeRef; value: { kind: "inline"; value: CanonicalValue }; range: StructuredElement["range"] }> = [{+ id: `${id}.draft`, type: endpoint.draftType,+ value: { kind: "inline", value: draft as unknown as CanonicalValue }, range: element.range,+ }];+ const inputs: Record<string, SurfaceResolvedReference["ref"] | { kind: "record"; id: string }> = {+ draft: { kind: "record", id: `${id}.draft` }, [exactModelTextInputName("prompt")]: prompt.ref,+ };+ const media = frames.map(({ port, image }, index) => {+ const name = `media-${String(index + 1).padStart(4, "0")}`;+ const bindingId = `${id}.${name}.binding`;+ const mediaPort = generationPort(endpoint.ports, port);+ assert(mediaPort.value.kind === "media", `PixVerse port ${port} is not media`);+ records.push({+ id: bindingId, type: endpoint.mediaBindings[port]!.type,+ value: { kind: "inline", value: sealGenerationMediaBinding(mediaPort as GenerationMediaPort, { role: "image" }) as unknown as CanonicalValue },+ range: element.range,+ });+ const names = exactModelMediaInputNames(name);+ inputs[names.binding] = { kind: "record", id: bindingId };+ inputs[names.artifact] = image.ref;+ return { name, port } as const;+ });+ const fragment = createExactModelPrimaryGenerationFragment(endpoint, media, [{ name: "prompt", port: "prompt" }]);+ return { records, fragments: [fragment], components: [{+ id, fragment: fragment.id, inputs, outputs: { video: `${id}.video` }, range: element.range,+ }] };+ };+}++export const decodePixverseVideoSurface = decoder(pixverseEndpoints.video!);diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex e70d85e4..8a3b97a8 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -174,6 +174,9 @@ importers:'@hypit/performance-studio':specifier: workspace:*version: link:packages/performance-studio+ '@hypit/pixverse':+ specifier: workspace:*+ version: link:packages/pixverse'@hypit/program-space':specifier: workspace:*version: link:packages/program-space@@ -2208,6 +2211,27 @@ importers:specifier: workspace:*version: link:../timeline+ packages/pixverse:+ dependencies:+ '@hypit/artifact':+ specifier: workspace:*+ version: link:../artifact+ '@hypit/generation':+ specifier: workspace:*+ version: link:../generation+ '@hypit/markup':+ specifier: workspace:*+ version: link:../markup+ '@hypit/model-kit':+ specifier: workspace:*+ version: link:../model-kit+ '@hypit/protocol':+ specifier: workspace:*+ version: link:../protocol+ '@hypit/text':+ specifier: workspace:*+ version: link:../text+packages/program-space:dependencies:'@hypit/markup':