Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

22.8 KB
Raw Blame
import { preserveExecutionLog } from "../execution-log.js";
import { currentFileReference, localExternalFiles } from "./file-reference.js";
import type { ExternalFileAccess } from "./file-reference.js";
import { randomUUID } from "node:crypto";
import {
  mkdir,
  open,
  readFile,
  readdir,
  rename,
  rm,
  stat,
  writeFile,
} from "node:fs/promises";
import { createReadStream } from "node:fs";
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";

import { assertOrderedBuildId, buildIdCreatedAt, sameType } from "@hypit/protocol";
import type { BlobRef, TypeRef } from "@hypit/protocol";

import type {
  BuildResultResourceSource,
  BuildResultFileRef,
  BuildResultFileRange,
  BuildResultFinish,
  BuildResultForward,
  BuildResultManifest,
  BuildResultPresentationUpdate,
  BuildResultSeed,
  BuildResultSync,
  ResolvedBuildResultOutput,
  BuildResultRepository,
  RepositoryBuildResultOutput,
  RepositoryBuildResultOutputDescription,
  RepositoryBuildResultOutputLocation,
  FinishedBuildResultManifest,
} from "../types.js";
import { assertBuildResultSeed } from "../types.js";
import {
  decodeBuildResultJson,
  decodeBuildResultManifest,
  decodeBuildResultValueDocument,
  decodeBuildResultWriterState,
  encodeBuildResultManifest,
} from "../decode.js";
import { syncBuildResultOutputs } from "../writer.js";
import { replaceFile } from "@hypit/atomic-file";

const manifestName = "result.json";
const writerStateName = ".writer.json";

function assert(condition: unknown, message: string): asserts condition {
  if (!condition) throw new Error(message);
}

export function applyBuildResultPresentation(
  manifest: BuildResultManifest,
  update: BuildResultPresentationUpdate,
): BuildResultManifest {
  assert(manifest.outcome !== undefined, `Build Result ${manifest.id} is not finished`);
  const base = { ...manifest } as {
    title?: string;
    note?: string;
    highlightedOutputs?: readonly string[];
  } & BuildResultManifest;
  if (update.title !== undefined) {
    if (update.title === null) delete base.title;
    else {
      const title = update.title.trim();
      assert(title.length > 0, "Build Result title must not be empty");
      base.title = title;
    }
  }
  if (update.note !== undefined) {
    if (update.note === null) delete base.note;
    else {
      const note = update.note.trim();
      assert(note.length > 0, "Build Result note must not be empty");
      base.note = note;
    }
  }
  if (update.highlightedOutputs !== undefined) {
    const highlighted = [...new Set(update.highlightedOutputs)];
    for (const output of highlighted) {
      assert(manifest.outputs[output] !== undefined,
        `Build Result ${manifest.id} has no Output ${output}`);
    }
    if (highlighted.length === 0) delete base.highlightedOutputs;
    else base.highlightedOutputs = highlighted;
  }
  if (update.outputDisplayNames !== undefined) {
    const outputs = { ...manifest.outputs };
    for (const [name, value] of Object.entries(update.outputDisplayNames)) {
      const entry = manifest.outputs[name];
      assert(Object.hasOwn(manifest.outputs, name) && entry !== undefined,
        `Build Result ${manifest.id} has no Output ${name}`);
      const { displayName: _priorName, ...rest } = entry;
      if (value === null) outputs[name] = rest;
      else {
        const displayName = value.trim();
        assert(displayName.length > 0, "Output display name must not be empty");
        outputs[name] = { ...rest, displayName };
      }
    }
    return { ...base, outputs };
  }
  return base;
}

async function exists(path: string): Promise<boolean> {
  return await stat(path).then((item) => item.isFile(), () => false);
}

async function writeJsonAtomic(path: string, value: unknown): Promise<void> {
  await mkdir(dirname(path), { recursive: true });
  const temporary = `${path}.part-${randomUUID()}`;
  try {
    await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx" });
    await replaceFile(temporary, path);
  } catch (error) {
    await rm(temporary, { force: true });
    throw error;
  }
}

async function writeManifestAtomic(path: string, manifest: BuildResultManifest): Promise<void> {
  await writeJsonAtomic(path, encodeBuildResultManifest(manifest));
}

async function copyArtifactAtomic(
  source: BuildResultResourceSource,
  artifact: BlobRef,
  destination: string,
): Promise<void> {
  await mkdir(dirname(destination), { recursive: true });
  const input = await source.open(artifact);
  if (input === undefined) throw new Error(`Build resource ${artifact.resource} is unavailable`);
  await writeStreamAtomic(destination, input);
}

export async function locateRepositoryBuildResultOutput(
  repository: Pick<BuildResultRepository, "read">,
  build: string,
  output: string,
): Promise<RepositoryBuildResultOutputLocation | undefined> {
  const seen = new Set<string>();
  let currentBuild = build;
  let currentOutput = output;
  while (true) {
    const address = `${currentBuild}\u0000${currentOutput}`;
    assert(!seen.has(address), `Build Output forwarding repeats ${currentBuild} / ${currentOutput}`);
    seen.add(address);
    const manifest = await repository.read(currentBuild);
    if (manifest === undefined) return undefined;
    assert(manifest.outcome !== undefined, `Build ${currentBuild} is not a finished Result`);
    const entry = manifest.outputs[currentOutput];
    if (entry === undefined) return undefined;
    if (entry.value.kind === "build-output") {
      currentBuild = entry.value.build;
      currentOutput = entry.value.output;
      continue;
    }
    return { build: currentBuild, output: currentOutput, type: entry.type };
  }
}

export async function locateBuildResultOutput(
  root: string,
  build: string,
  output: string,
): Promise<RepositoryBuildResultOutputLocation | undefined> {
  return await locateRepositoryBuildResultOutput({
    read: async (currentBuild) => await readBuildResult(buildResultDirectory(root, currentBuild)),
  }, build, output);
}

async function writeStreamAtomic(destination: string, input: AsyncIterable<Uint8Array>): Promise<void> {
  await mkdir(dirname(destination), { recursive: true });
  const temporary = `${destination}.part-${randomUUID()}`;
  const output = await open(temporary, "wx");
  try {
    for await (const value of input) {
      const chunk = Uint8Array.from(value);
      let offset = 0;
      while (offset < chunk.byteLength) {
        const { bytesWritten } = await output.write(chunk, offset, chunk.byteLength - offset);
        offset += bytesWritten;
      }
    }
    await output.close();
    await replaceFile(temporary, destination);
  } catch (error) {
    await output.close().catch(() => undefined);
    await rm(temporary, { force: true });
    throw error;
  }
}

function buildResultDateBucket(build: string): string | undefined {
  const createdAt = buildIdCreatedAt(build);
  return createdAt === undefined ? undefined : new Date(createdAt).toISOString().slice(0, 10);
}

export function buildResultDirectory(root: string, build: string): string {
  assertOrderedBuildId(build);
  const directory = resolve(root);
  const bucket = buildResultDateBucket(build)!;
  const target = resolve(directory, bucket, build);
  const relation = relative(directory, target);
  assert(relation.length > 0 && relation !== ".." && !relation.startsWith(`..${sep}`),
    `Build ${build} leaves Build Result root ${directory}`);
  return target;
}

export async function readBuildResult(directory: string): Promise<BuildResultManifest | undefined> {
  const path = join(resolve(directory), manifestName);
  const bytes = await readFile(path).catch((error: unknown) => {
    if (error instanceof Error && "code" in error && error.code === "ENOENT") return undefined;
    throw error;
  });
  if (bytes === undefined) return undefined;
  const build = basename(resolve(directory));
  return decodeBuildResultManifest(decodeBuildResultJson(bytes, path), build, path);
}

function containedResultPath(directory: string, path: string): string {
  assert(!isAbsolute(path), `Build Result path ${path} must be relative`);
  const root = resolve(directory);
  const absolute = resolve(root, path);
  const relation = relative(root, absolute);
  assert(relation !== ".." && !relation.startsWith(`..${sep}`),
    `Build Result path ${path} leaves ${root}`);
  return absolute;
}

export async function browseBuildResults(
  root: string,
  request: { readonly before?: string; readonly limit: number },
): Promise<{ readonly results: readonly FinishedBuildResultManifest[]; readonly next?: string }> {
  assert(Number.isSafeInteger(request.limit) && request.limit > 0, "Build Result browse limit must be positive");
  if (request.before !== undefined) assertOrderedBuildId(request.before);
  const directory = resolve(root);
  const entries = await readdir(directory, { withFileTypes: true }).catch((error: unknown) => {
    if (error instanceof Error && "code" in error && error.code === "ENOENT") return [];
    throw error;
  });
  const beforeBucket = request.before === undefined
    ? undefined
    : buildResultDateBucket(request.before)!;
  const buckets = entries
    .filter((entry) => entry.isDirectory() && /^\d{4}-\d{2}-\d{2}$/u.test(entry.name))
    .map((entry) => entry.name)
    .filter((bucket) => beforeBucket === undefined || bucket <= beforeBucket)
    .sort((left, right) => right.localeCompare(left));
  const found: FinishedBuildResultManifest[] = [];
  for (const bucket of buckets) {
    const bucketDirectory = join(directory, bucket);
    const builds = await readdir(bucketDirectory, { withFileTypes: true }).catch((error: unknown) => {
      if (error instanceof Error && "code" in error && error.code === "ENOENT") return [];
      throw error;
    });
    const ordered = builds
      .filter((entry) => entry.isDirectory())
      .map((entry) => entry.name)
      .filter((build) => {
        return buildResultDateBucket(build) === bucket
          && (request.before === undefined || build < request.before);
      })
      .sort((left, right) => right.localeCompare(left));
    for (const build of ordered) {
      const manifest = await readBuildResult(join(bucketDirectory, build));
      if (manifest?.outcome === undefined || manifest.finishedAt === undefined) continue;
      found.push(manifest as FinishedBuildResultManifest);
      if (found.length > request.limit) break;
    }
    if (found.length > request.limit) break;
  }
  const results = found.slice(0, request.limit);
  return {
    results,
    ...(found.length > request.limit && results.length > 0 ? { next: results[results.length - 1]!.id } : {}),
  };
}

export async function resolveBuildResultOutput(
  root: string,
  build: string,
  output: string,
  externalFiles: ExternalFileAccess = localExternalFiles,
): Promise<ResolvedBuildResultOutput | undefined> {
  const seen = new Set<string>();
  let currentBuild = build;
  let currentOutput = output;
  while (true) {
    const address = `${currentBuild}\u0000${currentOutput}`;
    assert(!seen.has(address), `Build Output forwarding repeats ${currentBuild} / ${currentOutput}`);
    seen.add(address);
    const directory = buildResultDirectory(root, currentBuild);
    const manifest = await readBuildResult(directory);
    if (manifest === undefined) return undefined;
    const entry = manifest.outputs[currentOutput];
    if (entry === undefined) return undefined;
    if (entry.value.kind === "build-output") {
      currentBuild = entry.value.build;
      currentOutput = entry.value.output;
      continue;
    }
    if (entry.value.kind === "value") {
      const path = containedResultPath(directory, entry.value.path);
      const document = decodeBuildResultValueDocument(
        decodeBuildResultJson(await readFile(path), path),
        `Build ${currentBuild} Output ${currentOutput}`,
      );
      return {
        build: currentBuild,
        output: currentOutput,
        directory,
        type: entry.type,
        value: { ...entry.value, document },
      };
    }
    const terminalKind = (entry.value as { readonly kind?: unknown }).kind;
    assert(terminalKind === "build-file" || terminalKind === "external-file" || terminalKind === "inline",
      `Build ${currentBuild} Output ${currentOutput} has unsupported Result value kind ${String(terminalKind)}`);
    return {
      build: currentBuild,
      output: currentOutput,
      directory,
      type: entry.type,
      value: entry.value.kind === "external-file" ? await currentFileReference(entry.value, externalFiles) : entry.value,
    };
  }
}

export async function describeBuildResultOutput(
  root: string,
  build: string,
  output: string,
  externalFiles: ExternalFileAccess = localExternalFiles,
): Promise<RepositoryBuildResultOutputDescription | undefined> {
  const seen = new Set<string>();
  let currentBuild = build;
  let currentOutput = output;
  while (true) {
    const address = `${currentBuild}\u0000${currentOutput}`;
    assert(!seen.has(address), `Build Output forwarding repeats ${currentBuild} / ${currentOutput}`);
    seen.add(address);
    const manifest = await readBuildResult(buildResultDirectory(root, currentBuild));
    const entry = manifest?.outputs[currentOutput];
    if (entry === undefined) return undefined;
    if (entry.value.kind === "build-output") {
      currentBuild = entry.value.build;
      currentOutput = entry.value.output;
      continue;
    }
    const file = entry.value.kind === "external-file" ? await currentFileReference(entry.value, externalFiles) : entry.value;
    return file.kind === "build-file" || file.kind === "external-file"
      ? { type: entry.type, kind: "resource", size: file.size, mediaType: file.mediaType }
      : entry.value.kind === "value"
        ? { type: entry.type, kind: "composite" }
        : { type: entry.type, kind: "scalar" };
  }
}

/**
 * Accept only finished historical Results and reduce every new Forward to the Result that owns its
 * terminal value. This follows only the addresses named by the caller; it needs no reverse index or
 * global Result scan.
 */
export async function normalizeBuildResultForwards(
  repository: {
    read(build: string): Promise<BuildResultManifest | undefined>;
  },
  forwards: readonly BuildResultForward[],
): Promise<readonly (BuildResultForward & { readonly type: TypeRef })[]> {
  return await Promise.all(forwards.map(async (forward) => {
    const source = await repository.read(forward.build);
    assert(source?.outcome !== undefined,
      `Build ${forward.build} is not a finished Result`);
    const resolved = await locateRepositoryBuildResultOutput(repository, forward.build, forward.sourceOutput);
    assert(resolved !== undefined,
      `Build ${forward.build} has no Output ${forward.sourceOutput}`);
    const owner = resolved.build === source.id ? source : await repository.read(resolved.build);
    assert(owner?.outcome !== undefined,
      `Build ${resolved.build} is not a finished Result`);
    assert(forward.type === undefined || sameType(forward.type, resolved.type),
      `Build ${forward.build} Output ${forward.sourceOutput} has the wrong type for its forwarded Logical Output`);
    return {
      output: forward.output,
      build: resolved.build,
      sourceOutput: resolved.output,
      type: resolved.type,
    };
  }));
}

export class FileBuildResult {
  readonly directory: string;

  private constructor(directory: string) {
    this.directory = resolve(directory);
  }

  static async create(root: string, seed: BuildResultSeed, externalFiles: ExternalFileAccess = localExternalFiles): Promise<FileBuildResult> {
    assertOrderedBuildId(seed.id);
    assertBuildResultSeed(seed);
    const forwards = await normalizeBuildResultForwards({
      read: async (build) => await readBuildResult(buildResultDirectory(root, build)),
    }, seed.forwards ?? []);
    const directory = buildResultDirectory(root, seed.id);
    await mkdir(dirname(directory), { recursive: true });
    const temporary = join(resolve(root), `.preparing-${seed.id}-${randomUUID()}`);
    await mkdir(temporary);
    const manifest: BuildResultManifest = {
      format: "hypit.build-result@1",
      id: seed.id,
      ...(seed.title === undefined ? {} : { title: seed.title }),
      source: seed.source,
      ...(seed.run === undefined ? {} : { run: seed.run }),
      targets: [...seed.targets],
      outputs: {},
    };
    try {
      await writeManifestAtomic(join(temporary, manifestName), manifest);
      await writeJsonAtomic(join(temporary, writerStateName), {
        resources: {},
        values: {},
        publishedOutputs: seed.publishedOutputs,
        ...(seed.resourceReferences === undefined ? {} : { resourceReferences: seed.resourceReferences }),
        forwards,
      });
      await rename(temporary, directory);
    } catch (error) {
      await rm(temporary, { recursive: true, force: true });
      throw error;
    }
    return new FileBuildResult(directory);
  }

  static async open(directory: string): Promise<FileBuildResult> {
    const absolute = resolve(directory);
    const manifest = await readBuildResult(absolute);
    if (manifest === undefined) throw new Error(`${absolute} has no Build Result`);
    return new FileBuildResult(absolute);
  }

  async read(): Promise<BuildResultManifest> {
    const manifest = await readBuildResult(this.directory);
    assert(manifest !== undefined, `${this.directory} has no Build Result`);
    return manifest;
  }

  async sync(input: BuildResultSync): Promise<BuildResultManifest> {
    const manifest = await this.read();
    if (manifest.outcome !== undefined) return manifest;
    const writerPath = join(this.directory, writerStateName);
    const writer = decodeBuildResultWriterState(
      decodeBuildResultJson(await readFile(writerPath), writerPath),
      writerPath,
    );
    const directory = this.directory;
    const updated = await syncBuildResultOutputs({
      manifest,
      writer,
      sync: input,
      target: {
        async writeResource(path, artifact, source) {
          await copyArtifactAtomic(source, artifact, join(directory, path));
        },
        async writeValue(path, document) {
          await writeJsonAtomic(join(directory, path), document);
        },
      },
    });
    if (!updated.changed) return manifest;
    await writeJsonAtomic(join(this.directory, writerStateName), updated.writer);
    await writeManifestAtomic(join(this.directory, manifestName), updated.manifest);
    return updated.manifest;
  }

  async finish(input: BuildResultFinish): Promise<BuildResultManifest> {
    const manifest = await this.read();
    if (manifest.outcome !== undefined) {
      assert(manifest.outcome === input.outcome && manifest.failure === input.failure,
        `Build Result ${manifest.id} is already finished with a different outcome`);
      await rm(join(this.directory, writerStateName), { force: true });
      return manifest;
    }
    const executionLog = await preserveExecutionLog(input.executionLog,
      async (path, chunks) => await writeStreamAtomic(join(this.directory, path), chunks));
    const now = Date.now();
    const updated: BuildResultManifest = {
      ...manifest,
      outcome: input.outcome,
      ...(executionLog === undefined ? {} : { executionLog }),
      ...(input.operations === undefined ? {} : { operations: input.operations }),
      finishedAt: manifest.finishedAt ?? now,
      ...(input.failure === undefined ? {} : { failure: input.failure }),
    };
    await writeManifestAtomic(join(this.directory, manifestName), updated);
    await rm(join(this.directory, writerStateName), { force: true });
    return updated;
  }
}

/** Default zero-configuration project repository backed by one ordinary directory tree. */
export class FileBuildResultRepository implements BuildResultRepository {
  readonly externalFiles: ExternalFileAccess;
  readonly root: string;

  constructor(root: string, externalFiles: ExternalFileAccess = localExternalFiles) {
    this.externalFiles = externalFiles;
    assert(root.trim().length > 0, "Build Result root must not be empty");
    this.root = resolve(root);
  }

  async create(seed: BuildResultSeed): Promise<FileBuildResult> {
    return await FileBuildResult.create(this.root, seed, this.externalFiles);
  }

  async openWriter(build: string): Promise<FileBuildResult | undefined> {
    const directory = buildResultDirectory(this.root, build);
    return await readBuildResult(directory) === undefined ? undefined : await FileBuildResult.open(directory);
  }

  async removeIncomplete(build: string): Promise<void> {
    const manifest = await this.read(build);
    if (manifest === undefined) return;
    assert(manifest.outcome === undefined, `Finished Build Result ${build} cannot be removed`);
    await rm(buildResultDirectory(this.root, build), { recursive: true, force: true });
  }

  async read(build: string): Promise<BuildResultManifest | undefined> {
    return await readBuildResult(buildResultDirectory(this.root, build));
  }

  async updatePresentation(
    build: string,
    update: BuildResultPresentationUpdate,
  ): Promise<BuildResultManifest> {
    const manifest = await this.read(build);
    assert(manifest !== undefined, `Build Result ${build} does not exist`);
    const updated = applyBuildResultPresentation(manifest, update);
    await writeManifestAtomic(join(buildResultDirectory(this.root, build), manifestName), updated);
    return updated;
  }

  async browse(request: { readonly before?: string; readonly limit: number }) {
    return await browseBuildResults(this.root, request);
  }

  async describeOutput(build: string, output: string): Promise<RepositoryBuildResultOutputDescription | undefined> {
    return await describeBuildResultOutput(this.root, build, output, this.externalFiles);
  }

  async resolve(build: string, output: string): Promise<RepositoryBuildResultOutput | undefined> {
    const resolvedOutput = await resolveBuildResultOutput(this.root, build, output, this.externalFiles);
    if (resolvedOutput === undefined) return undefined;
    const { directory: _directory, ...portable } = resolvedOutput;
    return portable;
  }

  async describeFile(_build: string, file: BuildResultFileRef): Promise<BuildResultFileRef> {
    return await currentFileReference(file, this.externalFiles);
  }

  async openFile(
    build: string,
    file: BuildResultFileRef,
    range?: BuildResultFileRange,
  ): Promise<AsyncIterable<Uint8Array> | undefined> {
    if (file.kind === "external-file") return await this.externalFiles.open(file.uri, range);
    const directory = buildResultDirectory(this.root, file.build ?? build);
    const path = containedResultPath(directory, file.path);
    if (!await exists(path)) return undefined;
    if (range === undefined) return createReadStream(path);
    assert(Number.isSafeInteger(range.start) && range.start >= 0, "Build Result file range start is invalid");
    assert(Number.isSafeInteger(range.endExclusive) && range.endExclusive > range.start,
      "Build Result file range end is invalid");
    assert(range.endExclusive <= file.size, "Build Result file range exceeds the declared file size");
    return createReadStream(path, { start: range.start, end: range.endExclusive - 1 });
  }
}