Skip to content

Write HTML. Render video. Built for agents.

10.3 KB
Raw Blame
// Equivalence tests for the regression harness's PSNR comparison.
//
// `psnrPerFrame()` replaced a loop that spawned one ffmpeg per checkpoint with
// `select='eq(n,N)'`. That filter has no index, so every spawn re-decoded from
// frame 0 and the comparison phase grew quadratically with checkpoint count.
// These tests pin the replacement to the behaviour of the method it replaced:
// the same frame must yield the same PSNR, within the two-decimal precision
// that ffmpeg's `stats_file` reports.
//
// Lives in the integration lane (see scripts/test-classification.mjs) because
// it shells out to a real ffmpeg.

import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import { spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { frameIndexForCheckpoint, psnrAtCheckpoint, psnrAtFrames } from "./regression-harness.js";

const FPS = 30;
const TOTAL_FRAMES = 60;
const SAMPLE_FRAMES = [0, 1, 15, 30, 45, 59];

let workDir: string;
let referenceVideo: string;
let degradedVideo: string;
/** Same pixels as `degradedVideo`, but a timeline framesync cannot align. */
let rebasedVideo: string;
/** Half the frames of `referenceVideo`, for one-sided end-of-stream cases. */
let halfLengthVideo: string;

/**
 * The pre-existing per-checkpoint implementation, kept here as the oracle.
 * Any divergence between this and `psnrPerFrame()` is a behaviour change in
 * the regression suite's pass/fail decisions.
 */
function legacyPsnrAtFrame(videoA: string, videoB: string, frameIndex: number): number {
  const filter =
    `[0:v]select='eq(n\\,${frameIndex})',setpts=PTS-STARTPTS[rv];` +
    `[1:v]select='eq(n\\,${frameIndex})',setpts=PTS-STARTPTS[gv];[rv][gv]psnr`;
  const result = spawnSync(
    "ffmpeg",
    [
      "-hide_banner",
      "-loglevel",
      "info",
      "-i",
      videoA,
      "-i",
      videoB,
      "-filter_complex",
      filter,
      "-frames:v",
      "1",
      "-f",
      "null",
      "-",
    ],
    { encoding: "utf-8" },
  );
  const match = result.stderr.match(/average:\s*(\S+)/i);
  if (!match) throw new Error(`legacy PSNR parse failed at frame ${frameIndex}`);
  const raw = (match[1] ?? "").trim().toLowerCase();
  return raw === "inf" || raw === "infinite" ? Number.POSITIVE_INFINITY : Number(raw);
}

function ffmpeg(args: string[]): void {
  const result = spawnSync("ffmpeg", args, { encoding: "utf-8" });
  if (result.status !== 0) throw new Error(`ffmpeg fixture setup failed: ${result.stderr}`);
}

beforeAll(() => {
  workDir = mkdtempSync(join(tmpdir(), "hf-psnr-test-"));
  referenceVideo = join(workDir, "reference.mp4");
  degradedVideo = join(workDir, "degraded.mp4");

  // testsrc2 is deterministic, so these fixtures are reproducible across hosts.
  ffmpeg([
    "-hide_banner",
    "-loglevel",
    "error",
    "-f",
    "lavfi",
    "-i",
    `testsrc2=size=320x240:rate=${FPS}:duration=${TOTAL_FRAMES / FPS}`,
    "-c:v",
    "libx264",
    "-crf",
    "18",
    "-pix_fmt",
    "yuv420p",
    "-y",
    referenceVideo,
  ]);
  // Re-encode hard enough to land well inside the PSNR range fixtures use.
  ffmpeg([
    "-hide_banner",
    "-loglevel",
    "error",
    "-i",
    referenceVideo,
    "-c:v",
    "libx264",
    "-crf",
    "40",
    "-pix_fmt",
    "yuv420p",
    "-y",
    degradedVideo,
  ]);

  // Identical frames to degradedVideo, re-timed to a different rate. Rendered
  // output does not carry its baseline's timestamps, and comparing the two
  // streams directly lets framesync pair frames by PTS instead of by index —
  // the bug that made style-3-prod fail 10 checkpoints. Frame N here must
  // still compare against frame N of the reference.
  rebasedVideo = join(workDir, "rebased.mp4");
  ffmpeg([
    "-hide_banner",
    "-loglevel",
    "error",
    "-i",
    degradedVideo,
    "-vf",
    "setpts=N/25/TB",
    "-r",
    "25",
    "-c:v",
    "libx264",
    "-crf",
    "40",
    "-pix_fmt",
    "yuv420p",
    "-y",
    rebasedVideo,
  ]);

  // Exactly half the frames, same rate. Used for one-sided EOF: requesting an
  // index this video does not have must fail rather than silently pair against
  // its repeated last frame.
  halfLengthVideo = join(workDir, "half.mp4");
  ffmpeg([
    "-hide_banner",
    "-loglevel",
    "error",
    "-f",
    "lavfi",
    "-i",
    `testsrc2=size=320x240:rate=${FPS}:duration=${TOTAL_FRAMES / 2 / FPS}`,
    "-c:v",
    "libx264",
    "-crf",
    "18",
    "-pix_fmt",
    "yuv420p",
    "-y",
    halfLengthVideo,
  ]);
});

afterAll(() => {
  rmSync(workDir, { recursive: true, force: true });
});

describe("psnrAtFrames()", () => {
  it("returns exactly the frames asked for", () => {
    const byFrame = psnrAtFrames(referenceVideo, degradedVideo, SAMPLE_FRAMES);
    expect([...byFrame.keys()].sort((a, b) => a - b)).toEqual(SAMPLE_FRAMES);
  });

  it("writes its stats file under a temp dir whose path holds filtergraph syntax", () => {
    const savedTmp = process.env.TMPDIR;
    try {
      process.env.TMPDIR = join(workDir, "tmp:with\\backslash, [o'brien];");
      mkdirSync(process.env.TMPDIR, { recursive: true });
      expect(psnrAtFrames(referenceVideo, degradedVideo, [0, 59]).size).toBe(2);
    } finally {
      if (savedTmp === undefined) delete process.env.TMPDIR;
      else process.env.TMPDIR = savedTmp;
    }
  });

  it("matches the per-checkpoint ffmpeg method it replaced", () => {
    const byFrame = psnrAtFrames(referenceVideo, degradedVideo, SAMPLE_FRAMES);
    // Sample across the whole range: the old method's cost grew with frame
    // index, so late frames are exactly where a regression would hide.
    for (const frameIndex of SAMPLE_FRAMES) {
      const actual = byFrame.get(frameIndex);
      const expected = legacyPsnrAtFrame(referenceVideo, degradedVideo, frameIndex);
      expect(actual).toBeDefined();
      // stats_file reports two decimals; the legacy stderr parse was full float.
      expect(Math.abs((actual as number) - expected)).toBeLessThanOrEqual(0.01);
    }
  });

  it("pairs by frame index even when the two videos carry different timelines", () => {
    // The regression this guards: a rendered output whose PTS cadence differs
    // from the baseline's must still be compared frame-for-frame. Letting
    // framesync align by timestamp silently compares unrelated frames.
    const byFrame = psnrAtFrames(referenceVideo, rebasedVideo, SAMPLE_FRAMES);
    expect([...byFrame.keys()].sort((a, b) => a - b)).toEqual(SAMPLE_FRAMES);
    for (const frameIndex of SAMPLE_FRAMES) {
      const expected = legacyPsnrAtFrame(referenceVideo, rebasedVideo, frameIndex);
      expect(Math.abs((byFrame.get(frameIndex) as number) - expected)).toBeLessThanOrEqual(0.01);
    }
  });

  it("deduplicates repeated frame indices without losing alignment", () => {
    // Fixtures shorter than 100 frames map several checkpoints onto one frame.
    // `select` emits that frame once, so the row-to-frame mapping has to
    // account for it or every later value shifts.
    const withRepeats = [0, 0, 15, 15, 15, 59];
    const byFrame = psnrAtFrames(referenceVideo, degradedVideo, withRepeats);
    expect([...byFrame.keys()].sort((a, b) => a - b)).toEqual([0, 15, 59]);
    for (const frameIndex of [0, 15, 59]) {
      const expected = legacyPsnrAtFrame(referenceVideo, degradedVideo, frameIndex);
      expect(Math.abs((byFrame.get(frameIndex) as number) - expected)).toBeLessThanOrEqual(0.01);
    }
  });

  it("is insensitive to the order frame indices are supplied in", () => {
    const ascending = psnrAtFrames(referenceVideo, degradedVideo, [1, 30, 59]);
    const shuffled = psnrAtFrames(referenceVideo, degradedVideo, [59, 1, 30]);
    expect([...shuffled.entries()].sort()).toEqual([...ascending.entries()].sort());
  });

  it("reports identical video as infinite PSNR, as the legacy method did", () => {
    const byFrame = psnrAtFrames(referenceVideo, referenceVideo, SAMPLE_FRAMES);
    expect([...byFrame.values()].every((value) => value === Number.POSITIVE_INFINITY)).toBe(true);
    expect(legacyPsnrAtFrame(referenceVideo, referenceVideo, 10)).toBe(Number.POSITIVE_INFINITY);
  });

  it("throws rather than mis-pairing when a frame is past the end of both videos", () => {
    expect(() => psnrAtFrames(referenceVideo, degradedVideo, [0, TOTAL_FRAMES + 50])).toThrow(
      /differ in frame count|Expected PSNR/,
    );
  });

  it("throws when only one input runs out of frames", () => {
    // The asymmetric case, and the one the test above cannot reach: there both
    // inputs end together, so framesync ends too. With only one side short,
    // framesync's repeatlast default holds that side's last selected frame and
    // pads the pairing to full length — ffmpeg then writes one row per
    // requested frame and the count guard sees nothing wrong, while the tail
    // rows silently compare against a stale frame.
    expect(() => psnrAtFrames(referenceVideo, halfLengthVideo, [0, 15, 45])).toThrow(
      /differ in frame count|Expected PSNR/,
    );
  });

  it("still compares every requested frame when one input is merely shorter than the last index", () => {
    // Guard against over-correcting: truncation must fail, but a shorter input
    // that still covers every requested index has to keep working.
    const byFrame = psnrAtFrames(referenceVideo, halfLengthVideo, [0, 10, 29]);
    expect([...byFrame.keys()].sort((a, b) => a - b)).toEqual([0, 10, 29]);
  });

  it("returns an empty map when asked for nothing", () => {
    expect(psnrAtFrames(referenceVideo, degradedVideo, []).size).toBe(0);
  });
});

describe("psnrAtCheckpoint()", () => {
  it("maps a checkpoint time to the same frame index the legacy code used", () => {
    const times = [0, 0.5, 1, 1.9];
    const byFrame = psnrAtFrames(
      referenceVideo,
      degradedVideo,
      times.map((time) => frameIndexForCheckpoint(time, FPS)),
    );
    for (const time of times) {
      // Math.round(time * fps) is the mapping the per-checkpoint code used.
      expect(frameIndexForCheckpoint(time, FPS)).toBe(Math.round(time * FPS));
      expect(psnrAtCheckpoint(byFrame, time, FPS)).toBe(
        byFrame.get(Math.round(time * FPS)) as number,
      );
    }
  });

  it("throws when the requested frame was not among those compared", () => {
    const byFrame = psnrAtFrames(referenceVideo, degradedVideo, [0, 15]);
    expect(() => psnrAtCheckpoint(byFrame, 1.5, FPS)).toThrow(/Unable to parse PSNR output/);
  });
});