Skip to content

Write HTML. Render video. Built for agents.

13.4 KB
Raw Blame
import {
  chmodSync,
  lstatSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  readdirSync,
  rmSync,
  statSync,
  symlinkSync,
  writeFileSync,
} from "node:fs";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { downloadToFile } from "./download.js";

function makeBytesFetch(bytes: Uint8Array, headers: Record<string, string> = {}): typeof fetch {
  return (async () =>
    new Response(new Blob([bytes as unknown as BlobPart]), {
      status: 200,
      headers: {
        "content-type": "application/octet-stream",
        ...headers,
      },
    })) as unknown as typeof fetch;
}

function makeErrorFetch(status: number, statusText = "Not Found"): typeof fetch {
  return (async () => new Response("nope", { status, statusText })) as unknown as typeof fetch;
}

interface RangeServer {
  url: string;
  ranges: (string | undefined)[];
  close: () => Promise<void>;
}

async function serveBytes(
  body: Buffer,
  // `drop` cuts the first `times` responses whose range covers byte `at`, right before it.
  options: { honourRange: boolean; drop?: { at: number; times: number } },
): Promise<RangeServer> {
  const ranges: (string | undefined)[] = [];
  const server: Server = createServer((req, res) => {
    ranges.push(req.headers.range);
    const match = /^bytes=(\d+)-(\d+)$/.exec(req.headers.range ?? "");
    if (!options.honourRange || !match) {
      res.writeHead(200, { "content-length": body.length }).end(body);
      return;
    }
    const start = Number(match[1]);
    if (start >= body.length) {
      res.writeHead(416, { "content-range": `bytes */${body.length}` }).end();
      return;
    }
    const end = Math.min(Number(match[2]), body.length - 1);
    res.writeHead(206, {
      "content-length": end - start + 1,
      "content-range": `bytes ${start}-${end}/${body.length}`,
    });
    const drop = options.drop;
    if (drop && drop.times > 0 && start <= drop.at && drop.at <= end) {
      drop.times--;
      res.write(body.subarray(start, drop.at), () => res.destroy());
      return;
    }
    res.end(body.subarray(start, end + 1));
  });
  // Windows completes a large loopback write at once, so the 5 s keep-alive idle clock
  // started before a slow reader finished and closed the socket under it.
  server.keepAliveTimeout = 0;
  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
  const { port } = server.address() as AddressInfo;
  return {
    url: `http://127.0.0.1:${port}/video.mp4`,
    ranges,
    close: () => new Promise((resolve) => server.close(() => resolve())),
  };
}

function patternBytes(length: number): Buffer {
  const bytes = Buffer.alloc(length);
  for (let i = 0; i < length; i++) bytes[i] = (i * 31 + (i >> 13)) & 0xff;
  return bytes;
}

describe("cloud/download", () => {
  let dir: string;
  beforeEach(() => {
    dir = mkdtempSync(join(tmpdir(), "hf-cloud-download-"));
  });
  afterEach(() => {
    rmSync(dir, { recursive: true, force: true });
  });

  it("streams the response body to disk and returns the byte count", async () => {
    const payload = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
    const dest = join(dir, "out.bin");
    const result = await downloadToFile("https://example/x", dest, {
      fetchImpl: makeBytesFetch(payload, { "content-length": String(payload.length) }),
    });
    expect(result.path).toBe(dest);
    expect(result.bytes).toBe(payload.length);
    const written = readFileSync(dest);
    expect(written.equals(Buffer.from(payload))).toBe(true);
    expect(statSync(dest).size).toBe(payload.length);
  });

  it("creates the destination's parent directory if missing", async () => {
    const payload = new Uint8Array([42]);
    const dest = join(dir, "nested", "subdir", "file.mp4");
    const result = await downloadToFile("https://example/x", dest, {
      fetchImpl: makeBytesFetch(payload),
    });
    expect(result.bytes).toBe(1);
    expect(readFileSync(dest).at(0)).toBe(42);
  });

  it("reports progress with bytes downloaded and total when content-length is set", async () => {
    const payload = new Uint8Array(64);
    const dest = join(dir, "progress.bin");
    const calls: { bytes: number; total: number | undefined }[] = [];
    await downloadToFile("https://example/x", dest, {
      fetchImpl: makeBytesFetch(payload, { "content-length": "64" }),
      onProgress: (bytes, total) => calls.push({ bytes, total }),
    });
    expect(calls.length).toBeGreaterThanOrEqual(1);
    const last = calls.at(-1)!;
    expect(last.bytes).toBe(64);
    expect(last.total).toBe(64);
  });

  it("throws on non-2xx responses", async () => {
    const dest = join(dir, "missing.bin");
    await expect(
      downloadToFile("https://example/x", dest, {
        fetchImpl: makeErrorFetch(404),
      }),
    ).rejects.toThrow(/HTTP 404/);
  });

  it("rejects truncated downloads when bytes received < content-length", async () => {
    // Returns 10 bytes but declares content-length: 20.
    const dest = join(dir, "truncated.bin");
    const lyingFetch: typeof fetch = (async () =>
      new Response(new Blob([new Uint8Array(10) as unknown as BlobPart]), {
        status: 200,
        headers: { "content-length": "20" },
      })) as unknown as typeof fetch;
    await expect(
      downloadToFile("https://example/x", dest, { fetchImpl: lyingFetch }),
    ).rejects.toThrow(/Truncated download/);
    // Partial file must be cleaned up.
    expect(() => statSync(dest)).toThrow();
  });

  it("deletes the partial file when the abort signal fires mid-stream", async () => {
    const dest = join(dir, "aborted.bin");
    // 1 MB payload should give the abort time to land mid-stream.
    const payload = new Uint8Array(1024 * 1024);
    const controller = new AbortController();
    const fetchImpl: typeof fetch = (async () =>
      new Response(new Blob([payload as unknown as BlobPart]), {
        status: 200,
        headers: { "content-length": String(payload.length) },
      })) as unknown as typeof fetch;
    // Abort almost immediately.
    queueMicrotask(() => controller.abort(new Error("user cancelled")));
    await expect(
      downloadToFile("https://example/x", dest, {
        fetchImpl,
        signal: controller.signal,
      }),
    ).rejects.toThrow();
    expect(() => statSync(dest)).toThrow();
  });

  async function expectFailedReplacement(dest: string, fetchImpl: typeof fetch, error: RegExp) {
    await expect(downloadToFile("https://example/x", dest, { fetchImpl })).rejects.toThrow(error);
    expect(readFileSync(dest, "utf8")).toBe("previous render");
    expect(readdirSync(dir)).toEqual(["out.mp4"]);
  }

  it("preserves an existing output when the response is truncated", async () => {
    const dest = join(dir, "out.mp4");
    writeFileSync(dest, "previous render");
    await expectFailedReplacement(
      dest,
      makeBytesFetch(new Uint8Array([1]), { "content-length": "2" }),
      /Truncated download/,
    );
  });

  it("keeps the old output visible until successful replacement and preserves its mode", async () => {
    const dest = join(dir, "out.mp4");
    writeFileSync(dest, "previous render");
    chmodSync(dest, 0o640);
    await downloadToFile("https://example/x", dest, {
      fetchImpl: makeBytesFetch(new Uint8Array([42])),
      onProgress: () => {
        expect(readFileSync(dest, "utf8")).toBe("previous render");
        expect(readdirSync(dir).filter((name) => name.startsWith(".hf-download-"))).toHaveLength(1);
      },
    });
    expect(readFileSync(dest)).toEqual(Buffer.from([42]));
    if (process.platform !== "win32") expect(statSync(dest).mode & 0o777).toBe(0o640);
    expect(readdirSync(dir)).toEqual(["out.mp4"]);
  });

  it("preserves the previous output when cancelled during progress", async () => {
    const dest = join(dir, "out.mp4");
    writeFileSync(dest, "previous render");
    const controller = new AbortController();
    await expect(
      downloadToFile("https://example/x", dest, {
        fetchImpl: makeBytesFetch(new Uint8Array(1024 * 1024)),
        signal: controller.signal,
        onProgress: () => controller.abort(new Error("user cancelled")),
      }),
    ).rejects.toThrow(/user cancelled/);
    expect(readFileSync(dest, "utf8")).toBe("previous render");
    expect(readdirSync(dir)).toEqual(["out.mp4"]);
  });

  it("preserves the previous output when the response stream errors", async () => {
    const dest = join(dir, "out.mp4");
    writeFileSync(dest, "previous render");
    let sent = false;
    const body = new ReadableStream<Uint8Array>({
      pull(controller) {
        if (sent) controller.error(new Error("network interrupted"));
        else {
          sent = true;
          controller.enqueue(new Uint8Array([42]));
        }
      },
    });
    await expectFailedReplacement(
      dest,
      (async () => new Response(body)) as typeof fetch,
      /network interrupted/,
    );
  });

  it.skipIf(process.platform === "win32")(
    "preserves existing and dangling output symlinks",
    async () => {
      const target = join(dir, "target.mp4");
      const alias = join(dir, "alias.mp4");
      symlinkSync("target.mp4", alias);
      for (const value of [1, 2]) {
        await downloadToFile("https://example/x", alias, {
          fetchImpl: makeBytesFetch(new Uint8Array([value])),
        });
        expect(lstatSync(alias).isSymbolicLink()).toBe(true);
        expect(readFileSync(target)).toEqual(Buffer.from([value]));
      }
      expect(readdirSync(dir).sort()).toEqual(["alias.mp4", "target.mp4"]);
    },
  );

  it.skipIf(process.platform === "win32")(
    "follows a relative output link inside a linked folder to the real target",
    async () => {
      mkdirSync(join(dir, "real/deep/sub"), { recursive: true });
      mkdirSync(join(dir, "root"));
      symlinkSync(join(dir, "real/deep/sub"), join(dir, "root/sub"));
      symlinkSync("../target.mp4", join(dir, "real/deep/sub/alias.mp4"));

      await downloadToFile("https://example/x", join(dir, "root/sub/alias.mp4"), {
        fetchImpl: makeBytesFetch(new Uint8Array([7])),
      });

      expect(readFileSync(join(dir, "real/deep/target.mp4"))).toEqual(Buffer.from([7]));
      expect(readdirSync(join(dir, "root")).sort()).toEqual(["sub"]);
    },
  );

  describe("over HTTP", () => {
    const servers: RangeServer[] = [];
    afterEach(async () => {
      await Promise.all(servers.splice(0).map((server) => server.close()));
    });
    // Small parts keep the split the same while moving little data: a Windows runner's loopback
    // has carried a single connection at ~1 MB/s, past the timeout for megabyte bodies.
    const PART = 64 * 1024;
    const options = { partBytes: PART };
    async function serve(...args: Parameters<typeof serveBytes>) {
      const server = await serveBytes(...args);
      servers.push(server);
      return server;
    }

    it("splits a large file into parallel ranged requests", async () => {
      const body = patternBytes(7.5 * PART + 123);
      const server = await serve(body, { honourRange: true });
      const dest = join(dir, "out.mp4");
      const result = await downloadToFile(server.url, dest, options);
      expect(result.bytes).toBe(body.length);
      expect(readFileSync(dest).equals(body)).toBe(true);
      expect(server.ranges).toHaveLength(8);
      expect(server.ranges[0]).toBe(`bytes=0-${PART - 1}`);
      expect(readdirSync(dir)).toEqual(["out.mp4"]);
    });

    it("fetches small and empty files in one request", async () => {
      for (const size of [1000, 0]) {
        const body = patternBytes(size);
        const server = await serve(body, { honourRange: true });
        const dest = join(dir, `out-${size}.mp4`);
        await downloadToFile(server.url, dest, options);
        expect(readFileSync(dest).equals(body)).toBe(true);
        expect(server.ranges[0]).toBe(`bytes=0-${PART - 1}`);
        expect(server.ranges.slice(1)).toEqual(size === 0 ? [undefined] : []);
      }
    });

    it("streams the whole file in one request when the server ignores Range", async () => {
      const body = patternBytes(2.5 * PART);
      const server = await serve(body, { honourRange: false });
      const dest = join(dir, "out.mp4");
      await downloadToFile(server.url, dest, options);
      expect(readFileSync(dest).equals(body)).toBe(true);
      expect(server.ranges).toEqual([`bytes=0-${PART - 1}`]);
    });

    it("resumes a range whose connection drops mid-body from the bytes it already has", async () => {
      const body = patternBytes(6.25 * PART);
      const at = PART + 1024;
      const server = await serve(body, { honourRange: true, drop: { at, times: 2 } });
      const dest = join(dir, "out.mp4");
      const result = await downloadToFile(server.url, dest, options);
      expect(result.bytes).toBe(body.length);
      expect(readFileSync(dest).equals(body)).toBe(true);
      expect(server.ranges.filter((range) => range?.startsWith(`bytes=${at}-`))).toHaveLength(2);
    });

    it("keeps the previous output and leaves no staged file when one part keeps dropping", async () => {
      const body = patternBytes(6.25 * PART);
      const server = await serve(body, {
        honourRange: true,
        drop: { at: PART + 1024, times: Infinity },
      });
      const dest = join(dir, "out.mp4");
      writeFileSync(dest, "previous render");
      await expect(downloadToFile(server.url, dest, options)).rejects.toThrow(/dropped 4 times/);
      expect(readFileSync(dest, "utf8")).toBe("previous render");
      expect(readdirSync(dir)).toEqual(["out.mp4"]);
    });
  });
});