Skip to content

Write HTML. Render video. Built for agents.

13.5 KB
Raw Blame
import { randomUUID } from "node:crypto";
import { MODELS_DIR, ensureModel } from "../whisper/manager.js";
import { EventEmitter } from "node:events";
import fs, { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
import { syncBuiltinESMExports } from "node:module";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { PassThrough } from "node:stream";
import { get as httpsGet } from "node:https";
import type { ClientRequest, IncomingMessage } from "node:http";
import { afterEach, describe, expect, it, vi } from "vitest";
import { downloadFile, REDIRECT_CODES, redirectTarget } from "./download.js";

vi.mock("node:https", () => ({ get: vi.fn() }));

const mockGet = vi.mocked(httpsGet);
const tempDirs: string[] = [];

function httpsResponse(
  statusCode: number,
  headers: Record<string, string> = {},
  body?: string | ((url: string) => string),
  onResponse?: (response: IncomingMessage) => void,
): typeof httpsGet {
  return ((_url: string, callback: (response: IncomingMessage) => void) => {
    const response = new PassThrough() as PassThrough & {
      statusCode: number;
      headers: Record<string, string>;
    };
    response.statusCode = statusCode;
    response.headers = headers;
    const request = new EventEmitter() as ClientRequest;
    request.setTimeout = vi.fn();
    const message = response as unknown as IncomingMessage;
    onResponse?.(message);
    callback(message);
    if (body !== undefined) {
      queueMicrotask(() => response.end(typeof body === "function" ? body(_url) : body));
    }
    return request;
  }) as typeof httpsGet;
}

afterEach(() => {
  vi.clearAllMocks();
  vi.restoreAllMocks();
  syncBuiltinESMExports();
  for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});

describe("downloadFile", () => {
  it("hands https.get the caller's signal, so aborting a model download removes the partial file", async () => {
    let seen: AbortSignal | undefined;
    mockGet.mockImplementation(((
      _url: string,
      options: { signal?: AbortSignal },
      callback: (response: IncomingMessage) => void,
    ) => {
      seen = options.signal;
      const response = Object.assign(new PassThrough(), { statusCode: 200, headers: {} });
      const request = new EventEmitter() as ClientRequest;
      request.setTimeout = vi.fn();
      callback(response as unknown as IncomingMessage);
      response.write("partial");
      // What Node does with a request whose signal aborts.
      options.signal?.addEventListener("abort", () =>
        request.emit(
          "error",
          Object.assign(new Error("The operation was aborted"), { name: "AbortError" }),
        ),
      );
      return request;
    }) as unknown as typeof httpsGet);
    const controller = new AbortController();
    const model = `test-${randomUUID()}`;

    const download = ensureModel(model, { signal: controller.signal });
    await new Promise((resolve) => setImmediate(resolve));
    controller.abort();

    await expect(download).rejects.toThrow("The operation was aborted");
    expect(seen).toBe(controller.signal);
    expect(readdirSync(MODELS_DIR).filter((name) => name.includes(model))).toEqual([]);
  });

  it("carries model download progress through the real pipeline and stays quiet on cache hits", async () => {
    mockGet.mockImplementation(httpsResponse(200, { "content-length": "5" }, "hello"));
    const events: unknown[] = [];
    const model = `test-${randomUUID()}`;
    const options = {
      onDownloadProgress: (receivedBytes: number, totalBytes: number | null) =>
        events.push({ receivedBytes, totalBytes }),
    };
    const modelPath = await ensureModel(model, options);
    try {
      expect(readFileSync(modelPath, "utf8")).toBe("hello");
      expect(events).toEqual([
        { receivedBytes: 0, totalBytes: 5 },
        { receivedBytes: 5, totalBytes: 5 },
      ]);
      events.length = 0;
      expect(await ensureModel(model, options)).toBe(modelPath);
      expect(events).toEqual([]);
      expect(mockGet).toHaveBeenCalledTimes(1);
    } finally {
      rmSync(modelPath);
    }
  });

  it.each([
    [{ "content-length": "5" }, 5],
    [{}, null],
    [{ "content-length": "invalid" }, null],
  ])("reports actual response bytes with total %j", async (headers, totalBytes) => {
    mockGet.mockImplementation(httpsResponse(200, headers as Record<string, string>, "hello"));
    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
    tempDirs.push(dir);
    const events: unknown[] = [];
    await downloadFile("https://example.test/model", join(dir, "model"), {
      onProgress: (receivedBytes, total) => events.push({ receivedBytes, totalBytes: total }),
    });
    expect(events).toContainEqual({ receivedBytes: 5, totalBytes });
    expect(readFileSync(join(dir, "model"), "utf8")).toBe("hello");
  });

  it("keeps concurrent partial downloads separate", async () => {
    mockGet.mockImplementation(httpsResponse(200, {}, (url) => url));

    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
    tempDirs.push(dir);
    const dest = join(dir, "model.onnx");
    const first = "https://example.test/first";
    const second = "https://example.test/second";

    await Promise.all([downloadFile(first, dest), downloadFile(second, dest)]);
    expect([first, second]).toContain(readFileSync(dest, "utf-8"));
  });

  it("rejects a response that exceeds its byte limit", async () => {
    mockGet.mockImplementation(httpsResponse(200, {}, "too large"));

    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
    tempDirs.push(dir);
    const dest = join(dir, "model.onnx");

    await expect(
      downloadFile("https://example.test/model.onnx", dest, { maxBytes: 3 }),
    ).rejects.toThrow("Download exceeded 3 bytes");
    expect(existsSync(dest)).toBe(false);
    expect(readdirSync(dir)).toEqual([]);
  });

  it("rejects an idle response and removes the partial file", async () => {
    let responseClosed = false;
    const realUnlinkSync = fs.unlinkSync;
    vi.spyOn(fs, "unlinkSync").mockImplementation((path) => {
      if (!responseClosed) {
        const error = new Error("file is locked");
        Object.assign(error, { code: "EBUSY" });
        throw error;
      }
      realUnlinkSync(path);
    });
    // Node's named exports of a builtin follow its default export only once synced.
    syncBuiltinESMExports();

    mockGet.mockImplementation(((_url: string, callback: (response: IncomingMessage) => void) => {
      const response = new PassThrough() as PassThrough & {
        statusCode: number;
        headers: Record<string, string>;
      };
      response.statusCode = 200;
      response.headers = {};
      response.once("close", () => {
        responseClosed = true;
      });

      class FakeRequest extends EventEmitter {
        private timeout: ReturnType<typeof setTimeout> | undefined;
        private fallback = setTimeout(() => this.destroy(new Error("late shutdown")), 50);
        private destroyed = false;

        setTimeout(ms: number, onTimeout: () => void): this {
          this.timeout = setTimeout(onTimeout, ms);
          return this;
        }

        destroy(error: Error): void {
          if (this.destroyed) return;
          this.destroyed = true;
          clearTimeout(this.timeout);
          clearTimeout(this.fallback);
          response.destroy(error);
          this.emit("error", error);
        }
      }

      const request = new FakeRequest();
      callback(response as unknown as IncomingMessage);
      return request as unknown as ClientRequest;
    }) as typeof httpsGet);

    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
    tempDirs.push(dir);
    const dest = join(dir, "model.onnx");

    await expect(
      downloadFile("https://example.test/model.onnx", dest, { timeoutMs: 10 }),
    ).rejects.toThrow("Download timed out after 10ms");
    expect(existsSync(`${dest}.tmp`)).toBe(false);
    expect(existsSync(dest)).toBe(false);
  });
});

describe("redirect handling", () => {
  it("closes each redirect response before opening the next request", async () => {
    const responses: IncomingMessage[] = [];
    const capture = (response: IncomingMessage) => responses.push(response);
    mockGet
      .mockImplementationOnce(httpsResponse(302, { location: "/second" }, undefined, capture))
      .mockImplementationOnce(
        httpsResponse(307, { location: "/model" }, undefined, (response) => {
          expect(responses[0]?.destroyed).toBe(true);
          capture(response);
        }),
      )
      .mockImplementationOnce(
        httpsResponse(200, {}, "hello", () => {
          expect(responses[1]?.destroyed).toBe(true);
        }),
      );
    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-redirect-"));
    tempDirs.push(dir);
    await downloadFile("https://example.test/first", join(dir, "model"));

    expect(responses).toHaveLength(2);
    expect(responses.every((response) => response.destroyed)).toBe(true);
    expect(mockGet).toHaveBeenCalledTimes(3);
    expect(readFileSync(join(dir, "model"), "utf8")).toBe("hello");
  });

  it("does not let a discarded redirect request fail the active download", async () => {
    mockGet
      .mockImplementationOnce(httpsResponse(302, { location: "/model" }))
      .mockImplementationOnce(httpsResponse(200, {}, "hello"));
    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-redirect-"));
    tempDirs.push(dir);
    const pending = downloadFile("https://example.test/first", join(dir, "model"));
    const firstRequest = mockGet.mock.results[0]?.value;
    firstRequest?.emit("error", new Error("old redirect socket failed"));

    await expect(pending).resolves.toBeUndefined();
    expect(readFileSync(join(dir, "model"), "utf8")).toBe("hello");
    expect(readdirSync(dir)).toEqual(["model"]);
  });

  it("closes redirect loop responses when reaching the hop limit", async () => {
    const responses: IncomingMessage[] = [];
    mockGet.mockImplementation(
      httpsResponse(302, { location: "/loop" }, undefined, (response) => responses.push(response)),
    );
    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-redirect-"));
    tempDirs.push(dir);

    await expect(downloadFile("https://example.test/loop", join(dir, "model"))).rejects.toThrow(
      "more than 10 redirects",
    );
    expect(responses).toHaveLength(11);
    expect(responses.every((response) => response.destroyed)).toBe(true);
    expect(readdirSync(dir)).toEqual([]);
  });

  it("keeps a signed query out of the redirect-limit error", async () => {
    mockGet.mockImplementation(httpsResponse(302, { location: "/loop" }));
    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-redirect-"));
    tempDirs.push(dir);

    const error = await downloadFile(
      "https://example.test/handler.zip?X-Amz-Signature=secret",
      join(dir, "handler.zip"),
    ).then(
      () => {
        throw new Error("expected the download to fail");
      },
      (caught: unknown) => caught as Error,
    );
    expect(error.message).toContain("https://example.test/handler.zip");
    expect(error.message).not.toContain("secret");
  });

  it.each([302, 404, 503])("closes rejected HTTP %s responses", async (status) => {
    const responses: IncomingMessage[] = [];
    mockGet.mockImplementation(
      httpsResponse(status, {}, undefined, (response) => responses.push(response)),
    );
    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-http-error-"));
    tempDirs.push(dir);

    await expect(downloadFile("https://example.test/model", join(dir, "model"))).rejects.toThrow(
      `HTTP ${status}`,
    );
    expect(responses[0]?.destroyed).toBe(true);
    expect(readdirSync(dir)).toEqual([]);
  });

  it("rejects a redirect that the HTTPS client cannot follow", async () => {
    mockGet
      .mockImplementationOnce(
        httpsResponse(307, { location: "http://cdn.example.test/model.onnx" }),
      )
      .mockImplementationOnce(() => {
        throw new TypeError('Protocol "http:" not supported');
      });

    const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
    tempDirs.push(dir);
    await expect(
      downloadFile("https://example.test/model.onnx", join(dir, "model.onnx")),
    ).rejects.toThrow('Protocol "http:" not supported');
  });

  it("follows every redirect a host may answer with", () => {
    // 307 is the one that broke: HuggingFace answers the tokenizer with it,
    // and the original set stopped at 302, so the download fell through to the
    // not-200 branch and hung.
    for (const code of [301, 302, 303, 307, 308]) {
      expect(REDIRECT_CODES.has(code)).toBe(true);
    }
  });

  it("does not treat a success or an error as a redirect", () => {
    for (const code of [200, 204, 404, 500]) {
      expect(REDIRECT_CODES.has(code)).toBe(false);
    }
  });

  it("resolves a relative location against the url that sent it", () => {
    // The actual failure: handing this path back as a request target is not a
    // valid URL, so nothing was ever fetched.
    expect(
      redirectTarget(
        "/api/resolve-cache/models/x/tokenizer.json",
        "https://huggingface.co/x/resolve/main/tokenizer.json",
      ),
    ).toBe("https://huggingface.co/api/resolve-cache/models/x/tokenizer.json");
  });

  it("keeps an absolute location as it is", () => {
    expect(redirectTarget("https://cdn.example.com/blob", "https://huggingface.co/a/b")).toBe(
      "https://cdn.example.com/blob",
    );
  });

  it("resolves a sibling-relative location", () => {
    expect(redirectTarget("other.json", "https://example.com/a/b/file.json")).toBe(
      "https://example.com/a/b/other.json",
    );
  });
});