Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

43.6 KB
Raw Blame
package gateway

import (
	"bytes"
	"cmp"
	"context"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"mime"
	"net/http"
	"net/url"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"time"

	"github.com/yetone/magpie/internal/catalog"
	"github.com/yetone/magpie/internal/provider"
	"github.com/yetone/magpie/internal/settings"
	"github.com/yetone/magpie/internal/usage"
)

// The gateway makes videos: POST /v1/videos, GET /v1/videos/{id} and
// GET /v1/videos/{id}/content take OpenAI's Videos API and answer in its
// shape, so its SDKs work unchanged. A video takes seconds to minutes, so a
// request starts one and answers at once with its id; the id is what asks
// for it again. It says who is making the video, video_<provider>.<the
// vendor's id>.<when it started>, so the gateway keeps nothing.
//
// A Grok subscription uses the Imagine API of the backend Grok Build talks
// to: <cli-chat-proxy.grok.com/v1>/videos/generations starts one and answers
// with its request_id. Volcengine Agent Plan uses Ark's asynchronous content
// generation tasks under its /api/plan/v3 base.
//
// Another magpie's videos (a Remote magpie's, #545) are asked of it, at its
// own videos API: its id for one goes, encoded, in the id given here, and
// what it answers comes back with this magpie's id and name for the model.
// A provider set up with a base URL whose own model list marks a model
// "kind": "video" (#1446) is asked the same way, at the OpenAI-shaped
// videos API under its base, on the key that started the video.

// grokVideoModels are the video models a Grok subscription makes videos with.
var grokVideoModels = []catalog.Model{
	{ID: "grok-imagine-video", Name: "Grok Imagine Video"},
	{ID: "grok-imagine-video-1.5", Name: "Grok Imagine Video 1.5"},
}

// grokVideoAspects are the aspect ratios Grok's video API takes.
var grokVideoAspects = []string{"1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"}

// maxVideoBytes is the most of a video the gateway passes on.
const maxVideoBytes = 512 << 20

// videoTimeout is how long a video's vendor is given to answer each of its
// calls: to take the video on, say how it goes, and send it. A video is
// made between them, so Settings' Image generation timeout isn't it.
const videoTimeout = 5 * time.Minute

// Videomakers are the models a provider can make videos with: a Grok
// subscription's, another magpie's, a preset plan's models when magpie
// implements that vendor's video API, a MiniMax API key's H3 models
// (video_minimax.go), and those a provider's own model list marks as video
// models (customVideomakers).
func Videomakers(p provider.Provider) []catalog.Model {
	if p.IsRemoteMagpie() {
		out := catalog.LiveVideomakers(p.ID)
		for i := range out {
			out[i].Provider = p.ID
		}
		return out
	}
	if out := p.PlanVideos(); len(out) > 0 {
		return out
	}
	if filmsMiniMax(p) {
		return minimaxVideomakers(p)
	}
	if !drawsGrok(p) {
		return customVideomakers(p)
	}
	out := slices.Clone(grokVideoModels)
	for i := range out {
		out[i].Provider = p.ID
	}
	return out
}

// customVideomakers are the video models of a provider set up with a base
// URL — not a subscription's account, not another magpie, not a preset plan
// whose own video API magpie speaks — that its fetched model list marks
// "kind": "video" (catalog.LiveVideomakers, #1446), or the user said make
// videos (provider.SetModelKind, #1399). They are asked at the
// OpenAI-shaped videos API under its base, as another magpie's are. A
// model's name alone makes none of them one: a list that doesn't say so,
// and a user who doesn't, leave the provider making no videos. A plugin's
// are the models it says give video (capabilities.output.video), asked
// through its fetch on the account (XYenon's Google AI Studio plugin, Veo).
func customVideomakers(p provider.Provider) []catalog.Model {
	if p.IsPlugin() && !drawsGrok(p) {
		var out []catalog.Model
		for _, m := range p.Available() {
			if slices.Contains(m.OutputModalities, "video") {
				m.Provider = p.ID
				out = append(out, m)
			}
		}
		return out
	}
	if p.Account != nil || p.IsRemoteMagpie() || len(p.PlanVideos()) > 0 || filmsMiniMax(p) || videoBase(p) == "" {
		return nil
	}
	out := catalog.LiveVideomakers(p.ID)
	for i := range out {
		out[i].Provider = p.ID
	}
	// those the user said make videos, whatever the list says, and none
	// they said draws (shen-joe, #1399: a relay whose /models marks none)
	for _, id := range p.KindModels(provider.KindVideo) {
		if !slices.ContainsFunc(out, func(m catalog.Model) bool { return m.ID == id }) {
			out = append(out, kindModel(p, id))
		}
	}
	return slices.DeleteFunc(out, func(m catalog.Model) bool { return p.ModelKind(m.ID) == provider.KindImage })
}

// videoBase is the root of the OpenAI-shaped videos API of another magpie
// or a provider set up with a base URL: its Chat base, or a Responses-only
// relay's Responses base, under which it serves the images API too; a
// plugin's fetch whatever API its chat is on (Provider.OpenAIBase).
func videoBase(p provider.Provider) string {
	return strings.TrimRight(cmp.Or(p.OpenAIBase(), p.Base(provider.Responses)), "/")
}

// filmsCustom is whether p makes model at its OpenAI-shaped videos API, as
// its model list says (customVideomakers).
func filmsCustom(p provider.Provider, model string) bool {
	return slices.ContainsFunc(customVideomakers(p), func(m catalog.Model) bool { return m.ID == model })
}

// keyMark is the mark of the key p sends with, in the id of a video it
// started at its OpenAI-shaped videos API: the vendor keeps a video for the
// key that started it, so it is asked after on that key, whichever comes
// first now. "0" for a provider with no key (a local server); a plugin's
// account's signer, the vendor keeping it for that account.
func keyMark(p provider.Provider) string {
	if p.IsPlugin() {
		return "a" + signer(p)
	}
	if p.Key == "" {
		return "0"
	}
	return "k" + provider.KeyID(p.Key)
}

// customVideoID is the id of a video p started at its OpenAI-shaped videos
// API, by the vendor's id for it, which goes in base64 (it may hold dots).
func customVideoID(p provider.Provider, theirs string, t time.Time) string {
	return "video_" + p.ID + "." + base64.RawURLEncoding.EncodeToString([]byte(theirs)) + "." + strconv.FormatInt(t.Unix(), 10) + "." + keyMark(p)
}

// markedKey is p on its key whose mark is by (keyMark), while that key is
// still on.
func markedKey(p provider.Provider, by string) (provider.Provider, bool) {
	if p.IsPlugin() {
		// the plugin's account that started it, while it is still on
		for _, q := range append([]provider.Provider{p}, p.AlsoOn()...) {
			if keyMark(q) == by {
				return q, true
			}
		}
		return p, false
	}
	if p.Key == "" {
		return p, by == "0"
	}
	for _, k := range p.KeysOn() {
		if q := p.WithKey(k); keyMark(q) == by && (videoBase(q) != "" || filmsMiniMax(q)) {
			return q, true
		}
	}
	return p, false
}

// videomaker is the model a request that names none makes its video with:
// AutoVideomaker's, unless the Settings turned image generation off, which
// turns video off with it. None when it is off or no provider makes videos.
func videomaker() (string, bool) {
	if settings.Load().ImageGen == "off" {
		return "", false
	}
	m := AutoVideomaker()
	return m, m != ""
}

// AutoVideomaker is the model a request that names none makes its video
// with: the first model of the first provider that makes any. "" when none
// can. A MiniMax API key's H3 comes after every other: it is paid by the
// second from the key's balance, and a key set up for chat would
// otherwise take videos a subscription made before.
func AutoVideomaker() string {
	paid := ""
	for _, p := range provider.All() {
		if !p.On() || p.DecideOnly() {
			continue
		}
		if ms := Videomakers(p); len(ms) > 0 {
			if filmsMiniMax(p) {
				if paid == "" {
					paid = p.ID + "/" + ms[0].ID
				}
				continue
			}
			return p.ID + "/" + ms[0].ID
		}
	}
	return paid
}

// bareVideomaker is the provider that makes videos with model named without
// its provider: grok-imagine-video, as the vendor names it, not only
// grok/grok-imagine-video.
func bareVideomaker(name string) (provider.Provider, string, bool) {
	if strings.Contains(name, "/") {
		return provider.Provider{}, "", false
	}
	for _, p := range provider.All() {
		if !p.On() || p.DecideOnly() {
			continue
		}
		for _, m := range Videomakers(p) {
			if m.ID == name {
				return p, name, true
			}
		}
	}
	// any grok-imagine-video*, as videosCreate takes with the provider
	// named: the vendor's newer ones work before magpie lists them (#1857)
	for _, p := range provider.All() {
		if p.On() && !p.DecideOnly() && drawsGrok(p) && strings.HasPrefix(name, grokVideoFamily) {
			return p, name, true
		}
	}
	return provider.Provider{}, "", false
}

// grokVideoFamily is the family of a Grok subscription's video models.
const grokVideoFamily = "grok-imagine-video"

// filming is one videos request, whichever shape it came in.
type filming struct {
	Model, Prompt, Seconds, Size string
	Start                        *picture  // the first frame to animate
	End                          *picture  // the last frame, where the video ends (MiniMax)
	References                   []picture // images the video draws its subjects from
	// the request's other fields, which magpie doesn't read: a provider's
	// own videos API takes them as they came (Agnes's mode, aspect_ratio;
	// #1399, shen-joe: "mode 不能为空")
	Extra map[string]any
}

// filmingRead are the fields of a videos request magpie reads itself.
var filmingRead = []string{"model", "prompt", "seconds", "size", "input_reference", "last_frame", "reference_images", "reference_images[]"}

// readFilming reads a videos request: JSON, or the multipart form OpenAI's
// takes, the image to start from its input_reference.
func readFilming(r *http.Request) (filming, error) {
	var f filming
	ct, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if ct == "multipart/form-data" {
		if err := r.ParseMultipartForm(64 << 20); err != nil {
			return f, err
		}
		m := r.MultipartForm
		get := func(k string) string {
			if v := m.Value[k]; len(v) > 0 {
				return strings.TrimSpace(v[0])
			}
			return ""
		}
		f.Model, f.Prompt, f.Seconds, f.Size = get("model"), get("prompt"), get("seconds"), get("size")
		for k := range m.Value {
			if !slices.Contains(filmingRead, k) {
				if f.Extra == nil {
					f.Extra = map[string]any{}
				}
				f.Extra[k] = get(k)
			}
		}
		if fhs := m.File["input_reference"]; len(fhs) > 0 {
			pic, err := readPart(fhs[0])
			if err != nil {
				return f, err
			}
			f.Start = &pic
		}
		if fhs := m.File["last_frame"]; len(fhs) > 0 {
			pic, err := readPart(fhs[0])
			if err != nil {
				return f, err
			}
			f.End = &pic
		}
		for _, k := range []string{"reference_images", "reference_images[]"} {
			for _, fh := range m.File[k] {
				pic, err := readPart(fh)
				if err != nil {
					return f, err
				}
				f.References = append(f.References, pic)
			}
		}
	} else {
		body, err := io.ReadAll(r.Body)
		if err != nil {
			return f, err
		}
		var in struct {
			Model           string          `json:"model"`
			Prompt          string          `json:"prompt"`
			Seconds         json.RawMessage `json:"seconds"`
			Size            string          `json:"size"`
			InputReference  json.RawMessage `json:"input_reference"`
			LastFrame       json.RawMessage `json:"last_frame"`
			ReferenceImages json.RawMessage `json:"reference_images"`
		}
		if err := json.Unmarshal(body, &in); err != nil {
			return f, fmt.Errorf("the request isn't JSON: %v", err)
		}
		f = filming{Model: in.Model, Prompt: in.Prompt, Size: in.Size, Seconds: strings.Trim(strings.TrimSpace(string(in.Seconds)), `"`)}
		var all map[string]any
		json.Unmarshal(body, &all)
		for k, v := range all {
			if !slices.Contains(filmingRead, k) {
				if f.Extra == nil {
					f.Extra = map[string]any{}
				}
				f.Extra[k] = v
			}
		}
		if f.Seconds == "null" {
			f.Seconds = ""
		}
		if srcs := imageSources(in.InputReference); len(srcs) > 0 {
			pic, err := pictureOf(srcs[0])
			if err != nil {
				return f, err
			}
			f.Start = &pic
		}
		if srcs := imageSources(in.LastFrame); len(srcs) > 0 {
			pic, err := pictureOf(srcs[0])
			if err != nil {
				return f, err
			}
			f.End = &pic
		}
		for _, src := range imageSources(in.ReferenceImages) {
			pic, err := pictureOf(src)
			if err != nil {
				return f, err
			}
			f.References = append(f.References, pic)
		}
	}
	f.Model = strings.TrimSpace(f.Model)
	if strings.TrimSpace(f.Prompt) == "" {
		return f, errors.New("say what to film in prompt")
	}
	if f.Seconds != "" {
		n, err := parseSeconds(f.Seconds)
		if err != nil {
			return f, err
		}
		f.Seconds = strconv.Itoa(n)
	}
	return f, nil
}

// decimalNumber is a number as JSON writes it, not as Go's ParseFloat also
// takes it (hex, inf, nan, underscores).
var decimalNumber = regexp.MustCompile(`^[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?$`)

// parseSeconds is a duration in whole seconds, however it was written: 6,
// "6", 6.0, 6e0. What the vendor's limit makes of a length it says itself.
func parseSeconds(v string) (int, error) {
	whole := func() (int, bool) {
		if !decimalNumber.MatchString(v) {
			return 0, false
		}
		n, err := strconv.ParseFloat(v, 64)
		if err != nil || n != math.Trunc(n) || n < 1 || n > math.MaxInt32 {
			return 0, false
		}
		return int(n), true
	}
	if n, ok := whole(); ok {
		return n, nil
	}
	return 0, fmt.Errorf("seconds is a whole number of at least 1, not %q", v)
}

// videoResolution is the resolution Grok's video API is asked for when size
// says WIDTHxHEIGHT: by the shorter side, and no more than the model makes
// (grok-imagine-video stops at 720p, which the vendor turns 1080p away for;
// 1.5 makes 1080p). "" when size names no pixels.
func videoResolution(model, size string) string {
	res := pixelResolution(size)
	if res == "1080p" && model == "grok-imagine-video" {
		return "720p"
	}
	return res
}

func pixelResolution(size string) string {
	w, h, ok := strings.Cut(strings.ToLower(strings.TrimSpace(size)), "x")
	if !ok {
		return ""
	}
	x, err1 := strconv.ParseFloat(w, 64)
	y, err2 := strconv.ParseFloat(h, 64)
	if err1 != nil || err2 != nil || x <= 0 || y <= 0 {
		return ""
	}
	switch short := math.Min(x, y); {
	case short >= 1080:
		return "1080p"
	case short >= 720:
		return "720p"
	}
	return "480p"
}

// volcengineVideoBody is f as Ark's content-generation API takes it: text
// and images in content, with image roles distinguishing the first frame from
// subject references, plus duration, ratio and resolution as separate fields.
func volcengineVideoBody(model string, f filming) []byte {
	content := []map[string]any{{"type": "text", "text": f.Prompt}}
	if f.Start != nil {
		content = append(content, map[string]any{"type": "image_url", "image_url": map[string]string{"url": f.Start.dataURL()}, "role": "first_frame"})
	}
	for _, pic := range f.References {
		content = append(content, map[string]any{"type": "image_url", "image_url": map[string]string{"url": pic.dataURL()}, "role": "reference_image"})
	}
	req := map[string]any{"model": model, "content": content}
	if f.Seconds != "" {
		req["duration"], _ = strconv.Atoi(f.Seconds)
	}
	if ar := aspectAmong(f.Size, grokVideoAspects); ar != "" {
		req["ratio"] = ar
	}
	if res := pixelResolution(f.Size); res != "" {
		req["resolution"] = res
	}
	b, _ := json.Marshal(req)
	return b
}

// grokVideoBody is f as Grok's video API takes it: the duration in whole
// seconds, an aspect ratio and a resolution rather than a size, the first
// frame as image and the subjects to draw from as reference_images. What
// the API turns away (a duration out of range) it says itself.
func grokVideoBody(model string, f filming) ([]byte, error) {
	req := map[string]any{"model": model, "prompt": f.Prompt}
	if f.Seconds != "" {
		n, err := parseSeconds(f.Seconds)
		if err != nil {
			return nil, err
		}
		req["duration"] = n
	}
	if ar := aspectAmong(f.Size, grokVideoAspects); ar != "" {
		req["aspect_ratio"] = ar
	}
	if res := videoResolution(model, f.Size); res != "" {
		req["resolution"] = res
	}
	if f.Start != nil {
		req["image"] = map[string]string{"url": f.Start.dataURL()}
	}
	if len(f.References) > 0 {
		var refs []map[string]string
		for _, pic := range f.References {
			refs = append(refs, map[string]string{"url": pic.dataURL()})
		}
		req["reference_images"] = refs
	}
	return json.Marshal(req)
}

// videoID is the id of a video p is making for vendorID, started at t. It
// names the provider, the vendor's id, the start and whose sign-in started it.
func videoID(p provider.Provider, vendorID string, t time.Time) string {
	return "video_" + p.ID + "." + vendorID + "." + strconv.FormatInt(t.Unix(), 10) + "." + signer(p)
}

// signer is a short mark of the account p signs in as, so a video can be
// told from those of the account signed in before a switch.
func signer(p provider.Provider) string {
	if p.Account == nil {
		return "0"
	}
	sum := sha256.Sum256([]byte(p.Account.User))
	return hex.EncodeToString(sum[:3])
}

// idPart is what a provider id or a vendor's request id is made of; an id
// is put in a URL, so nothing that would change its path gets through.
var idPart = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)

// parseVideoID is the provider, vendor id, start and signer of a video's id.
func parseVideoID(id string) (providerID, vendorID string, started time.Time, by string, ok bool) {
	rest, found := strings.CutPrefix(id, "video_")
	if !found {
		return
	}
	parts := strings.Split(rest, ".")
	if len(parts) != 4 || !idPart.MatchString(parts[0]) || !idPart.MatchString(parts[1]) || !idPart.MatchString(parts[3]) {
		return
	}
	sec, err := strconv.ParseInt(parts[2], 10, 64)
	if err != nil {
		return
	}
	return parts[0], parts[1], time.Unix(sec, 0), parts[3], true
}

// videoState is how the vendor says a video is going.
type videoState struct {
	Status   string `json:"status"`
	Progress int    `json:"progress"`
	Model    string `json:"model"`
	Video    struct {
		URL      string  `json:"url"`
		Duration float64 `json:"duration"`
	} `json:"video"`
	Content struct {
		VideoURL string `json:"video_url"`
	} `json:"content"`
	Duration float64 `json:"duration"`
	Error    struct {
		Code    string `json:"code"`
		Message string `json:"message"`
	} `json:"error"`
}

// videoObject is a video as OpenAI's API gives it.
func videoObject(id, model string, started time.Time, st videoState, f filming) map[string]any {
	status, progress := "queued", 0
	switch st.Status {
	case "done", "succeeded":
		status, progress = "completed", 100
	case "running":
		status, progress = "in_progress", st.Progress
	case "failed", "expired", "cancelled":
		status, progress = "failed", st.Progress
	default:
		if st.Progress > 0 {
			status, progress = "in_progress", st.Progress
		}
	}
	obj := map[string]any{"id": id, "object": "video", "created_at": started.Unix(), "status": status, "progress": progress, "model": model}
	if f.Prompt != "" {
		obj["prompt"] = f.Prompt
	}
	if f.Size != "" {
		obj["size"] = f.Size
	}
	switch {
	case st.Video.Duration > 0:
		obj["seconds"] = strconv.FormatFloat(st.Video.Duration, 'f', -1, 64)
	case st.Duration > 0:
		obj["seconds"] = strconv.FormatFloat(st.Duration, 'f', -1, 64)
	case f.Seconds != "":
		obj["seconds"] = f.Seconds
	}
	if status == "failed" {
		code, msg := st.Error.Code, st.Error.Message
		switch st.Status {
		case "expired":
			code, msg = "expired", "the vendor no longer keeps this video"
		case "cancelled":
			code, msg = "cancelled", "the video was cancelled"
		}
		if msg == "" {
			msg = "the video failed"
		}
		obj["error"] = map[string]string{"code": code, "message": msg}
	}
	return obj
}

// videoMaker is the provider a video's id names, when it makes videos.
func videoMaker(id string) (p provider.Provider, vendorID string, started time.Time, err error) {
	pid, vendorID, started, by, ok := parseVideoID(id)
	if !ok {
		return p, "", started, fmt.Errorf("%q isn't the id of a video magpie is making", id)
	}
	found, ferr := provider.Find(pid)
	if ferr == nil && filmsMiniMax(*found) {
		// a MiniMax task is its key's account's: asked on that key
		q, ok := markedKey(*found, by)
		if !ok {
			return p, "", started, fmt.Errorf("this video was started with a key of %s that is no longer on: turn it back on to get the video", found.Name)
		}
		return q, vendorID, started, nil
	}
	if ferr == nil && !drawsGrok(*found) && !found.IsRemoteMagpie() && len(found.PlanVideos()) == 0 && len(customVideomakers(*found)) > 0 {
		// asked on the key that started it
		q, ok := markedKey(*found, by)
		if !ok {
			return p, "", started, fmt.Errorf("this video was started with a key of %s that is no longer on: turn it back on to get the video", found.Name)
		}
		return q, vendorID, started, nil
	}
	if ferr != nil || !drawsGrok(*found) && !found.IsRemoteMagpie() && len(found.PlanVideos()) == 0 {
		return p, "", started, fmt.Errorf("no provider %q makes videos here", pid)
	}
	if found.Account == nil && found.PlansMixed() {
		// asked on the key that started it, at its plan's endpoint
		q, ok := markedKey(*found, by)
		if !ok {
			return p, "", started, fmt.Errorf("this video was started with a key of %s that is no longer on: turn it back on to get the video", found.Name)
		}
		return q, vendorID, started, nil
	}
	if signer(*found) != by {
		return p, "", started, fmt.Errorf("this video was started with another %s account than the one signed in now: sign back in to it to get the video", found.Name)
	}
	return *found, vendorID, started, nil
}

// remoteVideoID is the id of a video another magpie (p) is making, by its
// id there: that has dots in it, so it goes in base64.
func remoteVideoID(p provider.Provider, theirs string, t time.Time) string {
	return videoID(p, base64.RawURLEncoding.EncodeToString([]byte(theirs)), t)
}

// theirVideoID is the other magpie's id of a video, from remoteVideoID's.
func theirVideoID(vendorID string) string {
	b, _ := base64.RawURLEncoding.DecodeString(vendorID)
	return string(b)
}

// remoteVideoBody is f as another magpie's videos API takes it: OpenAI's,
// the images as data URLs.
func remoteVideoBody(model string, f filming) []byte {
	req := map[string]any{}
	for k, v := range f.Extra {
		req[k] = v
	}
	req["model"], req["prompt"] = model, f.Prompt
	if f.Seconds != "" {
		req["seconds"] = f.Seconds
	}
	if f.Size != "" {
		req["size"] = f.Size
	}
	if f.Start != nil {
		req["input_reference"] = f.Start.dataURL()
	}
	if f.End != nil {
		req["last_frame"] = f.End.dataURL()
	}
	var refs []string
	for _, pic := range f.References {
		refs = append(refs, pic.dataURL())
	}
	if len(refs) > 0 {
		req["reference_images"] = refs
	}
	b, _ := json.Marshal(req)
	return b
}

// asOurs is another magpie's video object with this magpie's id for it and
// name for its model.
func asOurs(b []byte, p provider.Provider, id string) (map[string]any, error) {
	var obj map[string]any
	if err := json.Unmarshal(b, &obj); err != nil || obj == nil {
		return nil, fmt.Errorf("%s's answer isn't a video's: %v", p.Name, err)
	}
	obj["id"] = id
	if m, _ := obj["model"].(string); m != "" {
		obj["model"] = p.ID + "/" + m
	}
	return obj, nil
}

// filmsOpenAI is whether p makes videos at an OpenAI-shaped videos API:
// another magpie, or a provider whose model list marks video models.
func filmsOpenAI(p provider.Provider) bool {
	return p.IsRemoteMagpie() || len(customVideomakers(p)) > 0
}

// remoteVideoURL is where another magpie, or a provider's OpenAI-shaped
// videos API, answers for a video, with suffix ("" or "/content").
func remoteVideoURL(p provider.Provider, vendorID, suffix string) string {
	return videoBase(p) + "/videos/" + url.PathEscape(theirVideoID(vendorID)) + suffix
}

// videoStatus asks the vendor how a video is going.
func (s *Server) videoStatus(ctx context.Context, p provider.Provider, vendorID string) (videoState, int, error) {
	if filmsMiniMax(p) {
		return s.minimaxVideoStatus(ctx, p, vendorID)
	}
	var st videoState
	at := strings.TrimRight(p.Base(provider.Responses), "/") + "/videos/" + url.PathEscape(vendorID)
	if p.Preset == "volcengine" {
		at = strings.TrimRight(p.Base(provider.Responses), "/") + "/contents/generations/tasks/" + url.PathEscape(vendorID)
	}
	b, code, err := s.sendAs(ctx, p, http.MethodGet, at, "", nil, true)
	if err != nil {
		return st, code, err
	}
	if err := json.Unmarshal(b, &st); err != nil {
		return st, 502, fmt.Errorf("%s's answer isn't a video's: %v", p.Name, err)
	}
	return st, code, nil
}

// videosCreate starts a video and answers with its id.
func (s *Server) videosCreate(w http.ResponseWriter, r *http.Request) {
	inputBody, admitted := s.requestBody(w, r, provider.Chat)
	if !admitted {
		return
	}
	r.Body = io.NopCloser(bytes.NewReader(inputBody))
	defer func() {
		if r.MultipartForm != nil {
			r.MultipartForm.RemoveAll()
		}
	}()
	start := time.Now()
	f, err := readFilming(r)
	if err != nil {
		writeError(w, provider.Chat, 400, err.Error())
		return
	}
	who := callerOf(r)
	call := Call{Time: start, From: provider.Chat, Agent: who.agent, Via: who.via, Model: f.Model}
	usage.Saw(agentOf(r))
	fail := func(code int, msg string) {
		call.Status, call.Error, call.Millis = code, msg, time.Since(start).Milliseconds()
		writeError(w, provider.Chat, code, msg)
		s.record(call)
	}
	named := f.Model != "" // a gateway key's models hold one the caller names (#882)
	if f.Model == "" {
		m, ok := videomaker()
		if !ok {
			fail(400, "no model to make videos with: sign in to a Grok subscription or set up a provider that makes videos (a MiniMax API key makes them with MiniMax-H3), and leave Settings → Images → Image generation on, or name one")
			return
		}
		f.Model, call.Model = m, m
	}
	p, model, ok := provider.Resolve(f.Model)
	if !ok {
		p, model, ok = bareVideomaker(f.Model)
	}
	if !ok {
		if off, isOff := provider.SwitchedOff(f.Model); isOff {
			fail(404, switchedOff(off, f.Model))
			return
		}
		fail(404, fmt.Sprintf("magpie knows no model %q to make videos with", f.Model))
		return
	}
	g, ms, isGroup := drawingGroup(f.Model, p, model)
	keyWho, keyHeld := keyHolds(r)
	if keyHeld && named && (isGroup && !groupAllowed(keyWho, g, ms) || !isGroup && !modelAllowed(keyWho, p, model)) {
		fail(403, keyModelError(keyWho, f.Model))
		return
	}
	if msg, off := modelOff(p, model, f.Model); off { // disabled by the user (#1867)
		fail(403, msg)
		return
	}
	// the models to start the video on, the next asked while one fails: a
	// routing group's members in its routing's order, as a drawing's
	// (#2005), else the one
	tries := []provider.Member{{Provider: p, Model: model}}
	if isGroup {
		var paused []provider.Paused
		if ms, paused = provider.PausedOut(g.Live(), ms, agentOf(r), "", ruleClock()); len(ms) == 0 && len(paused) > 0 {
			fail(503, pausedError(f.Model, paused))
			return
		}
		var keeps map[string]bool
		if keyHeld {
			keeps = groupKeeps(keyWho, g, ms)
		}
		t := s.drawingTries(g, ms, keeps)
		if len(t) == 0 && keeps != nil {
			fail(403, keyModelError(keyWho, f.Model))
			return
		}
		if len(t) > 0 {
			tries = t
		}
	}
	call.To = provider.Chat
	var unmask func()
	prompt := f.Prompt
	var hits maskHits
	w, f.Prompt, unmask, hits, err = redactedPromptHits(r.Context(), w, f.Prompt)
	defer unmask()
	// the fields passed on as they came are masked as the prompt is: a
	// negative_prompt is the user's text all the same
	var extra maskHits
	if err == nil {
		err = maskStrings(r.Context(), f.Extra, &extra)
	}
	if err != nil { // fails closed
		fail(maskStatus(err), err.Error())
		return
	}
	n := extra.n
	hits.add(extra)
	sent := f.Prompt
	if b, e := json.Marshal(f.Extra); e == nil && extra.n > 0 {
		sent += " " + string(b)
	}
	// the record names the member that had it last (noteMaskedUntil)
	// the Routing page lists it as it does a chat, each member asked a
	// try of its one route (sideRoute); its polls and download are the
	// same video, and add none
	var grp *GroupRef
	if isGroup {
		grp = groupRef(g, ms)
	}
	side := s.beginSide(r, start, call.Model, p.ID, grp)
	masked, noted := noteMaskedUntil(r, "video", hits, sent, side.id(), cmp.Or(call.Provider, p.ID), call.Model)
	side.masked(masked)
	defer func() { noted(call.Provider, model) }()
	if n > 0 && f.Prompt == prompt { // the prompt's mask wraps w already
		var done func()
		w, done = restoring(w)
		defer done()
	}
	ctx, cancel := drawWithin(r.Context(), videoTimeout)
	defer cancel()
	var (
		v       videoStart
		code    int
		skipped []string
	)
	for i, m := range tries {
		model = m.Model
		at := time.Now()
		v, code, err = s.startVideo(ctx, r, m.Provider, m.Model, f)
		p, call.Provider = v.p, v.p.ID
		side.tryAt(candidateOf(v.p, m.Model), at)(code, err)
		// a group's next member is asked as a chat's would be: while this
		// one can't make videos, its vendor fails, is out of quota or
		// doesn't serve the model; a refusal of the request is the request's
		if err == nil || ctx.Err() != nil || i+1 == len(tries) || !v.unfit && !retryable(code, []byte(err.Error())) {
			break
		}
		skipped = append(skipped, m.Provider.ID+"/"+m.Model+": "+err.Error())
	}
	call.Fallback = strings.Join(skipped, "; ")
	if err != nil && !v.asked {
		// turned away before any vendor was asked: nothing was spent
		msg := err.Error()
		if len(skipped) > 0 {
			msg += " (tried first: " + call.Fallback + ")"
		}
		side.end(p, model, code, errors.New(msg), Usage{})
		fail(code, msg)
		return
	}
	b, remote, custom, field, onKey := v.answer, v.remote, v.custom, v.field, v.onKey
	call.Millis, call.Status = time.Since(start).Milliseconds(), code
	var started map[string]any
	theirs := ""
	if err == nil {
		json.Unmarshal(b, &started)
		if theirs, _ = started[field].(string); theirs == "" {
			code, err = 502, fmt.Errorf("%s started no video: its answer had no %s", p.Name, field)
			call.Status = code
		}
	}
	side.end(p, model, call.Status, err, Usage{})
	appendUsage(r, usage.Record{RouteID: side.id(), Operation: "generate_content", Time: start, Agent: call.Agent, Via: call.Via, Provider: p.ID, Host: p.Where(), Model: model, Requested: call.Model, ProviderAccount: accountOf(p),
		Millis: call.Millis, Status: call.Status, Session: sessionOf(r.Header)})
	if err != nil {
		call.Error = err.Error()
		s.record(call)
		msg := err.Error()
		if len(skipped) > 0 {
			msg += " (tried first: " + call.Fallback + ")"
		}
		writeError(w, provider.Chat, code, msg)
		return
	}
	s.record(call)
	if remote {
		obj, _ := asOurs(b, p, remoteVideoID(p, theirs, start))
		writeJSON(w, 200, obj)
		return
	}
	if custom {
		obj, _ := asOurs(b, p, customVideoID(p, theirs, start))
		if _, ok := obj["model"]; !ok {
			obj["model"] = p.ID + "/" + model
		}
		writeJSON(w, 200, obj)
		return
	}
	id := videoID(p, theirs, start)
	if onKey {
		id = strings.TrimSuffix(id, "."+signer(p)) + "." + keyMark(p)
	}
	writeJSON(w, 200, videoObject(id, p.ID+"/"+model, start, videoState{}, f))
}

// videoStart is how starting a video on one model went: the key or
// account it went out on, and where and how its answer is read.
type videoStart struct {
	p              provider.Provider
	answer         []byte
	field          string // the answer's field naming the vendor's video
	remote, custom bool
	asked          bool // a vendor was asked: what it answered is recorded
	unfit          bool // the model makes no videos: none was asked
	// onKey: started on one of keys on plans of their own, which its id
	// names (keyMark) to be asked after on, at that plan's endpoint
	onKey bool
}

// startVideo starts f's video on p's model: at another magpie's videos
// API, a provider's OpenAI-shaped one, Volcengine's tasks or Grok's. A
// failure before any vendor is asked (the model makes no videos, a key may
// use none of the provider's keys) has asked unset.
func (s *Server) startVideo(ctx context.Context, r *http.Request, p provider.Provider, model string, f filming) (videoStart, int, error) {
	// the provider's other keys to start the video on, when the first
	// says it doesn't serve the model
	var keys []candidate
	onKey := false
	if !p.IsRemoteMagpie() && filmsCustom(p, model) {
		// at a provider's own videos API: sent on the first key a chat
		// for the model would go to (#1657), never one whose own list of
		// models leaves it out, nor one the calling key may not use
		// (#905); the video's id names the key, which it is asked after on
		cs, code, msg := s.onKeys(r, p, model, f.Model)
		if code != 0 {
			return videoStart{p: p}, code, errors.New(msg)
		}
		p, keys = cs[0].p, cs[1:]
	} else if makesMiniMaxVideo(p, model) {
		// on the first key a chat for the model would go to that the
		// calling key may use, the next while one's plan doesn't make H3
		// (a Token Plan's); the video's id names the key it is asked on
		cs, code, msg := s.onKeys(r, p, model, f.Model)
		if code != 0 {
			return videoStart{p: p}, code, errors.New(msg)
		}
		p, keys = cs[0].p, cs[1:]
		onKey = true
	} else if !p.IsRemoteMagpie() && p.PlansMixed() {
		// keys on plans of their own (KeyAccount.Plan): sent on a key
		// whose plan makes the model, the Agent Plan's Seedance on the
		// Agent Plan's key, never a pay-as-you-go key beside it
		cs, code, msg := s.onKeys(r, p, model, f.Model)
		if code != 0 {
			return videoStart{p: p}, code, errors.New(msg)
		}
		p = cs[0].p
		onKey = true
	} else if keyWho, held := accountHolds(r); held {
		// held to some accounts (#905): sent on a key the calling key may
		// use, of those in use — not the provider's first alone, which a
		// key held to a later one was refused by — or none, refused
		var ok bool
		if p, ok = allowedKey(keyWho, p, model); !ok {
			return videoStart{p: p}, 403, errors.New(keyAccountsError(keyWho, f.Model))
		}
	}
	v := videoStart{p: p, remote: p.IsRemoteMagpie(), onKey: onKey}
	v.custom = !v.remote && filmsCustom(p, model)
	volcengine := p.Preset == "volcengine" && slices.ContainsFunc(p.PlanVideos(), func(m catalog.Model) bool { return m.ID == model })
	minimax := !v.remote && !v.custom && makesMiniMaxVideo(p, model)
	// any grok-imagine-video*, not only those listed: the vendor's newer ones work before magpie names them
	if !v.remote && !v.custom && !volcengine && !minimax && (!drawsGrok(p) || !strings.HasPrefix(model, grokVideoFamily)) {
		v.unfit = true
		return v, 400, fmt.Errorf(`%s/%s can't make videos: magpie makes videos with a Grok subscription's grok-imagine-video, Volcengine Agent Plan's Seedance models, a MiniMax API key's MiniMax-H3 models, another magpie's, and a provider's models its own model list marks "kind": "video" or that you mark a Video model (right-click the model in the provider's editor: Makes…)`, p.ID, model)
	}
	// another magpie is asked at its videos API, which says itself what
	// it can't make
	at, field := strings.TrimRight(p.Base(provider.Responses), "/")+"/videos/generations", "request_id"
	var body []byte
	switch {
	case v.remote, v.custom:
		// a provider's OpenAI-shaped videos API is asked as another
		// magpie's is: OpenAI's body, its answer its own
		at, field, body = videoBase(p)+"/videos", "id", remoteVideoBody(model, f)
	case volcengine:
		at, field, body = strings.TrimRight(p.Base(provider.Responses), "/")+"/contents/generations/tasks", "id", volcengineVideoBody(model, f)
	case minimax:
		var err error
		if body, err = minimaxVideoBody(model, f); err != nil {
			return v, 400, err
		}
		at, field = minimaxHost(p)+"/v2/video_generation", "task_id"
	default:
		var err error
		if body, err = grokVideoBody(model, f); err != nil {
			return v, 400, err
		}
	}
	v.field, v.asked = field, true
	b, code, err := s.send(ctx, p, at, "application/json", body, true)
	// a key answering 404 that it doesn't serve the model hands it to the
	// next (#1657), as a drawing does; the id names the key that started it
	for ; err != nil && len(keys) > 0 && ctx.Err() == nil && (keyLacksModel(code, err) || minimax && minimaxLacksH3(b)); keys = keys[1:] {
		p = keys[0].p
		b, code, err = s.send(ctx, p, at, "application/json", body, true)
	}
	if minimax {
		if err != nil {
			err = minimaxError(code, b, err)
		} else if msg, mcode := minimaxSaid(b); msg != "" {
			// an answer of 200 that says it failed (base_resp)
			code, err = 502, minimaxError(0, b, fmt.Errorf("%s: %s%s", p.Name, msg, codeUnlessSaid(msg, mcode)))
		}
	}
	v.p, v.answer = p, b
	return v, code, err
}

// videoAsker is p on a key or account the calling key may use, to ask
// after a video (#905), as starting one is held: a video at the
// OpenAI-shaped videos API of a provider set up with a base URL on the key
// that started it, which the calling key must be allowed; any other on the
// first key or account it may use, as videosCreate picks. What the calling
// key is told when it may use none, else "".
func videoAsker(r *http.Request, p provider.Provider) (provider.Provider, string) {
	who, held := accountHolds(r)
	if !held {
		return p, ""
	}
	if !p.IsRemoteMagpie() && (filmsOpenAI(p) || filmsMiniMax(p)) {
		if p.Key == "" && p.Account == nil || accountAllowed(who, candidate{p: p}) {
			return p, ""
		}
		return p, keyAccountsError(who, p.ID)
	}
	q, ok := allowedKey(who, p, "")
	if !ok {
		return p, keyAccountsError(who, p.ID)
	}
	return q, ""
}

// videoModelHeld is what the calling key is told when its models hold it
// from a video of p's model (#882), as they hold it from starting one: ""
// when they don't. A video whose model the vendor doesn't say is held.
func videoModelHeld(r *http.Request, p provider.Provider, model string) string {
	who, held := keyHolds(r)
	if !held || model != "" && modelAllowed(who, p, model) {
		return ""
	}
	if model == "" {
		return fmt.Sprintf("The gateway key %q may not ask after this video: %s didn't say which model makes it.", who.KeyName, p.Name)
	}
	return keyModelError(who, p.ID+"/"+model)
}

// openAIVideo is how a video at an OpenAI-shaped videos API (another
// magpie's, a provider's) is going, with this magpie's id for it, and the
// model making it as that API names it.
func (s *Server) openAIVideo(ctx context.Context, p provider.Provider, vendorID, id string) (map[string]any, string, int, error) {
	b, code, err := s.sendAs(ctx, p, http.MethodGet, remoteVideoURL(p, vendorID, ""), "", nil, true)
	if err != nil {
		return nil, "", code, err
	}
	obj, err := asOurs(b, p, id)
	if err != nil {
		return nil, "", 502, err
	}
	model, _ := obj["model"].(string)
	return obj, strings.TrimPrefix(model, p.ID+"/"), 200, nil
}

// videosGet answers how a video is going.
func (s *Server) videosGet(w http.ResponseWriter, r *http.Request) {
	// A later poll can echo the prompt masked when the video was created.
	w, done := restoring(w)
	defer done()
	id := r.PathValue("id")
	p, vendorID, started, err := videoMaker(id)
	if err != nil {
		writeError(w, provider.Chat, 404, err.Error())
		return
	}
	p, held := videoAsker(r, p)
	if held != "" {
		writeError(w, provider.Chat, 403, held)
		return
	}
	ctx, cancel := drawWithin(r.Context(), videoTimeout)
	defer cancel()
	if filmsOpenAI(p) {
		obj, model, code, err := s.openAIVideo(ctx, p, vendorID, id)
		if err != nil {
			writeError(w, provider.Chat, code, err.Error())
			return
		}
		if held := videoModelHeld(r, p, model); held != "" {
			writeError(w, provider.Chat, 403, held)
			return
		}
		writeJSON(w, 200, obj)
		return
	}
	st, code, err := s.videoStatus(ctx, p, vendorID)
	if err != nil {
		writeError(w, provider.Chat, code, err.Error())
		return
	}
	model := st.Model
	if model == "" {
		model = "grok-imagine-video"
	}
	if held := videoModelHeld(r, p, model); held != "" {
		writeError(w, provider.Chat, 403, held)
		return
	}
	writeJSON(w, 200, videoObject(id, p.ID+"/"+model, started, st, filming{}))
}

// videosContent sends the video's bytes once it is made.
func (s *Server) videosContent(w http.ResponseWriter, r *http.Request) {
	id := r.PathValue("id")
	p, vendorID, _, err := videoMaker(id)
	if err != nil {
		writeError(w, provider.Chat, 404, err.Error())
		return
	}
	p, held := videoAsker(r, p)
	if held != "" {
		writeError(w, provider.Chat, 403, held)
		return
	}
	ctx, cancel := drawWithin(r.Context(), videoTimeout)
	defer cancel()
	if filmsOpenAI(p) {
		if _, models := keyHolds(r); models {
			// the bytes say no model: the video's own answer does
			_, model, code, err := s.openAIVideo(ctx, p, vendorID, id)
			if err != nil {
				writeError(w, provider.Chat, code, err.Error())
				return
			}
			if held := videoModelHeld(r, p, model); held != "" {
				writeError(w, provider.Chat, 403, held)
				return
			}
		}
		s.remoteVideoContent(ctx, w, p, vendorID)
		return
	}
	st, code, err := s.videoStatus(ctx, p, vendorID)
	if err != nil {
		writeError(w, provider.Chat, code, err.Error())
		return
	}
	if held := videoModelHeld(r, p, cmp.Or(st.Model, "grok-imagine-video")); held != "" {
		writeError(w, provider.Chat, 403, held)
		return
	}
	if st.Status != "done" && st.Status != "succeeded" {
		obj := videoObject(id, "", time.Time{}, st, filming{})
		msg := fmt.Sprintf("the video isn't ready: it is %v", obj["status"])
		if e, ok := obj["error"].(map[string]string); ok {
			msg += ": " + e["message"]
		}
		writeError(w, provider.Chat, 409, msg)
		return
	}
	if st.Video.URL == "" {
		st.Video.URL = st.Content.VideoURL
	}
	if st.Video.URL == "" {
		writeError(w, provider.Chat, 502, p.Name+" made the video but gave no URL for it")
		return
	}
	req, err := http.NewRequestWithContext(p.Via(ctx), http.MethodGet, st.Video.URL, nil)
	if err != nil {
		writeError(w, provider.Chat, 502, err.Error())
		return
	}
	// the URL is in the vendor's answer, so it is fetched as one a caller
	// names is: public addresses, or the vendor's own host (vendorClient)
	res, err := vendorClient(p).Do(req)
	if err != nil {
		writeError(w, provider.Chat, 502, fmt.Sprintf("the video couldn't be fetched: %v", err))
		return
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		writeError(w, provider.Chat, 502, fmt.Sprintf("the video couldn't be fetched: %d %s", res.StatusCode, http.StatusText(res.StatusCode)))
		return
	}
	w.Header().Set("Content-Type", "video/mp4")
	if res.ContentLength > 0 {
		w.Header().Set("Content-Length", strconv.FormatInt(res.ContentLength, 10))
	}
	w.WriteHeader(200)
	io.Copy(w, io.LimitReader(res.Body, maxVideoBytes))
}

// remoteVideoContent passes on the bytes of a video another magpie, or a
// provider's OpenAI-shaped videos API, made, or what it says of one it
// hasn't.
func (s *Server) remoteVideoContent(ctx context.Context, w http.ResponseWriter, p provider.Provider, vendorID string) {
	ctx = p.Via(ctx)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, remoteVideoURL(p, vendorID, "/content"), nil)
	if err != nil {
		writeError(w, provider.Chat, 502, err.Error())
		return
	}
	// magpie's, or the agent's own where the provider passes it on (#1851)
	stampUA(ctx, p, req.Header)
	if p.IsRemoteMagpie() {
		// vendors are told no caller
		req.Header.Set("User-Agent", "magpie/"+Version)
		passOnCaller(ctx, req)
	}
	if err := p.Sign(ctx, req, provider.Chat, nil); err != nil {
		writeError(w, provider.Chat, 502, err.Error())
		return
	}
	res, err := p.Do(s.client, req)
	if err != nil {
		writeError(w, provider.Chat, 502, fmt.Sprintf("the video couldn't be fetched: %v", err))
		return
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		b, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
		said, ok := provider.Unsaid(b)
		if !ok {
			said = vendorMessage(b)
		}
		writeError(w, provider.Chat, res.StatusCode, p.Name+": "+said)
		return
	}
	w.Header().Set("Content-Type", cmp.Or(res.Header.Get("Content-Type"), "video/mp4"))
	if res.ContentLength > 0 {
		w.Header().Set("Content-Length", strconv.FormatInt(res.ContentLength, 10))
	}
	w.WriteHeader(200)
	io.Copy(w, io.LimitReader(res.Body, maxVideoBytes))
}

// maskStrings masks every string in v (a decoded JSON value) in place as a
// prompt is masked, adding what it swapped to h; an error when it can't
// (fail closed).
func maskStrings(ctx context.Context, v any, h *maskHits) error {
	one := func(str string) (string, error) {
		_, m, got, err := maskTextHits(ctx, str)
		h.add(got)
		return m, err
	}
	var err error
	switch x := v.(type) {
	case map[string]any:
		for k, e := range x {
			if str, ok := e.(string); ok {
				x[k], err = one(str)
			} else {
				err = maskStrings(ctx, e, h)
			}
			if err != nil {
				return err
			}
		}
	case []any:
		for i, e := range x {
			if str, ok := e.(string); ok {
				x[i], err = one(str)
			} else {
				err = maskStrings(ctx, e, h)
			}
			if err != nil {
				return err
			}
		}
	}
	return nil
}