Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
"""The argv, grid and adapter contract both generative video-upscale runners share.
#6511 promises ONE argument set for both backends: `buildLtxUpscaleArgs()` in
`server/services/videoGen/renderArgs.js` emits a single argv whichever runtime
the plan resolved, and only the interpreter and script path differ. #6512 (MLX)
and #6513 (CUDA) each implement that contract against a different runtime, so a
second copy of it in each runner would be two tables free to drift — exactly the
failure the flag-passed reference bounds already exist to prevent.
What lives here is the part that is genuinely identical: the parser, the model
grid, and the header-only reads of the IC-LoRA adapter. What stays with each
runner is what actually differs — the runtime it imports, the host it gates on,
the pack layout it validates, and the rename maps its own loader fuses through.
Everything here is stdlib-only, deliberately. These are the checks that run
BEFORE a GPU, an MLX wheel, or a 68 GB model pack is committed, which is what
makes the whole contract testable on a machine that has none of them.
"""
from __future__ import annotations
import argparse
import json
import struct
from pathlib import Path
# The LTX-2.5 grid, mirrored from `LTX_GRID` in
# `server/services/videoGen/upscalePlan.js` and confirmed against both runtimes.
# The video VAE compresses spatially by 32, and the upscaler's reference has to
# land on that grid at HALF the output (see `assert_reference_scale_fits`), so
# the OUTPUT axis must be divisible by 64 — which is also what upstream's own
# `assert_resolution(..., is_two_stage=True)` demands of the request both
# runners make (`conditioned_stage_request` below). Temporal compression is 8
# and the reference encoder needs a (1 + 8k)-frame input, so `frames % 8 == 1`
# with a floor of 9.
# The VAE's spatial compression: the reference clip is resized onto this grid
# before it is encoded, so a reference that is not a multiple of it gets
# resampled — which for an upscaler means conditioning on a blurred source.
VAE_SPATIAL_COMPRESSION = 32
# The shipped adapter's declared `reference_downscale_factor` (measured, and
# declared by `icLoraWeights.js`); `assert_reference_scale_fits` re-derives the
# same rule from the factor read off the file actually fused.
SHIPPED_REFERENCE_DOWNSCALE = 2
SPATIAL_MULTIPLE = SHIPPED_REFERENCE_DOWNSCALE * VAE_SPATIAL_COMPRESSION
FRAME_MODULUS = 8
FRAME_REMAINDER = 1
MIN_FRAMES = 9
# How both `ICLoraPipeline`s interpret the `height`/`width` they are handed:
# as the dims of the OPTIONAL latent-upsample stage 2, with the IC-conditioned
# stage 1 rendering at exactly half of that. The Pixel Spatial Upscaler is the
# standard single-stage IC-LoRA video-to-video recipe (Lightricks' IC-LoRA
# guide and their ComfyUI reference workflow run it "single-stage distilled at
# your set resolution", reference at half), so a runner asks for TWICE the
# output and skips stage 2: stage 1 then renders AT the output size with the
# source as a native-resolution reference, and the decoded clip is the output.
# Requesting the output itself would render stage 1 at half the output —
# conditioning on the source downscaled by another 2x — and hand the second
# doubling to the latent upsampler, which is exactly the pixel-faithful
# refinement the adapter is not.
PIPELINE_REQUEST_MULTIPLIER = 2
# The upscale contract carries no prompt (#6511): the source clip is the whole
# conditioning signal, and inventing text steering would push synthesized detail
# toward content the user never asked for. The empty string is the neutral
# choice — the distilled schedule runs no CFG, so the prompt is pure
# conditioning rather than a guidance pole. `--prompt` exists so #6514's
# verification matrix can probe the effect without changing the argv contract.
DEFAULT_PROMPT = ""
# Full reference conditioning: unlike a control/pose IC render, the reference
# here is the picture itself, so there is nothing to attenuate.
REFERENCE_STRENGTH = 1.0
# A real safetensors header is a few KB to low-MB; anything past this is a
# corrupt length we refuse to allocate for. Mirrors the same bound in
# `server/lib/safetensors.js`.
MAX_HEADER_BYTES = 100 * 1024 * 1024
_LORA_HALVES = ("lora_A", "lora_B")
def add_upscale_arguments(parser: argparse.ArgumentParser, *, model_help: str) -> argparse.ArgumentParser:
"""Declare the flags `buildLtxUpscaleArgs()` emits, in one place.
The pass is an IC-LoRA render whose single reference IS the clip being
upscaled, which is why the alphabet is the IC one rather than a second
vocabulary. The reference-count bounds arrive as flags because
`server/lib/icLoraWeights.js` is the single source of truth across both
languages — a Python-side default would be a second table free to drift.
Only `--model`'s help text differs between backends: one names an MLX pack,
the other a split CUDA snapshot.
"""
parser.add_argument("--model", required=True, help=model_help)
parser.add_argument("--ic-lora-path", required=True,
help="local .safetensors of the Pixel Spatial Upscaler adapter")
parser.add_argument("--ic-reference", action="append", default=[],
help="the clip being upscaled, already aligned to the model grid")
parser.add_argument("--ic-min-references", type=int, required=True)
parser.add_argument("--ic-max-references", type=int, required=True)
parser.add_argument("--width", type=int, required=True)
parser.add_argument("--height", type=int, required=True)
parser.add_argument("--num-frames", type=int, required=True)
parser.add_argument("--fps", type=float, required=True)
parser.add_argument("--seed", type=int, required=True)
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
parser.add_argument("--output", required=True)
return parser
def validate_args(args: argparse.Namespace) -> None:
"""Everything checkable before a GPU is committed.
Both backends refuse the same sources, and both refuse them the same way a
direct/script caller would hit — so a queue replay, a hand-run helper and
the route all get one answer. Every refusal is a `SystemExit` string the
queue surfaces verbatim.
"""
if args.width % SPATIAL_MULTIPLE or args.height % SPATIAL_MULTIPLE:
raise SystemExit(
f"The LTX-2.5 upscale requires width and height divisible by {SPATIAL_MULTIPLE}; "
f"got {args.width}x{args.height}."
)
if args.num_frames < MIN_FRAMES or args.num_frames % FRAME_MODULUS != FRAME_REMAINDER:
raise SystemExit(
f"LTX-2.5 num-frames must be at least {MIN_FRAMES} and satisfy "
f"frames % {FRAME_MODULUS} == {FRAME_REMAINDER}; got {args.num_frames}."
)
if not args.fps > 0:
raise SystemExit(f"--fps must be positive; got {args.fps}.")
if args.seed < 0:
raise SystemExit(f"--seed must be non-negative; got {args.seed}.")
lo, hi = args.ic_min_references, args.ic_max_references
if lo < 1 or hi < lo:
raise SystemExit(
f"--ic-min-references/--ic-max-references must satisfy 1 <= min <= max; got {lo}/{hi}"
)
references = list(args.ic_reference or [])
if not (lo <= len(references) <= hi):
expected = f"exactly {lo}" if lo == hi else f"{lo}-{hi}"
raise SystemExit(
f"The upscale adapter needs {expected} --ic-reference clip(s); got {len(references)}"
)
for reference in references:
if not Path(reference).is_file():
raise SystemExit(f"--ic-reference does not exist: {reference}")
# A path, never a repo id. Both runtimes' LoRA resolvers fall back to a
# remote fetch for anything that is not an existing file, which for this
# gated adapter is a 401 deep inside a render — and for any repo, a pull
# PortOS never announced. The download surface owns every fetch.
if not Path(args.ic_lora_path).is_file():
raise SystemExit(
f"The Pixel Spatial Upscaler adapter is not on disk at {args.ic_lora_path} — "
"download it from the Video Gen model panel before upscaling."
)
def read_safetensors_header(path: str) -> "dict | None":
"""Parse a safetensors JSON header with the stdlib alone.
Deliberately not `safetensors.safe_open`: this runs during validation, in a
bare interpreter, before any runtime import — which is what keeps the whole
argument contract testable without an MLX or torch wheel. Returns None for a
missing, truncated, or non-safetensors file rather than raising, so the
caller states the refusal in its own words.
"""
try:
with open(path, "rb") as handle:
raw_len = handle.read(8)
if len(raw_len) < 8:
return None
header_len = struct.unpack("<Q", raw_len)[0]
if header_len <= 0 or header_len > MAX_HEADER_BYTES:
return None
raw = handle.read(header_len)
if len(raw) < header_len:
return None
parsed = json.loads(raw.decode("utf-8"))
return parsed if isinstance(parsed, dict) else None
except (OSError, ValueError, struct.error):
return None
def reference_downscale_factor(header: "dict | None") -> int:
"""The adapter's declared `reference_downscale_factor`, defaulting to 1.
Matches `read_lora_reference_downscale_factor` in both runtimes, which is
what the pipeline uses as the real value. Read here too so the resolution
rule can be stated BEFORE a multi-minute render commits to it — and so
PortOS can record the measured factor rather than the `null` its registry
honestly holds for a gated weight nobody has opened yet.
"""
metadata = (header or {}).get("__metadata__")
if not isinstance(metadata, dict):
return 1
try:
scale = int(metadata.get("reference_downscale_factor", 1))
except (TypeError, ValueError):
return 1
return scale if scale >= 1 else 1
def conditioned_stage_request(width: int, height: int) -> "tuple[int, int]":
"""The `(width, height)` a runner hands the pipeline so stage 1 renders at the output.
Paired with `skip_stage_2=True` on both runtimes — see
`PIPELINE_REQUEST_MULTIPLIER`. Kept as one helper so the two runners cannot
disagree about which stage the output comes from.
"""
return width * PIPELINE_REQUEST_MULTIPLIER, height * PIPELINE_REQUEST_MULTIPLIER
def assert_reference_scale_fits(scale: int, width: int, height: int) -> None:
"""Enforce the adapter's own resolution rule on the conditioned stage's dims.
`append_ic_lora_reference_video_conditionings` divides the dims it is HANDED
by the factor and snaps the result onto the VAE grid. With
`conditioned_stage_request` those dims ARE the output, so the reference is
`output / scale` — and for the upscaler that must be the (padded) source's
own size, or the source is resampled before it conditions anything. Hence
the output must divide by `scale * 32`. Stating the rule in OUTPUT terms is
what makes the message actionable — the user picked an output size.
"""
if scale <= 1:
return
required = scale * VAE_SPATIAL_COMPRESSION
if height % required == 0 and width % required == 0:
return
raise SystemExit(
f"This adapter downscales its reference by {scale}, so the output dimensions must be "
f"divisible by {required}; got {width}x{height}."
)
def lora_target_keys(names) -> "set[str]":
"""Base-weight keys an adapter can fuse into, from its ALREADY-RENAMED names.
Both loaders pair `<prefix>.lora_A.weight` with `<prefix>.lora_B.weight` and
skip a prefix missing either half (`_products_for_sd_key` on torch,
`_prepare_deltas` on MLX), so a half-pair contributes nothing and must not
count as coverage. Takes renamed names rather than raw ones on purpose: the
ComfyUI rename table belongs to the runtime, and copying it here would be a
second table free to drift from the one that actually fuses.
"""
prefixes = {half: set() for half in _LORA_HALVES}
for name in names:
for half, bucket in prefixes.items():
suffix = f".{half}.weight"
if isinstance(name, str) and name.endswith(suffix):
bucket.add(name[: -len(suffix)])
return {f"{prefix}.weight" for prefix in prefixes["lora_A"] & prefixes["lora_B"]}
def entry_shape(entry) -> "tuple[int, ...] | None":
"""One safetensors header entry's tensor shape, or None when it has no usable one.
Total on purpose: a header is untrusted input read before any runtime
validates it, so a malformed entry yields None and the caller decides
whether that is a refusal or simply a key it does not care about.
"""
shape = entry.get("shape") if isinstance(entry, dict) else None
if not isinstance(shape, list) or not all(isinstance(n, int) for n in shape):
return None
return tuple(shape)
def lora_delta_shapes(header: "dict | None", rename) -> "dict[str, tuple[int, int, int, int]]":
"""Per-target `(out_features, in_features, rank_a, rank_b)` for each fusable pair.
A safetensors header carries every tensor's shape, so the delta `B @ A` a
fusion would produce is fully described without reading one byte of tensor
data. That is what lets a runner check an adapter NUMERICALLY — against the
weights it claims to address — rather than settling for "the load raised
nothing", which on both runtimes it never would.
`rename` is the runtime's own key mapping; a key it rejects yields None and
is dropped, exactly as the loader drops it.
"""
renamed = {}
for raw, entry in (header or {}).items():
if raw == "__metadata__":
continue
name = rename(raw)
if isinstance(name, str):
renamed[name] = entry
shapes = {}
for target in lora_target_keys(renamed):
prefix = target[: -len(".weight")]
a = entry_shape(renamed.get(f"{prefix}.lora_A.weight"))
b = entry_shape(renamed.get(f"{prefix}.lora_B.weight"))
if a is None or b is None or len(a) != 2 or len(b) != 2:
continue
shapes[target] = (b[0], a[1], a[0], b[1])
return shapes