Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* LoRA service — install/list/delete + sidecar metadata management.
*
* Files live in `data/loras/<filename>.safetensors`; metadata lives next to
* them in `<filename>.metadata.json`. The sidecar is the source of truth for
* Civitai-derived info (trigger words, base model, recommended weight,
* preview image URL) — the .safetensors file alone has no such surface.
*
* Install flow: parse Civitai URL → fetch model metadata → pick version +
* primary .safetensors → stream-download to disk → write sidecar. The whole
* thing is one POST; progress is reported through the existing image-gen
* SSE channel (TBD — for v1 the client polls).
*
* No try/catch — errors bubble. `downloadToFile` Range-resumes a leftover
* `.partial` after a transport drop; a user-cancelled SSE disconnect still
* discards it. listLoras() filters `.partial` files out by extension.
*/
import { existsSync } from 'fs';
import { link, readFile, rename, stat } from 'fs/promises';
import { basename, join } from 'path';
import { ServerError } from '../lib/errorHandler.js';
import {
assessDownloadPreflight,
assertDownloadFits,
createDownloadSlot,
etagPathFor,
probeRemoteSize,
siblingDownloadMeta,
streamResumableDownload,
} from '../lib/downloadPreflight.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { atomicWrite, assertSafeFilename, ensureDir, listDirectoryByExtension, sha256File, PATHS, rmGuarded, unlinkGuarded } from '../lib/fileUtils.js';
import { verifySafetensorsStructure } from '../lib/hfCache.js';
import { isPlainObject } from '../lib/objects.js';
import { readCachedLoraEffectReport } from '../lib/loraEffect.js';
import { createKeyedFileWriteQueue } from '../lib/fileWriteQueue.js';
import { createSingleFlight } from '../lib/singleFlight.js';
import { loraInventoryRow } from '../lib/modelInventory.js';
import { recordModelInstall, recordModelUninstall } from './modelManifest.js';
import {
applyDownloadToken,
baseModelToRunner,
buildSidecar,
detectEarlyAccess,
fetchCivitaiModel,
flux2VariantFromBaseModel,
normalizeCivitaiImageUrl,
parseCivitaiUrl,
pickPrimaryFile,
pickVersion,
slugifyForFilename,
stillPreviewUrl,
} from '../lib/civitai.js';
import {
buildHfAuthHeaders,
buildHfLoraSidecar,
buildHfResolveUrl,
detectHfLoraFamily,
fetchHuggingfaceModel,
HF_LORA_FAMILIES,
parseHuggingfaceLoraRef,
pickHfLoraFile,
} from '../lib/huggingfaceLora.js';
import { RUNNER_FAMILIES, composeCompatKey } from '../lib/runners.js';
import {
classifyLoraKeyLayout,
classifyLoraKeyLayoutFromHeader,
detectFlux2VariantFromHeader,
isKnownLoraKeyLayout,
readSafetensorsHeader,
} from '../lib/safetensors.js';
import { getHfToken } from './hfToken.js';
import { getSettings } from './settings.js';
const SIDECAR_SUFFIX = '.metadata.json';
// listLoras() is called from several picker/setup paths. Cache the fully
// normalized public entry by the weight file's path plus weight/sidecar mtimes
// so repeated reads only pay the directory/stat pass, not one sidecar read (and
// potentially one safetensors header parse) per installed LoRA.
const loraMetadataCache = new Map();
// One install attempt per requested Civitai model version at a time. A duplicate
// submit shares the original result instead of racing it to the same destination
// files, while two explicitly different versions remain independent.
const civitaiInstalls = createSingleFlight();
// Keyed on the resolved destination path — what `isAnyDownloadInFlight` (and the
// orphaned-partial GC sweep) asks about, and what a second install of the same
// file would otherwise race. Not `exclusive`: LoRA files are small relative to a
// checkpoint, so installing several at once is normal, not a hazard. Cancel
// discards the `.partial` (`keepPartialOnCancel` defaults false) — a single
// abandoned file, not a multi-shard checkpoint worth resuming.
const downloadSlot = createDownloadSlot({ codePrefix: 'LORA' });
const sidecarPath = (loraFilename) => join(PATHS.loras, `${loraFilename}${SIDECAR_SUFFIX}`);
const invalidateLoraMetadataCache = (filename) => {
loraMetadataCache.delete(join(PATHS.loras, filename));
};
// Reads the sidecar JSON next to a LoRA file. Returns `null` when the
// sidecar is absent — calling code can fall back to filename inference for
// legacy LoRAs the user dropped in manually pre-Civitai. Permissions /
// I/O / parse errors get logged so an unreadable sidecar doesn't masquerade
// as a "legacy LoRA" in the manager UI.
export const readSidecar = async (filename) => {
const path = sidecarPath(filename);
const raw = await readFile(path, 'utf-8').catch((err) => {
if (err?.code === 'ENOENT') return null;
console.log(`⚠️ LoRA sidecar unreadable [${filename}]: ${err?.code || err?.message || err}`);
return null;
});
if (raw == null) return null;
let parsed = null;
try { parsed = JSON.parse(raw); } catch (err) {
console.log(`⚠️ LoRA sidecar malformed JSON [${filename}]: ${err?.message || err}`);
return null;
}
return isPlainObject(parsed) ? parsed : null;
};
// Read the `triggerWords` of several LoRAs at once, keyed by basename — the
// input to the render-time trigger weave (`lib/loraTriggers.js`, #4665).
// `filenames` may mix bare basenames (the current client surface) and absolute
// paths (legacy sidecar replays); both collapse to the basename key. A LoRA
// with no sidecar, or a legacy sidecar predating `triggerWords`, simply gets no
// entry, so the weave is a no-op for it rather than an error.
export const readTriggerWordsByFilename = async (filenames) => {
const names = [...new Set(
(Array.isArray(filenames) ? filenames : [])
.filter((f) => typeof f === 'string' && f)
.map((f) => basename(f)),
)];
if (!names.length) return {};
const sidecars = await Promise.all(names.map((n) => readSidecar(n)));
return Object.fromEntries(
names.map((n, i) => [n, sidecars[i]?.triggerWords]).filter(([, words]) => Array.isArray(words)),
);
};
// License + source URL as recorded on the sidecar at install time (#5638).
// Missing/legacy sidecars contribute `{ license: null, sourceUrl: null }` so
// a render still stamps the LoRA id with unknown terms rather than skipping it.
export const readLoraLicensesByFilename = async (filenames) => {
const names = [...new Set(
(Array.isArray(filenames) ? filenames : [])
.filter((f) => typeof f === 'string' && f)
.map((f) => basename(f)),
)];
if (!names.length) return {};
const sidecars = await Promise.all(names.map((n) => readSidecar(n)));
return Object.fromEntries(names.map((n, i) => {
const sc = sidecars[i] || {};
const sourceUrl = sc.civitai?.url || sc.huggingface?.url || null;
const license = typeof sc.license === 'string' && sc.license.trim() ? sc.license.trim() : null;
return [n, { license, sourceUrl, name: sc.name || null }];
}));
};
// Validate a basename so it can't escape PATHS.loras. Delegates to the
// shared `assertSafeFilename` helper in fileUtils.js (which also handles
// gallery .png assertions). Substring `..` is allowed because
// slugifyForFilename can produce names like `foo..bar` from non-ASCII input.
// `requiredMessage` preserves the historical "Filename required" wording
// (instead of the subject-derived "LoRA filename required") so any client
// or test that pattern-matches on the missing-input message keeps working.
export const assertSafeLoraFilename = (filename) => {
assertSafeFilename(filename, {
extensions: ['.safetensors'],
subject: 'LoRA filename',
requiredMessage: 'Filename required',
});
};
// LoRAs without sidecars get a minimal "legacy" entry with sensible defaults
// so the manager UI can still render LoRAs the user dropped in pre-Civitai.
export const listLoras = async () => {
if (!existsSync(PATHS.loras)) {
loraMetadataCache.clear();
return [];
}
const lorasStat = await stat(PATHS.loras).catch(() => null);
if (!lorasStat || !lorasStat.isDirectory()) {
loraMetadataCache.clear();
console.log(`⚠️ PATHS.loras exists but is not a directory: ${PATHS.loras}`);
return [];
}
// listDirectoryByExtension handles the readdir + extension filter + per-
// entry stat + isFile check (so directories named `foo.safetensors` are
// dropped before deleteLora would later trip on EISDIR).
const out = await listDirectoryByExtension(PATHS.loras, {
extensions: ['.safetensors'],
mapEntry: async (filename, _fullPath, s) => {
const sidecarStat = await stat(sidecarPath(filename)).catch(() => null);
const cached = loraMetadataCache.get(_fullPath);
if (cached?.mtimeMs === s.mtimeMs && cached?.sidecarMtimeMs === sidecarStat?.mtimeMs) {
return cached.entry;
}
const sidecar = await readSidecar(filename);
const fallbackName = filename.replace(/^lora-/, '').replace(/\.safetensors$/, '');
// Re-derive runnerFamily from civitai.baseModel at read time so
// sidecars written before a baseModelToRunner() mapping update (e.g.
// an install before 'Ernie' was a recognized base) don't permanently
// show as runnerFamily=null and leak across compat filters. Falls
// back to the stored value for legacy LoRAs without civitai metadata.
const baseModel = sidecar?.civitai?.baseModel;
const runnerFamily = baseModel
? baseModelToRunner(baseModel)
: (sidecar?.runnerFamily || null);
// Fine-grained FLUX.2 size variant ('4b'/'9b'). Resolve in order:
// stored sidecar value → Civitai baseModel string → the safetensors
// header (ground truth for self-trained / hand-dropped LoRAs with no
// Civitai metadata). Persist a header-detected value back so the header
// is read at most once per LoRA. Mirrors the "re-derive runnerFamily on
// read" healing above.
//
// Both header-derived fields below share ONE lazy header read (a LoRA
// whose sidecar already answers never opens the file), and they heal the
// sidecar through a SINGLE patch — two fire-and-forget patches would
// read-modify-write the same sidecar concurrently and drop one field.
let headerPromise = null;
const readHeaderOnce = () => {
if (!headerPromise) headerPromise = readSafetensorsHeader(_fullPath);
return headerPromise;
};
const backfill = {};
let fluxVariant = null;
if (runnerFamily === RUNNER_FAMILIES.FLUX2) {
fluxVariant = sidecar?.fluxVariant || flux2VariantFromBaseModel(baseModel);
if (!fluxVariant) {
fluxVariant = detectFlux2VariantFromHeader(await readHeaderOnce());
if (fluxVariant) backfill.fluxVariant = fluxVariant;
}
}
// Safetensors key layout ('bare'/'comfyui'/'diffusers'/'kohya'/
// 'not_a_lora'). Backfilled for LoRAs installed before installs recorded
// it. `null` means "couldn't read the header", which we deliberately
// don't persist: it's not a classification, and a later read may succeed.
// Validate the stored value: a sidecar hand-edited (or written by a
// future version that knows a layout this one doesn't) must fall back to
// re-classifying, not propagate a string nothing downstream understands.
let keyLayout = isKnownLoraKeyLayout(sidecar?.keyLayout) ? sidecar.keyLayout : null;
if (!keyLayout) {
keyLayout = classifyLoraKeyLayoutFromHeader(await readHeaderOnce());
if (keyLayout) backfill.keyLayout = keyLayout;
}
// Best-effort cache — never block or fail the list on a write error.
if (Object.keys(backfill).length > 0) patchLoraSidecar(filename, backfill).catch(() => {});
// The picker matches this against the selected model's `loraCompatKey`.
// FLUX.2 → size-specific (or bare 'flux2' when size is unknown, so it
// still shows for both sizes); every other family → its runner id.
const loraCompatKey = composeCompatKey(runnerFamily, fluxVariant);
const entry = {
filename,
name: sidecar?.name || fallbackName,
sizeBytes: s.size,
installedAt: sidecar?.installedAt || s.birthtime?.toISOString?.() || null,
// sidecar fields surfaced for the picker / manager UI:
civitai: sidecar?.civitai || null,
// HF-installed video LoRAs carry this instead of `civitai` — the
// manager matches curated video suggestions to installs by repo and
// links the card out to HuggingFace.
huggingface: sidecar?.huggingface || null,
runnerFamily,
fluxVariant,
keyLayout,
loraCompatKey,
triggerWords: sidecar?.triggerWords || [],
// Coerce non-finite values (NaN, Infinity, missing/malformed sidecar
// fields) to the default — `?? 1.0` alone wouldn't catch NaN.
recommendedScale: Number.isFinite(sidecar?.recommendedScale) ? sidecar.recommendedScale : 1.0,
// Normalize on read so already-installed LoRAs (sidecars written
// before the URL-normalize fix) also benefit without a reinstall.
// stillPreviewUrl drops a video URL for the same reason: Civitai's
// media list mixes clips in, and the card renders this in an <img>.
previewImageUrl: normalizeCivitaiImageUrl(stillPreviewUrl(sidecar?.previewImageUrl)) || null,
description: sidecar?.description || '',
license: typeof sidecar?.license === 'string' && sidecar.license.trim() ? sidecar.license.trim() : null,
// Trained-LoRA surfacing (sidecars written by services/loraTraining):
// 'trained' source + the character identity block let the library UI
// badge them and characterLoraResolver match them. Null for Civitai
// installs and legacy drops.
source: sidecar?.source || null,
character: sidecar?.character || null,
trainedFromDatasetId: sidecar?.datasetId || null,
// Adapter-effect diagnostic (#4872). CACHED ONLY — listing the library
// must never fan out into one Python child per installed LoRA, so this
// surfaces whatever the explicit probe last measured and drops it when
// the file has changed size underneath it. `null` = never measured (or
// measured against a different file), which the UI shows as an offer to
// run the check, not as a verdict.
effectReport: readCachedLoraEffectReport(sidecar?.effectReport, { sizeBytes: s.size, mtimeMs: s.mtimeMs }),
};
loraMetadataCache.set(_fullPath, {
mtimeMs: s.mtimeMs,
sidecarMtimeMs: sidecarStat?.mtimeMs,
entry,
});
return entry;
},
});
const listedPaths = new Set(out.map(({ filename }) => join(PATHS.loras, filename)));
for (const cachedPath of loraMetadataCache.keys()) {
if (!listedPaths.has(cachedPath)) loraMetadataCache.delete(cachedPath);
}
return out.sort((a, b) => (b.installedAt || '').localeCompare(a.installedAt || ''));
};
export const getLora = async (filename) => {
assertSafeLoraFilename(filename);
const fullPath = join(PATHS.loras, filename);
if (!existsSync(fullPath)) {
throw new ServerError(`LoRA not found: ${filename}`, { status: 404, code: 'NOT_FOUND' });
}
const list = await listLoras();
const lora = list.find((l) => l.filename === filename);
if (!lora) {
// File exists on disk but listLoras filtered it out — most likely because
// it's not a regular .safetensors file (directory, symlink, etc.).
throw new ServerError(
`LoRA "${filename}" exists but is not a valid regular .safetensors file`,
{ status: 404, code: 'NOT_FOUND' },
);
}
return lora;
};
// Run, dataset and character rows can keep naming a deleted LoRA, so the
// unlink holds backup admission: it waits out an open cut, and a cut waits out
// an unlink in flight (#9982).
export const deleteLora = async (filename) => {
assertSafeLoraFilename(filename);
const filePath = join(PATHS.loras, filename);
await withBackupAssetPublication(() => queueSidecarWrite(filename, async () => {
if (!existsSync(filePath)) {
throw new ServerError(`LoRA not found: ${filename}`, { status: 404, code: 'NOT_FOUND' });
}
const s = await stat(filePath).catch(() => null);
if (!s || !s.isFile()) {
throw new ServerError(
`Cannot delete "${filename}": not a regular file`,
{ status: 400, code: 'INVALID_LORA_FILE' },
);
}
// Capture metadata before removing it so a later model-unlink failure can
// restore the usable pair. readFile also fails before model removal when a
// malformed sidecar is a directory or otherwise unreadable.
const metadataPath = sidecarPath(filename);
const metadata = await readFile(metadataPath).then(
(contents) => contents,
(err) => (err.code === 'ENOENT' ? null : Promise.reject(err)),
);
if (metadata !== null) await rmGuarded(metadataPath);
const modelRemovalError = await rmGuarded(filePath).then(() => null, (err) => err);
if (modelRemovalError) {
if (metadata !== null) await atomicWrite(metadataPath, metadata);
throw modelRemovalError;
}
invalidateLoraMetadataCache(filename);
}));
await recordModelUninstall({ backend: 'lora', key: filename });
console.log(`🗑️ Deleted LoRA: ${filename}`);
return { ok: true, filename };
};
// Patch the sidecar with user-editable fields (name, recommendedScale, notes).
// Civitai-derived fields are passed through but the route layer scopes the
// patch so callers can't trample those.
// One sidecar write at a time per LoRA. A patch is a read-modify-write of a
// whole JSON document, and there are several writers: the user renaming a LoRA
// in the manager, listLoras() healing keyLayout/fluxVariant on read, the effect
// probe caching its measurement when it finishes, and the two installers
// replacing the sidecar wholesale. Interleaved, the last write wins the whole
// file and silently drops the other's field — a re-install over an existing
// filename could have its fresh civitai block overwritten by a patch that read
// the pre-install sidecar. EVERY writer goes through here, not just patches, or
// the serialization is a half-measure. Keyed, so different LoRAs still run in
// parallel.
const queueSidecarWrite = createKeyedFileWriteQueue();
// Replace a sidecar wholesale (the install paths), serialized against patches.
//
// This is also where the model manifest learns about a new adapter: all three
// installers (Civitai, Hugging Face, and a finished training run) finish by
// writing the sidecar wholesale, and none of them shares another step. Recording
// here rather than in each installer is what keeps a fourth one from arriving
// untracked — and a re-install over an existing filename keeps its original
// `installedAt`, so it does not read as newly downloaded.
//
// The sidecar is the record that first names freshly installed or trained
// weights, which are already on disk under their final name, so writing it is
// the commit that takes backup admission (#9982).
export const writeLoraSidecar = (filename, sidecar) => withBackupAssetPublication(() => queueSidecarWrite(filename, async () => {
const loraPath = join(PATHS.loras, filename);
if (!existsSync(loraPath)) {
throw new ServerError(`LoRA not found: ${filename}`, { status: 404, code: 'NOT_FOUND' });
}
await atomicWrite(sidecarPath(filename), JSON.stringify(sidecar, null, 2) + '\n');
invalidateLoraMetadataCache(filename);
await recordModelInstall(loraInventoryRow({
filename,
name: sidecar?.name || filename.replace(/^lora-/, '').replace(/\.safetensors$/, ''),
// Measured, not claimed: the Civitai sidecar carries the API's `sizeKB` and the
// HF one carries nothing, so the file itself is the only uniform source.
sizeBytes: await stat(loraPath).then((info) => info.size, () => null),
}));
}));
export const patchLoraSidecar = async (filename, patch) => {
assertSafeLoraFilename(filename);
const filePath = join(PATHS.loras, filename);
// The read has to be INSIDE the queued cycle — reading before joining the
// queue would merge against a snapshot the previous write already superseded.
return queueSidecarWrite(filename, async () => {
if (!existsSync(filePath)) {
throw new ServerError(`LoRA not found: ${filename}`, { status: 404, code: 'NOT_FOUND' });
}
// Match getLora/deleteLora: refuse non-regular files (directory named
// foo.safetensors, dangling symlink, etc.) so we don't quietly create a
// sidecar for an entry that listLoras() will then filter out.
const s = await stat(filePath).catch(() => null);
if (!s || !s.isFile()) {
throw new ServerError(
`Cannot patch "${filename}": not a regular file`,
{ status: 400, code: 'INVALID_LORA_FILE' },
);
}
const current = (await readSidecar(filename)) || { filename };
const next = { ...current, ...patch, filename };
await atomicWrite(sidecarPath(filename), JSON.stringify(next, null, 2) + '\n');
invalidateLoraMetadataCache(filename);
return next;
});
};
// Stamp the classified safetensors key layout onto a freshly-built sidecar so
// consumers (the video-LoRA fusion gate) never have to re-read the header.
// Omits the field entirely when classification fails — a `null` there would be
// indistinguishable from "already classified as nothing" and would defeat the
// heal-on-read backfill in listLoras().
const withKeyLayout = async (sidecar, destPath) => {
const keyLayout = await classifyLoraKeyLayout(destPath);
return keyLayout ? { ...sidecar, keyLayout } : sidecar;
};
/**
* Resolve one LoRA's safetensors key layout: the sidecar value when present,
* else classified from the file header. Returns `null` when the header can't
* be read (the "couldn't determine" sentinel — NOT `'not_a_lora'`).
*
* Deliberately does not persist a header-derived value: the callers are render
* hot paths, and listLoras() already owns the backfill-and-cache duty.
*/
export const getLoraKeyLayout = async (filename) => {
const sidecar = await readSidecar(filename);
if (isKnownLoraKeyLayout(sidecar?.keyLayout)) return sidecar.keyLayout;
return classifyLoraKeyLayout(join(PATHS.loras, filename));
};
// Resolve the active Civitai API key — either from settings (`civitai.apiKey`)
// or the CIVITAI_API_KEY env var. Settings wins so a user can override the
// env without restarting. Returns empty string for "no key", which the
// downstream helpers treat as anonymous.
export const resolveCivitaiKey = async () => {
const env = (process.env.CIVITAI_API_KEY || '').trim();
// getSettings reads the JSON every call; cheap and avoids stale-cache bugs.
const s = await getSettings();
const fromSettings = (s?.civitai?.apiKey || '').trim();
return fromSettings || env || '';
};
// Stream-download a URL to `${destPath}.partial` then link/rename into place.
// A leftover `.partial` is Range-resumed on retry; a user-cancelled SSE
// disconnect still discards it. The `.partial` suffix keeps listLoras from
// picking up a half-written file. fetchImpl is injectable for tests.
// `onProgress({ received, total })` (optional) fires with byte counts during
// the stream download — the manager's streaming install endpoint forwards these
// as SSE `progress` frames so the UI can show a percentage. `total` is 0 when
// the response carries no Content-Length (chunked / some CDN redirects), in
// which case the client renders an indeterminate bar.
const downloadToFile = async (url, destPath, { fetchImpl = fetch, headers = {} , hasApiKey = false, source = 'civitai', onProgress = null, signal = null } = {}) => {
const onHttpError = (res) => {
if (res.status === 401 || res.status === 403) {
if (source === 'huggingface') {
const message = hasApiKey
? `HuggingFace rejected the download (${res.status}) even with your saved token — accept the model's license on its HF page, or the token may be expired/scoped.`
: `HuggingFace download rejected (${res.status}) — this LoRA's repo is gated. Accept its license on HuggingFace and add your HF token in Image Gen settings, then retry.`;
throw new ServerError(message, { status: res.status, code: 'HF_AUTH' });
}
const message = hasApiKey
? `Civitai rejected the download (${res.status}) even with your saved API key. The LoRA is likely in early-access (Civitai supporters only) or your key has expired/been revoked.`
: `Civitai download rejected (${res.status}) — this LoRA may require an API key. Configure a Civitai API key in PortOS Settings (or set the CIVITAI_API_KEY env var) and retry.`;
throw new ServerError(message, { status: res.status, code: 'CIVITAI_AUTH' });
}
const label = source === 'huggingface' ? 'HuggingFace' : 'Civitai';
const code = source === 'huggingface' ? 'HF_DOWNLOAD_FAILED' : 'CIVITAI_DOWNLOAD_FAILED';
throw new ServerError(`${label} download failed: ${res.status} ${res.statusText}`, { status: 502, code });
};
// Claim before the first await: two presses landing inside the same tick
// would otherwise both pass the caller's existsSync pre-check and start a
// parallel transfer of the same file. Released in the finally below.
const slot = downloadSlot.claim(destPath, {
busyMessage: `${basename(destPath)} is already downloading`,
});
// An external abort signal (SSE client-disconnect) cancels through the slot
// rather than its own controller, so the typed CANCELLED/STALLED error from
// `wrapError` below applies uniformly regardless of who triggered the abort.
const onExternalAbort = () => downloadSlot.cancel(destPath);
if (signal) {
if (signal.aborted) onExternalAbort();
else signal.addEventListener('abort', onExternalAbort, { once: true });
}
const emitProgress = onProgress
? slot.throttle((received, total) => onProgress({ received, total }))
: null;
try {
let lastTick = { received: 0, total: 0 };
const { tmpPath } = await streamResumableDownload({
url,
destPath,
headers,
fetchImpl,
finalize: false,
onHttpError,
onBytes: (received, total) => {
lastTick = { received, total };
slot.track(received, total);
if (emitProgress) emitProgress(received, total);
},
...slot.downloadOptions(),
});
// `finalize: false` means streamResumableDownload never had a chance to
// clean up its own etag sidecar (that only happens on ITS finalize path).
// Reaching here means the stream completed successfully — every branch
// below either moves or deletes tmpPath outright, never leaves it for a
// future resume, so the sidecar describing it is equally done.
await rmGuarded(etagPathFor(destPath), { force: true }).catch(() => {});
if (onProgress) onProgress(lastTick);
// Atomic no-clobber finalize: `link` is POSIX-atomic and fails with EEXIST
// when destPath already exists (concurrent install that snuck past our
// pre-check). On success we unlink the tmp; on EEXIST we clean up and
// throw CIVITAI_ALREADY_INSTALLED. For other link errors (cross-device
// EXDEV, read-only fs, etc.) fall back to rename, which is the only
// portable option on those platforms.
const linkErr = await link(tmpPath, destPath).catch((e) => e);
if (!linkErr) {
await unlinkGuarded(tmpPath).catch(() => {});
return;
}
if (linkErr.code === 'EEXIST') {
await rmGuarded(tmpPath, { force: true }).catch(() => {});
const basename_ = basename(destPath);
throw new ServerError(
`Already installed: ${basename_}. Delete it first or pick a different version.`,
{ status: 409, code: 'CIVITAI_ALREADY_INSTALLED' },
);
}
// EXDEV or similar — fall back to rename. Re-check destPath right before
// the rename so a concurrent install that landed between our link attempt
// and now can't be silently clobbered (POSIX rename overwrites). Treat
// late-arriving dest as CIVITAI_ALREADY_INSTALLED, matching the EEXIST
// path above.
if (existsSync(destPath)) {
await rmGuarded(tmpPath, { force: true }).catch(() => {});
const basename_ = basename(destPath);
throw new ServerError(
`Already installed: ${basename_}. Delete it first or pick a different version.`,
{ status: 409, code: 'CIVITAI_ALREADY_INSTALLED' },
);
}
await rename(tmpPath, destPath).catch(async (err) => {
await rmGuarded(tmpPath, { force: true }).catch(() => {});
throw err;
});
} catch (err) {
throw slot.wrapError(err);
} finally {
if (signal) signal.removeEventListener('abort', onExternalAbort);
slot.release();
}
};
// After a LoRA finishes downloading, verify the on-disk `.safetensors` before
// we commit to it by writing the sidecar. A truncated (short download) or
// right-size-but-wrong-bytes file otherwise installs silently and renders
// garbage ("mosaic" output) at generate time — the leading non-Metal cause of
// corrupt output (issue #2199; mirrors the HF-cache integrity check in
// hfCache.js / #1324). Always runs the cheap structural header/size check
// (reads only the header region, never the multi-GB payload); adds a deep
// sha256 compare only when the source metadata carried a digest (Civitai
// `file.hashes.SHA256`). On any failure the file is deleted so the next install
// re-downloads instead of training/rendering against corrupt weights.
const verifyDownloadedLora = async (destPath, { expectedSha256 = null, source = 'civitai' } = {}) => {
const label = source === 'huggingface' ? 'HuggingFace' : 'Civitai';
const code = source === 'huggingface' ? 'HF_LORA_CORRUPT' : 'CIVITAI_LORA_CORRUPT';
const st = await stat(destPath).catch(() => null);
const structural = await verifySafetensorsStructure(destPath, st?.size ?? 0);
if (!structural.ok) {
await rmGuarded(destPath, { force: true }).catch(() => {});
throw new ServerError(
`${label} LoRA download is corrupt (${structural.reason}) — the partial file was deleted. Retry the install.`,
{ status: 502, code },
);
}
// Civitai hashes are uppercase hex; sha256File returns lowercase — compare
// case-insensitively and only when the digest is a well-formed sha256.
const want = typeof expectedSha256 === 'string' ? expectedSha256.trim().toLowerCase() : '';
if (/^[0-9a-f]{64}$/.test(want)) {
const actual = await sha256File(destPath).catch(() => null);
if (actual && actual.toLowerCase() !== want) {
await rmGuarded(destPath, { force: true }).catch(() => {});
throw new ServerError(
`${label} LoRA failed SHA-256 verification (expected ${want.slice(0, 12)}…, got ${actual.slice(0, 12)}…) — the file was deleted. Retry the install.`,
{ status: 502, code },
);
}
}
};
// Install a LoRA from a Civitai URL. Returns the new sidecar JSON so the
// client can render it immediately without a second list round-trip.
// Civitai's `type` casing varies in the wild (LORA / Lora / lora) and the
// family includes LoCon / LyCORIS / DoRA / LoHA — all of which load through
// diffusers' lora pipeline. Refuse only true non-LoRA checkpoints.
const ALLOWED_CIVITAI_LORA_TYPES = new Set(['lora', 'locon', 'lycoris', 'dora', 'loha']);
// Shared by performCivitaiInstall and previewCivitaiInstall so a preview can
// never promise a download the install would actually refuse — early-access
// gating, the LoRA-type check, and the missing-downloadUrl guard all throw
// from here now, instead of only guarding the real transfer.
const resolveCivitaiInstallPlan = async (input, { modelId, versionId, fetchImpl }) => {
const apiKey = (typeof input?.apiKey === 'string' && input.apiKey.trim()) || (await resolveCivitaiKey());
const model = await fetchCivitaiModel(modelId, { apiKey, fetchImpl });
const version = pickVersion(model, versionId);
// Refuse early-access versions up front. The download endpoint would
// 401 even with a valid API key (only Civitai supporters can download
// during the early-access window), and routing the user into the
// "set API key" modal is misleading because their key isn't the issue.
const ea = detectEarlyAccess(version);
if (ea.early) {
const when = ea.hoursRemaining != null
? (ea.hoursRemaining < 24
? `~${ea.hoursRemaining}h`
: `~${Math.round(ea.hoursRemaining / 24)}d`)
: 'soon';
throw new ServerError(
`"${model.name}" v${version.id} is in Civitai early-access — only Civitai supporters can download it for ${when} more${ea.endsAt ? ` (until ${ea.endsAt})` : ''}. Try again once it goes public.`,
{ status: 403, code: 'CIVITAI_EARLY_ACCESS' },
);
}
const file = pickPrimaryFile(version);
if (!file?.downloadUrl) {
throw new ServerError(
`Civitai version ${version?.id} has no downloadUrl — try selecting a different version`,
{ status: 422, code: 'CIVITAI_NO_DOWNLOAD' },
);
}
if (model?.type && !ALLOWED_CIVITAI_LORA_TYPES.has(String(model.type).toLowerCase())) {
throw new ServerError(
`Civitai model "${model.name}" is type "${model.type}", not a LoRA — refusing to install`,
{ status: 400, code: 'CIVITAI_NOT_A_LORA' },
);
}
// Build a stable filename: `lora-<slug>-<versionId>.safetensors`. The
// versionId suffix prevents collisions if a user installs two versions of
// the same model. The `lora-` prefix keeps it distinguishable from base
// model weights if they ever coexist in the same dir.
const slug = slugifyForFilename(model.name || file.name?.replace(/\.safetensors$/i, ''));
const filename = `lora-${slug}-v${version.id}.safetensors`;
const destPath = join(PATHS.loras, filename);
const expectedBytes = Number(file.sizeKB) > 0 ? Number(file.sizeKB) * 1024 : 0;
return { apiKey, model, version, file, filename, destPath, expectedBytes };
};
const performCivitaiInstall = async (input, { modelId, versionId, fetchImpl }) => {
const plan = await resolveCivitaiInstallPlan(input, { modelId, versionId, fetchImpl });
if (existsSync(plan.destPath)) {
throw new ServerError(
`Already installed: ${plan.filename}. Delete it first or pick a different version.`,
{ status: 409, code: 'CIVITAI_ALREADY_INSTALLED' },
);
}
await ensureDir(PATHS.loras);
assertDownloadFits(await assessDownloadPreflight({ destPath: plan.destPath, expectedBytes: plan.expectedBytes }));
console.log(`📥 Installing Civitai LoRA: ${plan.model.name} v${plan.version.id} → ${plan.filename} (${plan.file.sizeKB ? Math.round(plan.file.sizeKB / 1024) + ' MB' : 'size unknown'})`);
// Authenticate downloads via `?token=` only — the Authorization header
// doesn't survive the 302 to CDN, AND sending both means the token also
// ends up in CDN access logs. The metadata fetch (fetchCivitaiModel)
// still uses the header since /api/v1/* doesn't redirect.
const tokenized = applyDownloadToken(plan.file.downloadUrl, plan.apiKey);
await downloadToFile(tokenized, plan.destPath, {
fetchImpl,
headers: { 'User-Agent': 'PortOS/civitai-installer' },
hasApiKey: !!plan.apiKey,
});
await verifyDownloadedLora(plan.destPath, { expectedSha256: plan.file?.hashes?.SHA256 || null, source: 'civitai' });
const sidecar = await withKeyLayout(
buildSidecar({ model: plan.model, version: plan.version, file: plan.file, filename: plan.filename }),
plan.destPath,
);
await writeLoraSidecar(plan.filename, sidecar);
console.log(`✅ Installed Civitai LoRA: ${plan.filename} [layout=${sidecar.keyLayout || 'unknown'}]`);
return sidecar;
};
export const installFromCivitai = async (input, { fetchImpl = fetch } = {}) => {
const { modelId, versionId } = parseCivitaiUrl(input?.url);
const installKey = `${modelId}:${versionId ?? 'latest'}`;
return civitaiInstalls.run(
installKey,
() => performCivitaiInstall(input, { modelId, versionId, fetchImpl }),
);
};
/** Size / dest / free-disk numbers for the LoRA confirm step — no transfer. */
export const previewCivitaiInstall = async (input, { fetchImpl = fetch } = {}) => {
const { modelId, versionId } = parseCivitaiUrl(input?.url);
const plan = await resolveCivitaiInstallPlan(input, { modelId, versionId, fetchImpl });
const preflight = await assessDownloadPreflight({ destPath: plan.destPath, expectedBytes: plan.expectedBytes });
return {
kind: 'civitai',
...preflight,
destPath: plan.filename,
alreadyDownloaded: existsSync(plan.destPath),
};
};
// Set of recognized LoRA families an HF import may target (image runners +
// video families). Used to validate a user-supplied `family` override.
const HF_LORA_FAMILY_VALUES = new Set(HF_LORA_FAMILIES);
// Shared by installFromHuggingface and previewHuggingfaceInstall so a preview
// can never show a different file/filename than the install it precedes.
// `family` is resolved (override → autodetection → null) but NOT required —
// it only narrows `pickHfLoraFile`'s pick among flux2 variant files; the
// generated filename depends only on `repo` + the picked `file`. Preview
// therefore never throws HF_UNKNOWN_FAMILY: that check belongs to the actual
// install, which needs a concrete family for the sidecar metadata.
// Neither caller threads a cancellation signal into this metadata lookup
// (only the actual byte-stream download gets one) — a stalled-but-reachable
// HF would otherwise hang a preview, or an install, indefinitely on this
// call alone.
const METADATA_FETCH_TIMEOUT_MS = 10_000;
const resolveHfLoraInstallPlan = async (input, { fetchImpl = fetch } = {}) => {
const { repo, revision, file: parsedFile } = parseHuggingfaceLoraRef(input?.url);
// Stored/env/CLI HF token — only needed for gated repos, but harmless to
// send on public ones (HF ignores a bearer it doesn't require).
const token = (typeof input?.token === 'string' && input.token.trim()) || (await getHfToken()) || '';
const model = await fetchHuggingfaceModel(repo, {
token, revision, fetchImpl, signal: AbortSignal.timeout(METADATA_FETCH_TIMEOUT_MS),
});
const preferredFile = input?.file || parsedFile || null;
const detected = detectHfLoraFamily({ repo, model, file: preferredFile });
// Family: explicit override (validated by the caller) wins over autodetection.
const family = input?.family || detected?.family || null;
const file = pickHfLoraFile(
model,
preferredFile,
{ family, fluxVariant: family === RUNNER_FAMILIES.FLUX2 ? (detected?.fluxVariant || null) : null },
);
const refined = preferredFile ? detected : detectHfLoraFamily({ repo, model, file });
const fluxVariant = family === RUNNER_FAMILIES.FLUX2
? (refined?.fluxVariant || detected?.fluxVariant || null)
: null;
// Stable filename: `lora-<org>-<name>[-<file-variant>]-hf.safetensors`.
// Canonical single-file names (pytorch_lora_weights / lora) keep the original
// repo-only contract. Any other stem (CharacterSheet's QuadView_klein9b_v1,
// a `_v2` artifact) gets a distinct suffix so siblings can coexist.
const slug = slugifyForFilename(repo.replace('/', '-'));
const repoNameSlug = slugifyForFilename((repo.split('/')[1] || repo).replace(/_/g, '-'));
const pickedStem = file.replace(/\.safetensors$/i, '').split('/').pop();
const fileStemSlug = slugifyForFilename(pickedStem.replace(/_/g, '-'));
const isCanonicalStem = /^(pytorch_lora_weights|lora)$/i.test(pickedStem);
const explicitVariant = isCanonicalStem
? ''
: (fileStemSlug.startsWith(`${repoNameSlug}-`)
? fileStemSlug.slice(repoNameSlug.length + 1)
: (fileStemSlug === repoNameSlug ? '' : fileStemSlug));
const filename = `lora-${slug}${explicitVariant ? `-${explicitVariant}` : ''}-hf.safetensors`;
const destPath = join(PATHS.loras, filename);
const hfSibling = (Array.isArray(model?.siblings) ? model.siblings : []).find((row) => row?.rfilename === file);
// The model-metadata response never carries per-sibling sizes without the
// `blobs=true` expand we deliberately skip (see fetchHuggingfaceModel) —
// siblingDownloadMeta is ~always 0 here. Without a real number the preflight
// can never refuse an oversized LoRA, so probe the resolved file directly
// (same fallback specDecodeModels.js already uses for GGUF weights).
let expectedBytes = siblingDownloadMeta(hfSibling).bytes;
if (!expectedBytes) {
const probed = await probeRemoteSize(buildHfResolveUrl(repo, revision, file), {
headers: buildHfAuthHeaders(token),
fetchImpl,
signal: AbortSignal.timeout(METADATA_FETCH_TIMEOUT_MS),
});
expectedBytes = probed.bytes;
}
return { repo, revision, token, model, family, fluxVariant, file, filename, destPath, expectedBytes };
};
// Install a LoRA from a HuggingFace repo (Flux.2 Klein image adapters, fal /
// Lightricks LTX video LoRAs, MiniMax H3). Mirrors installFromCivitai: parse
// the ref → fetch repo metadata → pick the .safetensors → stream-download →
// write the sidecar. The family is auto-detected from the repo id / tags /
// base_model / filenames, or taken from an explicit `input.family` override
// (validated against the known image + video families). Returns the new sidecar.
export const installFromHuggingface = async (input, { fetchImpl = fetch, onProgress = null, signal = null } = {}) => {
// An unrecognized override is refused before the network round trip — a
// wrongly-tagged LoRA would surface under a model it can't actually load.
if (input?.family && !HF_LORA_FAMILY_VALUES.has(input.family)) {
throw new ServerError(
`Unknown LoRA family "${input.family}" — expected one of ${[...HF_LORA_FAMILY_VALUES].join(', ')}`,
{ status: 400, code: 'HF_BAD_FAMILY' },
);
}
const plan = await resolveHfLoraInstallPlan(input, { fetchImpl });
if (!plan.family) {
throw new ServerError(
`Couldn't determine a supported model for HuggingFace repo "${plan.repo}". PortOS can install Flux 2, Flux 1, Z-Image, ERNIE, HiDream, Qwen, LTX-Video, and MiniMax H3 LoRAs — pass an explicit family if you know which one this targets.`,
{ status: 422, code: 'HF_UNKNOWN_FAMILY' },
);
}
if (existsSync(plan.destPath)) {
throw new ServerError(
`Already installed: ${plan.filename}. Delete it first to reinstall.`,
{ status: 409, code: 'HF_ALREADY_INSTALLED' },
);
}
await ensureDir(PATHS.loras);
assertDownloadFits(await assessDownloadPreflight({ destPath: plan.destPath, expectedBytes: plan.expectedBytes }));
console.log(`📥 Installing HuggingFace LoRA: ${plan.repo} (${plan.file}) → ${plan.filename} [family=${plan.family}]`);
await downloadToFile(buildHfResolveUrl(plan.repo, plan.revision, plan.file), plan.destPath, {
fetchImpl,
headers: { 'User-Agent': 'PortOS/hf-lora-installer', ...buildHfAuthHeaders(plan.token) },
hasApiKey: !!plan.token,
source: 'huggingface',
onProgress,
signal,
});
// HF's model metadata doesn't expose a per-file digest through pickHfLoraFile,
// so the structural header/size check is the integrity guard here (no deep
// sha256 compare available for this path).
await verifyDownloadedLora(plan.destPath, { source: 'huggingface' });
const sidecar = await withKeyLayout(
buildHfLoraSidecar({ repo: plan.repo, revision: plan.revision, file: plan.file, model: plan.model, family: plan.family, filename: plan.filename, fluxVariant: plan.fluxVariant }),
plan.destPath,
);
await writeLoraSidecar(plan.filename, sidecar);
console.log(`✅ Installed HuggingFace LoRA: ${plan.filename} [layout=${sidecar.keyLayout || 'unknown'}]`);
return sidecar;
};
/** Size / dest / free-disk numbers for the HF LoRA confirm step — mirrors the
* exact file/filename installFromHuggingface will write, so the number shown
* never disagrees with what installs. No transfer starts. */
export const previewHuggingfaceInstall = async (input, { fetchImpl = fetch } = {}) => {
const plan = await resolveHfLoraInstallPlan(input, { fetchImpl });
const preflight = await assessDownloadPreflight({ destPath: plan.destPath, expectedBytes: plan.expectedBytes });
return {
kind: 'huggingface',
...preflight,
destPath: plan.filename,
alreadyDownloaded: existsSync(plan.destPath),
};
};