Open source CapCut alternative.
TypeScript
import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { accessSync, constants as fsConstants, realpathSync } from "node:fs";
import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
import path from "node:path";
import { Defuddle } from "defuddle/node";
import { parseHTML } from "linkedom";
import { convertToMp4, fitsAlready, streamCodecs } from "./convert";
import type { LibrarySource } from "./library";
import { videoDimensions } from "./util";
import { pexelsFileUrl, pexelsVideo } from "./pexels";
import { UserError } from "../lib/userError";
// Bring back whatever a URL points at. yt-dlp, resolved from PATH, pulls
// anything with a video or audio stream (TikTok, YouTube, Instagram, …); X's
// auth-free syndication endpoint covers the posts it can't touch — photos, an
// Article, a post that is only words; a Pexels video page brings its file
// from the CDN; and any other page falls back to its readable text. Shared by the local engine's import routes (urlImport.ts) and
// the cloud render worker's import_url jobs.
const MEDIA_EXT = /\.(mp4|mov|m4v|webm|mkv|mp3|m4a|aac|wav|ogg|flac)$/i;
interface YtMeta {
title?: string;
thumbnail?: string;
description?: string;
uploader?: string;
upload_date?: string;
webpage_url?: string;
extractor?: string;
}
/** What a finished download hands its consumer: the media files (inside a
* temp dir the wrapper deletes afterwards — one for yt-dlp's merged output,
* one per photo for a photo post) plus what the extractor knew about them.
* Each file may carry a `poster`: the source's own cover image, saved beside
* it. `text` is the source's own words — a post's body, an Article, a video's
* title and description — shown in the chat beside the media it came with. A
* source that is only words lands with no files and that text alone. */
export interface Downloaded {
files: { file: string; title: string; poster?: string }[];
source: LibrarySource;
text?: string;
}
export interface DownloadOptions {
/** Bring back only the sound: the source's audio track, landed as an audio
* file. Reads the media rung; a post's photos and a page's text come back as
* themselves. */
audio?: boolean;
}
// A video's own words for the chat: its title and description joined, capped so
// a long YouTube description (link dumps, chapter lists) can't flood the chat.
// TikTok/Instagram often repeat the caption as both fields, so drop a
// description that just restates the title.
const MAX_SOURCE_TEXT = 2000;
// An Article or a page is imported *for* its text, so it gets a wider cap.
const MAX_PAGE_TEXT = 8000;
function sourceTextFromMeta(meta: YtMeta): string | undefined {
const title = (meta.title || "").trim();
const desc = (meta.description || "").trim();
// A caption-only site has no title of its own, so yt-dlp cuts one out of the
// caption at a word and ends it with an ellipsis. The prefix test compares
// what survives that cut, so the same sentence doesn't land twice.
const stem = title.replace(/[.\u2026\s]+$/, "");
const restates = !desc || desc === title || (stem.length > 0 && desc.startsWith(stem));
const body = restates ? desc || title : title ? `${title}\n\n${desc}` : desc;
return clamp(body, MAX_SOURCE_TEXT) || undefined;
}
const X_STATUS_RE = /^https?:\/\/(?:www\.|mobile\.)?(?:x|twitter)\.com\/[^/]+\/status\/(\d+)/i;
const X_ARTICLE_RE = /^https?:\/\/(?:www\.)?(?:x|twitter)\.com\/i\/article\/\d+/i;
const X_URL_RE = /^https?:\/\/(?:www\.|mobile\.)?(?:x|twitter)\.com\//i;
/** Work down a ladder of extractors until one comes back with something: each
* rung handles what the rung before it can't, and a rung that recognizes
* nothing at the URL returns null so the next one gets its turn. An X post
* offers its Article or photos first, then yt-dlp for the video stream the
* syndication endpoint won't serve, then the stills that stream left behind,
* then the post's words alone. Every other URL tries yt-dlp, then the page
* itself — what it says and what it shows. When no rung produces anything, the
* first failure is what surfaces — it names the real cause. */
export async function download(
url: string,
tmp: string,
opts: DownloadOptions = {}
): Promise<Downloaded> {
if (IMAGE_URL_RE.test(url)) return downloadDirectImage(url, tmp);
const pexels = pexelsVideo(url);
if (pexels) return downloadPexels(pexels, url, tmp, opts);
const rungs: (() => Promise<Downloaded | null>)[] = [];
const postId = X_STATUS_RE.exec(url)?.[1];
// An X post's lower rungs are the post itself — its stills, its words — so a
// video stream that won't come down still imports as the post. Everywhere
// else the page under a video host is its thumbnail, and filing that as the
// import is worse than saying why the video didn't arrive.
const xLadder = !!postId || X_ARTICLE_RE.test(url);
if (postId) {
// One syndication fetch feeds every X rung.
const post = fetchPost(postId);
rungs.push(
() => xPostMedia(post, url, tmp, { deferVideo: true }),
() => downloadMedia(url, tmp, opts),
() => xPostMedia(post, url, tmp, { deferVideo: false }),
() => xPostText(post, url)
);
} else if (X_ARTICLE_RE.test(url)) {
rungs.push(() => xArticle({}, url, tmp));
} else {
rungs.push(() => downloadMedia(url, tmp, opts), () => readPage(url, tmp));
}
const failures: unknown[] = [];
for (const rung of rungs) {
try {
const got = await rung();
if (got) return got;
} catch (e) {
if (e instanceof MediaHostError && !xLadder) throw e;
failures.push(e);
}
}
const first = failures[0];
throw first instanceof Error ? first : new UserError("url_nothing_imported", "Nothing could be imported from that URL.");
}
/** A Pexels video by its page link: the page refuses a server, so its best
* rendition comes straight off the CDN, titled from the page's slug and kept
* against the page, so the asset says which Pexels video it is. */
async function downloadPexels(
video: { id: string; title: string; file?: string },
pageUrl: string,
tmp: string,
opts: DownloadOptions
): Promise<Downloaded> {
const file = video.file ?? (await pexelsFileUrl(video.id));
// Newer videos name their files by a second id only the page shows.
if (!file)
throw new UserError(
"pexels_files_hidden",
`Pexels video ${video.id} keeps its files under names only its page shows: read the page for its videos.pexels.com/video-files/${video.id}/… link (the 1920 or 1080 wide one) and import that link.`,
{ id: video.id }
);
const got = await downloadMedia(file, tmp, opts);
const title = (video.file ? video.title : `${video.title} (Pexels ${video.id})`).slice(0, 120);
return {
...got,
files: got.files.slice(0, 1).map((f) => ({ ...f, title })),
source: { url: video.file ? `https://www.pexels.com/video/${video.id}/` : pageUrl, title, uploader: "Pexels" },
text: undefined,
};
}
/** yt-dlp knows this site as a video host and still could not bring the video
* back. The import fails on it rather than falling through to the page. */
class MediaHostError extends Error {}
/** The bundled tool is missing — every retry lands here, so the first attempt
* is the last. */
class MissingToolError extends UserError {
constructor() {
super("url_tool_missing", "yt-dlp is not available in this build.");
}
}
async function downloadMedia(url: string, tmp: string, opts: DownloadOptions = {}): Promise<Downloaded> {
// Read what sits at the URL before pulling any bytes. The extractor name
// says whether this is a site that serves video, and that decides what a
// failed download means: on a video host the import stops with the real
// reason, on an ordinary page the page rung takes over. A URL yt-dlp can't
// read at all throws here, so a page import spawns it once.
let probe: YtMeta;
let bin: string;
try {
({ meta: probe, bin } = await probeUrl(url));
} catch (e) {
// The failure names the extractor that produced it. A media host that
// could not be read is the end of the import — falling through to the page
// rung here would land a video site's cover image and its markup in place
// of the video, and bury the reason the download failed.
if (e instanceof YtDlpError && e.extractor && !PAGE_EXTRACTORS.has(e.extractor))
throw new MediaHostError(e.message);
throw e;
}
const videoHost = !PAGE_EXTRACTORS.has((probe.extractor ?? "").toLowerCase());
let meta: YtMeta;
let out: string;
try {
({ meta, dir: out } = await runYtDlp(url, tmp, bin, opts));
} catch (e) {
if (!videoHost) throw e;
throw new MediaHostError(e instanceof Error ? e.message : "Download failed.");
}
// Pick the largest media file left behind (the merged output).
const names = (await readdir(out)).filter((f) => MEDIA_EXT.test(f));
if (names.length === 0) throw new UserError("url_nothing_downloadable", "Nothing downloadable was found at that URL.");
const sized = await Promise.all(
names.map(async (n) => ({ n, size: (await stat(path.join(out, n))).size }))
);
const file = await playableMedia(path.join(out, sized.sort((a, b) => b.size - a.size)[0].n));
// The site's own cover for this video, so a card and the viewer have a first
// frame to show before a byte of the video is decoded.
const poster = await savePoster(meta.thumbnail, out, path.parse(file).name);
const clip = {
file,
title: (meta.title || "Imported clip").slice(0, 120),
...(poster ? { poster } : {}),
};
const source = {
url: meta.webpage_url || url,
title: meta.title,
uploader: meta.uploader,
uploadDate: meta.upload_date,
};
// A page-scraping extractor means an ordinary web page that happened to
// embed this clip. The page is the import — its words and its pictures — and
// the clip is one of the things it showed.
if (PAGE_EXTRACTORS.has((meta.extractor ?? "").toLowerCase())) {
const page = await readPage(url, tmp).catch(() => null);
if (page) return { files: [clip, ...page.files], source, text: page.text };
}
return { files: [clip], source, text: sourceTextFromMeta(meta) };
}
// The extractors yt-dlp reports when it scraped an ordinary web page rather
// than a media host it knows.
const PAGE_EXTRACTORS = new Set(["generic", "html5"]);
const IMAGE_URL_RE = /\.(png|jpe?g|webp|gif|avif|bmp)(?:$|\?)/i;
/** What comes back has to play where the import is going — the phone's viewer,
* the Mac's player, a browser — and a site's best stream is often VP9 or AV1,
* which Apple decodes nowhere. A file whose picture is already H.264 is handed
* back untouched; anything else is re-encoded into one that plays. A
* conversion that fails leaves the original: a file that plays in fewer places
* beats no import at all. */
export async function playableMedia(file: string): Promise<string> {
const codecs = await streamCodecs(file);
// Sound alone decodes everywhere the import goes.
if (!codecs.video) return file;
if (fitsAlready(file, codecs, { shrink: false })) return file;
const out = path.join(path.dirname(file), `${path.parse(file).name}.h264.mp4`);
try {
await convertToMp4({ log: [], outPath: out, progress: 0, tmpDir: path.dirname(file) }, file, out);
return out;
} catch {
return file;
}
}
/** Save the source's cover image beside the media it belongs to. A cover that
* won't download is no reason to fail an import, so this answers nothing and
* the media lands without one. */
async function savePoster(
url: string | undefined,
dir: string,
base: string
): Promise<string | undefined> {
if (!url) return undefined;
try {
const res = await fetch(url, { signal: AbortSignal.timeout(20_000) });
if (!res.ok) return undefined;
const ext = IMAGE_TYPES[(res.headers.get("content-type") ?? "").split(";")[0].toLowerCase()];
if (!ext) return undefined;
const bytes = Buffer.from(await res.arrayBuffer());
if (bytes.byteLength === 0 || bytes.byteLength > MAX_IMAGE_BYTES) return undefined;
const file = path.join(dir, `${base}.poster.${ext}`);
await writeFile(file, bytes);
return file;
} catch {
return undefined;
}
}
async function downloadDirectImage(url: string, dir: string): Promise<Downloaded> {
const res = await fetch(url);
if (!res.ok) throw new UserError("image_download_failed", `Could not download that image (${res.status}).`, { status: res.status });
const name = path.basename(new URL(url).pathname) || "image.jpg";
const file = path.join(dir, name);
await writeFile(file, Buffer.from(await res.arrayBuffer()));
return { files: [{ file, title: name.slice(0, 120) }], source: { url } };
}
interface UrlEntity {
url?: string;
expanded_url?: string;
}
interface SyndicatedPost {
text?: string;
id_str?: string;
user?: { name?: string; screen_name?: string };
photos?: { url?: string }[];
entities?: { urls?: UrlEntity[]; media?: UrlEntity[] };
mediaDetails?: { type?: string }[];
video?: unknown;
article?: XArticle;
}
interface XArticle {
title?: string;
preview_text?: string;
cover_media?: { media_info?: { original_img_url?: string } };
}
/** Read a post through X's embed endpoint. It is auth-free but requires the
* token its embed script derives from the id — same float math here, precision
* loss included. Resolves to null when X won't answer, so the ladder moves on
* instead of failing the import. */
async function fetchPost(id: string): Promise<SyndicatedPost | null> {
const token = ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, "");
try {
const res = await fetch(
`https://cdn.syndication.twimg.com/tweet-result?id=${id}&token=${token}`
);
if (!res.ok) return null;
return (await res.json()) as SyndicatedPost;
} catch {
return null;
}
}
/** The media an X post carries: its Article (cover image and body) or its
* photos. The endpoint serves a video post's still but not its stream, so with
* `deferVideo` such a post passes to yt-dlp; the same rung runs again after
* yt-dlp, where the still is the best that's left. */
async function xPostMedia(
pending: Promise<SyndicatedPost | null>,
url: string,
dir: string,
{ deferVideo }: { deferVideo: boolean }
): Promise<Downloaded | null> {
const post = await pending;
if (!post) return null;
const uploader = uploaderOf(post);
if (post.article) return xArticle(post.article, url, dir, postText(post));
const hasVideo = !!post.video || (post.mediaDetails ?? []).some((m) => m.type !== "photo");
if (deferVideo && hasVideo) return null;
const photoUrls = (post.photos ?? []).map((p) => p.url).filter((u): u is string => !!u);
if (photoUrls.length === 0) return null;
const text = postText(post);
const baseTitle = text || `Photo from ${uploader ?? "X"}`;
const files: Downloaded["files"] = [];
for (const [i, photoUrl] of photoUrls.entries()) {
files.push(
await savePhoto(
photoUrl,
dir,
`${post.id_str ?? "post"}-${i + 1}`,
photoUrls.length > 1 ? `${baseTitle} (${i + 1}/${photoUrls.length})` : baseTitle
)
);
}
return {
files,
source: { url, title: text || undefined, uploader },
text: text || undefined,
};
}
/** The last rung for an X post: its words, with no media at all. This is what
* a text-only post imports as. */
async function xPostText(
pending: Promise<SyndicatedPost | null>,
url: string
): Promise<Downloaded | null> {
const post = await pending;
if (!post) return null;
const text = postText(post);
if (!text) return null;
return {
files: [],
source: { url, title: text.slice(0, 120), uploader: uploaderOf(post) },
text,
};
}
/** An X Article: the post carries the title and a cover image, and the article
* body rides in the post page's hydration payload. The syndicated preview is
* truncated to a couple of lines, so the full body is read from the page first
* and the preview stands in when that payload moves. */
async function xArticle(
article: XArticle,
url: string,
dir: string,
postWords?: string
): Promise<Downloaded> {
const scraped = await readArticlePayload(url).catch(() => ({}) as ArticlePayload);
const title = scraped.title || article.title;
const body = scraped.text || article.preview_text;
const cover = scraped.cover || article.cover_media?.media_info?.original_img_url;
const text = clamp([postWords, title, body].filter(Boolean).join("\n\n"), MAX_PAGE_TEXT);
if (!text && !cover) {
throw new UserError("x_article_unreadable", "Could not read that X Article — import the post that published it.");
}
const files = cover
? [await savePhoto(cover, dir, "article-cover", title || "Article cover")]
: [];
return { files, source: { url, title: title || undefined }, text: text || undefined };
}
interface ArticlePayload {
title?: string;
text?: string;
cover?: string;
}
/** X renders an Article in the browser, but its page ships the article in the
* hydration payload as plain JS string literals — title, full body, cover
* image. Read those three out of the payload; a shape change yields nothing
* and the caller falls back to the syndicated preview. */
async function readArticlePayload(pageUrl: string): Promise<ArticlePayload> {
const { html } = await fetchPage(pageUrl);
const at = html.search(/"?__typename"?\s*:\s*"ArticleEntity"/);
if (at < 0) return {};
const payload = html.slice(at);
return {
title: jsString(payload, "title"),
text: jsString(payload, "plain_text"),
cover: jsString(payload, "original_img_url"),
};
}
/** The value of `key:"…"` in a JS payload, unescaped. */
function jsString(payload: string, key: string): string | undefined {
const m = new RegExp(`"?${key}"?\\s*:\\s*"((?:[^"\\\\]|\\\\.)*)"`).exec(payload);
if (!m) return undefined;
try {
return JSON.parse(`"${m[1].replace(/\\'/g, "'")}"`) as string;
} catch {
return undefined;
}
}
/** A post's own words. The endpoint HTML-escapes them and ends them with a
* t.co link per attachment, so entities decide what each link becomes: an
* outside link expands to where it points, and X's own self-links — the
* photos, the Article, a quoted post — drop, since the import already carries
* what they point at. */
function postText(post: SyndicatedPost): string {
let text = decodeEntities(post.text || "");
for (const u of [...(post.entities?.urls ?? []), ...(post.entities?.media ?? [])]) {
if (!u.url) continue;
const target = u.expanded_url ?? "";
text = text.split(u.url).join(target && !X_URL_RE.test(target) ? target : "");
}
return text.replace(/\s+/g, " ").trim();
}
function uploaderOf(post: SyndicatedPost): string | undefined {
const handle = post.user?.screen_name;
return handle ? `@${handle}` : post.user?.name;
}
/** Fetch one pbs.twimg.com image at its original resolution (behind
* name=orig) and write it into the temp dir. */
async function savePhoto(
photoUrl: string,
dir: string,
stem: string,
title: string
): Promise<{ file: string; title: string }> {
let img = await fetch(`${photoUrl}${photoUrl.includes("?") ? "&" : "?"}name=orig`);
if (!img.ok) img = await fetch(photoUrl);
if (!img.ok) throw new UserError("post_photos_download_failed", "Could not download the post's photos.");
const ext = (
/\.(png|jpe?g|webp|gif)(?:$|\?)/i.exec(photoUrl)?.[1] ||
(img.headers.get("content-type")?.includes("png") ? "png" : "jpg")
).toLowerCase();
const file = path.join(dir, `${stem}.${ext}`);
await writeFile(file, Buffer.from(await img.arrayBuffer()));
return { file, title: title.slice(0, 120) };
}
// A browser user agent, since a page served to a crawler is often an error or
// a stub, and a ceiling on how much of a page is read into memory.
const PAGE_UA =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
const MAX_PAGE_BYTES = 4_000_000;
/** The page's markup plus the address it settled on after redirects, which is
* what its relative links and images resolve against. */
async function fetchPage(url: string): Promise<{ html: string; url: string }> {
const res = await fetch(url, {
headers: { "user-agent": PAGE_UA, accept: "text/html,application/xhtml+xml,*/*" },
redirect: "follow",
});
if (!res.ok) throw new UserError("page_read_failed", `That page could not be read (${res.status}).`, { status: res.status });
const type = res.headers.get("content-type") ?? "";
if (type && !/text\/|xml|json/i.test(type)) throw new UserError("url_not_readable_page", "That URL is not a readable page.");
const buf = await res.arrayBuffer();
const bytes = buf.byteLength > MAX_PAGE_BYTES ? buf.slice(0, MAX_PAGE_BYTES) : buf;
return { html: Buffer.from(bytes).toString("utf8"), url: res.url || url };
}
// Below this a page's readable text is chrome — a cookie notice, a nav bar —
// and worth nothing to the chat, so the ladder keeps the earlier failure.
const MIN_PAGE_TEXT = 40;
/** The last rung for anything that isn't a post: read the page itself. A
* landing page, an article or a docs page imports as what it says and what it
* shows — its readable text plus the pictures on it — so a link the user
* pasted for its content arrives as content. Returns null for a page that has
* neither, leaving the earlier failure to surface. */
async function readPage(url: string, dir: string): Promise<Downloaded | null> {
const page = await fetchPage(url);
const { document } = parseHTML(page.html);
// Defuddle (a Readability successor) picks the article out of the page and
// hands it back as markdown, so headings, lists and links survive and nav,
// sidebars and cookie banners don't.
const article = await Defuddle(document, page.url, { markdown: true }).catch(() => null);
const title = (article?.title || document.title || "").trim();
const body = [title, (article?.content ?? "").trim()].filter(Boolean).join("\n\n");
const text = clamp(body, MAX_PAGE_TEXT);
const files = await pageImages(article, document, page.url, title, dir);
if (files.length === 0 && text.length < MIN_PAGE_TEXT) return null;
return {
files,
source: { url, title: title || undefined, uploader: article?.site?.trim() || undefined },
text: text.length >= MIN_PAGE_TEXT ? text : undefined,
};
}
// How many of a page's pictures come along, and the size band a file has to
// land in to count as one: under the floor are icons, spacers, avatars and
// tracking pixels, over the ceiling is something no import should drag into a
// project. More candidates than that are fetched, since some won't qualify.
const MAX_PAGE_IMAGES = 6;
const MIN_IMAGE_BYTES = 10_000;
const MAX_IMAGE_BYTES = 15_000_000;
const IMAGE_CANDIDATES = 16;
/** Download the pictures a page shows, in the order it presents them, and
* write them into the temp dir beside anything else the import found. */
async function pageImages(
article: { image?: string; content?: string } | null,
document: Document,
pageUrl: string,
pageTitle: string,
dir: string
): Promise<Downloaded["files"]> {
const refs = imageCandidates(article, document, pageUrl).slice(0, IMAGE_CANDIDATES);
const saved = await Promise.all(
refs.map((ref, i) => savePageImage(ref, pageUrl, i, dir).catch(() => null))
);
const kept = (await distinctPictures(saved.filter((f): f is SavedImage => !!f))).slice(0, MAX_PAGE_IMAGES);
// A picture the page captioned keeps that caption; the rest are named after
// the page and counted, the way a multi-photo post's are.
return kept.map((img, i) => ({
file: img.file,
title: (
img.alt ||
(kept.length > 1 ? `${pageTitle} (${i + 1}/${kept.length})` : pageTitle) ||
"Image"
).slice(0, 120),
}));
}
interface SavedImage {
file: string;
alt?: string;
}
interface ImageRef {
url: string;
alt?: string;
}
// Vector icons and favicons are chrome, never content.
const SKIP_IMAGE_RE = /\.(svg|ico)(?:$|[?#])/i;
/** The pictures worth importing, best first: the one the page nominates for a
* link preview, then the ones the extractor kept with the article — its
* content rather than its logos and nav icons — then the rest of the
* document's images, each at the sharpest size its srcset offers. */
function imageCandidates(
article: { image?: string; content?: string } | null,
document: Document,
pageUrl: string
): ImageRef[] {
const seen = new Set<string>();
const refs: ImageRef[] = [];
const add = (raw: string | null | undefined, alt?: string | null) => {
const abs = absolute(raw ?? undefined, pageUrl);
if (!abs || seen.has(abs) || SKIP_IMAGE_RE.test(abs)) return;
seen.add(abs);
refs.push({ url: abs, alt: alt?.trim() || undefined });
};
add(article?.image);
// The extractor writes the article as markdown, so its pictures are the
// image links in it, in the order the page shows them.
for (const m of (article?.content ?? "").matchAll(/!\[([^\]]*)\]\(\s*([^)\s]+)/g)) add(m[2], m[1]);
for (const img of document.querySelectorAll("img")) {
add(widestSrc(img.getAttribute("srcset")) ?? img.getAttribute("src"), img.getAttribute("alt"));
}
return refs;
}
/** The sharpest candidate in a srcset — a responsive page lists one picture at
* several widths. */
function widestSrc(srcset?: string | null): string | undefined {
let best: { url: string; weight: number } | undefined;
for (const candidate of (srcset ?? "").split(",")) {
const [src, descriptor] = candidate.trim().split(/\s+/);
if (!src) continue;
const size = parseFloat(descriptor ?? "");
const weight = !Number.isFinite(size)
? 1
: descriptor?.endsWith("x")
? size * 1000
: size;
if (!best || weight > best.weight) best = { url: src, weight };
}
return best?.url;
}
const IMAGE_TYPES: Record<string, string> = {
"image/jpeg": "jpg",
"image/jpg": "jpg",
"image/png": "png",
"image/webp": "webp",
"image/gif": "gif",
"image/avif": "avif",
};
/** Fetch one of a page's images. The response's own type names the file, since
* a URL that ends in nothing recognizable still serves a real picture, and
* anything under the floor or not an image at all is skipped. */
async function savePageImage(
ref: ImageRef,
pageUrl: string,
index: number,
dir: string
): Promise<SavedImage | null> {
const res = await fetch(ref.url, {
headers: { "user-agent": PAGE_UA, accept: "image/*,*/*", referer: pageUrl },
});
if (!res.ok) return null;
const ext = IMAGE_TYPES[(res.headers.get("content-type") ?? "").split(";")[0].toLowerCase()];
if (!ext) return null;
if (Number(res.headers.get("content-length")) > MAX_IMAGE_BYTES) return null;
const bytes = Buffer.from(await res.arrayBuffer());
if (bytes.byteLength < MIN_IMAGE_BYTES || bytes.byteLength > MAX_IMAGE_BYTES) return null;
const file = path.join(dir, `page-${index + 1}.${ext}`);
await writeFile(file, bytes);
return { file, alt: ref.alt };
}
// Two files are one picture when their shapes agree and their 16×16
// thumbnails differ by at most this much per pixel on average, out of 255.
// A file page shows one logo as its preview, a thumbnail and a srcset entry.
const PRINT_SIDE = 16;
const SAME_PICTURE_DIFF = 6;
const SAME_SHAPE_TOLERANCE = 0.02;
interface PicturePrint {
pixels: number;
aspect: number;
thumb: Buffer;
}
interface PrintedImage {
image: SavedImage;
print: PicturePrint;
}
/** Keep each picture a page shows once, at its largest size, in the place it
* first appeared. A file that can't be read stays, since nothing says it
* repeats another. */
async function distinctPictures(images: SavedImage[]): Promise<SavedImage[]> {
const prints = await Promise.all(images.map((img) => picturePrint(img.file)));
const kept: { image: SavedImage; print: PicturePrint | null }[] = [];
for (const [i, image] of images.entries()) {
const print = prints[i];
const twin = print && kept.find((k): k is PrintedImage => !!k.print && samePicture(k.print, print));
if (!print || !twin) {
kept.push({ image, print });
continue;
}
// The sharper copy takes the first one's place; a caption from either stays.
const alt = twin.image.alt ?? image.alt;
if (print.pixels > twin.print.pixels) {
twin.image = { file: image.file, alt };
twin.print = print;
continue;
}
twin.image = { ...twin.image, alt };
}
return kept.map((k) => k.image);
}
function samePicture(a: PicturePrint, b: PicturePrint): boolean {
if (Math.abs(a.aspect - b.aspect) > SAME_SHAPE_TOLERANCE * a.aspect) {
return false;
}
let diff = 0;
for (let i = 0; i < a.thumb.length; i++) {
diff += Math.abs(a.thumb[i] - b.thumb[i]);
}
return diff / a.thumb.length <= SAME_PICTURE_DIFF;
}
async function picturePrint(file: string): Promise<PicturePrint | null> {
const [dims, thumb] = await Promise.all([videoDimensions(file), pictureThumb(file)]);
if (!dims || !thumb) {
return null;
}
return { pixels: dims.width * dims.height, aspect: dims.width / dims.height, thumb };
}
/** The picture shrunk to a gray PRINT_SIDE square, transparency laid on
* white so a logo's empty corners read the same in every rendition. */
function pictureThumb(file: string): Promise<Buffer | null> {
const side = `${PRINT_SIDE}x${PRINT_SIDE}`;
return new Promise((resolve) => {
const p = spawn("ffmpeg", [
"-v", "error",
"-i", file,
"-filter_complex",
`[0:v]scale=${side}:flags=area,format=rgba[pic];color=white:size=${side}[bg];[bg][pic]overlay=shortest=1,format=gray`,
"-frames:v", "1",
"-f", "rawvideo", "-",
]);
const chunks: Buffer[] = [];
const timer = setTimeout(() => {
p.kill("SIGKILL");
resolve(null);
}, 30_000);
timer.unref();
p.stdout.on("data", (d: Buffer) => chunks.push(d));
p.on("close", (code) => {
clearTimeout(timer);
const thumb = Buffer.concat(chunks);
resolve(code === 0 && thumb.length === PRINT_SIDE * PRINT_SIDE ? thumb : null);
});
p.on("error", () => {
clearTimeout(timer);
resolve(null);
});
});
}
/** An absolute http(s) address for a page's link, or nothing for the inline
* and non-web schemes an import can't fetch. */
function absolute(raw: string | undefined, base: string): string | undefined {
if (!raw) return undefined;
try {
const u = new URL(raw, base);
return u.protocol === "http:" || u.protocol === "https:" ? u.toString() : undefined;
} catch {
return undefined;
}
}
const ENTITIES: Record<string, string> = {
amp: "&",
lt: "<",
gt: ">",
quot: '"',
apos: "'",
nbsp: " ",
};
function decodeEntities(s: string): string {
return s.replace(/&(#x?[0-9a-f]+|[a-z]+);/gi, (whole, body: string) => {
if (!body.startsWith("#")) return ENTITIES[body.toLowerCase()] ?? whole;
const hex = body[1] === "x" || body[1] === "X";
const code = parseInt(hex ? body.slice(2) : body.slice(1), hex ? 16 : 10);
return Number.isFinite(code) && code > 0 && code <= 0x10ffff
? String.fromCodePoint(code)
: whole;
});
}
function clamp(text: string, max: number): string {
const t = text.trim();
return t.length > max ? `${t.slice(0, max).trimEnd()}…` : t;
}
// YouTube hands each of its player clients a different set of formats, and the
// one yt-dlp reaches for first can serve URLs that 403 the moment the download
// starts — the site breaks them one at a time. Each entry is a full attempt
// with a different client, so a broken one costs a retry instead of the
// import. Only YouTube reads the argument, so only YouTube pays for the ladder.
const YT_PLAYER_CLIENTS = [null, "mweb", "android"];
const YT_HOST = /(^|\.)(youtube\.com|youtu\.be)$/i;
function playerClients(url: string): (string | null)[] {
try {
return YT_HOST.test(new URL(url).hostname) ? YT_PLAYER_CLIENTS : [null];
} catch {
return [null];
}
}
/** Every yt-dlp this machine can run, in PATH order and deduped by the file
* they point at — the copy we ship comes first, since the engine puts its
* tools ahead of the rest of PATH.
*
* They are not interchangeable. The build we bundle carries curl_cffi, so it
* impersonates a browser on every request; a copy installed with pip or
* Homebrew usually can't, and takes yt-dlp's plain path instead. Sites tell
* the two apart: TikTok currently answers the impersonated request with a page
* the extractor can't read, while the plain path solves its challenge and gets
* the video. So a run that fails on one build is worth retrying on the next. */
function ytDlpBinaries(): string[] {
const seen = new Set<string>();
const bins: string[] = [];
for (const dir of (process.env.PATH ?? "").split(path.delimiter)) {
if (!dir) continue;
const file = path.join(dir, "yt-dlp");
try {
accessSync(file, fsConstants.X_OK);
const real = realpathSync(file);
if (seen.has(real)) continue;
seen.add(real);
bins.push(file);
} catch {
// Not on this entry of PATH.
}
}
return bins;
}
/** Read what sits at a URL, through whichever yt-dlp can read it. Hands back
* the build that answered so the download starts with the one that works. */
async function probeUrl(url: string): Promise<{ meta: YtMeta; bin: string }> {
const bins = ytDlpBinaries();
if (bins.length === 0) throw new MissingToolError();
let first: unknown;
for (const bin of bins) {
try {
return { meta: await ytDlpJson(bin, ["--skip-download", "--dump-json", url]), bin };
} catch (e) {
first ??= e;
}
}
throw first instanceof Error ? first : new UserError("url_nothing_readable", "Nothing could be read at that URL.");
}
// What the whole ladder may spend. Someone is watching a tile the entire time,
// and the cloud rung gives up on a job at ten minutes, so every retry together
// has to finish inside that.
const DOWNLOAD_BUDGET_MS = 8 * 60_000;
/** Pull the media at `url` into a fresh directory under `dir`, working through
* the builds and player clients until one comes back with the file. Each
* attempt writes into its own directory: a failed one can leave a finished
* video-only format behind, and the largest file in the directory is the one
* that lands — a leftover would beat a smaller merged file that has sound. */
async function runYtDlp(
url: string,
dir: string,
preferred: string,
opts: DownloadOptions = {}
): Promise<{ meta: YtMeta; dir: string }> {
const bins = [preferred, ...ytDlpBinaries().filter((b) => b !== preferred)];
const deadline = Date.now() + DOWNLOAD_BUDGET_MS;
const attempts = bins.flatMap((bin) => playerClients(url).map((client) => ({ bin, client })));
let first: unknown;
for (const [i, { bin, client }] of attempts.entries()) {
const left = deadline - Date.now();
if (left <= 0) break;
const out = path.join(dir, `try${i}`);
await mkdir(out, { recursive: true });
try {
return { meta: await ytDlp(bin, url, out, client, left, opts), dir: out };
} catch (e) {
if (e instanceof MissingToolError) throw e;
first ??= e;
}
}
throw first instanceof Error ? first : new UserError("download_failed", "Download failed.");
}
function ytDlp(
bin: string,
url: string,
dir: string,
client: string | null,
timeoutMs: number,
opts: DownloadOptions = {}
): Promise<YtMeta> {
// An audio import takes the best sound stream alone and lands it as m4a.
// AAC leads the sort — it decodes everywhere the import goes — and -x
// extracts the sound when the site only serves a combined file, transcoding
// an Opus-only stream through the bundled ffmpeg.
const format = opts.audio
? [
"-f", "bestaudio/best",
"-S", "acodec:aac",
"-x", "--audio-format", "m4a",
]
: [
// Take the best video and the best audio and put them together. Asking for
// a ready-made mp4 first would land whatever single file the site keeps
// around for compatibility — on YouTube that is 360p next to a 4K original.
//
// Codec leads the sort, resolution follows it. Apple decodes H.264 and
// HEVC and nothing else, so a taller VP9 or AV1 stream is a file that
// plays nowhere the import is going: the phone's viewer, the Mac's own
// player, Safari. Instagram serves 1440p VP9 beside a 1080p-class H.264
// file, and the H.264 one is worth having. Sound is compared last, after
// the picture: judged any earlier it would rank YouTube's 360p
// sound-and-picture file over the 1080p stream that carries no sound of
// its own. A source with no H.264 at all still comes down, and the
// conversion below makes it playable.
"-f", "bestvideo*+bestaudio/best",
"-S", "vcodec:h264,res,fps,hdr:sdr,acodec:aac",
"--merge-output-format", "mp4",
];
return ytDlpJson(bin, [
...(client ? ["--extractor-args", `youtube:player_client=${client}`] : []),
...format,
// Extractor IDs can contain signed URL queries longer than a disk filename.
"-o", path.join(dir, `${randomUUID()}.%(ext)s`),
"--print-json",
url,
], timeoutMs);
}
// yt-dlp reports failures as a stderr line aimed at people who file issues on
// its tracker. The library tile shows whatever comes back, so the shapes worth
// naming get a sentence someone can act on and the rest is trimmed down to the
// part that says what happened.
const YT_DLP_REASONS: [RegExp, string][] = [
[/IP address is blocked|blocked from accessing|rate.?limit|too many requests|HTTP Error 429/i,
"That site is blocking downloads from this network right now. Try again later."],
[/private video|login required|sign in|requires authentication|members-only|age.?restrict/i,
"That post is private or needs a sign-in."],
[/not available in your country|geo.?restrict|geo.?block/i,
"That post isn't available in this region."],
[/video unavailable|does not exist|has been removed|HTTP Error 404|account.*(private|banned)/i,
"That post isn't available any more."],
[/unsupported url|no video formats found|no media found/i,
"That link has nothing this can download."],
[/unable to extract|unexpected response|failed to parse|rehydration/i,
"That site changed how it serves this post, so the download tool can't read it."],
];
/** A yt-dlp run that ended in an error, holding the extractor named in its
* last line — `ERROR: [TikTok] 76720…: …` — so the caller can tell a media
* host's refusal from an ordinary page yt-dlp found no video in. */
class YtDlpError extends Error {
constructor(
message: string,
readonly extractor: string | null
) {
super(message);
}
}
function ytDlpExtractor(stderr: string): string | null {
const line = stderr.split("\n").filter(Boolean).slice(-1)[0] ?? "";
return line.match(/^ERROR:\s*\[([^\]]+)\]/i)?.[1].toLowerCase() ?? null;
}
function ytDlpReason(stderr: string): string {
const line = stderr.split("\n").filter(Boolean).slice(-1)[0] ?? "";
for (const [pattern, reason] of YT_DLP_REASONS) if (pattern.test(line)) return reason;
const clean = line
.replace(/^ERROR:\s*/i, "")
.replace(/^\[[^\]]+\]\s*[^:]*:\s*/, "")
.split(/;\s*please report/i)[0]
.trim();
return clean ? clean.slice(0, 200) : "Download failed.";
}
/** Run yt-dlp and read the info dict it prints. */
function ytDlpJson(bin: string, args: string[], timeoutMs = 300_000): Promise<YtMeta> {
return new Promise((resolve, reject) => {
const p = spawn(bin, ["--no-playlist", "--no-progress", ...args]);
const timer = setTimeout(() => {
p.kill("SIGKILL");
reject(new UserError("download_timed_out", "Download timed out."));
}, timeoutMs);
timer.unref();
let out = "";
let err = "";
p.stdout.on("data", (d) => (out += d.toString()));
p.stderr.on("data", (d) => (err = (err + d.toString()).slice(-2000)));
p.on("error", (e) => {
clearTimeout(timer);
// Bun and Node word a missing binary differently; the code is the same.
reject(
(e as NodeJS.ErrnoException).code === "ENOENT"
? new MissingToolError()
: e
);
});
p.on("close", (code) => {
clearTimeout(timer);
if (code !== 0) {
reject(new YtDlpError(ytDlpReason(err), ytDlpExtractor(err)));
return;
}
// The info dict comes back as JSON; take the last JSON line.
const line = out.trim().split("\n").reverse().find((l) => l.trimStart().startsWith("{"));
try {
resolve(line ? (JSON.parse(line) as YtMeta) : {});
} catch {
resolve({});
}
});
});
}