Skip to content

Open source CapCut alternative.

TypeScript

fix(site): import each page picture once [rebuild]

David Le2aad2c40de42
diff --git a/site/src/cut/server/urlDownload.e2e.test.ts b/site/src/cut/server/urlDownload.e2e.test.ts
index a60130b0..da6e4782 100644
--- a/site/src/cut/server/urlDownload.e2e.test.ts
+++ b/site/src/cut/server/urlDownload.e2e.test.ts
@@ -6,6 +6,7 @@ import { createServer } from "node:http";
import os from "node:os";
import path from "node:path";
import { download, playableMedia } from "./urlDownload";
+import { videoDimensions } from "./util";
// What a link import lands has to play on every surface it reaches: the
// phone's viewer and the Mac's player go through AVFoundation, which decodes
@@ -141,3 +142,72 @@ suite("playableMedia", () => {
}
});
});
+
+// Serve fixed bodies by path on a local port, so a page and its pictures
+// import the way a real site's would.
+async function serve(routes: Record<string, { type: string; body: Buffer | string; status?: number }>) {
+ const server = createServer((request, response) => {
+ const route = routes[new URL(request.url!, "http://localhost").pathname];
+ if (!route) {
+ response.writeHead(404).end();
+ return;
+ }
+ response.writeHead(route.status ?? 200, { "Content-Type": route.type }).end(route.body);
+ });
+ server.listen(0, "127.0.0.1");
+ await once(server, "listening");
+ const address = server.address();
+ if (!address || typeof address === "string") throw new Error("Missing test server address.");
+ return {
+ base: `http://127.0.0.1:${address.port}`,
+ close: () => new Promise<void>((resolve) => server.close(() => resolve())),
+ };
+}
+
+suite("page pictures", () => {
+ // A file page links one picture as its preview, a smaller rendition and a
+ // srcset entry; the import keeps it once, at its sharpest size.
+ test("one picture shown at several sizes imports once, at its largest", async () => {
+ const dir = await mkdtemp(path.join(os.tmpdir(), "cut-page-"));
+ const big = path.join(dir, "big.png");
+ const small = path.join(dir, "small.png");
+ const bars = path.join(dir, "bars.png");
+ ff(["-f", "lavfi", "-i", "testsrc=size=1280x1280", "-frames:v", "1", big]);
+ ff(["-i", big, "-vf", "scale=960:960", small]);
+ ff(["-f", "lavfi", "-i", "mandelbrot=size=960x540", "-frames:v", "1", bars]);
+ const prose = "A page about one logo, with enough words for a reader to keep. ".repeat(4);
+ const site = await serve({
+ "/page": {
+ type: "text/html",
+ body: `<html><head><title>Logo</title><meta property="og:image" content="/big.png?from=og"></head>
+ <body><article><h1>Logo</h1><p>${prose}</p>
+ <img src="/small.png" alt="Logo"><img srcset="/big.png?from=srcset 1280w"><img src="/bars.png" alt="Bars">
+ </article></body></html>`,
+ },
+ "/big.png": { type: "image/png", body: await readFile(big) },
+ "/small.png": { type: "image/png", body: await readFile(small) },
+ "/bars.png": { type: "image/png", body: await readFile(bars) },
+ });
+ try {
+ const result = await download(`${site.base}/page`, dir);
+ expect(result.files).toHaveLength(2);
+ expect((await videoDimensions(result.files[0].file))?.width).toBe(1280);
+ expect(result.files[1].title).toBe("Bars");
+ } finally {
+ await site.close();
+ await rm(dir, { force: true, recursive: true });
+ }
+ }, 60_000);
+
+ // The chat retries a refused image link only when it can see why it failed.
+ test("a refused image link says what the server answered", async () => {
+ const dir = await mkdtemp(path.join(os.tmpdir(), "cut-image-"));
+ const site = await serve({ "/logo.png": { type: "text/html", body: "Bad thumbnail width", status: 400 } });
+ try {
+ await expect(download(`${site.base}/logo.png`, dir)).rejects.toThrow("400");
+ } finally {
+ await site.close();
+ await rm(dir, { force: true, recursive: true });
+ }
+ });
+});
diff --git a/site/src/cut/server/urlDownload.ts b/site/src/cut/server/urlDownload.ts
index d001dff9..b4e87a08 100644
--- a/site/src/cut/server/urlDownload.ts
+++ b/site/src/cut/server/urlDownload.ts
@@ -7,6 +7,7 @@ import { Defuddle } from "defuddle/node";
import { parseHTML } from "linkedom";
import { convertToMp4, fitsAlready, streamCodecs } from "./convert";
import type { LibrarySource } from "./library";
+import { videoDimensions } from "./util";
import { pexelsFileUrl, pexelsVideo } from "./pexels";
import { UserError } from "../lib/userError";
@@ -274,7 +275,7 @@ async function savePoster(
async function downloadDirectImage(url: string, dir: string): Promise<Downloaded> {
const res = await fetch(url);
- if (!res.ok) throw new UserError("image_download_failed", "Could not download that image.");
+ if (!res.ok) throw new UserError("image_download_failed", `Could not download that image (${res.status}).`, { status: res.status });
const name = path.basename(new URL(url).pathname) || "image.jpg";
const file = path.join(dir, name);
await writeFile(file, Buffer.from(await res.arrayBuffer()));
@@ -543,7 +544,7 @@ async function pageImages(
const saved = await Promise.all(
refs.map((ref, i) => savePageImage(ref, pageUrl, i, dir).catch(() => null))
);
- const kept = saved.filter((f): f is SavedImage => !!f).slice(0, MAX_PAGE_IMAGES);
+ const kept = (await distinctPictures(saved.filter((f): f is SavedImage => !!f))).slice(0, MAX_PAGE_IMAGES);
// A picture the page captioned keeps that caption; the rest are named after
// the page and counted, the way a multi-photo post's are.
return kept.map((img, i) => ({
@@ -646,6 +647,101 @@ async function savePageImage(
return { file, alt: ref.alt };
}
+// Two files are one picture when their shapes agree and their 16×16
+// thumbnails differ by at most this much per pixel on average, out of 255.
+// A file page shows one logo as its preview, a thumbnail and a srcset entry.
+const PRINT_SIDE = 16;
+const SAME_PICTURE_DIFF = 6;
+const SAME_SHAPE_TOLERANCE = 0.02;
+
+interface PicturePrint {
+ pixels: number;
+ aspect: number;
+ thumb: Buffer;
+}
+
+interface PrintedImage {
+ image: SavedImage;
+ print: PicturePrint;
+}
+
+/** Keep each picture a page shows once, at its largest size, in the place it
+ * first appeared. A file that can't be read stays, since nothing says it
+ * repeats another. */
+async function distinctPictures(images: SavedImage[]): Promise<SavedImage[]> {
+ const prints = await Promise.all(images.map((img) => picturePrint(img.file)));
+ const kept: { image: SavedImage; print: PicturePrint | null }[] = [];
+ for (const [i, image] of images.entries()) {
+ const print = prints[i];
+ const twin = print && kept.find((k): k is PrintedImage => !!k.print && samePicture(k.print, print));
+ if (!print || !twin) {
+ kept.push({ image, print });
+ continue;
+ }
+
+ // The sharper copy takes the first one's place; a caption from either stays.
+ const alt = twin.image.alt ?? image.alt;
+ if (print.pixels > twin.print.pixels) {
+ twin.image = { file: image.file, alt };
+ twin.print = print;
+ continue;
+ }
+ twin.image = { ...twin.image, alt };
+ }
+ return kept.map((k) => k.image);
+}
+
+function samePicture(a: PicturePrint, b: PicturePrint): boolean {
+ if (Math.abs(a.aspect - b.aspect) > SAME_SHAPE_TOLERANCE * a.aspect) {
+ return false;
+ }
+ let diff = 0;
+ for (let i = 0; i < a.thumb.length; i++) {
+ diff += Math.abs(a.thumb[i] - b.thumb[i]);
+ }
+ return diff / a.thumb.length <= SAME_PICTURE_DIFF;
+}
+
+async function picturePrint(file: string): Promise<PicturePrint | null> {
+ const [dims, thumb] = await Promise.all([videoDimensions(file), pictureThumb(file)]);
+ if (!dims || !thumb) {
+ return null;
+ }
+ return { pixels: dims.width * dims.height, aspect: dims.width / dims.height, thumb };
+}
+
+/** The picture shrunk to a gray PRINT_SIDE square, transparency laid on
+ * white so a logo's empty corners read the same in every rendition. */
+function pictureThumb(file: string): Promise<Buffer | null> {
+ const side = `${PRINT_SIDE}x${PRINT_SIDE}`;
+ return new Promise((resolve) => {
+ const p = spawn("ffmpeg", [
+ "-v", "error",
+ "-i", file,
+ "-filter_complex",
+ `[0:v]scale=${side}:flags=area,format=rgba[pic];color=white:size=${side}[bg];[bg][pic]overlay=shortest=1,format=gray`,
+ "-frames:v", "1",
+ "-f", "rawvideo", "-",
+ ]);
+ const chunks: Buffer[] = [];
+ const timer = setTimeout(() => {
+ p.kill("SIGKILL");
+ resolve(null);
+ }, 30_000);
+ timer.unref();
+ p.stdout.on("data", (d: Buffer) => chunks.push(d));
+ p.on("close", (code) => {
+ clearTimeout(timer);
+ const thumb = Buffer.concat(chunks);
+ resolve(code === 0 && thumb.length === PRINT_SIDE * PRINT_SIDE ? thumb : null);
+ });
+ p.on("error", () => {
+ clearTimeout(timer);
+ resolve(null);
+ });
+ });
+}
+
/** An absolute http(s) address for a page's link, or nothing for the inline
* and non-web schemes an import can't fetch. */
function absolute(raw: string | undefined, base: string): string | undefined {
diff --git a/site/src/i18n/messages/.source-lock.json b/site/src/i18n/messages/.source-lock.json
index 8d9d4a60..347ee39e 100644
--- a/site/src/i18n/messages/.source-lock.json
+++ b/site/src/i18n/messages/.source-lock.json
@@ -761,7 +761,7 @@
"free_transcription_exhausted": "fe846b7f035b",
"image_add_failed": "bc0856874c2c",
"image_add_to_project_failed": "6f52878d6df8",
- "image_download_failed": "77e43cd1cd45",
+ "image_download_failed": "5871befe3472",
"image_generation_failed": "3f95bf82d7e5",
"image_import_failed": "d4c70fd1c009",
"image_output_missing": "1c6da081c7d5",
@@ -3800,7 +3800,7 @@
"free_transcription_exhausted": "fe846b7f035b",
"image_add_failed": "bc0856874c2c",
"image_add_to_project_failed": "6f52878d6df8",
- "image_download_failed": "77e43cd1cd45",
+ "image_download_failed": "5871befe3472",
"image_generation_failed": "3f95bf82d7e5",
"image_import_failed": "d4c70fd1c009",
"image_output_missing": "1c6da081c7d5",
@@ -6839,7 +6839,7 @@
"free_transcription_exhausted": "fe846b7f035b",
"image_add_failed": "bc0856874c2c",
"image_add_to_project_failed": "6f52878d6df8",
- "image_download_failed": "77e43cd1cd45",
+ "image_download_failed": "5871befe3472",
"image_generation_failed": "3f95bf82d7e5",
"image_import_failed": "d4c70fd1c009",
"image_output_missing": "1c6da081c7d5",
@@ -9878,7 +9878,7 @@
"free_transcription_exhausted": "fe846b7f035b",
"image_add_failed": "bc0856874c2c",
"image_add_to_project_failed": "6f52878d6df8",
- "image_download_failed": "77e43cd1cd45",
+ "image_download_failed": "5871befe3472",
"image_generation_failed": "3f95bf82d7e5",
"image_import_failed": "d4c70fd1c009",
"image_output_missing": "1c6da081c7d5",
@@ -12917,7 +12917,7 @@
"free_transcription_exhausted": "fe846b7f035b",
"image_add_failed": "bc0856874c2c",
"image_add_to_project_failed": "6f52878d6df8",
- "image_download_failed": "77e43cd1cd45",
+ "image_download_failed": "5871befe3472",
"image_generation_failed": "3f95bf82d7e5",
"image_import_failed": "d4c70fd1c009",
"image_output_missing": "1c6da081c7d5",
@@ -15956,7 +15956,7 @@
"free_transcription_exhausted": "fe846b7f035b",
"image_add_failed": "bc0856874c2c",
"image_add_to_project_failed": "6f52878d6df8",
- "image_download_failed": "77e43cd1cd45",
+ "image_download_failed": "5871befe3472",
"image_generation_failed": "3f95bf82d7e5",
"image_import_failed": "d4c70fd1c009",
"image_output_missing": "1c6da081c7d5",
@@ -18995,7 +18995,7 @@
"free_transcription_exhausted": "fe846b7f035b",
"image_add_failed": "bc0856874c2c",
"image_add_to_project_failed": "6f52878d6df8",
- "image_download_failed": "77e43cd1cd45",
+ "image_download_failed": "5871befe3472",
"image_generation_failed": "3f95bf82d7e5",
"image_import_failed": "d4c70fd1c009",
"image_output_missing": "1c6da081c7d5",
diff --git a/site/src/i18n/messages/de/errors.json b/site/src/i18n/messages/de/errors.json
index bed2654d..c03f489f 100644
--- a/site/src/i18n/messages/de/errors.json
+++ b/site/src/i18n/messages/de/errors.json
@@ -122,7 +122,7 @@
"free_transcription_exhausted": "Das Guthaben für kostenlose Transkriptionen ist aufgebraucht.",
"image_add_failed": "Bild konnte nicht hinzugefügt werden.",
"image_add_to_project_failed": "Bild konnte nicht zum Projekt hinzugefügt werden.",
- "image_download_failed": "Bild konnte nicht heruntergeladen werden.",
+ "image_download_failed": "Dieses Bild konnte nicht heruntergeladen werden ({status}).",
"image_generation_failed": "Bilderstellung fehlgeschlagen.",
"image_import_failed": "Bild konnte nicht importiert werden.",
"image_output_missing": "Der Anbieter hat kein Bild zurückgegeben.",
diff --git a/site/src/i18n/messages/en/errors.json b/site/src/i18n/messages/en/errors.json
index 7bddd204..76a27396 100644
--- a/site/src/i18n/messages/en/errors.json
+++ b/site/src/i18n/messages/en/errors.json
@@ -122,7 +122,7 @@
"free_transcription_exhausted": "The free transcription allowance is used up.",
"image_add_failed": "Could not add the image.",
"image_add_to_project_failed": "Could not add the image to the project.",
- "image_download_failed": "Could not download that image.",
+ "image_download_failed": "Could not download that image ({status}).",
"image_generation_failed": "Image generation failed.",
"image_import_failed": "Could not import the image.",
"image_output_missing": "The provider returned no image.",
diff --git a/site/src/i18n/messages/es/errors.json b/site/src/i18n/messages/es/errors.json
index 5694ec22..bd807781 100644
--- a/site/src/i18n/messages/es/errors.json
+++ b/site/src/i18n/messages/es/errors.json
@@ -122,7 +122,7 @@
"free_transcription_exhausted": "Se agotó el límite de transcripción gratuita.",
"image_add_failed": "No se pudo agregar la imagen.",
"image_add_to_project_failed": "No se pudo agregar la imagen al proyecto.",
- "image_download_failed": "No se pudo descargar esa imagen.",
+ "image_download_failed": "No se pudo descargar esa imagen ({status}).",
"image_generation_failed": "Falló la generación de la imagen.",
"image_import_failed": "No se pudo importar la imagen.",
"image_output_missing": "El proveedor no devolvió ninguna imagen.",
diff --git a/site/src/i18n/messages/fr/errors.json b/site/src/i18n/messages/fr/errors.json
index 8b9ed235..b77891b2 100644
--- a/site/src/i18n/messages/fr/errors.json
+++ b/site/src/i18n/messages/fr/errors.json
@@ -122,7 +122,7 @@
"free_transcription_exhausted": "Le quota de transcription gratuite est épuisé.",
"image_add_failed": "Impossible d'ajouter l'image.",
"image_add_to_project_failed": "Impossible d'ajouter l'image au projet.",
- "image_download_failed": "Impossible de télécharger cette image.",
+ "image_download_failed": "Impossible de télécharger cette image ({status}).",
"image_generation_failed": "Échec de la génération de l'image.",
"image_import_failed": "Impossible d'importer l'image.",
"image_output_missing": "Le fournisseur n'a renvoyé aucune image.",
diff --git a/site/src/i18n/messages/ja/errors.json b/site/src/i18n/messages/ja/errors.json
index 0f963a48..84e1a6cd 100644
--- a/site/src/i18n/messages/ja/errors.json
+++ b/site/src/i18n/messages/ja/errors.json
@@ -122,7 +122,7 @@
"free_transcription_exhausted": "無料の文字起こし枠を使い切りました。",
"image_add_failed": "画像を追加できませんでした。",
"image_add_to_project_failed": "プロジェクトに画像を追加できませんでした。",
- "image_download_failed": "その画像をダウンロードできませんでした。",
+ "image_download_failed": "その画像をダウンロードできませんでした ({status})。",
"image_generation_failed": "画像の生成に失敗しました。",
"image_import_failed": "画像をインポートできませんでした。",
"image_output_missing": "プロバイダーから画像が返されませんでした。",
diff --git a/site/src/i18n/messages/ko/errors.json b/site/src/i18n/messages/ko/errors.json
index 37493014..94b22b3e 100644
--- a/site/src/i18n/messages/ko/errors.json
+++ b/site/src/i18n/messages/ko/errors.json
@@ -122,7 +122,7 @@
"free_transcription_exhausted": "무료 전사 허용량을 모두 소진했어요.",
"image_add_failed": "이미지를 추가하지 못했어요.",
"image_add_to_project_failed": "프로젝트에 이미지를 추가하지 못했어요.",
- "image_download_failed": "해당 이미지를 다운로드하지 못했어요.",
+ "image_download_failed": "해당 이미지를 다운로드하지 못했어요 ({status}).",
"image_generation_failed": "이미지 생성에 실패했어요.",
"image_import_failed": "이미지를 가져오지 못했어요.",
"image_output_missing": "제공자로부터 반환된 이미지가 없어요.",
diff --git a/site/src/i18n/messages/pt-BR/errors.json b/site/src/i18n/messages/pt-BR/errors.json
index 6d41d96f..6b4aeff7 100644
--- a/site/src/i18n/messages/pt-BR/errors.json
+++ b/site/src/i18n/messages/pt-BR/errors.json
@@ -122,7 +122,7 @@
"free_transcription_exhausted": "O limite de transcrição gratuita foi atingido.",
"image_add_failed": "Não foi possível adicionar a imagem.",
"image_add_to_project_failed": "Não foi possível adicionar a imagem ao projeto.",
- "image_download_failed": "Não foi possível baixar essa imagem.",
+ "image_download_failed": "Não foi possível baixar essa imagem ({status}).",
"image_generation_failed": "Falha ao gerar a imagem.",
"image_import_failed": "Não foi possível importar a imagem.",
"image_output_missing": "O provedor não retornou nenhuma imagem.",
diff --git a/site/src/i18n/messages/zh-CN/errors.json b/site/src/i18n/messages/zh-CN/errors.json
index b5297ee0..ff82b63f 100644
--- a/site/src/i18n/messages/zh-CN/errors.json
+++ b/site/src/i18n/messages/zh-CN/errors.json
@@ -122,7 +122,7 @@
"free_transcription_exhausted": "免费转录额度已用尽。",
"image_add_failed": "无法添加图像。",
"image_add_to_project_failed": "无法将图像添加到项目中。",
- "image_download_failed": "无法下载该图像。",
+ "image_download_failed": "无法下载该图片({status})。",
"image_generation_failed": "图像生成失败。",
"image_import_failed": "无法导入图像。",
"image_output_missing": "服务提供方未返回任何图像。",