Open source CapCut alternative.
TypeScript
fix(site): import each page picture once [rebuild]
David Le2aad2c40de42
diff --git a/site/src/cut/server/urlDownload.e2e.test.ts b/site/src/cut/server/urlDownload.e2e.test.tsindex a60130b0..da6e4782 100644--- a/site/src/cut/server/urlDownload.e2e.test.ts+++ b/site/src/cut/server/urlDownload.e2e.test.ts@@ -6,6 +6,7 @@ import { createServer } from "node:http";import os from "node:os";import path from "node:path";import { download, playableMedia } from "./urlDownload";+import { videoDimensions } from "./util";// What a link import lands has to play on every surface it reaches: the// phone's viewer and the Mac's player go through AVFoundation, which decodes@@ -141,3 +142,72 @@ suite("playableMedia", () => {}});});++// Serve fixed bodies by path on a local port, so a page and its pictures+// import the way a real site's would.+async function serve(routes: Record<string, { type: string; body: Buffer | string; status?: number }>) {+ const server = createServer((request, response) => {+ const route = routes[new URL(request.url!, "http://localhost").pathname];+ if (!route) {+ response.writeHead(404).end();+ return;+ }+ response.writeHead(route.status ?? 200, { "Content-Type": route.type }).end(route.body);+ });+ server.listen(0, "127.0.0.1");+ await once(server, "listening");+ const address = server.address();+ if (!address || typeof address === "string") throw new Error("Missing test server address.");+ return {+ base: `http://127.0.0.1:${address.port}`,+ close: () => new Promise<void>((resolve) => server.close(() => resolve())),+ };+}++suite("page pictures", () => {+ // A file page links one picture as its preview, a smaller rendition and a+ // srcset entry; the import keeps it once, at its sharpest size.+ test("one picture shown at several sizes imports once, at its largest", async () => {+ const dir = await mkdtemp(path.join(os.tmpdir(), "cut-page-"));+ const big = path.join(dir, "big.png");+ const small = path.join(dir, "small.png");+ const bars = path.join(dir, "bars.png");+ ff(["-f", "lavfi", "-i", "testsrc=size=1280x1280", "-frames:v", "1", big]);+ ff(["-i", big, "-vf", "scale=960:960", small]);+ ff(["-f", "lavfi", "-i", "mandelbrot=size=960x540", "-frames:v", "1", bars]);+ const prose = "A page about one logo, with enough words for a reader to keep. ".repeat(4);+ const site = await serve({+ "/page": {+ type: "text/html",+ body: `<html><head><title>Logo</title><meta property="og:image" content="/big.png?from=og"></head>+ <body><article><h1>Logo</h1><p>${prose}</p>+ <img src="/small.png" alt="Logo"><img srcset="/big.png?from=srcset 1280w"><img src="/bars.png" alt="Bars">+ </article></body></html>`,+ },+ "/big.png": { type: "image/png", body: await readFile(big) },+ "/small.png": { type: "image/png", body: await readFile(small) },+ "/bars.png": { type: "image/png", body: await readFile(bars) },+ });+ try {+ const result = await download(`${site.base}/page`, dir);+ expect(result.files).toHaveLength(2);+ expect((await videoDimensions(result.files[0].file))?.width).toBe(1280);+ expect(result.files[1].title).toBe("Bars");+ } finally {+ await site.close();+ await rm(dir, { force: true, recursive: true });+ }+ }, 60_000);++ // The chat retries a refused image link only when it can see why it failed.+ test("a refused image link says what the server answered", async () => {+ const dir = await mkdtemp(path.join(os.tmpdir(), "cut-image-"));+ const site = await serve({ "/logo.png": { type: "text/html", body: "Bad thumbnail width", status: 400 } });+ try {+ await expect(download(`${site.base}/logo.png`, dir)).rejects.toThrow("400");+ } finally {+ await site.close();+ await rm(dir, { force: true, recursive: true });+ }+ });+});diff --git a/site/src/cut/server/urlDownload.ts b/site/src/cut/server/urlDownload.tsindex d001dff9..b4e87a08 100644--- a/site/src/cut/server/urlDownload.ts+++ b/site/src/cut/server/urlDownload.ts@@ -7,6 +7,7 @@ import { Defuddle } from "defuddle/node";import { parseHTML } from "linkedom";import { convertToMp4, fitsAlready, streamCodecs } from "./convert";import type { LibrarySource } from "./library";+import { videoDimensions } from "./util";import { pexelsFileUrl, pexelsVideo } from "./pexels";import { UserError } from "../lib/userError";@@ -274,7 +275,7 @@ async function savePoster(async function downloadDirectImage(url: string, dir: string): Promise<Downloaded> {const res = await fetch(url);- if (!res.ok) throw new UserError("image_download_failed", "Could not download that image.");+ if (!res.ok) throw new UserError("image_download_failed", `Could not download that image (${res.status}).`, { status: res.status });const name = path.basename(new URL(url).pathname) || "image.jpg";const file = path.join(dir, name);await writeFile(file, Buffer.from(await res.arrayBuffer()));@@ -543,7 +544,7 @@ async function pageImages(const saved = await Promise.all(refs.map((ref, i) => savePageImage(ref, pageUrl, i, dir).catch(() => null)));- const kept = saved.filter((f): f is SavedImage => !!f).slice(0, MAX_PAGE_IMAGES);+ const kept = (await distinctPictures(saved.filter((f): f is SavedImage => !!f))).slice(0, MAX_PAGE_IMAGES);// A picture the page captioned keeps that caption; the rest are named after// the page and counted, the way a multi-photo post's are.return kept.map((img, i) => ({@@ -646,6 +647,101 @@ async function savePageImage(return { file, alt: ref.alt };}+// Two files are one picture when their shapes agree and their 16×16+// thumbnails differ by at most this much per pixel on average, out of 255.+// A file page shows one logo as its preview, a thumbnail and a srcset entry.+const PRINT_SIDE = 16;+const SAME_PICTURE_DIFF = 6;+const SAME_SHAPE_TOLERANCE = 0.02;++interface PicturePrint {+ pixels: number;+ aspect: number;+ thumb: Buffer;+}++interface PrintedImage {+ image: SavedImage;+ print: PicturePrint;+}++/** Keep each picture a page shows once, at its largest size, in the place it+ * first appeared. A file that can't be read stays, since nothing says it+ * repeats another. */+async function distinctPictures(images: SavedImage[]): Promise<SavedImage[]> {+ const prints = await Promise.all(images.map((img) => picturePrint(img.file)));+ const kept: { image: SavedImage; print: PicturePrint | null }[] = [];+ for (const [i, image] of images.entries()) {+ const print = prints[i];+ const twin = print && kept.find((k): k is PrintedImage => !!k.print && samePicture(k.print, print));+ if (!print || !twin) {+ kept.push({ image, print });+ continue;+ }++ // The sharper copy takes the first one's place; a caption from either stays.+ const alt = twin.image.alt ?? image.alt;+ if (print.pixels > twin.print.pixels) {+ twin.image = { file: image.file, alt };+ twin.print = print;+ continue;+ }+ twin.image = { ...twin.image, alt };+ }+ return kept.map((k) => k.image);+}++function samePicture(a: PicturePrint, b: PicturePrint): boolean {+ if (Math.abs(a.aspect - b.aspect) > SAME_SHAPE_TOLERANCE * a.aspect) {+ return false;+ }+ let diff = 0;+ for (let i = 0; i < a.thumb.length; i++) {+ diff += Math.abs(a.thumb[i] - b.thumb[i]);+ }+ return diff / a.thumb.length <= SAME_PICTURE_DIFF;+}++async function picturePrint(file: string): Promise<PicturePrint | null> {+ const [dims, thumb] = await Promise.all([videoDimensions(file), pictureThumb(file)]);+ if (!dims || !thumb) {+ return null;+ }+ return { pixels: dims.width * dims.height, aspect: dims.width / dims.height, thumb };+}++/** The picture shrunk to a gray PRINT_SIDE square, transparency laid on+ * white so a logo's empty corners read the same in every rendition. */+function pictureThumb(file: string): Promise<Buffer | null> {+ const side = `${PRINT_SIDE}x${PRINT_SIDE}`;+ return new Promise((resolve) => {+ const p = spawn("ffmpeg", [+ "-v", "error",+ "-i", file,+ "-filter_complex",+ `[0:v]scale=${side}:flags=area,format=rgba[pic];color=white:size=${side}[bg];[bg][pic]overlay=shortest=1,format=gray`,+ "-frames:v", "1",+ "-f", "rawvideo", "-",+ ]);+ const chunks: Buffer[] = [];+ const timer = setTimeout(() => {+ p.kill("SIGKILL");+ resolve(null);+ }, 30_000);+ timer.unref();+ p.stdout.on("data", (d: Buffer) => chunks.push(d));+ p.on("close", (code) => {+ clearTimeout(timer);+ const thumb = Buffer.concat(chunks);+ resolve(code === 0 && thumb.length === PRINT_SIDE * PRINT_SIDE ? thumb : null);+ });+ p.on("error", () => {+ clearTimeout(timer);+ resolve(null);+ });+ });+}+/** An absolute http(s) address for a page's link, or nothing for the inline* and non-web schemes an import can't fetch. */function absolute(raw: string | undefined, base: string): string | undefined {diff --git a/site/src/i18n/messages/.source-lock.json b/site/src/i18n/messages/.source-lock.jsonindex 8d9d4a60..347ee39e 100644--- a/site/src/i18n/messages/.source-lock.json+++ b/site/src/i18n/messages/.source-lock.json@@ -761,7 +761,7 @@"free_transcription_exhausted": "fe846b7f035b","image_add_failed": "bc0856874c2c","image_add_to_project_failed": "6f52878d6df8",- "image_download_failed": "77e43cd1cd45",+ "image_download_failed": "5871befe3472","image_generation_failed": "3f95bf82d7e5","image_import_failed": "d4c70fd1c009","image_output_missing": "1c6da081c7d5",@@ -3800,7 +3800,7 @@"free_transcription_exhausted": "fe846b7f035b","image_add_failed": "bc0856874c2c","image_add_to_project_failed": "6f52878d6df8",- "image_download_failed": "77e43cd1cd45",+ "image_download_failed": "5871befe3472","image_generation_failed": "3f95bf82d7e5","image_import_failed": "d4c70fd1c009","image_output_missing": "1c6da081c7d5",@@ -6839,7 +6839,7 @@"free_transcription_exhausted": "fe846b7f035b","image_add_failed": "bc0856874c2c","image_add_to_project_failed": "6f52878d6df8",- "image_download_failed": "77e43cd1cd45",+ "image_download_failed": "5871befe3472","image_generation_failed": "3f95bf82d7e5","image_import_failed": "d4c70fd1c009","image_output_missing": "1c6da081c7d5",@@ -9878,7 +9878,7 @@"free_transcription_exhausted": "fe846b7f035b","image_add_failed": "bc0856874c2c","image_add_to_project_failed": "6f52878d6df8",- "image_download_failed": "77e43cd1cd45",+ "image_download_failed": "5871befe3472","image_generation_failed": "3f95bf82d7e5","image_import_failed": "d4c70fd1c009","image_output_missing": "1c6da081c7d5",@@ -12917,7 +12917,7 @@"free_transcription_exhausted": "fe846b7f035b","image_add_failed": "bc0856874c2c","image_add_to_project_failed": "6f52878d6df8",- "image_download_failed": "77e43cd1cd45",+ "image_download_failed": "5871befe3472","image_generation_failed": "3f95bf82d7e5","image_import_failed": "d4c70fd1c009","image_output_missing": "1c6da081c7d5",@@ -15956,7 +15956,7 @@"free_transcription_exhausted": "fe846b7f035b","image_add_failed": "bc0856874c2c","image_add_to_project_failed": "6f52878d6df8",- "image_download_failed": "77e43cd1cd45",+ "image_download_failed": "5871befe3472","image_generation_failed": "3f95bf82d7e5","image_import_failed": "d4c70fd1c009","image_output_missing": "1c6da081c7d5",@@ -18995,7 +18995,7 @@"free_transcription_exhausted": "fe846b7f035b","image_add_failed": "bc0856874c2c","image_add_to_project_failed": "6f52878d6df8",- "image_download_failed": "77e43cd1cd45",+ "image_download_failed": "5871befe3472","image_generation_failed": "3f95bf82d7e5","image_import_failed": "d4c70fd1c009","image_output_missing": "1c6da081c7d5",diff --git a/site/src/i18n/messages/de/errors.json b/site/src/i18n/messages/de/errors.jsonindex bed2654d..c03f489f 100644--- a/site/src/i18n/messages/de/errors.json+++ b/site/src/i18n/messages/de/errors.json@@ -122,7 +122,7 @@"free_transcription_exhausted": "Das Guthaben für kostenlose Transkriptionen ist aufgebraucht.","image_add_failed": "Bild konnte nicht hinzugefügt werden.","image_add_to_project_failed": "Bild konnte nicht zum Projekt hinzugefügt werden.",- "image_download_failed": "Bild konnte nicht heruntergeladen werden.",+ "image_download_failed": "Dieses Bild konnte nicht heruntergeladen werden ({status}).","image_generation_failed": "Bilderstellung fehlgeschlagen.","image_import_failed": "Bild konnte nicht importiert werden.","image_output_missing": "Der Anbieter hat kein Bild zurückgegeben.",diff --git a/site/src/i18n/messages/en/errors.json b/site/src/i18n/messages/en/errors.jsonindex 7bddd204..76a27396 100644--- a/site/src/i18n/messages/en/errors.json+++ b/site/src/i18n/messages/en/errors.json@@ -122,7 +122,7 @@"free_transcription_exhausted": "The free transcription allowance is used up.","image_add_failed": "Could not add the image.","image_add_to_project_failed": "Could not add the image to the project.",- "image_download_failed": "Could not download that image.",+ "image_download_failed": "Could not download that image ({status}).","image_generation_failed": "Image generation failed.","image_import_failed": "Could not import the image.","image_output_missing": "The provider returned no image.",diff --git a/site/src/i18n/messages/es/errors.json b/site/src/i18n/messages/es/errors.jsonindex 5694ec22..bd807781 100644--- a/site/src/i18n/messages/es/errors.json+++ b/site/src/i18n/messages/es/errors.json@@ -122,7 +122,7 @@"free_transcription_exhausted": "Se agotó el límite de transcripción gratuita.","image_add_failed": "No se pudo agregar la imagen.","image_add_to_project_failed": "No se pudo agregar la imagen al proyecto.",- "image_download_failed": "No se pudo descargar esa imagen.",+ "image_download_failed": "No se pudo descargar esa imagen ({status}).","image_generation_failed": "Falló la generación de la imagen.","image_import_failed": "No se pudo importar la imagen.","image_output_missing": "El proveedor no devolvió ninguna imagen.",diff --git a/site/src/i18n/messages/fr/errors.json b/site/src/i18n/messages/fr/errors.jsonindex 8b9ed235..b77891b2 100644--- a/site/src/i18n/messages/fr/errors.json+++ b/site/src/i18n/messages/fr/errors.json@@ -122,7 +122,7 @@"free_transcription_exhausted": "Le quota de transcription gratuite est épuisé.","image_add_failed": "Impossible d'ajouter l'image.","image_add_to_project_failed": "Impossible d'ajouter l'image au projet.",- "image_download_failed": "Impossible de télécharger cette image.",+ "image_download_failed": "Impossible de télécharger cette image ({status}).","image_generation_failed": "Échec de la génération de l'image.","image_import_failed": "Impossible d'importer l'image.","image_output_missing": "Le fournisseur n'a renvoyé aucune image.",diff --git a/site/src/i18n/messages/ja/errors.json b/site/src/i18n/messages/ja/errors.jsonindex 0f963a48..84e1a6cd 100644--- a/site/src/i18n/messages/ja/errors.json+++ b/site/src/i18n/messages/ja/errors.json@@ -122,7 +122,7 @@"free_transcription_exhausted": "無料の文字起こし枠を使い切りました。","image_add_failed": "画像を追加できませんでした。","image_add_to_project_failed": "プロジェクトに画像を追加できませんでした。",- "image_download_failed": "その画像をダウンロードできませんでした。",+ "image_download_failed": "その画像をダウンロードできませんでした ({status})。","image_generation_failed": "画像の生成に失敗しました。","image_import_failed": "画像をインポートできませんでした。","image_output_missing": "プロバイダーから画像が返されませんでした。",diff --git a/site/src/i18n/messages/ko/errors.json b/site/src/i18n/messages/ko/errors.jsonindex 37493014..94b22b3e 100644--- a/site/src/i18n/messages/ko/errors.json+++ b/site/src/i18n/messages/ko/errors.json@@ -122,7 +122,7 @@"free_transcription_exhausted": "무료 전사 허용량을 모두 소진했어요.","image_add_failed": "이미지를 추가하지 못했어요.","image_add_to_project_failed": "프로젝트에 이미지를 추가하지 못했어요.",- "image_download_failed": "해당 이미지를 다운로드하지 못했어요.",+ "image_download_failed": "해당 이미지를 다운로드하지 못했어요 ({status}).","image_generation_failed": "이미지 생성에 실패했어요.","image_import_failed": "이미지를 가져오지 못했어요.","image_output_missing": "제공자로부터 반환된 이미지가 없어요.",diff --git a/site/src/i18n/messages/pt-BR/errors.json b/site/src/i18n/messages/pt-BR/errors.jsonindex 6d41d96f..6b4aeff7 100644--- a/site/src/i18n/messages/pt-BR/errors.json+++ b/site/src/i18n/messages/pt-BR/errors.json@@ -122,7 +122,7 @@"free_transcription_exhausted": "O limite de transcrição gratuita foi atingido.","image_add_failed": "Não foi possível adicionar a imagem.","image_add_to_project_failed": "Não foi possível adicionar a imagem ao projeto.",- "image_download_failed": "Não foi possível baixar essa imagem.",+ "image_download_failed": "Não foi possível baixar essa imagem ({status}).","image_generation_failed": "Falha ao gerar a imagem.","image_import_failed": "Não foi possível importar a imagem.","image_output_missing": "O provedor não retornou nenhuma imagem.",diff --git a/site/src/i18n/messages/zh-CN/errors.json b/site/src/i18n/messages/zh-CN/errors.jsonindex b5297ee0..ff82b63f 100644--- a/site/src/i18n/messages/zh-CN/errors.json+++ b/site/src/i18n/messages/zh-CN/errors.json@@ -122,7 +122,7 @@"free_transcription_exhausted": "免费转录额度已用尽。","image_add_failed": "无法添加图像。","image_add_to_project_failed": "无法将图像添加到项目中。",- "image_download_failed": "无法下载该图像。",+ "image_download_failed": "无法下载该图片({status})。","image_generation_failed": "图像生成失败。","image_import_failed": "无法导入图像。","image_output_missing": "服务提供方未返回任何图像。",