Open source CapCut alternative.
TypeScript
feat(site): a link's sound can come down alone [rebuild]
David Le33374d01e49e
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex 3095eeda..7fb127ac 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -97,8 +97,16 @@ export const AI_PANEL_TOOLS = [{name: "import_url",description:- "Read any URL — TikTok, YouTube, Instagram Reels, an X/Twitter post or Article, an ordinary web page, or a direct video/audio/image link — with the bundled downloader and import what it holds into the project. Free and local. A web page comes back as its article text plus the pictures on it; a post as its video or photos; and the source's own words (returned as sourceText) are quoted for the user beside the media automatically — don't retype them in your reply. A source that is only words returns sourceText with no assets, which is a success: read it and answer from it. This is how you look something up: point it at the page and read what comes back. Media lands on a card in this chat and the user drags it to the timeline, Media, or the Library; place it yourself (add_clip) only when they asked for it in the cut. A short clip downloads in seconds; a long video can take a couple of minutes.",- inputSchema: obj({ url: str("The page or media URL to download") }, ["url"]),+ "Read any URL — TikTok, YouTube, Instagram Reels, an X/Twitter post or Article, an ordinary web page, or a direct video/audio/image link — with the bundled downloader and import what it holds into the project. Free and local. A web page comes back as its article text plus the pictures on it; a post as its video or photos; and the source's own words (returned as sourceText) are quoted for the user beside the media automatically — don't retype them in your reply. A source that is only words returns sourceText with no assets, which is a success: read it and answer from it. This is how you look something up: point it at the page and read what comes back. When the user wants only the sound — a song, a soundtrack, a podcast — pass audio_only and the source's audio track lands as an audio asset. Media lands on a card in this chat and the user drags it to the timeline, Media, or the Library; place it yourself (add_clip) only when they asked for it in the cut. A short clip downloads in seconds; a long video can take a couple of minutes.",+ inputSchema: obj(+ {+ url: str("The page or media URL to download"),+ audio_only: bool(+ "Import only the source's audio track, extracted to an audio file (default false)"+ ),+ },+ ["url"]+ ),},{name: "list_skills",diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 074603b6..09436fd0 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -2563,7 +2563,9 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {// Captured before the download: the media files under the chat that// asked, even if the user switches threads while it downloads.const chatId = chatOwner();- const { assets, text } = await importUrlMedia(projectId, url);+ const { assets, text } = await importUrlMedia(projectId, url, {+ audio: input.audio_only === true,+ });for (const asset of assets) tagChatAsset(asset.id, chatId);return {assets: assets.map((asset) => ({diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 1849b2b9..caccf68f 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -845,7 +845,8 @@ export async function importStockMusic(* an article, or a video's title and description. */export async function importUrlMedia(projectId: string,- url: string+ url: string,+ opts: { audio?: boolean } = {}): Promise<{ assets: MediaAsset[]; text?: string }> {url = normalizeLink(url);// Pinned at start: the download can outlast navigation into a project of@@ -854,13 +855,13 @@ export async function importUrlMedia(// A browser project has no server of its own to run the fetch, so its import// rides the cloud worker and the bytes come back down.if (backend.kind === "browser") {- const staged = await importUrlThroughCloud(projectId, backend, url);+ const staged = await importUrlThroughCloud(projectId, backend, url, opts);return adoptImportedFiles(projectId, backend, staged);}const res = await backend.fetch(`/api/cut/projects/${projectId}/import-url`, {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ url }),+ body: JSON.stringify({ url, ...(opts.audio ? { audio: true } : {}) }),});let body: { files?: { fileName: string; title: string }[]; text?: string; error?: string };if (backend.kind === "cloud") {@@ -911,12 +912,13 @@ async function adoptImportedFiles(async function importUrlThroughCloud(projectId: string,backend: CutBackend,- url: string+ url: string,+ opts: { audio?: boolean } = {}): Promise<{ files: { fileName: string; title: string }[]; text?: string }> {const res = await cloudBackend.fetch("/api/cut/library/import-url", {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ url }),+ body: JSON.stringify({ url, ...(opts.audio ? { audio: true } : {}) }),});const started = await apiJson<{ jobId?: string }>(res);if (!res.ok || !started.jobId) throw new Error(started.error ?? "Could not import that URL.");diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 6408da73..12b8a7e3 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -364,7 +364,7 @@ The Library is shared across every project: folders, reusable assets, and templa- Assets from the user's iOS app carry an origin: "camera" is a clip they recorded on their phone (their Camera Roll — "the video I shot on my phone" lives here), "inspiration" a reference saved into the Library's Inspiration folder. library_add imports either like any other asset. A camera clip's name is the clock it was shot on; its title, when it has one, was read off the clip itself and is what the user sees on the card — match a request like "the one about the launch" against the title.- notes_list reads the user's synced notes (written on their phone or the Notes tab) with the folders they are filed in and the labels they carry — the place to look when they point at "my note", or name a folder or label, for a script, caption, or voiceover text.- library_organize handles folders (create/rename/delete), filing (move_asset), and deletes (permanent — explicit ask only).-- import_url reads any URL (TikTok, YouTube, Instagram, an X post or Article, a web page, direct links): a page comes back as its article text and its pictures, a post as its video or photos, and a source that is only words as sourceText alone. Media lands as a card in the chat with that text quoted beside it; the user drags a card to the timeline, Media, or the Library. Place it with add_clip only when they asked for it in the cut.+- import_url reads any URL (TikTok, YouTube, Instagram, an X post or Article, a web page, direct links): a page comes back as its article text and its pictures, a post as its video or photos, and a source that is only words as sourceText alone. When the user asks for the sound alone — a song, a soundtrack — pass audio_only and the audio track lands as an audio asset. Media lands as a card in the chat with that text quoted beside it; the user drags a card to the timeline, Media, or the Library. Place it with add_clip only when they asked for it in the cut.Attachments: media files dropped on the chat import into project media by themselves; library attachments wait for library_add.`,"publish-and-export": `# Publish & exportdiff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.tsindex 89ef4160..263390be 100644--- a/site/src/cut/server/cloud/jobs.ts+++ b/site/src/cut/server/cloud/jobs.ts@@ -471,7 +471,7 @@ export const jobsCloud = {* engine's synchronous route. */async importUrl(userId: string, projectId: string, req: Request) {try {- const { url } = (await req.json()) as { url?: string };+ const { url, audio } = (await req.json()) as { url?: string; audio?: boolean };if (!url) return err("No URL provided.", 400);if (!(await getProject(userId, projectId))) return err("Project not found.", 404);const capped = await renderJobCheck(userId);@@ -481,7 +481,7 @@ export const jobsCloud = {userId,projectId,kind: "import_url",- spec: { url } as unknown as Prisma.InputJsonValue,+ spec: { url, ...(audio === true ? { audio: true } : {}) } as unknown as Prisma.InputJsonValue,},});wakeRenderWorker();@@ -496,7 +496,11 @@ export const jobsCloud = {* the assets, and the client polls this job for them. */async importUrlToLibrary(userId: string, req: Request) {try {- const { url, origin } = (await req.json()) as { url?: string; origin?: string };+ const { url, origin, audio } = (await req.json()) as {+ url?: string;+ origin?: string;+ audio?: boolean;+ };if (!url) return err("No URL provided.", 400);const capped = await renderJobCheck(userId);if (capped) return capped;@@ -507,6 +511,7 @@ export const jobsCloud = {spec: {url,target: "library",+ ...(audio === true ? { audio: true } : {}),// An inspiration link's downloads land in the Inspiration folder,// carrying the origin the Camera Roll and Library filters read....(origin === "inspiration" ? { origin } : {}),diff --git a/site/src/cut/server/http/library.ts b/site/src/cut/server/http/library.tsindex 1b14dc93..29b7960d 100644--- a/site/src/cut/server/http/library.ts+++ b/site/src/cut/server/http/library.ts@@ -113,9 +113,9 @@ export const libraryApi = {/** Download a media URL straight into the library. */async importUrl(req: Request) {try {- const { url } = (await req.json()) as { url?: string };+ const { url, audio } = (await req.json()) as { url?: string; audio?: boolean };if (!url) return err("No URL provided.", 400);- return Response.json(await importFromUrl(url));+ return Response.json(await importFromUrl(url, { audio: audio === true }));} catch (e) {return caught(e, "Could not import that URL.");}diff --git a/site/src/cut/server/http/projects.ts b/site/src/cut/server/http/projects.tsindex 0e6724e2..8f75c1a7 100644--- a/site/src/cut/server/http/projects.ts+++ b/site/src/cut/server/http/projects.ts@@ -188,9 +188,9 @@ export const projectsApi = {/** Download a media URL straight into the project's media folder. */async importUrl(req: Request, { id }: { id: string }) {try {- const { url } = (await req.json()) as { url?: string };+ const { url, audio } = (await req.json()) as { url?: string; audio?: boolean };if (!url) return err("No URL provided.", 400);- return Response.json(await importUrlToProject(id, url));+ return Response.json(await importUrlToProject(id, url, { audio: audio === true }));} catch (e) {return caught(e, "Could not import that URL.");}diff --git a/site/src/cut/server/urlDownload.ts b/site/src/cut/server/urlDownload.tsindex 14517c72..74a7ae1e 100644--- a/site/src/cut/server/urlDownload.ts+++ b/site/src/cut/server/urlDownload.ts@@ -39,6 +39,13 @@ export interface Downloaded {text?: string;}+export interface DownloadOptions {+ /** Bring back only the sound: the source's audio track, landed as an audio+ * file. Reads the media rung; a post's photos and a page's text come back as+ * themselves. */+ audio?: boolean;+}+// A video's own words for the chat: its title and description joined, capped so// a long YouTube description (link dumps, chapter lists) can't flood the chat.// TikTok/Instagram often repeat the caption as both fields, so drop a@@ -71,7 +78,11 @@ const X_URL_RE = /^https?:\/\/(?:www\.|mobile\.)?(?:x|twitter)\.com\//i;* then the post's words alone. Every other URL tries yt-dlp, then the page* itself — what it says and what it shows. When no rung produces anything, the* first failure is what surfaces — it names the real cause. */-export async function download(url: string, tmp: string): Promise<Downloaded> {+export async function download(+ url: string,+ tmp: string,+ opts: DownloadOptions = {}+): Promise<Downloaded> {if (IMAGE_URL_RE.test(url)) return downloadDirectImage(url, tmp);const rungs: (() => Promise<Downloaded | null>)[] = [];const postId = X_STATUS_RE.exec(url)?.[1];@@ -85,14 +96,14 @@ export async function download(url: string, tmp: string): Promise<Downloaded> {const post = fetchPost(postId);rungs.push(() => xPostMedia(post, url, tmp, { deferVideo: true }),- () => downloadMedia(url, tmp),+ () => downloadMedia(url, tmp, opts),() => xPostMedia(post, url, tmp, { deferVideo: false }),() => xPostText(post, url));} else if (X_ARTICLE_RE.test(url)) {rungs.push(() => xArticle({}, url, tmp));} else {- rungs.push(() => downloadMedia(url, tmp), () => readPage(url, tmp));+ rungs.push(() => downloadMedia(url, tmp, opts), () => readPage(url, tmp));}const failures: unknown[] = [];for (const rung of rungs) {@@ -116,7 +127,7 @@ class MediaHostError extends Error {}* is the last. */class MissingToolError extends Error {}-async function downloadMedia(url: string, tmp: string): Promise<Downloaded> {+async function downloadMedia(url: string, tmp: string, opts: DownloadOptions = {}): Promise<Downloaded> {// Read what sits at the URL before pulling any bytes. The extractor name// says whether this is a site that serves video, and that decides what a// failed download means: on a video host the import stops with the real@@ -139,7 +150,7 @@ async function downloadMedia(url: string, tmp: string): Promise<Downloaded> {let meta: YtMeta;let out: string;try {- ({ meta, dir: out } = await runYtDlp(url, tmp, bin));+ ({ meta, dir: out } = await runYtDlp(url, tmp, bin, opts));} catch (e) {if (!videoHost) throw e;throw new MediaHostError(e instanceof Error ? e.message : "Download failed.");@@ -710,7 +721,8 @@ const DOWNLOAD_BUDGET_MS = 8 * 60_000;async function runYtDlp(url: string,dir: string,- preferred: string+ preferred: string,+ opts: DownloadOptions = {}): Promise<{ meta: YtMeta; dir: string }> {const bins = [preferred, ...ytDlpBinaries().filter((b) => b !== preferred)];const deadline = Date.now() + DOWNLOAD_BUDGET_MS;@@ -722,7 +734,7 @@ async function runYtDlp(const out = path.join(dir, `try${i}`);await mkdir(out, { recursive: true });try {- return { meta: await ytDlp(bin, url, out, client, left), dir: out };+ return { meta: await ytDlp(bin, url, out, client, left, opts), dir: out };} catch (e) {if (e instanceof MissingToolError) throw e;first ??= e;@@ -736,26 +748,40 @@ function ytDlp(url: string,dir: string,client: string | null,- timeoutMs: number+ timeoutMs: number,+ opts: DownloadOptions = {}): Promise<YtMeta> {+ // An audio import takes the best sound stream alone and lands it as m4a.+ // AAC leads the sort — it decodes everywhere the import goes — and -x+ // extracts the sound when the site only serves a combined file, transcoding+ // an Opus-only stream through the bundled ffmpeg.+ const format = opts.audio+ ? [+ "-f", "bestaudio/best",+ "-S", "acodec:aac",+ "-x", "--audio-format", "m4a",+ ]+ : [+ // Take the best video and the best audio and put them together. Asking for+ // a ready-made mp4 first would land whatever single file the site keeps+ // around for compatibility — on YouTube that is 360p next to a 4K original.+ //+ // Codec leads the sort, resolution follows it. Apple decodes H.264 and+ // HEVC and nothing else, so a taller VP9 or AV1 stream is a file that+ // plays nowhere the import is going: the phone's viewer, the Mac's own+ // player, Safari. Instagram serves 1440p VP9 beside a 1080p-class H.264+ // file, and the H.264 one is worth having. Sound is compared last, after+ // the picture: judged any earlier it would rank YouTube's 360p+ // sound-and-picture file over the 1080p stream that carries no sound of+ // its own. A source with no H.264 at all still comes down, and the+ // conversion below makes it playable.+ "-f", "bestvideo*+bestaudio/best",+ "-S", "vcodec:h264,res,fps,hdr:sdr,acodec:aac",+ "--merge-output-format", "mp4",+ ];return ytDlpJson(bin, [...(client ? ["--extractor-args", `youtube:player_client=${client}`] : []),- // Take the best video and the best audio and put them together. Asking for- // a ready-made mp4 first would land whatever single file the site keeps- // around for compatibility — on YouTube that is 360p next to a 4K original.- //- // Codec leads the sort, resolution follows it. Apple decodes H.264 and- // HEVC and nothing else, so a taller VP9 or AV1 stream is a file that- // plays nowhere the import is going: the phone's viewer, the Mac's own- // player, Safari. Instagram serves 1440p VP9 beside a 1080p-class H.264- // file, and the H.264 one is worth having. Sound is compared last, after- // the picture: judged any earlier it would rank YouTube's 360p- // sound-and-picture file over the 1080p stream that carries no sound of- // its own. A source with no H.264 at all still comes down, and the- // conversion below makes it playable.- "-f", "bestvideo*+bestaudio/best",- "-S", "vcodec:h264,res,fps,hdr:sdr,acodec:aac",- "--merge-output-format", "mp4",+ ...format,"-o", path.join(dir, "%(id)s.%(ext)s"),"--print-json",url,diff --git a/site/src/cut/server/urlImport.ts b/site/src/cut/server/urlImport.tsindex b344a0d2..823a623b 100644--- a/site/src/cut/server/urlImport.ts+++ b/site/src/cut/server/urlImport.ts@@ -4,7 +4,7 @@ import path from "node:path";import { addDownloaded, type LibraryAsset, type LibrarySource } from "./library";import { assertLocalRuntime } from "./local-only";import { mediaDir, mediaPath as projectMediaPath, readProject } from "./projects";-import { download, type Downloaded } from "./urlDownload";+import { download, type Downloaded, type DownloadOptions } from "./urlDownload";import { uniqueName } from "./util";// Import a URL (TikTok, YouTube, Instagram, an X post, a page, …) into the@@ -21,7 +21,11 @@ let active = 0;* inside the try so a failure there still releases the slot (else a rejection* would leak it and, after MAX_ACTIVE failures, brick URL import until* restart). */-async function withDownload<T>(url: string, consume: (dl: Downloaded) => Promise<T>): Promise<T> {+async function withDownload<T>(+ url: string,+ opts: DownloadOptions,+ consume: (dl: Downloaded) => Promise<T>+): Promise<T> {assertLocalRuntime();if (!/^https?:\/\//i.test(url.trim())) throw new Error("Enter a valid http(s) URL.");if (active >= MAX_ACTIVE) {@@ -31,7 +35,7 @@ async function withDownload<T>(url: string, consume: (dl: Downloaded) => Promisetry {const tmp = await mkdtemp(path.join(os.tmpdir(), "cut-dl-"));try {- return await consume(await download(url.trim(), tmp));+ return await consume(await download(url.trim(), tmp, opts));} finally {void rm(tmp, { recursive: true, force: true });}@@ -44,8 +48,11 @@ async function withDownload<T>(url: string, consume: (dl: Downloaded) => Promise* A multi-photo post lands as one asset per photo. The Library holds media, so* a source that turned out to be only words fails here — the chat's import is* where text lands. */-export async function importFromUrl(url: string): Promise<LibraryAsset[]> {- return withDownload(url, async (dl) => {+export async function importFromUrl(+ url: string,+ opts: DownloadOptions = {}+): Promise<LibraryAsset[]> {+ return withDownload(url, opts, async (dl) => {if (dl.files.length === 0) throw new Error("That link has no media to import.");const assets: LibraryAsset[] = [];for (const f of dl.files) {@@ -61,10 +68,11 @@ export async function importFromUrl(url: string): Promise<LibraryAsset[]> {* the assets. */export async function importUrlToProject(projectId: string,- url: string+ url: string,+ opts: DownloadOptions = {}): Promise<{ files: { fileName: string; title: string }[]; source: LibrarySource; text?: string }> {if (!(await readProject(projectId))) throw new Error("Project not found.");- return withDownload(url, async (dl) => {+ return withDownload(url, opts, async (dl) => {await mkdir(mediaDir(projectId), { recursive: true });const files: { fileName: string; title: string }[] = [];for (const f of dl.files) {diff --git a/site/src/cut/worker/importUrlJob.ts b/site/src/cut/worker/importUrlJob.tsindex 7a903eaa..9d062b49 100644--- a/site/src/cut/worker/importUrlJob.ts+++ b/site/src/cut/worker/importUrlJob.ts@@ -35,10 +35,11 @@ export async function runImportUrlJob(job: ClaimedJob,isCanceled: () => boolean): Promise<ImportUrlResult | LibraryImportResult> {- const { url, target, origin } = (job.spec ?? {}) as {+ const { url, target, origin, audio } = (job.spec ?? {}) as {url?: string;target?: string;origin?: string;+ audio?: boolean;};if (!url || !/^https?:\/\//i.test(url.trim())) throw new Error("Enter a valid http(s) URL.");const toLibrary = target === "library";@@ -59,7 +60,7 @@ export async function runImportUrlJob(const tmp = await mkdtemp(path.join(os.tmpdir(), "cut-dl-"));try {- const dl = await download(url.trim(), tmp);+ const dl = await download(url.trim(), tmp, { audio: audio === true });if (isCanceled()) throw new Error("Import canceled.");const rows = await prisma.cutMediaObject.findMany({